Merged in ra_bauke/eigen (pull request PR-180) alias template for matrix and array classes, see also bug #864 Approved-by: Heiko Bauke <heiko.bauke@mail.de>
diff --git a/.hgignore b/.hgignore index 769a47f..ebbf746 100644 --- a/.hgignore +++ b/.hgignore
@@ -13,7 +13,7 @@ core.* *.bak *~ -build* +*build* *.moc.* *.moc ui_* @@ -28,7 +28,11 @@ *.rej log patch +*.patch a a.* lapack/testing lapack/reference +.*project +.settings +Makefile
diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b37533..76e0833 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -1,4 +1,4 @@ -project(Eigen) +project(Eigen3) cmake_minimum_required(VERSION 2.8.5) @@ -8,6 +8,12 @@ message(FATAL_ERROR "In-source builds not allowed. Please make a new directory (called a build directory) and run CMake from there. You may need to remove CMakeCache.txt. ") endif() + +# Alias Eigen_*_DIR to Eigen3_*_DIR: + +set(Eigen_SOURCE_DIR ${Eigen3_SOURCE_DIR}) +set(Eigen_BINARY_DIR ${Eigen3_BINARY_DIR}) + # guard against bad build-type strings if (NOT CMAKE_BUILD_TYPE) @@ -23,7 +29,7 @@ ############################################################################# -# retrieve version infomation # +# retrieve version information # ############################################################################# # automatically parse the version number @@ -36,10 +42,13 @@ set(EIGEN_MINOR_VERSION "${CMAKE_MATCH_1}") set(EIGEN_VERSION_NUMBER ${EIGEN_WORLD_VERSION}.${EIGEN_MAJOR_VERSION}.${EIGEN_MINOR_VERSION}) -# if the mercurial program is absent, this will leave the EIGEN_HG_CHANGESET string empty, -# but won't stop CMake. -execute_process(COMMAND hg tip -R ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE EIGEN_HGTIP_OUTPUT) -execute_process(COMMAND hg branch -R ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE EIGEN_BRANCH_OUTPUT) +# if we are not in a mercurial clone +if(IS_DIRECTORY ${CMAKE_SOURCE_DIR}/.hg) + # if the mercurial program is absent or this will leave the EIGEN_HG_CHANGESET string empty, + # but won't stop CMake. + execute_process(COMMAND hg tip -R ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE EIGEN_HGTIP_OUTPUT) + execute_process(COMMAND hg branch -R ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE EIGEN_BRANCH_OUTPUT) +endif() # if this is the default (aka development) branch, extract the mercurial changeset number from the hg tip output... if(EIGEN_BRANCH_OUTPUT MATCHES "default") @@ -59,6 +68,33 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) + +option(EIGEN_TEST_CXX11 "Enable testing with C++11 and C++11 features (e.g. Tensor module)." OFF) + + +macro(ei_add_cxx_compiler_flag FLAG) + string(REGEX REPLACE "-" "" SFLAG1 ${FLAG}) + string(REGEX REPLACE "\\+" "p" SFLAG ${SFLAG1}) + check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORT_${SFLAG}) + if(COMPILER_SUPPORT_${SFLAG}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG}") + endif() +endmacro(ei_add_cxx_compiler_flag) + +check_cxx_compiler_flag("-std=c++11" EIGEN_COMPILER_SUPPORT_CPP11) + +if(EIGEN_TEST_CXX11) + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_EXTENSIONS OFF) + if(EIGEN_COMPILER_SUPPORT_CPP11) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() +else() + #set(CMAKE_CXX_STANDARD 03) + #set(CMAKE_CXX_EXTENSIONS OFF) + ei_add_cxx_compiler_flag("-std=c++03") +endif() + ############################################################################# # find how to link to the standard libraries # ############################################################################# @@ -93,11 +129,13 @@ endif() option(EIGEN_BUILD_BTL "Build benchmark suite" OFF) -if(NOT WIN32) - option(EIGEN_BUILD_PKGCONFIG "Build pkg-config .pc file for Eigen" ON) -endif(NOT WIN32) -set(CMAKE_INCLUDE_CURRENT_DIR ON) +# Disable pkgconfig only for native Windows builds +if(NOT WIN32 OR NOT CMAKE_HOST_SYSTEM_NAME MATCHES Windows) + option(EIGEN_BUILD_PKGCONFIG "Build pkg-config .pc file for Eigen" ON) +endif() + +set(CMAKE_INCLUDE_CURRENT_DIR OFF) option(EIGEN_SPLIT_LARGE_TESTS "Split large tests into smaller executables" ON) @@ -108,30 +146,20 @@ set(EIGEN_TEST_MAX_SIZE "320" CACHE STRING "Maximal matrix/vector size, default is 320") -macro(ei_add_cxx_compiler_flag FLAG) - string(REGEX REPLACE "-" "" SFLAG1 ${FLAG}) - string(REGEX REPLACE "\\+" "p" SFLAG ${SFLAG1}) - check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORT_${SFLAG}) - if(COMPILER_SUPPORT_${SFLAG}) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG}") - endif() -endmacro(ei_add_cxx_compiler_flag) - if(NOT MSVC) # We assume that other compilers are partly compatible with GNUCC - # clang outputs some warnings for unknwon flags that are not caught by check_cxx_compiler_flag + # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag # adding -Werror turns such warnings into errors check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) if(COMPILER_SUPPORT_WERROR) set(CMAKE_REQUIRED_FLAGS "-Werror") endif() - ei_add_cxx_compiler_flag("-pedantic") ei_add_cxx_compiler_flag("-Wall") ei_add_cxx_compiler_flag("-Wextra") #ei_add_cxx_compiler_flag("-Weverything") # clang - + ei_add_cxx_compiler_flag("-Wundef") ei_add_cxx_compiler_flag("-Wcast-align") ei_add_cxx_compiler_flag("-Wchar-subscripts") @@ -141,31 +169,30 @@ ei_add_cxx_compiler_flag("-Wwrite-strings") ei_add_cxx_compiler_flag("-Wformat-security") ei_add_cxx_compiler_flag("-Wshorten-64-to-32") + ei_add_cxx_compiler_flag("-Wlogical-op") ei_add_cxx_compiler_flag("-Wenum-conversion") ei_add_cxx_compiler_flag("-Wc++11-extensions") - - # -Wshadow is insanely too strict with gcc, hopefully it will become usable with gcc 6 - # if(NOT CMAKE_COMPILER_IS_GNUCXX OR (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.0.0")) - if(NOT CMAKE_COMPILER_IS_GNUCXX) - ei_add_cxx_compiler_flag("-Wshadow") - endif() - + ei_add_cxx_compiler_flag("-Wdouble-promotion") +# ei_add_cxx_compiler_flag("-Wconversion") + + ei_add_cxx_compiler_flag("-Wshadow") + ei_add_cxx_compiler_flag("-Wno-psabi") ei_add_cxx_compiler_flag("-Wno-variadic-macros") ei_add_cxx_compiler_flag("-Wno-long-long") - + ei_add_cxx_compiler_flag("-fno-check-new") ei_add_cxx_compiler_flag("-fno-common") ei_add_cxx_compiler_flag("-fstrict-aliasing") ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark - ei_add_cxx_compiler_flag("-wd2304") # disbale ICC's "warning #2304: non-explicit constructor with single argument may cause implicit type conversion" produced by -Wnon-virtual-dtor - - + ei_add_cxx_compiler_flag("-wd2304") # disable ICC's "warning #2304: non-explicit constructor with single argument may cause implicit type conversion" produced by -Wnon-virtual-dtor + + # The -ansi flag must be added last, otherwise it is also used as a linker flag by check_cxx_compiler_flag making it fails # Moreover we should not set both -strict-ansi and -ansi check_cxx_compiler_flag("-strict-ansi" COMPILER_SUPPORT_STRICTANSI) ei_add_cxx_compiler_flag("-Qunused-arguments") # disable clang warning: argument unused during compilation: '-ansi' - + if(COMPILER_SUPPORT_STRICTANSI) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -strict-ansi") else() @@ -176,7 +203,7 @@ ei_add_cxx_compiler_flag("-pie") ei_add_cxx_compiler_flag("-fPIE") endif() - + set(CMAKE_REQUIRED_FLAGS "") option(EIGEN_TEST_SSE2 "Enable/Disable SSE2 in tests/examples" OFF) @@ -221,6 +248,15 @@ message(STATUS "Enabling FMA in tests/examples") endif() + option(EIGEN_TEST_AVX512 "Enable/Disable AVX512 in tests/examples" OFF) + if(EIGEN_TEST_AVX512) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mfma -DEIGEN_ENABLE_AVX512") + if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fabi-version=6") + endif() + message(STATUS "Enabling AVX512 in tests/examples") + endif() + option(EIGEN_TEST_F16C "Enable/Disable F16C in tests/examples" OFF) if(EIGEN_TEST_F16C) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mf16c") @@ -239,6 +275,12 @@ message(STATUS "Enabling VSX in tests/examples") endif() + option(EIGEN_TEST_MSA "Enable/Disable MSA in tests/examples" OFF) + if(EIGEN_TEST_MSA) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmsa") + message(STATUS "Enabling MSA in tests/examples") + endif() + option(EIGEN_TEST_NEON "Enable/Disable Neon in tests/examples" OFF) if(EIGEN_TEST_NEON) if(EIGEN_TEST_FMA) @@ -256,12 +298,18 @@ message(STATUS "Enabling NEON in tests/examples") endif() - option(EIGEN_TEST_ZVECTOR "Enable/Disable S390X(zEC13) ZVECTOR in tests/examples" OFF) - if(EIGEN_TEST_ZVECTOR) + option(EIGEN_TEST_Z13 "Enable/Disable S390X(zEC13) ZVECTOR in tests/examples" OFF) + if(EIGEN_TEST_Z13) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z13 -mzvector") message(STATUS "Enabling S390X(zEC13) ZVECTOR in tests/examples") endif() + option(EIGEN_TEST_Z14 "Enable/Disable S390X(zEC14) ZVECTOR in tests/examples" OFF) + if(EIGEN_TEST_Z14) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z14 -mzvector") + message(STATUS "Enabling S390X(zEC13) ZVECTOR in tests/examples") + endif() + check_cxx_compiler_flag("-fopenmp" COMPILER_SUPPORT_OPENMP) if(COMPILER_SUPPORT_OPENMP) option(EIGEN_TEST_OPENMP "Enable/Disable OpenMP in tests/examples" OFF) @@ -279,7 +327,7 @@ # because we are oftentimes returning objects that have a destructor or may # throw exceptions - in particular in the unit tests we are throwing extra many # exceptions to cover indexing errors. - # C4505 - unreferenced local function has been removed (impossible to deactive selectively) + # C4505 - unreferenced local function has been removed (impossible to deactivate selectively) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /wd4127 /wd4505 /wd4714") # replace all /Wx by /W4 @@ -302,6 +350,19 @@ endif(NOT CMAKE_CL_64) message(STATUS "Enabling SSE2 in tests/examples") endif(EIGEN_TEST_SSE2) + + option(EIGEN_TEST_AVX "Enable/Disable AVX in tests/examples" OFF) + if(EIGEN_TEST_AVX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX") + message(STATUS "Enabling AVX in tests/examples") + endif() + + option(EIGEN_TEST_FMA "Enable/Disable FMA/AVX2 in tests/examples" OFF) + if(EIGEN_TEST_FMA AND NOT EIGEN_TEST_NEON) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") + message(STATUS "Enabling FMA/AVX2 in tests/examples") + endif() + endif(NOT MSVC) option(EIGEN_TEST_NO_EXPLICIT_VECTORIZATION "Disable explicit vectorization in tests/examples" OFF) @@ -344,11 +405,9 @@ message(STATUS "Disabling exceptions in tests/examples") endif() -option(EIGEN_TEST_CXX11 "Enable testing with C++11 and C++11 features (e.g. Tensor module)." OFF) - set(EIGEN_CUDA_COMPUTE_ARCH 30 CACHE STRING "The CUDA compute architecture level to target when compiling CUDA code") -include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) # Backward compatibility support for EIGEN_INCLUDE_INSTALL_DIR if(EIGEN_INCLUDE_INSTALL_DIR) @@ -365,7 +424,7 @@ ) endif() set(CMAKEPACKAGE_INSTALL_DIR - "${CMAKE_INSTALL_LIBDIR}/cmake/eigen3" + "${CMAKE_INSTALL_DATADIR}/eigen3/cmake" CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen3Config.cmake is installed" ) set(PKGCONFIG_INSTALL_DIR @@ -395,22 +454,23 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eigen3.pc DESTINATION ${PKGCONFIG_INSTALL_DIR} ) -endif(EIGEN_BUILD_PKGCONFIG) +endif() add_subdirectory(Eigen) add_subdirectory(doc EXCLUDE_FROM_ALL) -include(EigenConfigureTesting) +option(BUILD_TESTING "Enable creation of Eigen tests." ON) +if(BUILD_TESTING) + include(EigenConfigureTesting) -# fixme, not sure this line is still needed: -enable_testing() # must be called from the root CMakeLists, see man page + if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) + add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest + else() + add_subdirectory(test EXCLUDE_FROM_ALL) + endif() - -if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) - add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest -else() - add_subdirectory(test EXCLUDE_FROM_ALL) + add_subdirectory(failtest) endif() if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) @@ -421,6 +481,20 @@ add_subdirectory(lapack EXCLUDE_FROM_ALL) endif() +# add SYCL +option(EIGEN_TEST_SYCL "Add Sycl support." OFF) +option(EIGEN_SYCL_TRISYCL "Use the triSYCL Sycl implementation (ComputeCPP by default)." OFF) +if(EIGEN_TEST_SYCL) + set (CMAKE_MODULE_PATH "${CMAKE_ROOT}/Modules" "cmake/Modules/" "${CMAKE_MODULE_PATH}") + if(EIGEN_SYCL_TRISYCL) + message(STATUS "Using triSYCL") + include(FindTriSYCL) + else(EIGEN_SYCL_TRISYCL) + message(STATUS "Using ComputeCPP SYCL") + include(FindComputeCpp) + endif(EIGEN_SYCL_TRISYCL) +endif(EIGEN_TEST_SYCL) + add_subdirectory(unsupported) add_subdirectory(demos EXCLUDE_FROM_ALL) @@ -439,17 +513,14 @@ configure_file(scripts/cdashtesting.cmake.in cdashtesting.cmake @ONLY) -ei_testing_print_summary() +if(BUILD_TESTING) + ei_testing_print_summary() +endif() message(STATUS "") message(STATUS "Configured Eigen ${EIGEN_VERSION_NUMBER}") message(STATUS "") -option(EIGEN_FAILTEST "Enable failtests." OFF) -if(EIGEN_FAILTEST) - add_subdirectory(failtest) -endif() - string(TOLOWER "${CMAKE_GENERATOR}" cmake_generator_tolower) if(cmake_generator_tolower MATCHES "makefile") message(STATUS "Some things you can do now:") @@ -466,8 +537,10 @@ message(STATUS " | Or:") message(STATUS " | cmake . -DINCLUDE_INSTALL_DIR=yourdir") message(STATUS "make doc | Generate the API documentation, requires Doxygen & LaTeX") - message(STATUS "make check | Build and run the unit-tests. Read this page:") - message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests") + if(BUILD_TESTING) + message(STATUS "make check | Build and run the unit-tests. Read this page:") + message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests") + endif() message(STATUS "make blas | Build BLAS library (not the same thing as Eigen)") message(STATUS "make uninstall| Removes files installed by make install") message(STATUS "--------------+--------------------------------------------------------------") @@ -485,18 +558,89 @@ set ( EIGEN_VERSION_PATCH ${EIGEN_MINOR_VERSION} ) set ( EIGEN_DEFINITIONS "") set ( EIGEN_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/${INCLUDE_INSTALL_DIR}" ) -set ( EIGEN_INCLUDE_DIRS ${EIGEN_INCLUDE_DIR} ) set ( EIGEN_ROOT_DIR ${CMAKE_INSTALL_PREFIX} ) -configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3Config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake - @ONLY ESCAPE_QUOTES - ) +# Interface libraries require at least CMake 3.0 +if (NOT CMAKE_VERSION VERSION_LESS 3.0) + include (CMakePackageConfigHelpers) + + # Imported target support + add_library (eigen INTERFACE) + add_library (Eigen3::Eigen ALIAS eigen) + target_compile_definitions (eigen INTERFACE ${EIGEN_DEFINITIONS}) + target_include_directories (eigen INTERFACE + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> + $<INSTALL_INTERFACE:${INCLUDE_INSTALL_DIR}> + ) + + # Export as title case Eigen + set_target_properties (eigen PROPERTIES EXPORT_NAME Eigen) + + install (TARGETS eigen EXPORT Eigen3Targets) + + configure_package_config_file ( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3Config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake + PATH_VARS EIGEN_INCLUDE_DIR EIGEN_ROOT_DIR + INSTALL_DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} + NO_CHECK_REQUIRED_COMPONENTS_MACRO # Eigen does not provide components + ) + # Remove CMAKE_SIZEOF_VOID_P from Eigen3ConfigVersion.cmake since Eigen does + # not depend on architecture specific settings or libraries. More + # specifically, an Eigen3Config.cmake generated from a 64 bit target can be + # used for 32 bit targets as well (and vice versa). + set (_Eigen3_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) + unset (CMAKE_SIZEOF_VOID_P) + write_basic_package_version_file (Eigen3ConfigVersion.cmake + VERSION ${EIGEN_VERSION_NUMBER} + COMPATIBILITY SameMajorVersion) + set (CMAKE_SIZEOF_VOID_P ${_Eigen3_CMAKE_SIZEOF_VOID_P}) + + # The Eigen target will be located in the Eigen3 namespace. Other CMake + # targets can refer to it using Eigen3::Eigen. + export (TARGETS eigen NAMESPACE Eigen3:: FILE Eigen3Targets.cmake) + # Export Eigen3 package to CMake registry such that it can be easily found by + # CMake even if it has not been installed to a standard directory. + export (PACKAGE Eigen3) + + install (EXPORT Eigen3Targets NAMESPACE Eigen3:: DESTINATION ${CMAKEPACKAGE_INSTALL_DIR}) + +else (NOT CMAKE_VERSION VERSION_LESS 3.0) + # Fallback to legacy Eigen3Config.cmake without the imported target + + # If CMakePackageConfigHelpers module is available (CMake >= 2.8.8) + # create a relocatable Config file, otherwise leave the hardcoded paths + include(CMakePackageConfigHelpers OPTIONAL RESULT_VARIABLE CPCH_PATH) + + if(CPCH_PATH) + configure_package_config_file ( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3ConfigLegacy.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake + PATH_VARS EIGEN_INCLUDE_DIR EIGEN_ROOT_DIR + INSTALL_DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} + NO_CHECK_REQUIRED_COMPONENTS_MACRO # Eigen does not provide components + ) + else() + # The PACKAGE_* variables are defined by the configure_package_config_file + # but without it we define them manually to the hardcoded paths + set(PACKAGE_INIT "") + set(PACKAGE_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR}) + set(PACKAGE_EIGEN_ROOT_DIR ${EIGEN_ROOT_DIR}) + configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3ConfigLegacy.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake + @ONLY ESCAPE_QUOTES ) + endif() + + write_basic_package_version_file( Eigen3ConfigVersion.cmake + VERSION ${EIGEN_VERSION_NUMBER} + COMPATIBILITY SameMajorVersion ) + +endif (NOT CMAKE_VERSION VERSION_LESS 3.0) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/UseEigen3.cmake ${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake - DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} - ) + ${CMAKE_CURRENT_BINARY_DIR}/Eigen3ConfigVersion.cmake + DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} ) # Add uninstall target add_custom_target ( uninstall
diff --git a/CTestConfig.cmake b/CTestConfig.cmake index 4c00278..8b4cd79 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake
@@ -11,7 +11,7 @@ set(CTEST_DROP_SITE "manao.inria.fr") set(CTEST_DROP_LOCATION "/CDash/submit.php?project=Eigen") set(CTEST_DROP_SITE_CDASH TRUE) -set(CTEST_PROJECT_SUBPROJECTS -Official -Unsupported -) +#set(CTEST_PROJECT_SUBPROJECTS +#Official +#Unsupported +#)
diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index 9fed9d3..89e487f 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in
@@ -1,3 +1,4 @@ set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "2000") set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "2000") +list(APPEND CTEST_CUSTOM_ERROR_EXCEPTION @EIGEN_CTEST_ERROR_EXCEPTION@)
diff --git a/Eigen/CMakeLists.txt b/Eigen/CMakeLists.txt index a92dd6f..9eb502b 100644 --- a/Eigen/CMakeLists.txt +++ b/Eigen/CMakeLists.txt
@@ -16,4 +16,4 @@ DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen COMPONENT Devel ) -add_subdirectory(src) +install(DIRECTORY src DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen COMPONENT Devel FILES_MATCHING PATTERN "*.h")
diff --git a/Eigen/Cholesky b/Eigen/Cholesky index 705a04c..1332b54 100644 --- a/Eigen/Cholesky +++ b/Eigen/Cholesky
@@ -9,6 +9,7 @@ #define EIGEN_CHOLESKY_MODULE_H #include "Core" +#include "Jacobi" #include "src/Core/util/DisableStupidWarnings.h" @@ -31,7 +32,12 @@ #include "src/Cholesky/LLT.h" #include "src/Cholesky/LDLT.h" #ifdef EIGEN_USE_LAPACKE -#include "src/Cholesky/LLT_MKL.h" +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "src/misc/lapacke.h" +#endif +#include "src/Cholesky/LLT_LAPACKE.h" #endif #include "src/Core/util/ReenableStupidWarnings.h"
diff --git a/Eigen/Core b/Eigen/Core index 5004013..759b1bb 100644 --- a/Eigen/Core +++ b/Eigen/Core
@@ -14,56 +14,26 @@ // first thing Eigen does: stop the compiler from committing suicide #include "src/Core/util/DisableStupidWarnings.h" -// Handle NVCC/CUDA -#ifdef __CUDACC__ - // Do not try asserts on CUDA! - #ifndef EIGEN_NO_DEBUG - #define EIGEN_NO_DEBUG - #endif +// then include this file where all our macros are defined. It's really important to do it first because +// it's where we do all the compiler/OS/arch detections and define most defaults. +#include "src/Core/util/Macros.h" - #ifdef EIGEN_INTERNAL_DEBUGGING - #undef EIGEN_INTERNAL_DEBUGGING - #endif +// This detects SSE/AVX/NEON/etc. and configure alignment settings +#include "src/Core/util/ConfigureVectorization.h" - // Do not try to vectorize on CUDA! - #ifndef EIGEN_DONT_VECTORIZE - #define EIGEN_DONT_VECTORIZE - #endif - - #ifdef EIGEN_EXCEPTIONS - #undef EIGEN_EXCEPTIONS - #endif - - // All functions callable from CUDA code must be qualified with __device__ - #define EIGEN_DEVICE_FUNC __host__ __device__ - -#else - #define EIGEN_DEVICE_FUNC - +// We need cuda_runtime.h/hip_runtime.h to ensure that +// the EIGEN_USING_STD_MATH macro works properly on the device side +#if defined(EIGEN_CUDACC) + #include <cuda_runtime.h> +#elif defined(EIGEN_HIPCC) + #include <hip/hip_runtime.h> #endif -// When compiling CUDA device code with NVCC, pull in math functions from the -// global namespace. In host mode, and when device doee with clang, use the -// std versions. -#if defined(__CUDA_ARCH__) && defined(__NVCC__) - #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC; -#else - #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC; -#endif - -#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(EIGEN_EXCEPTIONS) - #define EIGEN_EXCEPTIONS -#endif #ifdef EIGEN_EXCEPTIONS #include <new> #endif -// then include this file where all our macros are defined. It's really important to do it first because -// it's where we do all the alignment settings (platform detection and honoring the user's will if he -// defined e.g. EIGEN_DONT_ALIGN) so it needs to be done before we do anything with vectorization. -#include "src/Core/util/Macros.h" - // Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3) // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details. #if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6) @@ -76,150 +46,9 @@ // and inclusion of their respective header files #include "src/Core/util/MKL_support.h" -// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into -// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks -#if EIGEN_MAX_ALIGN_BYTES==0 - #ifndef EIGEN_DONT_VECTORIZE - #define EIGEN_DONT_VECTORIZE - #endif -#endif -#if EIGEN_COMP_MSVC - #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled - #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later - // Remember that usage of defined() in a #define is undefined by the standard. - // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. - #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 - #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER - #endif - #endif -#else - // Remember that usage of defined() in a #define is undefined by the standard - #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) - #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC - #endif -#endif - -#ifndef EIGEN_DONT_VECTORIZE - - #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) - - // Defines symbols for compile-time detection of which instructions are - // used. - // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used - #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_SSE - #define EIGEN_VECTORIZE_SSE2 - - // Detect sse3/ssse3/sse4: - // gcc and icc defines __SSE3__, ... - // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you - // want to force the use of those instructions with msvc. - #ifdef __SSE3__ - #define EIGEN_VECTORIZE_SSE3 - #endif - #ifdef __SSSE3__ - #define EIGEN_VECTORIZE_SSSE3 - #endif - #ifdef __SSE4_1__ - #define EIGEN_VECTORIZE_SSE4_1 - #endif - #ifdef __SSE4_2__ - #define EIGEN_VECTORIZE_SSE4_2 - #endif - #ifdef __AVX__ - #define EIGEN_VECTORIZE_AVX - #define EIGEN_VECTORIZE_SSE3 - #define EIGEN_VECTORIZE_SSSE3 - #define EIGEN_VECTORIZE_SSE4_1 - #define EIGEN_VECTORIZE_SSE4_2 - #endif - #ifdef __AVX2__ - #define EIGEN_VECTORIZE_AVX2 - #endif - #ifdef __FMA__ - #define EIGEN_VECTORIZE_FMA - #endif - - // include files - - // This extern "C" works around a MINGW-w64 compilation issue - // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354 - // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do). - // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations - // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know; - // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too. - // notice that since these are C headers, the extern "C" is theoretically needed anyways. - extern "C" { - // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. - // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: - #if EIGEN_COMP_ICC >= 1110 - #include <immintrin.h> - #else - #include <emmintrin.h> - #include <xmmintrin.h> - #ifdef EIGEN_VECTORIZE_SSE3 - #include <pmmintrin.h> - #endif - #ifdef EIGEN_VECTORIZE_SSSE3 - #include <tmmintrin.h> - #endif - #ifdef EIGEN_VECTORIZE_SSE4_1 - #include <smmintrin.h> - #endif - #ifdef EIGEN_VECTORIZE_SSE4_2 - #include <nmmintrin.h> - #endif - #ifdef EIGEN_VECTORIZE_AVX - #include <immintrin.h> - #endif - #endif - } // end extern "C" - #elif defined __VSX__ - #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_VSX - #include <altivec.h> - // We need to #undef all these ugly tokens defined in <altivec.h> - // => use __vector instead of vector - #undef bool - #undef vector - #undef pixel - #elif defined __ALTIVEC__ - #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_ALTIVEC - #include <altivec.h> - // We need to #undef all these ugly tokens defined in <altivec.h> - // => use __vector instead of vector - #undef bool - #undef vector - #undef pixel - #elif (defined __ARM_NEON) || (defined __ARM_NEON__) - #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_NEON - #include <arm_neon.h> - #elif (defined __s390x__ && defined __VEC__) - #define EIGEN_VECTORIZE - #define EIGEN_VECTORIZE_ZVECTOR - #include <vecintrin.h> - #endif -#endif - -#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG) - // We can use the optimized fp16 to float and float to fp16 conversion routines - #define EIGEN_HAS_FP16_C -#endif - -#if defined __CUDACC__ - #define EIGEN_VECTORIZE_CUDA - #include <vector_types.h> - #if defined __CUDACC_VER__ && __CUDACC_VER__ >= 70500 - #define EIGEN_HAS_CUDA_FP16 - #endif -#endif - -#if defined EIGEN_HAS_CUDA_FP16 - #include <host_defines.h> - #include <cuda_fp16.h> +#if defined(EIGEN_HAS_CUDA_FP16) || defined(EIGEN_HAS_HIP_FP16) + #define EIGEN_HAS_GPU_FP16 #endif #if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE) @@ -243,7 +72,9 @@ #include <cmath> #include <cassert> #include <functional> -#include <iosfwd> +#ifndef EIGEN_NO_IO + #include <iosfwd> +#endif #include <cstring> #include <string> #include <limits> @@ -251,6 +82,15 @@ // for min/max: #include <algorithm> +#if EIGEN_HAS_CXX11 +#include <array> +#endif + +// for std::is_nothrow_move_assignable +#ifdef EIGEN_INCLUDE_TYPE_TRAITS +#include <type_traits> +#endif + // for outputting debug info #ifdef EIGEN_DEBUG_ASSIGN #include <iostream> @@ -261,48 +101,31 @@ #include <intrin.h> #endif -/** \brief Namespace containing all symbols from the %Eigen library. */ -namespace Eigen { - -inline static const char *SimdInstructionSetsInUse(void) { -#if defined(EIGEN_VECTORIZE_AVX) - return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; -#elif defined(EIGEN_VECTORIZE_SSE4_2) - return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; -#elif defined(EIGEN_VECTORIZE_SSE4_1) - return "SSE, SSE2, SSE3, SSSE3, SSE4.1"; -#elif defined(EIGEN_VECTORIZE_SSSE3) - return "SSE, SSE2, SSE3, SSSE3"; -#elif defined(EIGEN_VECTORIZE_SSE3) - return "SSE, SSE2, SSE3"; -#elif defined(EIGEN_VECTORIZE_SSE2) - return "SSE, SSE2"; -#elif defined(EIGEN_VECTORIZE_ALTIVEC) - return "AltiVec"; -#elif defined(EIGEN_VECTORIZE_VSX) - return "VSX"; -#elif defined(EIGEN_VECTORIZE_NEON) - return "ARM NEON"; -#elif defined(EIGEN_VECTORIZE_ZVECTOR) - return "S390X ZVECTOR"; -#else - return "None"; +#if defined(__SYCL_DEVICE_ONLY__) + #undef min + #undef max + #undef isnan + #undef isinf + #undef isfinite + #include <SYCL/sycl.hpp> #endif -} -} // end namespace Eigen #if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT // This will generate an error message: #error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information #endif -// we use size_t frequently and we'll never remember to prepend it with std:: everytime just to +namespace Eigen { + +// we use size_t frequently and we'll never remember to prepend it with std:: every time just to // ensure QNX/QCC support using std::size_t; // gcc 4.6.0 wants std:: for ptrdiff_t using std::ptrdiff_t; +} + /** \defgroup Core_Module Core module * This is the main module of Eigen providing dense matrix and vector support * (both fixed and dynamic size) with all the features corresponding to a BLAS library @@ -319,57 +142,98 @@ #include "src/Core/util/StaticAssert.h" #include "src/Core/util/XprHelper.h" #include "src/Core/util/Memory.h" +#include "src/Core/util/IntegralConstant.h" +#include "src/Core/util/SymbolicIndex.h" #include "src/Core/NumTraits.h" #include "src/Core/MathFunctions.h" -#include "src/Core/SpecialFunctions.h" #include "src/Core/GenericPacketMath.h" +#include "src/Core/MathFunctionsImpl.h" +#include "src/Core/arch/Default/ConjHelper.h" -#if defined EIGEN_VECTORIZE_AVX +#if defined EIGEN_VECTORIZE_AVX512 + #include "src/Core/arch/SSE/PacketMath.h" + #include "src/Core/arch/SSE/TypeCasting.h" + #include "src/Core/arch/SSE/Complex.h" + #include "src/Core/arch/AVX/PacketMath.h" + #include "src/Core/arch/AVX/TypeCasting.h" + #include "src/Core/arch/AVX/Complex.h" + #include "src/Core/arch/AVX512/PacketMath.h" + #include "src/Core/arch/AVX512/Complex.h" + #include "src/Core/arch/SSE/MathFunctions.h" + #include "src/Core/arch/AVX/MathFunctions.h" + #include "src/Core/arch/AVX512/MathFunctions.h" +#elif defined EIGEN_VECTORIZE_AVX // Use AVX for floats and doubles, SSE for integers #include "src/Core/arch/SSE/PacketMath.h" + #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/Complex.h" - #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/AVX/PacketMath.h" - #include "src/Core/arch/AVX/MathFunctions.h" - #include "src/Core/arch/AVX/Complex.h" #include "src/Core/arch/AVX/TypeCasting.h" + #include "src/Core/arch/AVX/Complex.h" + #include "src/Core/arch/SSE/MathFunctions.h" + #include "src/Core/arch/AVX/MathFunctions.h" #elif defined EIGEN_VECTORIZE_SSE #include "src/Core/arch/SSE/PacketMath.h" + #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/SSE/Complex.h" - #include "src/Core/arch/SSE/TypeCasting.h" #elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/PacketMath.h" #include "src/Core/arch/AltiVec/MathFunctions.h" #include "src/Core/arch/AltiVec/Complex.h" #elif defined EIGEN_VECTORIZE_NEON #include "src/Core/arch/NEON/PacketMath.h" + #include "src/Core/arch/NEON/TypeCasting.h" #include "src/Core/arch/NEON/MathFunctions.h" #include "src/Core/arch/NEON/Complex.h" #elif defined EIGEN_VECTORIZE_ZVECTOR #include "src/Core/arch/ZVector/PacketMath.h" #include "src/Core/arch/ZVector/MathFunctions.h" #include "src/Core/arch/ZVector/Complex.h" +#elif defined EIGEN_VECTORIZE_MSA + #include "src/Core/arch/MSA/PacketMath.h" + #include "src/Core/arch/MSA/MathFunctions.h" + #include "src/Core/arch/MSA/Complex.h" #endif -#include "src/Core/arch/CUDA/Half.h" +// Half float support +#include "src/Core/arch/GPU/Half.h" +#include "src/Core/arch/GPU/PacketMathHalf.h" +#include "src/Core/arch/GPU/TypeCasting.h" -#if defined EIGEN_VECTORIZE_CUDA - #include "src/Core/arch/CUDA/PacketMath.h" - #include "src/Core/arch/CUDA/PacketMathHalf.h" - #include "src/Core/arch/CUDA/MathFunctions.h" - #include "src/Core/arch/CUDA/TypeCasting.h" +#if defined EIGEN_VECTORIZE_GPU + #include "src/Core/arch/GPU/PacketMath.h" + #include "src/Core/arch/GPU/MathFunctions.h" #endif +#if defined EIGEN_VECTORIZE_SYCL + #include "src/Core/arch/SYCL/InteropHeaders.h" + #include "src/Core/arch/SYCL/PacketMath.h" + #include "src/Core/arch/SYCL/MathFunctions.h" + #include "src/Core/arch/SYCL/TypeCasting.h" +#endif #include "src/Core/arch/Default/Settings.h" +#include "src/Core/functors/TernaryFunctors.h" #include "src/Core/functors/BinaryFunctors.h" #include "src/Core/functors/UnaryFunctors.h" #include "src/Core/functors/NullaryFunctors.h" #include "src/Core/functors/StlFunctors.h" #include "src/Core/functors/AssignmentFunctors.h" +// Specialized functors to enable the processing of complex numbers +// on CUDA devices +#ifdef EIGEN_CUDACC +#include "src/Core/arch/CUDA/Complex.h" +#endif + +#include "src/Core/util/IndexedViewHelper.h" +#include "src/Core/util/ReshapedHelper.h" +#include "src/Core/ArithmeticSequence.h" +#ifndef EIGEN_NO_IO + #include "src/Core/IO.h" +#endif #include "src/Core/DenseCoeffsBase.h" #include "src/Core/DenseBase.h" #include "src/Core/MatrixBase.h" @@ -396,6 +260,7 @@ #include "src/Core/PlainObjectBase.h" #include "src/Core/Matrix.h" #include "src/Core/Array.h" +#include "src/Core/CwiseTernaryOp.h" #include "src/Core/CwiseBinaryOp.h" #include "src/Core/CwiseUnaryOp.h" #include "src/Core/CwiseNullaryOp.h" @@ -409,6 +274,8 @@ #include "src/Core/Ref.h" #include "src/Core/Block.h" #include "src/Core/VectorBlock.h" +#include "src/Core/IndexedView.h" +#include "src/Core/Reshaped.h" #include "src/Core/Transpose.h" #include "src/Core/DiagonalMatrix.h" #include "src/Core/Diagonal.h" @@ -416,7 +283,6 @@ #include "src/Core/Redux.h" #include "src/Core/Visitor.h" #include "src/Core/Fuzzy.h" -#include "src/Core/IO.h" #include "src/Core/Swap.h" #include "src/Core/CommaInitializer.h" #include "src/Core/GeneralProduct.h" @@ -449,10 +315,12 @@ #include "src/Core/BooleanRedux.h" #include "src/Core/Select.h" #include "src/Core/VectorwiseOp.h" +#include "src/Core/PartialReduxEvaluator.h" #include "src/Core/Random.h" #include "src/Core/Replicate.h" #include "src/Core/Reverse.h" #include "src/Core/ArrayWrapper.h" +#include "src/Core/StlIterators.h" #ifdef EIGEN_USE_BLAS #include "src/Core/products/GeneralMatrixMatrix_BLAS.h"
diff --git a/Eigen/Eigenvalues b/Eigen/Eigenvalues index ea93eb3..7d6ac78 100644 --- a/Eigen/Eigenvalues +++ b/Eigen/Eigenvalues
@@ -10,14 +10,14 @@ #include "Core" -#include "src/Core/util/DisableStupidWarnings.h" - #include "Cholesky" #include "Jacobi" #include "Householder" #include "LU" #include "Geometry" +#include "src/Core/util/DisableStupidWarnings.h" + /** \defgroup Eigenvalues_Module Eigenvalues module * * @@ -32,6 +32,7 @@ * \endcode */ +#include "src/misc/RealSvd2x2.h" #include "src/Eigenvalues/Tridiagonalization.h" #include "src/Eigenvalues/RealSchur.h" #include "src/Eigenvalues/EigenSolver.h" @@ -44,9 +45,14 @@ #include "src/Eigenvalues/GeneralizedEigenSolver.h" #include "src/Eigenvalues/MatrixBaseEigenvalues.h" #ifdef EIGEN_USE_LAPACKE -#include "src/Eigenvalues/RealSchur_MKL.h" -#include "src/Eigenvalues/ComplexSchur_MKL.h" -#include "src/Eigenvalues/SelfAdjointEigenSolver_MKL.h" +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "src/misc/lapacke.h" +#endif +#include "src/Eigenvalues/RealSchur_LAPACKE.h" +#include "src/Eigenvalues/ComplexSchur_LAPACKE.h" +#include "src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h" #endif #include "src/Core/util/ReenableStupidWarnings.h"
diff --git a/Eigen/Geometry b/Eigen/Geometry index 716d529..04aa316 100644 --- a/Eigen/Geometry +++ b/Eigen/Geometry
@@ -10,12 +10,12 @@ #include "Core" -#include "src/Core/util/DisableStupidWarnings.h" - #include "SVD" #include "LU" #include <limits> +#include "src/Core/util/DisableStupidWarnings.h" + /** \defgroup Geometry_Module Geometry module * * This module provides support for: @@ -59,4 +59,3 @@ #endif // EIGEN_GEOMETRY_MODULE_H /* vim: set filetype=cpp et sw=2 ts=2 ai: */ -
diff --git a/Eigen/KLUSupport b/Eigen/KLUSupport new file mode 100644 index 0000000..b23d905 --- /dev/null +++ b/Eigen/KLUSupport
@@ -0,0 +1,41 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_KLUSUPPORT_MODULE_H +#define EIGEN_KLUSUPPORT_MODULE_H + +#include <Eigen/SparseCore> + +#include <Eigen/src/Core/util/DisableStupidWarnings.h> + +extern "C" { +#include <btf.h> +#include <klu.h> + } + +/** \ingroup Support_modules + * \defgroup KLUSupport_Module KLUSupport module + * + * This module provides an interface to the KLU library which is part of the <a href="http://www.suitesparse.com">suitesparse</a> package. + * It provides the following factorization class: + * - class KLU: a sparse LU factorization, well-suited for circuit simulation. + * + * \code + * #include <Eigen/KLUSupport> + * \endcode + * + * In order to use this module, the klu and btf headers must be accessible from the include paths, and your binary must be linked to the klu library and its dependencies. + * The dependencies depend on how umfpack has been compiled. + * For a cmake based project, you can use our FindKLU.cmake module to help you in this task. + * + */ + +#include "src/KLUSupport/KLUSupport.h" + +#include <Eigen/src/Core/util/ReenableStupidWarnings.h> + +#endif // EIGEN_KLUSUPPORT_MODULE_H
diff --git a/Eigen/LU b/Eigen/LU index 2d70c92..6418a86 100644 --- a/Eigen/LU +++ b/Eigen/LU
@@ -28,7 +28,12 @@ #include "src/LU/FullPivLU.h" #include "src/LU/PartialPivLU.h" #ifdef EIGEN_USE_LAPACKE -#include "src/LU/PartialPivLU_MKL.h" +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "src/misc/lapacke.h" +#endif +#include "src/LU/PartialPivLU_LAPACKE.h" #endif #include "src/LU/Determinant.h" #include "src/LU/InverseImpl.h"
diff --git a/Eigen/PaStiXSupport b/Eigen/PaStiXSupport index de3a63b..234619a 100644 --- a/Eigen/PaStiXSupport +++ b/Eigen/PaStiXSupport
@@ -36,6 +36,7 @@ * \endcode * * In order to use this module, the PaSTiX headers must be accessible from the include paths, and your binary must be linked to the PaSTiX library and its dependencies. + * This wrapper resuires PaStiX version 5.x compiled without MPI support. * The dependencies depend on how PaSTiX has been compiled. * For a cmake based project, you can use our FindPaSTiX.cmake module to help you in this task. *
diff --git a/Eigen/PardisoSupport b/Eigen/PardisoSupport old mode 100755 new mode 100644
diff --git a/Eigen/QR b/Eigen/QR index 25c781c..1be1863 100644 --- a/Eigen/QR +++ b/Eigen/QR
@@ -10,12 +10,12 @@ #include "Core" -#include "src/Core/util/DisableStupidWarnings.h" - #include "Cholesky" #include "Jacobi" #include "Householder" +#include "src/Core/util/DisableStupidWarnings.h" + /** \defgroup QR_Module QR module * * @@ -36,8 +36,13 @@ #include "src/QR/ColPivHouseholderQR.h" #include "src/QR/CompleteOrthogonalDecomposition.h" #ifdef EIGEN_USE_LAPACKE -#include "src/QR/HouseholderQR_MKL.h" -#include "src/QR/ColPivHouseholderQR_MKL.h" +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "src/misc/lapacke.h" +#endif +#include "src/QR/HouseholderQR_LAPACKE.h" +#include "src/QR/ColPivHouseholderQR_LAPACKE.h" #endif #include "src/Core/util/ReenableStupidWarnings.h"
diff --git a/Eigen/QtAlignedMalloc b/Eigen/QtAlignedMalloc index 4044d5a..4f07df0 100644 --- a/Eigen/QtAlignedMalloc +++ b/Eigen/QtAlignedMalloc
@@ -14,7 +14,7 @@ #include "src/Core/util/DisableStupidWarnings.h" -void *qMalloc(size_t size) +void *qMalloc(std::size_t size) { return Eigen::internal::aligned_malloc(size); } @@ -24,10 +24,10 @@ Eigen::internal::aligned_free(ptr); } -void *qRealloc(void *ptr, size_t size) +void *qRealloc(void *ptr, std::size_t size) { void* newPtr = Eigen::internal::aligned_malloc(size); - memcpy(newPtr, ptr, size); + std::memcpy(newPtr, ptr, size); Eigen::internal::aligned_free(ptr); return newPtr; }
diff --git a/Eigen/SVD b/Eigen/SVD index b353f3f..5d0e75f 100644 --- a/Eigen/SVD +++ b/Eigen/SVD
@@ -31,12 +31,18 @@ * \endcode */ +#include "src/misc/RealSvd2x2.h" #include "src/SVD/UpperBidiagonalization.h" #include "src/SVD/SVDBase.h" #include "src/SVD/JacobiSVD.h" #include "src/SVD/BDCSVD.h" #if defined(EIGEN_USE_LAPACKE) && !defined(EIGEN_USE_LAPACKE_STRICT) -#include "src/SVD/JacobiSVD_MKL.h" +#ifdef EIGEN_USE_MKL +#include "mkl_lapacke.h" +#else +#include "src/misc/lapacke.h" +#endif +#include "src/SVD/JacobiSVD_LAPACKE.h" #endif #include "src/Core/util/ReenableStupidWarnings.h"
diff --git a/Eigen/Sparse b/Eigen/Sparse index a2ef7a6..136e681 100644 --- a/Eigen/Sparse +++ b/Eigen/Sparse
@@ -25,7 +25,9 @@ #include "SparseCore" #include "OrderingMethods" +#ifndef EIGEN_MPL2_ONLY #include "SparseCholesky" +#endif #include "SparseLU" #include "SparseQR" #include "IterativeLinearSolvers"
diff --git a/Eigen/SparseLU b/Eigen/SparseLU index 38b38b5..37c4a5c 100644 --- a/Eigen/SparseLU +++ b/Eigen/SparseLU
@@ -23,6 +23,8 @@ // Ordering interface #include "OrderingMethods" +#include "src/Core/util/DisableStupidWarnings.h" + #include "src/SparseLU/SparseLU_gemm_kernel.h" #include "src/SparseLU/SparseLU_Structs.h" @@ -43,4 +45,6 @@ #include "src/SparseLU/SparseLU_Utils.h" #include "src/SparseLU/SparseLU.h" +#include "src/Core/util/ReenableStupidWarnings.h" + #endif // EIGEN_SPARSELU_MODULE_H
diff --git a/Eigen/SparseQR b/Eigen/SparseQR index a6f3b7f..f5fc5fa 100644 --- a/Eigen/SparseQR +++ b/Eigen/SparseQR
@@ -28,7 +28,6 @@ * */ -#include "OrderingMethods" #include "src/SparseCore/SparseColEtree.h" #include "src/SparseQR/SparseQR.h"
diff --git a/Eigen/StdDeque b/Eigen/StdDeque index be3a7f8..bc68397 100644 --- a/Eigen/StdDeque +++ b/Eigen/StdDeque
@@ -14,7 +14,7 @@ #include "Core" #include <deque> -#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 /* MSVC auto aligns in 64 bit builds */ +#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ #define EIGEN_DEFINE_STL_DEQUE_SPECIALIZATION(...)
diff --git a/Eigen/StdList b/Eigen/StdList index 07ba129..4c6262c 100644 --- a/Eigen/StdList +++ b/Eigen/StdList
@@ -13,7 +13,7 @@ #include "Core" #include <list> -#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 /* MSVC auto aligns in 64 bit builds */ +#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ #define EIGEN_DEFINE_STL_LIST_SPECIALIZATION(...)
diff --git a/Eigen/StdVector b/Eigen/StdVector index fdfc377..0c4697a 100644 --- a/Eigen/StdVector +++ b/Eigen/StdVector
@@ -14,7 +14,7 @@ #include "Core" #include <vector> -#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 /* MSVC auto aligns in 64 bit builds */ +#if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ #define EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION(...)
diff --git a/Eigen/SuperLUSupport b/Eigen/SuperLUSupport index 113f58e..59312a8 100644 --- a/Eigen/SuperLUSupport +++ b/Eigen/SuperLUSupport
@@ -43,7 +43,7 @@ * - class SuperLU: a supernodal sequential LU factorization. * - class SuperILU: a supernodal sequential incomplete LU factorization (to be used as a preconditioner for iterative methods). * - * \warning This wrapper is only for the 4.x versions of SuperLU. The 3.x and 5.x versions are not supported. + * \warning This wrapper requires at least versions 4.0 of SuperLU. The 3.x versions are not supported. * * \warning When including this module, you have to use SUPERLU_EMPTY instead of EMPTY which is no longer defined because it is too polluting. *
diff --git a/Eigen/src/CMakeLists.txt b/Eigen/src/CMakeLists.txt deleted file mode 100644 index c326f37..0000000 --- a/Eigen/src/CMakeLists.txt +++ /dev/null
@@ -1,7 +0,0 @@ -file(GLOB Eigen_src_subdirectories "*") -escape_string_as_regex(ESCAPED_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") -foreach(f ${Eigen_src_subdirectories}) - if(NOT f MATCHES "\\.txt" AND NOT f MATCHES "${ESCAPED_CMAKE_CURRENT_SOURCE_DIR}/[.].+" ) - add_subdirectory(${f}) - endif() -endforeach()
diff --git a/Eigen/src/Cholesky/CMakeLists.txt b/Eigen/src/Cholesky/CMakeLists.txt deleted file mode 100644 index d01488b..0000000 --- a/Eigen/src/Cholesky/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Cholesky_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Cholesky_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Cholesky COMPONENT Devel - )
diff --git a/Eigen/src/Cholesky/LDLT.h b/Eigen/src/Cholesky/LDLT.h index 538aff9..67e97ff 100644 --- a/Eigen/src/Cholesky/LDLT.h +++ b/Eigen/src/Cholesky/LDLT.h
@@ -16,6 +16,15 @@ namespace Eigen { namespace internal { + template<typename _MatrixType, int _UpLo> struct traits<LDLT<_MatrixType, _UpLo> > + : traits<_MatrixType> + { + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; + enum { Flags = 0 }; + }; + template<typename MatrixType, int UpLo> struct LDLT_Traits; // PositiveSemiDef means positive semi-definite and non-zero; same for NegativeSemiDef @@ -43,25 +52,25 @@ * Remember that Cholesky decompositions are not rank-revealing. Also, do not use a Cholesky * decomposition to determine whether a system of equations has a solution. * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::ldlt(), SelfAdjointView::ldlt(), class LLT */ template<typename _MatrixType, int _UpLo> class LDLT + : public SolverBase<LDLT<_MatrixType, _UpLo> > { public: typedef _MatrixType MatrixType; + typedef SolverBase<LDLT> Base; + friend class SolverBase<LDLT>; + + EIGEN_GENERIC_PUBLIC_INTERFACE(LDLT) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options & ~RowMajorBit, // these are the options for the TmpMatrixType, we need a ColMajor matrix here! MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, UpLo = _UpLo }; - typedef typename MatrixType::Scalar Scalar; - typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; - typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 - typedef typename MatrixType::StorageIndex StorageIndex; - typedef Matrix<Scalar, RowsAtCompileTime, 1, Options, MaxRowsAtCompileTime, 1> TmpMatrixType; + typedef Matrix<Scalar, RowsAtCompileTime, 1, 0, MaxRowsAtCompileTime, 1> TmpMatrixType; typedef Transpositions<RowsAtCompileTime, MaxRowsAtCompileTime> TranspositionType; typedef PermutationMatrix<RowsAtCompileTime, MaxRowsAtCompileTime> PermutationType; @@ -97,6 +106,7 @@ /** \brief Constructor with decomposition * * This calculates the decomposition for the input \a matrix. + * * \sa LDLT(Index size) */ template<typename InputType> @@ -110,6 +120,23 @@ compute(matrix.derived()); } + /** \brief Constructs a LDLT factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when \c MatrixType is a Eigen::Ref. + * + * \sa LDLT(const EigenBase&) + */ + template<typename InputType> + explicit LDLT(EigenBase<InputType>& matrix) + : m_matrix(matrix.derived()), + m_transpositions(matrix.rows()), + m_temporary(matrix.rows()), + m_sign(internal::ZeroSign), + m_isInitialized(false) + { + compute(matrix.derived()); + } + /** Clear any existing decomposition * \sa rankUpdate(w,sigma) */ @@ -161,6 +188,7 @@ return m_sign == internal::NegativeSemiDef || m_sign == internal::ZeroSign; } + #ifdef EIGEN_PARSED_BY_DOXYGEN /** \returns a solution x of \f$ A x = b \f$ using the current decomposition of A. * * This function also supports in-place solves using the syntax <tt>x = decompositionObject.solve(x)</tt> . @@ -178,13 +206,8 @@ */ template<typename Rhs> inline const Solve<LDLT, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "LDLT is not initialized."); - eigen_assert(m_matrix.rows()==b.rows() - && "LDLT::solve(): invalid number of rows of the right hand side matrix b"); - return Solve<LDLT, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif template<typename Derived> bool solveInPlace(MatrixBase<Derived> &bAndX) const; @@ -228,19 +251,21 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, - * \c NumericalIssue if the matrix.appears to be negative. + * \returns \c Success if computation was successful, + * \c NumericalIssue if the factorization failed because of a zero pivot. */ ComputationInfo info() const { eigen_assert(m_isInitialized && "LDLT is not initialized."); - return Success; + return m_info; } #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: @@ -262,6 +287,7 @@ TmpMatrixType m_temporary; internal::SignMatrix m_sign; bool m_isInitialized; + ComputationInfo m_info; }; namespace internal { @@ -279,11 +305,14 @@ typedef typename TranspositionType::StorageIndex IndexType; eigen_assert(mat.rows()==mat.cols()); const Index size = mat.rows(); + bool found_zero_pivot = false; + bool ret = true; if (size <= 1) { transpositions.setIdentity(); - if (numext::real(mat.coeff(0,0)) > static_cast<RealScalar>(0) ) sign = PositiveSemiDef; + if(size==0) sign = ZeroSign; + else if (numext::real(mat.coeff(0,0)) > static_cast<RealScalar>(0) ) sign = PositiveSemiDef; else if (numext::real(mat.coeff(0,0)) < static_cast<RealScalar>(0)) sign = NegativeSemiDef; else sign = ZeroSign; return true; @@ -337,8 +366,28 @@ // we should only make sure that we do not introduce INF or NaN values. // Remark that LAPACK also uses 0 as the cutoff value. RealScalar realAkk = numext::real(mat.coeffRef(k,k)); - if((rs>0) && (abs(realAkk) > RealScalar(0))) + bool pivot_is_valid = (abs(realAkk) > RealScalar(0)); + + if(k==0 && !pivot_is_valid) + { + // The entire diagonal is zero, there is nothing more to do + // except filling the transpositions, and checking whether the matrix is zero. + sign = ZeroSign; + for(Index j = 0; j<size; ++j) + { + transpositions.coeffRef(j) = IndexType(j); + ret = ret && (mat.col(j).tail(size-j-1).array()==Scalar(0)).all(); + } + return ret; + } + + if((rs>0) && pivot_is_valid) A21 /= realAkk; + else if(rs>0) + ret = ret && (A21.array()==Scalar(0)).all(); + + if(found_zero_pivot && pivot_is_valid) ret = false; // factorization failed + else if(!pivot_is_valid) found_zero_pivot = true; if (sign == PositiveSemiDef) { if (realAkk < static_cast<RealScalar>(0)) sign = Indefinite; @@ -350,7 +399,7 @@ } } - return true; + return ret; } // Reference for the algorithm: Davis and Hager, "Multiple Rank @@ -474,7 +523,7 @@ m_temporary.resize(size); m_sign = internal::ZeroSign; - internal::ldlt_inplace<UpLo>::unblocked(m_matrix, m_transpositions, m_temporary, m_sign); + m_info = internal::ldlt_inplace<UpLo>::unblocked(m_matrix, m_transpositions, m_temporary, m_sign) ? Success : NumericalIssue; m_isInitialized = true; return *this; @@ -517,25 +566,33 @@ template<typename RhsType, typename DstType> void LDLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const { - eigen_assert(rhs.rows() == rows()); + _solve_impl_transposed<true>(rhs, dst); +} + +template<typename _MatrixType,int _UpLo> +template<bool Conjugate, typename RhsType, typename DstType> +void LDLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ // dst = P b dst = m_transpositions * rhs; // dst = L^-1 (P b) - matrixL().solveInPlace(dst); + // dst = L^-*T (P b) + matrixL().template conjugateIf<!Conjugate>().solveInPlace(dst); - // dst = D^-1 (L^-1 P b) + // dst = D^-* (L^-1 P b) + // dst = D^-1 (L^-*T P b) // more precisely, use pseudo-inverse of D (see bug 241) using std::abs; const typename Diagonal<const MatrixType>::RealReturnType vecD(vectorD()); - // In some previous versions, tolerance was set to the max of 1/highest and the maximal diagonal entry * epsilon - // as motivated by LAPACK's xGELSS: + // In some previous versions, tolerance was set to the max of 1/highest (or rather numeric_limits::min()) + // and the maximal diagonal entry * epsilon as motivated by LAPACK's xGELSS: // RealScalar tolerance = numext::maxi(vecD.array().abs().maxCoeff() * NumTraits<RealScalar>::epsilon(),RealScalar(1) / NumTraits<RealScalar>::highest()); // However, LDLT is not rank revealing, and so adjusting the tolerance wrt to the highest // diagonal element is not well justified and leads to numerical issues in some cases. // Moreover, Lapack's xSYTRS routines use 0 for the tolerance. - RealScalar tolerance = RealScalar(1) / NumTraits<RealScalar>::highest(); - + // Using numeric_limits::min() gives us more robustness to denormals. + RealScalar tolerance = (std::numeric_limits<RealScalar>::min)(); for (Index i = 0; i < vecD.size(); ++i) { if(abs(vecD(i)) > tolerance) @@ -544,10 +601,12 @@ dst.row(i).setZero(); } - // dst = L^-T (D^-1 L^-1 P b) - matrixU().solveInPlace(dst); + // dst = L^-* (D^-* L^-1 P b) + // dst = L^-T (D^-1 L^-*T P b) + matrixL().transpose().template conjugateIf<Conjugate>().solveInPlace(dst); - // dst = P^-1 (L^-T D^-1 L^-1 P b) = A^-1 b + // dst = P^T (L^-* D^-* L^-1 P b) = A^-1 b + // dst = P^-T (L^-T D^-1 L^-*T P b) = A^-1 b dst = m_transpositions.transpose() * dst; } #endif @@ -602,7 +661,6 @@ return res; } -#ifndef __CUDACC__ /** \cholesky_module * \returns the Cholesky decomposition with full pivoting without square root of \c *this * \sa MatrixBase::ldlt() @@ -624,7 +682,6 @@ { return LDLT<PlainObject>(derived()); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/Cholesky/LLT.h b/Eigen/src/Cholesky/LLT.h index 19578b2..5876966 100644 --- a/Eigen/src/Cholesky/LLT.h +++ b/Eigen/src/Cholesky/LLT.h
@@ -13,6 +13,16 @@ namespace Eigen { namespace internal{ + +template<typename _MatrixType, int _UpLo> struct traits<LLT<_MatrixType, _UpLo> > + : traits<_MatrixType> +{ + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; + enum { Flags = 0 }; +}; + template<typename MatrixType, int UpLo> struct LLT_Traits; } @@ -24,7 +34,7 @@ * * \tparam _MatrixType the type of the matrix of which we are computing the LL^T Cholesky decomposition * \tparam _UpLo the triangular part that will be used for the decompositon: Lower (default) or Upper. - * The other triangular part won't be read. + * The other triangular part won't be read. * * This class performs a LL^T Cholesky decomposition of a symmetric, positive definite * matrix A such that A = LL^* = U^*U, where L is lower triangular. @@ -41,26 +51,30 @@ * Example: \include LLT_example.cpp * Output: \verbinclude LLT_example.out * + * \b Performance: for best performance, it is recommended to use a column-major storage format + * with the Lower triangular part (the default), or, equivalently, a row-major storage format + * with the Upper triangular part. Otherwise, you might get a 20% slowdown for the full factorization + * step, and rank-updates can be up to 3 times slower. + * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * + * Note that during the decomposition, only the lower (or upper, as defined by _UpLo) triangular part of A is considered. + * Therefore, the strict lower part does not have to store correct values. + * * \sa MatrixBase::llt(), SelfAdjointView::llt(), class LDLT */ - /* HEY THIS DOX IS DISABLED BECAUSE THERE's A BUG EITHER HERE OR IN LDLT ABOUT THAT (OR BOTH) - * Note that during the decomposition, only the upper triangular part of A is considered. Therefore, - * the strict lower part does not have to store correct values. - */ template<typename _MatrixType, int _UpLo> class LLT + : public SolverBase<LLT<_MatrixType, _UpLo> > { public: typedef _MatrixType MatrixType; + typedef SolverBase<LLT> Base; + friend class SolverBase<LLT>; + + EIGEN_GENERIC_PUBLIC_INTERFACE(LLT) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename MatrixType::Scalar Scalar; - typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; - typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 - typedef typename MatrixType::StorageIndex StorageIndex; enum { PacketSize = internal::packet_traits<Scalar>::size, @@ -95,6 +109,21 @@ compute(matrix.derived()); } + /** \brief Constructs a LLT factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when + * \c MatrixType is a Eigen::Ref. + * + * \sa LLT(const EigenBase&) + */ + template<typename InputType> + explicit LLT(EigenBase<InputType>& matrix) + : m_matrix(matrix.derived()), + m_isInitialized(false) + { + compute(matrix.derived()); + } + /** \returns a view of the upper triangular matrix U */ inline typename Traits::MatrixU matrixU() const { @@ -109,6 +138,7 @@ return Traits::getL(m_matrix); } + #ifdef EIGEN_PARSED_BY_DOXYGEN /** \returns the solution x of \f$ A x = b \f$ using the current decomposition of A. * * Since this LLT class assumes anyway that the matrix A is invertible, the solution @@ -121,16 +151,11 @@ */ template<typename Rhs> inline const Solve<LLT, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "LLT is not initialized."); - eigen_assert(m_matrix.rows()==b.rows() - && "LLT::solve(): invalid number of rows of the right hand side matrix b"); - return Solve<LLT, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif template<typename Derived> - void solveInPlace(MatrixBase<Derived> &bAndX) const; + void solveInPlace(const MatrixBase<Derived> &bAndX) const; template<typename InputType> LLT& compute(const EigenBase<InputType>& matrix); @@ -160,8 +185,8 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, - * \c NumericalIssue if the matrix.appears to be negative. + * \returns \c Success if computation was successful, + * \c NumericalIssue if the matrix.appears not to be positive definite. */ ComputationInfo info() const { @@ -180,12 +205,14 @@ inline Index cols() const { return m_matrix.cols(); } template<typename VectorType> - LLT rankUpdate(const VectorType& vec, const RealScalar& sigma = 1); + LLT & rankUpdate(const VectorType& vec, const RealScalar& sigma = 1); #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: @@ -335,7 +362,7 @@ Index ret; if((ret=unblocked(A11))>=0) return k+ret; if(rs>0) A11.adjoint().template triangularView<Upper>().template solveInPlace<OnTheRight>(A21); - if(rs>0) A22.template selfadjointView<Lower>().rankUpdate(A21,-1); // bottleneck + if(rs>0) A22.template selfadjointView<Lower>().rankUpdate(A21,typename NumTraits<RealScalar>::Literal(-1)); // bottleneck } return -1; } @@ -409,7 +436,8 @@ eigen_assert(a.rows()==a.cols()); const Index size = a.rows(); m_matrix.resize(size, size); - m_matrix = a.derived(); + if (!internal::is_same_dense(m_matrix, a.derived())) + m_matrix = a.derived(); // Compute matrix L1 norm = max abs column sum. m_l1_norm = RealScalar(0); @@ -438,7 +466,7 @@ */ template<typename _MatrixType, int _UpLo> template<typename VectorType> -LLT<_MatrixType,_UpLo> LLT<_MatrixType,_UpLo>::rankUpdate(const VectorType& v, const RealScalar& sigma) +LLT<_MatrixType,_UpLo> & LLT<_MatrixType,_UpLo>::rankUpdate(const VectorType& v, const RealScalar& sigma) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorType); eigen_assert(v.size()==m_matrix.cols()); @@ -456,8 +484,17 @@ template<typename RhsType, typename DstType> void LLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const { - dst = rhs; - solveInPlace(dst); + _solve_impl_transposed<true>(rhs, dst); +} + +template<typename _MatrixType,int _UpLo> +template<bool Conjugate, typename RhsType, typename DstType> +void LLT<_MatrixType,_UpLo>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + dst = rhs; + + matrixL().template conjugateIf<!Conjugate>().solveInPlace(dst); + matrixU().template conjugateIf<!Conjugate>().solveInPlace(dst); } #endif @@ -469,11 +506,14 @@ * * This version avoids a copy when the right hand side matrix b is not needed anymore. * + * \warning The parameter is only marked 'const' to make the C++ compiler accept a temporary expression here. + * This function will const_cast it, so constness isn't honored here. + * * \sa LLT::solve(), MatrixBase::llt() */ template<typename MatrixType, int _UpLo> template<typename Derived> -void LLT<MatrixType,_UpLo>::solveInPlace(MatrixBase<Derived> &bAndX) const +void LLT<MatrixType,_UpLo>::solveInPlace(const MatrixBase<Derived> &bAndX) const { eigen_assert(m_isInitialized && "LLT is not initialized."); eigen_assert(m_matrix.rows()==bAndX.rows()); @@ -491,7 +531,6 @@ return matrixL() * matrixL().adjoint().toDenseMatrix(); } -#ifndef __CUDACC__ /** \cholesky_module * \returns the LLT decomposition of \c *this * \sa SelfAdjointView::llt() @@ -513,7 +552,6 @@ { return LLT<PlainObject,UpLo>(m_matrix); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/Cholesky/LLT_MKL.h b/Eigen/src/Cholesky/LLT_LAPACKE.h similarity index 80% rename from Eigen/src/Cholesky/LLT_MKL.h rename to Eigen/src/Cholesky/LLT_LAPACKE.h index 0d42cb5..bc6489e 100644 --- a/Eigen/src/Cholesky/LLT_MKL.h +++ b/Eigen/src/Cholesky/LLT_LAPACKE.h
@@ -25,25 +25,22 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * LLt decomposition based on LAPACKE_?potrf function. ******************************************************************************** */ -#ifndef EIGEN_LLT_MKL_H -#define EIGEN_LLT_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" -#include <iostream> +#ifndef EIGEN_LLT_LAPACKE_H +#define EIGEN_LLT_LAPACKE_H namespace Eigen { namespace internal { -template<typename Scalar> struct mkl_llt; +template<typename Scalar> struct lapacke_llt; -#define EIGEN_MKL_LLT(EIGTYPE, MKLTYPE, MKLPREFIX) \ -template<> struct mkl_llt<EIGTYPE> \ +#define EIGEN_LAPACKE_LLT(EIGTYPE, BLASTYPE, LAPACKE_PREFIX) \ +template<> struct lapacke_llt<EIGTYPE> \ { \ template<typename MatrixType> \ static inline Index potrf(MatrixType& m, char uplo) \ @@ -53,13 +50,13 @@ EIGTYPE* a; \ eigen_assert(m.rows()==m.cols()); \ /* Set up parameters for ?potrf */ \ - size = m.rows(); \ + size = convert_index<lapack_int>(m.rows()); \ StorageOrder = MatrixType::Flags&RowMajorBit?RowMajor:ColMajor; \ matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ a = &(m.coeffRef(0,0)); \ - lda = m.outerStride(); \ + lda = convert_index<lapack_int>(m.outerStride()); \ \ - info = LAPACKE_##MKLPREFIX##potrf( matrix_order, uplo, size, (MKLTYPE*)a, lda ); \ + info = LAPACKE_##LAPACKE_PREFIX##potrf( matrix_order, uplo, size, (BLASTYPE*)a, lda ); \ info = (info==0) ? -1 : info>0 ? info-1 : size; \ return info; \ } \ @@ -69,7 +66,7 @@ template<typename MatrixType> \ static Index blocked(MatrixType& m) \ { \ - return mkl_llt<EIGTYPE>::potrf(m, 'L'); \ + return lapacke_llt<EIGTYPE>::potrf(m, 'L'); \ } \ template<typename MatrixType, typename VectorType> \ static Index rankUpdate(MatrixType& mat, const VectorType& vec, const typename MatrixType::RealScalar& sigma) \ @@ -80,7 +77,7 @@ template<typename MatrixType> \ static Index blocked(MatrixType& m) \ { \ - return mkl_llt<EIGTYPE>::potrf(m, 'U'); \ + return lapacke_llt<EIGTYPE>::potrf(m, 'U'); \ } \ template<typename MatrixType, typename VectorType> \ static Index rankUpdate(MatrixType& mat, const VectorType& vec, const typename MatrixType::RealScalar& sigma) \ @@ -90,13 +87,13 @@ } \ }; -EIGEN_MKL_LLT(double, double, d) -EIGEN_MKL_LLT(float, float, s) -EIGEN_MKL_LLT(dcomplex, MKL_Complex16, z) -EIGEN_MKL_LLT(scomplex, MKL_Complex8, c) +EIGEN_LAPACKE_LLT(double, double, d) +EIGEN_LAPACKE_LLT(float, float, s) +EIGEN_LAPACKE_LLT(dcomplex, lapack_complex_double, z) +EIGEN_LAPACKE_LLT(scomplex, lapack_complex_float, c) } // end namespace internal } // end namespace Eigen -#endif // EIGEN_LLT_MKL_H +#endif // EIGEN_LLT_LAPACKE_H
diff --git a/Eigen/src/CholmodSupport/CMakeLists.txt b/Eigen/src/CholmodSupport/CMakeLists.txt deleted file mode 100644 index 814dfa6..0000000 --- a/Eigen/src/CholmodSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_CholmodSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_CholmodSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/CholmodSupport COMPONENT Devel - )
diff --git a/Eigen/src/CholmodSupport/CholmodSupport.h b/Eigen/src/CholmodSupport/CholmodSupport.h index b8020a9..adaf528 100644 --- a/Eigen/src/CholmodSupport/CholmodSupport.h +++ b/Eigen/src/CholmodSupport/CholmodSupport.h
@@ -10,38 +10,44 @@ #ifndef EIGEN_CHOLMODSUPPORT_H #define EIGEN_CHOLMODSUPPORT_H -namespace Eigen { +namespace Eigen { namespace internal { -template<typename Scalar, typename CholmodType> -void cholmod_configure_matrix(CholmodType& mat) -{ - if (internal::is_same<Scalar,float>::value) - { - mat.xtype = CHOLMOD_REAL; - mat.dtype = CHOLMOD_SINGLE; - } - else if (internal::is_same<Scalar,double>::value) - { +template<typename Scalar> struct cholmod_configure_matrix; + +template<> struct cholmod_configure_matrix<double> { + template<typename CholmodType> + static void run(CholmodType& mat) { mat.xtype = CHOLMOD_REAL; mat.dtype = CHOLMOD_DOUBLE; } - else if (internal::is_same<Scalar,std::complex<float> >::value) - { - mat.xtype = CHOLMOD_COMPLEX; - mat.dtype = CHOLMOD_SINGLE; - } - else if (internal::is_same<Scalar,std::complex<double> >::value) - { +}; + +template<> struct cholmod_configure_matrix<std::complex<double> > { + template<typename CholmodType> + static void run(CholmodType& mat) { mat.xtype = CHOLMOD_COMPLEX; mat.dtype = CHOLMOD_DOUBLE; } - else - { - eigen_assert(false && "Scalar type not supported by CHOLMOD"); - } -} +}; + +// Other scalar types are not yet supported by Cholmod +// template<> struct cholmod_configure_matrix<float> { +// template<typename CholmodType> +// static void run(CholmodType& mat) { +// mat.xtype = CHOLMOD_REAL; +// mat.dtype = CHOLMOD_SINGLE; +// } +// }; +// +// template<> struct cholmod_configure_matrix<std::complex<float> > { +// template<typename CholmodType> +// static void run(CholmodType& mat) { +// mat.xtype = CHOLMOD_COMPLEX; +// mat.dtype = CHOLMOD_SINGLE; +// } +// }; } // namespace internal @@ -49,11 +55,11 @@ * Note that the data are shared. */ template<typename _Scalar, int _Options, typename _StorageIndex> -cholmod_sparse viewAsCholmod(SparseMatrix<_Scalar,_Options,_StorageIndex>& mat) +cholmod_sparse viewAsCholmod(Ref<SparseMatrix<_Scalar,_Options,_StorageIndex> > mat) { cholmod_sparse res; res.nzmax = mat.nonZeros(); - res.nrow = mat.rows();; + res.nrow = mat.rows(); res.ncol = mat.cols(); res.p = mat.outerIndexPtr(); res.i = mat.innerIndexPtr(); @@ -73,12 +79,12 @@ res.dtype = 0; res.stype = -1; - + if (internal::is_same<_StorageIndex,int>::value) { res.itype = CHOLMOD_INT; } - else if (internal::is_same<_StorageIndex,long>::value) + else if (internal::is_same<_StorageIndex,SuiteSparse_long>::value) { res.itype = CHOLMOD_LONG; } @@ -88,17 +94,24 @@ } // setup res.xtype - internal::cholmod_configure_matrix<_Scalar>(res); - + internal::cholmod_configure_matrix<_Scalar>::run(res); + res.stype = 0; - + return res; } template<typename _Scalar, int _Options, typename _Index> const cholmod_sparse viewAsCholmod(const SparseMatrix<_Scalar,_Options,_Index>& mat) { - cholmod_sparse res = viewAsCholmod(mat.const_cast_derived()); + cholmod_sparse res = viewAsCholmod(Ref<SparseMatrix<_Scalar,_Options,_Index> >(mat.const_cast_derived())); + return res; +} + +template<typename _Scalar, int _Options, typename _Index> +const cholmod_sparse viewAsCholmod(const SparseVector<_Scalar,_Options,_Index>& mat) +{ + cholmod_sparse res = viewAsCholmod(Ref<SparseMatrix<_Scalar,_Options,_Index> >(mat.const_cast_derived())); return res; } @@ -107,10 +120,13 @@ template<typename _Scalar, int _Options, typename _Index, unsigned int UpLo> cholmod_sparse viewAsCholmod(const SparseSelfAdjointView<const SparseMatrix<_Scalar,_Options,_Index>, UpLo>& mat) { - cholmod_sparse res = viewAsCholmod(mat.matrix().const_cast_derived()); - + cholmod_sparse res = viewAsCholmod(Ref<SparseMatrix<_Scalar,_Options,_Index> >(mat.matrix().const_cast_derived())); + if(UpLo==Upper) res.stype = 1; if(UpLo==Lower) res.stype = -1; + // swap stype for rowmajor matrices (only works for real matrices) + EIGEN_STATIC_ASSERT((_Options & RowMajorBit) == 0 || NumTraits<_Scalar>::IsComplex == 0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); + if(_Options & RowMajorBit) res.stype *=-1; return res; } @@ -131,7 +147,7 @@ res.x = (void*)(mat.derived().data()); res.z = 0; - internal::cholmod_configure_matrix<Scalar>(res); + internal::cholmod_configure_matrix<Scalar>::run(res); return res; } @@ -146,6 +162,44 @@ static_cast<StorageIndex*>(cm.p), static_cast<StorageIndex*>(cm.i),static_cast<Scalar*>(cm.x) ); } +namespace internal { + +// template specializations for int and long that call the correct cholmod method + +#define EIGEN_CHOLMOD_SPECIALIZE0(ret, name) \ + template<typename _StorageIndex> inline ret cm_ ## name (cholmod_common &Common) { return cholmod_ ## name (&Common); } \ + template<> inline ret cm_ ## name<SuiteSparse_long> (cholmod_common &Common) { return cholmod_l_ ## name (&Common); } + +#define EIGEN_CHOLMOD_SPECIALIZE1(ret, name, t1, a1) \ + template<typename _StorageIndex> inline ret cm_ ## name (t1& a1, cholmod_common &Common) { return cholmod_ ## name (&a1, &Common); } \ + template<> inline ret cm_ ## name<SuiteSparse_long> (t1& a1, cholmod_common &Common) { return cholmod_l_ ## name (&a1, &Common); } + +EIGEN_CHOLMOD_SPECIALIZE0(int, start) +EIGEN_CHOLMOD_SPECIALIZE0(int, finish) + +EIGEN_CHOLMOD_SPECIALIZE1(int, free_factor, cholmod_factor*, L) +EIGEN_CHOLMOD_SPECIALIZE1(int, free_dense, cholmod_dense*, X) +EIGEN_CHOLMOD_SPECIALIZE1(int, free_sparse, cholmod_sparse*, A) + +EIGEN_CHOLMOD_SPECIALIZE1(cholmod_factor*, analyze, cholmod_sparse, A) + +template<typename _StorageIndex> inline cholmod_dense* cm_solve (int sys, cholmod_factor& L, cholmod_dense& B, cholmod_common &Common) { return cholmod_solve (sys, &L, &B, &Common); } +template<> inline cholmod_dense* cm_solve<SuiteSparse_long> (int sys, cholmod_factor& L, cholmod_dense& B, cholmod_common &Common) { return cholmod_l_solve (sys, &L, &B, &Common); } + +template<typename _StorageIndex> inline cholmod_sparse* cm_spsolve (int sys, cholmod_factor& L, cholmod_sparse& B, cholmod_common &Common) { return cholmod_spsolve (sys, &L, &B, &Common); } +template<> inline cholmod_sparse* cm_spsolve<SuiteSparse_long> (int sys, cholmod_factor& L, cholmod_sparse& B, cholmod_common &Common) { return cholmod_l_spsolve (sys, &L, &B, &Common); } + +template<typename _StorageIndex> +inline int cm_factorize_p (cholmod_sparse* A, double beta[2], _StorageIndex* fset, std::size_t fsize, cholmod_factor* L, cholmod_common &Common) { return cholmod_factorize_p (A, beta, fset, fsize, L, &Common); } +template<> +inline int cm_factorize_p<SuiteSparse_long> (cholmod_sparse* A, double beta[2], SuiteSparse_long* fset, std::size_t fsize, cholmod_factor* L, cholmod_common &Common) { return cholmod_l_factorize_p (A, beta, fset, fsize, L, &Common); } + +#undef EIGEN_CHOLMOD_SPECIALIZE0 +#undef EIGEN_CHOLMOD_SPECIALIZE1 + +} // namespace internal + + enum CholmodMode { CholmodAuto, CholmodSimplicialLLt, CholmodSupernodalLLt, CholmodLDLt }; @@ -180,31 +234,33 @@ CholmodBase() : m_cholmodFactor(0), m_info(Success), m_factorizationIsOk(false), m_analysisIsOk(false) { - m_shiftOffset[0] = m_shiftOffset[1] = RealScalar(0.0); - cholmod_start(&m_cholmod); + EIGEN_STATIC_ASSERT((internal::is_same<double,RealScalar>::value), CHOLMOD_SUPPORTS_DOUBLE_PRECISION_ONLY); + m_shiftOffset[0] = m_shiftOffset[1] = 0.0; + internal::cm_start<StorageIndex>(m_cholmod); } explicit CholmodBase(const MatrixType& matrix) : m_cholmodFactor(0), m_info(Success), m_factorizationIsOk(false), m_analysisIsOk(false) { - m_shiftOffset[0] = m_shiftOffset[1] = RealScalar(0.0); - cholmod_start(&m_cholmod); + EIGEN_STATIC_ASSERT((internal::is_same<double,RealScalar>::value), CHOLMOD_SUPPORTS_DOUBLE_PRECISION_ONLY); + m_shiftOffset[0] = m_shiftOffset[1] = 0.0; + internal::cm_start<StorageIndex>(m_cholmod); compute(matrix); } ~CholmodBase() { if(m_cholmodFactor) - cholmod_free_factor(&m_cholmodFactor, &m_cholmod); - cholmod_finish(&m_cholmod); + internal::cm_free_factor<StorageIndex>(m_cholmodFactor, m_cholmod); + internal::cm_finish<StorageIndex>(m_cholmod); } - + inline StorageIndex cols() const { return internal::convert_index<StorageIndex, Index>(m_cholmodFactor->n); } inline StorageIndex rows() const { return internal::convert_index<StorageIndex, Index>(m_cholmodFactor->n); } - + /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix.appears to be negative. */ ComputationInfo info() const @@ -220,29 +276,29 @@ factorize(matrix); return derived(); } - + /** Performs a symbolic decomposition on the sparsity pattern of \a matrix. * * This function is particularly useful when solving for several problems having the same structure. - * + * * \sa factorize() */ void analyzePattern(const MatrixType& matrix) { if(m_cholmodFactor) { - cholmod_free_factor(&m_cholmodFactor, &m_cholmod); + internal::cm_free_factor<StorageIndex>(m_cholmodFactor, m_cholmod); m_cholmodFactor = 0; } cholmod_sparse A = viewAsCholmod(matrix.template selfadjointView<UpLo>()); - m_cholmodFactor = cholmod_analyze(&A, &m_cholmod); - + m_cholmodFactor = internal::cm_analyze<StorageIndex>(A, m_cholmod); + this->m_isInitialized = true; this->m_info = Success; m_analysisIsOk = true; m_factorizationIsOk = false; } - + /** Performs a numeric decomposition of \a matrix * * The given matrix must have the same sparsity pattern as the matrix on which the symbolic decomposition has been performed. @@ -253,17 +309,17 @@ { eigen_assert(m_analysisIsOk && "You must first call analyzePattern()"); cholmod_sparse A = viewAsCholmod(matrix.template selfadjointView<UpLo>()); - cholmod_factorize_p(&A, m_shiftOffset, 0, 0, m_cholmodFactor, &m_cholmod); - + internal::cm_factorize_p<StorageIndex>(&A, m_shiftOffset, 0, 0, m_cholmodFactor, m_cholmod); + // If the factorization failed, minor is the column at which it did. On success minor == n. this->m_info = (m_cholmodFactor->minor == m_cholmodFactor->n ? Success : NumericalIssue); m_factorizationIsOk = true; } - + /** Returns a reference to the Cholmod's configuration structure to get a full control over the performed operations. * See the Cholmod user guide for details. */ cholmod_common& cholmod() { return m_cholmod; } - + #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal */ template<typename Rhs,typename Dest> @@ -273,25 +329,26 @@ const Index size = m_cholmodFactor->n; EIGEN_UNUSED_VARIABLE(size); eigen_assert(size==b.rows()); - - // Cholmod needs column-major stoarge without inner-stride, which corresponds to the default behavior of Ref. + + // Cholmod needs column-major storage without inner-stride, which corresponds to the default behavior of Ref. Ref<const Matrix<typename Rhs::Scalar,Dynamic,Dynamic,ColMajor> > b_ref(b.derived()); cholmod_dense b_cd = viewAsCholmod(b_ref); - cholmod_dense* x_cd = cholmod_solve(CHOLMOD_A, m_cholmodFactor, &b_cd, &m_cholmod); + cholmod_dense* x_cd = internal::cm_solve<StorageIndex>(CHOLMOD_A, *m_cholmodFactor, b_cd, m_cholmod); if(!x_cd) { this->m_info = NumericalIssue; return; } // TODO optimize this copy by swapping when possible (be careful with alignment, etc.) + // NOTE Actually, the copy can be avoided by calling cholmod_solve2 instead of cholmod_solve dest = Matrix<Scalar,Dest::RowsAtCompileTime,Dest::ColsAtCompileTime>::Map(reinterpret_cast<Scalar*>(x_cd->x),b.rows(),b.cols()); - cholmod_free_dense(&x_cd, &m_cholmod); + internal::cm_free_dense<StorageIndex>(x_cd, m_cholmod); } - + /** \internal */ - template<typename RhsScalar, int RhsOptions, typename RhsIndex, typename DestScalar, int DestOptions, typename DestIndex> - void _solve_impl(const SparseMatrix<RhsScalar,RhsOptions,RhsIndex> &b, SparseMatrix<DestScalar,DestOptions,DestIndex> &dest) const + template<typename RhsDerived, typename DestDerived> + void _solve_impl(const SparseMatrixBase<RhsDerived> &b, SparseMatrixBase<DestDerived> &dest) const { eigen_assert(m_factorizationIsOk && "The decomposition is not in a valid state for solving, you must first call either compute() or symbolic()/numeric()"); const Index size = m_cholmodFactor->n; @@ -299,20 +356,22 @@ eigen_assert(size==b.rows()); // note: cs stands for Cholmod Sparse - cholmod_sparse b_cs = viewAsCholmod(b); - cholmod_sparse* x_cs = cholmod_spsolve(CHOLMOD_A, m_cholmodFactor, &b_cs, &m_cholmod); + Ref<SparseMatrix<typename RhsDerived::Scalar,ColMajor,typename RhsDerived::StorageIndex> > b_ref(b.const_cast_derived()); + cholmod_sparse b_cs = viewAsCholmod(b_ref); + cholmod_sparse* x_cs = internal::cm_spsolve<StorageIndex>(CHOLMOD_A, *m_cholmodFactor, b_cs, m_cholmod); if(!x_cs) { this->m_info = NumericalIssue; return; } // TODO optimize this copy by swapping when possible (be careful with alignment, etc.) - dest = viewAsEigen<DestScalar,DestOptions,DestIndex>(*x_cs); - cholmod_free_sparse(&x_cs, &m_cholmod); + // NOTE cholmod_spsolve in fact just calls the dense solver for blocks of 4 columns at a time (similar to Eigen's sparse solver) + dest.derived() = viewAsEigen<typename DestDerived::Scalar,ColMajor,typename DestDerived::StorageIndex>(*x_cs); + internal::cm_free_sparse<StorageIndex>(x_cs, m_cholmod); } #endif // EIGEN_PARSED_BY_DOXYGEN - - + + /** Sets the shift parameter that will be used to adjust the diagonal coefficients during the numerical factorization. * * During the numerical factorization, an offset term is added to the diagonal coefficients:\n @@ -324,10 +383,10 @@ */ Derived& setShift(const RealScalar& offset) { - m_shiftOffset[0] = offset; + m_shiftOffset[0] = double(offset); return derived(); } - + /** \returns the determinant of the underlying matrix from the current factorization */ Scalar determinant() const { @@ -382,11 +441,11 @@ template<typename Stream> void dumpMemory(Stream& /*s*/) {} - + protected: mutable cholmod_common m_cholmod; cholmod_factor* m_cholmodFactor; - RealScalar m_shiftOffset[2]; + double m_shiftOffset[2]; mutable ComputationInfo m_info; int m_factorizationIsOk; int m_analysisIsOk; @@ -410,6 +469,8 @@ * * This class supports all kind of SparseMatrix<>: row or column major; upper, lower, or both; compressed or non compressed. * + * \warning Only double precision real and complex scalar types are supported by Cholmod. + * * \sa \ref TutorialSparseSolverConcept, class CholmodSupernodalLLT, class SimplicialLLT */ template<typename _MatrixType, int _UpLo = Lower> @@ -417,11 +478,11 @@ { typedef CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLLT> Base; using Base::m_cholmod; - + public: - + typedef _MatrixType MatrixType; - + CholmodSimplicialLLT() : Base() { init(); } CholmodSimplicialLLT(const MatrixType& matrix) : Base() @@ -459,6 +520,8 @@ * * This class supports all kind of SparseMatrix<>: row or column major; upper, lower, or both; compressed or non compressed. * + * \warning Only double precision real and complex scalar types are supported by Cholmod. + * * \sa \ref TutorialSparseSolverConcept, class CholmodSupernodalLLT, class SimplicialLDLT */ template<typename _MatrixType, int _UpLo = Lower> @@ -466,11 +529,11 @@ { typedef CholmodBase<_MatrixType, _UpLo, CholmodSimplicialLDLT> Base; using Base::m_cholmod; - + public: - + typedef _MatrixType MatrixType; - + CholmodSimplicialLDLT() : Base() { init(); } CholmodSimplicialLDLT(const MatrixType& matrix) : Base() @@ -506,6 +569,8 @@ * * This class supports all kind of SparseMatrix<>: row or column major; upper, lower, or both; compressed or non compressed. * + * \warning Only double precision real and complex scalar types are supported by Cholmod. + * * \sa \ref TutorialSparseSolverConcept */ template<typename _MatrixType, int _UpLo = Lower> @@ -513,11 +578,11 @@ { typedef CholmodBase<_MatrixType, _UpLo, CholmodSupernodalLLT> Base; using Base::m_cholmod; - + public: - + typedef _MatrixType MatrixType; - + CholmodSupernodalLLT() : Base() { init(); } CholmodSupernodalLLT(const MatrixType& matrix) : Base() @@ -555,6 +620,8 @@ * * This class supports all kind of SparseMatrix<>: row or column major; upper, lower, or both; compressed or non compressed. * + * \warning Only double precision real and complex scalar types are supported by Cholmod. + * * \sa \ref TutorialSparseSolverConcept */ template<typename _MatrixType, int _UpLo = Lower> @@ -562,11 +629,11 @@ { typedef CholmodBase<_MatrixType, _UpLo, CholmodDecomposition> Base; using Base::m_cholmod; - + public: - + typedef _MatrixType MatrixType; - + CholmodDecomposition() : Base() { init(); } CholmodDecomposition(const MatrixType& matrix) : Base() @@ -576,7 +643,7 @@ } ~CholmodDecomposition() {} - + void setMode(CholmodMode mode) { switch(mode)
diff --git a/Eigen/src/Core/ArithmeticSequence.h b/Eigen/src/Core/ArithmeticSequence.h new file mode 100644 index 0000000..b6200fa --- /dev/null +++ b/Eigen/src/Core/ArithmeticSequence.h
@@ -0,0 +1,413 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_ARITHMETIC_SEQUENCE_H +#define EIGEN_ARITHMETIC_SEQUENCE_H + +namespace Eigen { + +namespace internal { + +#if (!EIGEN_HAS_CXX11) || !((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) +template<typename T> struct aseq_negate {}; + +template<> struct aseq_negate<Index> { + typedef Index type; +}; + +template<int N> struct aseq_negate<FixedInt<N> > { + typedef FixedInt<-N> type; +}; + +// Compilation error in the following case: +template<> struct aseq_negate<FixedInt<DynamicIndex> > {}; + +template<typename FirstType,typename SizeType,typename IncrType, + bool FirstIsSymbolic=symbolic::is_symbolic<FirstType>::value, + bool SizeIsSymbolic =symbolic::is_symbolic<SizeType>::value> +struct aseq_reverse_first_type { + typedef Index type; +}; + +template<typename FirstType,typename SizeType,typename IncrType> +struct aseq_reverse_first_type<FirstType,SizeType,IncrType,true,true> { + typedef symbolic::AddExpr<FirstType, + symbolic::ProductExpr<symbolic::AddExpr<SizeType,symbolic::ValueExpr<FixedInt<-1> > >, + symbolic::ValueExpr<IncrType> > + > type; +}; + +template<typename SizeType,typename IncrType,typename EnableIf = void> +struct aseq_reverse_first_type_aux { + typedef Index type; +}; + +template<typename SizeType,typename IncrType> +struct aseq_reverse_first_type_aux<SizeType,IncrType,typename internal::enable_if<bool((SizeType::value+IncrType::value)|0x1)>::type> { + typedef FixedInt<(SizeType::value-1)*IncrType::value> type; +}; + +template<typename FirstType,typename SizeType,typename IncrType> +struct aseq_reverse_first_type<FirstType,SizeType,IncrType,true,false> { + typedef typename aseq_reverse_first_type_aux<SizeType,IncrType>::type Aux; + typedef symbolic::AddExpr<FirstType,symbolic::ValueExpr<Aux> > type; +}; + +template<typename FirstType,typename SizeType,typename IncrType> +struct aseq_reverse_first_type<FirstType,SizeType,IncrType,false,true> { + typedef symbolic::AddExpr<symbolic::ProductExpr<symbolic::AddExpr<SizeType,symbolic::ValueExpr<FixedInt<-1> > >, + symbolic::ValueExpr<IncrType> >, + symbolic::ValueExpr<> > type; +}; +#endif + +// Helper to cleanup the type of the increment: +template<typename T> struct cleanup_seq_incr { + typedef typename cleanup_index_type<T,DynamicIndex>::type type; +}; + +} + +//-------------------------------------------------------------------------------- +// seq(first,last,incr) and seqN(first,size,incr) +//-------------------------------------------------------------------------------- + +template<typename FirstType=Index,typename SizeType=Index,typename IncrType=internal::FixedInt<1> > +class ArithmeticSequence; + +template<typename FirstType,typename SizeType,typename IncrType> +ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type, + typename internal::cleanup_index_type<SizeType>::type, + typename internal::cleanup_seq_incr<IncrType>::type > +seqN(FirstType first, SizeType size, IncrType incr); + +/** \class ArithmeticSequence + * \ingroup Core_Module + * + * This class represents an arithmetic progression \f$ a_0, a_1, a_2, ..., a_{n-1}\f$ defined by + * its \em first value \f$ a_0 \f$, its \em size (aka length) \em n, and the \em increment (aka stride) + * that is equal to \f$ a_{i+1}-a_{i}\f$ for any \em i. + * + * It is internally used as the return type of the Eigen::seq and Eigen::seqN functions, and as the input arguments + * of DenseBase::operator()(const RowIndices&, const ColIndices&), and most of the time this is the + * only way it is used. + * + * \tparam FirstType type of the first element, usually an Index, + * but internally it can be a symbolic expression + * \tparam SizeType type representing the size of the sequence, usually an Index + * or a compile time integral constant. Internally, it can also be a symbolic expression + * \tparam IncrType type of the increment, can be a runtime Index, or a compile time integral constant (default is compile-time 1) + * + * \sa Eigen::seq, Eigen::seqN, DenseBase::operator()(const RowIndices&, const ColIndices&), class IndexedView + */ +template<typename FirstType,typename SizeType,typename IncrType> +class ArithmeticSequence +{ +public: + ArithmeticSequence(FirstType first, SizeType size) : m_first(first), m_size(size) {} + ArithmeticSequence(FirstType first, SizeType size, IncrType incr) : m_first(first), m_size(size), m_incr(incr) {} + + enum { + SizeAtCompileTime = internal::get_fixed_value<SizeType>::value, + IncrAtCompileTime = internal::get_fixed_value<IncrType,DynamicIndex>::value + }; + + /** \returns the size, i.e., number of elements, of the sequence */ + Index size() const { return m_size; } + + /** \returns the first element \f$ a_0 \f$ in the sequence */ + Index first() const { return m_first; } + + /** \returns the value \f$ a_i \f$ at index \a i in the sequence. */ + Index operator[](Index i) const { return m_first + i * m_incr; } + + const FirstType& firstObject() const { return m_first; } + const SizeType& sizeObject() const { return m_size; } + const IncrType& incrObject() const { return m_incr; } + +protected: + FirstType m_first; + SizeType m_size; + IncrType m_incr; + +public: + +#if EIGEN_HAS_CXX11 && ((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) + auto reverse() const -> decltype(Eigen::seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr)) { + return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); + } +#else +protected: + typedef typename internal::aseq_negate<IncrType>::type ReverseIncrType; + typedef typename internal::aseq_reverse_first_type<FirstType,SizeType,IncrType>::type ReverseFirstType; +public: + ArithmeticSequence<ReverseFirstType,SizeType,ReverseIncrType> + reverse() const { + return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); + } +#endif +}; + +/** \returns an ArithmeticSequence starting at \a first, of length \a size, and increment \a incr + * + * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */ +template<typename FirstType,typename SizeType,typename IncrType> +ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,typename internal::cleanup_index_type<SizeType>::type,typename internal::cleanup_seq_incr<IncrType>::type > +seqN(FirstType first, SizeType size, IncrType incr) { + return ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,typename internal::cleanup_index_type<SizeType>::type,typename internal::cleanup_seq_incr<IncrType>::type>(first,size,incr); +} + +/** \returns an ArithmeticSequence starting at \a first, of length \a size, and unit increment + * + * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType) */ +template<typename FirstType,typename SizeType> +ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,typename internal::cleanup_index_type<SizeType>::type > +seqN(FirstType first, SizeType size) { + return ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,typename internal::cleanup_index_type<SizeType>::type>(first,size); +} + +#ifdef EIGEN_PARSED_BY_DOXYGEN + +/** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and with positive (or negative) increment \a incr + * + * It is essentially an alias to: + * \code + * seqN(f, (l-f+incr)/incr, incr); + * \endcode + * + * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType) + */ +template<typename FirstType,typename LastType, typename IncrType> +auto seq(FirstType f, LastType l, IncrType incr); + +/** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and unit increment + * + * It is essentially an alias to: + * \code + * seqN(f,l-f+1); + * \endcode + * + * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) + */ +template<typename FirstType,typename LastType> +auto seq(FirstType f, LastType l); + +#else // EIGEN_PARSED_BY_DOXYGEN + +#if EIGEN_HAS_CXX11 +template<typename FirstType,typename LastType> +auto seq(FirstType f, LastType l) -> decltype(seqN(typename internal::cleanup_index_type<FirstType>::type(f), + ( typename internal::cleanup_index_type<LastType>::type(l) + - typename internal::cleanup_index_type<FirstType>::type(f)+fix<1>()))) +{ + return seqN(typename internal::cleanup_index_type<FirstType>::type(f), + (typename internal::cleanup_index_type<LastType>::type(l) + -typename internal::cleanup_index_type<FirstType>::type(f)+fix<1>())); +} + +template<typename FirstType,typename LastType, typename IncrType> +auto seq(FirstType f, LastType l, IncrType incr) + -> decltype(seqN(typename internal::cleanup_index_type<FirstType>::type(f), + ( typename internal::cleanup_index_type<LastType>::type(l) + - typename internal::cleanup_index_type<FirstType>::type(f)+typename internal::cleanup_seq_incr<IncrType>::type(incr) + ) / typename internal::cleanup_seq_incr<IncrType>::type(incr), + typename internal::cleanup_seq_incr<IncrType>::type(incr))) +{ + typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType; + return seqN(typename internal::cleanup_index_type<FirstType>::type(f), + ( typename internal::cleanup_index_type<LastType>::type(l) + -typename internal::cleanup_index_type<FirstType>::type(f)+CleanedIncrType(incr)) / CleanedIncrType(incr), + CleanedIncrType(incr)); +} + +#else // EIGEN_HAS_CXX11 + +template<typename FirstType,typename LastType> +typename internal::enable_if<!(symbolic::is_symbolic<FirstType>::value || symbolic::is_symbolic<LastType>::value), + ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,Index> >::type +seq(FirstType f, LastType l) +{ + return seqN(typename internal::cleanup_index_type<FirstType>::type(f), + Index((typename internal::cleanup_index_type<LastType>::type(l)-typename internal::cleanup_index_type<FirstType>::type(f)+fix<1>()))); +} + +template<typename FirstTypeDerived,typename LastType> +typename internal::enable_if<!symbolic::is_symbolic<LastType>::value, + ArithmeticSequence<FirstTypeDerived, symbolic::AddExpr<symbolic::AddExpr<symbolic::NegateExpr<FirstTypeDerived>,symbolic::ValueExpr<> >, + symbolic::ValueExpr<internal::FixedInt<1> > > > >::type +seq(const symbolic::BaseExpr<FirstTypeDerived> &f, LastType l) +{ + return seqN(f.derived(),(typename internal::cleanup_index_type<LastType>::type(l)-f.derived()+fix<1>())); +} + +template<typename FirstType,typename LastTypeDerived> +typename internal::enable_if<!symbolic::is_symbolic<FirstType>::value, + ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type, + symbolic::AddExpr<symbolic::AddExpr<LastTypeDerived,symbolic::ValueExpr<> >, + symbolic::ValueExpr<internal::FixedInt<1> > > > >::type +seq(FirstType f, const symbolic::BaseExpr<LastTypeDerived> &l) +{ + return seqN(typename internal::cleanup_index_type<FirstType>::type(f),(l.derived()-typename internal::cleanup_index_type<FirstType>::type(f)+fix<1>())); +} + +template<typename FirstTypeDerived,typename LastTypeDerived> +ArithmeticSequence<FirstTypeDerived, + symbolic::AddExpr<symbolic::AddExpr<LastTypeDerived,symbolic::NegateExpr<FirstTypeDerived> >,symbolic::ValueExpr<internal::FixedInt<1> > > > +seq(const symbolic::BaseExpr<FirstTypeDerived> &f, const symbolic::BaseExpr<LastTypeDerived> &l) +{ + return seqN(f.derived(),(l.derived()-f.derived()+fix<1>())); +} + + +template<typename FirstType,typename LastType, typename IncrType> +typename internal::enable_if<!(symbolic::is_symbolic<FirstType>::value || symbolic::is_symbolic<LastType>::value), + ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,Index,typename internal::cleanup_seq_incr<IncrType>::type> >::type +seq(FirstType f, LastType l, IncrType incr) +{ + typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType; + return seqN(typename internal::cleanup_index_type<FirstType>::type(f), + Index((typename internal::cleanup_index_type<LastType>::type(l)-typename internal::cleanup_index_type<FirstType>::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr)), incr); +} + +template<typename FirstTypeDerived,typename LastType, typename IncrType> +typename internal::enable_if<!symbolic::is_symbolic<LastType>::value, + ArithmeticSequence<FirstTypeDerived, + symbolic::QuotientExpr<symbolic::AddExpr<symbolic::AddExpr<symbolic::NegateExpr<FirstTypeDerived>, + symbolic::ValueExpr<> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + typename internal::cleanup_seq_incr<IncrType>::type> >::type +seq(const symbolic::BaseExpr<FirstTypeDerived> &f, LastType l, IncrType incr) +{ + typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType; + return seqN(f.derived(),(typename internal::cleanup_index_type<LastType>::type(l)-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); +} + +template<typename FirstType,typename LastTypeDerived, typename IncrType> +typename internal::enable_if<!symbolic::is_symbolic<FirstType>::value, + ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type, + symbolic::QuotientExpr<symbolic::AddExpr<symbolic::AddExpr<LastTypeDerived,symbolic::ValueExpr<> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + typename internal::cleanup_seq_incr<IncrType>::type> >::type +seq(FirstType f, const symbolic::BaseExpr<LastTypeDerived> &l, IncrType incr) +{ + typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType; + return seqN(typename internal::cleanup_index_type<FirstType>::type(f), + (l.derived()-typename internal::cleanup_index_type<FirstType>::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr), incr); +} + +template<typename FirstTypeDerived,typename LastTypeDerived, typename IncrType> +ArithmeticSequence<FirstTypeDerived, + symbolic::QuotientExpr<symbolic::AddExpr<symbolic::AddExpr<LastTypeDerived, + symbolic::NegateExpr<FirstTypeDerived> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + symbolic::ValueExpr<typename internal::cleanup_seq_incr<IncrType>::type> >, + typename internal::cleanup_seq_incr<IncrType>::type> +seq(const symbolic::BaseExpr<FirstTypeDerived> &f, const symbolic::BaseExpr<LastTypeDerived> &l, IncrType incr) +{ + typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType; + return seqN(f.derived(),(l.derived()-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); +} +#endif // EIGEN_HAS_CXX11 + +#endif // EIGEN_PARSED_BY_DOXYGEN + + +#if EIGEN_HAS_CXX11 || defined(EIGEN_PARSED_BY_DOXYGEN) +/** \cpp11 + * \returns a symbolic ArithmeticSequence representing the last \a size elements with increment \a incr. + * + * It is a shortcut for: \code seqN(last-(size-fix<1>)*incr, size, incr) \endcode + * + * \sa lastN(SizeType), seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */ +template<typename SizeType,typename IncrType> +auto lastN(SizeType size, IncrType incr) +-> decltype(seqN(Eigen::last-(size-fix<1>())*incr, size, incr)) +{ + return seqN(Eigen::last-(size-fix<1>())*incr, size, incr); +} + +/** \cpp11 + * \returns a symbolic ArithmeticSequence representing the last \a size elements with a unit increment. + * + * It is a shortcut for: \code seq(last+fix<1>-size, last) \endcode + * + * \sa lastN(SizeType,IncrType, seqN(FirstType,SizeType), seq(FirstType,LastType) */ +template<typename SizeType> +auto lastN(SizeType size) +-> decltype(seqN(Eigen::last+fix<1>()-size, size)) +{ + return seqN(Eigen::last+fix<1>()-size, size); +} +#endif + +namespace internal { + +// Convert a symbolic span into a usable one (i.e., remove last/end "keywords") +template<typename T> +struct make_size_type { + typedef typename internal::conditional<symbolic::is_symbolic<T>::value, Index, T>::type type; +}; + +template<typename FirstType,typename SizeType,typename IncrType,int XprSize> +struct IndexedViewCompatibleType<ArithmeticSequence<FirstType,SizeType,IncrType>, XprSize> { + typedef ArithmeticSequence<Index,typename make_size_type<SizeType>::type,IncrType> type; +}; + +template<typename FirstType,typename SizeType,typename IncrType> +ArithmeticSequence<Index,typename make_size_type<SizeType>::type,IncrType> +makeIndexedViewCompatible(const ArithmeticSequence<FirstType,SizeType,IncrType>& ids, Index size,SpecializedType) { + return ArithmeticSequence<Index,typename make_size_type<SizeType>::type,IncrType>( + eval_expr_given_size(ids.firstObject(),size),eval_expr_given_size(ids.sizeObject(),size),ids.incrObject()); +} + +template<typename FirstType,typename SizeType,typename IncrType> +struct get_compile_time_incr<ArithmeticSequence<FirstType,SizeType,IncrType> > { + enum { value = get_fixed_value<IncrType,DynamicIndex>::value }; +}; + +} // end namespace internal + +/** \namespace Eigen::indexing + * \ingroup Core_Module + * + * The sole purpose of this namespace is to be able to import all functions + * and symbols that are expected to be used within operator() for indexing + * and slicing. If you already imported the whole Eigen namespace: + * \code using namespace Eigen; \endcode + * then you are already all set. Otherwise, if you don't want/cannot import + * the whole Eigen namespace, the following line: + * \code using namespace Eigen::indexing; \endcode + * is equivalent to: + * \code + using Eigen::all; + using Eigen::seq; + using Eigen::seqN; + using Eigen::lastN; // c++11 only + using Eigen::last; + using Eigen::lastp1; + using Eigen::fix; + \endcode + */ +namespace indexing { + using Eigen::all; + using Eigen::seq; + using Eigen::seqN; + #if EIGEN_HAS_CXX11 + using Eigen::lastN; + #endif + using Eigen::last; + using Eigen::lastp1; + using Eigen::fix; +} + +} // end namespace Eigen + +#endif // EIGEN_ARITHMETIC_SEQUENCE_H
diff --git a/Eigen/src/Core/Array.h b/Eigen/src/Core/Array.h index af04ad3..039f41a 100644 --- a/Eigen/src/Core/Array.h +++ b/Eigen/src/Core/Array.h
@@ -37,7 +37,7 @@ * storage layout. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_ARRAY_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAY_PLUGIN. * * \sa \blank \ref TutorialArrayClass, \ref TopicClassHierarchy */ @@ -147,17 +147,15 @@ } #endif -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC - Array(Array&& other) + Array(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible<Scalar>::value) : Base(std::move(other)) { Base::_check_template_params(); - if (RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic) - Base::_set_noalias(other); } EIGEN_DEVICE_FUNC - Array& operator=(Array&& other) + Array& operator=(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable<Scalar>::value) { other.swap(*this); return *this; @@ -180,6 +178,46 @@ Base::_check_template_params(); this->template _init2<T0,T1>(val0, val1); } + + #if EIGEN_HAS_CXX11 + /** \copydoc PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + * + * Example: \include Array_variadic_ctor_cxx11.cpp + * Output: \verbinclude Array_variadic_ctor_cxx11.out + * + * \sa Array(const std::initializer_list<std::initializer_list<Scalar>>&) + * \sa Array(Scalar), Array(Scalar,Scalar) + */ + template <typename... ArgTypes> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + : Base(a0, a1, a2, a3, args...) {} + + /** \brief Constructs an array and initializes it from the coefficients given as initializer-lists grouped by row. \cpp11 + * + * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients: + * + * Example: \include Array_initializer_list_23_cxx11.cpp + * Output: \verbinclude Array_initializer_list_23_cxx11.out + * + * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is triggered. + * + * In the case of a compile-time column 1D array, implicit transposition from a single row is allowed. + * Therefore <code> Array<int,Dynamic,1>{{1,2,3,4,5}}</code> is legal and the more verbose syntax + * <code>Array<int,Dynamic,1>{{1},{2},{3},{4},{5}}</code> can be avoided: + * + * Example: \include Array_initializer_list_vector_cxx11.cpp + * Output: \verbinclude Array_initializer_list_vector_cxx11.out + * + * In the case of fixed-sized arrays, the initializer list sizes must exactly match the array sizes, + * and implicit transposition is allowed for compile-time 1D arrays only. + * + * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + */ + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Array(const std::initializer_list<std::initializer_list<Scalar> >& list) : Base(list) {} + #endif // end EIGEN_HAS_CXX11 + #else /** \brief Constructs a fixed-sized array initialized with coefficients starting at \a data */ EIGEN_DEVICE_FUNC explicit Array(const Scalar *data); @@ -191,7 +229,8 @@ */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(Index dim); - /** constructs an initialized 1x1 Array with the given coefficient */ + /** constructs an initialized 1x1 Array with the given coefficient + * \sa const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args */ Array(const Scalar& value); /** constructs an uninitialized array with \a rows rows and \a cols columns. * @@ -199,11 +238,14 @@ * it is redundant to pass these parameters, so one should use the default constructor * Array() instead. */ Array(Index rows, Index cols); - /** constructs an initialized 2D vector with given coefficients */ + /** constructs an initialized 2D vector with given coefficients + * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ Array(const Scalar& val0, const Scalar& val1); - #endif + #endif // end EIGEN_PARSED_BY_DOXYGEN - /** constructs an initialized 3D vector with given coefficients */ + /** constructs an initialized 3D vector with given coefficients + * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2) { @@ -213,7 +255,9 @@ m_storage.data()[1] = val1; m_storage.data()[2] = val2; } - /** constructs an initialized 4D vector with given coefficients */ + /** constructs an initialized 4D vector with given coefficients + * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2, const Scalar& val3) { @@ -231,10 +275,16 @@ : Base(other) { } + private: + struct PrivateType {}; + public: + /** \sa MatrixBase::operator=(const EigenBase<OtherDerived>&) */ template<typename OtherDerived> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Array(const EigenBase<OtherDerived> &other) + EIGEN_STRONG_INLINE Array(const EigenBase<OtherDerived> &other, + typename internal::enable_if<internal::is_convertible<typename OtherDerived::Scalar,Scalar>::value, + PrivateType>::type = PrivateType()) : Base(other.derived()) { }
diff --git a/Eigen/src/Core/ArrayBase.h b/Eigen/src/Core/ArrayBase.h index 0443e30..9da960f 100644 --- a/Eigen/src/Core/ArrayBase.h +++ b/Eigen/src/Core/ArrayBase.h
@@ -32,7 +32,7 @@ * \tparam Derived is the derived type, e.g., an array or an expression type. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_ARRAYBASE_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAYBASE_PLUGIN. * * \sa class MatrixBase, \ref TopicClassHierarchy */ @@ -52,8 +52,6 @@ typedef typename NumTraits<Scalar>::Real RealScalar; typedef DenseBase<Derived> Base; - using Base::operator*; - using Base::operator/; using Base::RowsAtCompileTime; using Base::ColsAtCompileTime; using Base::SizeAtCompileTime; @@ -71,6 +69,7 @@ using Base::coeff; using Base::coeffRef; using Base::lazyAssign; + using Base::operator-; using Base::operator=; using Base::operator+=; using Base::operator-=; @@ -89,7 +88,7 @@ #endif // not EIGEN_PARSED_BY_DOXYGEN #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::ArrayBase -# include "../plugins/CommonCwiseUnaryOps.h" +#define EIGEN_DOC_UNARY_ADDONS(X,Y) # include "../plugins/MatrixCwiseUnaryOps.h" # include "../plugins/ArrayCwiseUnaryOps.h" # include "../plugins/CommonCwiseBinaryOps.h" @@ -99,6 +98,7 @@ # include EIGEN_ARRAYBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS +#undef EIGEN_DOC_UNARY_ADDONS /** Special case of the template operator=, in order to prevent the compiler * from generating a default operator= (issue hit with g++ 4.1) @@ -175,10 +175,10 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase<Derived>::operator-=(const ArrayBase<OtherDerived> &other) { - call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -188,10 +188,10 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase<Derived>::operator+=(const ArrayBase<OtherDerived>& other) { - call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -201,7 +201,7 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase<Derived>::operator*=(const ArrayBase<OtherDerived>& other) { call_assignment(derived(), other.derived(), internal::mul_assign_op<Scalar,typename OtherDerived::Scalar>()); @@ -214,10 +214,10 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase<Derived>::operator/=(const ArrayBase<OtherDerived>& other) { - call_assignment(derived(), other.derived(), internal::div_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::div_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); }
diff --git a/Eigen/src/Core/ArrayWrapper.h b/Eigen/src/Core/ArrayWrapper.h index 6013d4d..757b318 100644 --- a/Eigen/src/Core/ArrayWrapper.h +++ b/Eigen/src/Core/ArrayWrapper.h
@@ -32,7 +32,8 @@ // Let's remove NestByRefBit enum { Flags0 = traits<typename remove_all<typename ExpressionType::Nested>::type >::Flags, - Flags = Flags0 & ~NestByRefBit + LvalueBitFlag = is_lvalue<ExpressionType>::value ? LvalueBit : 0, + Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; }; } @@ -54,6 +55,8 @@ typedef typename internal::ref_selector<ExpressionType>::non_const_type NestedExpressionType; + using Base::coeffRef; + EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE ArrayWrapper(ExpressionType& matrix) : m_expression(matrix) {} @@ -72,71 +75,23 @@ inline const Scalar* data() const { return m_expression.data(); } EIGEN_DEVICE_FUNC - inline CoeffReturnType coeff(Index rowId, Index colId) const - { - return m_expression.coeff(rowId, colId); - } - - EIGEN_DEVICE_FUNC - inline Scalar& coeffRef(Index rowId, Index colId) - { - return m_expression.coeffRef(rowId, colId); - } - - EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { return m_expression.coeffRef(rowId, colId); } EIGEN_DEVICE_FUNC - inline CoeffReturnType coeff(Index index) const - { - return m_expression.coeff(index); - } - - EIGEN_DEVICE_FUNC - inline Scalar& coeffRef(Index index) - { - return m_expression.coeffRef(index); - } - - EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); } - template<int LoadMode> - inline const PacketScalar packet(Index rowId, Index colId) const - { - return m_expression.template packet<LoadMode>(rowId, colId); - } - - template<int LoadMode> - inline void writePacket(Index rowId, Index colId, const PacketScalar& val) - { - m_expression.template writePacket<LoadMode>(rowId, colId, val); - } - - template<int LoadMode> - inline const PacketScalar packet(Index index) const - { - return m_expression.template packet<LoadMode>(index); - } - - template<int LoadMode> - inline void writePacket(Index index, const PacketScalar& val) - { - m_expression.template writePacket<LoadMode>(index, val); - } - template<typename Dest> EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const { dst = m_expression; } - const typename internal::remove_all<NestedExpressionType>::type& EIGEN_DEVICE_FUNC + const typename internal::remove_all<NestedExpressionType>::type& nestedExpression() const { return m_expression; @@ -175,7 +130,8 @@ // Let's remove NestByRefBit enum { Flags0 = traits<typename remove_all<typename ExpressionType::Nested>::type >::Flags, - Flags = Flags0 & ~NestByRefBit + LvalueBitFlag = is_lvalue<ExpressionType>::value ? LvalueBit : 0, + Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; }; } @@ -197,6 +153,8 @@ typedef typename internal::ref_selector<ExpressionType>::non_const_type NestedExpressionType; + using Base::coeffRef; + EIGEN_DEVICE_FUNC explicit inline MatrixWrapper(ExpressionType& matrix) : m_expression(matrix) {} @@ -215,65 +173,17 @@ inline const Scalar* data() const { return m_expression.data(); } EIGEN_DEVICE_FUNC - inline CoeffReturnType coeff(Index rowId, Index colId) const - { - return m_expression.coeff(rowId, colId); - } - - EIGEN_DEVICE_FUNC - inline Scalar& coeffRef(Index rowId, Index colId) - { - return m_expression.coeffRef(rowId, colId); - } - - EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { return m_expression.derived().coeffRef(rowId, colId); } EIGEN_DEVICE_FUNC - inline CoeffReturnType coeff(Index index) const - { - return m_expression.coeff(index); - } - - EIGEN_DEVICE_FUNC - inline Scalar& coeffRef(Index index) - { - return m_expression.coeffRef(index); - } - - EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); } - template<int LoadMode> - inline const PacketScalar packet(Index rowId, Index colId) const - { - return m_expression.template packet<LoadMode>(rowId, colId); - } - - template<int LoadMode> - inline void writePacket(Index rowId, Index colId, const PacketScalar& val) - { - m_expression.template writePacket<LoadMode>(rowId, colId, val); - } - - template<int LoadMode> - inline const PacketScalar packet(Index index) const - { - return m_expression.template packet<LoadMode>(index); - } - - template<int LoadMode> - inline void writePacket(Index index, const PacketScalar& val) - { - m_expression.template writePacket<LoadMode>(index, val); - } - EIGEN_DEVICE_FUNC const typename internal::remove_all<NestedExpressionType>::type& nestedExpression() const
diff --git a/Eigen/src/Core/Assign.h b/Eigen/src/Core/Assign.h index 53806ba..655412e 100644 --- a/Eigen/src/Core/Assign.h +++ b/Eigen/src/Core/Assign.h
@@ -16,7 +16,7 @@ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived> ::lazyAssign(const DenseBase<OtherDerived>& other) { enum{
diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 9d4b315..229e258 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h
@@ -24,7 +24,7 @@ // copy_using_evaluator_traits is based on assign_traits -template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc> +template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc, int MaxPacketSize = -1> struct copy_using_evaluator_traits { typedef typename DstEvaluator::XprType Dst; @@ -39,7 +39,7 @@ enum { DstAlignment = DstEvaluator::Alignment, SrcAlignment = SrcEvaluator::Alignment, - DstHasDirectAccess = DstFlags & DirectAccessBit, + DstHasDirectAccess = (DstFlags & DirectAccessBit) == DirectAccessBit, JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment) }; @@ -51,13 +51,15 @@ InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime) : int(DstFlags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime) : int(Dst::MaxRowsAtCompileTime), + RestrictedInnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(InnerSize,MaxPacketSize), + RestrictedLinearSize = EIGEN_SIZE_MIN_PREFER_FIXED(Dst::SizeAtCompileTime,MaxPacketSize), OuterStride = int(outer_stride_at_compile_time<Dst>::ret), MaxSizeAtCompileTime = Dst::SizeAtCompileTime }; // TODO distinguish between linear traversal and inner-traversals - typedef typename find_best_packet<DstScalar,Dst::SizeAtCompileTime>::type LinearPacketType; - typedef typename find_best_packet<DstScalar,InnerSize>::type InnerPacketType; + typedef typename find_best_packet<DstScalar,RestrictedLinearSize>::type LinearPacketType; + typedef typename find_best_packet<DstScalar,RestrictedInnerSize>::type InnerPacketType; enum { LinearPacketSize = unpacket_traits<LinearPacketType>::size, @@ -75,28 +77,29 @@ DstIsRowMajor = DstFlags&RowMajorBit, SrcIsRowMajor = SrcFlags&RowMajorBit, StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)), - MightVectorize = StorageOrdersAgree + MightVectorize = bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) & ActualPacketAccessBit) - && (functor_traits<AssignFunc>::PacketAccess), + && bool(functor_traits<AssignFunc>::PacketAccess), MayInnerVectorize = MightVectorize && int(InnerSize)!=Dynamic && int(InnerSize)%int(InnerPacketSize)==0 && int(OuterStride)!=Dynamic && int(OuterStride)%int(InnerPacketSize)==0 - && int(JointAlignment)>=int(InnerRequiredAlignment), - MayLinearize = StorageOrdersAgree && (int(DstFlags) & int(SrcFlags) & LinearAccessBit), - MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess - && ((int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime == Dynamic), + && (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment)>=int(InnerRequiredAlignment)), + MayLinearize = bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) & LinearAccessBit), + MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess) + && (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime == Dynamic), /* If the destination isn't aligned, we have to do runtime checks and we don't unroll, so it's only good for large enough sizes. */ - MaySliceVectorize = MightVectorize && DstHasDirectAccess - && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=3*InnerPacketSize) + MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess) + && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=(EIGEN_UNALIGNED_VECTORIZE?InnerPacketSize:(3*InnerPacketSize))) /* slice vectorization can be slow, so we only want it if the slices are big, which is indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block - in a fixed-size matrix */ + in a fixed-size matrix + However, with EIGEN_UNALIGNED_VECTORIZE and unrolling, slice vectorization is still worth it */ }; public: enum { - Traversal = int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize) ? int(LinearVectorizedTraversal) + Traversal = (int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize)) ? int(LinearVectorizedTraversal) : int(MayInnerVectorize) ? int(InnerVectorizedTraversal) : int(MayLinearVectorize) ? int(LinearVectorizedTraversal) : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) @@ -116,9 +119,9 @@ : 1, UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize, MayUnrollCompletely = int(Dst::SizeAtCompileTime) != Dynamic - && int(Dst::SizeAtCompileTime) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit), + && int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <= int(UnrollingLimit), MayUnrollInner = int(InnerSize) != Dynamic - && int(InnerSize) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit) + && int(InnerSize) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <= int(UnrollingLimit) }; public: @@ -130,11 +133,17 @@ : int(NoUnrolling) ) : int(Traversal) == int(LinearVectorizedTraversal) - ? ( bool(MayUnrollCompletely) && (int(DstAlignment)>=int(LinearRequiredAlignment)) ? int(CompleteUnrolling) - : int(NoUnrolling) ) + ? ( bool(MayUnrollCompletely) && ( EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment))) + ? int(CompleteUnrolling) + : int(NoUnrolling) ) : int(Traversal) == int(LinearTraversal) ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling) : int(NoUnrolling) ) +#if EIGEN_UNALIGNED_VECTORIZE + : int(Traversal) == int(SliceVectorizedTraversal) + ? ( bool(MayUnrollInner) ? int(InnerUnrolling) + : int(NoUnrolling) ) +#endif : int(NoUnrolling) }; @@ -156,6 +165,7 @@ EIGEN_DEBUG_VAR(InnerMaxSize) EIGEN_DEBUG_VAR(LinearPacketSize) EIGEN_DEBUG_VAR(InnerPacketSize) + EIGEN_DEBUG_VAR(ActualPacketSize) EIGEN_DEBUG_VAR(StorageOrdersAgree) EIGEN_DEBUG_VAR(MightVectorize) EIGEN_DEBUG_VAR(MayLinearize) @@ -163,6 +173,9 @@ EIGEN_DEBUG_VAR(MayLinearVectorize) EIGEN_DEBUG_VAR(MaySliceVectorize) std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl; + EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost) + EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost) + EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime) EIGEN_DEBUG_VAR(UnrollingLimit) EIGEN_DEBUG_VAR(MayUnrollCompletely) EIGEN_DEBUG_VAR(MayUnrollInner) @@ -256,12 +269,13 @@ enum { outer = Index / DstXprType::InnerSizeAtCompileTime, inner = Index % DstXprType::InnerSizeAtCompileTime, - JointAlignment = Kernel::AssignmentTraits::JointAlignment + SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, + DstAlignment = Kernel::AssignmentTraits::DstAlignment }; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) { - kernel.template assignPacketByOuterInner<Aligned, JointAlignment, PacketType>(outer, inner); + kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner); enum { NextIndex = Index + unpacket_traits<PacketType>::size }; copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel); } @@ -273,20 +287,20 @@ EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { } }; -template<typename Kernel, int Index_, int Stop> +template<typename Kernel, int Index_, int Stop, int SrcAlignment, int DstAlignment> struct copy_using_evaluator_innervec_InnerUnrolling { typedef typename Kernel::PacketType PacketType; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer) { - kernel.template assignPacketByOuterInner<Aligned, Aligned, PacketType>(outer, Index_); + kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_); enum { NextIndex = Index_ + unpacket_traits<PacketType>::size }; - copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop>::run(kernel, outer); + copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel, outer); } }; -template<typename Kernel, int Stop> -struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop> +template<typename Kernel, int Stop, int SrcAlignment, int DstAlignment> +struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment> { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &, Index) { } }; @@ -397,7 +411,7 @@ : int(Kernel::AssignmentTraits::DstAlignment), srcAlignment = Kernel::AssignmentTraits::JointAlignment }; - const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(&kernel.dstEvaluator().coeffRef(0), size); + const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size); const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize; unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart); @@ -415,9 +429,10 @@ EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) { typedef typename Kernel::DstEvaluatorType::XprType DstXprType; + typedef typename Kernel::PacketType PacketType; enum { size = DstXprType::SizeAtCompileTime, - packetSize = packet_traits<typename Kernel::Scalar>::size, + packetSize =unpacket_traits<PacketType>::size, alignedSize = (size/packetSize)*packetSize }; copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel); @@ -433,6 +448,10 @@ struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling> { typedef typename Kernel::PacketType PacketType; + enum { + SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, + DstAlignment = Kernel::AssignmentTraits::DstAlignment + }; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) { const Index innerSize = kernel.innerSize(); @@ -440,7 +459,7 @@ const Index packetSize = unpacket_traits<PacketType>::size; for(Index outer = 0; outer < outerSize; ++outer) for(Index inner = 0; inner < innerSize; inner+=packetSize) - kernel.template assignPacketByOuterInner<Aligned, Aligned, PacketType>(outer, inner); + kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner); } }; @@ -460,9 +479,11 @@ EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) { typedef typename Kernel::DstEvaluatorType::XprType DstXprType; + typedef typename Kernel::AssignmentTraits Traits; const Index outerSize = kernel.outerSize(); for(Index outer = 0; outer < outerSize; ++outer) - copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer); + copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime, + Traits::SrcAlignment, Traits::DstAlignment>::run(kernel, outer); } }; @@ -498,7 +519,7 @@ template<typename Kernel> struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling> { - EIGEN_DEVICE_FUNC static inline void run(Kernel &kernel) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) { typedef typename Kernel::Scalar Scalar; typedef typename Kernel::PacketType PacketType; @@ -510,10 +531,10 @@ dstAlignment = alignable ? int(requestedAlignment) : int(Kernel::AssignmentTraits::DstAlignment) }; - const Scalar *dst_ptr = &kernel.dstEvaluator().coeffRef(0,0); - if((!bool(dstIsAligned)) && (size_t(dst_ptr) % sizeof(Scalar))>0) + const Scalar *dst_ptr = kernel.dstDataPtr(); + if((!bool(dstIsAligned)) && (UIntPtr(dst_ptr) % sizeof(Scalar))>0) { - // the pointer is not aligend-on scalar, so alignment is not possible + // the pointer is not aligned-on scalar, so alignment is not possible return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel); } const Index packetAlignedMask = packetSize - 1; @@ -537,11 +558,34 @@ for(Index inner = alignedEnd; inner<innerSize ; ++inner) kernel.assignCoeffByOuterInner(outer, inner); - alignedStart = std::min<Index>((alignedStart+alignedStep)%packetSize, innerSize); + alignedStart = numext::mini((alignedStart+alignedStep)%packetSize, innerSize); } } }; +#if EIGEN_UNALIGNED_VECTORIZE +template<typename Kernel> +struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling> +{ + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel) + { + typedef typename Kernel::DstEvaluatorType::XprType DstXprType; + typedef typename Kernel::PacketType PacketType; + + enum { size = DstXprType::InnerSizeAtCompileTime, + packetSize =unpacket_traits<PacketType>::size, + vectorizableSize = (size/packetSize)*packetSize }; + + for(Index outer = 0; outer < kernel.outerSize(); ++outer) + { + copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer); + copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, size>::run(kernel, outer); + } + } +}; +#endif + + /*************************************************************************** * Part 4 : Generic dense assignment kernel ***************************************************************************/ @@ -567,7 +611,8 @@ typedef typename AssignmentTraits::PacketType PacketType; - EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) { #ifdef EIGEN_DEBUG_ASSIGN @@ -643,6 +688,11 @@ : int(DstEvaluatorType::Flags)&RowMajorBit ? inner : outer; } + + EIGEN_DEVICE_FUNC const Scalar* dstDataPtr() const + { + return m_dstExpr.data(); + } protected: DstEvaluatorType& m_dst; @@ -652,31 +702,75 @@ DstXprType& m_dstExpr; }; +// Special kernel used when computing small products whose operands have dynamic dimensions. It ensures that the +// PacketSize used is no larger than 4, thereby increasing the chance that vectorized instructions will be used +// when computing the product. + +template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor> +class restricted_packet_dense_assignment_kernel : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> +{ +protected: + typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> Base; + public: + typedef typename Base::Scalar Scalar; + typedef typename Base::DstXprType DstXprType; + typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4> AssignmentTraits; + typedef typename AssignmentTraits::PacketType PacketType; + + EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) + : Base(dst, src, func, dstExpr) + { + } + }; + /*************************************************************************** * Part 5 : Entry point for dense rectangular assignment ***************************************************************************/ -template<typename DstXprType, typename SrcXprType, typename Functor> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func) +template<typename DstXprType,typename SrcXprType, typename Functor> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +void resize_if_allowed(DstXprType &dst, const SrcXprType& src, const Functor &/*func*/) { + EIGEN_ONLY_USED_FOR_DEBUG(dst); + EIGEN_ONLY_USED_FOR_DEBUG(src); eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); - +} + +template<typename DstXprType,typename SrcXprType, typename T1, typename T2> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +void resize_if_allowed(DstXprType &dst, const SrcXprType& src, const internal::assign_op<T1,T2> &/*func*/) +{ + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols))) + dst.resize(dstRows, dstCols); + eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols); +} + +template<typename DstXprType, typename SrcXprType, typename Functor> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func) +{ typedef evaluator<DstXprType> DstEvaluatorType; typedef evaluator<SrcXprType> SrcEvaluatorType; - DstEvaluatorType dstEvaluator(dst); SrcEvaluatorType srcEvaluator(src); + + // NOTE To properly handle A = (A*A.transpose())/s with A rectangular, + // we need to resize the destination after the source evaluator has been created. + resize_if_allowed(dst, src, func); + + DstEvaluatorType dstEvaluator(dst); typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel; Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived()); - + dense_assignment_loop<Kernel>::run(kernel); } template<typename DstXprType, typename SrcXprType> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src) { - call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar>()); + call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>()); } /*************************************************************************** @@ -688,7 +782,7 @@ // AssignmentKind must define a Kind typedef. template<typename DstShape, typename SrcShape> struct AssignmentKind; -// Assignement kind defined in this file: +// Assignment kind defined in this file: struct Dense2Dense {}; struct EigenBase2EigenBase {}; @@ -698,7 +792,7 @@ // This is the main assignment class template< typename DstXprType, typename SrcXprType, typename Functor, typename Kind = typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape , typename evaluator_traits<SrcXprType>::Shape >::Kind, - typename Scalar = typename DstXprType::Scalar> + typename EnableIf = void> struct Assignment; @@ -711,13 +805,13 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(Dst& dst, const Src& src) { - call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>()); + call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>()); } template<typename Dst, typename Src> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(const Dst& dst, const Src& src) { - call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>()); + call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>()); } // Deal with "assume-aliasing" @@ -756,11 +850,6 @@ ) && int(Dst::SizeAtCompileTime) != 1 }; - Index dstRows = NeedToTranspose ? src.cols() : src.rows(); - Index dstCols = NeedToTranspose ? src.rows() : src.cols(); - if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) - dst.resize(dstRows, dstCols); - typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned; typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType; ActualDstType actualDst(dst); @@ -772,47 +861,64 @@ Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func); } + +template<typename Dst, typename Src, typename Func> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +void call_restricted_packet_assignment_no_alias(Dst& dst, const Src& src, const Func& func) +{ + typedef evaluator<Dst> DstEvaluatorType; + typedef evaluator<Src> SrcEvaluatorType; + typedef restricted_packet_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Func> Kernel; + + EIGEN_STATIC_ASSERT_LVALUE(Dst) + EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename Dst::Scalar,typename Src::Scalar); + + SrcEvaluatorType srcEvaluator(src); + resize_if_allowed(dst, src, func); + + DstEvaluatorType dstEvaluator(dst); + Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived()); + + dense_assignment_loop<Kernel>::run(kernel); +} + template<typename Dst, typename Src> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment_no_alias(Dst& dst, const Src& src) { - call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar>()); + call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>()); } template<typename Dst, typename Src, typename Func> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func) { - Index dstRows = src.rows(); - Index dstCols = src.cols(); - if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) - dst.resize(dstRows, dstCols); - // TODO check whether this is the right place to perform these checks: EIGEN_STATIC_ASSERT_LVALUE(Dst) EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src) - + EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename Dst::Scalar,typename Src::Scalar); + Assignment<Dst,Src,Func>::run(dst, src, func); } template<typename Dst, typename Src> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src) { - call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar>()); + call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>()); } // forward declaration template<typename Dst, typename Src> void check_for_aliasing(const Dst &dst, const Src &src); // Generic Dense to Dense assignment -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Scalar> +// Note that the last template argument "Weak" is needed to make it possible to perform +// both partial specialization+SFINAE without ambiguous specialization +template< typename DstXprType, typename SrcXprType, typename Functor, typename Weak> +struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak> { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { - eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); - #ifndef EIGEN_NO_DEBUG internal::check_for_aliasing(dst, src); #endif @@ -823,15 +929,50 @@ // Generic assignment through evalTo. // TODO: not sure we have to keep that one, but it helps porting current code to new evaluator mechanism. -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Scalar> +// Note that the last template argument "Weak" is needed to make it possible to perform +// both partial specialization+SFINAE without ambiguous specialization +template< typename DstXprType, typename SrcXprType, typename Functor, typename Weak> +struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak> { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); src.evalTo(dst); } + + // NOTE The following two functions are templated to avoid their instantiation if not needed + // This is needed because some expressions supports evalTo only and/or have 'void' as scalar type. + template<typename SrcScalarType> + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar,SrcScalarType> &/*func*/) + { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + src.addTo(dst); + } + + template<typename SrcScalarType> + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar,SrcScalarType> &/*func*/) + { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + src.subTo(dst); + } }; } // namespace internal
diff --git a/Eigen/src/Core/Assign_MKL.h b/Eigen/src/Core/Assign_MKL.h old mode 100644 new mode 100755 index 897187a..c6140d1 --- a/Eigen/src/Core/Assign_MKL.h +++ b/Eigen/src/Core/Assign_MKL.h
@@ -68,27 +68,28 @@ #define EIGEN_PP_EXPAND(ARG) ARG #if !defined (EIGEN_FAST_MATH) || (EIGEN_FAST_MATH != 1) -#define EIGEN_VMLMODE_EXPAND_LA , VML_HA +#define EIGEN_VMLMODE_EXPAND_xLA , VML_HA #else -#define EIGEN_VMLMODE_EXPAND_LA , VML_LA +#define EIGEN_VMLMODE_EXPAND_xLA , VML_LA #endif -#define EIGEN_VMLMODE_EXPAND__ +#define EIGEN_VMLMODE_EXPAND_x_ -#define EIGEN_VMLMODE_PREFIX_LA vm -#define EIGEN_VMLMODE_PREFIX__ v -#define EIGEN_VMLMODE_PREFIX(VMLMODE) EIGEN_CAT(EIGEN_VMLMODE_PREFIX_,VMLMODE) +#define EIGEN_VMLMODE_PREFIX_xLA vm +#define EIGEN_VMLMODE_PREFIX_x_ v +#define EIGEN_VMLMODE_PREFIX(VMLMODE) EIGEN_CAT(EIGEN_VMLMODE_PREFIX_x,VMLMODE) #define EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ template< typename DstXprType, typename SrcXprNested> \ - struct Assignment<DstXprType, CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested>, assign_op<EIGENTYPE>, \ - Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml,EIGENTYPE>::type> { \ + struct Assignment<DstXprType, CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested>, assign_op<EIGENTYPE,EIGENTYPE>, \ + Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml>::type> { \ typedef CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested> SrcXprType; \ - static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE> &/*func*/) { \ + static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE,EIGENTYPE> &func) { \ + resize_if_allowed(dst, src, func); \ eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); \ if(vml_assign_traits<DstXprType,SrcXprNested>::Traversal==LinearTraversal) { \ VMLOP(dst.size(), (const VMLTYPE*)src.nestedExpression().data(), \ - (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE) ); \ + (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE) ); \ } else { \ const Index outerSize = dst.outerSize(); \ for(Index outer = 0; outer < outerSize; ++outer) { \ @@ -96,7 +97,7 @@ &(src.nestedExpression().coeffRef(0, outer)); \ EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer,0)) : &(dst.coeffRef(0, outer)); \ VMLOP( dst.innerSize(), (const VMLTYPE*)src_ptr, \ - (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE)); \ + (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE)); \ } \ } \ } \ @@ -138,25 +139,28 @@ EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(ceil, Ceil, _) #define EIGEN_MKL_VML_DECLARE_POW_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ - template< typename DstXprType, typename SrcXprNested> \ - struct Assignment<DstXprType, CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested>, assign_op<EIGENTYPE>, \ - Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml,EIGENTYPE>::type> { \ - typedef CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested> SrcXprType; \ - static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE> &/*func*/) { \ + template< typename DstXprType, typename SrcXprNested, typename Plain> \ + struct Assignment<DstXprType, CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE,EIGENTYPE>, SrcXprNested, \ + const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>,Plain> >, assign_op<EIGENTYPE,EIGENTYPE>, \ + Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml>::type> { \ + typedef CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE,EIGENTYPE>, SrcXprNested, \ + const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>,Plain> > SrcXprType; \ + static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE,EIGENTYPE> &func) { \ + resize_if_allowed(dst, src, func); \ eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); \ - VMLTYPE exponent = reinterpret_cast<const VMLTYPE&>(src.functor().m_exponent); \ + VMLTYPE exponent = reinterpret_cast<const VMLTYPE&>(src.rhs().functor().m_other); \ if(vml_assign_traits<DstXprType,SrcXprNested>::Traversal==LinearTraversal) \ { \ - VMLOP( dst.size(), (const VMLTYPE*)src.nestedExpression().data(), exponent, \ - (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE) ); \ + VMLOP( dst.size(), (const VMLTYPE*)src.lhs().data(), exponent, \ + (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE) ); \ } else { \ const Index outerSize = dst.outerSize(); \ for(Index outer = 0; outer < outerSize; ++outer) { \ - const EIGENTYPE *src_ptr = src.IsRowMajor ? &(src.nestedExpression().coeffRef(outer,0)) : \ - &(src.nestedExpression().coeffRef(0, outer)); \ + const EIGENTYPE *src_ptr = src.IsRowMajor ? &(src.lhs().coeffRef(outer,0)) : \ + &(src.lhs().coeffRef(0, outer)); \ EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer,0)) : &(dst.coeffRef(0, outer)); \ VMLOP( dst.innerSize(), (const VMLTYPE*)src_ptr, exponent, \ - (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE)); \ + (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE)); \ } \ } \ } \
diff --git a/Eigen/src/Core/Block.h b/Eigen/src/Core/Block.h index 11de45c..6e938ea 100644 --- a/Eigen/src/Core/Block.h +++ b/Eigen/src/Core/Block.h
@@ -114,8 +114,8 @@ /** Column or Row constructor */ - EIGEN_DEVICE_FUNC - inline Block(XprType& xpr, Index i) : Impl(xpr,i) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Block(XprType& xpr, Index i) : Impl(xpr,i) { eigen_assert( (i>=0) && ( ((BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) && i<xpr.rows()) @@ -124,8 +124,8 @@ /** Fixed-size constructor */ - EIGEN_DEVICE_FUNC - inline Block(XprType& xpr, Index startRow, Index startCol) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Block(XprType& xpr, Index startRow, Index startCol) : Impl(xpr, startRow, startCol) { EIGEN_STATIC_ASSERT(RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic,THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE) @@ -135,8 +135,8 @@ /** Dynamic-size constructor */ - EIGEN_DEVICE_FUNC - inline Block(XprType& xpr, + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Block(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) : Impl(xpr, startRow, startCol, blockRows, blockCols) @@ -159,10 +159,10 @@ public: typedef Impl Base; EIGEN_INHERIT_ASSIGNMENT_OPERATORS(BlockImpl) - EIGEN_DEVICE_FUNC inline BlockImpl(XprType& xpr, Index i) : Impl(xpr,i) {} - EIGEN_DEVICE_FUNC inline BlockImpl(XprType& xpr, Index startRow, Index startCol) : Impl(xpr, startRow, startCol) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index i) : Impl(xpr,i) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index startRow, Index startCol) : Impl(xpr, startRow, startCol) {} EIGEN_DEVICE_FUNC - inline BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) : Impl(xpr, startRow, startCol, blockRows, blockCols) {} }; @@ -294,22 +294,22 @@ EIGEN_DEVICE_FUNC inline Index outerStride() const; #endif - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename internal::remove_all<XprTypeNested>::type& nestedExpression() const { return m_xpr; } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE XprType& nestedExpression() { return m_xpr; } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex startRow() const { return m_startRow.value(); } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex startCol() const { return m_startCol.value(); @@ -342,8 +342,8 @@ /** Column or Row constructor */ - EIGEN_DEVICE_FUNC - inline BlockImpl_dense(XprType& xpr, Index i) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + BlockImpl_dense(XprType& xpr, Index i) : Base(xpr.data() + i * ( ((BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) && (!XprTypeIsRowMajor)) || ((BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) && ( XprTypeIsRowMajor)) ? xpr.innerStride() : xpr.outerStride()), BlockRows==1 ? 1 : xpr.rows(), @@ -357,8 +357,8 @@ /** Fixed-size constructor */ - EIGEN_DEVICE_FUNC - inline BlockImpl_dense(XprType& xpr, Index startRow, Index startCol) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + BlockImpl_dense(XprType& xpr, Index startRow, Index startCol) : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol)), m_xpr(xpr), m_startRow(startRow), m_startCol(startCol) { @@ -367,8 +367,8 @@ /** Dynamic-size constructor */ - EIGEN_DEVICE_FUNC - inline BlockImpl_dense(XprType& xpr, + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + BlockImpl_dense(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol), blockRows, blockCols), @@ -377,18 +377,18 @@ init(); } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename internal::remove_all<XprTypeNested>::type& nestedExpression() const { return m_xpr; } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE XprType& nestedExpression() { return m_xpr; } /** \sa MapBase::innerStride() */ - EIGEN_DEVICE_FUNC - inline Index innerStride() const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index innerStride() const { return internal::traits<BlockType>::HasSameStorageOrderAsXprType ? m_xpr.innerStride() @@ -396,19 +396,19 @@ } /** \sa MapBase::outerStride() */ - EIGEN_DEVICE_FUNC - inline Index outerStride() const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index outerStride() const { return m_outerStride; } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex startRow() const { return m_startRow.value(); } - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex startCol() const { return m_startCol.value(); @@ -422,8 +422,8 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal used by allowAligned() */ - EIGEN_DEVICE_FUNC - inline BlockImpl_dense(XprType& xpr, const Scalar* data, Index blockRows, Index blockCols) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + BlockImpl_dense(XprType& xpr, const Scalar* data, Index blockRows, Index blockCols) : Base(data, blockRows, blockCols), m_xpr(xpr) { init(); @@ -431,7 +431,7 @@ #endif protected: - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void init() { m_outerStride = internal::traits<BlockType>::HasSameStorageOrderAsXprType
diff --git a/Eigen/src/Core/BooleanRedux.h b/Eigen/src/Core/BooleanRedux.h index 8409d87..ccf5190 100644 --- a/Eigen/src/Core/BooleanRedux.h +++ b/Eigen/src/Core/BooleanRedux.h
@@ -14,56 +14,54 @@ namespace internal { -template<typename Derived, int UnrollCount> +template<typename Derived, int UnrollCount, int Rows> struct all_unroller { - typedef typename Derived::ExpressionTraits Traits; enum { - col = (UnrollCount-1) / Traits::RowsAtCompileTime, - row = (UnrollCount-1) % Traits::RowsAtCompileTime + col = (UnrollCount-1) / Rows, + row = (UnrollCount-1) % Rows }; static inline bool run(const Derived &mat) { - return all_unroller<Derived, UnrollCount-1>::run(mat) && mat.coeff(row, col); + return all_unroller<Derived, UnrollCount-1, Rows>::run(mat) && mat.coeff(row, col); } }; -template<typename Derived> -struct all_unroller<Derived, 0> +template<typename Derived, int Rows> +struct all_unroller<Derived, 0, Rows> { static inline bool run(const Derived &/*mat*/) { return true; } }; -template<typename Derived> -struct all_unroller<Derived, Dynamic> +template<typename Derived, int Rows> +struct all_unroller<Derived, Dynamic, Rows> { static inline bool run(const Derived &) { return false; } }; -template<typename Derived, int UnrollCount> +template<typename Derived, int UnrollCount, int Rows> struct any_unroller { - typedef typename Derived::ExpressionTraits Traits; enum { - col = (UnrollCount-1) / Traits::RowsAtCompileTime, - row = (UnrollCount-1) % Traits::RowsAtCompileTime + col = (UnrollCount-1) / Rows, + row = (UnrollCount-1) % Rows }; static inline bool run(const Derived &mat) { - return any_unroller<Derived, UnrollCount-1>::run(mat) || mat.coeff(row, col); + return any_unroller<Derived, UnrollCount-1, Rows>::run(mat) || mat.coeff(row, col); } }; -template<typename Derived> -struct any_unroller<Derived, 0> +template<typename Derived, int Rows> +struct any_unroller<Derived, 0, Rows> { static inline bool run(const Derived & /*mat*/) { return false; } }; -template<typename Derived> -struct any_unroller<Derived, Dynamic> +template<typename Derived, int Rows> +struct any_unroller<Derived, Dynamic, Rows> { static inline bool run(const Derived &) { return false; } }; @@ -78,7 +76,7 @@ * \sa any(), Cwise::operator<() */ template<typename Derived> -inline bool DenseBase<Derived>::all() const +EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::all() const { typedef internal::evaluator<Derived> Evaluator; enum { @@ -87,7 +85,7 @@ }; Evaluator evaluator(derived()); if(unroll) - return internal::all_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic>::run(evaluator); + return internal::all_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic, internal::traits<Derived>::RowsAtCompileTime>::run(evaluator); else { for(Index j = 0; j < cols(); ++j) @@ -102,7 +100,7 @@ * \sa all() */ template<typename Derived> -inline bool DenseBase<Derived>::any() const +EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::any() const { typedef internal::evaluator<Derived> Evaluator; enum { @@ -111,7 +109,7 @@ }; Evaluator evaluator(derived()); if(unroll) - return internal::any_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic>::run(evaluator); + return internal::any_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic, internal::traits<Derived>::RowsAtCompileTime>::run(evaluator); else { for(Index j = 0; j < cols(); ++j) @@ -126,7 +124,7 @@ * \sa all(), any() */ template<typename Derived> -inline Eigen::Index DenseBase<Derived>::count() const +EIGEN_DEVICE_FUNC inline Eigen::Index DenseBase<Derived>::count() const { return derived().template cast<bool>().template cast<Index>().sum(); }
diff --git a/Eigen/src/Core/CMakeLists.txt b/Eigen/src/Core/CMakeLists.txt deleted file mode 100644 index 38c3afd..0000000 --- a/Eigen/src/Core/CMakeLists.txt +++ /dev/null
@@ -1,11 +0,0 @@ -FILE(GLOB Eigen_Core_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core COMPONENT Devel - ) - -ADD_SUBDIRECTORY(products) -ADD_SUBDIRECTORY(util) -ADD_SUBDIRECTORY(arch) -ADD_SUBDIRECTORY(functors)
diff --git a/Eigen/src/Core/CommaInitializer.h b/Eigen/src/Core/CommaInitializer.h index 2abc660..35fdbb8 100644 --- a/Eigen/src/Core/CommaInitializer.h +++ b/Eigen/src/Core/CommaInitializer.h
@@ -80,9 +80,7 @@ EIGEN_DEVICE_FUNC CommaInitializer& operator,(const DenseBase<OtherDerived>& other) { - if(other.cols()==0 || other.rows()==0) - return *this; - if (m_col==m_xpr.cols()) + if (m_col==m_xpr.cols() && (other.cols()!=0 || other.rows()!=m_currentBlockRows)) { m_row+=m_currentBlockRows; m_col = 0; @@ -90,15 +88,11 @@ eigen_assert(m_row+m_currentBlockRows<=m_xpr.rows() && "Too many rows passed to comma initializer (operator<<)"); } - eigen_assert(m_col<m_xpr.cols() + eigen_assert((m_col + other.cols() <= m_xpr.cols()) && "Too many coefficients passed to comma initializer (operator<<)"); eigen_assert(m_currentBlockRows==other.rows()); - if (OtherDerived::SizeAtCompileTime != Dynamic) - m_xpr.template block<OtherDerived::RowsAtCompileTime != Dynamic ? OtherDerived::RowsAtCompileTime : 1, - OtherDerived::ColsAtCompileTime != Dynamic ? OtherDerived::ColsAtCompileTime : 1> - (m_row, m_col) = other; - else - m_xpr.block(m_row, m_col, other.rows(), other.cols()) = other; + m_xpr.template block<OtherDerived::RowsAtCompileTime, OtherDerived::ColsAtCompileTime> + (m_row, m_col, other.rows(), other.cols()) = other; m_col += other.cols(); return *this; } @@ -109,9 +103,7 @@ EIGEN_EXCEPTION_SPEC(Eigen::eigen_assert_exception) #endif { - eigen_assert((m_row+m_currentBlockRows) == m_xpr.rows() - && m_col == m_xpr.cols() - && "Too few coefficients passed to comma initializer (operator<<)"); + finished(); } /** \returns the built matrix once all its coefficients have been set. @@ -122,7 +114,12 @@ * \endcode */ EIGEN_DEVICE_FUNC - inline XprType& finished() { return m_xpr; } + inline XprType& finished() { + eigen_assert(((m_row+m_currentBlockRows) == m_xpr.rows() || m_xpr.cols() == 0) + && m_col == m_xpr.cols() + && "Too few coefficients passed to comma initializer (operator<<)"); + return m_xpr; + } XprType& m_xpr; // target expression Index m_row; // current row id @@ -144,7 +141,7 @@ * \sa CommaInitializer::finished(), class CommaInitializer */ template<typename Derived> -inline CommaInitializer<Derived> DenseBase<Derived>::operator<< (const Scalar& s) +EIGEN_DEVICE_FUNC inline CommaInitializer<Derived> DenseBase<Derived>::operator<< (const Scalar& s) { return CommaInitializer<Derived>(*static_cast<Derived*>(this), s); } @@ -152,7 +149,7 @@ /** \sa operator<<(const Scalar&) */ template<typename Derived> template<typename OtherDerived> -inline CommaInitializer<Derived> +EIGEN_DEVICE_FUNC inline CommaInitializer<Derived> DenseBase<Derived>::operator<<(const DenseBase<OtherDerived>& other) { return CommaInitializer<Derived>(*static_cast<Derived *>(this), other);
diff --git a/Eigen/src/Core/ConditionEstimator.h b/Eigen/src/Core/ConditionEstimator.h index 68c5e91..51a2e5f 100644 --- a/Eigen/src/Core/ConditionEstimator.h +++ b/Eigen/src/Core/ConditionEstimator.h
@@ -32,33 +32,6 @@ } }; -/** \brief Reciprocal condition number estimator. - * - * Computing a decomposition of a dense matrix takes O(n^3) operations, while - * this method estimates the condition number quickly and reliably in O(n^2) - * operations. - * - * \returns an estimate of the reciprocal condition number - * (1 / (||matrix||_1 * ||inv(matrix)||_1)) of matrix, given ||matrix||_1 and - * its decomposition. Supports the following decompositions: FullPivLU, - * PartialPivLU, LDLT, and LLT. - * - * \sa FullPivLU, PartialPivLU, LDLT, LLT. - */ -template <typename Decomposition> -typename Decomposition::RealScalar -rcond_estimate_helper(typename Decomposition::RealScalar matrix_norm, const Decomposition& dec) -{ - typedef typename Decomposition::RealScalar RealScalar; - eigen_assert(dec.rows() == dec.cols()); - if (dec.rows() == 0) return RealScalar(1); - if (matrix_norm == RealScalar(0)) return RealScalar(0); - if (dec.rows() == 1) return RealScalar(1); - const RealScalar inverse_matrix_norm = rcond_invmatrix_L1_norm_estimate(dec); - return (inverse_matrix_norm == RealScalar(0) ? RealScalar(0) - : (RealScalar(1) / inverse_matrix_norm) / matrix_norm); -} - /** * \returns an estimate of ||inv(matrix)||_1 given a decomposition of * \a matrix that implements .solve() and .adjoint().solve() methods. @@ -94,7 +67,15 @@ if (n == 0) return 0; + // Disable Index to float conversion warning +#ifdef __INTEL_COMPILER + #pragma warning push + #pragma warning ( disable : 2259 ) +#endif Vector v = dec.solve(Vector::Ones(n) / Scalar(n)); +#ifdef __INTEL_COMPILER + #pragma warning pop +#endif // lower_bound is a lower bound on // ||inv(matrix)||_1 = sup_v ||inv(matrix) v||_1 / ||v||_1 @@ -151,7 +132,8 @@ // Hager's algorithm to vastly underestimate ||matrix||_1. Scalar alternating_sign(RealScalar(1)); for (Index i = 0; i < n; ++i) { - v[i] = alternating_sign * (RealScalar(1) + (RealScalar(i) / (RealScalar(n - 1)))); + // The static_cast is needed when Scalar is a complex and RealScalar implements expression templates + v[i] = alternating_sign * static_cast<RealScalar>(RealScalar(1) + (RealScalar(i) / (RealScalar(n - 1)))); alternating_sign = -alternating_sign; } v = dec.solve(v); @@ -159,6 +141,33 @@ return numext::maxi(lower_bound, alternate_lower_bound); } +/** \brief Reciprocal condition number estimator. + * + * Computing a decomposition of a dense matrix takes O(n^3) operations, while + * this method estimates the condition number quickly and reliably in O(n^2) + * operations. + * + * \returns an estimate of the reciprocal condition number + * (1 / (||matrix||_1 * ||inv(matrix)||_1)) of matrix, given ||matrix||_1 and + * its decomposition. Supports the following decompositions: FullPivLU, + * PartialPivLU, LDLT, and LLT. + * + * \sa FullPivLU, PartialPivLU, LDLT, LLT. + */ +template <typename Decomposition> +typename Decomposition::RealScalar +rcond_estimate_helper(typename Decomposition::RealScalar matrix_norm, const Decomposition& dec) +{ + typedef typename Decomposition::RealScalar RealScalar; + eigen_assert(dec.rows() == dec.cols()); + if (dec.rows() == 0) return NumTraits<RealScalar>::infinity(); + if (matrix_norm == RealScalar(0)) return RealScalar(0); + if (dec.rows() == 1) return RealScalar(1); + const RealScalar inverse_matrix_norm = rcond_invmatrix_L1_norm_estimate(dec); + return (inverse_matrix_norm == RealScalar(0) ? RealScalar(0) + : (RealScalar(1) / inverse_matrix_norm) / matrix_norm); +} + } // namespace internal } // namespace Eigen
diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 388805f..670fa77 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h
@@ -41,11 +41,20 @@ // We currently distinguish the following kind of evaluators: // - unary_evaluator for expressions taking only one arguments (CwiseUnaryOp, CwiseUnaryView, Transpose, MatrixWrapper, ArrayWrapper, Reverse, Replicate) // - binary_evaluator for expression taking two arguments (CwiseBinaryOp) +// - ternary_evaluator for expression taking three arguments (CwiseTernaryOp) // - product_evaluator for linear algebra products (Product); special case of binary_evaluator because it requires additional tags for dispatching. // - mapbase_evaluator for Map, Block, Ref // - block_evaluator for Block (special dispatching to a mapbase_evaluator or unary_evaluator) template< typename T, + typename Arg1Kind = typename evaluator_traits<typename T::Arg1>::Kind, + typename Arg2Kind = typename evaluator_traits<typename T::Arg2>::Kind, + typename Arg3Kind = typename evaluator_traits<typename T::Arg3>::Kind, + typename Arg1Scalar = typename traits<typename T::Arg1>::Scalar, + typename Arg2Scalar = typename traits<typename T::Arg2>::Scalar, + typename Arg3Scalar = typename traits<typename T::Arg3>::Scalar> struct ternary_evaluator; + +template< typename T, typename LhsKind = typename evaluator_traits<typename T::Lhs>::Kind, typename RhsKind = typename evaluator_traits<typename T::Rhs>::Kind, typename LhsScalar = typename traits<typename T::Lhs>::Scalar, @@ -81,7 +90,8 @@ struct evaluator : public unary_evaluator<T> { typedef unary_evaluator<T> Base; - EIGEN_DEVICE_FUNC explicit evaluator(const T& xpr) : Base(xpr) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const T& xpr) : Base(xpr) {} }; @@ -90,14 +100,14 @@ struct evaluator<const T> : evaluator<T> { - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const T& xpr) : evaluator<T>(xpr) {} }; // ---------- base class for all evaluators ---------- template<typename ExpressionType> -struct evaluator_base : public noncopyable +struct evaluator_base { // TODO that's not very nice to have to propagate all these traits. They are currently only needed to handle outer,inner indices. typedef traits<ExpressionType> ExpressionTraits; @@ -105,6 +115,14 @@ enum { Alignment = 0 }; + // noncopyable: + // Don't make this class inherit noncopyable as this kills EBO (Empty Base Optimization) + // and make complex evaluator much larger than then should do. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator_base() {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~evaluator_base() {} +private: + EIGEN_DEVICE_FUNC evaluator_base(const evaluator_base&); + EIGEN_DEVICE_FUNC const evaluator_base& operator=(const evaluator_base&); }; // -------------------- Matrix and Array -------------------- @@ -114,6 +132,33 @@ // Here we directly specialize evaluator. This is not really a unary expression, and it is, by definition, dense, // so no need for more sophisticated dispatching. +// this helper permits to completely eliminate m_outerStride if it is known at compiletime. +template<typename Scalar,int OuterStride> class plainobjectbase_evaluator_data { +public: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride) : data(ptr) + { +#ifndef EIGEN_INTERNAL_DEBUGGING + EIGEN_UNUSED_VARIABLE(outerStride); +#endif + eigen_internal_assert(outerStride==OuterStride); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index outerStride() const { return OuterStride; } + const Scalar *data; +}; + +template<typename Scalar> class plainobjectbase_evaluator_data<Scalar,Dynamic> { +public: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride) : data(ptr), m_outerStride(outerStride) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index outerStride() const { return m_outerStride; } + const Scalar *data; +protected: + Index m_outerStride; +}; + template<typename Derived> struct evaluator<PlainObjectBase<Derived> > : evaluator_base<Derived> @@ -132,18 +177,23 @@ Flags = traits<Derived>::EvaluatorFlags, Alignment = traits<Derived>::Alignment }; - - EIGEN_DEVICE_FUNC evaluator() - : m_data(0), - m_outerStride(IsVectorAtCompileTime ? 0 - : int(IsRowMajor) ? ColsAtCompileTime - : RowsAtCompileTime) + enum { + // We do not need to know the outer stride for vectors + OuterStrideAtCompileTime = IsVectorAtCompileTime ? 0 + : int(IsRowMajor) ? ColsAtCompileTime + : RowsAtCompileTime + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + evaluator() + : m_d(0,OuterStrideAtCompileTime) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } - - EIGEN_DEVICE_FUNC explicit evaluator(const PlainObjectType& m) - : m_data(m.data()), m_outerStride(IsVectorAtCompileTime ? 0 : m.outerStride()) + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const PlainObjectType& m) + : m_d(m.data(),IsVectorAtCompileTime ? 0 : m.outerStride()) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } @@ -152,30 +202,30 @@ CoeffReturnType coeff(Index row, Index col) const { if (IsRowMajor) - return m_data[row * m_outerStride.value() + col]; + return m_d.data[row * m_d.outerStride() + col]; else - return m_data[row + col * m_outerStride.value()]; + return m_d.data[row + col * m_d.outerStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_data[index]; + return m_d.data[index]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { if (IsRowMajor) - return const_cast<Scalar*>(m_data)[row * m_outerStride.value() + col]; + return const_cast<Scalar*>(m_d.data)[row * m_d.outerStride() + col]; else - return const_cast<Scalar*>(m_data)[row + col * m_outerStride.value()]; + return const_cast<Scalar*>(m_d.data)[row + col * m_d.outerStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - return const_cast<Scalar*>(m_data)[index]; + return const_cast<Scalar*>(m_d.data)[index]; } template<int LoadMode, typename PacketType> @@ -183,16 +233,16 @@ PacketType packet(Index row, Index col) const { if (IsRowMajor) - return ploadt<PacketType, LoadMode>(m_data + row * m_outerStride.value() + col); + return ploadt<PacketType, LoadMode>(m_d.data + row * m_d.outerStride() + col); else - return ploadt<PacketType, LoadMode>(m_data + row + col * m_outerStride.value()); + return ploadt<PacketType, LoadMode>(m_d.data + row + col * m_d.outerStride()); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index index) const { - return ploadt<PacketType, LoadMode>(m_data + index); + return ploadt<PacketType, LoadMode>(m_d.data + index); } template<int StoreMode,typename PacketType> @@ -201,26 +251,22 @@ { if (IsRowMajor) return pstoret<Scalar, PacketType, StoreMode> - (const_cast<Scalar*>(m_data) + row * m_outerStride.value() + col, x); + (const_cast<Scalar*>(m_d.data) + row * m_d.outerStride() + col, x); else return pstoret<Scalar, PacketType, StoreMode> - (const_cast<Scalar*>(m_data) + row + col * m_outerStride.value(), x); + (const_cast<Scalar*>(m_d.data) + row + col * m_d.outerStride(), x); } template<int StoreMode, typename PacketType> EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { - return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_data) + index, x); + return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + index, x); } protected: - const Scalar *m_data; - // We do not need to know the outer stride for vectors - variable_if_dynamic<Index, IsVectorAtCompileTime ? 0 - : int(IsRowMajor) ? ColsAtCompileTime - : RowsAtCompileTime> m_outerStride; + plainobjectbase_evaluator_data<Scalar,OuterStrideAtCompileTime> m_d; }; template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> @@ -229,9 +275,11 @@ { typedef Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType; - EIGEN_DEVICE_FUNC evaluator() {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + evaluator() {} - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& m) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& m) : evaluator<PlainObjectBase<XprType> >(m) { } }; @@ -242,9 +290,11 @@ { typedef Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType; - EIGEN_DEVICE_FUNC evaluator() {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + evaluator() {} - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& m) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& m) : evaluator<PlainObjectBase<XprType> >(m) { } }; @@ -263,7 +313,8 @@ Alignment = evaluator<ArgType>::Alignment }; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& t) : m_argImpl(t.nestedExpression()) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& t) : m_argImpl(t.nestedExpression()) {} typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; @@ -328,6 +379,120 @@ // Like Matrix and Array, this is not really a unary expression, so we directly specialize evaluator. // Likewise, there is not need to more sophisticated dispatching here. +template<typename Scalar,typename NullaryOp, + bool has_nullary = has_nullary_operator<NullaryOp>::value, + bool has_unary = has_unary_operator<NullaryOp>::value, + bool has_binary = has_binary_operator<NullaryOp>::value> +struct nullary_wrapper +{ + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { return op(i,j); } + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { return op(i); } + + template <typename T, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { return op.template packetOp<T>(i,j); } + template <typename T, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { return op.template packetOp<T>(i); } +}; + +template<typename Scalar,typename NullaryOp> +struct nullary_wrapper<Scalar,NullaryOp,true,false,false> +{ + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType=0, IndexType=0) const { return op(); } + template <typename T, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType=0, IndexType=0) const { return op.template packetOp<T>(); } +}; + +template<typename Scalar,typename NullaryOp> +struct nullary_wrapper<Scalar,NullaryOp,false,false,true> +{ + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j=0) const { return op(i,j); } + template <typename T, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j=0) const { return op.template packetOp<T>(i,j); } +}; + +// We need the following specialization for vector-only functors assigned to a runtime vector, +// for instance, using linspace and assigning a RowVectorXd to a MatrixXd or even a row of a MatrixXd. +// In this case, i==0 and j is used for the actual iteration. +template<typename Scalar,typename NullaryOp> +struct nullary_wrapper<Scalar,NullaryOp,false,true,false> +{ + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { + eigen_assert(i==0 || j==0); + return op(i+j); + } + template <typename T, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { + eigen_assert(i==0 || j==0); + return op.template packetOp<T>(i+j); + } + + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { return op(i); } + template <typename T, typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { return op.template packetOp<T>(i); } +}; + +template<typename Scalar,typename NullaryOp> +struct nullary_wrapper<Scalar,NullaryOp,false,false,false> {}; + +#if 0 && EIGEN_COMP_MSVC>0 +// Disable this ugly workaround. This is now handled in traits<Ref>::match, +// but this piece of code might still become handly if some other weird compilation +// erros pop up again. + +// MSVC exhibits a weird compilation error when +// compiling: +// Eigen::MatrixXf A = MatrixXf::Random(3,3); +// Ref<const MatrixXf> R = 2.f*A; +// and that has_*ary_operator<scalar_constant_op<float>> have not been instantiated yet. +// The "problem" is that evaluator<2.f*A> is instantiated by traits<Ref>::match<2.f*A> +// and at that time has_*ary_operator<T> returns true regardless of T. +// Then nullary_wrapper is badly instantiated as nullary_wrapper<.,.,true,true,true>. +// The trick is thus to defer the proper instantiation of nullary_wrapper when coeff(), +// and packet() are really instantiated as implemented below: + +// This is a simple wrapper around Index to enforce the re-instantiation of +// has_*ary_operator when needed. +template<typename T> struct nullary_wrapper_workaround_msvc { + nullary_wrapper_workaround_msvc(const T&); + operator T()const; +}; + +template<typename Scalar,typename NullaryOp> +struct nullary_wrapper<Scalar,NullaryOp,true,true,true> +{ + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { + return nullary_wrapper<Scalar,NullaryOp, + has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().operator()(op,i,j); + } + template <typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { + return nullary_wrapper<Scalar,NullaryOp, + has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().operator()(op,i); + } + + template <typename T, typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { + return nullary_wrapper<Scalar,NullaryOp, + has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().template packetOp<T>(op,i,j); + } + template <typename T, typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { + return nullary_wrapper<Scalar,NullaryOp, + has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value, + has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().template packetOp<T>(op,i); + } +}; +#endif // MSVC workaround + template<typename NullaryOp, typename PlainObjectType> struct evaluator<CwiseNullaryOp<NullaryOp,PlainObjectType> > : evaluator_base<CwiseNullaryOp<NullaryOp,PlainObjectType> > @@ -347,41 +512,44 @@ }; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& n) - : m_functor(n.functor()) + : m_functor(n.functor()), m_wrapper() { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; + template <typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - CoeffReturnType coeff(Index row, Index col) const + CoeffReturnType coeff(IndexType row, IndexType col) const { - return m_functor(row, col); + return m_wrapper(m_functor, row, col); } + template <typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - CoeffReturnType coeff(Index index) const + CoeffReturnType coeff(IndexType index) const { - return m_functor(index); + return m_wrapper(m_functor,index); } - template<int LoadMode, typename PacketType> + template<int LoadMode, typename PacketType, typename IndexType> EIGEN_STRONG_INLINE - PacketType packet(Index row, Index col) const + PacketType packet(IndexType row, IndexType col) const { - return m_functor.template packetOp<Index,PacketType>(row, col); + return m_wrapper.template packetOp<PacketType>(m_functor, row, col); } - template<int LoadMode, typename PacketType> + template<int LoadMode, typename PacketType, typename IndexType> EIGEN_STRONG_INLINE - PacketType packet(Index index) const + PacketType packet(IndexType index) const { - return m_functor.template packetOp<Index,PacketType>(index); + return m_wrapper.template packetOp<PacketType>(m_functor, index); } protected: const NullaryOp m_functor; + const internal::nullary_wrapper<CoeffReturnType,NullaryOp> m_wrapper; }; // -------------------- CwiseUnaryOp -------------------- @@ -401,9 +569,7 @@ }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - explicit unary_evaluator(const XprType& op) - : m_functor(op.functor()), - m_argImpl(op.nestedExpression()) + explicit unary_evaluator(const XprType& op) : m_d(op) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<UnaryOp>::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); @@ -414,32 +580,138 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - return m_functor(m_argImpl.coeff(row, col)); + return m_d.func()(m_d.argImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_functor(m_argImpl.coeff(index)); + return m_d.func()(m_d.argImpl.coeff(index)); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { - return m_functor.packetOp(m_argImpl.template packet<LoadMode, PacketType>(row, col)); + return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(row, col)); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index index) const { - return m_functor.packetOp(m_argImpl.template packet<LoadMode, PacketType>(index)); + return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(index)); } protected: - const UnaryOp m_functor; - evaluator<ArgType> m_argImpl; + + // this helper permits to completely eliminate the functor if it is empty + class Data : private UnaryOp + { + public: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Data(const XprType& xpr) : UnaryOp(xpr.functor()), argImpl(xpr.nestedExpression()) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const UnaryOp& func() const { return static_cast<const UnaryOp&>(*this); } + evaluator<ArgType> argImpl; + }; + + Data m_d; +}; + +// -------------------- CwiseTernaryOp -------------------- + +// this is a ternary expression +template<typename TernaryOp, typename Arg1, typename Arg2, typename Arg3> +struct evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> > + : public ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> > +{ + typedef CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> XprType; + typedef ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> > Base; + + EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : Base(xpr) {} +}; + +template<typename TernaryOp, typename Arg1, typename Arg2, typename Arg3> +struct ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>, IndexBased, IndexBased> + : evaluator_base<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> > +{ + typedef CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> XprType; + + enum { + CoeffReadCost = evaluator<Arg1>::CoeffReadCost + evaluator<Arg2>::CoeffReadCost + evaluator<Arg3>::CoeffReadCost + functor_traits<TernaryOp>::Cost, + + Arg1Flags = evaluator<Arg1>::Flags, + Arg2Flags = evaluator<Arg2>::Flags, + Arg3Flags = evaluator<Arg3>::Flags, + SameType = is_same<typename Arg1::Scalar,typename Arg2::Scalar>::value && is_same<typename Arg1::Scalar,typename Arg3::Scalar>::value, + StorageOrdersAgree = (int(Arg1Flags)&RowMajorBit)==(int(Arg2Flags)&RowMajorBit) && (int(Arg1Flags)&RowMajorBit)==(int(Arg3Flags)&RowMajorBit), + Flags0 = (int(Arg1Flags) | int(Arg2Flags) | int(Arg3Flags)) & ( + HereditaryBits + | (int(Arg1Flags) & int(Arg2Flags) & int(Arg3Flags) & + ( (StorageOrdersAgree ? LinearAccessBit : 0) + | (functor_traits<TernaryOp>::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0) + ) + ) + ), + Flags = (Flags0 & ~RowMajorBit) | (Arg1Flags & RowMajorBit), + Alignment = EIGEN_PLAIN_ENUM_MIN( + EIGEN_PLAIN_ENUM_MIN(evaluator<Arg1>::Alignment, evaluator<Arg2>::Alignment), + evaluator<Arg3>::Alignment) + }; + + EIGEN_DEVICE_FUNC explicit ternary_evaluator(const XprType& xpr) : m_d(xpr) + { + EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<TernaryOp>::Cost); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + typedef typename XprType::CoeffReturnType CoeffReturnType; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + CoeffReturnType coeff(Index row, Index col) const + { + return m_d.func()(m_d.arg1Impl.coeff(row, col), m_d.arg2Impl.coeff(row, col), m_d.arg3Impl.coeff(row, col)); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + CoeffReturnType coeff(Index index) const + { + return m_d.func()(m_d.arg1Impl.coeff(index), m_d.arg2Impl.coeff(index), m_d.arg3Impl.coeff(index)); + } + + template<int LoadMode, typename PacketType> + EIGEN_STRONG_INLINE + PacketType packet(Index row, Index col) const + { + return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode,PacketType>(row, col), + m_d.arg2Impl.template packet<LoadMode,PacketType>(row, col), + m_d.arg3Impl.template packet<LoadMode,PacketType>(row, col)); + } + + template<int LoadMode, typename PacketType> + EIGEN_STRONG_INLINE + PacketType packet(Index index) const + { + return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode,PacketType>(index), + m_d.arg2Impl.template packet<LoadMode,PacketType>(index), + m_d.arg3Impl.template packet<LoadMode,PacketType>(index)); + } + +protected: + // this helper permits to completely eliminate the functor if it is empty + struct Data : private TernaryOp + { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Data(const XprType& xpr) : TernaryOp(xpr.functor()), arg1Impl(xpr.arg1()), arg2Impl(xpr.arg2()), arg3Impl(xpr.arg3()) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TernaryOp& func() const { return static_cast<const TernaryOp&>(*this); } + evaluator<Arg1> arg1Impl; + evaluator<Arg2> arg2Impl; + evaluator<Arg3> arg3Impl; + }; + + Data m_d; }; // -------------------- CwiseBinaryOp -------------------- @@ -452,7 +724,8 @@ typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; typedef binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > Base; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : Base(xpr) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& xpr) : Base(xpr) {} }; template<typename BinaryOp, typename Lhs, typename Rhs> @@ -480,10 +753,8 @@ Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator<Lhs>::Alignment,evaluator<Rhs>::Alignment) }; - EIGEN_DEVICE_FUNC explicit binary_evaluator(const XprType& xpr) - : m_functor(xpr.functor()), - m_lhsImpl(xpr.lhs()), - m_rhsImpl(xpr.rhs()) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit binary_evaluator(const XprType& xpr) : m_d(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<BinaryOp>::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); @@ -494,35 +765,45 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - return m_functor(m_lhsImpl.coeff(row, col), m_rhsImpl.coeff(row, col)); + return m_d.func()(m_d.lhsImpl.coeff(row, col), m_d.rhsImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_functor(m_lhsImpl.coeff(index), m_rhsImpl.coeff(index)); + return m_d.func()(m_d.lhsImpl.coeff(index), m_d.rhsImpl.coeff(index)); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { - return m_functor.packetOp(m_lhsImpl.template packet<LoadMode,PacketType>(row, col), - m_rhsImpl.template packet<LoadMode,PacketType>(row, col)); + return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode,PacketType>(row, col), + m_d.rhsImpl.template packet<LoadMode,PacketType>(row, col)); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index index) const { - return m_functor.packetOp(m_lhsImpl.template packet<LoadMode,PacketType>(index), - m_rhsImpl.template packet<LoadMode,PacketType>(index)); + return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode,PacketType>(index), + m_d.rhsImpl.template packet<LoadMode,PacketType>(index)); } protected: - const BinaryOp m_functor; - evaluator<Lhs> m_lhsImpl; - evaluator<Rhs> m_rhsImpl; + + // this helper permits to completely eliminate the functor if it is empty + struct Data : private BinaryOp + { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Data(const XprType& xpr) : BinaryOp(xpr.functor()), lhsImpl(xpr.lhs()), rhsImpl(xpr.rhs()) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const BinaryOp& func() const { return static_cast<const BinaryOp&>(*this); } + evaluator<Lhs> lhsImpl; + evaluator<Rhs> rhsImpl; + }; + + Data m_d; }; // -------------------- CwiseUnaryView -------------------- @@ -541,9 +822,7 @@ Alignment = 0 // FIXME it is not very clear why alignment is necessarily lost... }; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) - : m_unaryOp(op.functor()), - m_argImpl(op.nestedExpression()) + EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) : m_d(op) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<UnaryOp>::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); @@ -555,30 +834,40 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - return m_unaryOp(m_argImpl.coeff(row, col)); + return m_d.func()(m_d.argImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_unaryOp(m_argImpl.coeff(index)); + return m_d.func()(m_d.argImpl.coeff(index)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { - return m_unaryOp(m_argImpl.coeffRef(row, col)); + return m_d.func()(m_d.argImpl.coeffRef(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - return m_unaryOp(m_argImpl.coeffRef(index)); + return m_d.func()(m_d.argImpl.coeffRef(index)); } protected: - const UnaryOp m_unaryOp; - evaluator<ArgType> m_argImpl; + + // this helper permits to completely eliminate the functor if it is empty + struct Data : private UnaryOp + { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Data(const XprType& xpr) : UnaryOp(xpr.functor()), argImpl(xpr.nestedExpression()) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const UnaryOp& func() const { return static_cast<const UnaryOp&>(*this); } + evaluator<ArgType> argImpl; + }; + + Data m_d; }; // -------------------- Map -------------------- @@ -601,73 +890,80 @@ ColsAtCompileTime = XprType::ColsAtCompileTime, CoeffReadCost = NumTraits<Scalar>::ReadCost }; - - EIGEN_DEVICE_FUNC explicit mapbase_evaluator(const XprType& map) - : m_data(const_cast<PointerType>(map.data())), - m_xpr(map) + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit mapbase_evaluator(const XprType& map) + : m_data(const_cast<PointerType>(map.data())), + m_innerStride(map.innerStride()), + m_outerStride(map.outerStride()) { EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(evaluator<Derived>::Flags&PacketAccessBit, internal::inner_stride_at_compile_time<Derived>::ret==1), PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } - + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()]; + return m_data[col * colStride() + row * rowStride()]; } - + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_data[index * m_xpr.innerStride()]; + return m_data[index * m_innerStride.value()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { - return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()]; + return m_data[col * colStride() + row * rowStride()]; } - + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - return m_data[index * m_xpr.innerStride()]; + return m_data[index * m_innerStride.value()]; } - + template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE - PacketType packet(Index row, Index col) const + PacketType packet(Index row, Index col) const { - PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride(); + PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::ploadt<PacketType, LoadMode>(ptr); } template<int LoadMode, typename PacketType> EIGEN_STRONG_INLINE - PacketType packet(Index index) const + PacketType packet(Index index) const { - return internal::ploadt<PacketType, LoadMode>(m_data + index * m_xpr.innerStride()); + return internal::ploadt<PacketType, LoadMode>(m_data + index * m_innerStride.value()); } - + template<int StoreMode, typename PacketType> EIGEN_STRONG_INLINE - void writePacket(Index row, Index col, const PacketType& x) + void writePacket(Index row, Index col, const PacketType& x) { - PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride(); + PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::pstoret<Scalar, PacketType, StoreMode>(ptr, x); } - + template<int StoreMode, typename PacketType> EIGEN_STRONG_INLINE - void writePacket(Index index, const PacketType& x) + void writePacket(Index index, const PacketType& x) { - internal::pstoret<Scalar, PacketType, StoreMode>(m_data + index * m_xpr.innerStride(), x); + internal::pstoret<Scalar, PacketType, StoreMode>(m_data + index * m_innerStride.value(), x); } - protected: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index rowStride() const { return XprType::IsRowMajor ? m_outerStride.value() : m_innerStride.value(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index colStride() const { return XprType::IsRowMajor ? m_innerStride.value() : m_outerStride.value(); } + PointerType m_data; - const XprType& m_xpr; + const internal::variable_if_dynamic<Index, XprType::InnerStrideAtCompileTime> m_innerStride; + const internal::variable_if_dynamic<Index, XprType::OuterStrideAtCompileTime> m_outerStride; }; template<typename PlainObjectType, int MapOptions, typename StrideType> @@ -716,7 +1012,8 @@ Alignment = evaluator<Map<PlainObjectType, RefOptions, StrideType> >::Alignment }; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& ref) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& ref) : mapbase_evaluator<XprType, PlainObjectType>(ref) { } }; @@ -755,9 +1052,7 @@ OuterStrideAtCompileTime = HasSameStorageOrderAsArgType ? int(outer_stride_at_compile_time<ArgType>::ret) : int(inner_stride_at_compile_time<ArgType>::ret), - MaskPacketAccessBit = (InnerSize == Dynamic || (InnerSize % packet_traits<Scalar>::size) == 0) - && (InnerStrideAtCompileTime == 1) - ? PacketAccessBit : 0, + MaskPacketAccessBit = (InnerStrideAtCompileTime == 1 || HasSameStorageOrderAsArgType) ? PacketAccessBit : 0, FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1 || (InnerPanel && (evaluator<ArgType>::Flags&LinearAccessBit))) ? LinearAccessBit : 0, FlagsRowMajorBit = XprType::Flags&RowMajorBit, @@ -767,11 +1062,14 @@ Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit, PacketAlignment = unpacket_traits<PacketScalar>::alignment, - Alignment0 = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % int(PacketAlignment)) == 0)) ? int(PacketAlignment) : 0, + Alignment0 = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) + && (OuterStrideAtCompileTime!=0) + && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % int(PacketAlignment)) == 0)) ? int(PacketAlignment) : 0, Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator<ArgType>::Alignment, Alignment0) }; typedef block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel> block_evaluator_type; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& block) : block_evaluator_type(block) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& block) : block_evaluator_type(block) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } @@ -784,7 +1082,8 @@ { typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; - EIGEN_DEVICE_FUNC explicit block_evaluator(const XprType& block) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit block_evaluator(const XprType& block) : unary_evaluator<XprType>(block) {} }; @@ -795,17 +1094,20 @@ { typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& block) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& block) : m_argImpl(block.nestedExpression()), m_startRow(block.startRow()), - m_startCol(block.startCol()) + m_startCol(block.startCol()), + m_linear_offset(ForwardLinearAccess?(ArgType::IsRowMajor ? block.startRow()*block.nestedExpression().cols() + block.startCol() : block.startCol()*block.nestedExpression().rows() + block.startRow()):0) { } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { - RowsAtCompileTime = XprType::RowsAtCompileTime + RowsAtCompileTime = XprType::RowsAtCompileTime, + ForwardLinearAccess = (InnerPanel || int(XprType::IsRowMajor)==int(ArgType::IsRowMajor)) && bool(evaluator<ArgType>::Flags&LinearAccessBit) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -817,7 +1119,10 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); + if (ForwardLinearAccess) + return m_argImpl.coeff(m_linear_offset.value() + index); + else + return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -829,7 +1134,10 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); + if (ForwardLinearAccess) + return m_argImpl.coeffRef(m_linear_offset.value() + index); + else + return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } template<int LoadMode, typename PacketType> @@ -843,30 +1151,37 @@ EIGEN_STRONG_INLINE PacketType packet(Index index) const { - return packet<LoadMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + if (ForwardLinearAccess) + return m_argImpl.template packet<LoadMode,PacketType>(m_linear_offset.value() + index); + else + return packet<LoadMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); } template<int StoreMode, typename PacketType> EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) - { + { return m_argImpl.template writePacket<StoreMode,PacketType>(m_startRow.value() + row, m_startCol.value() + col, x); } template<int StoreMode, typename PacketType> EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) - { - return writePacket<StoreMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, - x); + { + if (ForwardLinearAccess) + return m_argImpl.template writePacket<StoreMode,PacketType>(m_linear_offset.value() + index, x); + else + return writePacket<StoreMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0, + x); } protected: evaluator<ArgType> m_argImpl; const variable_if_dynamic<Index, (ArgType::RowsAtCompileTime == 1 && BlockRows==1) ? 0 : Dynamic> m_startRow; const variable_if_dynamic<Index, (ArgType::ColsAtCompileTime == 1 && BlockCols==1) ? 0 : Dynamic> m_startCol; + const variable_if_dynamic<Index, ForwardLinearAccess ? Dynamic : 0> m_linear_offset; }; // TODO: This evaluator does not actually use the child evaluator; @@ -880,11 +1195,12 @@ typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; typedef typename XprType::Scalar Scalar; - EIGEN_DEVICE_FUNC explicit block_evaluator(const XprType& block) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit block_evaluator(const XprType& block) : mapbase_evaluator<XprType, typename XprType::PlainObject>(block) { // TODO: for the 3.3 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime - eigen_assert(((size_t(block.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator<XprType>::Alignment)) == 0) && "data is not aligned"); + eigen_assert(((internal::UIntPtr(block.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator<XprType>::Alignment)) == 0) && "data is not aligned"); } }; @@ -908,7 +1224,8 @@ Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator<ThenMatrixType>::Alignment, evaluator<ElseMatrixType>::Alignment) }; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& select) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& select) : m_conditionImpl(select.conditionMatrix()), m_thenImpl(select.thenMatrix()), m_elseImpl(select.elseMatrix()) @@ -965,7 +1282,8 @@ Alignment = evaluator<ArgTypeNestedCleaned>::Alignment }; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& replicate) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& replicate) : m_arg(replicate.nestedExpression()), m_argImpl(m_arg), m_rows(replicate.nestedExpression().rows()), @@ -1029,64 +1347,6 @@ const variable_if_dynamic<Index, ArgType::ColsAtCompileTime> m_cols; }; - -// -------------------- PartialReduxExpr -------------------- - -template< typename ArgType, typename MemberOp, int Direction> -struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> > - : evaluator_base<PartialReduxExpr<ArgType, MemberOp, Direction> > -{ - typedef PartialReduxExpr<ArgType, MemberOp, Direction> XprType; - typedef typename internal::nested_eval<ArgType,1>::type ArgTypeNested; - typedef typename internal::remove_all<ArgTypeNested>::type ArgTypeNestedCleaned; - typedef typename ArgType::Scalar InputScalar; - typedef typename XprType::Scalar Scalar; - enum { - TraversalSize = Direction==int(Vertical) ? int(ArgType::RowsAtCompileTime) : int(ArgType::ColsAtCompileTime) - }; - typedef typename MemberOp::template Cost<InputScalar,int(TraversalSize)> CostOpType; - enum { - CoeffReadCost = TraversalSize==Dynamic ? HugeCost - : TraversalSize * evaluator<ArgType>::CoeffReadCost + int(CostOpType::value), - - Flags = (traits<XprType>::Flags&RowMajorBit) | (evaluator<ArgType>::Flags&(HereditaryBits&(~RowMajorBit))) | LinearAccessBit, - - Alignment = 0 // FIXME this will need to be improved once PartialReduxExpr is vectorized - }; - - EIGEN_DEVICE_FUNC explicit evaluator(const XprType xpr) - : m_arg(xpr.nestedExpression()), m_functor(xpr.functor()) - { - EIGEN_INTERNAL_CHECK_COST_VALUE(TraversalSize==Dynamic ? HugeCost : int(CostOpType::value)); - EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); - } - - typedef typename XprType::CoeffReturnType CoeffReturnType; - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const Scalar coeff(Index i, Index j) const - { - if (Direction==Vertical) - return m_functor(m_arg.col(j)); - else - return m_functor(m_arg.row(i)); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const Scalar coeff(Index index) const - { - if (Direction==Vertical) - return m_functor(m_arg.col(index)); - else - return m_functor(m_arg.row(index)); - } - -protected: - const ArgTypeNested m_arg; - const MemberOp m_functor; -}; - - // -------------------- MatrixWrapper and ArrayWrapper -------------------- // // evaluator_wrapper_base<T> is a common base class for the @@ -1103,7 +1363,8 @@ Alignment = evaluator<ArgType>::Alignment }; - EIGEN_DEVICE_FUNC explicit evaluator_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} typedef typename ArgType::Scalar Scalar; typedef typename ArgType::CoeffReturnType CoeffReturnType; @@ -1170,7 +1431,8 @@ { typedef MatrixWrapper<TArgType> XprType; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& wrapper) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& wrapper) : evaluator_wrapper_base<MatrixWrapper<TArgType> >(wrapper.nestedExpression()) { } }; @@ -1181,7 +1443,8 @@ { typedef ArrayWrapper<TArgType> XprType; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& wrapper) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& wrapper) : evaluator_wrapper_base<ArrayWrapper<TArgType> >(wrapper.nestedExpression()) { } }; @@ -1223,7 +1486,8 @@ Alignment = 0 // FIXME in some rare cases, Alignment could be preserved, like a Vector4f. }; - EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& reverse) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit unary_evaluator(const XprType& reverse) : m_argImpl(reverse.nestedExpression()), m_rows(ReverseRow ? reverse.nestedExpression().rows() : 1), m_cols(ReverseCol ? reverse.nestedExpression().cols() : 1) @@ -1325,20 +1589,19 @@ enum { CoeffReadCost = evaluator<ArgType>::CoeffReadCost, - Flags = (unsigned int)evaluator<ArgType>::Flags & (HereditaryBits | LinearAccessBit | DirectAccessBit) & ~RowMajorBit, + Flags = (unsigned int)(evaluator<ArgType>::Flags & (HereditaryBits | DirectAccessBit) & ~RowMajorBit) | LinearAccessBit, Alignment = 0 }; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& diagonal) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit evaluator(const XprType& diagonal) : m_argImpl(diagonal.nestedExpression()), m_index(diagonal.index()) { } typedef typename XprType::Scalar Scalar; - // FIXME having to check whether ArgType is sparse here i not very nice. - typedef typename internal::conditional<!internal::is_same<typename ArgType::StorageKind,Sparse>::value, - typename XprType::CoeffReturnType,Scalar>::type CoeffReturnType; + typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index) const
diff --git a/Eigen/src/Core/CoreIterators.h b/Eigen/src/Core/CoreIterators.h index 4eb42b9..b967196 100644 --- a/Eigen/src/Core/CoreIterators.h +++ b/Eigen/src/Core/CoreIterators.h
@@ -48,6 +48,11 @@ * Explicit zeros are not skipped over. To skip explicit zeros, see class SparseView */ EIGEN_STRONG_INLINE InnerIterator& operator++() { m_iter.operator++(); return *this; } + EIGEN_STRONG_INLINE InnerIterator& operator+=(Index i) { m_iter.operator+=(i); return *this; } + EIGEN_STRONG_INLINE InnerIterator operator+(Index i) + { InnerIterator result(*this); result+=i; return result; } + + /// \returns the column or row index of the current coefficient. EIGEN_STRONG_INLINE Index index() const { return m_iter.index(); } /// \returns the row index of the current coefficient.
diff --git a/Eigen/src/Core/CwiseBinaryOp.h b/Eigen/src/Core/CwiseBinaryOp.h index 39820fd..8b8de83 100644 --- a/Eigen/src/Core/CwiseBinaryOp.h +++ b/Eigen/src/Core/CwiseBinaryOp.h
@@ -46,7 +46,7 @@ typedef typename remove_reference<LhsNested>::type _LhsNested; typedef typename remove_reference<RhsNested>::type _RhsNested; enum { - Flags = _LhsNested::Flags & RowMajorBit + Flags = cwise_promote_storage_order<typename traits<Lhs>::StorageKind,typename traits<Rhs>::StorageKind,_LhsNested::Flags & RowMajorBit,_RhsNested::Flags & RowMajorBit>::value }; }; } // end namespace internal @@ -84,6 +84,7 @@ { public: + typedef typename internal::remove_all<BinaryOp>::type Functor; typedef typename internal::remove_all<LhsType>::type Lhs; typedef typename internal::remove_all<RhsType>::type Rhs; @@ -99,8 +100,14 @@ typedef typename internal::remove_reference<LhsNested>::type _LhsNested; typedef typename internal::remove_reference<RhsNested>::type _RhsNested; - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE CwiseBinaryOp(const Lhs& aLhs, const Rhs& aRhs, const BinaryOp& func = BinaryOp()) +#if EIGEN_COMP_MSVC && EIGEN_HAS_CXX11 + //Required for Visual Studio or the Copy constructor will probably not get inlined! + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + CwiseBinaryOp(const CwiseBinaryOp<BinaryOp,LhsType,RhsType>&) = default; +#endif + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + CwiseBinaryOp(const Lhs& aLhs, const Rhs& aRhs, const BinaryOp& func = BinaryOp()) : m_lhs(aLhs), m_rhs(aRhs), m_functor(func) { EIGEN_CHECK_BINARY_COMPATIBILIY(BinaryOp,typename Lhs::Scalar,typename Rhs::Scalar); @@ -109,16 +116,16 @@ eigen_assert(aLhs.rows() == aRhs.rows() && aLhs.cols() == aRhs.cols()); } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Index rows() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index rows() const { // return the fixed size type if available to enable compile time optimizations if (internal::traits<typename internal::remove_all<LhsNested>::type>::RowsAtCompileTime==Dynamic) return m_rhs.rows(); else return m_lhs.rows(); } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Index cols() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index cols() const { // return the fixed size type if available to enable compile time optimizations if (internal::traits<typename internal::remove_all<LhsNested>::type>::ColsAtCompileTime==Dynamic) return m_rhs.cols(); @@ -127,13 +134,13 @@ } /** \returns the left hand side nested expression */ - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const _LhsNested& lhs() const { return m_lhs; } /** \returns the right hand side nested expression */ - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const _RhsNested& rhs() const { return m_rhs; } /** \returns the functor representing the binary operation */ - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& functor() const { return m_functor; } protected: @@ -157,10 +164,10 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & MatrixBase<Derived>::operator-=(const MatrixBase<OtherDerived> &other) { - call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -170,14 +177,13 @@ */ template<typename Derived> template<typename OtherDerived> -EIGEN_STRONG_INLINE Derived & +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & MatrixBase<Derived>::operator+=(const MatrixBase<OtherDerived>& other) { - call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } } // end namespace Eigen #endif // EIGEN_CWISE_BINARY_OP_H -
diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index 3c6508c..ef70819 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h
@@ -20,7 +20,8 @@ Flags = traits<PlainObjectType>::Flags & RowMajorBit }; }; -} + +} // namespace internal /** \class CwiseNullaryOp * \ingroup Core_Module @@ -37,7 +38,23 @@ * However, if you want to write a function returning such an expression, you * will need to use this class. * - * \sa class CwiseUnaryOp, class CwiseBinaryOp, DenseBase::NullaryExpr() + * The functor NullaryOp must expose one of the following method: + <table class="manual"> + <tr ><td>\c operator()() </td><td>if the procedural generation does not depend on the coefficient entries (e.g., random numbers)</td></tr> + <tr class="alt"><td>\c operator()(Index i)</td><td>if the procedural generation makes sense for vectors only and that it depends on the coefficient index \c i (e.g., linspace) </td></tr> + <tr ><td>\c operator()(Index i,Index j)</td><td>if the procedural generation depends on the matrix coordinates \c i, \c j (e.g., to generate a checkerboard with 0 and 1)</td></tr> + </table> + * It is also possible to expose the last two operators if the generation makes sense for matrices but can be optimized for vectors. + * + * See DenseBase::NullaryExpr(Index,const CustomNullaryOp&) for an example binding + * C++11 random number generators. + * + * A nullary expression can also be used to implement custom sophisticated matrix manipulations + * that cannot be covered by the existing set of natively supported matrix manipulations. + * See this \ref TopicCustomizing_NullaryExpr "page" for some examples and additional explanations + * on the behavior of CwiseNullaryOp. + * + * \sa class CwiseUnaryOp, class CwiseBinaryOp, DenseBase::NullaryExpr */ template<typename NullaryOp, typename PlainObjectType> class CwiseNullaryOp : public internal::dense_xpr_base< CwiseNullaryOp<NullaryOp, PlainObjectType> >::type, internal::no_assignment_operator @@ -62,30 +79,6 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index cols() const { return m_cols.value(); } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar coeff(Index rowId, Index colId) const - { - return m_functor(rowId, colId); - } - - template<int LoadMode> - EIGEN_STRONG_INLINE PacketScalar packet(Index rowId, Index colId) const - { - return m_functor.packetOp(rowId, colId); - } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar coeff(Index index) const - { - return m_functor(index); - } - - template<int LoadMode> - EIGEN_STRONG_INLINE PacketScalar packet(Index index) const - { - return m_functor.packetOp(index); - } - /** \returns the functor representing the nullary operation */ EIGEN_DEVICE_FUNC const NullaryOp& functor() const { return m_functor; } @@ -112,7 +105,12 @@ */ template<typename Derived> template<typename CustomNullaryOp> -EIGEN_STRONG_INLINE const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const CwiseNullaryOp<CustomNullaryOp,typename DenseBase<Derived>::PlainObject> +#else +const CwiseNullaryOp<CustomNullaryOp,PlainObject> +#endif DenseBase<Derived>::NullaryExpr(Index rows, Index cols, const CustomNullaryOp& func) { return CwiseNullaryOp<CustomNullaryOp, PlainObject>(rows, cols, func); @@ -138,7 +136,12 @@ */ template<typename Derived> template<typename CustomNullaryOp> -EIGEN_STRONG_INLINE const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject> +#else +const CwiseNullaryOp<CustomNullaryOp, PlainObject> +#endif DenseBase<Derived>::NullaryExpr(Index size, const CustomNullaryOp& func) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) @@ -157,7 +160,12 @@ */ template<typename Derived> template<typename CustomNullaryOp> -EIGEN_STRONG_INLINE const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject> +#else +const CwiseNullaryOp<CustomNullaryOp, PlainObject> +#endif DenseBase<Derived>::NullaryExpr(const CustomNullaryOp& func) { return CwiseNullaryOp<CustomNullaryOp, PlainObject>(RowsAtCompileTime, ColsAtCompileTime, func); @@ -177,7 +185,7 @@ * \sa class CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Constant(Index rows, Index cols, const Scalar& value) { return DenseBase<Derived>::NullaryExpr(rows, cols, internal::scalar_constant_op<Scalar>(value)); @@ -199,7 +207,7 @@ * \sa class CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Constant(Index size, const Scalar& value) { return DenseBase<Derived>::NullaryExpr(size, internal::scalar_constant_op<Scalar>(value)); @@ -215,49 +223,36 @@ * \sa class CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Constant(const Scalar& value) { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) return DenseBase<Derived>::NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_constant_op<Scalar>(value)); } -/** - * \brief Sets a linearly spaced vector. +/** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(Index,const Scalar&,const Scalar&) * - * The function generates 'size' equally spaced values in the closed interval [low,high]. - * This particular version of LinSpaced() uses sequential access, i.e. vector access is - * assumed to be a(0), a(1), ..., a(size). This assumption allows for better vectorization - * and yields faster code than the random access version. - * - * When size is set to 1, a vector of length 1 containing 'high' is returned. - * - * \only_for_vectors - * - * Example: \include DenseBase_LinSpaced_seq.cpp - * Output: \verbinclude DenseBase_LinSpaced_seq.out - * - * \sa setLinSpaced(Index,const Scalar&,const Scalar&), LinSpaced(Index,Scalar,Scalar), CwiseNullaryOp + * \sa LinSpaced(Index,Scalar,Scalar), setLinSpaced(Index,const Scalar&,const Scalar&) */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::SequentialLinSpacedReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType DenseBase<Derived>::LinSpaced(Sequential_t, Index size, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar,PacketScalar,false>(low,high,size)); + return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar>(low,high,size)); } -/** - * \copydoc DenseBase::LinSpaced(Sequential_t, Index, const Scalar&, const Scalar&) - * Special version for fixed size types which does not require the size parameter. +/** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(const Scalar&,const Scalar&) + * + * \sa LinSpaced(Scalar,Scalar) */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::SequentialLinSpacedReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType DenseBase<Derived>::LinSpaced(Sequential_t, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) - return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op<Scalar,PacketScalar,false>(low,high,Derived::SizeAtCompileTime)); + return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op<Scalar>(low,high,Derived::SizeAtCompileTime)); } /** @@ -271,14 +266,24 @@ * Example: \include DenseBase_LinSpaced.cpp * Output: \verbinclude DenseBase_LinSpaced.out * - * \sa setLinSpaced(Index,const Scalar&,const Scalar&), LinSpaced(Sequential_t,Index,const Scalar&,const Scalar&,Index), CwiseNullaryOp + * For integer scalar types, an even spacing is possible if and only if the length of the range, + * i.e., \c high-low is a scalar multiple of \c size-1, or if \c size is a scalar multiple of the + * number of values \c high-low+1 (meaning each value can be repeated the same number of time). + * If one of these two considions is not satisfied, then \c high is lowered to the largest value + * satisfying one of this constraint. + * Here are some examples: + * + * Example: \include DenseBase_LinSpacedInt.cpp + * Output: \verbinclude DenseBase_LinSpacedInt.out + * + * \sa setLinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType DenseBase<Derived>::LinSpaced(Index size, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar,PacketScalar,true>(low,high,size)); + return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar>(low,high,size)); } /** @@ -286,17 +291,17 @@ * Special version for fixed size types which does not require the size parameter. */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType DenseBase<Derived>::LinSpaced(const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) - return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op<Scalar,PacketScalar,true>(low,high,Derived::SizeAtCompileTime)); + return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op<Scalar>(low,high,Derived::SizeAtCompileTime)); } /** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */ template<typename Derived> -bool DenseBase<Derived>::isApproxToConstant +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isApproxToConstant (const Scalar& val, const RealScalar& prec) const { typename internal::nested_eval<Derived,1>::type self(derived()); @@ -311,7 +316,7 @@ * * \returns true if all coefficients in this matrix are approximately equal to \a value, to within precision \a prec */ template<typename Derived> -bool DenseBase<Derived>::isConstant +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isConstant (const Scalar& val, const RealScalar& prec) const { return isApproxToConstant(val, prec); @@ -322,7 +327,7 @@ * \sa setConstant(), Constant(), class CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE void DenseBase<Derived>::fill(const Scalar& val) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void DenseBase<Derived>::fill(const Scalar& val) { setConstant(val); } @@ -332,7 +337,7 @@ * \sa fill(), setConstant(Index,const Scalar&), setConstant(Index,Index,const Scalar&), setZero(), setOnes(), Constant(), class CwiseNullaryOp, setZero(), setOnes() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setConstant(const Scalar& val) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setConstant(const Scalar& val) { return derived() = Constant(rows(), cols(), val); } @@ -347,7 +352,7 @@ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(Index size, const Scalar& val) { resize(size); @@ -366,7 +371,7 @@ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(Index rows, Index cols, const Scalar& val) { resize(rows, cols); @@ -384,27 +389,33 @@ * Example: \include DenseBase_setLinSpaced.cpp * Output: \verbinclude DenseBase_setLinSpaced.out * - * \sa CwiseNullaryOp + * For integer scalar types, do not miss the explanations on the definition + * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink. + * + * \sa LinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(Index newSize, const Scalar& low, const Scalar& high) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(Index newSize, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return derived() = Derived::NullaryExpr(newSize, internal::linspaced_op<Scalar,PacketScalar,false>(low,high,newSize)); + return derived() = Derived::NullaryExpr(newSize, internal::linspaced_op<Scalar>(low,high,newSize)); } /** * \brief Sets a linearly spaced vector. * - * The function fill *this with equally spaced values in the closed interval [low,high]. + * The function fills \c *this with equally spaced values in the closed interval [low,high]. * When size is set to 1, a vector of length 1 containing 'high' is returned. * * \only_for_vectors * - * \sa setLinSpaced(Index, const Scalar&, const Scalar&), CwiseNullaryOp + * For integer scalar types, do not miss the explanations on the definition + * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink. + * + * \sa LinSpaced(Index,const Scalar&,const Scalar&), setLinSpaced(Index, const Scalar&, const Scalar&), CwiseNullaryOp */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(const Scalar& low, const Scalar& high) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return setLinSpaced(size(), low, high); @@ -427,7 +438,7 @@ * \sa Zero(), Zero(Index) */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Zero(Index rows, Index cols) { return Constant(rows, cols, Scalar(0)); @@ -450,7 +461,7 @@ * \sa Zero(), Zero(Index,Index) */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Zero(Index size) { return Constant(size, Scalar(0)); @@ -467,7 +478,7 @@ * \sa Zero(Index), Zero(Index,Index) */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Zero() { return Constant(Scalar(0)); @@ -482,7 +493,7 @@ * \sa class CwiseNullaryOp, Zero() */ template<typename Derived> -bool DenseBase<Derived>::isZero(const RealScalar& prec) const +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isZero(const RealScalar& prec) const { typename internal::nested_eval<Derived,1>::type self(derived()); for(Index j = 0; j < cols(); ++j) @@ -500,7 +511,7 @@ * \sa class CwiseNullaryOp, Zero() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setZero() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setZero() { return setConstant(Scalar(0)); } @@ -515,7 +526,7 @@ * \sa DenseBase::setZero(), setZero(Index,Index), class CwiseNullaryOp, DenseBase::Zero() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(Index newSize) { resize(newSize); @@ -533,7 +544,7 @@ * \sa DenseBase::setZero(), setZero(Index), class CwiseNullaryOp, DenseBase::Zero() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(Index rows, Index cols) { resize(rows, cols); @@ -557,7 +568,7 @@ * \sa Ones(), Ones(Index), isOnes(), class Ones */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones(Index rows, Index cols) { return Constant(rows, cols, Scalar(1)); @@ -580,7 +591,7 @@ * \sa Ones(), Ones(Index,Index), isOnes(), class Ones */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones(Index newSize) { return Constant(newSize, Scalar(1)); @@ -597,7 +608,7 @@ * \sa Ones(Index), Ones(Index,Index), isOnes(), class Ones */ template<typename Derived> -EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones() { return Constant(Scalar(1)); @@ -612,7 +623,7 @@ * \sa class CwiseNullaryOp, Ones() */ template<typename Derived> -bool DenseBase<Derived>::isOnes +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isOnes (const RealScalar& prec) const { return isApproxToConstant(Scalar(1), prec); @@ -626,7 +637,7 @@ * \sa class CwiseNullaryOp, Ones() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setOnes() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setOnes() { return setConstant(Scalar(1)); } @@ -641,7 +652,7 @@ * \sa MatrixBase::setOnes(), setOnes(Index,Index), class CwiseNullaryOp, MatrixBase::Ones() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(Index newSize) { resize(newSize); @@ -659,7 +670,7 @@ * \sa MatrixBase::setOnes(), setOnes(Index), class CwiseNullaryOp, MatrixBase::Ones() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(Index rows, Index cols) { resize(rows, cols); @@ -683,7 +694,7 @@ * \sa Identity(), setIdentity(), isIdentity() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType MatrixBase<Derived>::Identity(Index rows, Index cols) { return DenseBase<Derived>::NullaryExpr(rows, cols, internal::scalar_identity_op<Scalar>()); @@ -700,7 +711,7 @@ * \sa Identity(Index,Index), setIdentity(), isIdentity() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType MatrixBase<Derived>::Identity() { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) @@ -759,7 +770,7 @@ static EIGEN_STRONG_INLINE Derived& run(Derived& m) { m.setZero(); - const Index size = (std::min)(m.rows(), m.cols()); + const Index size = numext::mini(m.rows(), m.cols()); for(Index i = 0; i < size; ++i) m.coeffRef(i,i) = typename Derived::Scalar(1); return m; } @@ -775,7 +786,7 @@ * \sa class CwiseNullaryOp, Identity(), Identity(Index,Index), isIdentity() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity() { return internal::setIdentity_impl<Derived>::run(derived()); } @@ -791,7 +802,7 @@ * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Identity() */ template<typename Derived> -EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity(Index rows, Index cols) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity(Index rows, Index cols) { derived().resize(rows, cols); return setIdentity(); @@ -804,7 +815,7 @@ * \sa MatrixBase::Unit(Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(Index newSize, Index i) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(Index newSize, Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(newSize,newSize), i); @@ -819,7 +830,7 @@ * \sa MatrixBase::Unit(Index,Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(Index i) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(),i); @@ -832,7 +843,7 @@ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitX() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitX() { return Derived::Unit(0); } /** \returns an expression of the Y axis unit vector (0,1{,0}^*) @@ -842,7 +853,7 @@ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitY() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitY() { return Derived::Unit(1); } /** \returns an expression of the Z axis unit vector (0,0,1{,0}^*) @@ -852,7 +863,7 @@ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitZ() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitZ() { return Derived::Unit(2); } /** \returns an expression of the W axis unit vector (0,0,0,1) @@ -862,9 +873,45 @@ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template<typename Derived> -EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitW() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitW() { return Derived::Unit(3); } +/** \brief Set the coefficients of \c *this to the i-th unit (basis) vector + * + * \param i index of the unique coefficient to be set to 1 + * + * \only_for_vectors + * + * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Unit(Index,Index) + */ +template<typename Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setUnit(Index i) +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + eigen_assert(i<size()); + derived().setZero(); + derived().coeffRef(i) = Scalar(1); + return derived(); +} + +/** \brief Resizes to the given \a newSize, and writes the i-th unit (basis) vector into *this. + * + * \param newSize the new size of the vector + * \param i index of the unique coefficient to be set to 1 + * + * \only_for_vectors + * + * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Unit(Index,Index) + */ +template<typename Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setUnit(Index newSize, Index i) +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + eigen_assert(i<newSize); + derived().resize(newSize); + return setUnit(i); +} + } // end namespace Eigen #endif // EIGEN_CWISE_NULLARY_OP_H
diff --git a/Eigen/src/Core/CwiseTernaryOp.h b/Eigen/src/Core/CwiseTernaryOp.h new file mode 100644 index 0000000..9f3576f --- /dev/null +++ b/Eigen/src/Core/CwiseTernaryOp.h
@@ -0,0 +1,197 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> +// Copyright (C) 2016 Eugene Brevdo <ebrevdo@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CWISE_TERNARY_OP_H +#define EIGEN_CWISE_TERNARY_OP_H + +namespace Eigen { + +namespace internal { +template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3> +struct traits<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> > { + // we must not inherit from traits<Arg1> since it has + // the potential to cause problems with MSVC + typedef typename remove_all<Arg1>::type Ancestor; + typedef typename traits<Ancestor>::XprKind XprKind; + enum { + RowsAtCompileTime = traits<Ancestor>::RowsAtCompileTime, + ColsAtCompileTime = traits<Ancestor>::ColsAtCompileTime, + MaxRowsAtCompileTime = traits<Ancestor>::MaxRowsAtCompileTime, + MaxColsAtCompileTime = traits<Ancestor>::MaxColsAtCompileTime + }; + + // even though we require Arg1, Arg2, and Arg3 to have the same scalar type + // (see CwiseTernaryOp constructor), + // we still want to handle the case when the result type is different. + typedef typename result_of<TernaryOp( + const typename Arg1::Scalar&, const typename Arg2::Scalar&, + const typename Arg3::Scalar&)>::type Scalar; + + typedef typename internal::traits<Arg1>::StorageKind StorageKind; + typedef typename internal::traits<Arg1>::StorageIndex StorageIndex; + + typedef typename Arg1::Nested Arg1Nested; + typedef typename Arg2::Nested Arg2Nested; + typedef typename Arg3::Nested Arg3Nested; + typedef typename remove_reference<Arg1Nested>::type _Arg1Nested; + typedef typename remove_reference<Arg2Nested>::type _Arg2Nested; + typedef typename remove_reference<Arg3Nested>::type _Arg3Nested; + enum { Flags = _Arg1Nested::Flags & RowMajorBit }; +}; +} // end namespace internal + +template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3, + typename StorageKind> +class CwiseTernaryOpImpl; + +/** \class CwiseTernaryOp + * \ingroup Core_Module + * + * \brief Generic expression where a coefficient-wise ternary operator is + * applied to two expressions + * + * \tparam TernaryOp template functor implementing the operator + * \tparam Arg1Type the type of the first argument + * \tparam Arg2Type the type of the second argument + * \tparam Arg3Type the type of the third argument + * + * This class represents an expression where a coefficient-wise ternary + * operator is applied to three expressions. + * It is the return type of ternary operators, by which we mean only those + * ternary operators where + * all three arguments are Eigen expressions. + * For example, the return type of betainc(matrix1, matrix2, matrix3) is a + * CwiseTernaryOp. + * + * Most of the time, this is the only way that it is used, so you typically + * don't have to name + * CwiseTernaryOp types explicitly. + * + * \sa MatrixBase::ternaryExpr(const MatrixBase<Argument2> &, const + * MatrixBase<Argument3> &, const CustomTernaryOp &) const, class CwiseBinaryOp, + * class CwiseUnaryOp, class CwiseNullaryOp + */ +template <typename TernaryOp, typename Arg1Type, typename Arg2Type, + typename Arg3Type> +class CwiseTernaryOp : public CwiseTernaryOpImpl< + TernaryOp, Arg1Type, Arg2Type, Arg3Type, + typename internal::traits<Arg1Type>::StorageKind>, + internal::no_assignment_operator +{ + public: + typedef typename internal::remove_all<Arg1Type>::type Arg1; + typedef typename internal::remove_all<Arg2Type>::type Arg2; + typedef typename internal::remove_all<Arg3Type>::type Arg3; + + typedef typename CwiseTernaryOpImpl< + TernaryOp, Arg1Type, Arg2Type, Arg3Type, + typename internal::traits<Arg1Type>::StorageKind>::Base Base; + EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseTernaryOp) + + typedef typename internal::ref_selector<Arg1Type>::type Arg1Nested; + typedef typename internal::ref_selector<Arg2Type>::type Arg2Nested; + typedef typename internal::ref_selector<Arg3Type>::type Arg3Nested; + typedef typename internal::remove_reference<Arg1Nested>::type _Arg1Nested; + typedef typename internal::remove_reference<Arg2Nested>::type _Arg2Nested; + typedef typename internal::remove_reference<Arg3Nested>::type _Arg3Nested; + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE CwiseTernaryOp(const Arg1& a1, const Arg2& a2, + const Arg3& a3, + const TernaryOp& func = TernaryOp()) + : m_arg1(a1), m_arg2(a2), m_arg3(a3), m_functor(func) { + // require the sizes to match + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg2) + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg3) + + // The index types should match + EIGEN_STATIC_ASSERT((internal::is_same< + typename internal::traits<Arg1Type>::StorageKind, + typename internal::traits<Arg2Type>::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) + EIGEN_STATIC_ASSERT((internal::is_same< + typename internal::traits<Arg1Type>::StorageKind, + typename internal::traits<Arg3Type>::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) + + eigen_assert(a1.rows() == a2.rows() && a1.cols() == a2.cols() && + a1.rows() == a3.rows() && a1.cols() == a3.cols()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Index rows() const { + // return the fixed size type if available to enable compile time + // optimizations + if (internal::traits<typename internal::remove_all<Arg1Nested>::type>:: + RowsAtCompileTime == Dynamic && + internal::traits<typename internal::remove_all<Arg2Nested>::type>:: + RowsAtCompileTime == Dynamic) + return m_arg3.rows(); + else if (internal::traits<typename internal::remove_all<Arg1Nested>::type>:: + RowsAtCompileTime == Dynamic && + internal::traits<typename internal::remove_all<Arg3Nested>::type>:: + RowsAtCompileTime == Dynamic) + return m_arg2.rows(); + else + return m_arg1.rows(); + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Index cols() const { + // return the fixed size type if available to enable compile time + // optimizations + if (internal::traits<typename internal::remove_all<Arg1Nested>::type>:: + ColsAtCompileTime == Dynamic && + internal::traits<typename internal::remove_all<Arg2Nested>::type>:: + ColsAtCompileTime == Dynamic) + return m_arg3.cols(); + else if (internal::traits<typename internal::remove_all<Arg1Nested>::type>:: + ColsAtCompileTime == Dynamic && + internal::traits<typename internal::remove_all<Arg3Nested>::type>:: + ColsAtCompileTime == Dynamic) + return m_arg2.cols(); + else + return m_arg1.cols(); + } + + /** \returns the first argument nested expression */ + EIGEN_DEVICE_FUNC + const _Arg1Nested& arg1() const { return m_arg1; } + /** \returns the first argument nested expression */ + EIGEN_DEVICE_FUNC + const _Arg2Nested& arg2() const { return m_arg2; } + /** \returns the third argument nested expression */ + EIGEN_DEVICE_FUNC + const _Arg3Nested& arg3() const { return m_arg3; } + /** \returns the functor representing the ternary operation */ + EIGEN_DEVICE_FUNC + const TernaryOp& functor() const { return m_functor; } + + protected: + Arg1Nested m_arg1; + Arg2Nested m_arg2; + Arg3Nested m_arg3; + const TernaryOp m_functor; +}; + +// Generic API dispatcher +template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3, + typename StorageKind> +class CwiseTernaryOpImpl + : public internal::generic_xpr_base< + CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> >::type { + public: + typedef typename internal::generic_xpr_base< + CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> >::type Base; +}; + +} // end namespace Eigen + +#endif // EIGEN_CWISE_TERNARY_OP_H
diff --git a/Eigen/src/Core/CwiseUnaryView.h b/Eigen/src/Core/CwiseUnaryView.h index 2710330..21cf5ea 100644 --- a/Eigen/src/Core/CwiseUnaryView.h +++ b/Eigen/src/Core/CwiseUnaryView.h
@@ -81,7 +81,7 @@ /** \returns the nested expression */ typename internal::remove_reference<MatrixTypeNested>::type& - nestedExpression() { return m_matrix.const_cast_derived(); } + nestedExpression() { return m_matrix; } protected: MatrixTypeNested m_matrix;
diff --git a/Eigen/src/Core/DenseBase.h b/Eigen/src/Core/DenseBase.h index 5a38e5f..2289fe4 100644 --- a/Eigen/src/Core/DenseBase.h +++ b/Eigen/src/Core/DenseBase.h
@@ -34,17 +34,15 @@ * \tparam Derived is the derived type, e.g., a matrix type or an expression. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_DENSEBASE_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_DENSEBASE_PLUGIN. * * \sa \blank \ref TopicClassHierarchy */ template<typename Derived> class DenseBase #ifndef EIGEN_PARSED_BY_DOXYGEN - : public internal::special_scalar_op_base<Derived, typename internal::traits<Derived>::Scalar, - typename NumTraits<typename internal::traits<Derived>::Scalar>::Real, - DenseCoeffsBase<Derived> > + : public DenseCoeffsBase<Derived, internal::accessors_level<Derived>::value> #else - : public DenseCoeffsBase<Derived> + : public DenseCoeffsBase<Derived,DirectWriteAccessors> #endif // not EIGEN_PARSED_BY_DOXYGEN { public: @@ -73,10 +71,8 @@ typedef Scalar value_type; typedef typename NumTraits<Scalar>::Real RealScalar; - typedef internal::special_scalar_op_base<Derived,Scalar,RealScalar, DenseCoeffsBase<Derived> > Base; + typedef DenseCoeffsBase<Derived, internal::accessors_level<Derived>::value> Base; - using Base::operator*; - using Base::operator/; using Base::derived; using Base::const_cast_derived; using Base::rows; @@ -154,13 +150,18 @@ * \sa SizeAtCompileTime, MaxRowsAtCompileTime, MaxColsAtCompileTime */ - IsVectorAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime == 1 - || internal::traits<Derived>::MaxColsAtCompileTime == 1, + IsVectorAtCompileTime = internal::traits<Derived>::RowsAtCompileTime == 1 + || internal::traits<Derived>::ColsAtCompileTime == 1, /**< This is set to true if either the number of rows or the number of * columns is known at compile-time to be equal to 1. Indeed, in that case, * we are dealing with a column-vector (if there is only one column) or with * a row-vector (if there is only one row). */ + NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2, + /**< This value is equal to Tensor::NumDimensions, i.e. 0 for scalars, 1 for vectors, + * and 2 for matrices. + */ + Flags = internal::traits<Derived>::Flags, /**< This stores expression \ref flags flags which may or may not be inherited by new expressions * constructed from this one. See the \ref flags "list of flags". @@ -264,10 +265,10 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp<internal::scalar_constant_op<Scalar>,PlainObject> ConstantReturnType; - /** \internal Represents a vector with linearly spaced coefficients that allows sequential access only. */ - typedef CwiseNullaryOp<internal::linspaced_op<Scalar,PacketScalar,false>,PlainObject> SequentialLinSpacedReturnType; + /** \internal \deprecated Represents a vector with linearly spaced coefficients that allows sequential access only. */ + typedef CwiseNullaryOp<internal::linspaced_op<Scalar>,PlainObject> SequentialLinSpacedReturnType; /** \internal Represents a vector with linearly spaced coefficients that allows random access. */ - typedef CwiseNullaryOp<internal::linspaced_op<Scalar,PacketScalar,true>,PlainObject> RandomAccessLinSpacedReturnType; + typedef CwiseNullaryOp<internal::linspaced_op<Scalar>,PlainObject> RandomAccessLinSpacedReturnType; /** \internal the return type of MatrixBase::eigenvalues() */ typedef Matrix<typename NumTraits<typename internal::traits<Derived>::Scalar>::Real, internal::traits<Derived>::ColsAtCompileTime, 1> EigenvaluesReturnType; @@ -300,7 +301,7 @@ EIGEN_DEVICE_FUNC Derived& operator=(const ReturnByValue<OtherDerived>& func); - /** \ínternal + /** \internal * Copies \a other into *this without evaluating other. \returns a reference to *this. * \deprecated */ template<typename OtherDerived> @@ -399,7 +400,7 @@ * Notice that in the case of a plain matrix or vector (not an expression) this function just returns * a const reference, in order to avoid a useless copy. * - * \warning Be carefull with eval() and the auto C++ keyword, as detailed in this \link TopicPitfalls_auto_keyword page \endlink. + * \warning Be careful with eval() and the auto C++ keyword, as detailed in this \link TopicPitfalls_auto_keyword page \endlink. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvalReturnType eval() const @@ -414,7 +415,7 @@ * */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(const DenseBase<OtherDerived>& other) { EIGEN_STATIC_ASSERT(!OtherDerived::IsPlainObjectBase,THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY); @@ -426,7 +427,7 @@ * */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(PlainObjectBase<OtherDerived>& other) { eigen_assert(rows()==other.rows() && cols()==other.cols()); @@ -467,7 +468,17 @@ EIGEN_DEVICE_FUNC void visit(Visitor& func) const; - inline const WithFormat<Derived> format(const IOFormat& fmt) const; + /** \returns a WithFormat proxy object allowing to print a matrix the with given + * format \a fmt. + * + * See class IOFormat for some examples. + * + * \sa class IOFormat, class WithFormat + */ + inline const WithFormat<Derived> format(const IOFormat& fmt) const + { + return WithFormat<Derived>(derived(), fmt); + } /** \returns the unique coefficient of a 1x1 expression */ EIGEN_DEVICE_FUNC @@ -478,16 +489,16 @@ return derived().coeff(0,0); } - bool all() const; - bool any() const; - Index count() const; + EIGEN_DEVICE_FUNC bool all() const; + EIGEN_DEVICE_FUNC bool any() const; + EIGEN_DEVICE_FUNC Index count() const; typedef VectorwiseOp<Derived, Horizontal> RowwiseReturnType; typedef const VectorwiseOp<const Derived, Horizontal> ConstRowwiseReturnType; typedef VectorwiseOp<Derived, Vertical> ColwiseReturnType; typedef const VectorwiseOp<const Derived, Vertical> ConstColwiseReturnType; - /** \returns a VectorwiseOp wrapper of *this providing additional partial reduction operations + /** \returns a VectorwiseOp wrapper of *this for broadcasting and partial reductions * * Example: \include MatrixBase_rowwise.cpp * Output: \verbinclude MatrixBase_rowwise.out @@ -500,7 +511,7 @@ } EIGEN_DEVICE_FUNC RowwiseReturnType rowwise(); - /** \returns a VectorwiseOp wrapper of *this providing additional partial reduction operations + /** \returns a VectorwiseOp wrapper of *this broadcasting and partial reductions * * Example: \include MatrixBase_colwise.cpp * Output: \verbinclude MatrixBase_colwise.out @@ -561,13 +572,59 @@ } EIGEN_DEVICE_FUNC void reverseInPlace(); + #ifdef EIGEN_PARSED_BY_DOXYGEN + /** STL-like <a href="https://en.cppreference.com/w/cpp/named_req/RandomAccessIterator">RandomAccessIterator</a> + * iterator type as returned by the begin() and end() methods. + */ + typedef random_access_iterator_type iterator; + /** This is the const version of iterator (aka read-only) */ + typedef random_access_iterator_type const_iterator; + #else + typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, + internal::pointer_based_stl_iterator<Derived>, + internal::generic_randaccess_stl_iterator<Derived> + >::type iterator_type; + + typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, + internal::pointer_based_stl_iterator<const Derived>, + internal::generic_randaccess_stl_iterator<const Derived> + >::type const_iterator_type; + + // Stl-style iterators are supported only for vectors. + + typedef typename internal::conditional< IsVectorAtCompileTime, + iterator_type, + void + >::type iterator; + + typedef typename internal::conditional< IsVectorAtCompileTime, + const_iterator_type, + void + >::type const_iterator; + #endif + + inline iterator begin(); + inline const_iterator begin() const; + inline const_iterator cbegin() const; + inline iterator end(); + inline const_iterator end() const; + inline const_iterator cend() const; + #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::DenseBase +#define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +#define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) +#define EIGEN_DOC_UNARY_ADDONS(X,Y) +# include "../plugins/CommonCwiseUnaryOps.h" # include "../plugins/BlockMethods.h" +# include "../plugins/IndexedViewMethods.h" +# include "../plugins/ReshapedMethods.h" # ifdef EIGEN_DENSEBASE_PLUGIN # include EIGEN_DENSEBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS - +#undef EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +#undef EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF +#undef EIGEN_DOC_UNARY_ADDONS // disable the use of evalTo for dense objects with a nice compilation error template<typename Dest>
diff --git a/Eigen/src/Core/DenseCoeffsBase.h b/Eigen/src/Core/DenseCoeffsBase.h index 423ab16..c4af48a 100644 --- a/Eigen/src/Core/DenseCoeffsBase.h +++ b/Eigen/src/Core/DenseCoeffsBase.h
@@ -624,7 +624,7 @@ { static inline Index run(const Derived& m) { - return internal::first_aligned<Alignment>(&m.const_cast_derived().coeffRef(0,0), m.size()); + return internal::first_aligned<Alignment>(m.data(), m.size()); } };
diff --git a/Eigen/src/Core/DenseStorage.h b/Eigen/src/Core/DenseStorage.h index 3404846..a8bb8a6 100644 --- a/Eigen/src/Core/DenseStorage.h +++ b/Eigen/src/Core/DenseStorage.h
@@ -13,9 +13,9 @@ #define EIGEN_MATRIXSTORAGE_H #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN - #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN EIGEN_DENSE_STORAGE_CTOR_PLUGIN; + #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) X; EIGEN_DENSE_STORAGE_CTOR_PLUGIN; #else - #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) #endif namespace Eigen { @@ -61,19 +61,19 @@ #if defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT) #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) #elif EIGEN_GNUC_AT_LEAST(4,7) - // GCC 4.7 is too aggressive in its optimizations and remove the alignement test based on the fact the array is declared to be aligned. + // GCC 4.7 is too aggressive in its optimizations and remove the alignment test based on the fact the array is declared to be aligned. // See this bug report: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=53900 // Hiding the origin of the array pointer behind a function argument seems to do the trick even if the function is inlined: template<typename PtrType> EIGEN_ALWAYS_INLINE PtrType eigen_unaligned_array_assert_workaround_gcc47(PtrType array) { return array; } #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ - eigen_assert((reinterpret_cast<size_t>(eigen_unaligned_array_assert_workaround_gcc47(array)) & (sizemask)) == 0 \ + eigen_assert((internal::UIntPtr(eigen_unaligned_array_assert_workaround_gcc47(array)) & (sizemask)) == 0 \ && "this assertion is explained here: " \ "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ " **** READ THIS WEB PAGE !!! ****"); #else #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ - eigen_assert((reinterpret_cast<size_t>(array) & (sizemask)) == 0 \ + eigen_assert((internal::UIntPtr(array) & (sizemask)) == 0 \ && "this assertion is explained here: " \ "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ " **** READ THIS WEB PAGE !!! ****"); @@ -184,12 +184,16 @@ { internal::plain_array<T,Size,_Options> m_data; public: - EIGEN_DEVICE_FUNC DenseStorage() {} + EIGEN_DEVICE_FUNC DenseStorage() { + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) + } EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()) {} EIGEN_DEVICE_FUNC - DenseStorage(const DenseStorage& other) : m_data(other.m_data) {} + DenseStorage(const DenseStorage& other) : m_data(other.m_data) { + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) + } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { @@ -197,13 +201,15 @@ return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows==_Rows && cols==_Cols); EIGEN_UNUSED_VARIABLE(size); EIGEN_UNUSED_VARIABLE(rows); EIGEN_UNUSED_VARIABLE(cols); } - EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { std::swap(m_data,other.m_data); } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_data, other.m_data); + } EIGEN_DEVICE_FUNC static Index rows(void) {return _Rows;} EIGEN_DEVICE_FUNC static Index cols(void) {return _Cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index,Index,Index) {} @@ -263,7 +269,11 @@ } EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index cols) : m_rows(rows), m_cols(cols) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) - { std::swap(m_data,other.m_data); std::swap(m_rows,other.m_rows); std::swap(m_cols,other.m_cols); } + { + numext::swap(m_data,other.m_data); + numext::swap(m_rows,other.m_rows); + numext::swap(m_cols,other.m_cols); + } EIGEN_DEVICE_FUNC Index rows() const {return m_rows;} EIGEN_DEVICE_FUNC Index cols() const {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index cols) { m_rows = rows; m_cols = cols; } @@ -292,7 +302,11 @@ return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index) : m_rows(rows) {} - EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { std::swap(m_data,other.m_data); std::swap(m_rows,other.m_rows); } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) + { + numext::swap(m_data,other.m_data); + numext::swap(m_rows,other.m_rows); + } EIGEN_DEVICE_FUNC Index rows(void) const {return m_rows;} EIGEN_DEVICE_FUNC Index cols(void) const {return _Cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index) { m_rows = rows; } @@ -321,11 +335,14 @@ return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index, Index cols) : m_cols(cols) {} - EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { std::swap(m_data,other.m_data); std::swap(m_cols,other.m_cols); } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_data,other.m_data); + numext::swap(m_cols,other.m_cols); + } EIGEN_DEVICE_FUNC Index rows(void) const {return _Rows;} EIGEN_DEVICE_FUNC Index cols(void) const {return m_cols;} - void conservativeResize(Index, Index, Index cols) { m_cols = cols; } - void resize(Index, Index, Index cols) { m_cols = cols; } + EIGEN_DEVICE_FUNC void conservativeResize(Index, Index, Index cols) { m_cols = cols; } + EIGEN_DEVICE_FUNC void resize(Index, Index, Index cols) { m_cols = cols; } EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } EIGEN_DEVICE_FUNC T *data() { return m_data.array; } }; @@ -343,7 +360,7 @@ EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size)), m_rows(rows), m_cols(cols) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows>=0 && cols >=0); } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) @@ -351,6 +368,7 @@ , m_rows(other.m_rows) , m_cols(other.m_cols) { + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*m_cols) internal::smart_copy(other.m_data, other.m_data+other.m_rows*other.m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) @@ -362,9 +380,9 @@ } return *this; } -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC - DenseStorage(DenseStorage&& other) + DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_rows(std::move(other.m_rows)) , m_cols(std::move(other.m_cols)) @@ -374,18 +392,21 @@ other.m_cols = 0; } EIGEN_DEVICE_FUNC - DenseStorage& operator=(DenseStorage&& other) + DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { - using std::swap; - swap(m_data, other.m_data); - swap(m_rows, other.m_rows); - swap(m_cols, other.m_cols); + numext::swap(m_data, other.m_data); + numext::swap(m_rows, other.m_rows); + numext::swap(m_cols, other.m_cols); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, m_rows*m_cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) - { std::swap(m_data,other.m_data); std::swap(m_rows,other.m_rows); std::swap(m_cols,other.m_cols); } + { + numext::swap(m_data,other.m_data); + numext::swap(m_rows,other.m_rows); + numext::swap(m_cols,other.m_cols); + } EIGEN_DEVICE_FUNC Index rows(void) const {return m_rows;} EIGEN_DEVICE_FUNC Index cols(void) const {return m_cols;} void conservativeResize(Index size, Index rows, Index cols) @@ -399,11 +420,11 @@ if(size != m_rows*m_cols) { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, m_rows*m_cols); - if (size) + if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size); else m_data = 0; - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_rows = rows; m_cols = cols; @@ -422,7 +443,7 @@ explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_cols(0) {} EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size)), m_cols(cols) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows==_Rows && cols >=0); EIGEN_UNUSED_VARIABLE(rows); } @@ -430,6 +451,7 @@ : m_data(internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(_Rows*other.m_cols)) , m_cols(other.m_cols) { + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_cols*_Rows) internal::smart_copy(other.m_data, other.m_data+_Rows*m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) @@ -441,9 +463,9 @@ } return *this; } -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC - DenseStorage(DenseStorage&& other) + DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_cols(std::move(other.m_cols)) { @@ -451,16 +473,18 @@ other.m_cols = 0; } EIGEN_DEVICE_FUNC - DenseStorage& operator=(DenseStorage&& other) + DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { - using std::swap; - swap(m_data, other.m_data); - swap(m_cols, other.m_cols); + numext::swap(m_data, other.m_data); + numext::swap(m_cols, other.m_cols); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, _Rows*m_cols); } - EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { std::swap(m_data,other.m_data); std::swap(m_cols,other.m_cols); } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_data,other.m_data); + numext::swap(m_cols,other.m_cols); + } EIGEN_DEVICE_FUNC static Index rows(void) {return _Rows;} EIGEN_DEVICE_FUNC Index cols(void) const {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index size, Index, Index cols) @@ -473,11 +497,11 @@ if(size != _Rows*m_cols) { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, _Rows*m_cols); - if (size) + if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size); else m_data = 0; - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_cols = cols; } @@ -495,7 +519,7 @@ explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0) {} EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size)), m_rows(rows) { - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows>=0 && cols == _Cols); EIGEN_UNUSED_VARIABLE(cols); } @@ -503,6 +527,7 @@ : m_data(internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(other.m_rows*_Cols)) , m_rows(other.m_rows) { + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*_Cols) internal::smart_copy(other.m_data, other.m_data+other.m_rows*_Cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) @@ -514,9 +539,9 @@ } return *this; } -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC - DenseStorage(DenseStorage&& other) + DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_rows(std::move(other.m_rows)) { @@ -524,16 +549,18 @@ other.m_rows = 0; } EIGEN_DEVICE_FUNC - DenseStorage& operator=(DenseStorage&& other) + DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { - using std::swap; - swap(m_data, other.m_data); - swap(m_rows, other.m_rows); + numext::swap(m_data, other.m_data); + numext::swap(m_rows, other.m_rows); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, _Cols*m_rows); } - EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { std::swap(m_data,other.m_data); std::swap(m_rows,other.m_rows); } + EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { + numext::swap(m_data,other.m_data); + numext::swap(m_rows,other.m_rows); + } EIGEN_DEVICE_FUNC Index rows(void) const {return m_rows;} EIGEN_DEVICE_FUNC static Index cols(void) {return _Cols;} void conservativeResize(Index size, Index rows, Index) @@ -546,11 +573,11 @@ if(size != m_rows*_Cols) { internal::conditional_aligned_delete_auto<T,(_Options&DontAlign)==0>(m_data, _Cols*m_rows); - if (size) + if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto<T,(_Options&DontAlign)==0>(size); else m_data = 0; - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_rows = rows; }
diff --git a/Eigen/src/Core/Diagonal.h b/Eigen/src/Core/Diagonal.h index bfea058..563135f 100644 --- a/Eigen/src/Core/Diagonal.h +++ b/Eigen/src/Core/Diagonal.h
@@ -21,7 +21,7 @@ * \param MatrixType the type of the object in which we are taking a sub/main/super diagonal * \param DiagIndex the index of the sub/super diagonal. The default is 0 and it means the main diagonal. * A positive value means a superdiagonal, a negative value means a subdiagonal. - * You can also use Dynamic so the index can be set at runtime. + * You can also use DynamicIndex so the index can be set at runtime. * * The matrix is not required to be square. * @@ -70,7 +70,10 @@ EIGEN_DENSE_PUBLIC_INTERFACE(Diagonal) EIGEN_DEVICE_FUNC - explicit inline Diagonal(MatrixType& matrix, Index a_index = DiagIndex) : m_matrix(matrix), m_index(a_index) {} + explicit inline Diagonal(MatrixType& matrix, Index a_index = DiagIndex) : m_matrix(matrix), m_index(a_index) + { + eigen_assert( a_index <= m_matrix.cols() && -a_index <= m_matrix.rows() ); + } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Diagonal) @@ -184,7 +187,7 @@ * * \sa class Diagonal */ template<typename Derived> -inline typename MatrixBase<Derived>::DiagonalReturnType +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::DiagonalReturnType MatrixBase<Derived>::diagonal() { return DiagonalReturnType(derived()); @@ -192,7 +195,7 @@ /** This is the const version of diagonal(). */ template<typename Derived> -inline typename MatrixBase<Derived>::ConstDiagonalReturnType +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::ConstDiagonalReturnType MatrixBase<Derived>::diagonal() const { return ConstDiagonalReturnType(derived()); @@ -210,7 +213,7 @@ * * \sa MatrixBase::diagonal(), class Diagonal */ template<typename Derived> -inline typename MatrixBase<Derived>::DiagonalDynamicIndexReturnType +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::DiagonalDynamicIndexReturnType MatrixBase<Derived>::diagonal(Index index) { return DiagonalDynamicIndexReturnType(derived(), index); @@ -218,7 +221,7 @@ /** This is the const version of diagonal(Index). */ template<typename Derived> -inline typename MatrixBase<Derived>::ConstDiagonalDynamicIndexReturnType +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::ConstDiagonalDynamicIndexReturnType MatrixBase<Derived>::diagonal(Index index) const { return ConstDiagonalDynamicIndexReturnType(derived(), index); @@ -237,6 +240,7 @@ * \sa MatrixBase::diagonal(), class Diagonal */ template<typename Derived> template<int Index_> +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::template DiagonalIndexReturnType<Index_>::Type MatrixBase<Derived>::diagonal() { @@ -246,6 +250,7 @@ /** This is the const version of diagonal<int>(). */ template<typename Derived> template<int Index_> +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::template ConstDiagonalIndexReturnType<Index_>::Type MatrixBase<Derived>::diagonal() const {
diff --git a/Eigen/src/Core/DiagonalMatrix.h b/Eigen/src/Core/DiagonalMatrix.h index 5a9e3ab..afab2f1 100644 --- a/Eigen/src/Core/DiagonalMatrix.h +++ b/Eigen/src/Core/DiagonalMatrix.h
@@ -44,7 +44,7 @@ EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); } - + EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return derived().diagonal(); } EIGEN_DEVICE_FUNC @@ -71,18 +71,41 @@ return InverseReturnType(diagonal().cwiseInverse()); } - typedef DiagonalWrapper<const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DiagonalVectorType> > ScalarMultipleReturnType; EIGEN_DEVICE_FUNC - inline const ScalarMultipleReturnType + inline const DiagonalWrapper<const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DiagonalVectorType,Scalar,product) > operator*(const Scalar& scalar) const { - return ScalarMultipleReturnType(diagonal() * scalar); + return DiagonalWrapper<const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DiagonalVectorType,Scalar,product) >(diagonal() * scalar); } EIGEN_DEVICE_FUNC - friend inline const ScalarMultipleReturnType + friend inline const DiagonalWrapper<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar,DiagonalVectorType,product) > operator*(const Scalar& scalar, const DiagonalBase& other) { - return ScalarMultipleReturnType(other.diagonal() * scalar); + return DiagonalWrapper<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar,DiagonalVectorType,product) >(scalar * other.diagonal()); + } + + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + #ifdef EIGEN_PARSED_BY_DOXYGEN + inline unspecified_expression_type + #else + inline const DiagonalWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(DiagonalVectorType,typename OtherDerived::DiagonalVectorType,sum) > + #endif + operator+(const DiagonalBase<OtherDerived>& other) const + { + return (diagonal() + other.diagonal()).asDiagonal(); + } + + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + #ifdef EIGEN_PARSED_BY_DOXYGEN + inline unspecified_expression_type + #else + inline const DiagonalWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(DiagonalVectorType,typename OtherDerived::DiagonalVectorType,difference) > + #endif + operator-(const DiagonalBase<OtherDerived>& other) const + { + return (diagonal() - other.diagonal()).asDiagonal(); } }; @@ -274,7 +297,7 @@ * \sa class DiagonalWrapper, class DiagonalMatrix, diagonal(), isDiagonal() **/ template<typename Derived> -inline const DiagonalWrapper<const Derived> +EIGEN_DEVICE_FUNC inline const DiagonalWrapper<const Derived> MatrixBase<Derived>::asDiagonal() const { return DiagonalWrapper<const Derived>(derived()); @@ -291,12 +314,11 @@ template<typename Derived> bool MatrixBase<Derived>::isDiagonal(const RealScalar& prec) const { - using std::abs; if(cols() != rows()) return false; RealScalar maxAbsOnDiagonal = static_cast<RealScalar>(-1); for(Index j = 0; j < cols(); ++j) { - RealScalar absOnDiagonal = abs(coeff(j,j)); + RealScalar absOnDiagonal = numext::abs(coeff(j,j)); if(absOnDiagonal > maxAbsOnDiagonal) maxAbsOnDiagonal = absOnDiagonal; } for(Index j = 0; j < cols(); ++j) @@ -317,19 +339,24 @@ template<> struct AssignmentKind<DenseShape,DiagonalShape> { typedef Diagonal2Dense Kind; }; // Diagonal matrix to Dense assignment -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Dense, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Dense> { - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + dst.setZero(); dst.diagonal() = src.diagonal(); } - static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) { dst.diagonal() += src.diagonal(); } - static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) { dst.diagonal() -= src.diagonal(); } };
diff --git a/Eigen/src/Core/DiagonalProduct.h b/Eigen/src/Core/DiagonalProduct.h index d372b93..7911d1c 100644 --- a/Eigen/src/Core/DiagonalProduct.h +++ b/Eigen/src/Core/DiagonalProduct.h
@@ -17,7 +17,7 @@ */ template<typename Derived> template<typename DiagonalDerived> -inline const Product<Derived, DiagonalDerived, LazyProduct> +EIGEN_DEVICE_FUNC inline const Product<Derived, DiagonalDerived, LazyProduct> MatrixBase<Derived>::operator*(const DiagonalBase<DiagonalDerived> &a_diagonal) const { return Product<Derived, DiagonalDerived, LazyProduct>(derived(),a_diagonal.derived());
diff --git a/Eigen/src/Core/Dot.h b/Eigen/src/Core/Dot.h index 82d58fc..11da432 100644 --- a/Eigen/src/Core/Dot.h +++ b/Eigen/src/Core/Dot.h
@@ -28,28 +28,33 @@ > struct dot_nocheck { - typedef typename scalar_product_traits<typename traits<T>::Scalar,typename traits<U>::Scalar>::ReturnType ResScalar; + typedef scalar_conj_product_op<typename traits<T>::Scalar,typename traits<U>::Scalar> conj_prod; + typedef typename conj_prod::result_type ResScalar; EIGEN_DEVICE_FUNC - static inline ResScalar run(const MatrixBase<T>& a, const MatrixBase<U>& b) + EIGEN_STRONG_INLINE + static ResScalar run(const MatrixBase<T>& a, const MatrixBase<U>& b) { - return a.template binaryExpr<scalar_conj_product_op<typename traits<T>::Scalar,typename traits<U>::Scalar> >(b).sum(); + return a.template binaryExpr<conj_prod>(b).sum(); } }; template<typename T, typename U> struct dot_nocheck<T, U, true> { - typedef typename scalar_product_traits<typename traits<T>::Scalar,typename traits<U>::Scalar>::ReturnType ResScalar; + typedef scalar_conj_product_op<typename traits<T>::Scalar,typename traits<U>::Scalar> conj_prod; + typedef typename conj_prod::result_type ResScalar; EIGEN_DEVICE_FUNC - static inline ResScalar run(const MatrixBase<T>& a, const MatrixBase<U>& b) + EIGEN_STRONG_INLINE + static ResScalar run(const MatrixBase<T>& a, const MatrixBase<U>& b) { - return a.transpose().template binaryExpr<scalar_conj_product_op<typename traits<T>::Scalar,typename traits<U>::Scalar> >(b).sum(); + return a.transpose().template binaryExpr<conj_prod>(b).sum(); } }; } // end namespace internal -/** \returns the dot product of *this with other. +/** \fn MatrixBase::dot + * \returns the dot product of *this with other. * * \only_for_vectors * @@ -62,15 +67,18 @@ template<typename Derived> template<typename OtherDerived> EIGEN_DEVICE_FUNC -typename internal::scalar_product_traits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType +EIGEN_STRONG_INLINE +typename ScalarBinaryOpTraits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType MatrixBase<Derived>::dot(const MatrixBase<OtherDerived>& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Derived,OtherDerived) +#if !(defined(EIGEN_NO_STATIC_ASSERT) && defined(EIGEN_NO_DEBUG)) typedef internal::scalar_conj_product_op<Scalar,typename OtherDerived::Scalar> func; EIGEN_CHECK_BINARY_COMPATIBILIY(func,Scalar,typename OtherDerived::Scalar); - +#endif + eigen_assert(size() == other.size()); return internal::dot_nocheck<Derived,OtherDerived>::run(*this, other); @@ -85,7 +93,7 @@ * \sa dot(), norm(), lpNorm() */ template<typename Derived> -EIGEN_STRONG_INLINE typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::squaredNorm() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::squaredNorm() const { return numext::real((*this).cwiseAbs2().sum()); } @@ -97,7 +105,7 @@ * \sa lpNorm(), dot(), squaredNorm() */ template<typename Derived> -inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::norm() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::norm() const { return numext::sqrt(squaredNorm()); } @@ -112,7 +120,7 @@ * \sa norm(), normalize() */ template<typename Derived> -inline const typename MatrixBase<Derived>::PlainObject +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::normalized() const { typedef typename internal::nested_eval<Derived,2>::type _Nested; @@ -134,7 +142,7 @@ * \sa norm(), normalized() */ template<typename Derived> -inline void MatrixBase<Derived>::normalize() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase<Derived>::normalize() { RealScalar z = squaredNorm(); // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU @@ -155,7 +163,7 @@ * \sa stableNorm(), stableNormalize(), normalized() */ template<typename Derived> -inline const typename MatrixBase<Derived>::PlainObject +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::stableNormalized() const { typedef typename internal::nested_eval<Derived,3>::type _Nested; @@ -180,7 +188,7 @@ * \sa stableNorm(), stableNormalized(), normalize() */ template<typename Derived> -inline void MatrixBase<Derived>::stableNormalize() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase<Derived>::stableNormalize() { RealScalar w = cwiseAbs().maxCoeff(); RealScalar z = (derived()/w).squaredNorm(); @@ -227,9 +235,12 @@ template<typename Derived> struct lpNorm_selector<Derived, Infinity> { + typedef typename NumTraits<typename traits<Derived>::Scalar>::Real RealScalar; EIGEN_DEVICE_FUNC - static inline typename NumTraits<typename traits<Derived>::Scalar>::Real run(const MatrixBase<Derived>& m) + static inline RealScalar run(const MatrixBase<Derived>& m) { + if(Derived::SizeAtCompileTime==0 || (Derived::SizeAtCompileTime==Dynamic && m.size()==0)) + return RealScalar(0); return m.cwiseAbs().maxCoeff(); } }; @@ -240,6 +251,8 @@ * of the coefficients of \c *this. If \a p is the special value \a Eigen::Infinity, this function returns the \f$ \ell^\infty \f$ * norm, that is the maximum of the absolute values of the coefficients of \c *this. * + * In all cases, if \c *this is empty, then the value 0 is returned. + * * \note For matrices, this function does not compute the <a href="https://en.wikipedia.org/wiki/Operator_norm">operator-norm</a>. That is, if \c *this is a matrix, then its coefficients are interpreted as a 1D vector. Nonetheless, you can easily compute the 1-norm and \f$\infty\f$-norm matrix operator norms using \link TutorialReductionsVisitorsBroadcastingReductionsNorm partial reductions \endlink. * * \sa norm() @@ -247,9 +260,9 @@ template<typename Derived> template<int p> #ifndef EIGEN_PARSED_BY_DOXYGEN -inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real +EIGEN_DEVICE_FUNC inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real #else -MatrixBase<Derived>::RealScalar +EIGEN_DEVICE_FUNC MatrixBase<Derived>::RealScalar #endif MatrixBase<Derived>::lpNorm() const {
diff --git a/Eigen/src/Core/EigenBase.h b/Eigen/src/Core/EigenBase.h index ba8e096..b195506 100644 --- a/Eigen/src/Core/EigenBase.h +++ b/Eigen/src/Core/EigenBase.h
@@ -14,6 +14,7 @@ namespace Eigen { /** \class EigenBase + * \ingroup Core_Module * * Common base class for all classes T such that MatrixBase has an operator=(T) and a constructor MatrixBase(T). * @@ -128,6 +129,7 @@ */ template<typename Derived> template<typename OtherDerived> +EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator=(const EigenBase<OtherDerived> &other) { call_assignment(derived(), other.derived()); @@ -136,17 +138,19 @@ template<typename Derived> template<typename OtherDerived> +EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator+=(const EigenBase<OtherDerived> &other) { - call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } template<typename Derived> template<typename OtherDerived> +EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator-=(const EigenBase<OtherDerived> &other) { - call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); }
diff --git a/Eigen/src/Core/Fuzzy.h b/Eigen/src/Core/Fuzzy.h index 3e403a0..43aa49b 100644 --- a/Eigen/src/Core/Fuzzy.h +++ b/Eigen/src/Core/Fuzzy.h
@@ -100,7 +100,7 @@ */ template<typename Derived> template<typename OtherDerived> -bool DenseBase<Derived>::isApprox( +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isApprox( const DenseBase<OtherDerived>& other, const RealScalar& prec ) const @@ -122,7 +122,7 @@ * \sa isApprox(), isMuchSmallerThan(const DenseBase<OtherDerived>&, RealScalar) const */ template<typename Derived> -bool DenseBase<Derived>::isMuchSmallerThan( +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isMuchSmallerThan( const typename NumTraits<Scalar>::Real& other, const RealScalar& prec ) const @@ -142,7 +142,7 @@ */ template<typename Derived> template<typename OtherDerived> -bool DenseBase<Derived>::isMuchSmallerThan( +EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isMuchSmallerThan( const DenseBase<OtherDerived>& other, const RealScalar& prec ) const
diff --git a/Eigen/src/Core/GeneralProduct.h b/Eigen/src/Core/GeneralProduct.h index f7c5f42..bf7ef54 100644 --- a/Eigen/src/Core/GeneralProduct.h +++ b/Eigen/src/Core/GeneralProduct.h
@@ -18,17 +18,33 @@ Small = 3 }; +// Define the threshold value to fallback from the generic matrix-matrix product +// implementation (heavy) to the lightweight coeff-based product one. +// See generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> +// in products/GeneralMatrixMatrix.h for more details. +// TODO This threshold should also be used in the compile-time selector below. +#ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD +// This default value has been obtained on a Haswell architecture. +#define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 20 +#endif + namespace internal { template<int Rows, int Cols, int Depth> struct product_type_selector; template<int Size, int MaxSize> struct product_size_category { - enum { is_large = MaxSize == Dynamic || - Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD, - value = is_large ? Large - : Size == 1 ? 1 - : Small + enum { + #ifndef EIGEN_GPU_COMPILE_PHASE + is_large = MaxSize == Dynamic || + Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD || + (Size==Dynamic && MaxSize>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD), + #else + is_large = 0, + #endif + value = is_large ? Large + : Size == 1 ? 1 + : Small }; }; @@ -147,32 +163,32 @@ template<typename Scalar,int Size,int MaxSize> struct gemv_static_vector_if<Scalar,Size,MaxSize,false> { - EIGEN_STRONG_INLINE Scalar* data() { eigen_internal_assert(false && "should never be called"); return 0; } + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Scalar* data() { eigen_internal_assert(false && "should never be called"); return 0; } }; template<typename Scalar,int Size> struct gemv_static_vector_if<Scalar,Size,Dynamic,true> { - EIGEN_STRONG_INLINE Scalar* data() { return 0; } + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Scalar* data() { return 0; } }; template<typename Scalar,int Size,int MaxSize> struct gemv_static_vector_if<Scalar,Size,MaxSize,true> { - #if EIGEN_MAX_STATIC_ALIGN_BYTES!=0 - internal::plain_array<Scalar,EIGEN_SIZE_MIN_PREFER_FIXED(Size,MaxSize),0> m_data; - EIGEN_STRONG_INLINE Scalar* data() { return m_data.array; } - #else - // Some architectures cannot align on the stack, - // => let's manually enforce alignment by allocating more data and return the address of the first aligned element. enum { ForceAlignment = internal::packet_traits<Scalar>::Vectorizable, PacketSize = internal::packet_traits<Scalar>::size }; - internal::plain_array<Scalar,EIGEN_SIZE_MIN_PREFER_FIXED(Size,MaxSize)+(ForceAlignment?PacketSize:0),0> m_data; + #if EIGEN_MAX_STATIC_ALIGN_BYTES!=0 + internal::plain_array<Scalar,EIGEN_SIZE_MIN_PREFER_FIXED(Size,MaxSize),0,EIGEN_PLAIN_ENUM_MIN(AlignedMax,PacketSize)> m_data; + EIGEN_STRONG_INLINE Scalar* data() { return m_data.array; } + #else + // Some architectures cannot align on the stack, + // => let's manually enforce alignment by allocating more data and return the address of the first aligned element. + internal::plain_array<Scalar,EIGEN_SIZE_MIN_PREFER_FIXED(Size,MaxSize)+(ForceAlignment?EIGEN_MAX_ALIGN_BYTES:0),0> m_data; EIGEN_STRONG_INLINE Scalar* data() { return ForceAlignment - ? reinterpret_cast<Scalar*>((reinterpret_cast<size_t>(m_data.array) & ~(size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES) + ? reinterpret_cast<Scalar*>((internal::UIntPtr(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES) : m_data.array; } #endif @@ -207,7 +223,7 @@ typedef internal::blas_traits<Rhs> RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; + typedef Map<Matrix<ResScalar,Dynamic,1>, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits<ResScalar>::size)> MappedDest; ActualLhsType actualLhs = LhsBlasTraits::extract(lhs); ActualRhsType actualRhs = RhsBlasTraits::extract(rhs); @@ -223,50 +239,65 @@ // on, the other hand it is good for the cache to pack the vector anyways... EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime==1), ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex), - MightCannotUseDest = (ActualDest::InnerStrideAtCompileTime!=1) || ComplexByReal + MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime!=0) }; - gemv_static_vector_if<ResScalar,ActualDest::SizeAtCompileTime,ActualDest::MaxSizeAtCompileTime,MightCannotUseDest> static_dest; - - const bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); - const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; - - RhsScalar compatibleAlpha = get_factor<ResScalar,RhsScalar>::run(actualAlpha); - - ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), - evalToDest ? dest.data() : static_dest.data()); - - if(!evalToDest) - { - #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN - Index size = dest.size(); - EIGEN_DENSE_STORAGE_CTOR_PLUGIN - #endif - if(!alphaIsCompatible) - { - MappedDest(actualDestPtr, dest.size()).setZero(); - compatibleAlpha = RhsScalar(1); - } - else - MappedDest(actualDestPtr, dest.size()) = dest; - } - typedef const_blas_data_mapper<LhsScalar,Index,ColMajor> LhsMapper; typedef const_blas_data_mapper<RhsScalar,Index,RowMajor> RhsMapper; - general_matrix_vector_product - <Index,LhsScalar,LhsMapper,ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run( - actualLhs.rows(), actualLhs.cols(), - LhsMapper(actualLhs.data(), actualLhs.outerStride()), - RhsMapper(actualRhs.data(), actualRhs.innerStride()), - actualDestPtr, 1, - compatibleAlpha); + RhsScalar compatibleAlpha = get_factor<ResScalar,RhsScalar>::run(actualAlpha); - if (!evalToDest) + if(!MightCannotUseDest) { - if(!alphaIsCompatible) - dest += actualAlpha * MappedDest(actualDestPtr, dest.size()); - else - dest = MappedDest(actualDestPtr, dest.size()); + // shortcut if we are sure to be able to use dest directly, + // this ease the compiler to generate cleaner and more optimzized code for most common cases + general_matrix_vector_product + <Index,LhsScalar,LhsMapper,ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run( + actualLhs.rows(), actualLhs.cols(), + LhsMapper(actualLhs.data(), actualLhs.outerStride()), + RhsMapper(actualRhs.data(), actualRhs.innerStride()), + dest.data(), 1, + compatibleAlpha); + } + else + { + gemv_static_vector_if<ResScalar,ActualDest::SizeAtCompileTime,ActualDest::MaxSizeAtCompileTime,MightCannotUseDest> static_dest; + + const bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); + const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; + + ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), + evalToDest ? dest.data() : static_dest.data()); + + if(!evalToDest) + { + #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN + Index size = dest.size(); + EIGEN_DENSE_STORAGE_CTOR_PLUGIN + #endif + if(!alphaIsCompatible) + { + MappedDest(actualDestPtr, dest.size()).setZero(); + compatibleAlpha = RhsScalar(1); + } + else + MappedDest(actualDestPtr, dest.size()) = dest; + } + + general_matrix_vector_product + <Index,LhsScalar,LhsMapper,ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run( + actualLhs.rows(), actualLhs.cols(), + LhsMapper(actualLhs.data(), actualLhs.outerStride()), + RhsMapper(actualRhs.data(), actualRhs.innerStride()), + actualDestPtr, 1, + compatibleAlpha); + + if (!evalToDest) + { + if(!alphaIsCompatible) + dest.matrix() += actualAlpha * MappedDest(actualDestPtr, dest.size()); + else + dest = MappedDest(actualDestPtr, dest.size()); + } } } }; @@ -295,7 +326,7 @@ enum { // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1 // on, the other hand it is good for the cache to pack the vector anyways... - DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 + DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime==0 }; gemv_static_vector_if<RhsScalar,ActualRhsTypeCleaned::SizeAtCompileTime,ActualRhsTypeCleaned::MaxSizeAtCompileTime,!DirectlyUseRhs> static_rhs; @@ -329,6 +360,7 @@ template<typename Lhs, typename Rhs, typename Dest> static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { + EIGEN_STATIC_ASSERT((!nested_eval<Lhs,1>::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE); // TODO if rhs is large enough it might be beneficial to make sure that dest is sequentially stored in memory, otherwise use a temp typename nested_eval<Rhs,1>::type actual_rhs(rhs); const Index size = rhs.rows(); @@ -342,6 +374,7 @@ template<typename Lhs, typename Rhs, typename Dest> static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { + EIGEN_STATIC_ASSERT((!nested_eval<Lhs,1>::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE); typename nested_eval<Rhs,Lhs::RowsAtCompileTime>::type actual_rhs(rhs); const Index rows = dest.rows(); for(Index i=0; i<rows; ++i) @@ -361,11 +394,10 @@ * * \sa lazyProduct(), operator*=(const MatrixBase&), Cwise::operator*() */ -#ifndef __CUDACC__ - template<typename Derived> template<typename OtherDerived> -inline const Product<Derived, OtherDerived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const Product<Derived, OtherDerived> MatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const { // A note regarding the function declaration: In MSVC, this function will sometimes @@ -394,8 +426,6 @@ return Product<Derived, OtherDerived>(derived(), other.derived()); } -#endif // __CUDACC__ - /** \returns an expression of the matrix product of \c *this and \a other without implicit evaluation. * * The returned product will behave like any other expressions: the coefficients of the product will be @@ -409,6 +439,7 @@ */ template<typename Derived> template<typename OtherDerived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product<Derived,OtherDerived,LazyProduct> MatrixBase<Derived>::lazyProduct(const MatrixBase<OtherDerived> &other) const {
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index 001c2ff..04a321b 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -56,12 +56,15 @@ HasConj = 1, HasSetLinear = 1, HasBlend = 0, + HasReduxp = 1, HasDiv = 0, HasSqrt = 0, HasRsqrt = 0, HasExp = 0, + HasExpm1 = 0, HasLog = 0, + HasLog1p = 0, HasLog10 = 0, HasPow = 0, @@ -80,8 +83,13 @@ HasPolygamma = 0, HasErf = 0, HasErfc = 0, + HasI0e = 0, + HasI1e = 0, HasIGamma = 0, + HasIGammaDerA = 0, + HasGammaSampleDerAlpha = 0, HasIGammac = 0, + HasBetaInc = 0, HasRound = 0, HasFloor = 0, @@ -144,15 +152,18 @@ return static_cast<TgtPacket>(a); } +/** \internal \returns reinterpret_cast<Target>(a) */ +template <typename Target, typename Packet> +EIGEN_DEVICE_FUNC inline Target +preinterpret(const Packet& a); /* { return reinterpret_cast<const Target&>(a); } */ + /** \internal \returns a + b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -padd(const Packet& a, - const Packet& b) { return a+b; } +padd(const Packet& a, const Packet& b) { return a+b; } /** \internal \returns a - b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -psub(const Packet& a, - const Packet& b) { return a-b; } +psub(const Packet& a, const Packet& b) { return a-b; } /** \internal \returns -a (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet @@ -165,23 +176,19 @@ /** \internal \returns a * b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -pmul(const Packet& a, - const Packet& b) { return a*b; } +pmul(const Packet& a, const Packet& b) { return a*b; } /** \internal \returns a / b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -pdiv(const Packet& a, - const Packet& b) { return a/b; } +pdiv(const Packet& a, const Packet& b) { return a/b; } /** \internal \returns the min of \a a and \a b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -pmin(const Packet& a, - const Packet& b) { return numext::mini(a, b); } +pmin(const Packet& a, const Packet& b) { return numext::mini(a, b); } /** \internal \returns the max of \a a and \a b (coeff-wise) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -pmax(const Packet& a, - const Packet& b) { return numext::maxi(a, b); } +pmax(const Packet& a, const Packet& b) { return numext::maxi(a, b); } /** \internal \returns the absolute value of \a a */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet @@ -205,7 +212,72 @@ /** \internal \returns the bitwise andnot of \a a and \a b */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet -pandnot(const Packet& a, const Packet& b) { return a & (!b); } +pandnot(const Packet& a, const Packet& b) { return a & (~b); } + +/** \internal \returns ones */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +ptrue(const Packet& /*a*/) { Packet b; memset((void*)&b, 0xff, sizeof(b)); return b;} + +template <typename RealScalar> +EIGEN_DEVICE_FUNC inline std::complex<RealScalar> ptrue(const std::complex<RealScalar>& /*a*/) { + RealScalar b; + b = ptrue(b); + return std::complex<RealScalar>(b, b); +} + +/** \internal \returns the bitwise not of \a a */ +template <typename Packet> EIGEN_DEVICE_FUNC inline Packet +pnot(const Packet& a) { return pxor(ptrue(a), a);} + +/** \internal \returns \a a shifted by N bits to the right */ +template<int N> EIGEN_DEVICE_FUNC inline int +pshiftright(const int& a) { return a >> N; } +template<int N> EIGEN_DEVICE_FUNC inline long int +pshiftright(const long int& a) { return a >> N; } + +/** \internal \returns \a a shifted by N bits to the left */ +template<int N> EIGEN_DEVICE_FUNC inline int +pshiftleft(const int& a) { return a << N; } +template<int N> EIGEN_DEVICE_FUNC inline long int +pshiftleft(const long int& a) { return a << N; } + +/** \internal \returns the significant and exponent of the underlying floating point numbers + * See https://en.cppreference.com/w/cpp/numeric/math/frexp + */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pfrexp(const Packet &a, Packet &exponent) { return std::frexp(a,&exponent); } + +/** \internal \returns a * 2^exponent + * See https://en.cppreference.com/w/cpp/numeric/math/ldexp + */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pldexp(const Packet &a, const Packet &exponent) { return std::ldexp(a,exponent); } + +/** \internal \returns zeros */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pzero(const Packet& a) { return pxor(a,a); } + +/** \internal \returns bits of \a or \b according to the input bit mask \a mask */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pselect(const Packet& mask, const Packet& a, const Packet& b) { + return por(pand(a,mask),pandnot(b,mask)); +} + +/** \internal \returns a <= b as a bit mask */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pcmp_le(const Packet& a, const Packet& b) { return a<=b ? ptrue(a) : pzero(a); } + +/** \internal \returns a < b as a bit mask */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pcmp_lt(const Packet& a, const Packet& b) { return a<b ? ptrue(a) : pzero(a); } + +/** \internal \returns a == b as a bit mask */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pcmp_eq(const Packet& a, const Packet& b) { return a==b ? ptrue(a) : pzero(a); } + +/** \internal \returns a < b or a==NaN or b==NaN as a bit mask */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pcmp_lt_or_nan(const Packet& a, const Packet& b) { return pnot(pcmp_le(b,a)); } /** \internal \returns a packet version of \a *from, from must be 16 bytes aligned */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet @@ -219,6 +291,10 @@ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pset1(const typename unpacket_traits<Packet>::type& a) { return a; } +/** \internal \returns a packet with constant coefficients set from bits */ +template<typename Packet,typename BitsType> EIGEN_DEVICE_FUNC inline Packet +pset1frombits(BitsType a); + /** \internal \returns a packet with constant coefficients \a a[0], e.g.: (a[0],a[0],a[0],a[0]) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pload1(const typename unpacket_traits<Packet>::type *a) { return pset1<Packet>(*a); } @@ -228,7 +304,7 @@ * duplicated to form: {from[0],from[0],from[1],from[1],from[2],from[2],from[3],from[3]} * Currently, this function is only used for scalar * complex products. */ -template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet ploaddup(const typename unpacket_traits<Packet>::type* from) { return *from; } /** \internal \returns a packet with elements of \a *from quadrupled. @@ -276,7 +352,7 @@ } /** \internal \brief Returns a packet with coefficients (a,a+1,...,a+packet_size-1). */ -template<typename Packet> inline Packet +template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet plset(const typename unpacket_traits<Packet>::type& a) { return a; } /** \internal copy the packet \a from to \a *to, \a to must be 16 bytes aligned */ @@ -296,7 +372,9 @@ /** \internal tries to do cache prefetching of \a addr */ template<typename Scalar> EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr) { -#ifdef __CUDA_ARCH__ +#if defined(EIGEN_HIP_DEVICE_COMPILE) + // do nothing +#elif defined(EIGEN_CUDA_ARCH) #if defined(__LP64__) // 64-bit pointer operand constraint for inlined asm asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr)); @@ -304,7 +382,7 @@ // 32-bit pointer operand constraint for inlined asm asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr)); #endif -#elif !EIGEN_COMP_MSVC +#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC) __builtin_prefetch(addr); #endif } @@ -321,47 +399,52 @@ template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux(const Packet& a) { return a; } -/** \internal \returns the sum of the elements of \a a by block of 4 elements. +/** \internal \returns the sum of the elements of upper and lower half of \a a if \a a is larger than 4. * For a packet {a0, a1, a2, a3, a4, a5, a6, a7}, it returns a half packet {a0+a4, a1+a5, a2+a6, a3+a7} * For packet-size smaller or equal to 4, this boils down to a noop. */ template<typename Packet> EIGEN_DEVICE_FUNC inline typename conditional<(unpacket_traits<Packet>::size%8)==0,typename unpacket_traits<Packet>::half,Packet>::type -predux4(const Packet& a) +predux_half_dowto4(const Packet& a) { return a; } -/** \internal \returns the product of the elements of \a a*/ +/** \internal \returns the product of the elements of \a a */ template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_mul(const Packet& a) { return a; } -/** \internal \returns the min of the elements of \a a*/ +/** \internal \returns the min of the elements of \a a */ template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(const Packet& a) { return a; } -/** \internal \returns the max of the elements of \a a*/ +/** \internal \returns the max of the elements of \a a */ template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(const Packet& a) { return a; } +/** \internal \returns true if all coeffs of \a a means "true" + * It is supposed to be called on values returned by pcmp_*. + */ +// not needed yet +// template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_all(const Packet& a) +// { return bool(a); } + +/** \internal \returns true if any coeffs of \a a means "true" + * It is supposed to be called on values returned by pcmp_*. + */ +template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_any(const Packet& a) +{ + // Dirty but generic implementation where "true" is assumed to be non 0 and all the sames. + // It is expected that "true" is either: + // - Scalar(1) + // - bits full of ones (NaN for floats), + // - or first bit equals to 1 (1 for ints, smallest denormal for floats). + // For all these cases, taking the sum is just fine, and this boils down to a no-op for scalars. + return bool(predux(a)); +} + /** \internal \returns the reversed elements of \a a*/ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet preverse(const Packet& a) { return a; } -template<size_t offset, typename Packet> -struct protate_impl -{ - // Empty so attempts to use this unimplemented path will fail to compile. - // Only specializations of this template should be used. -}; - -/** \internal \returns a packet with the coefficients rotated to the right in little-endian convention, - * by the given offset, e.g. for offset == 1: - * (packet[3], packet[2], packet[1], packet[0]) becomes (packet[0], packet[3], packet[2], packet[1]) - */ -template<size_t offset, typename Packet> EIGEN_DEVICE_FUNC inline Packet protate(const Packet& a) -{ - return offset ? protate_impl<offset, Packet>::run(a) : a; -} - /** \internal \returns \a a with real and imaginary part flipped (for complex type only) */ template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pcplxflip(const Packet& a) { @@ -415,10 +498,18 @@ template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp(const Packet& a) { using std::exp; return exp(a); } +/** \internal \returns the expm1 of \a a (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pexpm1(const Packet& a) { return numext::expm1(a); } + /** \internal \returns the log of \a a (coeff-wise) */ template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog(const Packet& a) { using std::log; return log(a); } +/** \internal \returns the log1p of \a a (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet plog1p(const Packet& a) { return numext::log1p(a); } + /** \internal \returns the log10 of \a a (coeff-wise) */ template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog10(const Packet& a) { using std::log10; return log10(a); } @@ -445,43 +536,11 @@ template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pceil(const Packet& a) { using numext::ceil; return ceil(a); } -/** \internal \returns the ln(|gamma(\a a)|) (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet plgamma(const Packet& a) { using numext::lgamma; return lgamma(a); } - -/** \internal \returns the derivative of lgamma, psi(\a a) (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet pdigamma(const Packet& a) { using numext::digamma; return digamma(a); } - -/** \internal \returns the zeta function of two arguments (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet pzeta(const Packet& x, const Packet& q) { using numext::zeta; return zeta(x, q); } - -/** \internal \returns the polygamma function (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet ppolygamma(const Packet& n, const Packet& x) { using numext::polygamma; return polygamma(n, x); } - -/** \internal \returns the erf(\a a) (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet perf(const Packet& a) { using numext::erf; return erf(a); } - -/** \internal \returns the erfc(\a a) (coeff-wise) */ -template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -Packet perfc(const Packet& a) { using numext::erfc; return erfc(a); } - -/** \internal \returns the incomplete gamma function igamma(\a a, \a x) */ -template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -Packet pigamma(const Packet& a, const Packet& x) { using numext::igamma; return igamma(a, x); } - -/** \internal \returns the complementary incomplete gamma function igammac(\a a, \a x) */ -template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -Packet pigammac(const Packet& a, const Packet& x) { using numext::igammac; return igammac(a, x); } - /*************************************************************************** * The following functions might not have to be overwritten for vectorized types ***************************************************************************/ -/** \internal copy a packet with constant coeficient \a a (e.g., [a,a,a,a]) to \a *to. \a to must be 16 bytes aligned */ +/** \internal copy a packet with constant coefficient \a a (e.g., [a,a,a,a]) to \a *to. \a to must be 16 bytes aligned */ // NOTE: this function must really be templated on the packet type (think about different packet types for the same scalar type) template<typename Packet> inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename unpacket_traits<Packet>::type& a) @@ -524,7 +583,7 @@ * by the current computation. */ template<typename Packet, int LoadMode> -inline Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from) +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from) { return ploadt<Packet, LoadMode>(from); } @@ -563,7 +622,7 @@ ***************************************************************************/ // Eigen+CUDA does not support complexes. -#ifndef __CUDACC__ +#if !defined(EIGEN_GPUCC) template<> inline std::complex<float> pmul(const std::complex<float>& a, const std::complex<float>& b) { return std::complex<float>(real(a)*real(b) - imag(a)*imag(b), imag(a)*real(b) + real(a)*imag(b)); } @@ -600,6 +659,50 @@ return ifPacket.select[0] ? thenPacket : elsePacket; } +/** \internal \returns \a a with the first coefficient replaced by the scalar b */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pinsertfirst(const Packet& a, typename unpacket_traits<Packet>::type b) +{ + // Default implementation based on pblend. + // It must be specialized for higher performance. + Selector<unpacket_traits<Packet>::size> mask; + mask.select[0] = true; + // This for loop should be optimized away by the compiler. + for(Index i=1; i<unpacket_traits<Packet>::size; ++i) + mask.select[i] = false; + return pblend(mask, pset1<Packet>(b), a); +} + +/** \internal \returns \a a with the last coefficient replaced by the scalar b */ +template<typename Packet> EIGEN_DEVICE_FUNC inline Packet +pinsertlast(const Packet& a, typename unpacket_traits<Packet>::type b) +{ + // Default implementation based on pblend. + // It must be specialized for higher performance. + Selector<unpacket_traits<Packet>::size> mask; + // This for loop should be optimized away by the compiler. + for(Index i=0; i<unpacket_traits<Packet>::size-1; ++i) + mask.select[i] = false; + mask.select[unpacket_traits<Packet>::size-1] = true; + return pblend(mask, pset1<Packet>(b), a); +} + +/*************************************************************************** + * Some generic implementations to be used by implementors +***************************************************************************/ + +/** Default implementation of pfrexp for float. + * It is expected to be called by implementers of template<> pfrexp. + */ +template<typename Packet> EIGEN_STRONG_INLINE Packet +pfrexp_float(const Packet& a, Packet& exponent); + +/** Default implementation of pldexp for float. + * It is expected to be called by implementers of template<> pldexp. + */ +template<typename Packet> EIGEN_STRONG_INLINE Packet +pldexp_float(Packet a, Packet exponent); + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/GlobalFunctions.h b/Eigen/src/Core/GlobalFunctions.h index 05ba6dd..71377ce 100644 --- a/Eigen/src/Core/GlobalFunctions.h +++ b/Eigen/src/Core/GlobalFunctions.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2010-2012 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2010-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2010 Benoit Jacob <jacob.benoit.1@gmail.com> // // This Source Code Form is subject to the terms of the Mozilla @@ -11,13 +11,30 @@ #ifndef EIGEN_GLOBAL_FUNCTIONS_H #define EIGEN_GLOBAL_FUNCTIONS_H -#define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME,FUNCTOR) \ +#ifdef EIGEN_PARSED_BY_DOXYGEN + +#define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME,FUNCTOR,DOC_OP,DOC_DETAILS) \ + /** \returns an expression of the coefficient-wise DOC_OP of \a x + + DOC_DETAILS + + \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_##NAME">Math functions</a>, class CwiseUnaryOp + */ \ + template<typename Derived> \ + inline const Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived> \ + NAME(const Eigen::ArrayBase<Derived>& x); + +#else + +#define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME,FUNCTOR,DOC_OP,DOC_DETAILS) \ template<typename Derived> \ inline const Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived> \ (NAME)(const Eigen::ArrayBase<Derived>& x) { \ return Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived>(x.derived()); \ } +#endif // EIGEN_PARSED_BY_DOXYGEN + #define EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(NAME,FUNCTOR) \ \ template<typename Derived> \ @@ -36,47 +53,76 @@ namespace Eigen { - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(real,scalar_real_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(imag,scalar_imag_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(conj,scalar_conjugate_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(inverse,scalar_inverse_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sin,scalar_sin_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cos,scalar_cos_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tan,scalar_tan_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atan,scalar_atan_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asin,scalar_asin_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acos,scalar_acos_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sinh,scalar_sinh_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cosh,scalar_cosh_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tanh,scalar_tanh_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(lgamma,scalar_lgamma_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(digamma,scalar_digamma_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(zeta,scalar_zeta_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(polygamma,scalar_polygamma_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erf,scalar_erf_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erfc,scalar_erfc_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(exp,scalar_exp_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log,scalar_log_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log10,scalar_log10_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs,scalar_abs_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs2,scalar_abs2_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(arg,scalar_arg_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sqrt,scalar_sqrt_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(square,scalar_square_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cube,scalar_cube_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(round,scalar_round_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(floor,scalar_floor_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(ceil,scalar_ceil_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isnan,scalar_isnan_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isinf,scalar_isinf_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isfinite,scalar_isfinite_op) - EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sign,scalar_sign_op) - - template<typename Derived> - inline const Eigen::CwiseUnaryOp<Eigen::internal::scalar_pow_op<typename Derived::Scalar>, const Derived> - pow(const Eigen::ArrayBase<Derived>& x, const typename Derived::Scalar& exponent) { - return x.derived().pow(exponent); + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(real,scalar_real_op,real part,\sa ArrayBase::real) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(imag,scalar_imag_op,imaginary part,\sa ArrayBase::imag) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(conj,scalar_conjugate_op,complex conjugate,\sa ArrayBase::conjugate) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(inverse,scalar_inverse_op,inverse,\sa ArrayBase::inverse) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sin,scalar_sin_op,sine,\sa ArrayBase::sin) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cos,scalar_cos_op,cosine,\sa ArrayBase::cos) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tan,scalar_tan_op,tangent,\sa ArrayBase::tan) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atan,scalar_atan_op,arc-tangent,\sa ArrayBase::atan) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asin,scalar_asin_op,arc-sine,\sa ArrayBase::asin) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acos,scalar_acos_op,arc-consine,\sa ArrayBase::acos) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sinh,scalar_sinh_op,hyperbolic sine,\sa ArrayBase::sinh) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cosh,scalar_cosh_op,hyperbolic cosine,\sa ArrayBase::cosh) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tanh,scalar_tanh_op,hyperbolic tangent,\sa ArrayBase::tanh) +#if EIGEN_HAS_CXX11_MATH + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asinh,scalar_asinh_op,inverse hyperbolic sine,\sa ArrayBase::asinh) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acosh,scalar_acosh_op,inverse hyperbolic cosine,\sa ArrayBase::acosh) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atanh,scalar_atanh_op,inverse hyperbolic tangent,\sa ArrayBase::atanh) +#endif + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(logistic,scalar_logistic_op,logistic function,\sa ArrayBase::logistic) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(lgamma,scalar_lgamma_op,natural logarithm of the gamma function,\sa ArrayBase::lgamma) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(digamma,scalar_digamma_op,derivative of lgamma,\sa ArrayBase::digamma) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erf,scalar_erf_op,error function,\sa ArrayBase::erf) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erfc,scalar_erfc_op,complement error function,\sa ArrayBase::erfc) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(exp,scalar_exp_op,exponential,\sa ArrayBase::exp) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(expm1,scalar_expm1_op,exponential of a value minus 1,\sa ArrayBase::expm1) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log,scalar_log_op,natural logarithm,\sa Eigen::log10 DOXCOMMA ArrayBase::log) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log1p,scalar_log1p_op,natural logarithm of 1 plus the value,\sa ArrayBase::log1p) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log10,scalar_log10_op,base 10 logarithm,\sa Eigen::log DOXCOMMA ArrayBase::log) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs,scalar_abs_op,absolute value,\sa ArrayBase::abs DOXCOMMA MatrixBase::cwiseAbs) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs2,scalar_abs2_op,squared absolute value,\sa ArrayBase::abs2 DOXCOMMA MatrixBase::cwiseAbs2) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(arg,scalar_arg_op,complex argument,\sa ArrayBase::arg) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sqrt,scalar_sqrt_op,square root,\sa ArrayBase::sqrt DOXCOMMA MatrixBase::cwiseSqrt) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rsqrt,scalar_rsqrt_op,reciprocal square root,\sa ArrayBase::rsqrt) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(square,scalar_square_op,square (power 2),\sa Eigen::abs2 DOXCOMMA Eigen::pow DOXCOMMA ArrayBase::square) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cube,scalar_cube_op,cube (power 3),\sa Eigen::pow DOXCOMMA ArrayBase::cube) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(round,scalar_round_op,nearest integer,\sa Eigen::floor DOXCOMMA Eigen::ceil DOXCOMMA ArrayBase::round) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(floor,scalar_floor_op,nearest integer not greater than the giben value,\sa Eigen::ceil DOXCOMMA ArrayBase::floor) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(ceil,scalar_ceil_op,nearest integer not less than the giben value,\sa Eigen::floor DOXCOMMA ArrayBase::ceil) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isnan,scalar_isnan_op,not-a-number test,\sa Eigen::isinf DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isnan) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isinf,scalar_isinf_op,infinite value test,\sa Eigen::isnan DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isinf) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isfinite,scalar_isfinite_op,finite value test,\sa Eigen::isinf DOXCOMMA Eigen::isnan DOXCOMMA ArrayBase::isfinite) + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sign,scalar_sign_op,sign (or 0),\sa ArrayBase::sign) + + /** \returns an expression of the coefficient-wise power of \a x to the given constant \a exponent. + * + * \tparam ScalarExponent is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression (\c Derived::Scalar). + * + * \sa ArrayBase::pow() + * + * \relates ArrayBase + */ +#ifdef EIGEN_PARSED_BY_DOXYGEN + template<typename Derived,typename ScalarExponent> + inline const CwiseBinaryOp<internal::scalar_pow_op<Derived::Scalar,ScalarExponent>,Derived,Constant<ScalarExponent> > + pow(const Eigen::ArrayBase<Derived>& x, const ScalarExponent& exponent); +#else + template <typename Derived,typename ScalarExponent> + EIGEN_DEVICE_FUNC inline + EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg<typename Derived::Scalar + EIGEN_COMMA ScalarExponent EIGEN_COMMA + EIGEN_SCALAR_BINARY_SUPPORTED(pow,typename Derived::Scalar,ScalarExponent)>::type,pow)) + pow(const Eigen::ArrayBase<Derived>& x, const ScalarExponent& exponent) + { + typedef typename internal::promote_scalar_arg<typename Derived::Scalar,ScalarExponent, + EIGEN_SCALAR_BINARY_SUPPORTED(pow,typename Derived::Scalar,ScalarExponent)>::type PromotedExponent; + return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedExponent,pow)(x.derived(), + typename internal::plain_constant_type<Derived,PromotedExponent>::type(x.derived().rows(), x.derived().cols(), internal::scalar_constant_op<PromotedExponent>(exponent))); } +#endif /** \returns an expression of the coefficient-wise power of \a x to the given array of \a exponents. * @@ -84,82 +130,53 @@ * * Example: \include Cwise_array_power_array.cpp * Output: \verbinclude Cwise_array_power_array.out - * + * * \sa ArrayBase::pow() + * + * \relates ArrayBase */ template<typename Derived,typename ExponentDerived> - inline const Eigen::CwiseBinaryOp<Eigen::internal::scalar_binary_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived> - pow(const Eigen::ArrayBase<Derived>& x, const Eigen::ArrayBase<ExponentDerived>& exponents) + inline const Eigen::CwiseBinaryOp<Eigen::internal::scalar_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived> + pow(const Eigen::ArrayBase<Derived>& x, const Eigen::ArrayBase<ExponentDerived>& exponents) { - return Eigen::CwiseBinaryOp<Eigen::internal::scalar_binary_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived>( + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived>( x.derived(), exponents.derived() ); } - + /** \returns an expression of the coefficient-wise power of the scalar \a x to the given array of \a exponents. * * This function computes the coefficient-wise power between a scalar and an array of exponents. - * Beaware that the scalar type of the input scalar \a x and the exponents \a exponents must be the same. + * + * \tparam Scalar is the scalar type of \a x. It must be compatible with the scalar type of the given array expression (\c Derived::Scalar). * * Example: \include Cwise_scalar_power_array.cpp * Output: \verbinclude Cwise_scalar_power_array.out - * + * * \sa ArrayBase::pow() + * + * \relates ArrayBase */ - template<typename Derived> - inline const Eigen::CwiseBinaryOp<Eigen::internal::scalar_binary_pow_op<typename Derived::Scalar, typename Derived::Scalar>, const typename Derived::ConstantReturnType, const Derived> - pow(const typename Derived::Scalar& x, const Eigen::ArrayBase<Derived>& exponents) - { - typename Derived::ConstantReturnType constant_x(exponents.rows(), exponents.cols(), x); - return Eigen::CwiseBinaryOp<Eigen::internal::scalar_binary_pow_op<typename Derived::Scalar, typename Derived::Scalar>, const typename Derived::ConstantReturnType, const Derived>( - constant_x, - exponents.derived() - ); +#ifdef EIGEN_PARSED_BY_DOXYGEN + template<typename Scalar,typename Derived> + inline const CwiseBinaryOp<internal::scalar_pow_op<Scalar,Derived::Scalar>,Constant<Scalar>,Derived> + pow(const Scalar& x,const Eigen::ArrayBase<Derived>& x); +#else + template <typename Scalar, typename Derived> + EIGEN_DEVICE_FUNC inline + EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( + const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg<typename Derived::Scalar + EIGEN_COMMA Scalar EIGEN_COMMA + EIGEN_SCALAR_BINARY_SUPPORTED(pow,Scalar,typename Derived::Scalar)>::type,Derived,pow)) + pow(const Scalar& x, const Eigen::ArrayBase<Derived>& exponents) { + typedef typename internal::promote_scalar_arg<typename Derived::Scalar,Scalar, + EIGEN_SCALAR_BINARY_SUPPORTED(pow,Scalar,typename Derived::Scalar)>::type PromotedScalar; + return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedScalar,Derived,pow)( + typename internal::plain_constant_type<Derived,PromotedScalar>::type(exponents.derived().rows(), exponents.derived().cols(), internal::scalar_constant_op<PromotedScalar>(x)), exponents.derived()); } - - /** - * \brief Component-wise division of a scalar by array elements. - **/ - template <typename Derived> - inline const Eigen::CwiseUnaryOp<Eigen::internal::scalar_inverse_mult_op<typename Derived::Scalar>, const Derived> - operator/(const typename Derived::Scalar& s, const Eigen::ArrayBase<Derived>& a) - { - return Eigen::CwiseUnaryOp<Eigen::internal::scalar_inverse_mult_op<typename Derived::Scalar>, const Derived>( - a.derived(), - Eigen::internal::scalar_inverse_mult_op<typename Derived::Scalar>(s) - ); - } +#endif - /** \returns an expression of the coefficient-wise igamma(\a a, \a x) to the given arrays. - * - * This function computes the coefficient-wise incomplete gamma function. - * - */ - template<typename Derived,typename ExponentDerived> - inline const Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_op<typename Derived::Scalar>, const Derived, const ExponentDerived> - igamma(const Eigen::ArrayBase<Derived>& a, const Eigen::ArrayBase<ExponentDerived>& x) - { - return Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_op<typename Derived::Scalar>, const Derived, const ExponentDerived>( - a.derived(), - x.derived() - ); - } - - /** \returns an expression of the coefficient-wise igammac(\a a, \a x) to the given arrays. - * - * This function computes the coefficient-wise complementary incomplete gamma function. - * - */ - template<typename Derived,typename ExponentDerived> - inline const Eigen::CwiseBinaryOp<Eigen::internal::scalar_igammac_op<typename Derived::Scalar>, const Derived, const ExponentDerived> - igammac(const Eigen::ArrayBase<Derived>& a, const Eigen::ArrayBase<ExponentDerived>& x) - { - return Eigen::CwiseBinaryOp<Eigen::internal::scalar_igammac_op<typename Derived::Scalar>, const Derived, const ExponentDerived>( - a.derived(), - x.derived() - ); - } namespace internal {
diff --git a/Eigen/src/Core/IO.h b/Eigen/src/Core/IO.h index dfd9097..063511f 100644 --- a/Eigen/src/Core/IO.h +++ b/Eigen/src/Core/IO.h
@@ -41,6 +41,7 @@ * - \b rowSuffix string printed at the end of each row * - \b matPrefix string printed at the beginning of the matrix * - \b matSuffix string printed at the end of the matrix + * - \b fill character printed to fill the empty space in aligned columns * * Example: \include IOFormat.cpp * Output: \verbinclude IOFormat.out @@ -53,9 +54,9 @@ IOFormat(int _precision = StreamPrecision, int _flags = 0, const std::string& _coeffSeparator = " ", const std::string& _rowSeparator = "\n", const std::string& _rowPrefix="", const std::string& _rowSuffix="", - const std::string& _matPrefix="", const std::string& _matSuffix="") + const std::string& _matPrefix="", const std::string& _matSuffix="", const char _fill=' ') : matPrefix(_matPrefix), matSuffix(_matSuffix), rowPrefix(_rowPrefix), rowSuffix(_rowSuffix), rowSeparator(_rowSeparator), - rowSpacer(""), coeffSeparator(_coeffSeparator), precision(_precision), flags(_flags) + rowSpacer(""), coeffSeparator(_coeffSeparator), fill(_fill), precision(_precision), flags(_flags) { // TODO check if rowPrefix, rowSuffix or rowSeparator contains a newline // don't add rowSpacer if columns are not to be aligned @@ -71,6 +72,7 @@ std::string matPrefix, matSuffix; std::string rowPrefix, rowSuffix, rowSeparator, rowSpacer; std::string coeffSeparator; + char fill; int precision; int flags; }; @@ -105,51 +107,23 @@ } protected: - const typename ExpressionType::Nested m_matrix; + typename ExpressionType::Nested m_matrix; IOFormat m_format; }; -/** \returns a WithFormat proxy object allowing to print a matrix the with given - * format \a fmt. - * - * See class IOFormat for some examples. - * - * \sa class IOFormat, class WithFormat - */ -template<typename Derived> -inline const WithFormat<Derived> -DenseBase<Derived>::format(const IOFormat& fmt) const -{ - return WithFormat<Derived>(derived(), fmt); -} - namespace internal { -template<typename Scalar, bool IsInteger> -struct significant_decimals_default_impl -{ - typedef typename NumTraits<Scalar>::Real RealScalar; - static inline int run() - { - using std::ceil; - using std::log; - return cast<RealScalar,int>(ceil(-log(NumTraits<RealScalar>::epsilon())/log(RealScalar(10)))); - } -}; - -template<typename Scalar> -struct significant_decimals_default_impl<Scalar, true> -{ - static inline int run() - { - return 0; - } -}; - +// NOTE: This helper is kept for backward compatibility with previous code specializing +// this internal::significant_decimals_impl structure. In the future we should directly +// call digits10() which has been introduced in July 2016 in 3.3. template<typename Scalar> struct significant_decimals_impl - : significant_decimals_default_impl<Scalar, NumTraits<Scalar>::IsInteger> -{}; +{ + static inline int run() + { + return NumTraits<Scalar>::digits10(); + } +}; /** \internal * print the matrix \a _m to the output stream \a s using the output format \a fmt */ @@ -204,18 +178,26 @@ width = std::max<Index>(width, Index(sstr.str().length())); } } + std::streamsize old_width = s.width(); + char old_fill_character = s.fill(); s << fmt.matPrefix; for(Index i = 0; i < m.rows(); ++i) { if (i) s << fmt.rowSpacer; s << fmt.rowPrefix; - if(width) s.width(width); + if(width) { + s.fill(fmt.fill); + s.width(width); + } s << m.coeff(i, 0); for(Index j = 1; j < m.cols(); ++j) { s << fmt.coeffSeparator; - if (width) s.width(width); + if(width) { + s.fill(fmt.fill); + s.width(width); + } s << m.coeff(i, j); } s << fmt.rowSuffix; @@ -224,6 +206,10 @@ } s << fmt.matSuffix; if(explicit_precision) s.precision(old_precision); + if(width) { + s.fill(old_fill_character); + s.width(old_width); + } return s; }
diff --git a/Eigen/src/Core/IndexedView.h b/Eigen/src/Core/IndexedView.h new file mode 100644 index 0000000..377f8a5 --- /dev/null +++ b/Eigen/src/Core/IndexedView.h
@@ -0,0 +1,207 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_INDEXED_VIEW_H +#define EIGEN_INDEXED_VIEW_H + +namespace Eigen { + +namespace internal { + +template<typename XprType, typename RowIndices, typename ColIndices> +struct traits<IndexedView<XprType, RowIndices, ColIndices> > + : traits<XprType> +{ + enum { + RowsAtCompileTime = int(array_size<RowIndices>::value), + ColsAtCompileTime = int(array_size<ColIndices>::value), + MaxRowsAtCompileTime = RowsAtCompileTime != Dynamic ? int(RowsAtCompileTime) : Dynamic, + MaxColsAtCompileTime = ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime) : Dynamic, + + XprTypeIsRowMajor = (int(traits<XprType>::Flags)&RowMajorBit) != 0, + IsRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 + : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 + : XprTypeIsRowMajor, + + RowIncr = int(get_compile_time_incr<RowIndices>::value), + ColIncr = int(get_compile_time_incr<ColIndices>::value), + InnerIncr = IsRowMajor ? ColIncr : RowIncr, + OuterIncr = IsRowMajor ? RowIncr : ColIncr, + + HasSameStorageOrderAsXprType = (IsRowMajor == XprTypeIsRowMajor), + XprInnerStride = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time<XprType>::ret) : int(outer_stride_at_compile_time<XprType>::ret), + XprOuterstride = HasSameStorageOrderAsXprType ? int(outer_stride_at_compile_time<XprType>::ret) : int(inner_stride_at_compile_time<XprType>::ret), + + InnerSize = XprTypeIsRowMajor ? ColsAtCompileTime : RowsAtCompileTime, + IsBlockAlike = InnerIncr==1 && OuterIncr==1, + IsInnerPannel = HasSameStorageOrderAsXprType && is_same<AllRange<InnerSize>,typename conditional<XprTypeIsRowMajor,ColIndices,RowIndices>::type>::value, + + InnerStrideAtCompileTime = InnerIncr<0 || InnerIncr==DynamicIndex || XprInnerStride==Dynamic ? Dynamic : XprInnerStride * InnerIncr, + OuterStrideAtCompileTime = OuterIncr<0 || OuterIncr==DynamicIndex || XprOuterstride==Dynamic ? Dynamic : XprOuterstride * OuterIncr, + + ReturnAsScalar = is_same<RowIndices,SingleRange>::value && is_same<ColIndices,SingleRange>::value, + ReturnAsBlock = (!ReturnAsScalar) && IsBlockAlike, + ReturnAsIndexedView = (!ReturnAsScalar) && (!ReturnAsBlock), + + // FIXME we deal with compile-time strides if and only if we have DirectAccessBit flag, + // but this is too strict regarding negative strides... + DirectAccessMask = (int(InnerIncr)!=UndefinedIncr && int(OuterIncr)!=UndefinedIncr && InnerIncr>=0 && OuterIncr>=0) ? DirectAccessBit : 0, + FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0, + FlagsLvalueBit = is_lvalue<XprType>::value ? LvalueBit : 0, + Flags = (traits<XprType>::Flags & (HereditaryBits | DirectAccessMask)) | FlagsLvalueBit | FlagsRowMajorBit + }; + + typedef Block<XprType,RowsAtCompileTime,ColsAtCompileTime,IsInnerPannel> BlockType; +}; + +} + +template<typename XprType, typename RowIndices, typename ColIndices, typename StorageKind> +class IndexedViewImpl; + + +/** \class IndexedView + * \ingroup Core_Module + * + * \brief Expression of a non-sequential sub-matrix defined by arbitrary sequences of row and column indices + * + * \tparam XprType the type of the expression in which we are taking the intersections of sub-rows and sub-columns + * \tparam RowIndices the type of the object defining the sequence of row indices + * \tparam ColIndices the type of the object defining the sequence of column indices + * + * This class represents an expression of a sub-matrix (or sub-vector) defined as the intersection + * of sub-sets of rows and columns, that are themself defined by generic sequences of row indices \f$ \{r_0,r_1,..r_{m-1}\} \f$ + * and column indices \f$ \{c_0,c_1,..c_{n-1} \}\f$. Let \f$ A \f$ be the nested matrix, then the resulting matrix \f$ B \f$ has \c m + * rows and \c n columns, and its entries are given by: \f$ B(i,j) = A(r_i,c_j) \f$. + * + * The \c RowIndices and \c ColIndices types must be compatible with the following API: + * \code + * <integral type> operator[](Index) const; + * Index size() const; + * \endcode + * + * Typical supported types thus include: + * - std::vector<int> + * - std::valarray<int> + * - std::array<int> + * - Plain C arrays: int[N] + * - Eigen::ArrayXi + * - decltype(ArrayXi::LinSpaced(...)) + * - Any view/expressions of the previous types + * - Eigen::ArithmeticSequence + * - Eigen::internal::AllRange (helper for Eigen::all) + * - Eigen::internal::SingleRange (helper for single index) + * - etc. + * + * In typical usages of %Eigen, this class should never be used directly. It is the return type of + * DenseBase::operator()(const RowIndices&, const ColIndices&). + * + * \sa class Block + */ +template<typename XprType, typename RowIndices, typename ColIndices> +class IndexedView : public IndexedViewImpl<XprType, RowIndices, ColIndices, typename internal::traits<XprType>::StorageKind> +{ +public: + typedef typename IndexedViewImpl<XprType, RowIndices, ColIndices, typename internal::traits<XprType>::StorageKind>::Base Base; + EIGEN_GENERIC_PUBLIC_INTERFACE(IndexedView) + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(IndexedView) + + typedef typename internal::ref_selector<XprType>::non_const_type MatrixTypeNested; + typedef typename internal::remove_all<XprType>::type NestedExpression; + + template<typename T0, typename T1> + IndexedView(XprType& xpr, const T0& rowIndices, const T1& colIndices) + : m_xpr(xpr), m_rowIndices(rowIndices), m_colIndices(colIndices) + {} + + /** \returns number of rows */ + Index rows() const { return internal::size(m_rowIndices); } + + /** \returns number of columns */ + Index cols() const { return internal::size(m_colIndices); } + + /** \returns the nested expression */ + const typename internal::remove_all<XprType>::type& + nestedExpression() const { return m_xpr; } + + /** \returns the nested expression */ + typename internal::remove_reference<XprType>::type& + nestedExpression() { return m_xpr; } + + /** \returns a const reference to the object storing/generating the row indices */ + const RowIndices& rowIndices() const { return m_rowIndices; } + + /** \returns a const reference to the object storing/generating the column indices */ + const ColIndices& colIndices() const { return m_colIndices; } + +protected: + MatrixTypeNested m_xpr; + RowIndices m_rowIndices; + ColIndices m_colIndices; +}; + + +// Generic API dispatcher +template<typename XprType, typename RowIndices, typename ColIndices, typename StorageKind> +class IndexedViewImpl + : public internal::generic_xpr_base<IndexedView<XprType, RowIndices, ColIndices> >::type +{ +public: + typedef typename internal::generic_xpr_base<IndexedView<XprType, RowIndices, ColIndices> >::type Base; +}; + +namespace internal { + + +template<typename ArgType, typename RowIndices, typename ColIndices> +struct unary_evaluator<IndexedView<ArgType, RowIndices, ColIndices>, IndexBased> + : evaluator_base<IndexedView<ArgType, RowIndices, ColIndices> > +{ + typedef IndexedView<ArgType, RowIndices, ColIndices> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost /* TODO + cost of row/col index */, + + Flags = (evaluator<ArgType>::Flags & (HereditaryBits /*| LinearAccessBit | DirectAccessBit*/)), + + Alignment = 0 + }; + + EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) + { + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + CoeffReturnType coeff(Index row, Index col) const + { + return m_argImpl.coeff(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); + } + +protected: + + evaluator<ArgType> m_argImpl; + const XprType& m_xpr; + +}; + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_INDEXED_VIEW_H
diff --git a/Eigen/src/Core/Inverse.h b/Eigen/src/Core/Inverse.h index f3ec849..b76f043 100644 --- a/Eigen/src/Core/Inverse.h +++ b/Eigen/src/Core/Inverse.h
@@ -45,12 +45,13 @@ public: typedef typename XprType::StorageIndex StorageIndex; typedef typename XprType::PlainObject PlainObject; + typedef typename XprType::Scalar Scalar; typedef typename internal::ref_selector<XprType>::type XprTypeNested; typedef typename internal::remove_all<XprTypeNested>::type XprTypeNestedCleaned; typedef typename internal::ref_selector<Inverse>::type Nested; typedef typename internal::remove_all<XprType>::type NestedExpression; - explicit Inverse(const XprType &xpr) + explicit EIGEN_DEVICE_FUNC Inverse(const XprType &xpr) : m_xpr(xpr) {}
diff --git a/Eigen/src/Core/Map.h b/Eigen/src/Core/Map.h index 06d1967..c437f1a 100644 --- a/Eigen/src/Core/Map.h +++ b/Eigen/src/Core/Map.h
@@ -20,11 +20,17 @@ { typedef traits<PlainObjectType> TraitsBase; enum { + PlainObjectTypeInnerSize = ((traits<PlainObjectType>::Flags&RowMajorBit)==RowMajorBit) + ? PlainObjectType::ColsAtCompileTime + : PlainObjectType::RowsAtCompileTime, + InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0 ? int(PlainObjectType::InnerStrideAtCompileTime) : int(StrideType::InnerStrideAtCompileTime), OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 - ? int(PlainObjectType::OuterStrideAtCompileTime) + ? (InnerStrideAtCompileTime==Dynamic || PlainObjectTypeInnerSize==Dynamic + ? Dynamic + : int(InnerStrideAtCompileTime) * int(PlainObjectTypeInnerSize)) : int(StrideType::OuterStrideAtCompileTime), Alignment = int(MapOptions)&int(AlignedMask), Flags0 = TraitsBase::Flags & (~NestByRefBit), @@ -108,9 +114,10 @@ inline Index outerStride() const { return StrideType::OuterStrideAtCompileTime != 0 ? m_stride.outer() - : IsVectorAtCompileTime ? this->size() - : int(Flags)&RowMajorBit ? this->cols() - : this->rows(); + : internal::traits<Map>::OuterStrideAtCompileTime != Dynamic ? Index(internal::traits<Map>::OuterStrideAtCompileTime) + : IsVectorAtCompileTime ? (this->size() * innerStride()) + : int(Flags)&RowMajorBit ? (this->cols() * innerStride()) + : (this->rows() * innerStride()); } /** Constructor in the fixed-size case.
diff --git a/Eigen/src/Core/MapBase.h b/Eigen/src/Core/MapBase.h index 12c464a..668922f 100644 --- a/Eigen/src/Core/MapBase.h +++ b/Eigen/src/Core/MapBase.h
@@ -17,10 +17,20 @@ namespace Eigen { -/** \class MapBase - * \ingroup Core_Module +/** \ingroup Core_Module * - * \brief Base class for Map and Block expression with direct access + * \brief Base class for dense Map and Block expression with direct access + * + * This base class provides the const low-level accessors (e.g. coeff, coeffRef) of dense + * Map and Block objects with direct access. + * Typical users do not have to directly deal with this class. + * + * This class can be extended by through the macro plugin \c EIGEN_MAPBASE_PLUGIN. + * See \link TopicCustomizing_Plugins customizing Eigen \endlink for details. + * + * The \c Derived class has to provide the following two methods describing the memory layout: + * \code Index innerStride() const; \endcode + * \code Index outerStride() const; \endcode * * \sa class Map, class Block */ @@ -33,6 +43,7 @@ enum { RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime, ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime, + InnerStrideAtCompileTime = internal::traits<Derived>::InnerStrideAtCompileTime, SizeAtCompileTime = Base::SizeAtCompileTime }; @@ -75,7 +86,9 @@ typedef typename Base::CoeffReturnType CoeffReturnType; + /** \copydoc DenseBase::rows() */ EIGEN_DEVICE_FUNC inline Index rows() const { return m_rows.value(); } + /** \copydoc DenseBase::cols() */ EIGEN_DEVICE_FUNC inline Index cols() const { return m_cols.value(); } /** Returns a pointer to the first coefficient of the matrix or vector. @@ -86,12 +99,14 @@ */ EIGEN_DEVICE_FUNC inline const Scalar* data() const { return m_data; } + /** \copydoc PlainObjectBase::coeff(Index,Index) const */ EIGEN_DEVICE_FUNC inline const Scalar& coeff(Index rowId, Index colId) const { return m_data[colId * colStride() + rowId * rowStride()]; } + /** \copydoc PlainObjectBase::coeff(Index) const */ EIGEN_DEVICE_FUNC inline const Scalar& coeff(Index index) const { @@ -99,12 +114,14 @@ return m_data[index * innerStride()]; } + /** \copydoc PlainObjectBase::coeffRef(Index,Index) const */ EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { return this->m_data[colId * colStride() + rowId * rowStride()]; } + /** \copydoc PlainObjectBase::coeffRef(Index) const */ EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { @@ -112,6 +129,7 @@ return this->m_data[index * innerStride()]; } + /** \internal */ template<int LoadMode> inline PacketScalar packet(Index rowId, Index colId) const { @@ -119,6 +137,7 @@ (m_data + (colId * colStride() + rowId * rowStride())); } + /** \internal */ template<int LoadMode> inline PacketScalar packet(Index index) const { @@ -126,6 +145,7 @@ return internal::ploadt<PacketScalar, LoadMode>(m_data + index * innerStride()); } + /** \internal Constructor for fixed size matrices or vectors */ EIGEN_DEVICE_FUNC explicit inline MapBase(PointerType dataPtr) : m_data(dataPtr), m_rows(RowsAtCompileTime), m_cols(ColsAtCompileTime) { @@ -133,6 +153,7 @@ checkSanity<Derived>(); } + /** \internal Constructor for dynamically sized vectors */ EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index vecSize) : m_data(dataPtr), @@ -145,6 +166,7 @@ checkSanity<Derived>(); } + /** \internal Constructor for dynamically sized matrices */ EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index rows, Index cols) : m_data(dataPtr), m_rows(rows), m_cols(cols) @@ -166,8 +188,11 @@ void checkSanity(typename internal::enable_if<(internal::traits<T>::Alignment>0),void*>::type = 0) const { #if EIGEN_MAX_ALIGN_BYTES>0 - eigen_assert(( ((size_t(m_data) % internal::traits<Derived>::Alignment) == 0) - || (cols() * rows() * innerStride() * sizeof(Scalar)) < internal::traits<Derived>::Alignment ) && "data is not aligned"); + // innerStride() is not set yet when this function is called, so we optimistically assume the lowest plausible value: + const Index minInnerStride = InnerStrideAtCompileTime == Dynamic ? 1 : Index(InnerStrideAtCompileTime); + EIGEN_ONLY_USED_FOR_DEBUG(minInnerStride); + eigen_assert(( ((internal::UIntPtr(m_data) % internal::traits<Derived>::Alignment) == 0) + || (cols() * rows() * minInnerStride * sizeof(Scalar)) < internal::traits<Derived>::Alignment ) && "data is not aligned"); #endif } @@ -181,6 +206,16 @@ const internal::variable_if_dynamic<Index, ColsAtCompileTime> m_cols; }; +/** \ingroup Core_Module + * + * \brief Base class for non-const dense Map and Block expression with direct access + * + * This base class provides the non-const low-level accessors (e.g. coeff and coeffRef) of + * dense Map and Block objects with direct access. + * It inherits MapBase<Derived, ReadOnlyAccessors> which defines the const variant for reading specific entries. + * + * \sa class Map, class Block + */ template<typename Derived> class MapBase<Derived, WriteAccessors> : public MapBase<Derived, ReadOnlyAccessors> {
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 5771abf..34dd15d 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -11,7 +11,8 @@ #define EIGEN_MATHFUNCTIONS_H // source: http://www.geom.uiuc.edu/~huberty/math5337/groupe/digits.html -#define EIGEN_PI 3.141592653589793238462643383279502884197169399375105820974944592307816406 +// TODO this should better be moved to NumTraits +#define EIGEN_PI 3.141592653589793238462643383279502884197169399375105820974944592307816406L namespace Eigen { @@ -95,6 +96,19 @@ template<typename Scalar> struct real_impl : real_default_impl<Scalar> {}; +#if defined(EIGEN_GPU_COMPILE_PHASE) +template<typename T> +struct real_impl<std::complex<T> > +{ + typedef T RealScalar; + EIGEN_DEVICE_FUNC + static inline T run(const std::complex<T>& x) + { + return x.real(); + } +}; +#endif + template<typename Scalar> struct real_retval { @@ -130,6 +144,19 @@ template<typename Scalar> struct imag_impl : imag_default_impl<Scalar> {}; +#if defined(EIGEN_GPU_COMPILE_PHASE) +template<typename T> +struct imag_impl<std::complex<T> > +{ + typedef T RealScalar; + EIGEN_DEVICE_FUNC + static inline T run(const std::complex<T>& x) + { + return x.imag(); + } +}; +#endif + template<typename Scalar> struct imag_retval { @@ -211,7 +238,7 @@ ****************************************************************************/ template<typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex> -struct conj_impl +struct conj_default_impl { EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) @@ -221,7 +248,7 @@ }; template<typename Scalar> -struct conj_impl<Scalar,true> +struct conj_default_impl<Scalar,true> { EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) @@ -231,6 +258,20 @@ } }; +template<typename Scalar> struct conj_impl : conj_default_impl<Scalar> {}; + +#if defined(EIGEN_GPU_COMPILE_PHASE) +template<typename T> +struct conj_impl<std::complex<T> > +{ + EIGEN_DEVICE_FUNC + static inline std::complex<T> run(const std::complex<T>& x) + { + return std::complex<T>(x.real(), -x.imag()); + } +}; +#endif + template<typename Scalar> struct conj_retval { @@ -320,31 +361,7 @@ * Implementation of hypot * ****************************************************************************/ -template<typename Scalar> -struct hypot_impl -{ - typedef typename NumTraits<Scalar>::Real RealScalar; - static inline RealScalar run(const Scalar& x, const Scalar& y) - { - EIGEN_USING_STD_MATH(abs); - EIGEN_USING_STD_MATH(sqrt); - RealScalar _x = abs(x); - RealScalar _y = abs(y); - Scalar p, qp; - if(_x>_y) - { - p = _x; - qp = _y / p; - } - else - { - p = _y; - qp = _x / p; - } - if(p==RealScalar(0)) return RealScalar(0); - return p * sqrt(RealScalar(1) + qp*qp); - } -}; +template<typename Scalar> struct hypot_impl; template<typename Scalar> struct hypot_retval @@ -385,7 +402,7 @@ static inline Scalar run(const Scalar& x) { EIGEN_STATIC_ASSERT((!NumTraits<Scalar>::IsComplex), NUMERIC_TYPE_MUST_BE_REAL) - using std::round; + EIGEN_USING_STD_MATH(round); return round(x); } }; @@ -418,7 +435,12 @@ struct arg_impl { static inline Scalar run(const Scalar& x) { + #if defined(EIGEN_HIP_DEVICE_COMPILE) + // HIP does not seem to have a native device side implementation for the math routine "arg" + using std::arg; + #else EIGEN_USING_STD_MATH(arg); + #endif return arg(x); } }; @@ -455,32 +477,84 @@ }; /**************************************************************************** +* Implementation of expm1 * +****************************************************************************/ + +// This implementation is based on GSL Math's expm1. +namespace std_fallback { + // fallback expm1 implementation in case there is no expm1(Scalar) function in namespace of Scalar, + // or that there is no suitable std::expm1 function available. Implementation + // attributed to Kahan. See: http://www.plunk.org/~hatch/rightway.php. + template<typename Scalar> + EIGEN_DEVICE_FUNC inline Scalar expm1(const Scalar& x) { + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + typedef typename NumTraits<Scalar>::Real RealScalar; + + EIGEN_USING_STD_MATH(exp); + Scalar u = exp(x); + if (numext::equal_strict(u, Scalar(1))) { + return x; + } + Scalar um1 = u - RealScalar(1); + if (numext::equal_strict(um1, Scalar(-1))) { + return RealScalar(-1); + } + + EIGEN_USING_STD_MATH(log); + return (u - RealScalar(1)) * x / log(u); + } +} + +template<typename Scalar> +struct expm1_impl { + EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) + { + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + #if EIGEN_HAS_CXX11_MATH + using std::expm1; + #endif + using std_fallback::expm1; + return expm1(x); + } +}; + + +template<typename Scalar> +struct expm1_retval +{ + typedef Scalar type; +}; + +/**************************************************************************** * Implementation of log1p * ****************************************************************************/ -template<typename Scalar, bool isComplex = NumTraits<Scalar>::IsComplex > -struct log1p_impl -{ - static inline Scalar run(const Scalar& x) - { + +namespace std_fallback { + // fallback log1p implementation in case there is no log1p(Scalar) function in namespace of Scalar, + // or that there is no suitable std::log1p function available + template<typename Scalar> + EIGEN_DEVICE_FUNC inline Scalar log1p(const Scalar& x) { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) typedef typename NumTraits<Scalar>::Real RealScalar; EIGEN_USING_STD_MATH(log); Scalar x1p = RealScalar(1) + x; - return ( x1p == Scalar(1) ) ? x : x * ( log(x1p) / (x1p - RealScalar(1)) ); + return numext::equal_strict(x1p, Scalar(1)) ? x : x * ( log(x1p) / (x1p - RealScalar(1)) ); } -}; +} -#if EIGEN_HAS_CXX11_MATH template<typename Scalar> -struct log1p_impl<Scalar, false> { - static inline Scalar run(const Scalar& x) +struct log1p_impl { + EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + #if EIGEN_HAS_CXX11_MATH using std::log1p; + #endif + using std_fallback::log1p; return log1p(x); } }; -#endif + template<typename Scalar> struct log1p_retval @@ -492,24 +566,26 @@ * Implementation of pow * ****************************************************************************/ -template<typename Scalar, bool IsInteger> -struct pow_default_impl +template<typename ScalarX,typename ScalarY, bool IsInteger = NumTraits<ScalarX>::IsInteger&&NumTraits<ScalarY>::IsInteger> +struct pow_impl { - typedef Scalar retval; - static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) + //typedef Scalar retval; + typedef typename ScalarBinaryOpTraits<ScalarX,ScalarY,internal::scalar_pow_op<ScalarX,ScalarY> >::ReturnType result_type; + static EIGEN_DEVICE_FUNC inline result_type run(const ScalarX& x, const ScalarY& y) { EIGEN_USING_STD_MATH(pow); return pow(x, y); } }; -template<typename Scalar> -struct pow_default_impl<Scalar, true> +template<typename ScalarX,typename ScalarY> +struct pow_impl<ScalarX,ScalarY, true> { - static EIGEN_DEVICE_FUNC inline Scalar run(Scalar x, Scalar y) + typedef ScalarX result_type; + static EIGEN_DEVICE_FUNC inline ScalarX run(ScalarX x, ScalarY y) { - Scalar res(1); - eigen_assert(!NumTraits<Scalar>::IsSigned || y >= 0); + ScalarX res(1); + eigen_assert(!NumTraits<ScalarY>::IsSigned || y >= 0); if(y & 1) res *= x; y >>= 1; while(y) @@ -522,15 +598,6 @@ } }; -template<typename Scalar> -struct pow_impl : pow_default_impl<Scalar, NumTraits<Scalar>::IsInteger> {}; - -template<typename Scalar> -struct pow_retval -{ - typedef Scalar type; -}; - /**************************************************************************** * Implementation of random * ****************************************************************************/ @@ -616,20 +683,29 @@ struct random_default_impl<Scalar, false, true> { static inline Scalar run(const Scalar& x, const Scalar& y) - { - typedef typename conditional<NumTraits<Scalar>::IsSigned,std::ptrdiff_t,std::size_t>::type ScalarX; - if(y<x) + { + if (y <= x) return x; - std::size_t range = ScalarX(y)-ScalarX(x); - std::size_t offset = 0; - // rejection sampling - std::size_t divisor = (range+RAND_MAX-1)/(range+1); - std::size_t multiplier = (range+RAND_MAX-1)/std::size_t(RAND_MAX); - + // ScalarU is the unsigned counterpart of Scalar, possibly Scalar itself. + typedef typename make_unsigned<Scalar>::type ScalarU; + // ScalarX is the widest of ScalarU and unsigned int. + // We'll deal only with ScalarX and unsigned int below thus avoiding signed + // types and arithmetic and signed overflows (which are undefined behavior). + typedef typename conditional<(ScalarU(-1) > unsigned(-1)), ScalarU, unsigned>::type ScalarX; + // The following difference doesn't overflow, provided our integer types are two's + // complement and have the same number of padding bits in signed and unsigned variants. + // This is the case in most modern implementations of C++. + ScalarX range = ScalarX(y) - ScalarX(x); + ScalarX offset = 0; + ScalarX divisor = 1; + ScalarX multiplier = 1; + const unsigned rand_max = RAND_MAX; + if (range <= rand_max) divisor = (rand_max + 1) / (range + 1); + else multiplier = 1 + range / (rand_max + 1); + // Rejection sampling. do { - offset = ( (std::size_t(std::rand()) * multiplier) / divisor ); + offset = (unsigned(std::rand()) * multiplier) / divisor; } while (offset > range); - return Scalar(ScalarX(x) + offset); } @@ -675,7 +751,7 @@ return EIGEN_MATHFUNC_IMPL(random, Scalar)::run(); } -// Implementatin of is* functions +// Implementation of is* functions // std::is* do not work with fast-math and gcc, std::is* are available on MSVC 2013 and newer, as well as in clang. #if (EIGEN_HAS_CXX11_MATH && !(EIGEN_COMP_GNUC_STRICT && __FINITE_MATH_ONLY__)) || (EIGEN_COMP_MSVC>=1800) || (EIGEN_COMP_CLANG) @@ -704,7 +780,7 @@ typename internal::enable_if<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool>::type isfinite_impl(const T& x) { - #ifdef __CUDA_ARCH__ + #if defined(EIGEN_GPU_COMPILE_PHASE) return (::isfinite)(x); #elif EIGEN_USE_STD_FPCLASSIFY using std::isfinite; @@ -719,7 +795,7 @@ typename internal::enable_if<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool>::type isinf_impl(const T& x) { - #ifdef __CUDA_ARCH__ + #if defined(EIGEN_GPU_COMPILE_PHASE) return (::isinf)(x); #elif EIGEN_USE_STD_FPCLASSIFY using std::isinf; @@ -734,7 +810,7 @@ typename internal::enable_if<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool>::type isnan_impl(const T& x) { - #ifdef __CUDA_ARCH__ + #if defined(EIGEN_GPU_COMPILE_PHASE) return (::isnan)(x); #elif EIGEN_USE_STD_FPCLASSIFY using std::isnan; @@ -790,6 +866,8 @@ template<typename T> EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x); template<typename T> EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x); +template<typename T> T generic_fast_tanh_float(const T& a_x); + } // end namespace internal /**************************************************************************** @@ -798,7 +876,7 @@ namespace numext { -#ifndef __CUDA_ARCH__ +#if (!defined(EIGEN_GPUCC) || defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T mini(const T& x, const T& y) @@ -825,8 +903,26 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float mini(const float& x, const float& y) { + return fminf(x, y); +} +template<> +EIGEN_DEVICE_FUNC +EIGEN_ALWAYS_INLINE double mini(const double& x, const double& y) +{ return fmin(x, y); } +template<> +EIGEN_DEVICE_FUNC +EIGEN_ALWAYS_INLINE long double mini(const long double& x, const long double& y) +{ +#if defined(EIGEN_HIPCC) + // no "fminl" on HIP yet + return (x < y) ? x : y; +#else + return fminl(x, y); +#endif +} + template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T maxi(const T& x, const T& y) @@ -837,9 +933,95 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float maxi(const float& x, const float& y) { + return fmaxf(x, y); +} +template<> +EIGEN_DEVICE_FUNC +EIGEN_ALWAYS_INLINE double maxi(const double& x, const double& y) +{ return fmax(x, y); } +template<> +EIGEN_DEVICE_FUNC +EIGEN_ALWAYS_INLINE long double maxi(const long double& x, const long double& y) +{ +#if defined(EIGEN_HIPCC) + // no "fmaxl" on HIP yet + return (x > y) ? x : y; +#else + return fmaxl(x, y); #endif +} +#endif + +#if defined(__SYCL_DEVICE_ONLY__) + + +#define SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_char) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_short) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_int) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_long) +#define SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_char) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_short) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_int) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_long) +#define SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_uchar) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_ushort) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_uint) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_ulong) +#define SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_uchar) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_ushort) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_uint) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_ulong) +#define SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(NAME, FUNC) \ + SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \ + SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) +#define SYCL_SPECIALIZE_INTEGER_TYPES_UNARY(NAME, FUNC) \ + SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \ + SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) +#define SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(NAME, FUNC) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_float) \ + SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC,cl::sycl::cl_double) +#define SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(NAME, FUNC) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_float) \ + SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC,cl::sycl::cl_double) +#define SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(NAME, FUNC, RET_TYPE) \ + SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, cl::sycl::cl_float) \ + SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, cl::sycl::cl_double) + +#define SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE) \ +template<> \ + EIGEN_DEVICE_FUNC \ + EIGEN_ALWAYS_INLINE RET_TYPE NAME(const ARG_TYPE& x) { \ + return cl::sycl::FUNC(x); \ + } + +#define SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, TYPE) \ + SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, TYPE, TYPE) + +#define SYCL_SPECIALIZE_GEN1_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE1, ARG_TYPE2) \ + template<> \ + EIGEN_DEVICE_FUNC \ + EIGEN_ALWAYS_INLINE RET_TYPE NAME(const ARG_TYPE1& x, const ARG_TYPE2& y) { \ + return cl::sycl::FUNC(x, y); \ + } + +#define SYCL_SPECIALIZE_GEN2_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE) \ + SYCL_SPECIALIZE_GEN1_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE, ARG_TYPE) + +#define SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, TYPE) \ + SYCL_SPECIALIZE_GEN2_BINARY_FUNC(NAME, FUNC, TYPE, TYPE) + +SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(mini, min) +SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(mini, fmin) +SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(maxi, max) +SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(maxi, fmax) + +#endif // defined(__SYCL_DEVICE_ONLY__) template<typename Scalar> @@ -847,7 +1029,7 @@ inline EIGEN_MATHFUNC_RETVAL(real, Scalar) real(const Scalar& x) { return EIGEN_MATHFUNC_IMPL(real, Scalar)::run(x); -} +} template<typename Scalar> EIGEN_DEVICE_FUNC @@ -905,6 +1087,9 @@ return EIGEN_MATHFUNC_IMPL(abs2, Scalar)::run(x); } +EIGEN_DEVICE_FUNC +inline bool abs2(bool x) { return x; } + template<typename Scalar> EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(norm1, Scalar) norm1(const Scalar& x) @@ -919,6 +1104,10 @@ return EIGEN_MATHFUNC_IMPL(hypot, Scalar)::run(x, y); } +#if defined(__SYCL_DEVICE_ONLY__) + SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(hypot, hypot) +#endif // defined(__SYCL_DEVICE_ONLY__) + template<typename Scalar> EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(log1p, Scalar) log1p(const Scalar& x) @@ -926,17 +1115,39 @@ return EIGEN_MATHFUNC_IMPL(log1p, Scalar)::run(x); } -template<typename Scalar> +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(log1p, log1p) +#endif //defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +float log1p(const float &x) { return ::log1pf(x); } + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +double log1p(const double &x) { return ::log1p(x); } +#endif + +template<typename ScalarX,typename ScalarY> EIGEN_DEVICE_FUNC -inline EIGEN_MATHFUNC_RETVAL(pow, Scalar) pow(const Scalar& x, const Scalar& y) +inline typename internal::pow_impl<ScalarX,ScalarY>::result_type pow(const ScalarX& x, const ScalarY& y) { - return EIGEN_MATHFUNC_IMPL(pow, Scalar)::run(x, y); + return internal::pow_impl<ScalarX,ScalarY>::run(x, y); } +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(pow, pow) +#endif // defined(__SYCL_DEVICE_ONLY__) + template<typename T> EIGEN_DEVICE_FUNC bool (isnan) (const T &x) { return internal::isnan_impl(x); } template<typename T> EIGEN_DEVICE_FUNC bool (isinf) (const T &x) { return internal::isinf_impl(x); } template<typename T> EIGEN_DEVICE_FUNC bool (isfinite)(const T &x) { return internal::isfinite_impl(x); } +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isnan, isnan, bool) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isinf, isinf, bool) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isfinite, isfinite, bool) +#endif // defined(__SYCL_DEVICE_ONLY__) + template<typename Scalar> EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(round, Scalar) round(const Scalar& x) @@ -944,6 +1155,10 @@ return EIGEN_MATHFUNC_IMPL(round, Scalar)::run(x); } +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(round, round) +#endif // defined(__SYCL_DEVICE_ONLY__) + template<typename T> EIGEN_DEVICE_FUNC T (floor)(const T& x) @@ -952,7 +1167,11 @@ return floor(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(floor, floor) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float floor(const float &x) { return ::floorf(x); } @@ -968,7 +1187,11 @@ return ceil(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(ceil, ceil) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float ceil(const float &x) { return ::ceilf(x); } @@ -994,7 +1217,8 @@ /** \returns the square root of \a x. * - * It is essentially equivalent to \code using std::sqrt; return sqrt(x); \endcode, + * It is essentially equivalent to + * \code using std::sqrt; return sqrt(x); \endcode * but slightly faster for float/double and some compilers (e.g., gcc), thanks to * specializations when SSE is enabled. * @@ -1008,6 +1232,10 @@ return sqrt(x); } +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sqrt, sqrt) +#endif // defined(__SYCL_DEVICE_ONLY__) + template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T log(const T &x) { @@ -1015,7 +1243,12 @@ return log(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(log, log) +#endif // defined(__SYCL_DEVICE_ONLY__) + + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float log(const float &x) { return ::logf(x); } @@ -1025,17 +1258,40 @@ template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -typename NumTraits<T>::Real abs(const T &x) { +typename internal::enable_if<NumTraits<T>::IsSigned || NumTraits<T>::IsComplex,typename NumTraits<T>::Real>::type +abs(const T &x) { EIGEN_USING_STD_MATH(abs); return abs(x); } -#ifdef __CUDACC__ +template<typename T> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +typename internal::enable_if<!(NumTraits<T>::IsSigned || NumTraits<T>::IsComplex),typename NumTraits<T>::Real>::type +abs(const T &x) { + return x; +} + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_INTEGER_TYPES_UNARY(abs, abs) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(abs, fabs) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float abs(const float &x) { return ::fabsf(x); } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double abs(const double &x) { return ::fabs(x); } + +template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +float abs(const std::complex<float>& x) { + return ::hypotf(x.real(), x.imag()); +} + +template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +double abs(const std::complex<double>& x) { + return ::hypot(x.real(), x.imag()); +} #endif template<typename T> @@ -1045,12 +1301,51 @@ return exp(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(exp, exp) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float exp(const float &x) { return ::expf(x); } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double exp(const double &x) { return ::exp(x); } + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +std::complex<float> exp(const std::complex<float>& x) { + float com = ::expf(x.real()); + float res_real = com * ::cosf(x.imag()); + float res_imag = com * ::sinf(x.imag()); + return std::complex<float>(res_real, res_imag); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +std::complex<double> exp(const std::complex<double>& x) { + double com = ::exp(x.real()); + double res_real = com * ::cos(x.imag()); + double res_imag = com * ::sin(x.imag()); + return std::complex<double>(res_real, res_imag); +} +#endif + +template<typename Scalar> +EIGEN_DEVICE_FUNC +inline EIGEN_MATHFUNC_RETVAL(expm1, Scalar) expm1(const Scalar& x) +{ + return EIGEN_MATHFUNC_IMPL(expm1, Scalar)::run(x); +} + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(expm1, expm1) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +float expm1(const float &x) { return ::expm1f(x); } + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +double expm1(const double &x) { return ::expm1(x); } #endif template<typename T> @@ -1060,7 +1355,11 @@ return cos(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(cos,cos) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float cos(const float &x) { return ::cosf(x); } @@ -1075,7 +1374,11 @@ return sin(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sin, sin) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sin(const float &x) { return ::sinf(x); } @@ -1090,7 +1393,11 @@ return tan(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(tan, tan) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float tan(const float &x) { return ::tanf(x); } @@ -1105,7 +1412,21 @@ return acos(x); } -#ifdef __CUDACC__ +#if EIGEN_HAS_CXX11_MATH +template<typename T> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +T acosh(const T &x) { + EIGEN_USING_STD_MATH(acosh); + return acosh(x); +} +#endif + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(acos, acos) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(acosh, acosh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float acos(const float &x) { return ::acosf(x); } @@ -1120,7 +1441,21 @@ return asin(x); } -#ifdef __CUDACC__ +#if EIGEN_HAS_CXX11_MATH +template<typename T> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +T asinh(const T &x) { + EIGEN_USING_STD_MATH(asinh); + return asinh(x); +} +#endif + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(asin, asin) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(asinh, asinh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float asin(const float &x) { return ::asinf(x); } @@ -1135,7 +1470,21 @@ return atan(x); } -#ifdef __CUDACC__ +#if EIGEN_HAS_CXX11_MATH +template<typename T> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +T atanh(const T &x) { + EIGEN_USING_STD_MATH(atanh); + return atanh(x); +} +#endif + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(atan, atan) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(atanh, atanh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float atan(const float &x) { return ::atanf(x); } @@ -1151,7 +1500,11 @@ return cosh(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(cosh, cosh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float cosh(const float &x) { return ::coshf(x); } @@ -1166,7 +1519,11 @@ return sinh(x); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sinh, sinh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sinh(const float &x) { return ::sinhf(x); } @@ -1181,7 +1538,16 @@ return tanh(x); } -#ifdef __CUDACC__ +#if (!defined(EIGEN_GPUCC)) && EIGEN_FAST_MATH && (!defined(__SYCL_DEVICE_ONLY__)) +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +float tanh(float x) { return internal::generic_fast_tanh_float(x); } +#endif + +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(tanh, tanh) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float tanh(const float &x) { return ::tanhf(x); } @@ -1192,11 +1558,15 @@ template <typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T fmod(const T& a, const T& b) { - EIGEN_USING_STD_MATH(floor); + EIGEN_USING_STD_MATH(fmod); return fmod(a, b); } -#ifdef __CUDACC__ +#if defined(__SYCL_DEVICE_ONLY__) +SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(fmod, fmod) +#endif // defined(__SYCL_DEVICE_ONLY__) + +#if defined(EIGEN_GPUCC) template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float fmod(const float& a, const float& b) { @@ -1210,6 +1580,23 @@ } #endif +#if defined(__SYCL_DEVICE_ONLY__) +#undef SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY +#undef SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY +#undef SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY +#undef SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY +#undef SYCL_SPECIALIZE_INTEGER_TYPES_BINARY +#undef SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY +#undef SYCL_SPECIALIZE_FLOATING_TYPES_BINARY +#undef SYCL_SPECIALIZE_FLOATING_TYPES_UNARY +#undef SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE +#undef SYCL_SPECIALIZE_GEN_UNARY_FUNC +#undef SYCL_SPECIALIZE_UNARY_FUNC +#undef SYCL_SPECIALIZE_GEN1_BINARY_FUNC +#undef SYCL_SPECIALIZE_GEN2_BINARY_FUNC +#undef SYCL_SPECIALIZE_BINARY_FUNC +#endif // defined(__SYCL_DEVICE_ONLY__) + } // end namespace numext namespace internal { @@ -1287,11 +1674,12 @@ struct scalar_fuzzy_default_impl<Scalar, true, false> { typedef typename NumTraits<Scalar>::Real RealScalar; - template<typename OtherScalar> + template<typename OtherScalar> EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const Scalar& x, const OtherScalar& y, const RealScalar& prec) { return numext::abs2(x) <= numext::abs2(y) * prec * prec; } + EIGEN_DEVICE_FUNC static inline bool isApprox(const Scalar& x, const Scalar& y, const RealScalar& prec) { return numext::abs2(x - y) <= numext::mini(numext::abs2(x), numext::abs2(y)) * prec * prec; @@ -1337,13 +1725,13 @@ template<> struct scalar_fuzzy_impl<bool> { typedef bool RealScalar; - + template<typename OtherScalar> EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const bool& x, const bool&, const bool&) { return !x; } - + EIGEN_DEVICE_FUNC static inline bool isApprox(bool x, bool y, bool) { @@ -1355,10 +1743,10 @@ { return (!x) || y; } - + }; - + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/MathFunctionsImpl.h b/Eigen/src/Core/MathFunctionsImpl.h new file mode 100644 index 0000000..a23e93c --- /dev/null +++ b/Eigen/src/Core/MathFunctionsImpl.h
@@ -0,0 +1,97 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATHFUNCTIONSIMPL_H +#define EIGEN_MATHFUNCTIONSIMPL_H + +namespace Eigen { + +namespace internal { + +/** \internal \returns the hyperbolic tan of \a a (coeff-wise) + Doesn't do anything fancy, just a 13/6-degree rational interpolant which + is accurate up to a couple of ulp in the range [-9, 9], outside of which + the tanh(x) = +/-1. + + This implementation works on both scalars and packets. +*/ +template<typename T> +T generic_fast_tanh_float(const T& a_x) +{ + // Clamp the inputs to the range [-9, 9] since anything outside + // this range is +/-1.0f in single-precision. + const T plus_9 = pset1<T>(9.f); + const T minus_9 = pset1<T>(-9.f); + const T x = pmax(pmin(a_x, plus_9), minus_9); + // The monomial coefficients of the numerator polynomial (odd). + const T alpha_1 = pset1<T>(4.89352455891786e-03f); + const T alpha_3 = pset1<T>(6.37261928875436e-04f); + const T alpha_5 = pset1<T>(1.48572235717979e-05f); + const T alpha_7 = pset1<T>(5.12229709037114e-08f); + const T alpha_9 = pset1<T>(-8.60467152213735e-11f); + const T alpha_11 = pset1<T>(2.00018790482477e-13f); + const T alpha_13 = pset1<T>(-2.76076847742355e-16f); + + // The monomial coefficients of the denominator polynomial (even). + const T beta_0 = pset1<T>(4.89352518554385e-03f); + const T beta_2 = pset1<T>(2.26843463243900e-03f); + const T beta_4 = pset1<T>(1.18534705686654e-04f); + const T beta_6 = pset1<T>(1.19825839466702e-06f); + + // Since the polynomials are odd/even, we need x^2. + const T x2 = pmul(x, x); + + // Evaluate the numerator polynomial p. + T p = pmadd(x2, alpha_13, alpha_11); + p = pmadd(x2, p, alpha_9); + p = pmadd(x2, p, alpha_7); + p = pmadd(x2, p, alpha_5); + p = pmadd(x2, p, alpha_3); + p = pmadd(x2, p, alpha_1); + p = pmul(x, p); + + // Evaluate the denominator polynomial p. + T q = pmadd(x2, beta_6, beta_4); + q = pmadd(x2, q, beta_2); + q = pmadd(x2, q, beta_0); + + // Divide the numerator by the denominator. + return pdiv(p, q); +} + +template<typename RealScalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +RealScalar positive_real_hypot(const RealScalar& x, const RealScalar& y) +{ + EIGEN_USING_STD_MATH(sqrt); + RealScalar p, qp; + p = numext::maxi(x,y); + if(p==RealScalar(0)) return RealScalar(0); + qp = numext::mini(y,x) / p; + return p * sqrt(RealScalar(1) + qp*qp); +} + +template<typename Scalar> +struct hypot_impl +{ + typedef typename NumTraits<Scalar>::Real RealScalar; + static EIGEN_DEVICE_FUNC + inline RealScalar run(const Scalar& x, const Scalar& y) + { + EIGEN_USING_STD_MATH(abs); + return positive_real_hypot<RealScalar>(abs(x), abs(y)); + } +}; + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MATHFUNCTIONSIMPL_H
diff --git a/Eigen/src/Core/Matrix.h b/Eigen/src/Core/Matrix.h index a558477..ea5f7da 100644 --- a/Eigen/src/Core/Matrix.h +++ b/Eigen/src/Core/Matrix.h
@@ -27,7 +27,7 @@ default_alignment = compute_default_alignment<_Scalar,max_size>::value, actual_alignment = ((_Options&DontAlign)==0) ? default_alignment : 0, required_alignment = unpacket_traits<PacketScalar>::alignment, - packet_access_bit = packet_traits<_Scalar>::Vectorizable && (actual_alignment>=required_alignment) ? PacketAccessBit : 0 + packet_access_bit = (packet_traits<_Scalar>::Vectorizable && (EIGEN_UNALIGNED_VECTORIZE || (actual_alignment>=required_alignment))) ? PacketAccessBit : 0 }; public: @@ -106,7 +106,7 @@ * \endcode * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_MATRIX_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIX_PLUGIN. * * <i><b>Some notes:</b></i> * @@ -255,30 +255,28 @@ * * \sa resize(Index,Index) */ - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Matrix() : Base() + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Matrix() : Base() { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } // FIXME is it still needed - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(internal::constructor_without_unaligned_array_assert) : Base(internal::constructor_without_unaligned_array_assert()) { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } -#ifdef EIGEN_HAVE_RVALUE_REFERENCES - EIGEN_DEVICE_FUNC - Matrix(Matrix&& other) +#if EIGEN_HAS_RVALUE_REFERENCES + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Matrix(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible<Scalar>::value) : Base(std::move(other)) { Base::_check_template_params(); - if (RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic) - Base::_set_noalias(other); } - EIGEN_DEVICE_FUNC - Matrix& operator=(Matrix&& other) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Matrix& operator=(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable<Scalar>::value) { other.swap(*this); return *this; @@ -289,20 +287,59 @@ // This constructor is for both 1x1 matrices and dynamic vectors template<typename T> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE explicit Matrix(const T& x) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit Matrix(const T& x) { Base::_check_template_params(); Base::template _init1<T>(x); } template<typename T0, typename T1> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Matrix(const T0& x, const T1& y) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Matrix(const T0& x, const T1& y) { Base::_check_template_params(); Base::template _init2<T0,T1>(x, y); } + + #if EIGEN_HAS_CXX11 + /** \copydoc PlainObjectBase(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) + * + * Example: \include Matrix_variadic_ctor_cxx11.cpp + * Output: \verbinclude Matrix_variadic_ctor_cxx11.out + * + * \sa Matrix(const std::initializer_list<std::initializer_list<Scalar>>&) + */ + template <typename... ArgTypes> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + : Base(a0, a1, a2, a3, args...) {} + + /** \brief Constructs a Matrix and initializes it from the coefficients given as initializer-lists grouped by row. \cpp11 + * + * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients: + * + * Example: \include Matrix_initializer_list_23_cxx11.cpp + * Output: \verbinclude Matrix_initializer_list_23_cxx11.out + * + * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is triggered. + * + * In the case of a compile-time column vector, implicit transposition from a single row is allowed. + * Therefore <code>VectorXd{{1,2,3,4,5}}</code> is legal and the more verbose syntax + * <code>RowVectorXd{{1},{2},{3},{4},{5}}</code> can be avoided: + * + * Example: \include Matrix_initializer_list_vector_cxx11.cpp + * Output: \verbinclude Matrix_initializer_list_vector_cxx11.out + * + * In the case of fixed-sized matrices, the initializer list sizes must exactly match the matrix sizes, + * and implicit transposition is allowed for compile-time vectors only. + * + * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) + */ + EIGEN_DEVICE_FUNC + explicit EIGEN_STRONG_INLINE Matrix(const std::initializer_list<std::initializer_list<Scalar>>& list) : Base(list) {} + #endif // end EIGEN_HAS_CXX11 + #else /** \brief Constructs a fixed-sized matrix initialized with coefficients starting at \a data */ EIGEN_DEVICE_FUNC @@ -321,7 +358,8 @@ * \c EIGEN_INITIALIZE_MATRICES_BY_{ZERO,\c NAN} macros (see \ref TopicPreprocessorDirectives). */ EIGEN_STRONG_INLINE explicit Matrix(Index dim); - /** \brief Constructs an initialized 1x1 matrix with the given coefficient */ + /** \brief Constructs an initialized 1x1 matrix with the given coefficient + * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ Matrix(const Scalar& x); /** \brief Constructs an uninitialized matrix with \a rows rows and \a cols columns. * @@ -338,11 +376,14 @@ EIGEN_DEVICE_FUNC Matrix(Index rows, Index cols); - /** \brief Constructs an initialized 2D vector with given coefficients */ + /** \brief Constructs an initialized 2D vector with given coefficients + * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ Matrix(const Scalar& x, const Scalar& y); - #endif + #endif // end EIGEN_PARSED_BY_DOXYGEN - /** \brief Constructs an initialized 3D vector with given coefficients */ + /** \brief Constructs an initialized 3D vector with given coefficients + * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) + */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z) { @@ -352,7 +393,9 @@ m_storage.data()[1] = y; m_storage.data()[2] = z; } - /** \brief Constructs an initialized 4D vector with given coefficients */ + /** \brief Constructs an initialized 4D vector with given coefficients + * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) + */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) {
diff --git a/Eigen/src/Core/MatrixBase.h b/Eigen/src/Core/MatrixBase.h index 1e66b4e..4744e5c 100644 --- a/Eigen/src/Core/MatrixBase.h +++ b/Eigen/src/Core/MatrixBase.h
@@ -41,7 +41,7 @@ * \endcode * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_MATRIXBASE_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIXBASE_PLUGIN. * * \sa \blank \ref TopicClassHierarchy */ @@ -76,12 +76,11 @@ using Base::coeffRef; using Base::lazyAssign; using Base::eval; + using Base::operator-; using Base::operator+=; using Base::operator-=; using Base::operator*=; using Base::operator/=; - using Base::operator*; - using Base::operator/; typedef typename Base::CoeffReturnType CoeffReturnType; typedef typename Base::ConstTransposeReturnType ConstTransposeReturnType; @@ -100,7 +99,7 @@ /** \returns the size of the main diagonal, which is min(rows(),cols()). * \sa rows(), cols(), SizeAtCompileTime. */ EIGEN_DEVICE_FUNC - inline Index diagonalSize() const { return (std::min)(rows(),cols()); } + inline Index diagonalSize() const { return (numext::mini)(rows(),cols()); } typedef typename Base::PlainObject PlainObject; @@ -123,7 +122,7 @@ #endif // not EIGEN_PARSED_BY_DOXYGEN #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::MatrixBase -# include "../plugins/CommonCwiseUnaryOps.h" +#define EIGEN_DOC_UNARY_ADDONS(X,Y) # include "../plugins/CommonCwiseBinaryOps.h" # include "../plugins/MatrixCwiseUnaryOps.h" # include "../plugins/MatrixCwiseBinaryOps.h" @@ -131,6 +130,7 @@ # include EIGEN_MATRIXBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS +#undef EIGEN_DOC_UNARY_ADDONS /** Special case of the template operator=, in order to prevent the compiler * from generating a default operator= (issue hit with g++ 4.1) @@ -160,20 +160,11 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const MatrixBase<OtherDerived>& other); -#ifdef __CUDACC__ template<typename OtherDerived> EIGEN_DEVICE_FUNC - const Product<Derived,OtherDerived,LazyProduct> - operator*(const MatrixBase<OtherDerived> &other) const - { return this->lazyProduct(other); } -#else - - template<typename OtherDerived> const Product<Derived,OtherDerived> operator*(const MatrixBase<OtherDerived> &other) const; -#endif - template<typename OtherDerived> EIGEN_DEVICE_FUNC const Product<Derived,OtherDerived,LazyProduct> @@ -195,7 +186,7 @@ template<typename OtherDerived> EIGEN_DEVICE_FUNC - typename internal::scalar_product_traits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType + typename ScalarBinaryOpTraits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType dot(const MatrixBase<OtherDerived>& other) const; EIGEN_DEVICE_FUNC RealScalar squaredNorm() const; @@ -277,6 +268,8 @@ Derived& setIdentity(); EIGEN_DEVICE_FUNC Derived& setIdentity(Index rows, Index cols); + EIGEN_DEVICE_FUNC Derived& setUnit(Index i); + EIGEN_DEVICE_FUNC Derived& setUnit(Index newSize, Index i); bool isIdentity(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const; bool isDiagonal(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const; @@ -294,7 +287,7 @@ * fuzzy comparison such as isApprox() * \sa isApprox(), operator!= */ template<typename OtherDerived> - inline bool operator==(const MatrixBase<OtherDerived>& other) const + EIGEN_DEVICE_FUNC inline bool operator==(const MatrixBase<OtherDerived>& other) const { return cwiseEqual(other).all(); } /** \returns true if at least one pair of coefficients of \c *this and \a other are not exactly equal to each other. @@ -302,10 +295,10 @@ * fuzzy comparison such as isApprox() * \sa isApprox(), operator== */ template<typename OtherDerived> - inline bool operator!=(const MatrixBase<OtherDerived>& other) const + EIGEN_DEVICE_FUNC inline bool operator!=(const MatrixBase<OtherDerived>& other) const { return cwiseNotEqual(other).any(); } - NoAlias<Derived,Eigen::MatrixBase > noalias(); + NoAlias<Derived,Eigen::MatrixBase > EIGEN_DEVICE_FUNC noalias(); // TODO forceAlignedAccess is temporarily disabled // Need to find a nicer workaround. @@ -330,12 +323,9 @@ /////////// LU module /////////// - EIGEN_DEVICE_FUNC inline const FullPivLU<PlainObject> fullPivLu() const; - EIGEN_DEVICE_FUNC inline const PartialPivLU<PlainObject> partialPivLu() const; - EIGEN_DEVICE_FUNC inline const PartialPivLU<PlainObject> lu() const; EIGEN_DEVICE_FUNC @@ -348,12 +338,15 @@ bool& invertible, const RealScalar& absDeterminantThreshold = NumTraits<Scalar>::dummy_precision() ) const; + template<typename ResultType> inline void computeInverseWithCheck( ResultType& inverse, bool& invertible, const RealScalar& absDeterminantThreshold = NumTraits<Scalar>::dummy_precision() ) const; + + EIGEN_DEVICE_FUNC Scalar determinant() const; /////////// Cholesky module /////////// @@ -383,7 +376,7 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN /// \internal helper struct to form the return type of the cross product template<typename OtherDerived> struct cross_product_return_type { - typedef typename internal::scalar_product_traits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType Scalar; + typedef typename ScalarBinaryOpTraits<typename internal::traits<Derived>::Scalar,typename internal::traits<OtherDerived>::Scalar>::ReturnType Scalar; typedef Matrix<Scalar,MatrixBase::RowsAtCompileTime,MatrixBase::ColsAtCompileTime> type; }; #endif // EIGEN_PARSED_BY_DOXYGEN @@ -403,13 +396,14 @@ EIGEN_DEVICE_FUNC inline PlainObject unitOrthogonal(void) const; + EIGEN_DEVICE_FUNC inline Matrix<Scalar,3,1> eulerAngles(Index a0, Index a1, Index a2) const; - inline ScalarMultipleReturnType operator*(const UniformScaling<Scalar>& s) const; // put this as separate enum value to work around possible GCC 4.3 bug (?) enum { HomogeneousReturnTypeDirection = ColsAtCompileTime==1&&RowsAtCompileTime==1 ? ((internal::traits<Derived>::Flags&RowMajorBit)==RowMajorBit ? Horizontal : Vertical) : ColsAtCompileTime==1 ? Vertical : Horizontal }; typedef Homogeneous<Derived, HomogeneousReturnTypeDirection> HomogeneousReturnType; + EIGEN_DEVICE_FUNC inline HomogeneousReturnType homogeneous() const; enum { @@ -418,22 +412,25 @@ typedef Block<const Derived, internal::traits<Derived>::ColsAtCompileTime==1 ? SizeMinusOne : 1, internal::traits<Derived>::ColsAtCompileTime==1 ? 1 : SizeMinusOne> ConstStartMinusOne; - typedef CwiseUnaryOp<internal::scalar_quotient1_op<typename internal::traits<Derived>::Scalar>, - const ConstStartMinusOne > HNormalizedReturnType; - + typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(ConstStartMinusOne,Scalar,quotient) HNormalizedReturnType; + EIGEN_DEVICE_FUNC inline const HNormalizedReturnType hnormalized() const; ////////// Householder module /////////// + EIGEN_DEVICE_FUNC void makeHouseholderInPlace(Scalar& tau, RealScalar& beta); template<typename EssentialPart> + EIGEN_DEVICE_FUNC void makeHouseholder(EssentialPart& essential, Scalar& tau, RealScalar& beta) const; template<typename EssentialPart> + EIGEN_DEVICE_FUNC void applyHouseholderOnTheLeft(const EssentialPart& essential, const Scalar& tau, Scalar* workspace); template<typename EssentialPart> + EIGEN_DEVICE_FUNC void applyHouseholderOnTheRight(const EssentialPart& essential, const Scalar& tau, Scalar* workspace); @@ -441,8 +438,10 @@ ///////// Jacobi module ///////// template<typename OtherScalar> + EIGEN_DEVICE_FUNC void applyOnTheLeft(Index p, Index q, const JacobiRotation<OtherScalar>& j); template<typename OtherScalar> + EIGEN_DEVICE_FUNC void applyOnTheRight(Index p, Index q, const JacobiRotation<OtherScalar>& j); ///////// SparseCore module ///////// @@ -457,16 +456,29 @@ ///////// MatrixFunctions module ///////// typedef typename internal::stem_function<Scalar>::type StemFunction; - const MatrixExponentialReturnValue<Derived> exp() const; +#define EIGEN_MATRIX_FUNCTION(ReturnType, Name, Description) \ + /** \returns an expression of the matrix Description of \c *this. \brief This function requires the <a href="unsupported/group__MatrixFunctions__Module.html"> unsupported MatrixFunctions module</a>. To compute the coefficient-wise Description use ArrayBase::##Name . */ \ + const ReturnType<Derived> Name() const; +#define EIGEN_MATRIX_FUNCTION_1(ReturnType, Name, Description, Argument) \ + /** \returns an expression of the matrix Description of \c *this. \brief This function requires the <a href="unsupported/group__MatrixFunctions__Module.html"> unsupported MatrixFunctions module</a>. To compute the coefficient-wise Description use ArrayBase::##Name . */ \ + const ReturnType<Derived> Name(Argument) const; + + EIGEN_MATRIX_FUNCTION(MatrixExponentialReturnValue, exp, exponential) + /** \brief Helper function for the <a href="unsupported/group__MatrixFunctions__Module.html"> unsupported MatrixFunctions module</a>.*/ const MatrixFunctionReturnValue<Derived> matrixFunction(StemFunction f) const; - const MatrixFunctionReturnValue<Derived> cosh() const; - const MatrixFunctionReturnValue<Derived> sinh() const; - const MatrixFunctionReturnValue<Derived> cos() const; - const MatrixFunctionReturnValue<Derived> sin() const; - const MatrixSquareRootReturnValue<Derived> sqrt() const; - const MatrixLogarithmReturnValue<Derived> log() const; - const MatrixPowerReturnValue<Derived> pow(const RealScalar& p) const; - const MatrixComplexPowerReturnValue<Derived> pow(const std::complex<RealScalar>& p) const; + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cosh, hyperbolic cosine) + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sinh, hyperbolic sine) +#if EIGEN_HAS_CXX11_MATH + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, atanh, inverse hyperbolic cosine) + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, acosh, inverse hyperbolic cosine) + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, asinh, inverse hyperbolic sine) +#endif + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cos, cosine) + EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sin, sine) + EIGEN_MATRIX_FUNCTION(MatrixSquareRootReturnValue, sqrt, square root) + EIGEN_MATRIX_FUNCTION(MatrixLogarithmReturnValue, log, logarithm) + EIGEN_MATRIX_FUNCTION_1(MatrixPowerReturnValue, pow, power to \c p, const RealScalar& p) + EIGEN_MATRIX_FUNCTION_1(MatrixComplexPowerReturnValue, pow, power to \c p, const std::complex<RealScalar>& p) protected: EIGEN_DEVICE_FUNC MatrixBase() : Base() {}
diff --git a/Eigen/src/Core/NestByValue.h b/Eigen/src/Core/NestByValue.h index 13adf07..239bbba 100644 --- a/Eigen/src/Core/NestByValue.h +++ b/Eigen/src/Core/NestByValue.h
@@ -16,7 +16,11 @@ namespace internal { template<typename ExpressionType> struct traits<NestByValue<ExpressionType> > : public traits<ExpressionType> -{}; +{ + enum { + Flags = traits<ExpressionType>::Flags & ~NestByRefBit + }; +}; } /** \class NestByValue @@ -43,55 +47,11 @@ EIGEN_DEVICE_FUNC inline Index rows() const { return m_expression.rows(); } EIGEN_DEVICE_FUNC inline Index cols() const { return m_expression.cols(); } - EIGEN_DEVICE_FUNC inline Index outerStride() const { return m_expression.outerStride(); } - EIGEN_DEVICE_FUNC inline Index innerStride() const { return m_expression.innerStride(); } - - EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index row, Index col) const - { - return m_expression.coeff(row, col); - } - - EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) - { - return m_expression.const_cast_derived().coeffRef(row, col); - } - - EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index index) const - { - return m_expression.coeff(index); - } - - EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index index) - { - return m_expression.const_cast_derived().coeffRef(index); - } - - template<int LoadMode> - inline const PacketScalar packet(Index row, Index col) const - { - return m_expression.template packet<LoadMode>(row, col); - } - - template<int LoadMode> - inline void writePacket(Index row, Index col, const PacketScalar& x) - { - m_expression.const_cast_derived().template writePacket<LoadMode>(row, col, x); - } - - template<int LoadMode> - inline const PacketScalar packet(Index index) const - { - return m_expression.template packet<LoadMode>(index); - } - - template<int LoadMode> - inline void writePacket(Index index, const PacketScalar& x) - { - m_expression.const_cast_derived().template writePacket<LoadMode>(index, x); - } EIGEN_DEVICE_FUNC operator const ExpressionType&() const { return m_expression; } + EIGEN_DEVICE_FUNC const ExpressionType& nestedExpression() const { return m_expression; } + protected: const ExpressionType m_expression; }; @@ -99,12 +59,27 @@ /** \returns an expression of the temporary version of *this. */ template<typename Derived> -inline const NestByValue<Derived> +EIGEN_DEVICE_FUNC inline const NestByValue<Derived> DenseBase<Derived>::nestByValue() const { return NestByValue<Derived>(derived()); } +namespace internal { + +// Evaluator of Solve -> eval into a temporary +template<typename ArgType> +struct evaluator<NestByValue<ArgType> > + : public evaluator<ArgType> +{ + typedef evaluator<ArgType> Base; + + EIGEN_DEVICE_FUNC explicit evaluator(const NestByValue<ArgType>& xpr) + : Base(xpr.nestedExpression()) + {} +}; +} + } // end namespace Eigen #endif // EIGEN_NESTBYVALUE_H
diff --git a/Eigen/src/Core/NoAlias.h b/Eigen/src/Core/NoAlias.h index ffb673c..570283d 100644 --- a/Eigen/src/Core/NoAlias.h +++ b/Eigen/src/Core/NoAlias.h
@@ -33,13 +33,14 @@ public: typedef typename ExpressionType::Scalar Scalar; + EIGEN_DEVICE_FUNC explicit NoAlias(ExpressionType& expression) : m_expression(expression) {} template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator=(const StorageBase<OtherDerived>& other) { - call_assignment_no_alias(m_expression, other.derived(), internal::assign_op<Scalar>()); + call_assignment_no_alias(m_expression, other.derived(), internal::assign_op<Scalar,typename OtherDerived::Scalar>()); return m_expression; } @@ -47,7 +48,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator+=(const StorageBase<OtherDerived>& other) { - call_assignment_no_alias(m_expression, other.derived(), internal::add_assign_op<Scalar>()); + call_assignment_no_alias(m_expression, other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); return m_expression; } @@ -55,7 +56,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator-=(const StorageBase<OtherDerived>& other) { - call_assignment_no_alias(m_expression, other.derived(), internal::sub_assign_op<Scalar>()); + call_assignment_no_alias(m_expression, other.derived(), internal::sub_assign_op<Scalar,typename OtherDerived::Scalar>()); return m_expression; } @@ -74,10 +75,10 @@ * * More precisely, noalias() allows to bypass the EvalBeforeAssignBit flag. * Currently, even though several expressions may alias, only product - * expressions have this flag. Therefore, noalias() is only usefull when + * expressions have this flag. Therefore, noalias() is only useful when * the source expression contains a matrix product. * - * Here are some examples where noalias is usefull: + * Here are some examples where noalias is useful: * \code * D.noalias() = A * B; * D.noalias() += A.transpose() * B; @@ -98,7 +99,7 @@ * \sa class NoAlias */ template<typename Derived> -NoAlias<Derived,MatrixBase> MatrixBase<Derived>::noalias() +NoAlias<Derived,MatrixBase> EIGEN_DEVICE_FUNC MatrixBase<Derived>::noalias() { return NoAlias<Derived, Eigen::MatrixBase >(derived()); }
diff --git a/Eigen/src/Core/NumTraits.h b/Eigen/src/Core/NumTraits.h index e065fa7..b053cff 100644 --- a/Eigen/src/Core/NumTraits.h +++ b/Eigen/src/Core/NumTraits.h
@@ -12,6 +12,71 @@ namespace Eigen { +namespace internal { + +// default implementation of digits10(), based on numeric_limits if specialized, +// 0 for integer types, and log10(epsilon()) otherwise. +template< typename T, + bool use_numeric_limits = std::numeric_limits<T>::is_specialized, + bool is_integer = NumTraits<T>::IsInteger> +struct default_digits10_impl +{ + EIGEN_DEVICE_FUNC + static int run() { return std::numeric_limits<T>::digits10; } +}; + +template<typename T> +struct default_digits10_impl<T,false,false> // Floating point +{ + EIGEN_DEVICE_FUNC + static int run() { + using std::log10; + using std::ceil; + typedef typename NumTraits<T>::Real Real; + return int(ceil(-log10(NumTraits<Real>::epsilon()))); + } +}; + +template<typename T> +struct default_digits10_impl<T,false,true> // Integer +{ + EIGEN_DEVICE_FUNC + static int run() { return 0; } +}; + + +// default implementation of digits(), based on numeric_limits if specialized, +// 0 for integer types, and log2(epsilon()) otherwise. +template< typename T, + bool use_numeric_limits = std::numeric_limits<T>::is_specialized, + bool is_integer = NumTraits<T>::IsInteger> +struct default_digits_impl +{ + EIGEN_DEVICE_FUNC + static int run() { return std::numeric_limits<T>::digits; } +}; + +template<typename T> +struct default_digits_impl<T,false,false> // Floating point +{ + EIGEN_DEVICE_FUNC + static int run() { + using std::log; + using std::ceil; + typedef typename NumTraits<T>::Real Real; + return int(ceil(-log(NumTraits<Real>::epsilon())/log(static_cast<Real>(2)))); + } +}; + +template<typename T> +struct default_digits_impl<T,false,true> // Integer +{ + EIGEN_DEVICE_FUNC + static int run() { return 0; } +}; + +} // end namespace internal + /** \class NumTraits * \ingroup Core_Module * @@ -22,14 +87,16 @@ * This class stores enums, typedefs and static methods giving information about a numeric type. * * The provided data consists of: - * \li A typedef \a Real, giving the "real part" type of \a T. If \a T is already real, - * then \a Real is just a typedef to \a T. If \a T is \c std::complex<U> then \a Real + * \li A typedef \c Real, giving the "real part" type of \a T. If \a T is already real, + * then \c Real is just a typedef to \a T. If \a T is \c std::complex<U> then \c Real * is a typedef to \a U. - * \li A typedef \a NonInteger, giving the type that should be used for operations producing non-integral values, + * \li A typedef \c NonInteger, giving the type that should be used for operations producing non-integral values, * such as quotients, square roots, etc. If \a T is a floating-point type, then this typedef just gives * \a T again. Note however that many Eigen functions such as internal::sqrt simply refuse to * take integers. Outside of a few cases, Eigen doesn't do automatic type promotion. Thus, this typedef is * only intended as a helper for code that needs to explicitly promote types. + * \li A typedef \c Literal giving the type to use for numeric literals such as "2" or "0.5". For instance, for \c std::complex<U>, Literal is defined as \c U. + * Of course, this type must be fully compatible with \a T. In doubt, just use \a T here. * \li A typedef \a Nested giving the type to use to nest a value inside of the expression tree. If you don't know what * this means, just use \a T here. * \li An enum value \a IsComplex. It is equal to 1 if \a T is a \c std::complex @@ -38,14 +105,18 @@ * and to \c 0 otherwise. * \li Enum values ReadCost, AddCost and MulCost representing a rough estimate of the number of CPU cycles needed * to by move / add / mul instructions respectively, assuming the data is already stored in CPU registers. - * Stay vague here. No need to do architecture-specific stuff. + * Stay vague here. No need to do architecture-specific stuff. If you don't know what this means, just use \c Eigen::HugeCost. * \li An enum value \a IsSigned. It is equal to \c 1 if \a T is a signed type and to 0 if \a T is unsigned. * \li An enum value \a RequireInitialization. It is equal to \c 1 if the constructor of the numeric type \a T must * be called, and to 0 if it is safe not to call it. Default is 0 if \a T is an arithmetic type, and 1 otherwise. - * \li An epsilon() function which, unlike std::numeric_limits::epsilon(), returns a \a Real instead of a \a T. + * \li An epsilon() function which, unlike <a href="http://en.cppreference.com/w/cpp/types/numeric_limits/epsilon">std::numeric_limits::epsilon()</a>, + * it returns a \a Real instead of a \a T. * \li A dummy_precision() function returning a weak epsilon value. It is mainly used as a default * value by the fuzzy comparison operators. * \li highest() and lowest() functions returning the highest and lowest possible values respectively. + * \li digits10() function returning the number of decimal digits that can be represented without change. This is + * the analogue of <a href="http://en.cppreference.com/w/cpp/types/numeric_limits/digits10">std::numeric_limits<T>::digits10</a> + * which is used as the default implementation if specialized. */ template<typename T> struct GenericNumTraits @@ -60,23 +131,6 @@ MulCost = 1 }; - // Division is messy but important, because it is expensive and throughput - // varies significantly. The following numbers are based on min division - // throughput on Haswell. - template<bool Vectorized> - struct Div { - enum { -#ifdef EIGEN_VECTORIZE_AVX - AVX = true, -#else - AVX = false, -#endif - Cost = IsInteger ? (sizeof(T) == 8 ? (IsSigned ? 24 : 21) : (IsSigned ? 8 : 9)): - Vectorized ? (sizeof(T) == 8 ? (AVX ? 16 : 8) : (AVX ? 14 : 7)) : 8 - }; - }; - - typedef T Real; typedef typename internal::conditional< IsInteger, @@ -84,12 +138,26 @@ T >::type NonInteger; typedef T Nested; + typedef T Literal; EIGEN_DEVICE_FUNC static inline Real epsilon() { return numext::numeric_limits<T>::epsilon(); } + + EIGEN_DEVICE_FUNC + static inline int digits10() + { + return internal::default_digits10_impl<T>::run(); + } + + EIGEN_DEVICE_FUNC + static inline int digits() + { + return internal::default_digits_impl<T>::run(); + } + EIGEN_DEVICE_FUNC static inline Real dummy_precision() { @@ -145,6 +213,7 @@ : GenericNumTraits<std::complex<_Real> > { typedef _Real Real; + typedef typename NumTraits<_Real>::Literal Literal; enum { IsComplex = 1, RequireInitialization = NumTraits<_Real>::RequireInitialization, @@ -157,6 +226,8 @@ static inline Real epsilon() { return NumTraits<Real>::epsilon(); } EIGEN_DEVICE_FUNC static inline Real dummy_precision() { return NumTraits<Real>::dummy_precision(); } + EIGEN_DEVICE_FUNC + static inline int digits10() { return NumTraits<Real>::digits10(); } }; template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> @@ -168,6 +239,7 @@ typedef typename NumTraits<Scalar>::NonInteger NonIntegerScalar; typedef Array<NonIntegerScalar, Rows, Cols, Options, MaxRows, MaxCols> NonInteger; typedef ArrayType & Nested; + typedef typename NumTraits<Scalar>::Literal Literal; enum { IsComplex = NumTraits<Scalar>::IsComplex, @@ -183,8 +255,34 @@ static inline RealScalar epsilon() { return NumTraits<RealScalar>::epsilon(); } EIGEN_DEVICE_FUNC static inline RealScalar dummy_precision() { return NumTraits<RealScalar>::dummy_precision(); } + + static inline int digits10() { return NumTraits<Scalar>::digits10(); } }; +template<> struct NumTraits<std::string> + : GenericNumTraits<std::string> +{ + enum { + RequireInitialization = 1, + ReadCost = HugeCost, + AddCost = HugeCost, + MulCost = HugeCost + }; + + static inline int digits10() { return 0; } + +private: + static inline std::string epsilon(); + static inline std::string dummy_precision(); + static inline std::string lowest(); + static inline std::string highest(); + static inline std::string infinity(); + static inline std::string quiet_NaN(); +}; + +// Empty specialization for void to allow template specialization based on NumTraits<T>::Real with T==void and SFINAE. +template<> struct NumTraits<void> {}; + } // end namespace Eigen #endif // EIGEN_NUMTRAITS_H
diff --git a/Eigen/src/Core/PartialReduxEvaluator.h b/Eigen/src/Core/PartialReduxEvaluator.h new file mode 100644 index 0000000..0be6942 --- /dev/null +++ b/Eigen/src/Core/PartialReduxEvaluator.h
@@ -0,0 +1,232 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2011-2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PARTIALREDUX_H +#define EIGEN_PARTIALREDUX_H + +namespace Eigen { + +namespace internal { + + +/*************************************************************************** +* +* This file provides evaluators for partial reductions. +* There are two modes: +* +* - scalar path: simply calls the respective function on the column or row. +* -> nothing special here, all the tricky part is handled by the return +* types of VectorwiseOp's members. They embed the functor calling the +* respective DenseBase's member function. +* +* - vectorized path: implements a packet-wise reductions followed by +* some (optional) processing of the outcome, e.g., division by n for mean. +* +* For the vectorized path let's observe that the packet-size and outer-unrolling +* are both decided by the assignement logic. So all we have to do is to decide +* on the inner unrolling. +* +* For the unrolling, we can reuse "internal::redux_vec_unroller" from Redux.h, +* but be need to be careful to specify correct increment. +* +***************************************************************************/ + + +/* logic deciding a strategy for unrolling of vectorized paths */ +template<typename Func, typename Evaluator> +struct packetwise_redux_traits +{ + enum { + OuterSize = int(Evaluator::IsRowMajor) ? Evaluator::RowsAtCompileTime : Evaluator::ColsAtCompileTime, + Cost = OuterSize == Dynamic ? HugeCost + : OuterSize * Evaluator::CoeffReadCost + (OuterSize-1) * functor_traits<Func>::Cost, + Unrolling = Cost <= EIGEN_UNROLLING_LIMIT ? CompleteUnrolling : NoUnrolling + }; + +}; + +/* Value to be returned when size==0 , by default let's return 0 */ +template<typename PacketType,typename Func> +EIGEN_DEVICE_FUNC +PacketType packetwise_redux_empty_value(const Func& ) { return pset1<PacketType>(0); } + +/* For products the default is 1 */ +template<typename PacketType,typename Scalar> +EIGEN_DEVICE_FUNC +PacketType packetwise_redux_empty_value(const scalar_product_op<Scalar,Scalar>& ) { return pset1<PacketType>(1); } + +/* Perform the actual reduction */ +template<typename Func, typename Evaluator, + int Unrolling = packetwise_redux_traits<Func, Evaluator>::Unrolling +> +struct packetwise_redux_impl; + +/* Perform the actual reduction with unrolling */ +template<typename Func, typename Evaluator> +struct packetwise_redux_impl<Func, Evaluator, CompleteUnrolling> +{ + typedef redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> Base; + typedef typename Evaluator::Scalar Scalar; + + template<typename PacketType> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + PacketType run(const Evaluator &eval, const Func& func, Index /*size*/) + { + return redux_vec_unroller<Func, Evaluator, 0, packetwise_redux_traits<Func, Evaluator>::OuterSize>::template run<PacketType>(eval,func); + } +}; + +/* Add a specialization of redux_vec_unroller for size==0 at compiletime. + * This specialization is not required for general reductions, which is + * why it is defined here. + */ +template<typename Func, typename Evaluator, int Start> +struct redux_vec_unroller<Func, Evaluator, Start, 0> +{ + template<typename PacketType> + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &, const Func& f) + { + return packetwise_redux_empty_value<PacketType>(f); + } +}; + +/* Perform the actual reduction for dynamic sizes */ +template<typename Func, typename Evaluator> +struct packetwise_redux_impl<Func, Evaluator, NoUnrolling> +{ + typedef typename Evaluator::Scalar Scalar; + typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar; + + template<typename PacketType> + EIGEN_DEVICE_FUNC + static PacketType run(const Evaluator &eval, const Func& func, Index size) + { + if(size==0) + return packetwise_redux_empty_value<PacketType>(func); + + const Index size4 = (size-1)&(~3); + PacketType p = eval.template packetByOuterInner<Unaligned,PacketType>(0,0); + Index i = 1; + // This loop is optimized for instruction pipelining: + // - each iteration generates two independent instructions + // - thanks to branch prediction and out-of-order execution we have independent instructions across loops + for(; i<size4; i+=4) + p = func.packetOp(p, + func.packetOp( + func.packetOp(eval.template packetByOuterInner<Unaligned,PacketType>(i+0,0),eval.template packetByOuterInner<Unaligned,PacketType>(i+1,0)), + func.packetOp(eval.template packetByOuterInner<Unaligned,PacketType>(i+2,0),eval.template packetByOuterInner<Unaligned,PacketType>(i+3,0)))); + for(; i<size; ++i) + p = func.packetOp(p, eval.template packetByOuterInner<Unaligned,PacketType>(i,0)); + return p; + } +}; + +template< typename ArgType, typename MemberOp, int Direction> +struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> > + : evaluator_base<PartialReduxExpr<ArgType, MemberOp, Direction> > +{ + typedef PartialReduxExpr<ArgType, MemberOp, Direction> XprType; + typedef typename internal::nested_eval<ArgType,1>::type ArgTypeNested; + typedef typename internal::add_const_on_value_type<ArgTypeNested>::type ConstArgTypeNested; + typedef typename internal::remove_all<ArgTypeNested>::type ArgTypeNestedCleaned; + typedef typename ArgType::Scalar InputScalar; + typedef typename XprType::Scalar Scalar; + enum { + TraversalSize = Direction==int(Vertical) ? int(ArgType::RowsAtCompileTime) : int(ArgType::ColsAtCompileTime) + }; + typedef typename MemberOp::template Cost<int(TraversalSize)> CostOpType; + enum { + CoeffReadCost = TraversalSize==Dynamic ? HugeCost + : TraversalSize==0 ? 1 + : TraversalSize * evaluator<ArgType>::CoeffReadCost + int(CostOpType::value), + + _ArgFlags = evaluator<ArgType>::Flags, + + _Vectorizable = bool(int(_ArgFlags)&PacketAccessBit) + && bool(MemberOp::Vectorizable) + && (Direction==int(Vertical) ? bool(_ArgFlags&RowMajorBit) : (_ArgFlags&RowMajorBit)==0) + && (TraversalSize!=0), + + Flags = (traits<XprType>::Flags&RowMajorBit) + | (evaluator<ArgType>::Flags&(HereditaryBits&(~RowMajorBit))) + | (_Vectorizable ? PacketAccessBit : 0) + | LinearAccessBit, + + Alignment = 0 // FIXME this will need to be improved once PartialReduxExpr is vectorized + }; + + EIGEN_DEVICE_FUNC explicit evaluator(const XprType xpr) + : m_arg(xpr.nestedExpression()), m_functor(xpr.functor()) + { + EIGEN_INTERNAL_CHECK_COST_VALUE(TraversalSize==Dynamic ? HugeCost : (TraversalSize==0 ? 1 : int(CostOpType::value))); + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + typedef typename XprType::CoeffReturnType CoeffReturnType; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar coeff(Index i, Index j) const + { + return coeff(Direction==Vertical ? j : i); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar coeff(Index index) const + { + return m_functor(m_arg.template subVector<DirectionType(Direction)>(index)); + } + + template<int LoadMode,typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + PacketType packet(Index i, Index j) const + { + return packet<LoadMode,PacketType>(Direction==Vertical ? j : i); + } + + template<int LoadMode,typename PacketType> + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC + PacketType packet(Index idx) const + { + enum { PacketSize = internal::unpacket_traits<PacketType>::size }; + typedef Block<const ArgTypeNestedCleaned, + Direction==Vertical ? int(ArgType::RowsAtCompileTime) : int(PacketSize), + Direction==Vertical ? int(PacketSize) : int(ArgType::ColsAtCompileTime), + true /* InnerPanel */> PanelType; + + PanelType panel(m_arg, + Direction==Vertical ? 0 : idx, + Direction==Vertical ? idx : 0, + Direction==Vertical ? m_arg.rows() : Index(PacketSize), + Direction==Vertical ? Index(PacketSize) : m_arg.cols()); + + // FIXME + // See bug 1612, currently if PacketSize==1 (i.e. complex<double> with 128bits registers) then the storage-order of panel get reversed + // and methods like packetByOuterInner do not make sense anymore in this context. + // So let's just by pass "vectorization" in this case: + if(PacketSize==1) + return internal::pset1<PacketType>(coeff(idx)); + + typedef typename internal::redux_evaluator<PanelType> PanelEvaluator; + PanelEvaluator panel_eval(panel); + typedef typename MemberOp::BinaryOp BinaryOp; + PacketType p = internal::packetwise_redux_impl<BinaryOp,PanelEvaluator>::template run<PacketType>(panel_eval,m_functor.binaryFunc(),m_arg.outerSize()); + return p; + } + +protected: + ConstArgTypeNested m_arg; + const MemberOp m_functor; +}; + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_PARTIALREDUX_H
diff --git a/Eigen/src/Core/PermutationMatrix.h b/Eigen/src/Core/PermutationMatrix.h index b1fb455..acd0853 100644 --- a/Eigen/src/Core/PermutationMatrix.h +++ b/Eigen/src/Core/PermutationMatrix.h
@@ -99,13 +99,13 @@ #endif /** \returns the number of rows */ - inline Index rows() const { return Index(indices().size()); } + inline EIGEN_DEVICE_FUNC Index rows() const { return Index(indices().size()); } /** \returns the number of columns */ - inline Index cols() const { return Index(indices().size()); } + inline EIGEN_DEVICE_FUNC Index cols() const { return Index(indices().size()); } /** \returns the size of a side of the respective square matrix, i.e., the number of indices */ - inline Index size() const { return Index(indices().size()); } + inline EIGEN_DEVICE_FUNC Index size() const { return Index(indices().size()); } #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename DenseDerived>
diff --git a/Eigen/src/Core/PlainObjectBase.h b/Eigen/src/Core/PlainObjectBase.h index b7a4fce..bae186e 100644 --- a/Eigen/src/Core/PlainObjectBase.h +++ b/Eigen/src/Core/PlainObjectBase.h
@@ -41,7 +41,7 @@ { // http://hg.mozilla.org/mozilla-central/file/6c8a909977d3/xpcom/ds/CheckedInt.h#l242 // we assume Index is signed - Index max_index = (size_t(1) << (8 * sizeof(Index) - 1)) - 1; // assume Index is signed + Index max_index = (std::size_t(1) << (8 * sizeof(Index) - 1)) - 1; // assume Index is signed bool error = (rows == 0 || cols == 0) ? false : (rows > max_index / cols); if (error) @@ -58,34 +58,41 @@ } // end namespace internal +#ifdef EIGEN_PARSED_BY_DOXYGEN +namespace doxygen { + +// This is a workaround to doxygen not being able to understand the inheritance logic +// when it is hidden by the dense_xpr_base helper struct. +// Moreover, doxygen fails to include members that are not documented in the declaration body of +// MatrixBase if we inherits MatrixBase<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> >, +// this is why we simply inherits MatrixBase, though this does not make sense. + +/** This class is just a workaround for Doxygen and it does not not actually exist. */ +template<typename Derived> struct dense_xpr_base_dispatcher; +/** This class is just a workaround for Doxygen and it does not not actually exist. */ +template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> +struct dense_xpr_base_dispatcher<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > + : public MatrixBase {}; +/** This class is just a workaround for Doxygen and it does not not actually exist. */ +template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> +struct dense_xpr_base_dispatcher<Array<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > + : public ArrayBase {}; + +} // namespace doxygen + /** \class PlainObjectBase + * \ingroup Core_Module * \brief %Dense storage base class for matrices and arrays. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_PLAINOBJECTBASE_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_PLAINOBJECTBASE_PLUGIN. + * + * \tparam Derived is the derived type, e.g., a Matrix or Array * * \sa \ref TopicClassHierarchy */ -#ifdef EIGEN_PARSED_BY_DOXYGEN -namespace internal { - -// this is a workaround to doxygen not being able to understand the inheritance logic -// when it is hidden by the dense_xpr_base helper struct. -/** This class is just a workaround for Doxygen and it does not not actually exist. */ -template<typename Derived> struct dense_xpr_base_dispatcher_for_doxygen;// : public MatrixBase<Derived> {}; -/** This class is just a workaround for Doxygen and it does not not actually exist. */ -template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> -struct dense_xpr_base_dispatcher_for_doxygen<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > - : public MatrixBase<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > {}; -/** This class is just a workaround for Doxygen and it does not not actually exist. */ -template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> -struct dense_xpr_base_dispatcher_for_doxygen<Array<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > - : public ArrayBase<Array<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > {}; - -} // namespace internal - template<typename Derived> -class PlainObjectBase : public internal::dense_xpr_base_dispatcher_for_doxygen<Derived> +class PlainObjectBase : public doxygen::dense_xpr_base_dispatcher<Derived> #else template<typename Derived> class PlainObjectBase : public internal::dense_xpr_base<Derived>::type @@ -97,7 +104,7 @@ typedef typename internal::traits<Derived>::StorageKind StorageKind; typedef typename internal::traits<Derived>::Scalar Scalar; - + typedef typename internal::packet_traits<Scalar>::type PacketScalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Derived DenseType; @@ -145,6 +152,10 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index cols() const { return m_storage.cols(); } + /** This is an overloaded version of DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index,Index) const + * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. + * + * See DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index) const for details. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index rowId, Index colId) const { @@ -154,12 +165,20 @@ return m_storage.data()[rowId + colId * m_storage.rows()]; } + /** This is an overloaded version of DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index) const + * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. + * + * See DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index) const for details. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index index) const { return m_storage.data()[index]; } + /** This is an overloaded version of DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index,Index) const + * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. + * + * See DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index,Index) const for details. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index rowId, Index colId) { @@ -169,12 +188,18 @@ return m_storage.data()[rowId + colId * m_storage.rows()]; } + /** This is an overloaded version of DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index) const + * provided to by-pass the creation of an evaluator of the expression, thus saving compilation efforts. + * + * See DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index) const for details. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_storage.data()[index]; } + /** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index). + * It is provided for convenience. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index rowId, Index colId) const { @@ -184,6 +209,8 @@ return m_storage.data()[rowId + colId * m_storage.rows()]; } + /** This is the const version of coeffRef(Index) which is thus synonym of coeff(Index). + * It is provided for convenience. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index index) const { @@ -331,7 +358,7 @@ * remain row-vectors and vectors remain vectors. */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resizeLike(const EigenBase<OtherDerived>& _other) { const OtherDerived& other = _other.derived(); @@ -356,7 +383,7 @@ * of rows and/or of columns, you can use conservativeResize(NoChange_t, Index) or * conservativeResize(Index, NoChange_t). * - * Matrices are resized relative to the top-left element. In case values need to be + * Matrices are resized relative to the top-left element. In case values need to be * appended to the matrix they will be uninitialized. */ EIGEN_DEVICE_FUNC @@ -413,7 +440,7 @@ * of rows and/or of columns, you can use conservativeResize(NoChange_t, Index) or * conservativeResize(Index, NoChange_t). * - * Matrices are resized relative to the top-left element. In case values need to be + * Matrices are resized relative to the top-left element. In case values need to be * appended to the matrix they will copied from \c other. */ template<typename OtherDerived> @@ -471,15 +498,15 @@ } #endif -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC - PlainObjectBase(PlainObjectBase&& other) + PlainObjectBase(PlainObjectBase&& other) EIGEN_NOEXCEPT : m_storage( std::move(other.m_storage) ) { } EIGEN_DEVICE_FUNC - PlainObjectBase& operator=(PlainObjectBase&& other) + PlainObjectBase& operator=(PlainObjectBase&& other) EIGEN_NOEXCEPT { using std::swap; swap(m_storage, other.m_storage); @@ -499,6 +526,71 @@ // EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } + #if EIGEN_HAS_CXX11 + /** \brief Construct a row of column vector with fixed size from an arbitrary number of coefficients. \cpp11 + * + * \only_for_vectors + * + * This constructor is for 1D array or vectors with more than 4 coefficients. + * There exists c++98 anologue constructors for fixed-size array/vector having 1, 2, 3, or 4 coefficients. + * + * \warning To construct a column (resp. row) vector of fixed length, the number of values passed to this + * constructor must match the the fixed number of rows (resp. columns) of \c *this. + */ + template <typename... ArgTypes> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) + : m_storage() + { + _check_template_params(); + EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, sizeof...(args) + 4); + m_storage.data()[0] = a0; + m_storage.data()[1] = a1; + m_storage.data()[2] = a2; + m_storage.data()[3] = a3; + int i = 4; + auto x = {(m_storage.data()[i++] = args, 0)...}; + static_cast<void>(x); + } + + /** \brief Constructs a Matrix or Array and initializes it by elements given by an initializer list of initializer + * lists \cpp11 + */ + EIGEN_DEVICE_FUNC + explicit EIGEN_STRONG_INLINE PlainObjectBase(const std::initializer_list<std::initializer_list<Scalar>>& list) + : m_storage() + { + _check_template_params(); + + size_t list_size = 0; + if (list.begin() != list.end()) { + list_size = list.begin()->size(); + } + + // This is to allow syntax like VectorXi {{1, 2, 3, 4}} + if (ColsAtCompileTime == 1 && list.size() == 1) { + eigen_assert(list_size == static_cast<size_t>(RowsAtCompileTime) || RowsAtCompileTime == Dynamic); + resize(list_size, ColsAtCompileTime); + std::copy(list.begin()->begin(), list.begin()->end(), m_storage.data()); + } else { + eigen_assert(list.size() == static_cast<size_t>(RowsAtCompileTime) || RowsAtCompileTime == Dynamic); + eigen_assert(list_size == static_cast<size_t>(ColsAtCompileTime) || ColsAtCompileTime == Dynamic); + resize(list.size(), list_size); + + Index row_index = 0; + for (const std::initializer_list<Scalar>& row : list) { + eigen_assert(list_size == row.size()); + Index col_index = 0; + for (const Scalar& e : row) { + coeffRef(row_index, col_index) = e; + ++col_index; + } + ++row_index; + } + } + } + #endif // end EIGEN_HAS_CXX11 + /** \sa PlainObjectBase::operator=(const EigenBase<OtherDerived>&) */ template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -533,10 +625,11 @@ public: - /** \copydoc DenseBase::operator=(const EigenBase<OtherDerived>&) + /** \brief Copies the generic expression \a other into *this. + * \copydetails DenseBase::operator=(const EigenBase<OtherDerived> &other) */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const EigenBase<OtherDerived> &other) { _resize_to_match(other); @@ -549,6 +642,10 @@ * while the AlignedMap() functions return aligned Map objects and thus should be called only with 16-byte-aligned * \a data pointers. * + * Here is an example using strides: + * \include Matrix_Map_stride.cpp + * Output: \verbinclude Matrix_Map_stride.out + * * \see class Map */ //@{ @@ -646,7 +743,7 @@ * remain row-vectors and vectors remain vectors. */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _resize_to_match(const EigenBase<OtherDerived>& other) { #ifdef EIGEN_NO_AUTOMATIC_RESIZING @@ -673,10 +770,10 @@ * * \internal */ - // aliasing is dealt once in internall::call_assignment + // aliasing is dealt once in internal::call_assignment // so at this stage we have to assume aliasing... and resising has to be done later. template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& _set(const DenseBase<OtherDerived>& other) { internal::call_assignment(this->derived(), other.derived()); @@ -689,7 +786,7 @@ * \sa operator=(const MatrixBase<OtherDerived>&), _set() */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& _set_noalias(const DenseBase<OtherDerived>& other) { // I don't think we need this resize call since the lazyAssign will anyways resize @@ -697,7 +794,7 @@ //_resize_to_match(other); // the 'false' below means to enforce lazy evaluation. We don't use lazyAssign() because // it wouldn't allow to copy a row-vector into a column-vector. - internal::call_assignment_no_alias(this->derived(), other.derived(), internal::assign_op<Scalar>()); + internal::call_assignment_no_alias(this->derived(), other.derived(), internal::assign_op<Scalar,typename OtherDerived::Scalar>()); return this->derived(); } @@ -710,18 +807,18 @@ FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED) resize(rows,cols); } - + template<typename T0, typename T1> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void _init2(const Scalar& val0, const Scalar& val1, typename internal::enable_if<Base::SizeAtCompileTime==2,T0>::type* = 0) + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE void _init2(const T0& val0, const T1& val1, typename internal::enable_if<Base::SizeAtCompileTime==2,T0>::type* = 0) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 2) - m_storage.data()[0] = val0; - m_storage.data()[1] = val1; + m_storage.data()[0] = Scalar(val0); + m_storage.data()[1] = Scalar(val1); } - + template<typename T0, typename T1> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init2(const Index& val0, const Index& val1, typename internal::enable_if< (!internal::is_same<Index,Scalar>::value) && (internal::is_same<T0,Index>::value) @@ -742,12 +839,13 @@ { // NOTE MSVC 2008 complains if we directly put bool(NumTraits<T>::IsInteger) as the EIGEN_STATIC_ASSERT argument. const bool is_integer = NumTraits<T>::IsInteger; + EIGEN_UNUSED_VARIABLE(is_integer); EIGEN_STATIC_ASSERT(is_integer, FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED) resize(size); } - - // We have a 1x1 matrix/array => the argument is interpreted as the value of the unique coefficient (case where scalar type can be implicitely converted) + + // We have a 1x1 matrix/array => the argument is interpreted as the value of the unique coefficient (case where scalar type can be implicitly converted) template<typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Scalar& val0, typename internal::enable_if<Base::SizeAtCompileTime==1 && internal::is_convertible<T, Scalar>::value,T>::type* = 0) @@ -755,7 +853,7 @@ EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, 1) m_storage.data()[0] = val0; } - + // We have a 1x1 matrix/array => the argument is interpreted as the value of the unique coefficient (case where scalar type match the index type) template<typename T> EIGEN_DEVICE_FUNC @@ -783,6 +881,13 @@ this->_set_noalias(other); } + // Initialize an arbitrary matrix from an object convertible to the Derived type. + template<typename T> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE void _init1(const Derived& other){ + this->_set_noalias(other); + } + // Initialize an arbitrary matrix from a generic Eigen expression template<typename T, typename OtherDerived> EIGEN_DEVICE_FUNC @@ -804,8 +909,8 @@ { this->derived() = r; } - - // For fixed -size arrays: + + // For fixed-size Array<Scalar,...> template<typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Scalar& val0, @@ -816,7 +921,8 @@ { Base::setConstant(val0); } - + + // For fixed-size Array<Index,...> template<typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _init1(const Index& val0, @@ -829,34 +935,34 @@ { Base::setConstant(val0); } - + template<typename MatrixTypeA, typename MatrixTypeB, bool SwapPointers> friend struct internal::matrix_swap_impl; public: - + #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal * \brief Override DenseBase::swap() since for dynamic-sized matrices * of same type it is enough to swap the data pointers. */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(DenseBase<OtherDerived> & other) { enum { SwapPointers = internal::is_same<Derived, OtherDerived>::value && Base::SizeAtCompileTime==Dynamic }; internal::matrix_swap_impl<Derived, OtherDerived, bool(SwapPointers)>::run(this->derived(), other.derived()); } - + /** \internal * \brief const version forwarded to DenseBase::swap */ template<typename OtherDerived> - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(DenseBase<OtherDerived> const & other) { Base::swap(other.derived()); } - - EIGEN_DEVICE_FUNC + + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void _check_template_params() { EIGEN_STATIC_ASSERT((EIGEN_IMPLIES(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, (Options&RowMajor)==RowMajor) @@ -880,13 +986,19 @@ template <typename Derived, typename OtherDerived, bool IsVector> struct conservative_resize_like_impl { + #if EIGEN_HAS_TYPE_TRAITS + static const bool IsRelocatable = std::is_trivially_copyable<typename Derived::Scalar>::value; + #else + static const bool IsRelocatable = !NumTraits<typename Derived::Scalar>::RequireInitialization; + #endif static void run(DenseBase<Derived>& _this, Index rows, Index cols) { if (_this.rows() == rows && _this.cols() == cols) return; EIGEN_STATIC_ASSERT_DYNAMIC_SIZE(Derived) - if ( ( Derived::IsRowMajor && _this.cols() == cols) || // row-major and we change only the number of rows - (!Derived::IsRowMajor && _this.rows() == rows) ) // column-major and we change only the number of columns + if ( IsRelocatable + && (( Derived::IsRowMajor && _this.cols() == cols) || // row-major and we change only the number of rows + (!Derived::IsRowMajor && _this.rows() == rows) )) // column-major and we change only the number of columns { internal::check_rows_cols_for_overflow<Derived::MaxSizeAtCompileTime>::run(rows, cols); _this.derived().m_storage.conservativeResize(rows*cols,rows,cols); @@ -895,8 +1007,8 @@ { // The storage order does not allow us to use reallocation. typename Derived::PlainObject tmp(rows,cols); - const Index common_rows = (std::min)(rows, _this.rows()); - const Index common_cols = (std::min)(cols, _this.cols()); + const Index common_rows = numext::mini(rows, _this.rows()); + const Index common_cols = numext::mini(cols, _this.cols()); tmp.block(0,0,common_rows,common_cols) = _this.block(0,0,common_rows,common_cols); _this.derived().swap(tmp); } @@ -914,8 +1026,9 @@ EIGEN_STATIC_ASSERT_DYNAMIC_SIZE(Derived) EIGEN_STATIC_ASSERT_DYNAMIC_SIZE(OtherDerived) - if ( ( Derived::IsRowMajor && _this.cols() == other.cols()) || // row-major and we change only the number of rows - (!Derived::IsRowMajor && _this.rows() == other.rows()) ) // column-major and we change only the number of columns + if ( IsRelocatable && + (( Derived::IsRowMajor && _this.cols() == other.cols()) || // row-major and we change only the number of rows + (!Derived::IsRowMajor && _this.rows() == other.rows()) )) // column-major and we change only the number of columns { const Index new_rows = other.rows() - _this.rows(); const Index new_cols = other.cols() - _this.cols(); @@ -929,8 +1042,8 @@ { // The storage order does not allow us to use reallocation. typename Derived::PlainObject tmp(other); - const Index common_rows = (std::min)(tmp.rows(), _this.rows()); - const Index common_cols = (std::min)(tmp.cols(), _this.cols()); + const Index common_rows = numext::mini(tmp.rows(), _this.rows()); + const Index common_cols = numext::mini(tmp.cols(), _this.cols()); tmp.block(0,0,common_rows,common_cols) = _this.block(0,0,common_rows,common_cols); _this.derived().swap(tmp); } @@ -943,13 +1056,18 @@ struct conservative_resize_like_impl<Derived,OtherDerived,true> : conservative_resize_like_impl<Derived,OtherDerived,false> { - using conservative_resize_like_impl<Derived,OtherDerived,false>::run; - + typedef conservative_resize_like_impl<Derived,OtherDerived,false> Base; + using Base::run; + using Base::IsRelocatable; + static void run(DenseBase<Derived>& _this, Index size) { const Index new_rows = Derived::RowsAtCompileTime==1 ? 1 : size; const Index new_cols = Derived::RowsAtCompileTime==1 ? size : 1; - _this.derived().m_storage.conservativeResize(size,new_rows,new_cols); + if(IsRelocatable) + _this.derived().m_storage.conservativeResize(size,new_rows,new_cols); + else + Base::run(_this.derived(), new_rows, new_cols); } static void run(DenseBase<Derived>& _this, const DenseBase<OtherDerived>& other) @@ -960,7 +1078,10 @@ const Index new_rows = Derived::RowsAtCompileTime==1 ? 1 : other.rows(); const Index new_cols = Derived::RowsAtCompileTime==1 ? other.cols() : 1; - _this.derived().m_storage.conservativeResize(other.size(),new_rows,new_cols); + if(IsRelocatable) + _this.derived().m_storage.conservativeResize(other.size(),new_rows,new_cols); + else + Base::run(_this.derived(), new_rows, new_cols); if (num_new_elements > 0) _this.tail(num_new_elements) = other.tail(num_new_elements); @@ -971,7 +1092,7 @@ struct matrix_swap_impl { EIGEN_DEVICE_FUNC - static inline void run(MatrixTypeA& a, MatrixTypeB& b) + static EIGEN_STRONG_INLINE void run(MatrixTypeA& a, MatrixTypeB& b) { a.base().swap(b); }
diff --git a/Eigen/src/Core/Product.h b/Eigen/src/Core/Product.h index 8aa1de0..13d5662 100644 --- a/Eigen/src/Core/Product.h +++ b/Eigen/src/Core/Product.h
@@ -16,39 +16,6 @@ namespace internal { -// Determine the scalar of Product<Lhs, Rhs>. This is normally the same as Lhs::Scalar times -// Rhs::Scalar, but product with permutation matrices inherit the scalar of the other factor. -template<typename Lhs, typename Rhs, typename LhsShape = typename evaluator_traits<Lhs>::Shape, - typename RhsShape = typename evaluator_traits<Rhs>::Shape > -struct product_result_scalar -{ - typedef typename scalar_product_traits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; -}; - -template<typename Lhs, typename Rhs, typename RhsShape> -struct product_result_scalar<Lhs, Rhs, PermutationShape, RhsShape> -{ - typedef typename Rhs::Scalar Scalar; -}; - -template<typename Lhs, typename Rhs, typename LhsShape> - struct product_result_scalar<Lhs, Rhs, LhsShape, PermutationShape> -{ - typedef typename Lhs::Scalar Scalar; -}; - -template<typename Lhs, typename Rhs, typename RhsShape> -struct product_result_scalar<Lhs, Rhs, TranspositionsShape, RhsShape> -{ - typedef typename Rhs::Scalar Scalar; -}; - -template<typename Lhs, typename Rhs, typename LhsShape> - struct product_result_scalar<Lhs, Rhs, LhsShape, TranspositionsShape> -{ - typedef typename Lhs::Scalar Scalar; -}; - template<typename Lhs, typename Rhs, int Option> struct traits<Product<Lhs, Rhs, Option> > { @@ -59,7 +26,7 @@ typedef MatrixXpr XprKind; - typedef typename product_result_scalar<LhsCleaned,RhsCleaned>::Scalar Scalar; + typedef typename ScalarBinaryOpTraits<typename traits<LhsCleaned>::Scalar, typename traits<RhsCleaned>::Scalar>::ReturnType Scalar; typedef typename product_promote_storage_type<typename LhsTraits::StorageKind, typename RhsTraits::StorageKind, internal::product_type<Lhs,Rhs>::ret>::ret StorageKind; @@ -123,18 +90,23 @@ typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned; typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned; - EIGEN_DEVICE_FUNC Product(const Lhs& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Product(const Lhs& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) { eigen_assert(lhs.cols() == rhs.rows() && "invalid matrix product" && "if you wanted a coeff-wise or a dot product use the respective explicit functions"); } - EIGEN_DEVICE_FUNC inline Index rows() const { return m_lhs.rows(); } - EIGEN_DEVICE_FUNC inline Index cols() const { return m_rhs.cols(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index rows() const { return m_lhs.rows(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index cols() const { return m_rhs.cols(); } - EIGEN_DEVICE_FUNC const LhsNestedCleaned& lhs() const { return m_lhs; } - EIGEN_DEVICE_FUNC const RhsNestedCleaned& rhs() const { return m_rhs; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const LhsNestedCleaned& lhs() const { return m_lhs; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const RhsNestedCleaned& rhs() const { return m_rhs; } protected: @@ -149,7 +121,7 @@ : public internal::dense_xpr_base<Product<Lhs,Rhs,Option> >::type {}; -/** Convertion to scalar for inner-products */ +/** Conversion to scalar for inner-products */ template<typename Lhs, typename Rhs, int Option> class dense_product_base<Lhs, Rhs, Option, InnerProduct> : public internal::dense_xpr_base<Product<Lhs,Rhs,Option> >::type @@ -160,7 +132,7 @@ using Base::derived; typedef typename Base::Scalar Scalar; - operator const Scalar() const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator const Scalar() const { return internal::evaluator<ProductXpr>(derived()).coeff(0,0); } @@ -195,7 +167,7 @@ public: - EIGEN_DEVICE_FUNC Scalar coeff(Index row, Index col) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS); eigen_assert( (Option==LazyProduct) || (this->rows() == 1 && this->cols() == 1) ); @@ -203,7 +175,7 @@ return internal::evaluator<Derived>(derived()).coeff(row,col); } - EIGEN_DEVICE_FUNC Scalar coeff(Index i) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index i) const { EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS); eigen_assert( (Option==LazyProduct) || (this->rows() == 1 && this->cols() == 1) );
diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index d9fd888..d53dc30 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h
@@ -20,7 +20,7 @@ /** \internal * Evaluator of a product expression. * Since products require special treatments to handle all possible cases, - * we simply deffer the evaluation logic to a product_evaluator class + * we simply defer the evaluation logic to a product_evaluator class * which offers more partial specialization possibilities. * * \sa class product_evaluator @@ -32,25 +32,31 @@ typedef Product<Lhs, Rhs, Options> XprType; typedef product_evaluator<XprType> Base; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : Base(xpr) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(xpr) {} }; -// Catch scalar * ( A * B ) and transform it to (A*scalar) * B +// Catch "scalar * ( A * B )" and transform it to "(A*scalar) * B" // TODO we should apply that rule only if that's really helpful -template<typename Lhs, typename Rhs, typename Scalar> -struct evaluator_assume_aliasing<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > > +template<typename Lhs, typename Rhs, typename Scalar1, typename Scalar2, typename Plain1> +struct evaluator_assume_aliasing<CwiseBinaryOp<internal::scalar_product_op<Scalar1,Scalar2>, + const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>, + const Product<Lhs, Rhs, DefaultProduct> > > { static const bool value = true; }; -template<typename Lhs, typename Rhs, typename Scalar> -struct evaluator<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > > - : public evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,const Lhs>, Rhs, DefaultProduct> > +template<typename Lhs, typename Rhs, typename Scalar1, typename Scalar2, typename Plain1> +struct evaluator<CwiseBinaryOp<internal::scalar_product_op<Scalar1,Scalar2>, + const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>, + const Product<Lhs, Rhs, DefaultProduct> > > + : public evaluator<Product<EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar1,Lhs,product), Rhs, DefaultProduct> > { - typedef CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > XprType; - typedef evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,const Lhs>, Rhs, DefaultProduct> > Base; - - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) - : Base(xpr.functor().m_other * xpr.nestedExpression().lhs() * xpr.nestedExpression().rhs()) + typedef CwiseBinaryOp<internal::scalar_product_op<Scalar1,Scalar2>, + const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>, + const Product<Lhs, Rhs, DefaultProduct> > XprType; + typedef evaluator<Product<EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar1,Lhs,product), Rhs, DefaultProduct> > Base; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) + : Base(xpr.lhs().functor().m_other * xpr.rhs().lhs() * xpr.rhs().rhs()) {} }; @@ -62,7 +68,7 @@ typedef Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> XprType; typedef evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> > Base; - EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>( Product<Lhs, Rhs, LazyProduct>(xpr.nestedExpression().lhs(), xpr.nestedExpression().rhs()), xpr.index() )) @@ -122,14 +128,22 @@ PlainObject m_result; }; +// The following three shortcuts are enabled only if the scalar types match exactly. +// TODO: we could enable them for different scalar types when the product is not vectorized. + // Dense = Product template< typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::assign_op<Scalar>, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type> +struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::assign_op<Scalar,Scalar>, Dense2Dense, + typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> { typedef Product<Lhs,Rhs,Options> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); // FIXME shall we handle nested_eval here? generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs()); } @@ -137,12 +151,14 @@ // Dense += Product template< typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::add_assign_op<Scalar>, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type> +struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::add_assign_op<Scalar,Scalar>, Dense2Dense, + typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> { typedef Product<Lhs,Rhs,Options> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar> &) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar,Scalar> &) { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); // FIXME shall we handle nested_eval here? generic_product_impl<Lhs, Rhs>::addTo(dst, src.lhs(), src.rhs()); } @@ -150,12 +166,14 @@ // Dense -= Product template< typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::sub_assign_op<Scalar>, Dense2Dense, - typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct),Scalar>::type> +struct Assignment<DstXprType, Product<Lhs,Rhs,Options>, internal::sub_assign_op<Scalar,Scalar>, Dense2Dense, + typename enable_if<(Options==DefaultProduct || Options==AliasFreeProduct)>::type> { typedef Product<Lhs,Rhs,Options> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar> &) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar,Scalar> &) { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); // FIXME shall we handle nested_eval here? generic_product_impl<Lhs, Rhs>::subTo(dst, src.lhs(), src.rhs()); } @@ -165,74 +183,82 @@ // Dense ?= scalar * Product // TODO we should apply that rule if that's really helpful // for instance, this is not good for inner products -template< typename DstXprType, typename Lhs, typename Rhs, typename AssignFunc, typename Scalar, typename ScalarBis> -struct Assignment<DstXprType, CwiseUnaryOp<internal::scalar_multiple_op<ScalarBis>, - const Product<Lhs,Rhs,DefaultProduct> >, AssignFunc, Dense2Dense, Scalar> +template< typename DstXprType, typename Lhs, typename Rhs, typename AssignFunc, typename Scalar, typename ScalarBis, typename Plain> +struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_product_op<ScalarBis,Scalar>, const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>,Plain>, + const Product<Lhs,Rhs,DefaultProduct> >, AssignFunc, Dense2Dense> { - typedef CwiseUnaryOp<internal::scalar_multiple_op<ScalarBis>, - const Product<Lhs,Rhs,DefaultProduct> > SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const AssignFunc& func) + typedef CwiseBinaryOp<internal::scalar_product_op<ScalarBis,Scalar>, + const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>,Plain>, + const Product<Lhs,Rhs,DefaultProduct> > SrcXprType; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const AssignFunc& func) { - call_assignment_no_alias(dst, (src.functor().m_other * src.nestedExpression().lhs())*src.nestedExpression().rhs(), func); + call_assignment_no_alias(dst, (src.lhs().functor().m_other * src.rhs().lhs())*src.rhs().rhs(), func); } }; //---------------------------------------- // Catch "Dense ?= xpr + Product<>" expression to save one temporary // FIXME we could probably enable these rules for any product, i.e., not only Dense and DefaultProduct -// TODO enable it for "Dense ?= xpr - Product<>" as well. template<typename OtherXpr, typename Lhs, typename Rhs> -struct evaluator_assume_aliasing<CwiseBinaryOp<internal::scalar_sum_op<typename OtherXpr::Scalar>, const OtherXpr, +struct evaluator_assume_aliasing<CwiseBinaryOp<internal::scalar_sum_op<typename OtherXpr::Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, const OtherXpr, const Product<Lhs,Rhs,DefaultProduct> >, DenseShape > { static const bool value = true; }; -template<typename DstXprType, typename OtherXpr, typename ProductType, typename Scalar, typename Func1, typename Func2> -struct assignment_from_xpr_plus_product +template<typename OtherXpr, typename Lhs, typename Rhs> +struct evaluator_assume_aliasing<CwiseBinaryOp<internal::scalar_difference_op<typename OtherXpr::Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, const OtherXpr, + const Product<Lhs,Rhs,DefaultProduct> >, DenseShape > { + static const bool value = true; +}; + +template<typename DstXprType, typename OtherXpr, typename ProductType, typename Func1, typename Func2> +struct assignment_from_xpr_op_product { - typedef CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const OtherXpr, const ProductType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const Func1& func) + template<typename SrcXprType, typename InitialFunc> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const InitialFunc& /*func*/) { - call_assignment_no_alias(dst, src.lhs(), func); + call_assignment_no_alias(dst, src.lhs(), Func1()); call_assignment_no_alias(dst, src.rhs(), Func2()); } }; -template< typename DstXprType, typename OtherXpr, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const OtherXpr, - const Product<Lhs,Rhs,DefaultProduct> >, internal::assign_op<Scalar>, Dense2Dense> - : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::assign_op<Scalar>, internal::add_assign_op<Scalar> > -{}; -template< typename DstXprType, typename OtherXpr, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const OtherXpr, - const Product<Lhs,Rhs,DefaultProduct> >, internal::add_assign_op<Scalar>, Dense2Dense> - : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::add_assign_op<Scalar>, internal::add_assign_op<Scalar> > -{}; -template< typename DstXprType, typename OtherXpr, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const OtherXpr, - const Product<Lhs,Rhs,DefaultProduct> >, internal::sub_assign_op<Scalar>, Dense2Dense> - : assignment_from_xpr_plus_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, Scalar, internal::sub_assign_op<Scalar>, internal::sub_assign_op<Scalar> > -{}; +#define EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(ASSIGN_OP,BINOP,ASSIGN_OP2) \ + template< typename DstXprType, typename OtherXpr, typename Lhs, typename Rhs, typename DstScalar, typename SrcScalar, typename OtherScalar,typename ProdScalar> \ + struct Assignment<DstXprType, CwiseBinaryOp<internal::BINOP<OtherScalar,ProdScalar>, const OtherXpr, \ + const Product<Lhs,Rhs,DefaultProduct> >, internal::ASSIGN_OP<DstScalar,SrcScalar>, Dense2Dense> \ + : assignment_from_xpr_op_product<DstXprType, OtherXpr, Product<Lhs,Rhs,DefaultProduct>, internal::ASSIGN_OP<DstScalar,OtherScalar>, internal::ASSIGN_OP2<DstScalar,ProdScalar> > \ + {} + +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(assign_op, scalar_sum_op,add_assign_op); +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(add_assign_op,scalar_sum_op,add_assign_op); +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(sub_assign_op,scalar_sum_op,sub_assign_op); + +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(assign_op, scalar_difference_op,sub_assign_op); +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(add_assign_op,scalar_difference_op,sub_assign_op); +EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(sub_assign_op,scalar_difference_op,add_assign_op); + //---------------------------------------- template<typename Lhs, typename Rhs> struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,InnerProduct> { template<typename Dst> - static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { dst.coeffRef(0,0) = (lhs.transpose().cwiseProduct(rhs)).sum(); } template<typename Dst> - static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { dst.coeffRef(0,0) += (lhs.transpose().cwiseProduct(rhs)).sum(); } template<typename Dst> - static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { dst.coeffRef(0,0) -= (lhs.transpose().cwiseProduct(rhs)).sum(); } }; @@ -243,28 +269,28 @@ // Column major result template<typename Dst, typename Lhs, typename Rhs, typename Func> -EIGEN_DONT_INLINE void outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const false_type&) +void EIGEN_DEVICE_FUNC outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const false_type&) { evaluator<Rhs> rhsEval(rhs); - typename nested_eval<Lhs,Rhs::SizeAtCompileTime>::type actual_lhs(lhs); + ei_declare_local_nested_eval(Lhs,lhs,Rhs::SizeAtCompileTime,actual_lhs); // FIXME if cols is large enough, then it might be useful to make sure that lhs is sequentially stored // FIXME not very good if rhs is real and lhs complex while alpha is real too const Index cols = dst.cols(); for (Index j=0; j<cols; ++j) - func(dst.col(j), rhsEval.coeff(0,j) * actual_lhs); + func(dst.col(j), rhsEval.coeff(Index(0),j) * actual_lhs); } // Row major result template<typename Dst, typename Lhs, typename Rhs, typename Func> -EIGEN_DONT_INLINE void outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const true_type&) +void EIGEN_DEVICE_FUNC outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const true_type&) { evaluator<Lhs> lhsEval(lhs); - typename nested_eval<Rhs,Lhs::SizeAtCompileTime>::type actual_rhs(rhs); + ei_declare_local_nested_eval(Rhs,rhs,Lhs::SizeAtCompileTime,actual_rhs); // FIXME if rows is large enough, then it might be useful to make sure that rhs is sequentially stored // FIXME not very good if lhs is real and rhs complex while alpha is real too const Index rows = dst.rows(); for (Index i=0; i<rows; ++i) - func(dst.row(i), lhsEval.coeff(i,0) * actual_rhs); + func(dst.row(i), lhsEval.coeff(i,Index(0)) * actual_rhs); } template<typename Lhs, typename Rhs> @@ -274,37 +300,37 @@ typedef typename Product<Lhs,Rhs>::Scalar Scalar; // TODO it would be nice to be able to exploit our *_assign_op functors for that purpose - struct set { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() = src; } }; - struct add { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() += src; } }; - struct sub { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() -= src; } }; + struct set { template<typename Dst, typename Src> EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() = src; } }; + struct add { template<typename Dst, typename Src> EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() += src; } }; + struct sub { template<typename Dst, typename Src> EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() -= src; } }; struct adds { Scalar m_scale; explicit adds(const Scalar& s) : m_scale(s) {} - template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { + template<typename Dst, typename Src> void EIGEN_DEVICE_FUNC operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() += m_scale * src; } }; template<typename Dst> - static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { internal::outer_product_selector_run(dst, lhs, rhs, set(), is_row_major<Dst>()); } template<typename Dst> - static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { internal::outer_product_selector_run(dst, lhs, rhs, add(), is_row_major<Dst>()); } template<typename Dst> - static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { internal::outer_product_selector_run(dst, lhs, rhs, sub(), is_row_major<Dst>()); } template<typename Dst> - static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { internal::outer_product_selector_run(dst, lhs, rhs, adds(alpha), is_row_major<Dst>()); } @@ -319,19 +345,19 @@ typedef typename Product<Lhs,Rhs>::Scalar Scalar; template<typename Dst> - static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { dst.setZero(); scaleAndAddTo(dst, lhs, rhs, Scalar(1)); } template<typename Dst> - static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { scaleAndAddTo(dst,lhs, rhs, Scalar(1)); } template<typename Dst> - static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); } template<typename Dst> - static void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { Derived::scaleAndAddTo(dst,lhs,rhs,alpha); } }; @@ -340,17 +366,21 @@ struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> > { + typedef typename nested_eval<Lhs,1>::type LhsNested; + typedef typename nested_eval<Rhs,1>::type RhsNested; typedef typename Product<Lhs,Rhs>::Scalar Scalar; enum { Side = Lhs::IsVectorAtCompileTime ? OnTheLeft : OnTheRight }; - typedef typename internal::conditional<int(Side)==OnTheRight,Lhs,Rhs>::type MatrixType; + typedef typename internal::remove_all<typename internal::conditional<int(Side)==OnTheRight,LhsNested,RhsNested>::type>::type MatrixType; template<typename Dest> - static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { + LhsNested actual_lhs(lhs); + RhsNested actual_rhs(rhs); internal::gemv_dense_selector<Side, (int(MatrixType::Flags)&RowMajorBit) ? RowMajor : ColMajor, bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess) - >::run(lhs, rhs, dst, alpha); + >::run(actual_lhs, actual_rhs, dst, alpha); } }; @@ -360,30 +390,79 @@ typedef typename Product<Lhs,Rhs>::Scalar Scalar; template<typename Dst> - static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { // Same as: dst.noalias() = lhs.lazyProduct(rhs); // but easier on the compiler side - call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::assign_op<Scalar>()); + call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::assign_op<typename Dst::Scalar,Scalar>()); } - + template<typename Dst> - static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { // dst.noalias() += lhs.lazyProduct(rhs); - call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::add_assign_op<Scalar>()); + call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::add_assign_op<typename Dst::Scalar,Scalar>()); } template<typename Dst> - static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { // dst.noalias() -= lhs.lazyProduct(rhs); - call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::sub_assign_op<Scalar>()); + call_assignment_no_alias(dst, lhs.lazyProduct(rhs), internal::sub_assign_op<typename Dst::Scalar,Scalar>()); } - -// template<typename Dst> -// static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) -// { dst.noalias() += alpha * lhs.lazyProduct(rhs); } + + // This is a special evaluation path called from generic_product_impl<...,GemmProduct> in file GeneralMatrixMatrix.h + // This variant tries to extract scalar multiples from both the LHS and RHS and factor them out. For instance: + // dst {,+,-}= (s1*A)*(B*s2) + // will be rewritten as: + // dst {,+,-}= (s1*s2) * (A.lazyProduct(B)) + // There are at least four benefits of doing so: + // 1 - huge performance gain for heap-allocated matrix types as it save costly allocations. + // 2 - it is faster than simply by-passing the heap allocation through stack allocation. + // 3 - it makes this fallback consistent with the heavy GEMM routine. + // 4 - it fully by-passes huge stack allocation attempts when multiplying huge fixed-size matrices. + // (see https://stackoverflow.com/questions/54738495) + // For small fixed sizes matrices, howver, the gains are less obvious, it is sometimes x2 faster, but sometimes x3 slower, + // and the behavior depends also a lot on the compiler... This is why this re-writting strategy is currently + // enabled only when falling back from the main GEMM. + template<typename Dst, typename Func> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void eval_dynamic(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func &func) + { + enum { + HasScalarFactor = blas_traits<Lhs>::HasScalarFactor || blas_traits<Rhs>::HasScalarFactor, + ConjLhs = blas_traits<Lhs>::NeedToConjugate, + ConjRhs = blas_traits<Rhs>::NeedToConjugate + }; + // FIXME: in c++11 this should be auto, and extractScalarFactor should also return auto + // this is important for real*complex_mat + Scalar actualAlpha = blas_traits<Lhs>::extractScalarFactor(lhs) + * blas_traits<Rhs>::extractScalarFactor(rhs); + eval_dynamic_impl(dst, + blas_traits<Lhs>::extract(lhs).template conjugateIf<ConjLhs>(), + blas_traits<Rhs>::extract(rhs).template conjugateIf<ConjRhs>(), + func, + actualAlpha, + typename conditional<HasScalarFactor,true_type,false_type>::type()); + } + +protected: + + template<typename Dst, typename LhsT, typename RhsT, typename Func, typename Scalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void eval_dynamic_impl(Dst& dst, const LhsT& lhs, const RhsT& rhs, const Func &func, const Scalar& s /* == 1 */, false_type) + { + EIGEN_UNUSED_VARIABLE(s); + eigen_internal_assert(s==Scalar(1)); + call_restricted_packet_assignment_no_alias(dst, lhs.lazyProduct(rhs), func); + } + + template<typename Dst, typename LhsT, typename RhsT, typename Func, typename Scalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void eval_dynamic_impl(Dst& dst, const LhsT& lhs, const RhsT& rhs, const Func &func, const Scalar& s, true_type) + { + call_restricted_packet_assignment_no_alias(dst, s * lhs.lazyProduct(rhs), func); + } }; // This specialization enforces the use of a coefficient-based evaluation strategy @@ -423,6 +502,18 @@ EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost); EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::AddCost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); +#if 0 + std::cerr << "LhsOuterStrideBytes= " << LhsOuterStrideBytes << "\n"; + std::cerr << "RhsOuterStrideBytes= " << RhsOuterStrideBytes << "\n"; + std::cerr << "LhsAlignment= " << LhsAlignment << "\n"; + std::cerr << "RhsAlignment= " << RhsAlignment << "\n"; + std::cerr << "CanVectorizeLhs= " << CanVectorizeLhs << "\n"; + std::cerr << "CanVectorizeRhs= " << CanVectorizeRhs << "\n"; + std::cerr << "CanVectorizeInner= " << CanVectorizeInner << "\n"; + std::cerr << "EvalToRowMajor= " << EvalToRowMajor << "\n"; + std::cerr << "Alignment= " << Alignment << "\n"; + std::cerr << "Flags= " << Flags << "\n"; +#endif } // Everything below here is taken from CoeffBasedProduct.h @@ -473,15 +564,12 @@ SameType = is_same<typename LhsNestedCleaned::Scalar,typename RhsNestedCleaned::Scalar>::value, - CanVectorizeRhs = RhsRowMajor && (RhsFlags & PacketAccessBit) - && (ColsAtCompileTime == Dynamic || ((ColsAtCompileTime % RhsVecPacketSize) == 0) ), - - CanVectorizeLhs = (!LhsRowMajor) && (LhsFlags & PacketAccessBit) - && (RowsAtCompileTime == Dynamic || ((RowsAtCompileTime % LhsVecPacketSize) == 0) ), + CanVectorizeRhs = bool(RhsRowMajor) && (RhsFlags & PacketAccessBit) && (ColsAtCompileTime!=1), + CanVectorizeLhs = (!LhsRowMajor) && (LhsFlags & PacketAccessBit) && (RowsAtCompileTime!=1), EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 - : (RhsRowMajor && !CanVectorizeLhs), + : (bool(RhsRowMajor) && !CanVectorizeLhs), Flags = ((unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit) | (EvalToRowMajor ? RowMajorBit : 0) @@ -492,8 +580,8 @@ LhsOuterStrideBytes = int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename LhsNestedCleaned::Scalar)), RhsOuterStrideBytes = int(RhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename RhsNestedCleaned::Scalar)), - Alignment = CanVectorizeLhs ? (LhsOuterStrideBytes<0 || (int(LhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,LhsAlignment))!=0 ? 0 : LhsAlignment) - : CanVectorizeRhs ? (RhsOuterStrideBytes<0 || (int(RhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,RhsAlignment))!=0 ? 0 : RhsAlignment) + Alignment = bool(CanVectorizeLhs) ? (LhsOuterStrideBytes<=0 || (int(LhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,LhsAlignment))!=0 ? 0 : LhsAlignment) + : bool(CanVectorizeRhs) ? (RhsOuterStrideBytes<=0 || (int(RhsOuterStrideBytes) % EIGEN_PLAIN_ENUM_MAX(1,RhsAlignment))!=0 ? 0 : RhsAlignment) : 0, /* CanVectorizeInner deserves special explanation. It does not affect the product flags. It is not used outside @@ -517,14 +605,16 @@ * which is why we don't set the LinearAccessBit. * TODO: this seems possible when the result is a vector */ - EIGEN_DEVICE_FUNC const CoeffReturnType coeff(Index index) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const CoeffReturnType coeff(Index index) const { - const Index row = RowsAtCompileTime == 1 ? 0 : index; - const Index col = RowsAtCompileTime == 1 ? index : 0; + const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime==1) ? 0 : index; + const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime==1) ? index : 0; return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum(); } template<int LoadMode, typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packet(Index row, Index col) const { PacketType res; @@ -536,16 +626,17 @@ } template<int LoadMode, typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packet(Index index) const { - const Index row = RowsAtCompileTime == 1 ? 0 : index; - const Index col = RowsAtCompileTime == 1 ? index : 0; + const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime==1) ? 0 : index; + const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime==1) ? index : 0; return packet<LoadMode,PacketType>(row,col); } protected: - const LhsNested m_lhs; - const RhsNested m_rhs; + typename internal::add_const_on_value_type<LhsNested>::type m_lhs; + typename internal::add_const_on_value_type<RhsNested>::type m_rhs; LhsEtorType m_lhsImpl; RhsEtorType m_rhsImpl; @@ -564,7 +655,8 @@ enum { Flags = Base::Flags | EvalBeforeNestingBit }; - EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit product_evaluator(const XprType& xpr) : Base(BaseProduct(xpr.lhs(),xpr.rhs())) {} }; @@ -579,7 +671,7 @@ static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) { etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); - res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex-1)), rhs.template packet<LoadMode,Packet>(UnrollingIndex-1, col), res); + res = pmadd(pset1<Packet>(lhs.coeff(row, Index(UnrollingIndex-1))), rhs.template packet<LoadMode,Packet>(Index(UnrollingIndex-1), col), res); } }; @@ -589,7 +681,7 @@ static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) { etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); - res = pmadd(lhs.template packet<LoadMode,Packet>(row, UnrollingIndex-1), pset1<Packet>(rhs.coeff(UnrollingIndex-1, col)), res); + res = pmadd(lhs.template packet<LoadMode,Packet>(row, Index(UnrollingIndex-1)), pset1<Packet>(rhs.coeff(Index(UnrollingIndex-1), col)), res); } }; @@ -598,7 +690,7 @@ { static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) { - res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode,Packet>(0, col)); + res = pmul(pset1<Packet>(lhs.coeff(row, Index(0))),rhs.template packet<LoadMode,Packet>(Index(0), col)); } }; @@ -607,7 +699,7 @@ { static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) { - res = pmul(lhs.template packet<LoadMode,Packet>(row, 0), pset1<Packet>(rhs.coeff(0, col))); + res = pmul(lhs.template packet<LoadMode,Packet>(row, Index(0)), pset1<Packet>(rhs.coeff(Index(0), col))); } }; @@ -616,7 +708,7 @@ { static EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/, const Rhs& /*rhs*/, Index /*innerDim*/, Packet &res) { - res = pset1<Packet>(0); + res = pset1<Packet>(typename unpacket_traits<Packet>::type(0)); } }; @@ -625,7 +717,7 @@ { static EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/, const Rhs& /*rhs*/, Index /*innerDim*/, Packet &res) { - res = pset1<Packet>(0); + res = pset1<Packet>(typename unpacket_traits<Packet>::type(0)); } }; @@ -634,7 +726,7 @@ { static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) { - res = pset1<Packet>(0); + res = pset1<Packet>(typename unpacket_traits<Packet>::type(0)); for(Index i = 0; i < innerDim; ++i) res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode,Packet>(i, col), res); } @@ -645,7 +737,7 @@ { static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) { - res = pset1<Packet>(0); + res = pset1<Packet>(typename unpacket_traits<Packet>::type(0)); for(Index i = 0; i < innerDim; ++i) res = pmadd(lhs.template packet<LoadMode,Packet>(row, i), pset1<Packet>(rhs.coeff(i, col)), res); } @@ -702,7 +794,8 @@ typedef typename Product<Lhs,Rhs>::Scalar Scalar; template<typename Dest> - static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + static EIGEN_DEVICE_FUNC + void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { selfadjoint_product_impl<typename Lhs::MatrixType,Lhs::Mode,false,Rhs,0,Rhs::IsVectorAtCompileTime>::run(dst, lhs.nestedExpression(), rhs, alpha); } @@ -730,23 +823,35 @@ struct diagonal_product_evaluator_base : evaluator_base<Derived> { - typedef typename scalar_product_traits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType Scalar; + typedef typename ScalarBinaryOpTraits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType Scalar; public: enum { CoeffReadCost = NumTraits<Scalar>::MulCost + evaluator<MatrixType>::CoeffReadCost + evaluator<DiagonalType>::CoeffReadCost, MatrixFlags = evaluator<MatrixType>::Flags, DiagFlags = evaluator<DiagonalType>::Flags, - _StorageOrder = MatrixFlags & RowMajorBit ? RowMajor : ColMajor, + + _StorageOrder = (Derived::MaxRowsAtCompileTime==1 && Derived::MaxColsAtCompileTime!=1) ? RowMajor + : (Derived::MaxColsAtCompileTime==1 && Derived::MaxRowsAtCompileTime!=1) ? ColMajor + : MatrixFlags & RowMajorBit ? RowMajor : ColMajor, + _SameStorageOrder = _StorageOrder == (MatrixFlags & RowMajorBit ? RowMajor : ColMajor), + _ScalarAccessOnDiag = !((int(_StorageOrder) == ColMajor && int(ProductOrder) == OnTheLeft) ||(int(_StorageOrder) == RowMajor && int(ProductOrder) == OnTheRight)), _SameTypes = is_same<typename MatrixType::Scalar, typename DiagonalType::Scalar>::value, // FIXME currently we need same types, but in the future the next rule should be the one //_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagFlags)&PacketAccessBit))), - _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), + _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) + && _SameTypes + && (_SameStorageOrder || (MatrixFlags&LinearAccessBit)==LinearAccessBit) + && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0, Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0), - Alignment = evaluator<MatrixType>::Alignment + Alignment = evaluator<MatrixType>::Alignment, + + AsScalarProduct = (DiagonalType::SizeAtCompileTime==1) + || (DiagonalType::SizeAtCompileTime==Dynamic && MatrixType::RowsAtCompileTime==1 && ProductOrder==OnTheLeft) + || (DiagonalType::SizeAtCompileTime==Dynamic && MatrixType::ColsAtCompileTime==1 && ProductOrder==OnTheRight) }; diagonal_product_evaluator_base(const MatrixType &mat, const DiagonalType &diag) @@ -758,7 +863,10 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index idx) const { - return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx); + if(AsScalarProduct) + return m_diagImpl.coeff(0) * m_matImpl.coeff(idx); + else + return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx); } protected: @@ -797,10 +905,10 @@ typedef Product<Lhs, Rhs, ProductKind> XprType; typedef typename XprType::PlainObject PlainObject; + typedef typename Lhs::DiagonalVectorType DiagonalType; + - enum { - StorageOrder = int(Rhs::Flags) & RowMajorBit ? RowMajor : ColMajor - }; + enum { StorageOrder = Base::_StorageOrder }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.rhs(), xpr.lhs().diagonal()) @@ -812,7 +920,7 @@ return m_diagImpl.coeff(row) * m_matImpl.coeff(row, col); } -#ifndef __CUDACC__ +#ifndef EIGEN_GPUCC template<int LoadMode,typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { @@ -844,7 +952,7 @@ typedef Product<Lhs, Rhs, ProductKind> XprType; typedef typename XprType::PlainObject PlainObject; - enum { StorageOrder = int(Lhs::Flags) & RowMajorBit ? RowMajor : ColMajor }; + enum { StorageOrder = Base::_StorageOrder }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.lhs(), xpr.rhs().diagonal()) @@ -856,7 +964,7 @@ return m_matImpl.coeff(row, col) * m_diagImpl.coeff(col); } -#ifndef __CUDACC__ +#ifndef EIGEN_GPUCC template<int LoadMode,typename PacketType> EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
diff --git a/Eigen/src/Core/Random.h b/Eigen/src/Core/Random.h index 02038e9..486e9ed 100644 --- a/Eigen/src/Core/Random.h +++ b/Eigen/src/Core/Random.h
@@ -16,8 +16,7 @@ template<typename Scalar> struct scalar_random_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_random_op) - template<typename Index> - inline const Scalar operator() (Index, Index = 0) const { return random<Scalar>(); } + inline const Scalar operator() () const { return random<Scalar>(); } }; template<typename Scalar> @@ -129,7 +128,7 @@ * \sa class CwiseNullaryOp, setRandom(Index), setRandom(Index,Index) */ template<typename Derived> -inline Derived& DenseBase<Derived>::setRandom() +EIGEN_DEVICE_FUNC inline Derived& DenseBase<Derived>::setRandom() { return *this = Random(rows(), cols()); }
diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 98b2fd8..2eef5ab 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h
@@ -23,23 +23,29 @@ * Part 1 : the logic deciding a strategy for vectorization and unrolling ***************************************************************************/ -template<typename Func, typename Derived> +template<typename Func, typename Evaluator> struct redux_traits { public: - typedef typename find_best_packet<typename Derived::Scalar,Derived::SizeAtCompileTime>::type PacketType; + typedef typename find_best_packet<typename Evaluator::Scalar,Evaluator::SizeAtCompileTime>::type PacketType; enum { PacketSize = unpacket_traits<PacketType>::size, - InnerMaxSize = int(Derived::IsRowMajor) - ? Derived::MaxColsAtCompileTime - : Derived::MaxRowsAtCompileTime + InnerMaxSize = int(Evaluator::IsRowMajor) + ? Evaluator::MaxColsAtCompileTime + : Evaluator::MaxRowsAtCompileTime, + OuterMaxSize = int(Evaluator::IsRowMajor) + ? Evaluator::MaxRowsAtCompileTime + : Evaluator::MaxColsAtCompileTime, + SliceVectorizedWork = int(InnerMaxSize)==Dynamic ? Dynamic + : int(OuterMaxSize)==Dynamic ? (int(InnerMaxSize)>=int(PacketSize) ? Dynamic : 0) + : (int(InnerMaxSize)/int(PacketSize)) * int(OuterMaxSize) }; enum { - MightVectorize = (int(Derived::Flags)&ActualPacketAccessBit) + MightVectorize = (int(Evaluator::Flags)&ActualPacketAccessBit) && (functor_traits<Func>::PacketAccess), - MayLinearVectorize = MightVectorize && (int(Derived::Flags)&LinearAccessBit), - MaySliceVectorize = MightVectorize && int(InnerMaxSize)>=3*PacketSize + MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&LinearAccessBit), + MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork)==Dynamic || int(SliceVectorizedWork)>=3) }; public: @@ -51,8 +57,8 @@ public: enum { - Cost = Derived::SizeAtCompileTime == Dynamic ? HugeCost - : Derived::SizeAtCompileTime * Derived::CoeffReadCost + (Derived::SizeAtCompileTime-1) * functor_traits<Func>::Cost, + Cost = Evaluator::SizeAtCompileTime == Dynamic ? HugeCost + : Evaluator::SizeAtCompileTime * Evaluator::CoeffReadCost + (Evaluator::SizeAtCompileTime-1) * functor_traits<Func>::Cost, UnrollingLimit = EIGEN_UNROLLING_LIMIT * (int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize)) }; @@ -64,18 +70,20 @@ #ifdef EIGEN_DEBUG_ASSIGN static void debug() { - std::cerr << "Xpr: " << typeid(typename Derived::XprType).name() << std::endl; + std::cerr << "Xpr: " << typeid(typename Evaluator::XprType).name() << std::endl; std::cerr.setf(std::ios::hex, std::ios::basefield); - EIGEN_DEBUG_VAR(Derived::Flags) + EIGEN_DEBUG_VAR(Evaluator::Flags) std::cerr.unsetf(std::ios::hex); EIGEN_DEBUG_VAR(InnerMaxSize) + EIGEN_DEBUG_VAR(OuterMaxSize) + EIGEN_DEBUG_VAR(SliceVectorizedWork) EIGEN_DEBUG_VAR(PacketSize) EIGEN_DEBUG_VAR(MightVectorize) EIGEN_DEBUG_VAR(MayLinearVectorize) EIGEN_DEBUG_VAR(MaySliceVectorize) - EIGEN_DEBUG_VAR(Traversal) + std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl; EIGEN_DEBUG_VAR(UnrollingLimit) - EIGEN_DEBUG_VAR(Unrolling) + std::cerr << "Unrolling" << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl; std::cerr << std::endl; } #endif @@ -87,88 +95,86 @@ /*** no vectorization ***/ -template<typename Func, typename Derived, int Start, int Length> +template<typename Func, typename Evaluator, int Start, int Length> struct redux_novec_unroller { enum { HalfLength = Length/2 }; - typedef typename Derived::Scalar Scalar; + typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func) { - return func(redux_novec_unroller<Func, Derived, Start, HalfLength>::run(mat,func), - redux_novec_unroller<Func, Derived, Start+HalfLength, Length-HalfLength>::run(mat,func)); + return func(redux_novec_unroller<Func, Evaluator, Start, HalfLength>::run(eval,func), + redux_novec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::run(eval,func)); } }; -template<typename Func, typename Derived, int Start> -struct redux_novec_unroller<Func, Derived, Start, 1> +template<typename Func, typename Evaluator, int Start> +struct redux_novec_unroller<Func, Evaluator, Start, 1> { enum { - outer = Start / Derived::InnerSizeAtCompileTime, - inner = Start % Derived::InnerSizeAtCompileTime + outer = Start / Evaluator::InnerSizeAtCompileTime, + inner = Start % Evaluator::InnerSizeAtCompileTime }; - typedef typename Derived::Scalar Scalar; + typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func&) + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func&) { - return mat.coeffByOuterInner(outer, inner); + return eval.coeffByOuterInner(outer, inner); } }; // This is actually dead code and will never be called. It is required // to prevent false warnings regarding failed inlining though // for 0 length run() will never be called at all. -template<typename Func, typename Derived, int Start> -struct redux_novec_unroller<Func, Derived, Start, 0> +template<typename Func, typename Evaluator, int Start> +struct redux_novec_unroller<Func, Evaluator, Start, 0> { - typedef typename Derived::Scalar Scalar; + typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Derived&, const Func&) { return Scalar(); } + static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } }; /*** vectorization ***/ -template<typename Func, typename Derived, int Start, int Length> +template<typename Func, typename Evaluator, int Start, int Length> struct redux_vec_unroller { - enum { - PacketSize = redux_traits<Func, Derived>::PacketSize, - HalfLength = Length/2 - }; - - typedef typename Derived::Scalar Scalar; - typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; - - static EIGEN_STRONG_INLINE PacketScalar run(const Derived &mat, const Func& func) + template<typename PacketType> + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func) { + enum { + PacketSize = unpacket_traits<PacketType>::size, + HalfLength = Length/2 + }; + return func.packetOp( - redux_vec_unroller<Func, Derived, Start, HalfLength>::run(mat,func), - redux_vec_unroller<Func, Derived, Start+HalfLength, Length-HalfLength>::run(mat,func) ); + redux_vec_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval,func), + redux_vec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::template run<PacketType>(eval,func) ); } }; -template<typename Func, typename Derived, int Start> -struct redux_vec_unroller<Func, Derived, Start, 1> +template<typename Func, typename Evaluator, int Start> +struct redux_vec_unroller<Func, Evaluator, Start, 1> { - enum { - index = Start * redux_traits<Func, Derived>::PacketSize, - outer = index / int(Derived::InnerSizeAtCompileTime), - inner = index % int(Derived::InnerSizeAtCompileTime), - alignment = Derived::Alignment - }; - - typedef typename Derived::Scalar Scalar; - typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; - - static EIGEN_STRONG_INLINE PacketScalar run(const Derived &mat, const Func&) + template<typename PacketType> + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func&) { - return mat.template packetByOuterInner<alignment,PacketScalar>(outer, inner); + enum { + PacketSize = unpacket_traits<PacketType>::size, + index = Start * PacketSize, + outer = index / int(Evaluator::InnerSizeAtCompileTime), + inner = index % int(Evaluator::InnerSizeAtCompileTime), + alignment = Evaluator::Alignment + }; + return eval.template packetByOuterInner<alignment,PacketType>(outer, inner); } }; @@ -176,53 +182,65 @@ * Part 3 : implementation of all cases ***************************************************************************/ -template<typename Func, typename Derived, - int Traversal = redux_traits<Func, Derived>::Traversal, - int Unrolling = redux_traits<Func, Derived>::Unrolling +template<typename Func, typename Evaluator, + int Traversal = redux_traits<Func, Evaluator>::Traversal, + int Unrolling = redux_traits<Func, Evaluator>::Unrolling > struct redux_impl; -template<typename Func, typename Derived> -struct redux_impl<Func, Derived, DefaultTraversal, NoUnrolling> +template<typename Func, typename Evaluator> +struct redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling> { - typedef typename Derived::Scalar Scalar; - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) + typedef typename Evaluator::Scalar Scalar; + + template<typename XprType> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { - eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); + eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); Scalar res; - res = mat.coeffByOuterInner(0, 0); - for(Index i = 1; i < mat.innerSize(); ++i) - res = func(res, mat.coeffByOuterInner(0, i)); - for(Index i = 1; i < mat.outerSize(); ++i) - for(Index j = 0; j < mat.innerSize(); ++j) - res = func(res, mat.coeffByOuterInner(i, j)); + res = eval.coeffByOuterInner(0, 0); + for(Index i = 1; i < xpr.innerSize(); ++i) + res = func(res, eval.coeffByOuterInner(0, i)); + for(Index i = 1; i < xpr.outerSize(); ++i) + for(Index j = 0; j < xpr.innerSize(); ++j) + res = func(res, eval.coeffByOuterInner(i, j)); return res; } }; -template<typename Func, typename Derived> -struct redux_impl<Func,Derived, DefaultTraversal, CompleteUnrolling> - : public redux_novec_unroller<Func,Derived, 0, Derived::SizeAtCompileTime> -{}; - -template<typename Func, typename Derived> -struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling> +template<typename Func, typename Evaluator> +struct redux_impl<Func,Evaluator, DefaultTraversal, CompleteUnrolling> + : redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> { - typedef typename Derived::Scalar Scalar; - typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; - - static Scalar run(const Derived &mat, const Func& func) + typedef redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> Base; + typedef typename Evaluator::Scalar Scalar; + template<typename XprType> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& /*xpr*/) { - const Index size = mat.size(); + return Base::run(eval,func); + } +}; + +template<typename Func, typename Evaluator> +struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, NoUnrolling> +{ + typedef typename Evaluator::Scalar Scalar; + typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar; + + template<typename XprType> + static Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) + { + const Index size = xpr.size(); - const Index packetSize = redux_traits<Func, Derived>::PacketSize; + const Index packetSize = redux_traits<Func, Evaluator>::PacketSize; const int packetAlignment = unpacket_traits<PacketScalar>::alignment; enum { - alignment0 = (bool(Derived::Flags & DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned), - alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Derived::Alignment) + alignment0 = (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned), + alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Evaluator::Alignment) }; - const Index alignedStart = internal::first_default_aligned(mat.nestedExpression()); + const Index alignedStart = internal::first_default_aligned(xpr); const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize); const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize); const Index alignedEnd2 = alignedStart + alignedSize2; @@ -230,34 +248,34 @@ Scalar res; if(alignedSize) { - PacketScalar packet_res0 = mat.template packet<alignment,PacketScalar>(alignedStart); + PacketScalar packet_res0 = eval.template packet<alignment,PacketScalar>(alignedStart); if(alignedSize>packetSize) // we have at least two packets to partly unroll the loop { - PacketScalar packet_res1 = mat.template packet<alignment,PacketScalar>(alignedStart+packetSize); + PacketScalar packet_res1 = eval.template packet<alignment,PacketScalar>(alignedStart+packetSize); for(Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize) { - packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(index)); - packet_res1 = func.packetOp(packet_res1, mat.template packet<alignment,PacketScalar>(index+packetSize)); + packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(index)); + packet_res1 = func.packetOp(packet_res1, eval.template packet<alignment,PacketScalar>(index+packetSize)); } packet_res0 = func.packetOp(packet_res0,packet_res1); if(alignedEnd>alignedEnd2) - packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(alignedEnd2)); + packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(alignedEnd2)); } res = func.predux(packet_res0); for(Index index = 0; index < alignedStart; ++index) - res = func(res,mat.coeff(index)); + res = func(res,eval.coeff(index)); for(Index index = alignedEnd; index < size; ++index) - res = func(res,mat.coeff(index)); + res = func(res,eval.coeff(index)); } else // too small to vectorize anything. // since this is dynamic-size hence inefficient anyway for such small sizes, don't try to optimize. { - res = mat.coeff(0); + res = eval.coeff(0); for(Index index = 1; index < size; ++index) - res = func(res,mat.coeff(index)); + res = func(res,eval.coeff(index)); } return res; @@ -265,130 +283,108 @@ }; // NOTE: for SliceVectorizedTraversal we simply bypass unrolling -template<typename Func, typename Derived, int Unrolling> -struct redux_impl<Func, Derived, SliceVectorizedTraversal, Unrolling> +template<typename Func, typename Evaluator, int Unrolling> +struct redux_impl<Func, Evaluator, SliceVectorizedTraversal, Unrolling> { - typedef typename Derived::Scalar Scalar; - typedef typename redux_traits<Func, Derived>::PacketType PacketType; + typedef typename Evaluator::Scalar Scalar; + typedef typename redux_traits<Func, Evaluator>::PacketType PacketType; - EIGEN_DEVICE_FUNC static Scalar run(const Derived &mat, const Func& func) + template<typename XprType> + EIGEN_DEVICE_FUNC static Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { - eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); - const Index innerSize = mat.innerSize(); - const Index outerSize = mat.outerSize(); + eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); + const Index innerSize = xpr.innerSize(); + const Index outerSize = xpr.outerSize(); enum { - packetSize = redux_traits<Func, Derived>::PacketSize + packetSize = redux_traits<Func, Evaluator>::PacketSize }; const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize; Scalar res; if(packetedInnerSize) { - PacketType packet_res = mat.template packet<Unaligned,PacketType>(0,0); + PacketType packet_res = eval.template packet<Unaligned,PacketType>(0,0); for(Index j=0; j<outerSize; ++j) for(Index i=(j==0?packetSize:0); i<packetedInnerSize; i+=Index(packetSize)) - packet_res = func.packetOp(packet_res, mat.template packetByOuterInner<Unaligned,PacketType>(j,i)); + packet_res = func.packetOp(packet_res, eval.template packetByOuterInner<Unaligned,PacketType>(j,i)); res = func.predux(packet_res); for(Index j=0; j<outerSize; ++j) for(Index i=packetedInnerSize; i<innerSize; ++i) - res = func(res, mat.coeffByOuterInner(j,i)); + res = func(res, eval.coeffByOuterInner(j,i)); } else // too small to vectorize anything. // since this is dynamic-size hence inefficient anyway for such small sizes, don't try to optimize. { - res = redux_impl<Func, Derived, DefaultTraversal, NoUnrolling>::run(mat, func); + res = redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>::run(eval, func, xpr); } return res; } }; -template<typename Func, typename Derived> -struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> +template<typename Func, typename Evaluator> +struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, CompleteUnrolling> { - typedef typename Derived::Scalar Scalar; + typedef typename Evaluator::Scalar Scalar; - typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; + typedef typename redux_traits<Func, Evaluator>::PacketType PacketType; enum { - PacketSize = redux_traits<Func, Derived>::PacketSize, - Size = Derived::SizeAtCompileTime, + PacketSize = redux_traits<Func, Evaluator>::PacketSize, + Size = Evaluator::SizeAtCompileTime, VectorizedSize = (Size / PacketSize) * PacketSize }; - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) + + template<typename XprType> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType &xpr) { - eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); + EIGEN_ONLY_USED_FOR_DEBUG(xpr) + eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); if (VectorizedSize > 0) { - Scalar res = func.predux(redux_vec_unroller<Func, Derived, 0, Size / PacketSize>::run(mat,func)); + Scalar res = func.predux(redux_vec_unroller<Func, Evaluator, 0, Size / PacketSize>::template run<PacketType>(eval,func)); if (VectorizedSize != Size) - res = func(res,redux_novec_unroller<Func, Derived, VectorizedSize, Size-VectorizedSize>::run(mat,func)); + res = func(res,redux_novec_unroller<Func, Evaluator, VectorizedSize, Size-VectorizedSize>::run(eval,func)); return res; } else { - return redux_novec_unroller<Func, Derived, 0, Size>::run(mat,func); + return redux_novec_unroller<Func, Evaluator, 0, Size>::run(eval,func); } } }; // evaluator adaptor template<typename _XprType> -class redux_evaluator +class redux_evaluator : public internal::evaluator<_XprType> { + typedef internal::evaluator<_XprType> Base; public: typedef _XprType XprType; - EIGEN_DEVICE_FUNC explicit redux_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit redux_evaluator(const XprType &xpr) : Base(xpr) {} typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename XprType::PacketScalar PacketScalar; - typedef typename XprType::PacketReturnType PacketReturnType; enum { MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime, MaxColsAtCompileTime = XprType::MaxColsAtCompileTime, // TODO we should not remove DirectAccessBit and rather find an elegant way to query the alignment offset at runtime from the evaluator - Flags = evaluator<XprType>::Flags & ~DirectAccessBit, + Flags = Base::Flags & ~DirectAccessBit, IsRowMajor = XprType::IsRowMajor, SizeAtCompileTime = XprType::SizeAtCompileTime, - InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime, - CoeffReadCost = evaluator<XprType>::CoeffReadCost, - Alignment = evaluator<XprType>::Alignment + InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime }; - EIGEN_DEVICE_FUNC Index rows() const { return m_xpr.rows(); } - EIGEN_DEVICE_FUNC Index cols() const { return m_xpr.cols(); } - EIGEN_DEVICE_FUNC Index size() const { return m_xpr.size(); } - EIGEN_DEVICE_FUNC Index innerSize() const { return m_xpr.innerSize(); } - EIGEN_DEVICE_FUNC Index outerSize() const { return m_xpr.outerSize(); } - - EIGEN_DEVICE_FUNC - CoeffReturnType coeff(Index row, Index col) const - { return m_evaluator.coeff(row, col); } - - EIGEN_DEVICE_FUNC - CoeffReturnType coeff(Index index) const - { return m_evaluator.coeff(index); } - - template<int LoadMode, typename PacketType> - PacketType packet(Index row, Index col) const - { return m_evaluator.template packet<LoadMode,PacketType>(row, col); } - - template<int LoadMode, typename PacketType> - PacketType packet(Index index) const - { return m_evaluator.template packet<LoadMode,PacketType>(index); } - - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffByOuterInner(Index outer, Index inner) const - { return m_evaluator.coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + { return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } template<int LoadMode, typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetByOuterInner(Index outer, Index inner) const - { return m_evaluator.template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + { return Base::template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } - const XprType & nestedExpression() const { return m_xpr; } - -protected: - internal::evaluator<XprType> m_evaluator; - const XprType &m_xpr; }; } // end namespace internal @@ -403,52 +399,60 @@ * The template parameter \a BinaryOp is the type of the functor \a func which must be * an associative operator. Both current C++98 and C++11 functor styles are handled. * + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * * \sa DenseBase::sum(), DenseBase::minCoeff(), DenseBase::maxCoeff(), MatrixBase::colwise(), MatrixBase::rowwise() */ template<typename Derived> template<typename Func> -typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::redux(const Func& func) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); typedef typename internal::redux_evaluator<Derived> ThisEvaluator; ThisEvaluator thisEval(derived()); - - return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func); + + // The initial expression is passed to the reducer as an additional argument instead of + // passing it as a member of redux_evaluator to help + return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func, derived()); } /** \returns the minimum of all coefficients of \c *this. + * \warning the matrix must be not empty, otherwise an assertion is triggered. * \warning the result is undefined if \c *this contains NaN. */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff() const { - return derived().redux(Eigen::internal::scalar_min_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_min_op<Scalar,Scalar>()); } /** \returns the maximum of all coefficients of \c *this. + * \warning the matrix must be not empty, otherwise an assertion is triggered. * \warning the result is undefined if \c *this contains NaN. */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff() const { - return derived().redux(Eigen::internal::scalar_max_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_max_op<Scalar,Scalar>()); } -/** \returns the sum of all coefficients of *this +/** \returns the sum of all coefficients of \c *this + * + * If \c *this is empty, then the value 0 is returned. * * \sa trace(), prod(), mean() */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::sum() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) return Scalar(0); - return derived().redux(Eigen::internal::scalar_sum_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>()); } /** \returns the mean of all coefficients of *this @@ -456,10 +460,17 @@ * \sa trace(), prod(), sum() */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::mean() const { - return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar>())) / Scalar(this->size()); +#ifdef __INTEL_COMPILER + #pragma warning push + #pragma warning ( disable : 2259 ) +#endif + return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>())) / Scalar(this->size()); +#ifdef __INTEL_COMPILER + #pragma warning pop +#endif } /** \returns the product of all coefficients of *this @@ -470,7 +481,7 @@ * \sa sum(), mean(), trace() */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::prod() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) @@ -485,7 +496,7 @@ * \sa diagonal(), sum() */ template<typename Derived> -EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar MatrixBase<Derived>::trace() const { return derived().diagonal().sum();
diff --git a/Eigen/src/Core/Ref.h b/Eigen/src/Core/Ref.h index 6e94181..ac9502b 100644 --- a/Eigen/src/Core/Ref.h +++ b/Eigen/src/Core/Ref.h
@@ -35,7 +35,13 @@ || (int(StrideType::InnerStrideAtCompileTime)==0 && int(Derived::InnerStrideAtCompileTime)==1), OuterStrideMatch = Derived::IsVectorAtCompileTime || int(StrideType::OuterStrideAtCompileTime)==int(Dynamic) || int(StrideType::OuterStrideAtCompileTime)==int(Derived::OuterStrideAtCompileTime), - AlignmentMatch = (int(traits<PlainObjectType>::Alignment)==int(Unaligned)) || (int(evaluator<Derived>::Alignment) >= int(Alignment)), // FIXME the first condition is not very clear, it should be replaced by the required alignment + // NOTE, this indirection of evaluator<Derived>::Alignment is needed + // to workaround a very strange bug in MSVC related to the instantiation + // of has_*ary_operator in evaluator<CwiseNullaryOp>. + // This line is surprisingly very sensitive. For instance, simply adding parenthesis + // as "DerivedAlignment = (int(evaluator<Derived>::Alignment))," will make MSVC fail... + DerivedAlignment = int(evaluator<Derived>::Alignment), + AlignmentMatch = (int(traits<PlainObjectType>::Alignment)==int(Unaligned)) || (DerivedAlignment >= int(Alignment)), // FIXME the first condition is not very clear, it should be replaced by the required alignment ScalarTypeMatch = internal::is_same<typename PlainObjectType::Scalar, typename Derived::Scalar>::value, MatchAtCompileTime = HasDirectAccess && StorageOrderMatch && InnerStrideMatch && OuterStrideMatch && AlignmentMatch && ScalarTypeMatch }; @@ -89,6 +95,8 @@ template<typename Expression> EIGEN_DEVICE_FUNC void construct(Expression& expr) { + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(PlainObjectType,Expression); + if(PlainObjectType::RowsAtCompileTime==1) { eigen_assert(expr.rows()==1 || expr.cols()==1); @@ -178,6 +186,8 @@ * void foo(const Ref<MatrixXf,0,Stride<> >& A) { foo_impl(A); } * \endcode * + * See also the following stackoverflow questions for further references: + * - <a href="http://stackoverflow.com/questions/21132538/correct-usage-of-the-eigenref-class">Correct usage of the Eigen::Ref<> class</a> * * \sa PlainObjectBase::Map(), \ref TopicStorageOrders */ @@ -262,7 +272,7 @@ template<typename Expression> EIGEN_DEVICE_FUNC void construct(const Expression& expr, internal::false_type) { - internal::call_assignment_no_alias(m_object,expr,internal::assign_op<Scalar>()); + internal::call_assignment_no_alias(m_object,expr,internal::assign_op<Scalar,Scalar>()); Base::construct(m_object); }
diff --git a/Eigen/src/Core/Replicate.h b/Eigen/src/Core/Replicate.h index 9960ef8..0b2d6d7 100644 --- a/Eigen/src/Core/Replicate.h +++ b/Eigen/src/Core/Replicate.h
@@ -115,7 +115,7 @@ */ template<typename Derived> template<int RowFactor, int ColFactor> -const Replicate<Derived,RowFactor,ColFactor> +EIGEN_DEVICE_FUNC const Replicate<Derived,RowFactor,ColFactor> DenseBase<Derived>::replicate() const { return Replicate<Derived,RowFactor,ColFactor>(derived()); @@ -130,7 +130,7 @@ * \sa VectorwiseOp::replicate(), DenseBase::replicate(), class Replicate */ template<typename ExpressionType, int Direction> -const typename VectorwiseOp<ExpressionType,Direction>::ReplicateReturnType +EIGEN_DEVICE_FUNC const typename VectorwiseOp<ExpressionType,Direction>::ReplicateReturnType VectorwiseOp<ExpressionType,Direction>::replicate(Index factor) const { return typename VectorwiseOp<ExpressionType,Direction>::ReplicateReturnType
diff --git a/Eigen/src/Core/Reshaped.h b/Eigen/src/Core/Reshaped.h new file mode 100644 index 0000000..c955815 --- /dev/null +++ b/Eigen/src/Core/Reshaped.h
@@ -0,0 +1,453 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2014 yoco <peter.xiau@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_RESHAPED_H +#define EIGEN_RESHAPED_H + +namespace Eigen { + +/** \class Reshaped + * \ingroup Core_Module + * + * \brief Expression of a fixed-size or dynamic-size reshape + * + * \tparam XprType the type of the expression in which we are taking a reshape + * \tparam Rows the number of rows of the reshape we are taking at compile time (optional) + * \tparam Cols the number of columns of the reshape we are taking at compile time (optional) + * \tparam Order can be ColMajor or RowMajor, default is ColMajor. + * + * This class represents an expression of either a fixed-size or dynamic-size reshape. + * It is the return type of DenseBase::reshaped(NRowsType,NColsType) and + * most of the time this is the only way it is used. + * + * However, in C++98, if you want to directly maniputate reshaped expressions, + * for instance if you want to write a function returning such an expression, you + * will need to use this class. In C++11, it is advised to use the \em auto + * keyword for such use cases. + * + * Here is an example illustrating the dynamic case: + * \include class_Reshaped.cpp + * Output: \verbinclude class_Reshaped.out + * + * Here is an example illustrating the fixed-size case: + * \include class_FixedReshaped.cpp + * Output: \verbinclude class_FixedReshaped.out + * + * \sa DenseBase::reshaped(NRowsType,NColsType) + */ + +namespace internal { +template<typename XprType, int Rows, int Cols, int Order> +struct traits<Reshaped<XprType, Rows, Cols, Order> > : traits<XprType> +{ + typedef typename traits<XprType>::Scalar Scalar; + typedef typename traits<XprType>::StorageKind StorageKind; + typedef typename traits<XprType>::XprKind XprKind; + enum{ + MatrixRows = traits<XprType>::RowsAtCompileTime, + MatrixCols = traits<XprType>::ColsAtCompileTime, + RowsAtCompileTime = Rows, + ColsAtCompileTime = Cols, + MaxRowsAtCompileTime = Rows, + MaxColsAtCompileTime = Cols, + XpxStorageOrder = ((int(traits<XprType>::Flags) & RowMajorBit) == RowMajorBit) ? RowMajor : ColMajor, + ReshapedStorageOrder = (RowsAtCompileTime == 1 && ColsAtCompileTime != 1) ? RowMajor + : (ColsAtCompileTime == 1 && RowsAtCompileTime != 1) ? ColMajor + : XpxStorageOrder, + HasSameStorageOrderAsXprType = (ReshapedStorageOrder == XpxStorageOrder), + InnerSize = (ReshapedStorageOrder==int(RowMajor)) ? int(ColsAtCompileTime) : int(RowsAtCompileTime), + InnerStrideAtCompileTime = HasSameStorageOrderAsXprType + ? int(inner_stride_at_compile_time<XprType>::ret) + : Dynamic, + OuterStrideAtCompileTime = Dynamic, + + HasDirectAccess = internal::has_direct_access<XprType>::ret + && (Order==int(XpxStorageOrder)) + && ((evaluator<XprType>::Flags&LinearAccessBit)==LinearAccessBit), + + MaskPacketAccessBit = (InnerSize == Dynamic || (InnerSize % packet_traits<Scalar>::size) == 0) + && (InnerStrideAtCompileTime == 1) + ? PacketAccessBit : 0, + //MaskAlignedBit = ((OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % 16) == 0)) ? AlignedBit : 0, + FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1) ? LinearAccessBit : 0, + FlagsLvalueBit = is_lvalue<XprType>::value ? LvalueBit : 0, + FlagsRowMajorBit = (ReshapedStorageOrder==int(RowMajor)) ? RowMajorBit : 0, + FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0, + Flags0 = traits<XprType>::Flags & ( (HereditaryBits & ~RowMajorBit) | MaskPacketAccessBit), + + Flags = (Flags0 | FlagsLinearAccessBit | FlagsLvalueBit | FlagsRowMajorBit | FlagsDirectAccessBit) + }; +}; + +template<typename XprType, int Rows, int Cols, int Order, bool HasDirectAccess> class ReshapedImpl_dense; + +} // end namespace internal + +template<typename XprType, int Rows, int Cols, int Order, typename StorageKind> class ReshapedImpl; + +template<typename XprType, int Rows, int Cols, int Order> class Reshaped + : public ReshapedImpl<XprType, Rows, Cols, Order, typename internal::traits<XprType>::StorageKind> +{ + typedef ReshapedImpl<XprType, Rows, Cols, Order, typename internal::traits<XprType>::StorageKind> Impl; + public: + //typedef typename Impl::Base Base; + typedef Impl Base; + EIGEN_GENERIC_PUBLIC_INTERFACE(Reshaped) + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Reshaped) + + /** Fixed-size constructor + */ + EIGEN_DEVICE_FUNC + inline Reshaped(XprType& xpr) + : Impl(xpr) + { + EIGEN_STATIC_ASSERT(RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic,THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE) + eigen_assert(Rows * Cols == xpr.rows() * xpr.cols()); + } + + /** Dynamic-size constructor + */ + EIGEN_DEVICE_FUNC + inline Reshaped(XprType& xpr, + Index reshapeRows, Index reshapeCols) + : Impl(xpr, reshapeRows, reshapeCols) + { + eigen_assert((RowsAtCompileTime==Dynamic || RowsAtCompileTime==reshapeRows) + && (ColsAtCompileTime==Dynamic || ColsAtCompileTime==reshapeCols)); + eigen_assert(reshapeRows * reshapeCols == xpr.rows() * xpr.cols()); + } +}; + +// The generic default implementation for dense reshape simply forward to the internal::ReshapedImpl_dense +// that must be specialized for direct and non-direct access... +template<typename XprType, int Rows, int Cols, int Order> +class ReshapedImpl<XprType, Rows, Cols, Order, Dense> + : public internal::ReshapedImpl_dense<XprType, Rows, Cols, Order,internal::traits<Reshaped<XprType,Rows,Cols,Order> >::HasDirectAccess> +{ + typedef internal::ReshapedImpl_dense<XprType, Rows, Cols, Order,internal::traits<Reshaped<XprType,Rows,Cols,Order> >::HasDirectAccess> Impl; + public: + typedef Impl Base; + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl) + EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr) : Impl(xpr) {} + EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr, Index reshapeRows, Index reshapeCols) + : Impl(xpr, reshapeRows, reshapeCols) {} +}; + +namespace internal { + +/** \internal Internal implementation of dense Reshaped in the general case. */ +template<typename XprType, int Rows, int Cols, int Order> +class ReshapedImpl_dense<XprType,Rows,Cols,Order,false> + : public internal::dense_xpr_base<Reshaped<XprType, Rows, Cols, Order> >::type +{ + typedef Reshaped<XprType, Rows, Cols, Order> ReshapedType; + public: + + typedef typename internal::dense_xpr_base<ReshapedType>::type Base; + EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType) + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense) + + typedef typename internal::ref_selector<XprType>::non_const_type MatrixTypeNested; + typedef typename internal::remove_all<XprType>::type NestedExpression; + + class InnerIterator; + + /** Fixed-size constructor + */ + EIGEN_DEVICE_FUNC + inline ReshapedImpl_dense(XprType& xpr) + : m_xpr(xpr), m_rows(Rows), m_cols(Cols) + {} + + /** Dynamic-size constructor + */ + EIGEN_DEVICE_FUNC + inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols) + : m_xpr(xpr), m_rows(nRows), m_cols(nCols) + {} + + EIGEN_DEVICE_FUNC Index rows() const { return m_rows; } + EIGEN_DEVICE_FUNC Index cols() const { return m_cols; } + + #ifdef EIGEN_PARSED_BY_DOXYGEN + /** \sa MapBase::data() */ + EIGEN_DEVICE_FUNC inline const Scalar* data() const; + EIGEN_DEVICE_FUNC inline Index innerStride() const; + EIGEN_DEVICE_FUNC inline Index outerStride() const; + #endif + + /** \returns the nested expression */ + EIGEN_DEVICE_FUNC + const typename internal::remove_all<XprType>::type& + nestedExpression() const { return m_xpr; } + + /** \returns the nested expression */ + EIGEN_DEVICE_FUNC + typename internal::remove_reference<XprType>::type& + nestedExpression() { return m_xpr; } + + protected: + + MatrixTypeNested m_xpr; + const internal::variable_if_dynamic<Index, Rows> m_rows; + const internal::variable_if_dynamic<Index, Cols> m_cols; +}; + + +/** \internal Internal implementation of dense Reshaped in the direct access case. */ +template<typename XprType, int Rows, int Cols, int Order> +class ReshapedImpl_dense<XprType, Rows, Cols, Order, true> + : public MapBase<Reshaped<XprType, Rows, Cols, Order> > +{ + typedef Reshaped<XprType, Rows, Cols, Order> ReshapedType; + typedef typename internal::ref_selector<XprType>::non_const_type XprTypeNested; + public: + + typedef MapBase<ReshapedType> Base; + EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType) + EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense) + + /** Fixed-size constructor + */ + EIGEN_DEVICE_FUNC + inline ReshapedImpl_dense(XprType& xpr) + : Base(xpr.data()), m_xpr(xpr) + {} + + /** Dynamic-size constructor + */ + EIGEN_DEVICE_FUNC + inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols) + : Base(xpr.data(), nRows, nCols), + m_xpr(xpr) + {} + + EIGEN_DEVICE_FUNC + const typename internal::remove_all<XprTypeNested>::type& nestedExpression() const + { + return m_xpr; + } + + EIGEN_DEVICE_FUNC + XprType& nestedExpression() { return m_xpr; } + + /** \sa MapBase::innerStride() */ + EIGEN_DEVICE_FUNC + inline Index innerStride() const + { + return m_xpr.innerStride(); + } + + /** \sa MapBase::outerStride() */ + EIGEN_DEVICE_FUNC + inline Index outerStride() const + { + return ((Flags&RowMajorBit)==RowMajorBit) ? this->cols() : this->rows(); + } + + protected: + + XprTypeNested m_xpr; +}; + +// Evaluators +template<typename ArgType, int Rows, int Cols, int Order, bool HasDirectAccess> struct reshaped_evaluator; + +template<typename ArgType, int Rows, int Cols, int Order> +struct evaluator<Reshaped<ArgType, Rows, Cols, Order> > + : reshaped_evaluator<ArgType, Rows, Cols, Order, traits<Reshaped<ArgType,Rows,Cols,Order> >::HasDirectAccess> +{ + typedef Reshaped<ArgType, Rows, Cols, Order> XprType; + typedef typename XprType::Scalar Scalar; + // TODO: should check for smaller packet types + typedef typename packet_traits<Scalar>::type PacketScalar; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + HasDirectAccess = traits<XprType>::HasDirectAccess, + +// RowsAtCompileTime = traits<XprType>::RowsAtCompileTime, +// ColsAtCompileTime = traits<XprType>::ColsAtCompileTime, +// MaxRowsAtCompileTime = traits<XprType>::MaxRowsAtCompileTime, +// MaxColsAtCompileTime = traits<XprType>::MaxColsAtCompileTime, +// +// InnerStrideAtCompileTime = traits<XprType>::HasSameStorageOrderAsXprType +// ? int(inner_stride_at_compile_time<ArgType>::ret) +// : Dynamic, +// OuterStrideAtCompileTime = Dynamic, + + FlagsLinearAccessBit = (traits<XprType>::RowsAtCompileTime == 1 || traits<XprType>::ColsAtCompileTime == 1 || HasDirectAccess) ? LinearAccessBit : 0, + FlagsRowMajorBit = (traits<XprType>::ReshapedStorageOrder==int(RowMajor)) ? RowMajorBit : 0, + FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0, + Flags0 = evaluator<ArgType>::Flags & (HereditaryBits & ~RowMajorBit), + Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit | FlagsDirectAccessBit, + + PacketAlignment = unpacket_traits<PacketScalar>::alignment, + Alignment = evaluator<ArgType>::Alignment + }; + typedef reshaped_evaluator<ArgType, Rows, Cols, Order, HasDirectAccess> reshaped_evaluator_type; + EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : reshaped_evaluator_type(xpr) + { + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } +}; + +template<typename ArgType, int Rows, int Cols, int Order> +struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ false> + : evaluator_base<Reshaped<ArgType, Rows, Cols, Order> > +{ + typedef Reshaped<ArgType, Rows, Cols, Order> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost /* TODO + cost of index computations */, + + Flags = (evaluator<ArgType>::Flags & (HereditaryBits /*| LinearAccessBit | DirectAccessBit*/)), + + Alignment = 0 + }; + + EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) + { + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + typedef std::pair<Index, Index> RowCol; + + inline RowCol index_remap(Index rowId, Index colId) const + { + if(Order==ColMajor) + { + const Index nth_elem_idx = colId * m_xpr.rows() + rowId; + return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), + nth_elem_idx / m_xpr.nestedExpression().rows()); + } + else + { + const Index nth_elem_idx = colId + rowId * m_xpr.cols(); + return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), + nth_elem_idx % m_xpr.nestedExpression().cols()); + } + } + + EIGEN_DEVICE_FUNC + inline Scalar& coeffRef(Index rowId, Index colId) + { + EIGEN_STATIC_ASSERT_LVALUE(XprType) + const RowCol row_col = index_remap(rowId, colId); + return m_argImpl.coeffRef(row_col.first, row_col.second); + } + + EIGEN_DEVICE_FUNC + inline const Scalar& coeffRef(Index rowId, Index colId) const + { + const RowCol row_col = index_remap(rowId, colId); + return m_argImpl.coeffRef(row_col.first, row_col.second); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index rowId, Index colId) const + { + const RowCol row_col = index_remap(rowId, colId); + return m_argImpl.coeff(row_col.first, row_col.second); + } + + EIGEN_DEVICE_FUNC + inline Scalar& coeffRef(Index index) + { + EIGEN_STATIC_ASSERT_LVALUE(XprType) + const RowCol row_col = index_remap(Rows == 1 ? 0 : index, + Rows == 1 ? index : 0); + return m_argImpl.coeffRef(row_col.first, row_col.second); + + } + + EIGEN_DEVICE_FUNC + inline const Scalar& coeffRef(Index index) const + { + const RowCol row_col = index_remap(Rows == 1 ? 0 : index, + Rows == 1 ? index : 0); + return m_argImpl.coeffRef(row_col.first, row_col.second); + } + + EIGEN_DEVICE_FUNC + inline const CoeffReturnType coeff(Index index) const + { + const RowCol row_col = index_remap(Rows == 1 ? 0 : index, + Rows == 1 ? index : 0); + return m_argImpl.coeff(row_col.first, row_col.second); + } +#if 0 + EIGEN_DEVICE_FUNC + template<int LoadMode> + inline PacketScalar packet(Index rowId, Index colId) const + { + const RowCol row_col = index_remap(rowId, colId); + return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second); + + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC + inline void writePacket(Index rowId, Index colId, const PacketScalar& val) + { + const RowCol row_col = index_remap(rowId, colId); + m_argImpl.const_cast_derived().template writePacket<Unaligned> + (row_col.first, row_col.second, val); + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC + inline PacketScalar packet(Index index) const + { + const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second); + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC + inline void writePacket(Index index, const PacketScalar& val) + { + const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second, val); + } +#endif +protected: + + evaluator<ArgType> m_argImpl; + const XprType& m_xpr; + +}; + +template<typename ArgType, int Rows, int Cols, int Order> +struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ true> +: mapbase_evaluator<Reshaped<ArgType, Rows, Cols, Order>, + typename Reshaped<ArgType, Rows, Cols, Order>::PlainObject> +{ + typedef Reshaped<ArgType, Rows, Cols, Order> XprType; + typedef typename XprType::Scalar Scalar; + + EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr) + : mapbase_evaluator<XprType, typename XprType::PlainObject>(xpr) + { + // TODO: for the 3.4 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime + eigen_assert(((internal::UIntPtr(xpr.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator<XprType>::Alignment)) == 0) && "data is not aligned"); + } +}; + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_RESHAPED_H
diff --git a/Eigen/src/Core/ReturnByValue.h b/Eigen/src/Core/ReturnByValue.h index c44b767..11dc86d 100644 --- a/Eigen/src/Core/ReturnByValue.h +++ b/Eigen/src/Core/ReturnByValue.h
@@ -79,7 +79,7 @@ template<typename Derived> template<typename OtherDerived> -Derived& DenseBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) +EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) { other.evalTo(derived()); return derived();
diff --git a/Eigen/src/Core/Reverse.h b/Eigen/src/Core/Reverse.h index 0640cda..711dbcf 100644 --- a/Eigen/src/Core/Reverse.h +++ b/Eigen/src/Core/Reverse.h
@@ -114,7 +114,7 @@ * */ template<typename Derived> -inline typename DenseBase<Derived>::ReverseReturnType +EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::ReverseReturnType DenseBase<Derived>::reverse() { return ReverseReturnType(derived()); @@ -136,7 +136,7 @@ * * \sa VectorwiseOp::reverseInPlace(), reverse() */ template<typename Derived> -inline void DenseBase<Derived>::reverseInPlace() +EIGEN_DEVICE_FUNC inline void DenseBase<Derived>::reverseInPlace() { if(cols()>rows()) { @@ -201,9 +201,9 @@ * * \sa DenseBase::reverseInPlace(), reverse() */ template<typename ExpressionType, int Direction> -void VectorwiseOp<ExpressionType,Direction>::reverseInPlace() +EIGEN_DEVICE_FUNC void VectorwiseOp<ExpressionType,Direction>::reverseInPlace() { - internal::vectorwise_reverse_inplace_impl<Direction>::run(_expression().const_cast_derived()); + internal::vectorwise_reverse_inplace_impl<Direction>::run(m_matrix); } } // end namespace Eigen
diff --git a/Eigen/src/Core/SelfAdjointView.h b/Eigen/src/Core/SelfAdjointView.h index 9fda026..2173799 100644 --- a/Eigen/src/Core/SelfAdjointView.h +++ b/Eigen/src/Core/SelfAdjointView.h
@@ -45,7 +45,7 @@ }; } -// FIXME could also be called SelfAdjointWrapper to be consistent with DiagonalWrapper ?? + template<typename _MatrixType, unsigned int UpLo> class SelfAdjointView : public TriangularBase<SelfAdjointView<_MatrixType, UpLo> > { @@ -55,20 +55,26 @@ typedef TriangularBase<SelfAdjointView> Base; typedef typename internal::traits<SelfAdjointView>::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits<SelfAdjointView>::MatrixTypeNestedCleaned MatrixTypeNestedCleaned; + typedef MatrixTypeNestedCleaned NestedExpression; /** \brief The type of coefficients in this matrix */ typedef typename internal::traits<SelfAdjointView>::Scalar Scalar; typedef typename MatrixType::StorageIndex StorageIndex; + typedef typename internal::remove_all<typename MatrixType::ConjugateReturnType>::type MatrixConjugateReturnType; + typedef SelfAdjointView<typename internal::add_const<MatrixType>::type, UpLo> ConstSelfAdjointView; enum { Mode = internal::traits<SelfAdjointView>::Mode, - Flags = internal::traits<SelfAdjointView>::Flags + Flags = internal::traits<SelfAdjointView>::Flags, + TransposeMode = ((Mode & Upper) ? Lower : 0) | ((Mode & Lower) ? Upper : 0) }; typedef typename MatrixType::PlainObject PlainObject; EIGEN_DEVICE_FUNC explicit inline SelfAdjointView(MatrixType& matrix) : m_matrix(matrix) - {} + { + EIGEN_STATIC_ASSERT(UpLo==Lower || UpLo==Upper,SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY); + } EIGEN_DEVICE_FUNC inline Index rows() const { return m_matrix.rows(); } @@ -128,7 +134,7 @@ } friend EIGEN_DEVICE_FUNC - const SelfAdjointView<const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,MatrixType>,UpLo> + const SelfAdjointView<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar,MatrixType,product),UpLo> operator*(const Scalar& s, const SelfAdjointView& mat) { return (s*mat.nestedExpression()).template selfadjointView<UpLo>(); @@ -162,6 +168,83 @@ EIGEN_DEVICE_FUNC SelfAdjointView& rankUpdate(const MatrixBase<DerivedU>& u, const Scalar& alpha = Scalar(1)); + /** \returns an expression of a triangular view extracted from the current selfadjoint view of a given triangular part + * + * The parameter \a TriMode can have the following values: \c #Upper, \c #StrictlyUpper, \c #UnitUpper, + * \c #Lower, \c #StrictlyLower, \c #UnitLower. + * + * If \c TriMode references the same triangular part than \c *this, then this method simply return a \c TriangularView of the nested expression, + * otherwise, the nested expression is first transposed, thus returning a \c TriangularView<Transpose<MatrixType>> object. + * + * \sa MatrixBase::triangularView(), class TriangularView + */ + template<unsigned int TriMode> + EIGEN_DEVICE_FUNC + typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), + TriangularView<MatrixType,TriMode>, + TriangularView<typename MatrixType::AdjointReturnType,TriMode> >::type + triangularView() const + { + typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::ConstTransposeReturnType>::type tmp1(m_matrix); + typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::AdjointReturnType>::type tmp2(tmp1); + return typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), + TriangularView<MatrixType,TriMode>, + TriangularView<typename MatrixType::AdjointReturnType,TriMode> >::type(tmp2); + } + + typedef SelfAdjointView<const MatrixConjugateReturnType,UpLo> ConjugateReturnType; + /** \sa MatrixBase::conjugate() const */ + EIGEN_DEVICE_FUNC + inline const ConjugateReturnType conjugate() const + { return ConjugateReturnType(m_matrix.conjugate()); } + + /** \returns an expression of the complex conjugate of \c *this if Cond==true, + * returns \c *this otherwise. + */ + template<bool Cond> + EIGEN_DEVICE_FUNC + inline typename internal::conditional<Cond,ConjugateReturnType,ConstSelfAdjointView>::type + conjugateIf() const + { + typedef typename internal::conditional<Cond,ConjugateReturnType,ConstSelfAdjointView>::type ReturnType; + return ReturnType(m_matrix.template conjugateIf<Cond>()); + } + + typedef SelfAdjointView<const typename MatrixType::AdjointReturnType,TransposeMode> AdjointReturnType; + /** \sa MatrixBase::adjoint() const */ + EIGEN_DEVICE_FUNC + inline const AdjointReturnType adjoint() const + { return AdjointReturnType(m_matrix.adjoint()); } + + typedef SelfAdjointView<typename MatrixType::TransposeReturnType,TransposeMode> TransposeReturnType; + /** \sa MatrixBase::transpose() */ + EIGEN_DEVICE_FUNC + inline TransposeReturnType transpose() + { + EIGEN_STATIC_ASSERT_LVALUE(MatrixType) + typename MatrixType::TransposeReturnType tmp(m_matrix); + return TransposeReturnType(tmp); + } + + typedef SelfAdjointView<const typename MatrixType::ConstTransposeReturnType,TransposeMode> ConstTransposeReturnType; + /** \sa MatrixBase::transpose() const */ + EIGEN_DEVICE_FUNC + inline const ConstTransposeReturnType transpose() const + { + return ConstTransposeReturnType(m_matrix.transpose()); + } + + /** \returns a const expression of the main diagonal of the matrix \c *this + * + * This method simply returns the diagonal of the nested expression, thus by-passing the SelfAdjointView decorator. + * + * \sa MatrixBase::diagonal(), class Diagonal */ + EIGEN_DEVICE_FUNC + typename MatrixType::ConstDiagonalReturnType diagonal() const + { + return typename MatrixType::ConstDiagonalReturnType(m_matrix); + } + /////////// Cholesky module /////////// const LLT<PlainObject, UpLo> llt() const; @@ -251,17 +334,27 @@ * Implementation of MatrixBase methods ***************************************************************************/ +/** This is the const version of MatrixBase::selfadjointView() */ template<typename Derived> template<unsigned int UpLo> -typename MatrixBase<Derived>::template ConstSelfAdjointViewReturnType<UpLo>::Type +EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template ConstSelfAdjointViewReturnType<UpLo>::Type MatrixBase<Derived>::selfadjointView() const { return typename ConstSelfAdjointViewReturnType<UpLo>::Type(derived()); } +/** \returns an expression of a symmetric/self-adjoint view extracted from the upper or lower triangular part of the current matrix + * + * The parameter \a UpLo can be either \c #Upper or \c #Lower + * + * Example: \include MatrixBase_selfadjointView.cpp + * Output: \verbinclude MatrixBase_selfadjointView.out + * + * \sa class SelfAdjointView + */ template<typename Derived> template<unsigned int UpLo> -typename MatrixBase<Derived>::template SelfAdjointViewReturnType<UpLo>::Type +EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template SelfAdjointViewReturnType<UpLo>::Type MatrixBase<Derived>::selfadjointView() { return typename SelfAdjointViewReturnType<UpLo>::Type(derived());
diff --git a/Eigen/src/Core/SelfCwiseBinaryOp.h b/Eigen/src/Core/SelfCwiseBinaryOp.h index 78fff15..7c89c2e 100644 --- a/Eigen/src/Core/SelfCwiseBinaryOp.h +++ b/Eigen/src/Core/SelfCwiseBinaryOp.h
@@ -12,35 +12,33 @@ namespace Eigen { +// TODO generalize the scalar type of 'other' + template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator*=(const Scalar& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator*=(const Scalar& other) { - typedef typename Derived::PlainObject PlainObject; - internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::mul_assign_op<Scalar>()); + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::mul_assign_op<Scalar,Scalar>()); return derived(); } template<typename Derived> -EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator+=(const Scalar& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator+=(const Scalar& other) { - typedef typename Derived::PlainObject PlainObject; - internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::add_assign_op<Scalar>()); + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::add_assign_op<Scalar,Scalar>()); return derived(); } template<typename Derived> -EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator-=(const Scalar& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator-=(const Scalar& other) { - typedef typename Derived::PlainObject PlainObject; - internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::sub_assign_op<Scalar>()); + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::sub_assign_op<Scalar,Scalar>()); return derived(); } template<typename Derived> -EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator/=(const Scalar& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator/=(const Scalar& other) { - typedef typename Derived::PlainObject PlainObject; - internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::div_assign_op<Scalar>()); + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::div_assign_op<Scalar,Scalar>()); return derived(); }
diff --git a/Eigen/src/Core/Solve.h b/Eigen/src/Core/Solve.h index ba2ee53..ec4b4a9 100644 --- a/Eigen/src/Core/Solve.h +++ b/Eigen/src/Core/Solve.h
@@ -19,7 +19,7 @@ * * \brief Pseudo expression representing a solving operation * - * \tparam Decomposition the type of the matrix or decomposion object + * \tparam Decomposition the type of the matrix or decomposition object * \tparam Rhstype the type of the right-hand side * * This class represents an expression of A.solve(B) @@ -34,12 +34,12 @@ template<typename Decomposition, typename RhsType> struct solve_traits<Decomposition,RhsType,Dense> { - typedef Matrix<typename RhsType::Scalar, + typedef typename make_proper_matrix_type<typename RhsType::Scalar, Decomposition::ColsAtCompileTime, RhsType::ColsAtCompileTime, RhsType::PlainObject::Options, Decomposition::MaxColsAtCompileTime, - RhsType::MaxColsAtCompileTime> PlainObject; + RhsType::MaxColsAtCompileTime>::type PlainObject; }; template<typename Decomposition, typename RhsType> @@ -134,39 +134,54 @@ // Specialization for "dst = dec.solve(rhs)" // NOTE we need to specialize it for Dense2Dense to avoid ambiguous specialization error and a Sparse2Sparse specialization must exist somewhere template<typename DstXprType, typename DecType, typename RhsType, typename Scalar> -struct Assignment<DstXprType, Solve<DecType,RhsType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, Solve<DecType,RhsType>, internal::assign_op<Scalar,Scalar>, Dense2Dense> { typedef Solve<DecType,RhsType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { - // FIXME shall we resize dst here? + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + src.dec()._solve_impl(src.rhs(), dst); } }; // Specialization for "dst = dec.transpose().solve(rhs)" template<typename DstXprType, typename DecType, typename RhsType, typename Scalar> -struct Assignment<DstXprType, Solve<Transpose<const DecType>,RhsType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, Solve<Transpose<const DecType>,RhsType>, internal::assign_op<Scalar,Scalar>, Dense2Dense> { typedef Solve<Transpose<const DecType>,RhsType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + src.dec().nestedExpression().template _solve_impl_transposed<false>(src.rhs(), dst); } }; // Specialization for "dst = dec.adjoint().solve(rhs)" template<typename DstXprType, typename DecType, typename RhsType, typename Scalar> -struct Assignment<DstXprType, Solve<CwiseUnaryOp<internal::scalar_conjugate_op<typename DecType::Scalar>, const Transpose<const DecType> >,RhsType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, Solve<CwiseUnaryOp<internal::scalar_conjugate_op<typename DecType::Scalar>, const Transpose<const DecType> >,RhsType>, + internal::assign_op<Scalar,Scalar>, Dense2Dense> { typedef Solve<CwiseUnaryOp<internal::scalar_conjugate_op<typename DecType::Scalar>, const Transpose<const DecType> >,RhsType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + src.dec().nestedExpression().nestedExpression().template _solve_impl_transposed<true>(src.rhs(), dst); } }; -} // end namepsace internal +} // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/SolveTriangular.h b/Eigen/src/Core/SolveTriangular.h index a333564..2e061ca 100644 --- a/Eigen/src/Core/SolveTriangular.h +++ b/Eigen/src/Core/SolveTriangular.h
@@ -161,15 +161,19 @@ * TriangularView methods ***************************************************************************/ +#ifndef EIGEN_PARSED_BY_DOXYGEN template<typename MatrixType, unsigned int Mode> template<int Side, typename OtherDerived> -void TriangularViewImpl<MatrixType,Mode,Dense>::solveInPlace(const MatrixBase<OtherDerived>& _other) const +EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType,Mode,Dense>::solveInPlace(const MatrixBase<OtherDerived>& _other) const { OtherDerived& other = _other.const_cast_derived(); eigen_assert( derived().cols() == derived().rows() && ((Side==OnTheLeft && derived().cols() == other.rows()) || (Side==OnTheRight && derived().cols() == other.cols())) ); eigen_assert((!(Mode & ZeroDiag)) && bool(Mode & (Upper|Lower))); + // If solving for a 0x0 matrix, nothing to do, simply return. + if (derived().cols() == 0) + return; - enum { copy = internal::traits<OtherDerived>::Flags & RowMajorBit && OtherDerived::IsVectorAtCompileTime }; + enum { copy = (internal::traits<OtherDerived>::Flags & RowMajorBit) && OtherDerived::IsVectorAtCompileTime && OtherDerived::SizeAtCompileTime!=1}; typedef typename internal::conditional<copy, typename internal::plain_matrix_type_column_major<OtherDerived>::type, OtherDerived&>::type OtherCopy; OtherCopy otherCopy(other); @@ -188,6 +192,7 @@ { return internal::triangular_solve_retval<Side,TriangularViewType,Other>(derived(), other.derived()); } +#endif namespace internal {
diff --git a/Eigen/src/Core/SolverBase.h b/Eigen/src/Core/SolverBase.h index 8a4adc2..5014610 100644 --- a/Eigen/src/Core/SolverBase.h +++ b/Eigen/src/Core/SolverBase.h
@@ -14,8 +14,35 @@ namespace internal { +template<typename Derived> +struct solve_assertion { + template<bool Transpose_, typename Rhs> + static void run(const Derived& solver, const Rhs& b) { solver.template _check_solve_assertion<Transpose_>(b); } +}; +template<typename Derived> +struct solve_assertion<Transpose<Derived> > +{ + typedef Transpose<Derived> type; + template<bool Transpose_, typename Rhs> + static void run(const type& transpose, const Rhs& b) + { + internal::solve_assertion<typename internal::remove_all<Derived>::type>::template run<true>(transpose.nestedExpression(), b); + } +}; + +template<typename Scalar, typename Derived> +struct solve_assertion<CwiseUnaryOp<Eigen::internal::scalar_conjugate_op<Scalar>, const Transpose<Derived> > > +{ + typedef CwiseUnaryOp<Eigen::internal::scalar_conjugate_op<Scalar>, const Transpose<Derived> > type; + + template<bool Transpose_, typename Rhs> + static void run(const type& adjoint, const Rhs& b) + { + internal::solve_assertion<typename internal::remove_all<Transpose<Derived> >::type>::template run<true>(adjoint.nestedExpression(), b); + } +}; } // end namespace internal /** \class SolverBase @@ -35,7 +62,7 @@ * * \warning Currently, any other usage of transpose() and adjoint() are not supported and will produce compilation errors. * - * \sa class PartialPivLU, class FullPivLU + * \sa class PartialPivLU, class FullPivLU, class HouseholderQR, class ColPivHouseholderQR, class FullPivHouseholderQR, class CompleteOrthogonalDecomposition, class LLT, class LDLT, class SVDBase */ template<typename Derived> class SolverBase : public EigenBase<Derived> @@ -46,6 +73,9 @@ typedef typename internal::traits<Derived>::Scalar Scalar; typedef Scalar CoeffReturnType; + template<typename Derived_> + friend struct internal::solve_assertion; + enum { RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime, ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime, @@ -56,7 +86,8 @@ MaxSizeAtCompileTime = (internal::size_at_compile_time<internal::traits<Derived>::MaxRowsAtCompileTime, internal::traits<Derived>::MaxColsAtCompileTime>::ret), IsVectorAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime == 1 - || internal::traits<Derived>::MaxColsAtCompileTime == 1 + || internal::traits<Derived>::MaxColsAtCompileTime == 1, + NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2 }; /** Default constructor */ @@ -74,7 +105,7 @@ inline const Solve<Derived, Rhs> solve(const MatrixBase<Rhs>& b) const { - eigen_assert(derived().rows()==b.rows() && "solve(): invalid number of rows of the right hand side matrix b"); + internal::solve_assertion<typename internal::remove_all<Derived>::type>::template run<false>(derived(), b); return Solve<Derived, Rhs>(derived(), b.derived()); } @@ -112,6 +143,13 @@ } protected: + + template<bool Transpose_, typename Rhs> + void _check_solve_assertion(const Rhs& b) const { + EIGEN_ONLY_USED_FOR_DEBUG(b); + eigen_assert(derived().m_isInitialized && "Solver is not initialized."); + eigen_assert((Transpose_?derived().cols():derived().rows())==b.rows() && "SolverBase::solve(): invalid number of rows of the right hand side matrix b"); + } }; namespace internal {
diff --git a/Eigen/src/Core/SpecialFunctions.h b/Eigen/src/Core/SpecialFunctions.h deleted file mode 100644 index a3857ae..0000000 --- a/Eigen/src/Core/SpecialFunctions.h +++ /dev/null
@@ -1,1048 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2015 Eugene Brevdo <ebrevdo@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_SPECIAL_FUNCTIONS_H -#define EIGEN_SPECIAL_FUNCTIONS_H - -namespace Eigen { -namespace internal { - -// Parts of this code are based on the Cephes Math Library. -// -// Cephes Math Library Release 2.8: June, 2000 -// Copyright 1984, 1987, 1992, 2000 by Stephen L. Moshier -// -// Permission has been kindly provided by the original author -// to incorporate the Cephes software into the Eigen codebase: -// -// From: Stephen Moshier -// To: Eugene Brevdo -// Subject: Re: Permission to wrap several cephes functions in Eigen -// -// Hello Eugene, -// -// Thank you for writing. -// -// If your licensing is similar to BSD, the formal way that has been -// handled is simply to add a statement to the effect that you are incorporating -// the Cephes software by permission of the author. -// -// Good luck with your project, -// Steve - -namespace cephes { - -/* polevl (modified for Eigen) - * - * Evaluate polynomial - * - * - * - * SYNOPSIS: - * - * int N; - * Scalar x, y, coef[N+1]; - * - * y = polevl<decltype(x), N>( x, coef); - * - * - * - * DESCRIPTION: - * - * Evaluates polynomial of degree N: - * - * 2 N - * y = C + C x + C x +...+ C x - * 0 1 2 N - * - * Coefficients are stored in reverse order: - * - * coef[0] = C , ..., coef[N] = C . - * N 0 - * - * The function p1evl() assumes that coef[N] = 1.0 and is - * omitted from the array. Its calling arguments are - * otherwise the same as polevl(). - * - * - * The Eigen implementation is templatized. For best speed, store - * coef as a const array (constexpr), e.g. - * - * const double coef[] = {1.0, 2.0, 3.0, ...}; - * - */ -template <typename Scalar, int N> -struct polevl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x, const Scalar coef[]) { - EIGEN_STATIC_ASSERT((N > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); - - return polevl<Scalar, N - 1>::run(x, coef) * x + coef[N]; - } -}; - -template <typename Scalar> -struct polevl<Scalar, 0> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar, const Scalar coef[]) { - return coef[0]; - } -}; - -} // end namespace cephes - -/**************************************************************************** - * Implementation of lgamma * - ****************************************************************************/ - -template <typename Scalar> -struct lgamma_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -template <typename Scalar> -struct lgamma_retval { - typedef Scalar type; -}; - -#ifdef EIGEN_HAS_C99_MATH -template <> -struct lgamma_impl<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float run(float x) { return ::lgammaf(x); } -}; - -template <> -struct lgamma_impl<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double run(double x) { return ::lgamma(x); } -}; -#endif - -/**************************************************************************** - * Implementation of digamma (psi) * - ****************************************************************************/ - -template <typename Scalar> -struct digamma_retval { - typedef Scalar type; -}; - -#ifndef EIGEN_HAS_C99_MATH - -template <typename Scalar> -struct digamma_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(Scalar x) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -#else - -/* - * - * Polynomial evaluation helper for the Psi (digamma) function. - * - * digamma_impl_maybe_poly::run(s) evaluates the asymptotic Psi expansion for - * input Scalar s, assuming s is above 10.0. - * - * If s is above a certain threshold for the given Scalar type, zero - * is returned. Otherwise the polynomial is evaluated with enough - * coefficients for results matching Scalar machine precision. - * - * - */ -template <typename Scalar> -struct digamma_impl_maybe_poly { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - - -template <> -struct digamma_impl_maybe_poly<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float run(const float s) { - const float A[] = { - -4.16666666666666666667E-3f, - 3.96825396825396825397E-3f, - -8.33333333333333333333E-3f, - 8.33333333333333333333E-2f - }; - - float z; - if (s < 1.0e8f) { - z = 1.0f / (s * s); - return z * cephes::polevl<float, 3>::run(z, A); - } else return 0.0f; - } -}; - -template <> -struct digamma_impl_maybe_poly<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double run(const double s) { - const double A[] = { - 8.33333333333333333333E-2, - -2.10927960927960927961E-2, - 7.57575757575757575758E-3, - -4.16666666666666666667E-3, - 3.96825396825396825397E-3, - -8.33333333333333333333E-3, - 8.33333333333333333333E-2 - }; - - double z; - if (s < 1.0e17) { - z = 1.0 / (s * s); - return z * cephes::polevl<double, 6>::run(z, A); - } - else return 0.0; - } -}; - -template <typename Scalar> -struct digamma_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar x) { - /* - * - * Psi (digamma) function (modified for Eigen) - * - * - * SYNOPSIS: - * - * double x, y, psi(); - * - * y = psi( x ); - * - * - * DESCRIPTION: - * - * d - - * psi(x) = -- ln | (x) - * dx - * - * is the logarithmic derivative of the gamma function. - * For integer x, - * n-1 - * - - * psi(n) = -EUL + > 1/k. - * - - * k=1 - * - * If x is negative, it is transformed to a positive argument by the - * reflection formula psi(1-x) = psi(x) + pi cot(pi x). - * For general positive x, the argument is made greater than 10 - * using the recurrence psi(x+1) = psi(x) + 1/x. - * Then the following asymptotic expansion is applied: - * - * inf. B - * - 2k - * psi(x) = log(x) - 1/2x - > ------- - * - 2k - * k=1 2k x - * - * where the B2k are Bernoulli numbers. - * - * ACCURACY (float): - * Relative error (except absolute when |psi| < 1): - * arithmetic domain # trials peak rms - * IEEE 0,30 30000 1.3e-15 1.4e-16 - * IEEE -30,0 40000 1.5e-15 2.2e-16 - * - * ACCURACY (double): - * Absolute error, relative when |psi| > 1 : - * arithmetic domain # trials peak rms - * IEEE -33,0 30000 8.2e-7 1.2e-7 - * IEEE 0,33 100000 7.3e-7 7.7e-8 - * - * ERROR MESSAGES: - * message condition value returned - * psi singularity x integer <=0 INFINITY - */ - - Scalar p, q, nz, s, w, y; - bool negative = false; - - const Scalar maxnum = NumTraits<Scalar>::infinity(); - const Scalar m_pi(EIGEN_PI); - - const Scalar zero = Scalar(0); - const Scalar one = Scalar(1); - const Scalar half = Scalar(0.5); - nz = zero; - - if (x <= zero) { - negative = true; - q = x; - p = numext::floor(q); - if (p == q) { - return maxnum; - } - /* Remove the zeros of tan(m_pi x) - * by subtracting the nearest integer from x - */ - nz = q - p; - if (nz != half) { - if (nz > half) { - p += one; - nz = q - p; - } - nz = m_pi / numext::tan(m_pi * nz); - } - else { - nz = zero; - } - x = one - x; - } - - /* use the recurrence psi(x+1) = psi(x) + 1/x. */ - s = x; - w = zero; - while (s < Scalar(10)) { - w += one / s; - s += one; - } - - y = digamma_impl_maybe_poly<Scalar>::run(s); - - y = numext::log(s) - (half / s) - y - w; - - return (negative) ? y - nz : y; - } -}; - -#endif // EIGEN_HAS_C99_MATH - -/**************************************************************************** - * Implementation of erf * - ****************************************************************************/ - -template <typename Scalar> -struct erf_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -template <typename Scalar> -struct erf_retval { - typedef Scalar type; -}; - -#ifdef EIGEN_HAS_C99_MATH -template <> -struct erf_impl<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float run(float x) { return ::erff(x); } -}; - -template <> -struct erf_impl<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double run(double x) { return ::erf(x); } -}; -#endif // EIGEN_HAS_C99_MATH - -/*************************************************************************** -* Implementation of erfc * -****************************************************************************/ - -template <typename Scalar> -struct erfc_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -template <typename Scalar> -struct erfc_retval { - typedef Scalar type; -}; - -#ifdef EIGEN_HAS_C99_MATH -template <> -struct erfc_impl<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float run(const float x) { return ::erfcf(x); } -}; - -template <> -struct erfc_impl<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double run(const double x) { return ::erfc(x); } -}; -#endif // EIGEN_HAS_C99_MATH - -/**************************************************************************** - * Implementation of igammac (complemented incomplete gamma integral) * - ****************************************************************************/ - -template <typename Scalar> -struct igammac_retval { - typedef Scalar type; -}; - -#ifndef EIGEN_HAS_C99_MATH - -template <typename Scalar> -struct igammac_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar a, Scalar x) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -#else - -template <typename Scalar> struct igamma_impl; // predeclare igamma_impl - -template <typename Scalar> -struct igamma_helper { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar machep() { assert(false && "machep not supported for this type"); return 0.0; } - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar big() { assert(false && "big not supported for this type"); return 0.0; } -}; - -template <> -struct igamma_helper<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float machep() { - return NumTraits<float>::epsilon() / 2; // 1.0 - machep == 1.0 - } - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE float big() { - // use epsneg (1.0 - epsneg == 1.0) - return 1.0 / (NumTraits<float>::epsilon() / 2); - } -}; - -template <> -struct igamma_helper<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double machep() { - return NumTraits<double>::epsilon() / 2; // 1.0 - machep == 1.0 - } - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE double big() { - return 1.0 / NumTraits<double>::epsilon(); - } -}; - -template <typename Scalar> -struct igammac_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar a, Scalar x) { - /* igamc() - * - * Incomplete gamma integral (modified for Eigen) - * - * - * - * SYNOPSIS: - * - * double a, x, y, igamc(); - * - * y = igamc( a, x ); - * - * DESCRIPTION: - * - * The function is defined by - * - * - * igamc(a,x) = 1 - igam(a,x) - * - * inf. - * - - * 1 | | -t a-1 - * = ----- | e t dt. - * - | | - * | (a) - - * x - * - * - * In this implementation both arguments must be positive. - * The integral is evaluated by either a power series or - * continued fraction expansion, depending on the relative - * values of a and x. - * - * ACCURACY (float): - * - * Relative error: - * arithmetic domain # trials peak rms - * IEEE 0,30 30000 7.8e-6 5.9e-7 - * - * - * ACCURACY (double): - * - * Tested at random a, x. - * a x Relative error: - * arithmetic domain domain # trials peak rms - * IEEE 0.5,100 0,100 200000 1.9e-14 1.7e-15 - * IEEE 0.01,0.5 0,100 200000 1.4e-13 1.6e-15 - * - */ - /* - Cephes Math Library Release 2.2: June, 1992 - Copyright 1985, 1987, 1992 by Stephen L. Moshier - Direct inquiries to 30 Frost Street, Cambridge, MA 02140 - */ - const Scalar zero = 0; - const Scalar one = 1; - const Scalar two = 2; - const Scalar machep = igamma_helper<Scalar>::machep(); - const Scalar maxlog = numext::log(NumTraits<Scalar>::highest()); - const Scalar big = igamma_helper<Scalar>::big(); - const Scalar biginv = 1 / big; - const Scalar nan = NumTraits<Scalar>::quiet_NaN(); - const Scalar inf = NumTraits<Scalar>::infinity(); - - Scalar ans, ax, c, yc, r, t, y, z; - Scalar pk, pkm1, pkm2, qk, qkm1, qkm2; - - if ((x < zero) || ( a <= zero)) { - // domain error - return nan; - } - - if ((x < one) || (x < a)) { - return (one - igamma_impl<Scalar>::run(a, x)); - } - - if (x == inf) return zero; // std::isinf crashes on CUDA - - /* Compute x**a * exp(-x) / gamma(a) */ - ax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a); - if (ax < -maxlog) { // underflow - return zero; - } - ax = numext::exp(ax); - - // continued fraction - y = one - a; - z = x + y + one; - c = zero; - pkm2 = one; - qkm2 = x; - pkm1 = x + one; - qkm1 = z * x; - ans = pkm1 / qkm1; - - while (true) { - c += one; - y += one; - z += two; - yc = y * c; - pk = pkm1 * z - pkm2 * yc; - qk = qkm1 * z - qkm2 * yc; - if (qk != zero) { - r = pk / qk; - t = numext::abs((ans - r) / r); - ans = r; - } else { - t = one; - } - pkm2 = pkm1; - pkm1 = pk; - qkm2 = qkm1; - qkm1 = qk; - if (numext::abs(pk) > big) { - pkm2 *= biginv; - pkm1 *= biginv; - qkm2 *= biginv; - qkm1 *= biginv; - } - if (t <= machep) break; - } - - return (ans * ax); - } -}; - -#endif // EIGEN_HAS_C99_MATH - -/**************************************************************************** - * Implementation of igamma (incomplete gamma integral) * - ****************************************************************************/ - -template <typename Scalar> -struct igamma_retval { - typedef Scalar type; -}; - -#ifndef EIGEN_HAS_C99_MATH - -template <typename Scalar> -struct igamma_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(Scalar a, Scalar x) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -#else - -template <typename Scalar> -struct igamma_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar a, Scalar x) { - /* igam() - * Incomplete gamma integral - * - * - * - * SYNOPSIS: - * - * double a, x, y, igam(); - * - * y = igam( a, x ); - * - * DESCRIPTION: - * - * The function is defined by - * - * x - * - - * 1 | | -t a-1 - * igam(a,x) = ----- | e t dt. - * - | | - * | (a) - - * 0 - * - * - * In this implementation both arguments must be positive. - * The integral is evaluated by either a power series or - * continued fraction expansion, depending on the relative - * values of a and x. - * - * ACCURACY (double): - * - * Relative error: - * arithmetic domain # trials peak rms - * IEEE 0,30 200000 3.6e-14 2.9e-15 - * IEEE 0,100 300000 9.9e-14 1.5e-14 - * - * - * ACCURACY (float): - * - * Relative error: - * arithmetic domain # trials peak rms - * IEEE 0,30 20000 7.8e-6 5.9e-7 - * - */ - /* - Cephes Math Library Release 2.2: June, 1992 - Copyright 1985, 1987, 1992 by Stephen L. Moshier - Direct inquiries to 30 Frost Street, Cambridge, MA 02140 - */ - - - /* left tail of incomplete gamma function: - * - * inf. k - * a -x - x - * x e > ---------- - * - - - * k=0 | (a+k+1) - * - */ - const Scalar zero = 0; - const Scalar one = 1; - const Scalar machep = igamma_helper<Scalar>::machep(); - const Scalar maxlog = numext::log(NumTraits<Scalar>::highest()); - const Scalar nan = NumTraits<Scalar>::quiet_NaN(); - - double ans, ax, c, r; - - if (x == zero) return zero; - - if ((x < zero) || ( a <= zero)) { // domain error - return nan; - } - - if ((x > one) && (x > a)) { - return (one - igammac_impl<Scalar>::run(a, x)); - } - - /* Compute x**a * exp(-x) / gamma(a) */ - ax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a); - if (ax < -maxlog) { - // underflow - return zero; - } - ax = numext::exp(ax); - - /* power series */ - r = a; - c = one; - ans = one; - - while (true) { - r += one; - c *= x/r; - ans += c; - if (c/ans <= machep) break; - } - - return (ans * ax / a); - } -}; - -#endif // EIGEN_HAS_C99_MATH - -/**************************************************************************** - * Implementation of Riemann zeta function of two arguments * - ****************************************************************************/ - -template <typename Scalar> -struct zeta_retval { - typedef Scalar type; -}; - -#ifndef EIGEN_HAS_C99_MATH - -template <typename Scalar> -struct zeta_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(Scalar x, Scalar q) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -#else - -template <typename Scalar> -struct zeta_impl_series { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -template <> -struct zeta_impl_series<float> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE bool run(float& a, float& b, float& s, const float x, const float machep) { - int i = 0; - while(i < 9) - { - i += 1; - a += 1.0f; - b = numext::pow( a, -x ); - s += b; - if( numext::abs(b/s) < machep ) - return true; - } - - //Return whether we are done - return false; - } -}; - -template <> -struct zeta_impl_series<double> { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE bool run(double& a, double& b, double& s, const double x, const double machep) { - int i = 0; - while( (i < 9) || (a <= 9.0) ) - { - i += 1; - a += 1.0; - b = numext::pow( a, -x ); - s += b; - if( numext::abs(b/s) < machep ) - return true; - } - - //Return whether we are done - return false; - } -}; - -template <typename Scalar> -struct zeta_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar x, Scalar q) { - /* zeta.c - * - * Riemann zeta function of two arguments - * - * - * - * SYNOPSIS: - * - * double x, q, y, zeta(); - * - * y = zeta( x, q ); - * - * - * - * DESCRIPTION: - * - * - * - * inf. - * - -x - * zeta(x,q) = > (k+q) - * - - * k=0 - * - * where x > 1 and q is not a negative integer or zero. - * The Euler-Maclaurin summation formula is used to obtain - * the expansion - * - * n - * - -x - * zeta(x,q) = > (k+q) - * - - * k=1 - * - * 1-x inf. B x(x+1)...(x+2j) - * (n+q) 1 - 2j - * + --------- - ------- + > -------------------- - * x-1 x - x+2j+1 - * 2(n+q) j=1 (2j)! (n+q) - * - * where the B2j are Bernoulli numbers. Note that (see zetac.c) - * zeta(x,1) = zetac(x) + 1. - * - * - * - * ACCURACY: - * - * Relative error for single precision: - * arithmetic domain # trials peak rms - * IEEE 0,25 10000 6.9e-7 1.0e-7 - * - * Large arguments may produce underflow in powf(), in which - * case the results are inaccurate. - * - * REFERENCE: - * - * Gradshteyn, I. S., and I. M. Ryzhik, Tables of Integrals, - * Series, and Products, p. 1073; Academic Press, 1980. - * - */ - - int i; - Scalar p, r, a, b, k, s, t, w; - - const Scalar A[] = { - Scalar(12.0), - Scalar(-720.0), - Scalar(30240.0), - Scalar(-1209600.0), - Scalar(47900160.0), - Scalar(-1.8924375803183791606e9), /*1.307674368e12/691*/ - Scalar(7.47242496e10), - Scalar(-2.950130727918164224e12), /*1.067062284288e16/3617*/ - Scalar(1.1646782814350067249e14), /*5.109094217170944e18/43867*/ - Scalar(-4.5979787224074726105e15), /*8.028576626982912e20/174611*/ - Scalar(1.8152105401943546773e17), /*1.5511210043330985984e23/854513*/ - Scalar(-7.1661652561756670113e18) /*1.6938241367317436694528e27/236364091*/ - }; - - const Scalar maxnum = NumTraits<Scalar>::infinity(); - const Scalar zero = 0.0, half = 0.5, one = 1.0; - const Scalar machep = igamma_helper<Scalar>::machep(); - const Scalar nan = NumTraits<Scalar>::quiet_NaN(); - - if( x == one ) - return maxnum; - - if( x < one ) - { - return nan; - } - - if( q <= zero ) - { - if(q == numext::floor(q)) - { - return maxnum; - } - p = x; - r = numext::floor(p); - if (p != r) - return nan; - } - - /* Permit negative q but continue sum until n+q > +9 . - * This case should be handled by a reflection formula. - * If q<0 and x is an integer, there is a relation to - * the polygamma function. - */ - s = numext::pow( q, -x ); - a = q; - b = zero; - // Run the summation in a helper function that is specific to the floating precision - if (zeta_impl_series<Scalar>::run(a, b, s, x, machep)) { - return s; - } - - w = a; - s += b*w/(x-one); - s -= half * b; - a = one; - k = zero; - for( i=0; i<12; i++ ) - { - a *= x + k; - b /= w; - t = a*b/A[i]; - s = s + t; - t = numext::abs(t/s); - if( t < machep ) - return s; - k += one; - a *= x + k; - b /= w; - k += one; - } - return s; - } -}; - -#endif // EIGEN_HAS_C99_MATH - -/**************************************************************************** - * Implementation of polygamma function * - ****************************************************************************/ - -template <typename Scalar> -struct polygamma_retval { - typedef Scalar type; -}; - -#ifndef EIGEN_HAS_C99_MATH - -template <typename Scalar> -struct polygamma_impl { - EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(Scalar n, Scalar x) { - EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), - THIS_TYPE_IS_NOT_SUPPORTED); - return Scalar(0); - } -}; - -#else - -template <typename Scalar> -struct polygamma_impl { - EIGEN_DEVICE_FUNC - static Scalar run(Scalar n, Scalar x) { - Scalar zero = 0.0, one = 1.0; - Scalar nplus = n + one; - const Scalar nan = NumTraits<Scalar>::quiet_NaN(); - - // Check that n is an integer - if (numext::floor(n) != n) { - return nan; - } - // Just return the digamma function for n = 1 - else if (n == zero) { - return digamma_impl<Scalar>::run(x); - } - // Use the same implementation as scipy - else { - Scalar factorial = numext::exp(lgamma_impl<Scalar>::run(nplus)); - return numext::pow(-one, nplus) * factorial * zeta_impl<Scalar>::run(nplus, x); - } - } -}; - -#endif // EIGEN_HAS_C99_MATH - -} // end namespace internal - -namespace numext { - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(lgamma, Scalar) - lgamma(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(lgamma, Scalar)::run(x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(digamma, Scalar) - digamma(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(digamma, Scalar)::run(x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(zeta, Scalar) -zeta(const Scalar& x, const Scalar& q) { - return EIGEN_MATHFUNC_IMPL(zeta, Scalar)::run(x, q); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(polygamma, Scalar) -polygamma(const Scalar& n, const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(polygamma, Scalar)::run(n, x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(erf, Scalar) - erf(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(erf, Scalar)::run(x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(erfc, Scalar) - erfc(const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(erfc, Scalar)::run(x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(igamma, Scalar) - igamma(const Scalar& a, const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(igamma, Scalar)::run(a, x); -} - -template <typename Scalar> -EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(igammac, Scalar) - igammac(const Scalar& a, const Scalar& x) { - return EIGEN_MATHFUNC_IMPL(igammac, Scalar)::run(a, x); -} - -} // end namespace numext - - -} // end namespace Eigen - -#endif // EIGEN_SPECIAL_FUNCTIONS_H
diff --git a/Eigen/src/Core/StableNorm.h b/Eigen/src/Core/StableNorm.h index d2fe1e1..77ea3c2 100644 --- a/Eigen/src/Core/StableNorm.h +++ b/Eigen/src/Core/StableNorm.h
@@ -50,6 +50,71 @@ ssq += (bl*invScale).squaredNorm(); } +template<typename VectorType, typename RealScalar> +void stable_norm_impl_inner_step(const VectorType &vec, RealScalar& ssq, RealScalar& scale, RealScalar& invScale) +{ + typedef typename VectorType::Scalar Scalar; + const Index blockSize = 4096; + + typedef typename internal::nested_eval<VectorType,2>::type VectorTypeCopy; + typedef typename internal::remove_all<VectorTypeCopy>::type VectorTypeCopyClean; + const VectorTypeCopy copy(vec); + + enum { + CanAlign = ( (int(VectorTypeCopyClean::Flags)&DirectAccessBit) + || (int(internal::evaluator<VectorTypeCopyClean>::Alignment)>0) // FIXME Alignment)>0 might not be enough + ) && (blockSize*sizeof(Scalar)*2<EIGEN_STACK_ALLOCATION_LIMIT) + && (EIGEN_MAX_STATIC_ALIGN_BYTES>0) // if we cannot allocate on the stack, then let's not bother about this optimization + }; + typedef typename internal::conditional<CanAlign, Ref<const Matrix<Scalar,Dynamic,1,0,blockSize,1>, internal::evaluator<VectorTypeCopyClean>::Alignment>, + typename VectorTypeCopyClean::ConstSegmentReturnType>::type SegmentWrapper; + Index n = vec.size(); + + Index bi = internal::first_default_aligned(copy); + if (bi>0) + internal::stable_norm_kernel(copy.head(bi), ssq, scale, invScale); + for (; bi<n; bi+=blockSize) + internal::stable_norm_kernel(SegmentWrapper(copy.segment(bi,numext::mini(blockSize, n - bi))), ssq, scale, invScale); +} + +template<typename VectorType> +typename VectorType::RealScalar +stable_norm_impl(const VectorType &vec, typename enable_if<VectorType::IsVectorAtCompileTime>::type* = 0 ) +{ + using std::sqrt; + using std::abs; + + Index n = vec.size(); + + if(n==1) + return abs(vec.coeff(0)); + + typedef typename VectorType::RealScalar RealScalar; + RealScalar scale(0); + RealScalar invScale(1); + RealScalar ssq(0); // sum of squares + + stable_norm_impl_inner_step(vec, ssq, scale, invScale); + + return scale * sqrt(ssq); +} + +template<typename MatrixType> +typename MatrixType::RealScalar +stable_norm_impl(const MatrixType &mat, typename enable_if<!MatrixType::IsVectorAtCompileTime>::type* = 0 ) +{ + using std::sqrt; + + typedef typename MatrixType::RealScalar RealScalar; + RealScalar scale(0); + RealScalar invScale(1); + RealScalar ssq(0); // sum of squares + + for(Index j=0; j<mat.outerSize(); ++j) + stable_norm_impl_inner_step(mat.innerVector(j), ssq, scale, invScale); + return scale * sqrt(ssq); +} + template<typename Derived> inline typename NumTraits<typename traits<Derived>::Scalar>::Real blueNorm_impl(const EigenBase<Derived>& _vec) @@ -74,7 +139,7 @@ // are used. For any specific computer, each of the assignment // statements can be replaced ibeta = std::numeric_limits<RealScalar>::radix; // base for floating-point numbers - it = std::numeric_limits<RealScalar>::digits; // number of base-beta digits in mantissa + it = NumTraits<RealScalar>::digits(); // number of base-beta digits in mantissa iemin = std::numeric_limits<RealScalar>::min_exponent; // minimum exponent iemax = std::numeric_limits<RealScalar>::max_exponent; // maximum exponent rbig = (std::numeric_limits<RealScalar>::max)(); // largest floating-point number @@ -98,12 +163,16 @@ RealScalar asml = RealScalar(0); RealScalar amed = RealScalar(0); RealScalar abig = RealScalar(0); - for(typename Derived::InnerIterator it(vec, 0); it; ++it) + + for(Index j=0; j<vec.outerSize(); ++j) { - RealScalar ax = abs(it.value()); - if(ax > ab2) abig += numext::abs2(ax*s2m); - else if(ax < b1) asml += numext::abs2(ax*s1m); - else amed += numext::abs2(ax); + for(typename Derived::InnerIterator it(vec, j); it; ++it) + { + RealScalar ax = abs(it.value()); + if(ax > ab2) abig += numext::abs2(ax*s2m); + else if(ax < b1) asml += numext::abs2(ax*s1m); + else amed += numext::abs2(ax); + } } if(amed!=amed) return amed; // we got a NaN @@ -156,35 +225,7 @@ inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::stableNorm() const { - using std::sqrt; - using std::abs; - const Index blockSize = 4096; - RealScalar scale(0); - RealScalar invScale(1); - RealScalar ssq(0); // sum of square - - typedef typename internal::nested_eval<Derived,2>::type DerivedCopy; - typedef typename internal::remove_all<DerivedCopy>::type DerivedCopyClean; - DerivedCopy copy(derived()); - - enum { - CanAlign = ( (int(DerivedCopyClean::Flags)&DirectAccessBit) - || (int(internal::evaluator<DerivedCopyClean>::Alignment)>0) // FIXME Alignment)>0 might not be enough - ) && (blockSize*sizeof(Scalar)*2<EIGEN_STACK_ALLOCATION_LIMIT) // ifwe cannot allocate on the stack, then let's not bother about this optimization - }; - typedef typename internal::conditional<CanAlign, Ref<const Matrix<Scalar,Dynamic,1,0,blockSize,1>, internal::evaluator<DerivedCopyClean>::Alignment>, - typename DerivedCopyClean::ConstSegmentReturnType>::type SegmentWrapper; - Index n = size(); - - if(n==1) - return abs(this->coeff(0)); - - Index bi = internal::first_default_aligned(copy); - if (bi>0) - internal::stable_norm_kernel(copy.head(bi), ssq, scale, invScale); - for (; bi<n; bi+=blockSize) - internal::stable_norm_kernel(SegmentWrapper(copy.segment(bi,numext::mini(blockSize, n - bi))), ssq, scale, invScale); - return scale * sqrt(ssq); + return internal::stable_norm_impl(derived()); } /** \returns the \em l2 norm of \c *this using the Blue's algorithm. @@ -212,7 +253,10 @@ inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::hypotNorm() const { - return this->cwiseAbs().redux(internal::scalar_hypot_op<RealScalar>()); + if(size()==1) + return numext::abs(coeff(0,0)); + else + return this->cwiseAbs().redux(internal::scalar_hypot_op<RealScalar>()); } } // end namespace Eigen
diff --git a/Eigen/src/Core/StlIterators.h b/Eigen/src/Core/StlIterators.h new file mode 100644 index 0000000..4f25a60 --- /dev/null +++ b/Eigen/src/Core/StlIterators.h
@@ -0,0 +1,318 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +namespace Eigen { + +namespace internal { + +template<typename IteratorType> +struct indexed_based_stl_iterator_traits; + +template<typename Derived> +class indexed_based_stl_iterator_base +{ +protected: + typedef indexed_based_stl_iterator_traits<Derived> traits; + typedef typename traits::XprType XprType; + typedef indexed_based_stl_iterator_base<typename traits::non_const_iterator> non_const_iterator; + typedef indexed_based_stl_iterator_base<typename traits::const_iterator> const_iterator; + typedef typename internal::conditional<internal::is_const<XprType>::value,non_const_iterator,const_iterator>::type other_iterator; + // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class: + friend class indexed_based_stl_iterator_base<typename traits::const_iterator>; + friend class indexed_based_stl_iterator_base<typename traits::non_const_iterator>; +public: + typedef Index difference_type; + typedef std::random_access_iterator_tag iterator_category; + + indexed_based_stl_iterator_base() : mp_xpr(0), m_index(0) {} + indexed_based_stl_iterator_base(XprType& xpr, Index index) : mp_xpr(&xpr), m_index(index) {} + + indexed_based_stl_iterator_base(const non_const_iterator& other) + : mp_xpr(other.mp_xpr), m_index(other.m_index) + {} + + indexed_based_stl_iterator_base& operator=(const non_const_iterator& other) + { + mp_xpr = other.mp_xpr; + m_index = other.m_index; + return *this; + } + + Derived& operator++() { ++m_index; return derived(); } + Derived& operator--() { --m_index; return derived(); } + + Derived operator++(int) { Derived prev(derived()); operator++(); return prev;} + Derived operator--(int) { Derived prev(derived()); operator--(); return prev;} + + friend Derived operator+(const indexed_based_stl_iterator_base& a, Index b) { Derived ret(a.derived()); ret += b; return ret; } + friend Derived operator-(const indexed_based_stl_iterator_base& a, Index b) { Derived ret(a.derived()); ret -= b; return ret; } + friend Derived operator+(Index a, const indexed_based_stl_iterator_base& b) { Derived ret(b.derived()); ret += a; return ret; } + friend Derived operator-(Index a, const indexed_based_stl_iterator_base& b) { Derived ret(b.derived()); ret -= a; return ret; } + + Derived& operator+=(Index b) { m_index += b; return derived(); } + Derived& operator-=(Index b) { m_index -= b; return derived(); } + + difference_type operator-(const indexed_based_stl_iterator_base& other) const + { + eigen_assert(mp_xpr == other.mp_xpr); + return m_index - other.m_index; + } + + difference_type operator-(const other_iterator& other) const + { + eigen_assert(mp_xpr == other.mp_xpr); + return m_index - other.m_index; + } + + bool operator==(const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index == other.m_index; } + bool operator!=(const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index != other.m_index; } + bool operator< (const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index < other.m_index; } + bool operator<=(const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index <= other.m_index; } + bool operator> (const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index > other.m_index; } + bool operator>=(const indexed_based_stl_iterator_base& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index >= other.m_index; } + + bool operator==(const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index == other.m_index; } + bool operator!=(const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index != other.m_index; } + bool operator< (const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index < other.m_index; } + bool operator<=(const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index <= other.m_index; } + bool operator> (const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index > other.m_index; } + bool operator>=(const other_iterator& other) const { eigen_assert(mp_xpr == other.mp_xpr); return m_index >= other.m_index; } + +protected: + + Derived& derived() { return static_cast<Derived&>(*this); } + const Derived& derived() const { return static_cast<const Derived&>(*this); } + + XprType *mp_xpr; + Index m_index; +}; + +template<typename XprType> +class pointer_based_stl_iterator +{ + enum { is_lvalue = internal::is_lvalue<XprType>::value }; + typedef pointer_based_stl_iterator<typename internal::remove_const<XprType>::type> non_const_iterator; + typedef pointer_based_stl_iterator<typename internal::add_const<XprType>::type> const_iterator; + typedef typename internal::conditional<internal::is_const<XprType>::value,non_const_iterator,const_iterator>::type other_iterator; + // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class: + friend class pointer_based_stl_iterator<typename internal::add_const<XprType>::type>; + friend class pointer_based_stl_iterator<typename internal::remove_const<XprType>::type>; +public: + typedef Index difference_type; + typedef typename XprType::Scalar value_type; + typedef std::random_access_iterator_tag iterator_category; + typedef typename internal::conditional<bool(is_lvalue), value_type*, const value_type*>::type pointer; + typedef typename internal::conditional<bool(is_lvalue), value_type&, const value_type&>::type reference; + + + pointer_based_stl_iterator() : m_ptr(0) {} + pointer_based_stl_iterator(XprType& xpr, Index index) : m_incr(xpr.innerStride()) + { + m_ptr = xpr.data() + index * m_incr.value(); + } + + pointer_based_stl_iterator(const non_const_iterator& other) + : m_ptr(other.m_ptr), m_incr(other.m_incr) + {} + + pointer_based_stl_iterator& operator=(const non_const_iterator& other) + { + m_ptr = other.m_ptr; + m_incr.setValue(other.m_incr); + return *this; + } + + reference operator*() const { return *m_ptr; } + reference operator[](Index i) const { return *(m_ptr+i*m_incr.value()); } + pointer operator->() const { return m_ptr; } + + pointer_based_stl_iterator& operator++() { m_ptr += m_incr.value(); return *this; } + pointer_based_stl_iterator& operator--() { m_ptr -= m_incr.value(); return *this; } + + pointer_based_stl_iterator operator++(int) { pointer_based_stl_iterator prev(*this); operator++(); return prev;} + pointer_based_stl_iterator operator--(int) { pointer_based_stl_iterator prev(*this); operator--(); return prev;} + + friend pointer_based_stl_iterator operator+(const pointer_based_stl_iterator& a, Index b) { pointer_based_stl_iterator ret(a); ret += b; return ret; } + friend pointer_based_stl_iterator operator-(const pointer_based_stl_iterator& a, Index b) { pointer_based_stl_iterator ret(a); ret -= b; return ret; } + friend pointer_based_stl_iterator operator+(Index a, const pointer_based_stl_iterator& b) { pointer_based_stl_iterator ret(b); ret += a; return ret; } + friend pointer_based_stl_iterator operator-(Index a, const pointer_based_stl_iterator& b) { pointer_based_stl_iterator ret(b); ret -= a; return ret; } + + pointer_based_stl_iterator& operator+=(Index b) { m_ptr += b*m_incr.value(); return *this; } + pointer_based_stl_iterator& operator-=(Index b) { m_ptr -= b*m_incr.value(); return *this; } + + difference_type operator-(const pointer_based_stl_iterator& other) const { + return (m_ptr - other.m_ptr)/m_incr.value(); + } + + difference_type operator-(const other_iterator& other) const { + return (m_ptr - other.m_ptr)/m_incr.value(); + } + + bool operator==(const pointer_based_stl_iterator& other) const { return m_ptr == other.m_ptr; } + bool operator!=(const pointer_based_stl_iterator& other) const { return m_ptr != other.m_ptr; } + bool operator< (const pointer_based_stl_iterator& other) const { return m_ptr < other.m_ptr; } + bool operator<=(const pointer_based_stl_iterator& other) const { return m_ptr <= other.m_ptr; } + bool operator> (const pointer_based_stl_iterator& other) const { return m_ptr > other.m_ptr; } + bool operator>=(const pointer_based_stl_iterator& other) const { return m_ptr >= other.m_ptr; } + + bool operator==(const other_iterator& other) const { return m_ptr == other.m_ptr; } + bool operator!=(const other_iterator& other) const { return m_ptr != other.m_ptr; } + bool operator< (const other_iterator& other) const { return m_ptr < other.m_ptr; } + bool operator<=(const other_iterator& other) const { return m_ptr <= other.m_ptr; } + bool operator> (const other_iterator& other) const { return m_ptr > other.m_ptr; } + bool operator>=(const other_iterator& other) const { return m_ptr >= other.m_ptr; } + +protected: + + pointer m_ptr; + internal::variable_if_dynamic<Index, XprType::InnerStrideAtCompileTime> m_incr; +}; + +template<typename _XprType> +struct indexed_based_stl_iterator_traits<generic_randaccess_stl_iterator<_XprType> > +{ + typedef _XprType XprType; + typedef generic_randaccess_stl_iterator<typename internal::remove_const<XprType>::type> non_const_iterator; + typedef generic_randaccess_stl_iterator<typename internal::add_const<XprType>::type> const_iterator; +}; + +template<typename XprType> +class generic_randaccess_stl_iterator : public indexed_based_stl_iterator_base<generic_randaccess_stl_iterator<XprType> > +{ +public: + typedef typename XprType::Scalar value_type; + +protected: + + enum { + has_direct_access = (internal::traits<XprType>::Flags & DirectAccessBit) ? 1 : 0, + is_lvalue = internal::is_lvalue<XprType>::value + }; + + typedef indexed_based_stl_iterator_base<generic_randaccess_stl_iterator> Base; + using Base::m_index; + using Base::mp_xpr; + + // TODO currently const Transpose/Reshape expressions never returns const references, + // so lets return by value too. + //typedef typename internal::conditional<bool(has_direct_access), const value_type&, const value_type>::type read_only_ref_t; + typedef const value_type read_only_ref_t; + +public: + + typedef typename internal::conditional<bool(is_lvalue), value_type *, const value_type *>::type pointer; + typedef typename internal::conditional<bool(is_lvalue), value_type&, read_only_ref_t>::type reference; + + generic_randaccess_stl_iterator() : Base() {} + generic_randaccess_stl_iterator(XprType& xpr, Index index) : Base(xpr,index) {} + generic_randaccess_stl_iterator(const typename Base::non_const_iterator& other) : Base(other) {} + using Base::operator=; + + reference operator*() const { return (*mp_xpr)(m_index); } + reference operator[](Index i) const { return (*mp_xpr)(m_index+i); } + pointer operator->() const { return &((*mp_xpr)(m_index)); } +}; + +template<typename _XprType, DirectionType Direction> +struct indexed_based_stl_iterator_traits<subvector_stl_iterator<_XprType,Direction> > +{ + typedef _XprType XprType; + typedef subvector_stl_iterator<typename internal::remove_const<XprType>::type, Direction> non_const_iterator; + typedef subvector_stl_iterator<typename internal::add_const<XprType>::type, Direction> const_iterator; +}; + +template<typename XprType, DirectionType Direction> +class subvector_stl_iterator : public indexed_based_stl_iterator_base<subvector_stl_iterator<XprType,Direction> > +{ +protected: + + enum { is_lvalue = internal::is_lvalue<XprType>::value }; + + typedef indexed_based_stl_iterator_base<subvector_stl_iterator> Base; + using Base::m_index; + using Base::mp_xpr; + + typedef typename internal::conditional<Direction==Vertical,typename XprType::ColXpr,typename XprType::RowXpr>::type SubVectorType; + typedef typename internal::conditional<Direction==Vertical,typename XprType::ConstColXpr,typename XprType::ConstRowXpr>::type ConstSubVectorType; + +public: + typedef typename internal::conditional<bool(is_lvalue), SubVectorType, ConstSubVectorType>::type value_type; + typedef value_type* pointer; + typedef value_type reference; + + subvector_stl_iterator() : Base() {} + subvector_stl_iterator(XprType& xpr, Index index) : Base(xpr,index) {} + + reference operator*() const { return (*mp_xpr).template subVector<Direction>(m_index); } + reference operator[](Index i) const { return (*mp_xpr).template subVector<Direction>(m_index+i); } + pointer operator->() const { return &((*mp_xpr).template subVector<Direction>(m_index)); } +}; + +} // namespace internal + + +/** returns an iterator to the first element of the 1D vector or array + * \only_for_vectors + * \sa end(), cbegin() + */ +template<typename Derived> +inline typename DenseBase<Derived>::iterator DenseBase<Derived>::begin() +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + return iterator(derived(), 0); +} + +/** const version of begin() */ +template<typename Derived> +inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::begin() const +{ + return cbegin(); +} + +/** returns a read-only const_iterator to the first element of the 1D vector or array + * \only_for_vectors + * \sa cend(), begin() + */ +template<typename Derived> +inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::cbegin() const +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + return const_iterator(derived(), 0); +} + +/** returns an iterator to the element following the last element of the 1D vector or array + * \only_for_vectors + * \sa begin(), cend() + */ +template<typename Derived> +inline typename DenseBase<Derived>::iterator DenseBase<Derived>::end() +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + return iterator(derived(), size()); +} + +/** const version of end() */ +template<typename Derived> +inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::end() const +{ + return cend(); +} + +/** returns a read-only const_iterator to the element following the last element of the 1D vector or array + * \only_for_vectors + * \sa begin(), cend() + */ +template<typename Derived> +inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::cend() const +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); + return const_iterator(derived(), size()); +} + +} // namespace Eigen
diff --git a/Eigen/src/Core/Swap.h b/Eigen/src/Core/Swap.h index d702009..180a4e5 100644 --- a/Eigen/src/Core/Swap.h +++ b/Eigen/src/Core/Swap.h
@@ -30,12 +30,13 @@ typedef typename Base::DstXprType DstXprType; typedef swap_assign_op<Scalar> Functor; - EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + generic_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) : Base(dst, src, func, dstExpr) {} template<int StoreMode, int LoadMode, typename PacketType> - void assignPacket(Index row, Index col) + EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) { PacketType tmp = m_src.template packet<LoadMode,PacketType>(row,col); const_cast<SrcEvaluatorTypeT&>(m_src).template writePacket<LoadMode>(row,col, m_dst.template packet<StoreMode,PacketType>(row,col)); @@ -43,7 +44,7 @@ } template<int StoreMode, int LoadMode, typename PacketType> - void assignPacket(Index index) + EIGEN_STRONG_INLINE void assignPacket(Index index) { PacketType tmp = m_src.template packet<LoadMode,PacketType>(index); const_cast<SrcEvaluatorTypeT&>(m_src).template writePacket<LoadMode>(index, m_dst.template packet<StoreMode,PacketType>(index)); @@ -52,7 +53,7 @@ // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael) template<int StoreMode, int LoadMode, typename PacketType> - void assignPacketByOuterInner(Index outer, Index inner) + EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) { Index row = Base::rowIndexByOuterInner(outer, inner); Index col = Base::colIndexByOuterInner(outer, inner);
diff --git a/Eigen/src/Core/Transpose.h b/Eigen/src/Core/Transpose.h index bc23252..c513f7f 100644 --- a/Eigen/src/Core/Transpose.h +++ b/Eigen/src/Core/Transpose.h
@@ -61,23 +61,31 @@ typedef typename internal::remove_all<MatrixType>::type NestedExpression; EIGEN_DEVICE_FUNC - explicit inline Transpose(MatrixType& matrix) : m_matrix(matrix) {} + explicit EIGEN_STRONG_INLINE Transpose(MatrixType& matrix) : m_matrix(matrix) {} EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Transpose) - EIGEN_DEVICE_FUNC inline Index rows() const { return m_matrix.cols(); } - EIGEN_DEVICE_FUNC inline Index cols() const { return m_matrix.rows(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index rows() const { return m_matrix.cols(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index cols() const { return m_matrix.rows(); } /** \returns the nested expression */ - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename internal::remove_all<MatrixTypeNested>::type& nestedExpression() const { return m_matrix; } /** \returns the nested expression */ - EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::remove_reference<MatrixTypeNested>::type& nestedExpression() { return m_matrix; } + /** \internal */ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void resize(Index nrows, Index ncols) { + m_matrix.resize(ncols,nrows); + } + protected: typename internal::ref_selector<MatrixType>::non_const_type m_matrix; }; @@ -117,8 +125,10 @@ EIGEN_DENSE_PUBLIC_INTERFACE(Transpose<MatrixType>) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TransposeImpl) - EIGEN_DEVICE_FUNC inline Index innerStride() const { return derived().nestedExpression().innerStride(); } - EIGEN_DEVICE_FUNC inline Index outerStride() const { return derived().nestedExpression().outerStride(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index innerStride() const { return derived().nestedExpression().innerStride(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Index outerStride() const { return derived().nestedExpression().outerStride(); } typedef typename internal::conditional< internal::is_lvalue<MatrixType>::value, @@ -126,18 +136,20 @@ const Scalar >::type ScalarWithConstIfNotLvalue; - EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return derived().nestedExpression().data(); } - EIGEN_DEVICE_FUNC inline const Scalar* data() const { return derived().nestedExpression().data(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ScalarWithConstIfNotLvalue* data() { return derived().nestedExpression().data(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar* data() const { return derived().nestedExpression().data(); } // FIXME: shall we keep the const version of coeffRef? - EIGEN_DEVICE_FUNC - inline const Scalar& coeffRef(Index rowId, Index colId) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar& coeffRef(Index rowId, Index colId) const { return derived().nestedExpression().coeffRef(colId, rowId); } - EIGEN_DEVICE_FUNC - inline const Scalar& coeffRef(Index index) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar& coeffRef(Index index) const { return derived().nestedExpression().coeffRef(index); } @@ -163,7 +175,8 @@ * * \sa transposeInPlace(), adjoint() */ template<typename Derived> -inline Transpose<Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Transpose<Derived> DenseBase<Derived>::transpose() { return TransposeReturnType(derived()); @@ -175,7 +188,8 @@ * * \sa transposeInPlace(), adjoint() */ template<typename Derived> -inline typename DenseBase<Derived>::ConstTransposeReturnType +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename DenseBase<Derived>::ConstTransposeReturnType DenseBase<Derived>::transpose() const { return ConstTransposeReturnType(derived()); @@ -201,7 +215,7 @@ * * \sa adjointInPlace(), transpose(), conjugate(), class Transpose, class internal::scalar_conjugate_op */ template<typename Derived> -inline const typename MatrixBase<Derived>::AdjointReturnType +EIGEN_DEVICE_FUNC inline const typename MatrixBase<Derived>::AdjointReturnType MatrixBase<Derived>::adjoint() const { return AdjointReturnType(this->transpose()); @@ -276,7 +290,7 @@ * * \sa transpose(), adjoint(), adjointInPlace() */ template<typename Derived> -inline void DenseBase<Derived>::transposeInPlace() +EIGEN_DEVICE_FUNC inline void DenseBase<Derived>::transposeInPlace() { eigen_assert((rows() == cols() || (RowsAtCompileTime == Dynamic && ColsAtCompileTime == Dynamic)) && "transposeInPlace() called on a non-square non-resizable matrix"); @@ -307,7 +321,7 @@ * * \sa transpose(), adjoint(), transposeInPlace() */ template<typename Derived> -inline void MatrixBase<Derived>::adjointInPlace() +EIGEN_DEVICE_FUNC inline void MatrixBase<Derived>::adjointInPlace() { derived() = adjoint().eval(); } @@ -386,7 +400,8 @@ template<typename Dst, typename Src> void check_for_aliasing(const Dst &dst, const Src &src) { - internal::checkTransposeAliasing_impl<Dst, Src>::run(dst, src); + if((!Dst::IsVectorAtCompileTime) && dst.rows()>1 && dst.cols()>1) + internal::checkTransposeAliasing_impl<Dst, Src>::run(dst, src); } } // end namespace internal
diff --git a/Eigen/src/Core/Transpositions.h b/Eigen/src/Core/Transpositions.h index 19c17bb..81a4a58 100644 --- a/Eigen/src/Core/Transpositions.h +++ b/Eigen/src/Core/Transpositions.h
@@ -84,7 +84,7 @@ } // FIXME: do we want such methods ? - // might be usefull when the target matrix expression is complex, e.g.: + // might be useful when the target matrix expression is complex, e.g.: // object.matrix().block(..,..,..,..) = trans * object.matrix().block(..,..,..,..); /* template<typename MatrixType> @@ -384,7 +384,7 @@ const Product<OtherDerived, Transpose, AliasFreeProduct> operator*(const MatrixBase<OtherDerived>& matrix, const Transpose& trt) { - return Product<OtherDerived, Transpose, AliasFreeProduct>(matrix.derived(), trt.derived()); + return Product<OtherDerived, Transpose, AliasFreeProduct>(matrix.derived(), trt); } /** \returns the \a matrix with the inverse transpositions applied to the rows.
diff --git a/Eigen/src/Core/TriangularMatrix.h b/Eigen/src/Core/TriangularMatrix.h index 5c5e502..cf3532f 100644 --- a/Eigen/src/Core/TriangularMatrix.h +++ b/Eigen/src/Core/TriangularMatrix.h
@@ -65,6 +65,7 @@ inline Index innerStride() const { return derived().innerStride(); } // dummy resize function + EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { EIGEN_UNUSED_VARIABLE(rows); @@ -197,6 +198,7 @@ typedef typename internal::traits<TriangularView>::MatrixTypeNestedNonRef MatrixTypeNestedNonRef; typedef typename internal::remove_all<typename MatrixType::ConjugateReturnType>::type MatrixConjugateReturnType; + typedef TriangularView<typename internal::add_const<MatrixType>::type, _Mode> ConstTriangularView; public: @@ -242,6 +244,18 @@ inline const ConjugateReturnType conjugate() const { return ConjugateReturnType(m_matrix.conjugate()); } + /** \returns an expression of the complex conjugate of \c *this if Cond==true, + * returns \c *this otherwise. + */ + template<bool Cond> + EIGEN_DEVICE_FUNC + inline typename internal::conditional<Cond,ConjugateReturnType,ConstTriangularView>::type + conjugateIf() const + { + typedef typename internal::conditional<Cond,ConjugateReturnType,ConstTriangularView>::type ReturnType; + return ReturnType(m_matrix.template conjugateIf<Cond>()); + } + typedef TriangularView<const typename MatrixType::AdjointReturnType,TransposeMode> AdjointReturnType; /** \sa MatrixBase::adjoint() const */ EIGEN_DEVICE_FUNC @@ -367,14 +381,14 @@ template<typename Other> EIGEN_DEVICE_FUNC TriangularViewType& operator+=(const DenseBase<Other>& other) { - internal::call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op<Scalar>()); + internal::call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op<Scalar,typename Other::Scalar>()); return derived(); } /** \sa MatrixBase::operator-=() */ template<typename Other> EIGEN_DEVICE_FUNC TriangularViewType& operator-=(const DenseBase<Other>& other) { - internal::call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + internal::call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op<Scalar,typename Other::Scalar>()); return derived(); } @@ -470,6 +484,8 @@ * \a Side==OnTheLeft (the default), or the right-inverse-multiply \a other * inverse(\c *this) if * \a Side==OnTheRight. * + * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft + * * The matrix \c *this must be triangular and invertible (i.e., all the coefficients of the * diagonal must be non zero). It works as a forward (resp. backward) substitution if \c *this * is an upper (resp. lower) triangular matrix. @@ -486,7 +502,6 @@ * \sa TriangularView::solveInPlace() */ template<int Side, typename Other> - EIGEN_DEVICE_FUNC inline const internal::triangular_solve_retval<Side,TriangularViewType, Other> solve(const MatrixBase<Other>& other) const; @@ -495,6 +510,8 @@ * \warning The parameter is only marked 'const' to make the C++ compiler accept a temporary expression here. * This function will const_cast it, so constness isn't honored here. * + * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft + * * See TriangularView:solve() for the details. */ template<int Side, typename OtherDerived> @@ -539,27 +556,28 @@ template<typename ProductType> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularViewType& _assignProduct(const ProductType& prod, const Scalar& alpha); + EIGEN_STRONG_INLINE TriangularViewType& _assignProduct(const ProductType& prod, const Scalar& alpha, bool beta); }; /*************************************************************************** * Implementation of triangular evaluation/assignment ***************************************************************************/ +#ifndef EIGEN_PARSED_BY_DOXYGEN // FIXME should we keep that possibility template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -inline TriangularView<MatrixType, Mode>& +EIGEN_DEVICE_FUNC inline TriangularView<MatrixType, Mode>& TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const MatrixBase<OtherDerived>& other) { - internal::call_assignment_no_alias(derived(), other.derived(), internal::assign_op<Scalar>()); + internal::call_assignment_no_alias(derived(), other.derived(), internal::assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } // FIXME should we keep that possibility template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const MatrixBase<OtherDerived>& other) +EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const MatrixBase<OtherDerived>& other) { internal::call_assignment_no_alias(derived(), other.template triangularView<Mode>()); } @@ -568,7 +586,7 @@ template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -inline TriangularView<MatrixType, Mode>& +EIGEN_DEVICE_FUNC inline TriangularView<MatrixType, Mode>& TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const TriangularBase<OtherDerived>& other) { eigen_assert(Mode == int(OtherDerived::Mode)); @@ -578,11 +596,12 @@ template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const TriangularBase<OtherDerived>& other) +EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const TriangularBase<OtherDerived>& other) { eigen_assert(Mode == int(OtherDerived::Mode)); internal::call_assignment_no_alias(derived(), other.derived()); } +#endif /*************************************************************************** * Implementation of TriangularBase methods @@ -592,7 +611,7 @@ * If the matrix is triangular, the opposite part is set to zero. */ template<typename Derived> template<typename DenseDerived> -void TriangularBase<Derived>::evalTo(MatrixBase<DenseDerived> &other) const +EIGEN_DEVICE_FUNC void TriangularBase<Derived>::evalTo(MatrixBase<DenseDerived> &other) const { evalToLazy(other.derived()); } @@ -618,6 +637,7 @@ */ template<typename Derived> template<unsigned int Mode> +EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template TriangularViewReturnType<Mode>::Type MatrixBase<Derived>::triangularView() { @@ -627,6 +647,7 @@ /** This is the const version of MatrixBase::triangularView() */ template<typename Derived> template<unsigned int Mode> +EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template ConstTriangularViewReturnType<Mode>::Type MatrixBase<Derived>::triangularView() const { @@ -641,21 +662,20 @@ template<typename Derived> bool MatrixBase<Derived>::isUpperTriangular(const RealScalar& prec) const { - using std::abs; RealScalar maxAbsOnUpperPart = static_cast<RealScalar>(-1); for(Index j = 0; j < cols(); ++j) { - Index maxi = (std::min)(j, rows()-1); + Index maxi = numext::mini(j, rows()-1); for(Index i = 0; i <= maxi; ++i) { - RealScalar absValue = abs(coeff(i,j)); + RealScalar absValue = numext::abs(coeff(i,j)); if(absValue > maxAbsOnUpperPart) maxAbsOnUpperPart = absValue; } } RealScalar threshold = maxAbsOnUpperPart * prec; for(Index j = 0; j < cols(); ++j) for(Index i = j+1; i < rows(); ++i) - if(abs(coeff(i, j)) > threshold) return false; + if(numext::abs(coeff(i, j)) > threshold) return false; return true; } @@ -667,20 +687,19 @@ template<typename Derived> bool MatrixBase<Derived>::isLowerTriangular(const RealScalar& prec) const { - using std::abs; RealScalar maxAbsOnLowerPart = static_cast<RealScalar>(-1); for(Index j = 0; j < cols(); ++j) for(Index i = j; i < rows(); ++i) { - RealScalar absValue = abs(coeff(i,j)); + RealScalar absValue = numext::abs(coeff(i,j)); if(absValue > maxAbsOnLowerPart) maxAbsOnLowerPart = absValue; } RealScalar threshold = maxAbsOnLowerPart * prec; for(Index j = 1; j < cols(); ++j) { - Index maxi = (std::min)(j, rows()-1); + Index maxi = numext::mini(j, rows()-1); for(Index i = 0; i < maxi; ++i) - if(abs(coeff(i, j)) > threshold) return false; + if(numext::abs(coeff(i, j)) > threshold) return false; } return true; } @@ -711,6 +730,7 @@ { typedef TriangularView<MatrixType,Mode> XprType; typedef evaluator<typename internal::remove_all<MatrixType>::type> Base; + EIGEN_DEVICE_FUNC unary_evaluator(const XprType &xpr) : Base(xpr.nestedExpression()) {} }; @@ -777,15 +797,18 @@ template<int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType, typename Functor> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -void call_triangular_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func) +void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func) { - eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); - typedef evaluator<DstXprType> DstEvaluatorType; typedef evaluator<SrcXprType> SrcEvaluatorType; - DstEvaluatorType dstEvaluator(dst); SrcEvaluatorType srcEvaluator(src); + + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + DstEvaluatorType dstEvaluator(dst); typedef triangular_dense_assignment_kernel< Mode&(Lower|Upper),Mode&(UnitDiag|ZeroDiag|SelfAdjoint),SetOpposite, DstEvaluatorType,SrcEvaluatorType,Functor> Kernel; @@ -794,7 +817,7 @@ enum { unroll = DstXprType::SizeAtCompileTime != Dynamic && SrcEvaluatorType::CoeffReadCost < HugeCost - && DstXprType::SizeAtCompileTime * SrcEvaluatorType::CoeffReadCost / 2 <= EIGEN_UNROLLING_LIMIT + && DstXprType::SizeAtCompileTime * (DstEvaluatorType::CoeffReadCost+SrcEvaluatorType::CoeffReadCost) / 2 <= EIGEN_UNROLLING_LIMIT }; triangular_assignment_loop<Kernel, Mode, unroll ? int(DstXprType::SizeAtCompileTime) : Dynamic, SetOpposite>::run(kernel); @@ -802,9 +825,9 @@ template<int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -void call_triangular_assignment_loop(const DstXprType& dst, const SrcXprType& src) +void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src) { - call_triangular_assignment_loop<Mode,SetOpposite>(dst, src, internal::assign_op<typename DstXprType::Scalar>()); + call_triangular_assignment_loop<Mode,SetOpposite>(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>()); } template<> struct AssignmentKind<TriangularShape,TriangularShape> { typedef Triangular2Triangular Kind; }; @@ -812,8 +835,8 @@ template<> struct AssignmentKind<TriangularShape,DenseShape> { typedef Dense2Triangular Kind; }; -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Triangular, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Triangular> { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { @@ -823,8 +846,8 @@ } }; -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Dense, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Dense> { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { @@ -832,8 +855,8 @@ } }; -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Dense2Triangular, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Dense2Triangular> { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { @@ -893,7 +916,7 @@ { for(Index j = 0; j < kernel.cols(); ++j) { - Index maxi = (std::min)(j, kernel.rows()); + Index maxi = numext::mini(j, kernel.rows()); Index i = 0; if (((Mode&Lower) && SetOpposite) || (Mode&Upper)) { @@ -923,7 +946,7 @@ * If the matrix is triangular, the opposite part is set to zero. */ template<typename Derived> template<typename DenseDerived> -void TriangularBase<Derived>::evalToLazy(MatrixBase<DenseDerived> &other) const +EIGEN_DEVICE_FUNC void TriangularBase<Derived>::evalToLazy(MatrixBase<DenseDerived> &other) const { other.derived().resize(this->rows(), this->cols()); internal::call_triangular_assignment_loop<Derived::Mode,(Derived::Mode&SelfAdjoint)==0 /* SetOpposite */>(other.derived(), derived().nestedExpression()); @@ -933,35 +956,39 @@ // Triangular = Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::assign_op<Scalar>, Dense2Triangular, Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::assign_op<Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, Dense2Triangular> { typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,typename SrcXprType::Scalar> &) { - dst.setZero(); - dst._assignProduct(src, 1); + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + + dst._assignProduct(src, 1, 0); } }; // Triangular += Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::add_assign_op<Scalar>, Dense2Triangular, Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::add_assign_op<Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, Dense2Triangular> { typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar,typename SrcXprType::Scalar> &) { - dst._assignProduct(src, 1); + dst._assignProduct(src, 1, 1); } }; // Triangular -= Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> -struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::sub_assign_op<Scalar>, Dense2Triangular, Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::sub_assign_op<Scalar,typename Product<Lhs,Rhs,DefaultProduct>::Scalar>, Dense2Triangular> { typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar,typename SrcXprType::Scalar> &) { - dst._assignProduct(src, -1); + dst._assignProduct(src, -1, 1); } };
diff --git a/Eigen/src/Core/VectorBlock.h b/Eigen/src/Core/VectorBlock.h index d72fbf7..71c5b95 100644 --- a/Eigen/src/Core/VectorBlock.h +++ b/Eigen/src/Core/VectorBlock.h
@@ -35,7 +35,7 @@ * It is the return type of DenseBase::segment(Index,Index) and DenseBase::segment<int>(Index) and * most of the time this is the only way it is used. * - * However, if you want to directly maniputate sub-vector expressions, + * However, if you want to directly manipulate sub-vector expressions, * for instance if you want to write a function returning such an expression, you * will need to use this class. * @@ -71,8 +71,8 @@ /** Dynamic-size constructor */ - EIGEN_DEVICE_FUNC - inline VectorBlock(VectorType& vector, Index start, Index size) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + VectorBlock(VectorType& vector, Index start, Index size) : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start, IsColVector ? size : 1, IsColVector ? 1 : size) @@ -82,8 +82,8 @@ /** Fixed-size constructor */ - EIGEN_DEVICE_FUNC - inline VectorBlock(VectorType& vector, Index start) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + VectorBlock(VectorType& vector, Index start) : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock);
diff --git a/Eigen/src/Core/VectorwiseOp.h b/Eigen/src/Core/VectorwiseOp.h index 1938911..db0b9f8 100644 --- a/Eigen/src/Core/VectorwiseOp.h +++ b/Eigen/src/Core/VectorwiseOp.h
@@ -81,39 +81,46 @@ const MemberOp m_functor; }; -#define EIGEN_MEMBER_FUNCTOR(MEMBER,COST) \ - template <typename ResultType> \ - struct member_##MEMBER { \ - EIGEN_EMPTY_STRUCT_CTOR(member_##MEMBER) \ - typedef ResultType result_type; \ - template<typename Scalar, int Size> struct Cost \ - { enum { value = COST }; }; \ - template<typename XprType> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ - ResultType operator()(const XprType& mat) const \ - { return mat.MEMBER(); } \ +template<typename A,typename B> struct partial_redux_dummy_func; + +#define EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER,COST,VECTORIZABLE,BINARYOP) \ + template <typename ResultType,typename Scalar> \ + struct member_##MEMBER { \ + EIGEN_EMPTY_STRUCT_CTOR(member_##MEMBER) \ + typedef ResultType result_type; \ + typedef BINARYOP<Scalar,Scalar> BinaryOp; \ + template<int Size> struct Cost { enum { value = COST }; }; \ + enum { Vectorizable = VECTORIZABLE }; \ + template<typename XprType> \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ + ResultType operator()(const XprType& mat) const \ + { return mat.MEMBER(); } \ + BinaryOp binaryFunc() const { return BinaryOp(); } \ } +#define EIGEN_MEMBER_FUNCTOR(MEMBER,COST) \ + EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER,COST,0,partial_redux_dummy_func) + namespace internal { -EIGEN_MEMBER_FUNCTOR(squaredNorm, Size * NumTraits<Scalar>::MulCost + (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(norm, (Size+5) * NumTraits<Scalar>::MulCost + (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(stableNorm, (Size+5) * NumTraits<Scalar>::MulCost + (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(blueNorm, (Size+5) * NumTraits<Scalar>::MulCost + (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(hypotNorm, (Size-1) * functor_traits<scalar_hypot_op<Scalar> >::Cost ); -EIGEN_MEMBER_FUNCTOR(sum, (Size-1)*NumTraits<Scalar>::AddCost); -EIGEN_MEMBER_FUNCTOR(mean, (Size-1)*NumTraits<Scalar>::AddCost + NumTraits<Scalar>::MulCost); -EIGEN_MEMBER_FUNCTOR(minCoeff, (Size-1)*NumTraits<Scalar>::AddCost); -EIGEN_MEMBER_FUNCTOR(maxCoeff, (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(all, (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(any, (Size-1)*NumTraits<Scalar>::AddCost); EIGEN_MEMBER_FUNCTOR(count, (Size-1)*NumTraits<Scalar>::AddCost); -EIGEN_MEMBER_FUNCTOR(prod, (Size-1)*NumTraits<Scalar>::MulCost); -template <int p, typename ResultType> +EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(sum, (Size-1)*NumTraits<Scalar>::AddCost, 1, internal::scalar_sum_op); +EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(minCoeff, (Size-1)*NumTraits<Scalar>::AddCost, 1, internal::scalar_min_op); +EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(maxCoeff, (Size-1)*NumTraits<Scalar>::AddCost, 1, internal::scalar_max_op); +EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(prod, (Size-1)*NumTraits<Scalar>::MulCost, 1, internal::scalar_product_op); + +template <int p, typename ResultType,typename Scalar> struct member_lpnorm { typedef ResultType result_type; - template<typename Scalar, int Size> struct Cost + enum { Vectorizable = 0 }; + template<int Size> struct Cost { enum { value = (Size+5) * NumTraits<Scalar>::MulCost + (Size-1)*NumTraits<Scalar>::AddCost }; }; EIGEN_DEVICE_FUNC member_lpnorm() {} template<typename XprType> @@ -121,17 +128,20 @@ { return mat.template lpNorm<p>(); } }; -template <typename BinaryOp, typename Scalar> +template <typename BinaryOpT, typename Scalar> struct member_redux { + typedef BinaryOpT BinaryOp; typedef typename result_of< BinaryOp(const Scalar&,const Scalar&) >::type result_type; - template<typename _Scalar, int Size> struct Cost - { enum { value = (Size-1) * functor_traits<BinaryOp>::Cost }; }; + + enum { Vectorizable = functor_traits<BinaryOp>::PacketAccess }; + template<int Size> struct Cost { enum { value = (Size-1) * functor_traits<BinaryOp>::Cost }; }; EIGEN_DEVICE_FUNC explicit member_redux(const BinaryOp func) : m_functor(func) {} template<typename Derived> EIGEN_DEVICE_FUNC inline result_type operator()(const DenseBase<Derived>& mat) const { return mat.redux(m_functor); } + const BinaryOp& binaryFunc() const { return m_functor; } const BinaryOp m_functor; }; } @@ -139,18 +149,38 @@ /** \class VectorwiseOp * \ingroup Core_Module * - * \brief Pseudo expression providing partial reduction operations + * \brief Pseudo expression providing broadcasting and partial reduction operations * * \tparam ExpressionType the type of the object on which to do partial reductions - * \tparam Direction indicates the direction of the redux (#Vertical or #Horizontal) + * \tparam Direction indicates whether to operate on columns (#Vertical) or rows (#Horizontal) * - * This class represents a pseudo expression with partial reduction features. + * This class represents a pseudo expression with broadcasting and partial reduction features. * It is the return type of DenseBase::colwise() and DenseBase::rowwise() - * and most of the time this is the only way it is used. + * and most of the time this is the only way it is explicitly used. * + * To understand the logic of rowwise/colwise expression, let's consider a generic case `A.colwise().foo()` + * where `foo` is any method of `VectorwiseOp`. This expression is equivalent to applying `foo()` to each + * column of `A` and then re-assemble the outputs in a matrix expression: + * \code [A.col(0).foo(), A.col(1).foo(), ..., A.col(A.cols()-1).foo()] \endcode + * * Example: \include MatrixBase_colwise.cpp * Output: \verbinclude MatrixBase_colwise.out * + * The begin() and end() methods are obviously exceptions to the previous rule as they + * return STL-compatible begin/end iterators to the rows or columns of the nested expression. + * Typical use cases include for-range-loop and calls to STL algorithms: + * + * Example: \include MatrixBase_colwise_iterator_cxx11.cpp + * Output: \verbinclude MatrixBase_colwise_iterator_cxx11.out + * + * For a partial reduction on an empty input, some rules apply. + * For the sake of clarity, let's consider a vertical reduction: + * - If the number of columns is zero, then a 1x0 row-major vector expression is returned. + * - Otherwise, if the number of rows is zero, then + * - a row vector of zeros is returned for sum-like reductions (sum, squaredNorm, norm, etc.) + * - a row vector of ones is returned for a product reduction (e.g., <code>MatrixXd(n,0).colwise().prod()</code>) + * - an assert is triggered for all other reductions (minCoeff,maxCoeff,redux(bin_op)) + * * \sa DenseBase::colwise(), DenseBase::rowwise(), class PartialReduxExpr */ template<typename ExpressionType, int Direction> class VectorwiseOp @@ -163,11 +193,11 @@ typedef typename internal::ref_selector<ExpressionType>::non_const_type ExpressionTypeNested; typedef typename internal::remove_all<ExpressionTypeNested>::type ExpressionTypeNestedCleaned; - template<template<typename _Scalar> class Functor, - typename Scalar_=Scalar> struct ReturnType + template<template<typename OutScalar,typename InputScalar> class Functor, + typename ReturnScalar=Scalar> struct ReturnType { typedef PartialReduxExpr<ExpressionType, - Functor<Scalar_>, + Functor<ReturnScalar,Scalar>, Direction > Type; }; @@ -186,24 +216,7 @@ }; protected: - - typedef typename internal::conditional<isVertical, - typename ExpressionType::ColXpr, - typename ExpressionType::RowXpr>::type SubVector; - /** \internal - * \returns the i-th subvector according to the \c Direction */ - EIGEN_DEVICE_FUNC - SubVector subVector(Index i) - { - return SubVector(m_matrix.derived(),i); - } - - /** \internal - * \returns the number of subvectors in the direction \c Direction */ - EIGEN_DEVICE_FUNC - Index subVectors() const - { return isVertical?m_matrix.cols():m_matrix.rows(); } - + template<typename OtherDerived> struct ExtendedType { typedef Replicate<OtherDerived, isVertical ? 1 : ExpressionType::RowsAtCompileTime, @@ -258,41 +271,77 @@ EIGEN_DEVICE_FUNC inline const ExpressionType& _expression() const { return m_matrix; } + #ifdef EIGEN_PARSED_BY_DOXYGEN + /** STL-like <a href="https://en.cppreference.com/w/cpp/named_req/RandomAccessIterator">RandomAccessIterator</a> + * iterator type over the columns or rows as returned by the begin() and end() methods. + */ + random_access_iterator_type iterator; + /** This is the const version of iterator (aka read-only) */ + random_access_iterator_type const_iterator; + #else + typedef internal::subvector_stl_iterator<ExpressionType, DirectionType(Direction)> iterator; + typedef internal::subvector_stl_iterator<const ExpressionType, DirectionType(Direction)> const_iterator; + #endif + + /** returns an iterator to the first row (rowwise) or column (colwise) of the nested expression. + * \sa end(), cbegin() + */ + iterator begin() const { return iterator (m_matrix, 0); } + /** const version of begin() */ + const_iterator cbegin() const { return const_iterator(m_matrix, 0); } + + /** returns an iterator to the row (resp. column) following the last row (resp. column) of the nested expression + * \sa begin(), cend() + */ + iterator end() const { return iterator (m_matrix, m_matrix.template subVectors<DirectionType(Direction)>()); } + /** const version of end() */ + const_iterator cend() const { return const_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>()); } + /** \returns a row or column vector expression of \c *this reduxed by \a func * * The template parameter \a BinaryOp is the type of the functor * of the custom redux operator. Note that func must be an associative operator. * + * \warning the size along the reduction direction must be strictly positive, + * otherwise an assertion is triggered. + * * \sa class VectorwiseOp, DenseBase::colwise(), DenseBase::rowwise() */ template<typename BinaryOp> EIGEN_DEVICE_FUNC const typename ReduxReturnType<BinaryOp>::Type redux(const BinaryOp& func = BinaryOp()) const - { return typename ReduxReturnType<BinaryOp>::Type(_expression(), internal::member_redux<BinaryOp,Scalar>(func)); } + { + eigen_assert(redux_length()>0 && "you are using an empty matrix"); + return typename ReduxReturnType<BinaryOp>::Type(_expression(), internal::member_redux<BinaryOp,Scalar>(func)); + } typedef typename ReturnType<internal::member_minCoeff>::Type MinCoeffReturnType; typedef typename ReturnType<internal::member_maxCoeff>::Type MaxCoeffReturnType; - typedef typename ReturnType<internal::member_squaredNorm,RealScalar>::Type SquaredNormReturnType; - typedef typename ReturnType<internal::member_norm,RealScalar>::Type NormReturnType; + typedef PartialReduxExpr<const CwiseUnaryOp<internal::scalar_abs2_op<Scalar>, const ExpressionTypeNestedCleaned>,internal::member_sum<RealScalar,RealScalar>,Direction> SquaredNormReturnType; + typedef CwiseUnaryOp<internal::scalar_sqrt_op<RealScalar>, const SquaredNormReturnType> NormReturnType; typedef typename ReturnType<internal::member_blueNorm,RealScalar>::Type BlueNormReturnType; typedef typename ReturnType<internal::member_stableNorm,RealScalar>::Type StableNormReturnType; typedef typename ReturnType<internal::member_hypotNorm,RealScalar>::Type HypotNormReturnType; typedef typename ReturnType<internal::member_sum>::Type SumReturnType; - typedef typename ReturnType<internal::member_mean>::Type MeanReturnType; + typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(SumReturnType,Scalar,quotient) MeanReturnType; typedef typename ReturnType<internal::member_all>::Type AllReturnType; typedef typename ReturnType<internal::member_any>::Type AnyReturnType; - typedef PartialReduxExpr<ExpressionType, internal::member_count<Index>, Direction> CountReturnType; + typedef PartialReduxExpr<ExpressionType, internal::member_count<Index,Scalar>, Direction> CountReturnType; typedef typename ReturnType<internal::member_prod>::Type ProdReturnType; + typedef Reverse<const ExpressionType, Direction> ConstReverseReturnType; typedef Reverse<ExpressionType, Direction> ReverseReturnType; template<int p> struct LpNormReturnType { - typedef PartialReduxExpr<ExpressionType, internal::member_lpnorm<p,RealScalar>,Direction> Type; + typedef PartialReduxExpr<ExpressionType, internal::member_lpnorm<p,RealScalar,Scalar>,Direction> Type; }; /** \returns a row (or column) vector expression of the smallest coefficient * of each column (or row) of the referenced expression. * + * \warning the size along the reduction direction must be strictly positive, + * otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * Example: \include PartialRedux_minCoeff.cpp @@ -301,11 +350,17 @@ * \sa DenseBase::minCoeff() */ EIGEN_DEVICE_FUNC const MinCoeffReturnType minCoeff() const - { return MinCoeffReturnType(_expression()); } + { + eigen_assert(redux_length()>0 && "you are using an empty matrix"); + return MinCoeffReturnType(_expression()); + } /** \returns a row (or column) vector expression of the largest coefficient * of each column (or row) of the referenced expression. * + * \warning the size along the reduction direction must be strictly positive, + * otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * Example: \include PartialRedux_maxCoeff.cpp @@ -314,7 +369,10 @@ * \sa DenseBase::maxCoeff() */ EIGEN_DEVICE_FUNC const MaxCoeffReturnType maxCoeff() const - { return MaxCoeffReturnType(_expression()); } + { + eigen_assert(redux_length()>0 && "you are using an empty matrix"); + return MaxCoeffReturnType(_expression()); + } /** \returns a row (or column) vector expression of the squared norm * of each column (or row) of the referenced expression. @@ -326,7 +384,7 @@ * \sa DenseBase::squaredNorm() */ EIGEN_DEVICE_FUNC const SquaredNormReturnType squaredNorm() const - { return SquaredNormReturnType(_expression()); } + { return SquaredNormReturnType(m_matrix.cwiseAbs2()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression. @@ -338,7 +396,7 @@ * \sa DenseBase::norm() */ EIGEN_DEVICE_FUNC const NormReturnType norm() const - { return NormReturnType(_expression()); } + { return NormReturnType(squaredNorm()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression. @@ -403,7 +461,7 @@ * \sa DenseBase::mean() */ EIGEN_DEVICE_FUNC const MeanReturnType mean() const - { return MeanReturnType(_expression()); } + { return sum() / Scalar(Direction==Vertical?m_matrix.rows():m_matrix.cols()); } /** \returns a row (or column) vector expression representing * whether \b all coefficients of each respective column (or row) are \c true. @@ -456,7 +514,15 @@ * * \sa DenseBase::reverse() */ EIGEN_DEVICE_FUNC - const ReverseReturnType reverse() const + const ConstReverseReturnType reverse() const + { return ConstReverseReturnType( _expression() ); } + + /** \returns a writable matrix expression + * where each column (or row) are reversed. + * + * \sa reverse() const */ + EIGEN_DEVICE_FUNC + ReverseReturnType reverse() { return ReverseReturnType( _expression() ); } typedef Replicate<ExpressionType,(isVertical?Dynamic:1),(isHorizontal?Dynamic:1)> ReplicateReturnType; @@ -491,7 +557,7 @@ EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) //eigen_assert((m_matrix.isNull()) == (other.isNull())); FIXME - return const_cast<ExpressionType&>(m_matrix = extendedTo(other.derived())); + return m_matrix = extendedTo(other.derived()); } /** Adds the vector \a other to each subvector of \c *this */ @@ -501,7 +567,7 @@ { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) - return const_cast<ExpressionType&>(m_matrix += extendedTo(other.derived())); + return m_matrix += extendedTo(other.derived()); } /** Substracts the vector \a other to each subvector of \c *this */ @@ -511,7 +577,7 @@ { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) - return const_cast<ExpressionType&>(m_matrix -= extendedTo(other.derived())); + return m_matrix -= extendedTo(other.derived()); } /** Multiples each subvector of \c *this by the vector \a other */ @@ -523,7 +589,7 @@ EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) m_matrix *= extendedTo(other.derived()); - return const_cast<ExpressionType&>(m_matrix); + return m_matrix; } /** Divides each subvector of \c *this by the vector \a other */ @@ -535,12 +601,12 @@ EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) m_matrix /= extendedTo(other.derived()); - return const_cast<ExpressionType&>(m_matrix); + return m_matrix; } /** Returns the expression of the sum of the vector \a other to each subvector of \c *this */ template<typename OtherDerived> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC - CwiseBinaryOp<internal::scalar_sum_op<Scalar>, + CwiseBinaryOp<internal::scalar_sum_op<Scalar,typename OtherDerived::Scalar>, const ExpressionTypeNestedCleaned, const typename ExtendedType<OtherDerived>::Type> operator+(const DenseBase<OtherDerived>& other) const @@ -553,7 +619,7 @@ /** Returns the expression of the difference between each subvector of \c *this and the vector \a other */ template<typename OtherDerived> EIGEN_DEVICE_FUNC - CwiseBinaryOp<internal::scalar_difference_op<Scalar>, + CwiseBinaryOp<internal::scalar_difference_op<Scalar,typename OtherDerived::Scalar>, const ExpressionTypeNestedCleaned, const typename ExtendedType<OtherDerived>::Type> operator-(const DenseBase<OtherDerived>& other) const @@ -593,14 +659,14 @@ return m_matrix / extendedTo(other.derived()); } - /** \returns an expression where each column of row of the referenced matrix are normalized. + /** \returns an expression where each column (or row) of the referenced matrix are normalized. * The referenced matrix is \b not modified. * \sa MatrixBase::normalized(), normalize() */ EIGEN_DEVICE_FUNC CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const ExpressionTypeNestedCleaned, - const typename OppositeExtendedType<typename ReturnType<internal::member_norm,RealScalar>::Type>::Type> + const typename OppositeExtendedType<NormReturnType>::Type> normalized() const { return m_matrix.cwiseQuotient(extendedToOpposite(this->norm())); } @@ -616,6 +682,7 @@ /////////// Geometry module /////////// typedef Homogeneous<ExpressionType,Direction> HomogeneousReturnType; + EIGEN_DEVICE_FUNC HomogeneousReturnType homogeneous() const; typedef typename ExpressionType::PlainObject CrossReturnType; @@ -645,9 +712,14 @@ Direction==Horizontal ? HNormalized_SizeMinusOne : 1> > HNormalizedReturnType; + EIGEN_DEVICE_FUNC const HNormalizedReturnType hnormalized() const; protected: + Index redux_length() const + { + return Direction==Vertical ? m_matrix.rows() : m_matrix.cols(); + } ExpressionTypeNested m_matrix; }; @@ -659,7 +731,7 @@ * \sa rowwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ template<typename Derived> -inline typename DenseBase<Derived>::ColwiseReturnType +EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::ColwiseReturnType DenseBase<Derived>::colwise() { return ColwiseReturnType(derived()); @@ -673,7 +745,7 @@ * \sa colwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ template<typename Derived> -inline typename DenseBase<Derived>::RowwiseReturnType +EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::RowwiseReturnType DenseBase<Derived>::rowwise() { return RowwiseReturnType(derived());
diff --git a/Eigen/src/Core/Visitor.h b/Eigen/src/Core/Visitor.h index d71dfc9..f67d83b 100644 --- a/Eigen/src/Core/Visitor.h +++ b/Eigen/src/Core/Visitor.h
@@ -40,6 +40,14 @@ } }; +// This specialization enables visitors on empty matrices at compile-time +template<typename Visitor, typename Derived> +struct visitor_impl<Visitor, Derived, 0> { + EIGEN_DEVICE_FUNC + static inline void run(const Derived &/*mat*/, Visitor& /*visitor*/) + {} +}; + template<typename Visitor, typename Derived> struct visitor_impl<Visitor, Derived, Dynamic> { @@ -98,6 +106,8 @@ * * \note compared to one or two \em for \em loops, visitors offer automatic * unrolling for small fixed size matrix. + * + * \note if the matrix is empty, then the visitor is left unchanged. * * \sa minCoeff(Index*,Index*), maxCoeff(Index*,Index*), DenseBase::redux() */ @@ -106,6 +116,9 @@ EIGEN_DEVICE_FUNC void DenseBase<Derived>::visit(Visitor& visitor) const { + if(size()==0) + return; + typedef typename internal::visitor_evaluator<Derived> ThisEvaluator; ThisEvaluator thisEval(derived()); @@ -124,6 +137,8 @@ template <typename Derived> struct coeff_visitor { + // default initialization to avoid countless invalid maybe-uninitialized warnings by gcc + coeff_visitor() : row(-1), col(-1), res(0) {} typedef typename Derived::Scalar Scalar; Index row, col; Scalar res; @@ -194,7 +209,11 @@ } // end namespace internal -/** \returns the minimum of all coefficients of *this and puts in *row and *col its location. +/** \fn DenseBase<Derived>::minCoeff(IndexType* rowId, IndexType* colId) const + * \returns the minimum of all coefficients of *this and puts in *row and *col its location. + * + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * \sa DenseBase::minCoeff(Index*), DenseBase::maxCoeff(Index*,Index*), DenseBase::visit(), DenseBase::minCoeff() @@ -205,6 +224,8 @@ typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff(IndexType* rowId, IndexType* colId) const { + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); + internal::min_coeff_visitor<Derived> minVisitor; this->visit(minVisitor); *rowId = minVisitor.row; @@ -213,6 +234,9 @@ } /** \returns the minimum of all coefficients of *this and puts in *index its location. + * + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::minCoeff() @@ -223,6 +247,8 @@ typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff(IndexType* index) const { + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) internal::min_coeff_visitor<Derived> minVisitor; this->visit(minVisitor); @@ -230,7 +256,11 @@ return minVisitor.res; } -/** \returns the maximum of all coefficients of *this and puts in *row and *col its location. +/** \fn DenseBase<Derived>::maxCoeff(IndexType* rowId, IndexType* colId) const + * \returns the maximum of all coefficients of *this and puts in *row and *col its location. + * + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::maxCoeff() @@ -241,6 +271,8 @@ typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff(IndexType* rowPtr, IndexType* colPtr) const { + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); + internal::max_coeff_visitor<Derived> maxVisitor; this->visit(maxVisitor); *rowPtr = maxVisitor.row; @@ -249,6 +281,9 @@ } /** \returns the maximum of all coefficients of *this and puts in *index its location. + * + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * * \warning the result is undefined if \c *this contains NaN. * * \sa DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visitor(), DenseBase::maxCoeff() @@ -259,6 +294,8 @@ typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff(IndexType* index) const { + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) internal::max_coeff_visitor<Derived> maxVisitor; this->visit(maxVisitor);
diff --git a/Eigen/src/Core/arch/AVX/CMakeLists.txt b/Eigen/src/Core/arch/AVX/CMakeLists.txt deleted file mode 100644 index bdb71ab..0000000 --- a/Eigen/src/Core/arch/AVX/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_AVX_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_AVX_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/AVX COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/AVX/Complex.h b/Eigen/src/Core/arch/AVX/Complex.h index b16e0dd..5b8ff59 100644 --- a/Eigen/src/Core/arch/AVX/Complex.h +++ b/Eigen/src/Core/arch/AVX/Complex.h
@@ -22,6 +22,7 @@ __m256 v; }; +#ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits<std::complex<float> > : default_packet_traits { typedef Packet4cf type; @@ -44,8 +45,9 @@ HasSetLinear = 0 }; }; +#endif -template<> struct unpacket_traits<Packet4cf> { typedef std::complex<float> type; enum {size=4, alignment=Aligned32}; typedef Packet2cf half; }; +template<> struct unpacket_traits<Packet4cf> { typedef std::complex<float> type; enum {size=4, alignment=Aligned32, vectorizable=true}; typedef Packet2cf half; }; template<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); } @@ -67,10 +69,18 @@ return Packet4cf(result); } +template <> +EIGEN_STRONG_INLINE Packet4cf pcmp_eq(const Packet4cf& a, const Packet4cf& b) { + __m256 eq = _mm256_cmp_ps(a.v, b.v, _CMP_EQ_OQ); + return Packet4cf(_mm256_and_ps(eq, _mm256_permute_ps(eq, 0xb1))); +} + +template<> EIGEN_STRONG_INLINE Packet4cf ptrue<Packet4cf>(const Packet4cf& a) { return Packet4cf(ptrue(Packet8f(a.v))); } +template<> EIGEN_STRONG_INLINE Packet4cf pnot<Packet4cf>(const Packet4cf& a) { return Packet4cf(pnot(Packet8f(a.v))); } template<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); } @@ -204,23 +214,7 @@ } }; -template<> struct conj_helper<Packet8f, Packet4cf, false,false> -{ - EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet8f& x, const Packet4cf& y, const Packet4cf& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet4cf pmul(const Packet8f& x, const Packet4cf& y) const - { return Packet4cf(Eigen::internal::pmul(x, y.v)); } -}; - -template<> struct conj_helper<Packet4cf, Packet8f, false,false> -{ - EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet8f& y, const Packet4cf& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& x, const Packet8f& y) const - { return Packet4cf(Eigen::internal::pmul(x.v, y)); } -}; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cf,Packet8f) template<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { @@ -244,6 +238,7 @@ __m256d v; }; +#ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits<std::complex<double> > : default_packet_traits { typedef Packet2cd type; @@ -266,8 +261,9 @@ HasSetLinear = 0 }; }; +#endif -template<> struct unpacket_traits<Packet2cd> { typedef std::complex<double> type; enum {size=2, alignment=Aligned32}; typedef Packet1cd half; }; +template<> struct unpacket_traits<Packet2cd> { typedef std::complex<double> type; enum {size=2, alignment=Aligned32, vectorizable=true}; typedef Packet1cd half; }; template<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); } @@ -288,10 +284,18 @@ return Packet2cd(_mm256_addsub_pd(even, odd)); } +template <> +EIGEN_STRONG_INLINE Packet2cd pcmp_eq(const Packet2cd& a, const Packet2cd& b) { + __m256d eq = _mm256_cmp_pd(a.v, b.v, _CMP_EQ_OQ); + return Packet2cd(pand(eq, _mm256_permute_pd(eq, 0x5))); +} + +template<> EIGEN_STRONG_INLINE Packet2cd ptrue<Packet2cd>(const Packet2cd& a) { return Packet2cd(ptrue(Packet4d(a.v))); } +template<> EIGEN_STRONG_INLINE Packet2cd pnot<Packet2cd>(const Packet2cd& a) { return Packet2cd(pnot(Packet4d(a.v))); } template<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(const std::complex<double>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload<Packet4d>((const double*)from)); } @@ -400,23 +404,7 @@ } }; -template<> struct conj_helper<Packet4d, Packet2cd, false,false> -{ - EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet4d& x, const Packet2cd& y, const Packet2cd& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet2cd pmul(const Packet4d& x, const Packet2cd& y) const - { return Packet2cd(Eigen::internal::pmul(x, y.v)); } -}; - -template<> struct conj_helper<Packet2cd, Packet4d, false,false> -{ - EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet4d& y, const Packet2cd& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& x, const Packet4d& y) const - { return Packet2cd(Eigen::internal::pmul(x.v, y)); } -}; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cd,Packet4d) template<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { @@ -456,6 +444,26 @@ kernel.packet[0].v = tmp; } +template<> EIGEN_STRONG_INLINE Packet4cf pinsertfirst(const Packet4cf& a, std::complex<float> b) +{ + return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,1|2)); +} + +template<> EIGEN_STRONG_INLINE Packet2cd pinsertfirst(const Packet2cd& a, std::complex<double> b) +{ + return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,1|2)); +} + +template<> EIGEN_STRONG_INLINE Packet4cf pinsertlast(const Packet4cf& a, std::complex<float> b) +{ + return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,(1<<7)|(1<<6))); +} + +template<> EIGEN_STRONG_INLINE Packet2cd pinsertlast(const Packet2cd& a, std::complex<double> b) +{ + return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,(1<<3)|(1<<2))); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX/MathFunctions.h b/Eigen/src/Core/arch/AVX/MathFunctions.h index 98d8e02..9f375ed 100644 --- a/Eigen/src/Core/arch/AVX/MathFunctions.h +++ b/Eigen/src/Core/arch/AVX/MathFunctions.h
@@ -10,7 +10,7 @@ #ifndef EIGEN_MATH_FUNCTIONS_AVX_H #define EIGEN_MATH_FUNCTIONS_AVX_H -/* The sin, cos, exp, and log functions of this file are loosely derived from +/* The sin and cos functions of this file are loosely derived from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ @@ -18,187 +18,22 @@ namespace internal { -inline Packet8i pshiftleft(Packet8i v, int n) -{ -#ifdef EIGEN_VECTORIZE_AVX2 - return _mm256_slli_epi32(v, n); -#else - __m128i lo = _mm_slli_epi32(_mm256_extractf128_si256(v, 0), n); - __m128i hi = _mm_slli_epi32(_mm256_extractf128_si256(v, 1), n); - return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); -#endif -} - -inline Packet8f pshiftright(Packet8f v, int n) -{ -#ifdef EIGEN_VECTORIZE_AVX2 - return _mm256_cvtepi32_ps(_mm256_srli_epi32(_mm256_castps_si256(v), n)); -#else - __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(_mm256_castps_si256(v), 0), n); - __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(_mm256_castps_si256(v), 1), n); - return _mm256_cvtepi32_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1)); -#endif -} - -// Sine function -// Computes sin(x) by wrapping x to the interval [-Pi/4,3*Pi/4] and -// evaluating interpolants in [-Pi/4,Pi/4] or [Pi/4,3*Pi/4]. The interpolants -// are (anti-)symmetric and thus have only odd/even coefficients template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psin<Packet8f>(const Packet8f& _x) { - Packet8f x = _x; - - // Some useful values. - _EIGEN_DECLARE_CONST_Packet8i(one, 1); - _EIGEN_DECLARE_CONST_Packet8f(one, 1.0f); - _EIGEN_DECLARE_CONST_Packet8f(two, 2.0f); - _EIGEN_DECLARE_CONST_Packet8f(one_over_four, 0.25f); - _EIGEN_DECLARE_CONST_Packet8f(one_over_pi, 3.183098861837907e-01f); - _EIGEN_DECLARE_CONST_Packet8f(neg_pi_first, -3.140625000000000e+00f); - _EIGEN_DECLARE_CONST_Packet8f(neg_pi_second, -9.670257568359375e-04f); - _EIGEN_DECLARE_CONST_Packet8f(neg_pi_third, -6.278329571784980e-07f); - _EIGEN_DECLARE_CONST_Packet8f(four_over_pi, 1.273239544735163e+00f); - - // Map x from [-Pi/4,3*Pi/4] to z in [-1,3] and subtract the shifted period. - Packet8f z = pmul(x, p8f_one_over_pi); - Packet8f shift = _mm256_floor_ps(padd(z, p8f_one_over_four)); - x = pmadd(shift, p8f_neg_pi_first, x); - x = pmadd(shift, p8f_neg_pi_second, x); - x = pmadd(shift, p8f_neg_pi_third, x); - z = pmul(x, p8f_four_over_pi); - - // Make a mask for the entries that need flipping, i.e. wherever the shift - // is odd. - Packet8i shift_ints = _mm256_cvtps_epi32(shift); - Packet8i shift_isodd = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(shift_ints), _mm256_castsi256_ps(p8i_one))); - Packet8i sign_flip_mask = pshiftleft(shift_isodd, 31); - - // Create a mask for which interpolant to use, i.e. if z > 1, then the mask - // is set to ones for that entry. - Packet8f ival_mask = _mm256_cmp_ps(z, p8f_one, _CMP_GT_OQ); - - // Evaluate the polynomial for the interval [1,3] in z. - _EIGEN_DECLARE_CONST_Packet8f(coeff_right_0, 9.999999724233232e-01f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_right_2, -3.084242535619928e-01f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_right_4, 1.584991525700324e-02f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_right_6, -3.188805084631342e-04f); - Packet8f z_minus_two = psub(z, p8f_two); - Packet8f z_minus_two2 = pmul(z_minus_two, z_minus_two); - Packet8f right = pmadd(p8f_coeff_right_6, z_minus_two2, p8f_coeff_right_4); - right = pmadd(right, z_minus_two2, p8f_coeff_right_2); - right = pmadd(right, z_minus_two2, p8f_coeff_right_0); - - // Evaluate the polynomial for the interval [-1,1] in z. - _EIGEN_DECLARE_CONST_Packet8f(coeff_left_1, 7.853981525427295e-01f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_left_3, -8.074536727092352e-02f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_left_5, 2.489871967827018e-03f); - _EIGEN_DECLARE_CONST_Packet8f(coeff_left_7, -3.587725841214251e-05f); - Packet8f z2 = pmul(z, z); - Packet8f left = pmadd(p8f_coeff_left_7, z2, p8f_coeff_left_5); - left = pmadd(left, z2, p8f_coeff_left_3); - left = pmadd(left, z2, p8f_coeff_left_1); - left = pmul(left, z); - - // Assemble the results, i.e. select the left and right polynomials. - left = _mm256_andnot_ps(ival_mask, left); - right = _mm256_and_ps(ival_mask, right); - Packet8f res = _mm256_or_ps(left, right); - - // Flip the sign on the odd intervals and return the result. - res = _mm256_xor_ps(res, _mm256_castsi256_ps(sign_flip_mask)); - return res; + return psin_float(_x); } -// Natural logarithm -// Computes log(x) as log(2^e * m) = C*e + log(m), where the constant C =log(2) -// and m is in the range [sqrt(1/2),sqrt(2)). In this range, the logarithm can -// be easily approximated by a polynomial centered on m=1 for stability. -// TODO(gonnet): Further reduce the interval allowing for lower-degree -// polynomial interpolants -> ... -> profit! +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f +pcos<Packet8f>(const Packet8f& _x) { + return pcos_float(_x); +} + template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f plog<Packet8f>(const Packet8f& _x) { - Packet8f x = _x; - _EIGEN_DECLARE_CONST_Packet8f(1, 1.0f); - _EIGEN_DECLARE_CONST_Packet8f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet8f(126f, 126.0f); - - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(inv_mant_mask, ~0x7f800000); - - // The smallest non denormalized float number. - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(min_norm_pos, 0x00800000); - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(minus_inf, 0xff800000); - - // Polynomial coefficients. - _EIGEN_DECLARE_CONST_Packet8f(cephes_SQRTHF, 0.707106781186547524f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p0, 7.0376836292E-2f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p1, -1.1514610310E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p2, 1.1676998740E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p3, -1.2420140846E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p4, +1.4249322787E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p5, -1.6668057665E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p6, +2.0000714765E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p7, -2.4999993993E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_p8, +3.3333331174E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_q1, -2.12194440e-4f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_log_q2, 0.693359375f); - - Packet8f invalid_mask = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_NGE_UQ); // not greater equal is true if x is NaN - Packet8f iszero_mask = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_EQ_OQ); - - // Truncate input values to the minimum positive normal. - x = pmax(x, p8f_min_norm_pos); - - Packet8f emm0 = pshiftright(x,23); - Packet8f e = _mm256_sub_ps(emm0, p8f_126f); - - // Set the exponents to -1, i.e. x are in the range [0.5,1). - x = _mm256_and_ps(x, p8f_inv_mant_mask); - x = _mm256_or_ps(x, p8f_half); - - // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) - // and shift by -1. The values are then centered around 0, which improves - // the stability of the polynomial evaluation. - // if( x < SQRTHF ) { - // e -= 1; - // x = x + x - 1.0; - // } else { x = x - 1.0; } - Packet8f mask = _mm256_cmp_ps(x, p8f_cephes_SQRTHF, _CMP_LT_OQ); - Packet8f tmp = _mm256_and_ps(x, mask); - x = psub(x, p8f_1); - e = psub(e, _mm256_and_ps(p8f_1, mask)); - x = padd(x, tmp); - - Packet8f x2 = pmul(x, x); - Packet8f x3 = pmul(x2, x); - - // Evaluate the polynomial approximant of degree 8 in three parts, probably - // to improve instruction-level parallelism. - Packet8f y, y1, y2; - y = pmadd(p8f_cephes_log_p0, x, p8f_cephes_log_p1); - y1 = pmadd(p8f_cephes_log_p3, x, p8f_cephes_log_p4); - y2 = pmadd(p8f_cephes_log_p6, x, p8f_cephes_log_p7); - y = pmadd(y, x, p8f_cephes_log_p2); - y1 = pmadd(y1, x, p8f_cephes_log_p5); - y2 = pmadd(y2, x, p8f_cephes_log_p8); - y = pmadd(y, x3, y1); - y = pmadd(y, x3, y2); - y = pmul(y, x3); - - // Add the logarithm of the exponent back to the result of the interpolation. - y1 = pmul(e, p8f_cephes_log_q1); - tmp = pmul(x2, p8f_half); - y = padd(y, y1); - x = psub(x, tmp); - y2 = pmul(e, p8f_cephes_log_q2); - x = padd(x, y); - x = padd(x, y2); - - // Filter out invalid inputs, i.e. negative arg will be NAN, 0 will be -INF. - return _mm256_or_ps( - _mm256_andnot_ps(iszero_mask, _mm256_or_ps(x, invalid_mask)), - _mm256_and_ps(iszero_mask, p8f_minus_inf)); + return plog_float(_x); } // Exponential function. Works by writing "x = m*log(2) + r" where @@ -207,220 +42,46 @@ template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f pexp<Packet8f>(const Packet8f& _x) { - _EIGEN_DECLARE_CONST_Packet8f(1, 1.0f); - _EIGEN_DECLARE_CONST_Packet8f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet8f(127, 127.0f); - - _EIGEN_DECLARE_CONST_Packet8f(exp_hi, 88.3762626647950f); - _EIGEN_DECLARE_CONST_Packet8f(exp_lo, -88.3762626647949f); - - _EIGEN_DECLARE_CONST_Packet8f(cephes_LOG2EF, 1.44269504088896341f); - - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p0, 1.9875691500E-4f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p1, 1.3981999507E-3f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p2, 8.3334519073E-3f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p3, 4.1665795894E-2f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p4, 1.6666665459E-1f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_p5, 5.0000001201E-1f); - - // Clamp x. - Packet8f x = pmax(pmin(_x, p8f_exp_hi), p8f_exp_lo); - - // Express exp(x) as exp(m*ln(2) + r), start by extracting - // m = floor(x/ln(2) + 0.5). - Packet8f m = _mm256_floor_ps(pmadd(x, p8f_cephes_LOG2EF, p8f_half)); - -// Get r = x - m*ln(2). If no FMA instructions are available, m*ln(2) is -// subtracted out in two parts, m*C1+m*C2 = m*ln(2), to avoid accumulating -// truncation errors. Note that we don't use the "pmadd" function here to -// ensure that a precision-preserving FMA instruction is used. -#ifdef EIGEN_VECTORIZE_FMA - _EIGEN_DECLARE_CONST_Packet8f(nln2, -0.6931471805599453f); - Packet8f r = _mm256_fmadd_ps(m, p8f_nln2, x); -#else - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_C1, 0.693359375f); - _EIGEN_DECLARE_CONST_Packet8f(cephes_exp_C2, -2.12194440e-4f); - Packet8f r = psub(x, pmul(m, p8f_cephes_exp_C1)); - r = psub(r, pmul(m, p8f_cephes_exp_C2)); -#endif - - Packet8f r2 = pmul(r, r); - - // TODO(gonnet): Split into odd/even polynomials and try to exploit - // instruction-level parallelism. - Packet8f y = p8f_cephes_exp_p0; - y = pmadd(y, r, p8f_cephes_exp_p1); - y = pmadd(y, r, p8f_cephes_exp_p2); - y = pmadd(y, r, p8f_cephes_exp_p3); - y = pmadd(y, r, p8f_cephes_exp_p4); - y = pmadd(y, r, p8f_cephes_exp_p5); - y = pmadd(y, r2, r); - y = padd(y, p8f_1); - - // Build emm0 = 2^m. - Packet8i emm0 = _mm256_cvttps_epi32(padd(m, p8f_127)); - emm0 = pshiftleft(emm0, 23); - - // Return 2^m * exp(r). - return pmax(pmul(y, _mm256_castsi256_ps(emm0)), _x); + return pexp_float(_x); } // Hyperbolic Tangent function. -// Doesn't do anything fancy, just a 13/6-degree rational interpolant which -// is accurate up to a couple of ulp in the range [-9, 9], outside of which the -// fl(tanh(x)) = +/-1. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -ptanh<Packet8f>(const Packet8f& _x) { - // Clamp the inputs to the range [-9, 9] since anything outside - // this range is +/-1.0f in single-precision. - _EIGEN_DECLARE_CONST_Packet8f(plus_9, 9.0f); - _EIGEN_DECLARE_CONST_Packet8f(minus_9, -9.0f); - const Packet8f x = pmax(p8f_minus_9, pmin(p8f_plus_9, _x)); - - // The monomial coefficients of the numerator polynomial (odd). - _EIGEN_DECLARE_CONST_Packet8f(alpha_1, 4.89352455891786e-03f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_3, 6.37261928875436e-04f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_5, 1.48572235717979e-05f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_7, 5.12229709037114e-08f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_9, -8.60467152213735e-11f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_11, 2.00018790482477e-13f); - _EIGEN_DECLARE_CONST_Packet8f(alpha_13, -2.76076847742355e-16f); - - // The monomial coefficients of the denominator polynomial (even). - _EIGEN_DECLARE_CONST_Packet8f(beta_0, 4.89352518554385e-03f); - _EIGEN_DECLARE_CONST_Packet8f(beta_2, 2.26843463243900e-03f); - _EIGEN_DECLARE_CONST_Packet8f(beta_4, 1.18534705686654e-04f); - _EIGEN_DECLARE_CONST_Packet8f(beta_6, 1.19825839466702e-06f); - - // Since the polynomials are odd/even, we need x^2. - const Packet8f x2 = pmul(x, x); - - // Evaluate the numerator polynomial p. - Packet8f p = pmadd(x2, p8f_alpha_13, p8f_alpha_11); - p = pmadd(x2, p, p8f_alpha_9); - p = pmadd(x2, p, p8f_alpha_7); - p = pmadd(x2, p, p8f_alpha_5); - p = pmadd(x2, p, p8f_alpha_3); - p = pmadd(x2, p, p8f_alpha_1); - p = pmul(x, p); - - // Evaluate the denominator polynomial p. - Packet8f q = pmadd(x2, p8f_beta_6, p8f_beta_4); - q = pmadd(x2, q, p8f_beta_2); - q = pmadd(x2, q, p8f_beta_0); - - // Divide the numerator by the denominator. - return pdiv(p, q); +ptanh<Packet8f>(const Packet8f& x) { + return internal::generic_fast_tanh_float(x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d -pexp<Packet4d>(const Packet4d& _x) { - Packet4d x = _x; - - _EIGEN_DECLARE_CONST_Packet4d(1, 1.0); - _EIGEN_DECLARE_CONST_Packet4d(2, 2.0); - _EIGEN_DECLARE_CONST_Packet4d(half, 0.5); - - _EIGEN_DECLARE_CONST_Packet4d(exp_hi, 709.437); - _EIGEN_DECLARE_CONST_Packet4d(exp_lo, -709.436139303); - - _EIGEN_DECLARE_CONST_Packet4d(cephes_LOG2EF, 1.4426950408889634073599); - - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_p0, 1.26177193074810590878e-4); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_p1, 3.02994407707441961300e-2); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_p2, 9.99999999999999999910e-1); - - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_q0, 3.00198505138664455042e-6); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_q1, 2.52448340349684104192e-3); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_q2, 2.27265548208155028766e-1); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_q3, 2.00000000000000000009e0); - - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_C1, 0.693145751953125); - _EIGEN_DECLARE_CONST_Packet4d(cephes_exp_C2, 1.42860682030941723212e-6); - _EIGEN_DECLARE_CONST_Packet4i(1023, 1023); - - Packet4d tmp, fx; - - // clamp x - x = pmax(pmin(x, p4d_exp_hi), p4d_exp_lo); - // Express exp(x) as exp(g + n*log(2)). - fx = pmadd(p4d_cephes_LOG2EF, x, p4d_half); - - // Get the integer modulus of log(2), i.e. the "n" described above. - fx = _mm256_floor_pd(fx); - - // Get the remainder modulo log(2), i.e. the "g" described above. Subtract - // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last - // digits right. - tmp = pmul(fx, p4d_cephes_exp_C1); - Packet4d z = pmul(fx, p4d_cephes_exp_C2); - x = psub(x, tmp); - x = psub(x, z); - - Packet4d x2 = pmul(x, x); - - // Evaluate the numerator polynomial of the rational interpolant. - Packet4d px = p4d_cephes_exp_p0; - px = pmadd(px, x2, p4d_cephes_exp_p1); - px = pmadd(px, x2, p4d_cephes_exp_p2); - px = pmul(px, x); - - // Evaluate the denominator polynomial of the rational interpolant. - Packet4d qx = p4d_cephes_exp_q0; - qx = pmadd(qx, x2, p4d_cephes_exp_q1); - qx = pmadd(qx, x2, p4d_cephes_exp_q2); - qx = pmadd(qx, x2, p4d_cephes_exp_q3); - - // I don't really get this bit, copied from the SSE2 routines, so... - // TODO(gonnet): Figure out what is going on here, perhaps find a better - // rational interpolant? - x = _mm256_div_pd(px, psub(qx, px)); - x = pmadd(p4d_2, x, p4d_1); - - // Build e=2^n by constructing the exponents in a 128-bit vector and - // shifting them to where they belong in double-precision values. - __m128i emm0 = _mm256_cvtpd_epi32(fx); - emm0 = _mm_add_epi32(emm0, p4i_1023); - emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(3, 1, 2, 0)); - __m128i lo = _mm_slli_epi64(emm0, 52); - __m128i hi = _mm_slli_epi64(_mm_srli_epi64(emm0, 32), 52); - __m256i e = _mm256_insertf128_si256(_mm256_setzero_si256(), lo, 0); - e = _mm256_insertf128_si256(e, hi, 1); - - // Construct the result 2^n * exp(g) = e * x. The max is used to catch - // non-finite values in the input. - return pmax(pmul(x, _mm256_castsi256_pd(e)), _x); +pexp<Packet4d>(const Packet4d& x) { + return pexp_double(x); } // Functions for sqrt. // The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step // of Newton's method, at a cost of 1-2 bits of precision as opposed to the -// exact solution. The main advantage of this approach is not just speed, but -// also the fact that it can be inlined and pipelined with other computations, -// further reducing its effective latency. +// exact solution. It does not handle +inf, or denormalized numbers correctly. +// The main advantage of this approach is not just speed, but also the fact that +// it can be inlined and pipelined with other computations, further reducing its +// effective latency. This is similar to Quake3's fast inverse square root. +// For detail see here: http://www.beyond3d.com/content/articles/8/ #if EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psqrt<Packet8f>(const Packet8f& _x) { - _EIGEN_DECLARE_CONST_Packet8f(one_point_five, 1.5f); - _EIGEN_DECLARE_CONST_Packet8f(minus_half, -0.5f); - _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(flt_min, 0x00800000); + Packet8f half = pmul(_x, pset1<Packet8f>(.5f)); + Packet8f denormal_mask = _mm256_and_ps( + _mm256_cmp_ps(_x, pset1<Packet8f>((std::numeric_limits<float>::min)()), + _CMP_LT_OQ), + _mm256_cmp_ps(_x, _mm256_setzero_ps(), _CMP_GE_OQ)); - Packet8f neg_half = pmul(_x, p8f_minus_half); - - // select only the inverse sqrt of positive normal inputs (denormals are - // flushed to zero and cause infs as well). - Packet8f non_zero_mask = _mm256_cmp_ps(_x, p8f_flt_min, _CMP_GE_OQ); - Packet8f x = _mm256_and_ps(non_zero_mask, _mm256_rsqrt_ps(_x)); - + // Compute approximate reciprocal sqrt. + Packet8f x = _mm256_rsqrt_ps(_x); // Do a single step of Newton's iteration. - x = pmul(x, pmadd(neg_half, pmul(x, x), p8f_one_point_five)); - - // Multiply the original _x by it's reciprocal square root to extract the - // square root. - return pmul(_x, x); + x = pmul(x, psub(pset1<Packet8f>(1.5f), pmul(half, pmul(x,x)))); + // Flush results for denormals to zero. + return _mm256_andnot_ps(denormal_mask, pmul(_x,x)); } #else template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index 4fec14f..ee00f1f 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -18,11 +18,11 @@ #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif -#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS -#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*)) +#if !defined(EIGEN_VECTORIZE_AVX512) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 #endif -#ifdef __FMA__ +#ifdef EIGEN_VECTORIZE_FMA #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif @@ -48,7 +48,9 @@ #define _EIGEN_DECLARE_CONST_Packet8i(NAME,X) \ const Packet8i p8i_##NAME = pset1<Packet8i>(X) - +// Use the packet_traits defined in AVX512/PacketMath.h instead if we're going +// to leverage AVX512 instructions. +#ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits<float> : default_packet_traits { typedef Packet8f type; @@ -61,7 +63,7 @@ HasDiv = 1, HasSin = EIGEN_FAST_MATH, - HasCos = 0, + HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, HasSqrt = 1, @@ -93,6 +95,10 @@ HasCeil = 1 }; }; +#endif + +template<> struct scalar_div_cost<float,true> { enum { value = 14 }; }; +template<> struct scalar_div_cost<double,true> { enum { value = 16 }; }; /* Proper support for integers is only provided by AVX2. In the meantime, we'll use SSE instructions and packets to deal with integers. @@ -107,14 +113,29 @@ }; */ -template<> struct unpacket_traits<Packet8f> { typedef float type; typedef Packet4f half; enum {size=8, alignment=Aligned32}; }; -template<> struct unpacket_traits<Packet4d> { typedef double type; typedef Packet2d half; enum {size=4, alignment=Aligned32}; }; -template<> struct unpacket_traits<Packet8i> { typedef int type; typedef Packet4i half; enum {size=8, alignment=Aligned32}; }; +template<> struct unpacket_traits<Packet8f> { + typedef float type; + typedef Packet4f half; + typedef Packet8i integer_packet; + enum {size=8, alignment=Aligned32, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet4d> { + typedef double type; + typedef Packet2d half; + enum {size=4, alignment=Aligned32, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet8i> { typedef int type; typedef Packet4i half; enum {size=8, alignment=Aligned32, vectorizable=false}; }; template<> EIGEN_STRONG_INLINE Packet8f pset1<Packet8f>(const float& from) { return _mm256_set1_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pset1<Packet4d>(const double& from) { return _mm256_set1_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i pset1<Packet8i>(const int& from) { return _mm256_set1_epi32(from); } +template<> EIGEN_STRONG_INLINE Packet8f pset1frombits<Packet8f>(unsigned int from) { return _mm256_castsi256_ps(pset1<Packet8i>(from)); } + +template<> EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f& /*a*/) { return _mm256_setzero_ps(); } +template<> EIGEN_STRONG_INLINE Packet4d pzero(const Packet4d& /*a*/) { return _mm256_setzero_pd(); } +template<> EIGEN_STRONG_INLINE Packet8i pzero(const Packet8i& /*a*/) { return _mm256_setzero_si256(); } + template<> EIGEN_STRONG_INLINE Packet8f pload1<Packet8f>(const float* from) { return _mm256_broadcast_ss(from); } template<> EIGEN_STRONG_INLINE Packet4d pload1<Packet4d>(const double* from) { return _mm256_broadcast_sd(from); } @@ -123,6 +144,15 @@ template<> EIGEN_STRONG_INLINE Packet8f padd<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_add_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d padd<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_add_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8i padd<Packet8i>(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_add_epi32(a,b); +#else + __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f psub<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_sub_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d psub<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_sub_pd(a,b); } @@ -151,13 +181,14 @@ return pset1<Packet8i>(0); } -#ifdef __FMA__ +#ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE Packet8f pmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) { -#if EIGEN_COMP_GNUC || EIGEN_COMP_CLANG - // clang stupidly generates a vfmadd213ps instruction plus some vmovaps on registers, - // and gcc stupidly generates a vfmadd132ps instruction, - // so let's enforce it to generate a vfmadd231ps instruction since the most common use case is to accumulate - // the result of the product. +#if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) + // Clang stupidly generates a vfmadd213ps instruction plus some vmovaps on registers, + // and even register spilling with clang>=6.0 (bug 1637). + // Gcc stupidly generates a vfmadd132ps instruction. + // So let's enforce it to generate a vfmadd231ps instruction since the most common use + // case is to accumulate the result of the product. Packet8f res = c; __asm__("vfmadd231ps %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); return res; @@ -166,7 +197,7 @@ #endif } template<> EIGEN_STRONG_INLINE Packet4d pmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) { -#if EIGEN_COMP_GNUC || EIGEN_COMP_CLANG +#if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) // see above Packet4d res = c; __asm__("vfmadd231pd %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); @@ -177,11 +208,38 @@ } #endif -template<> EIGEN_STRONG_INLINE Packet8f pmin<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_min_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet4d pmin<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_min_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8f pmin<Packet8f>(const Packet8f& a, const Packet8f& b) { + // Arguments are swapped to match NaN propagation behavior of std::min. + return _mm256_min_ps(b,a); +} +template<> EIGEN_STRONG_INLINE Packet4d pmin<Packet4d>(const Packet4d& a, const Packet4d& b) { + // Arguments are swapped to match NaN propagation behavior of std::min. + return _mm256_min_pd(b,a); +} +template<> EIGEN_STRONG_INLINE Packet8f pmax<Packet8f>(const Packet8f& a, const Packet8f& b) { + // Arguments are swapped to match NaN propagation behavior of std::max. + return _mm256_max_ps(b,a); +} +template<> EIGEN_STRONG_INLINE Packet4d pmax<Packet4d>(const Packet4d& a, const Packet4d& b) { + // Arguments are swapped to match NaN propagation behavior of std::max. + return _mm256_max_pd(b,a); +} -template<> EIGEN_STRONG_INLINE Packet8f pmax<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_max_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet4d pmax<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_max_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LE_OQ); } +template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LT_OQ); } +template<> EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_EQ_OQ); } +template<> EIGEN_STRONG_INLINE Packet4d pcmp_eq(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_EQ_OQ); } +template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a, b, _CMP_NGE_UQ); } + +template<> EIGEN_STRONG_INLINE Packet8i pcmp_eq(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_cmpeq_epi32(a,b); +#else + __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); + __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pround<Packet8f>(const Packet8f& a) { return _mm256_round_ps(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet4d pround<Packet4d>(const Packet4d& a) { return _mm256_round_pd(a, _MM_FROUND_CUR_DIRECTION); } @@ -192,17 +250,101 @@ template<> EIGEN_STRONG_INLINE Packet8f pfloor<Packet8f>(const Packet8f& a) { return _mm256_floor_ps(a); } template<> EIGEN_STRONG_INLINE Packet4d pfloor<Packet4d>(const Packet4d& a) { return _mm256_floor_pd(a); } + +template<> EIGEN_STRONG_INLINE Packet8i ptrue<Packet8i>(const Packet8i& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + // vpcmpeqd has lower latency than the more general vcmpps + return _mm256_cmpeq_epi32(a,a); +#else + const __m256 b = _mm256_castsi256_ps(a); + return _mm256_castps_si256(_mm256_cmp_ps(b,b,_CMP_TRUE_UQ)); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet8f ptrue<Packet8f>(const Packet8f& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + // vpcmpeqd has lower latency than the more general vcmpps + const __m256i b = _mm256_castps_si256(a); + return _mm256_castsi256_ps(_mm256_cmpeq_epi32(b,b)); +#else + return _mm256_cmp_ps(a,a,_CMP_TRUE_UQ); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet4d ptrue<Packet4d>(const Packet4d& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + // vpcmpeqq has lower latency than the more general vcmppd + const __m256i b = _mm256_castpd_si256(a); + return _mm256_castsi256_pd(_mm256_cmpeq_epi64(b,b)); +#else + return _mm256_cmp_pd(a,a,_CMP_TRUE_UQ); +#endif +} + template<> EIGEN_STRONG_INLINE Packet8f pand<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_and_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pand<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_and_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8i pand<Packet8i>(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_and_si256(a,b); +#else + return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f por<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_or_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d por<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_or_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8i por<Packet8i>(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_or_si256(a,b); +#else + return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pxor<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_xor_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pxor<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_xor_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8i pxor<Packet8i>(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_xor_si256(a,b); +#else + return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); +#endif +} -template<> EIGEN_STRONG_INLINE Packet8f pandnot<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_andnot_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet4d pandnot<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_andnot_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet8f pandnot<Packet8f>(const Packet8f& a, const Packet8f& b) { return _mm256_andnot_ps(b,a); } +template<> EIGEN_STRONG_INLINE Packet4d pandnot<Packet4d>(const Packet4d& a, const Packet4d& b) { return _mm256_andnot_pd(b,a); } +template<> EIGEN_STRONG_INLINE Packet8i pandnot<Packet8i>(const Packet8i& a, const Packet8i& b) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_andnot_si256(b,a); +#else + return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b),_mm256_castsi256_ps(a))); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet8f pselect<Packet8f>(const Packet8f& mask, const Packet8f& a, const Packet8f& b) +{ return _mm256_blendv_ps(b,a,mask); } +template<> EIGEN_STRONG_INLINE Packet4d pselect<Packet4d>(const Packet4d& mask, const Packet4d& a, const Packet4d& b) +{ return _mm256_blendv_pd(b,a,mask); } + +template<int N> EIGEN_STRONG_INLINE Packet8i pshiftright(Packet8i a) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_srli_epi32(a, N); +#else + __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(a, 0), N); + __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(a, 1), N); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} + +template<int N> EIGEN_STRONG_INLINE Packet8i pshiftleft(Packet8i a) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_slli_epi32(a, N); +#else + __m128i lo = _mm_slli_epi32(_mm256_extractf128_si256(a, 0), N); + __m128i hi = _mm_slli_epi32(_mm256_extractf128_si256(a, 1), N); + return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); +#endif +} template<> EIGEN_STRONG_INLINE Packet8f pload<Packet8f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pload<Packet4d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_pd(from); } @@ -219,7 +361,7 @@ // Packet8f tmp = _mm256_castps128_ps256(_mm_loadu_ps(from)); // tmp = _mm256_insertf128_ps(tmp, _mm_movehl_ps(_mm256_castps256_ps128(tmp),_mm256_castps256_ps128(tmp)), 1); // return _mm256_unpacklo_ps(tmp,tmp); - + // _mm256_insertf128_ps is very slow on Haswell, thus: Packet8f tmp = _mm256_broadcast_ps((const __m128*)(const void*)from); // mimic an "inplace" permutation of the lower 128bits using a blend @@ -301,9 +443,11 @@ pstore(to, pa); } -template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } -template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } -template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } +#ifndef EIGEN_VECTORIZE_AVX512 +template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +#endif template<> EIGEN_STRONG_INLINE float pfirst<Packet8f>(const Packet8f& a) { return _mm_cvtss_f32(_mm256_castps256_ps128(a)); @@ -325,9 +469,12 @@ { __m256d tmp = _mm256_shuffle_pd(a,a,5); return _mm256_permute2f128_pd(tmp, tmp, 1); - + #if 0 + // This version is unlikely to be faster as _mm256_shuffle_ps and _mm256_permute_pd + // exhibit the same latency/throughput, but it is here for future reference/benchmarking... __m256d swap_halves = _mm256_permute2f128_pd(a,a,1); return _mm256_permute_pd(swap_halves,5); + #endif } // pabs should be ok @@ -342,6 +489,28 @@ return _mm256_and_pd(a,mask); } +template<> EIGEN_STRONG_INLINE Packet8f pfrexp<Packet8f>(const Packet8f& a, Packet8f& exponent) { + return pfrexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet8f pldexp<Packet8f>(const Packet8f& a, const Packet8f& exponent) { + return pldexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet4d pldexp<Packet4d>(const Packet4d& a, const Packet4d& exponent) { + // Build e=2^n by constructing the exponents in a 128-bit vector and + // shifting them to where they belong in double-precision values. + Packet4i cst_1023 = pset1<Packet4i>(1023); + __m128i emm0 = _mm256_cvtpd_epi32(exponent); + emm0 = _mm_add_epi32(emm0, cst_1023); + emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(3, 1, 2, 0)); + __m128i lo = _mm_slli_epi64(emm0, 52); + __m128i hi = _mm_slli_epi64(_mm_srli_epi64(emm0, 32), 52); + __m256i e = _mm256_insertf128_si256(_mm256_setzero_si256(), lo, 0); + e = _mm256_insertf128_si256(e, hi, 1); + return pmul(a,_mm256_castsi256_pd(e)); +} + // preduxp should be ok // FIXME: why is this ok? why isn't the simply implementation working as expected? template<> EIGEN_STRONG_INLINE Packet8f preduxp<Packet8f>(const Packet8f* vecs) @@ -387,17 +556,14 @@ template<> EIGEN_STRONG_INLINE float predux<Packet8f>(const Packet8f& a) { - Packet8f tmp0 = _mm256_hadd_ps(a,_mm256_permute2f128_ps(a,a,1)); - tmp0 = _mm256_hadd_ps(tmp0,tmp0); - return pfirst(_mm256_hadd_ps(tmp0, tmp0)); + return predux(Packet4f(_mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,1)))); } template<> EIGEN_STRONG_INLINE double predux<Packet4d>(const Packet4d& a) { - Packet4d tmp0 = _mm256_hadd_pd(a,_mm256_permute2f128_pd(a,a,1)); - return pfirst(_mm256_hadd_pd(tmp0,tmp0)); + return predux(Packet2d(_mm_add_pd(_mm256_castpd256_pd128(a),_mm256_extractf128_pd(a,1)))); } -template<> EIGEN_STRONG_INLINE Packet4f predux4<Packet8f>(const Packet8f& a) +template<> EIGEN_STRONG_INLINE Packet4f predux_half_dowto4<Packet8f>(const Packet8f& a) { return _mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,1)); } @@ -441,6 +607,16 @@ return pfirst(_mm256_max_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1))); } +// not needed yet +// template<> EIGEN_STRONG_INLINE bool predux_all(const Packet8f& x) +// { +// return _mm256_movemask_ps(x)==0xFF; +// } + +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet8f& x) +{ + return _mm256_movemask_ps(x)!=0; +} template<int Offset> struct palign_impl<Offset,Packet8f> @@ -601,6 +777,26 @@ return _mm256_blendv_pd(thenPacket, elsePacket, false_mask); } +template<> EIGEN_STRONG_INLINE Packet8f pinsertfirst(const Packet8f& a, float b) +{ + return _mm256_blend_ps(a,pset1<Packet8f>(b),1); +} + +template<> EIGEN_STRONG_INLINE Packet4d pinsertfirst(const Packet4d& a, double b) +{ + return _mm256_blend_pd(a,pset1<Packet4d>(b),1); +} + +template<> EIGEN_STRONG_INLINE Packet8f pinsertlast(const Packet8f& a, float b) +{ + return _mm256_blend_ps(a,pset1<Packet8f>(b),(1<<7)); +} + +template<> EIGEN_STRONG_INLINE Packet4d pinsertlast(const Packet4d& a, double b) +{ + return _mm256_blend_pd(a,pset1<Packet4d>(b),(1<<3)); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX/TypeCasting.h b/Eigen/src/Core/arch/AVX/TypeCasting.h index 83bfdc6..7d2e1e6 100644 --- a/Eigen/src/Core/arch/AVX/TypeCasting.h +++ b/Eigen/src/Core/arch/AVX/TypeCasting.h
@@ -37,13 +37,21 @@ template<> EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(const Packet8f& a) { - return _mm256_cvtps_epi32(a); + return _mm256_cvttps_epi32(a); } template<> EIGEN_STRONG_INLINE Packet8f pcast<Packet8i, Packet8f>(const Packet8i& a) { return _mm256_cvtepi32_ps(a); } +template<> EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i,Packet8f>(const Packet8f& a) { + return _mm256_castps_si256(a); +} + +template<> EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f,Packet8i>(const Packet8i& a) { + return _mm256_castsi256_ps(a); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/Complex.h b/Eigen/src/Core/arch/AVX512/Complex.h new file mode 100644 index 0000000..9a89dd0 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/Complex.h
@@ -0,0 +1,488 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_COMPLEX_AVX512_H +#define EIGEN_COMPLEX_AVX512_H + +namespace Eigen { + +namespace internal { + +//---------- float ---------- +struct Packet8cf +{ + EIGEN_STRONG_INLINE Packet8cf() {} + EIGEN_STRONG_INLINE explicit Packet8cf(const __m512& a) : v(a) {} + __m512 v; +}; + +template<> struct packet_traits<std::complex<float> > : default_packet_traits +{ + typedef Packet8cf type; + typedef Packet4cf half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + HasHalfPacket = 1, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0, + HasReduxp = 0 + }; +}; + +template<> struct unpacket_traits<Packet8cf> { + typedef std::complex<float> type; + enum { + size = 8, + alignment=unpacket_traits<Packet16f>::alignment, + vectorizable=true + }; + typedef Packet4cf half; +}; + +template<> EIGEN_STRONG_INLINE Packet8cf ptrue<Packet8cf>(const Packet8cf& a) { return Packet8cf(ptrue(Packet16f(a.v))); } +template<> EIGEN_STRONG_INLINE Packet8cf pnot<Packet8cf>(const Packet8cf& a) { return Packet8cf(pnot(Packet16f(a.v))); } +template<> EIGEN_STRONG_INLINE Packet8cf padd<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_add_ps(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet8cf psub<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_sub_ps(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet8cf pnegate(const Packet8cf& a) +{ + return Packet8cf(pnegate(a.v)); +} +template<> EIGEN_STRONG_INLINE Packet8cf pconj(const Packet8cf& a) +{ + const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32( + 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000, + 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000)); + return Packet8cf(pxor(a.v,mask)); +} + +template<> EIGEN_STRONG_INLINE Packet8cf pmul<Packet8cf>(const Packet8cf& a, const Packet8cf& b) +{ + __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1))); + return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2)); +} + +template<> EIGEN_STRONG_INLINE Packet8cf pand <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pand(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet8cf por <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(por(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet8cf pxor <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pxor(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet8cf pandnot<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pandnot(a.v,b.v)); } + +template <> +EIGEN_STRONG_INLINE Packet8cf pcmp_eq(const Packet8cf& a, const Packet8cf& b) { + __m512 eq = pcmp_eq<Packet16f>(a.v, b.v); + return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1))); +} + +template<> EIGEN_STRONG_INLINE Packet8cf pload <Packet8cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload<Packet16f>(&numext::real_ref(*from))); } +template<> EIGEN_STRONG_INLINE Packet8cf ploadu<Packet8cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from))); } + + +template<> EIGEN_STRONG_INLINE Packet8cf pset1<Packet8cf>(const std::complex<float>& from) +{ + return Packet8cf(_mm512_castpd_ps(pload1<Packet8d>((const double*)(const void*)&from))); +} + +template<> EIGEN_STRONG_INLINE Packet8cf ploaddup<Packet8cf>(const std::complex<float>* from) +{ + return Packet8cf( _mm512_castpd_ps( ploaddup<Packet8d>((const double*)(const void*)from )) ); +} +template<> EIGEN_STRONG_INLINE Packet8cf ploadquad<Packet8cf>(const std::complex<float>* from) +{ + return Packet8cf( _mm512_castpd_ps( ploadquad<Packet8d>((const double*)(const void*)from )) ); +} + +template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet8cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } + +template<> EIGEN_DEVICE_FUNC inline Packet8cf pgather<std::complex<float>, Packet8cf>(const std::complex<float>* from, Index stride) +{ + return Packet8cf(_mm512_castpd_ps(pgather<double,Packet8d>((const double*)(const void*)from, stride))); +} + +template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from, Index stride) +{ + pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet8cf>(const Packet8cf& a) +{ + return pfirst(Packet2cf(_mm512_castps512_ps128(a.v))); +} + +template<> EIGEN_STRONG_INLINE Packet8cf preverse(const Packet8cf& a) { + return Packet8cf(_mm512_castsi512_ps( + _mm512_permutexvar_epi64( _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), + _mm512_castps_si512(a.v)))); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet8cf>(const Packet8cf& a) +{ + return predux(padd(Packet4cf(extract256<0>(a.v)), + Packet4cf(extract256<1>(a.v)))); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet8cf>(const Packet8cf& a) +{ + return predux_mul(pmul(Packet4cf(extract256<0>(a.v)), + Packet4cf(extract256<1>(a.v)))); +} + +template <> +EIGEN_STRONG_INLINE Packet4cf predux_half_dowto4<Packet8cf>(const Packet8cf& a) { + __m256 lane0 = extract256<0>(a.v); + __m256 lane1 = extract256<1>(a.v); + __m256 res = _mm256_add_ps(lane0, lane1); + return Packet4cf(res); +} + +template<int Offset> +struct palign_impl<Offset,Packet8cf> +{ + static EIGEN_STRONG_INLINE void run(Packet8cf& first, const Packet8cf& second) + { + if (Offset==0) return; + palign_impl<Offset*2,Packet16f>::run(first.v, second.v); + } +}; + +template<> struct conj_helper<Packet8cf, Packet8cf, false,true> +{ + EIGEN_STRONG_INLINE Packet8cf pmadd(const Packet8cf& x, const Packet8cf& y, const Packet8cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet8cf pmul(const Packet8cf& a, const Packet8cf& b) const + { + return internal::pmul(a, pconj(b)); + } +}; + +template<> struct conj_helper<Packet8cf, Packet8cf, true,false> +{ + EIGEN_STRONG_INLINE Packet8cf pmadd(const Packet8cf& x, const Packet8cf& y, const Packet8cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet8cf pmul(const Packet8cf& a, const Packet8cf& b) const + { + return internal::pmul(pconj(a), b); + } +}; + +template<> struct conj_helper<Packet8cf, Packet8cf, true,true> +{ + EIGEN_STRONG_INLINE Packet8cf pmadd(const Packet8cf& x, const Packet8cf& y, const Packet8cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet8cf pmul(const Packet8cf& a, const Packet8cf& b) const + { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf,Packet16f) + +template<> EIGEN_STRONG_INLINE Packet8cf pdiv<Packet8cf>(const Packet8cf& a, const Packet8cf& b) +{ + Packet8cf num = pmul(a, pconj(b)); + __m512 tmp = _mm512_mul_ps(b.v, b.v); + __m512 tmp2 = _mm512_shuffle_ps(tmp,tmp,0xB1); + __m512 denom = _mm512_add_ps(tmp, tmp2); + return Packet8cf(_mm512_div_ps(num.v, denom)); +} + +template<> EIGEN_STRONG_INLINE Packet8cf pcplxflip<Packet8cf>(const Packet8cf& x) +{ + return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1))); +} + +//---------- double ---------- +struct Packet4cd +{ + EIGEN_STRONG_INLINE Packet4cd() {} + EIGEN_STRONG_INLINE explicit Packet4cd(const __m512d& a) : v(a) {} + __m512d v; +}; + +template<> struct packet_traits<std::complex<double> > : default_packet_traits +{ + typedef Packet4cd type; + typedef Packet2cd half; + enum { + Vectorizable = 1, + AlignedOnScalar = 0, + size = 4, + HasHalfPacket = 1, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0, + HasReduxp = 0 + }; +}; + +template<> struct unpacket_traits<Packet4cd> { + typedef std::complex<double> type; + enum { + size = 4, + alignment = unpacket_traits<Packet8d>::alignment, + vectorizable=true + }; + typedef Packet2cd half; +}; + +template<> EIGEN_STRONG_INLINE Packet4cd padd<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_add_pd(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd psub<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_sub_pd(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd pnegate(const Packet4cd& a) { return Packet4cd(pnegate(a.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd pconj(const Packet4cd& a) +{ + const __m512d mask = _mm512_castsi512_pd( + _mm512_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0, + 0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0)); + return Packet4cd(pxor(a.v,mask)); +} + +template<> EIGEN_STRONG_INLINE Packet4cd pmul<Packet4cd>(const Packet4cd& a, const Packet4cd& b) +{ + __m512d tmp1 = _mm512_shuffle_pd(a.v,a.v,0x0); + __m512d tmp2 = _mm512_shuffle_pd(a.v,a.v,0xFF); + __m512d tmp3 = _mm512_shuffle_pd(b.v,b.v,0x55); + __m512d odd = _mm512_mul_pd(tmp2, tmp3); + return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd)); +} + +template<> EIGEN_STRONG_INLINE Packet4cd ptrue<Packet4cd>(const Packet4cd& a) { return Packet4cd(ptrue(Packet8d(a.v))); } +template<> EIGEN_STRONG_INLINE Packet4cd pnot<Packet4cd>(const Packet4cd& a) { return Packet4cd(pnot(Packet8d(a.v))); } +template<> EIGEN_STRONG_INLINE Packet4cd pand <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pand(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd por <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(por(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd pxor <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pxor(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet4cd pandnot<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pandnot(a.v,b.v)); } + +template <> +EIGEN_STRONG_INLINE Packet4cd pcmp_eq(const Packet4cd& a, const Packet4cd& b) { + __m512d eq = pcmp_eq<Packet8d>(a.v, b.v); + return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55))); +} + +template<> EIGEN_STRONG_INLINE Packet4cd pload <Packet4cd>(const std::complex<double>* from) +{ EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload<Packet8d>((const double*)from)); } +template<> EIGEN_STRONG_INLINE Packet4cd ploadu<Packet4cd>(const std::complex<double>* from) +{ EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu<Packet8d>((const double*)from)); } + +template<> EIGEN_STRONG_INLINE Packet4cd pset1<Packet4cd>(const std::complex<double>& from) +{ + #ifdef EIGEN_VECTORIZE_AVX512DQ + return Packet4cd(_mm512_broadcast_f64x2(pset1<Packet1cd>(from).v)); + #else + return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4( _mm_castpd_ps(pset1<Packet1cd>(from).v)))); + #endif +} + +template<> EIGEN_STRONG_INLINE Packet4cd ploaddup<Packet4cd>(const std::complex<double>* from) { + return Packet4cd(_mm512_insertf64x4( + _mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from+1).v, 1)); +} + +template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet4cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet4cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } + +template<> EIGEN_DEVICE_FUNC inline Packet4cd pgather<std::complex<double>, Packet4cd>(const std::complex<double>* from, Index stride) +{ + return Packet4cd(_mm512_insertf64x4(_mm512_castpd256_pd512( + _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+0*stride).v), ploadu<Packet1cd>(from+1*stride).v,1)), + _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+2*stride).v), ploadu<Packet1cd>(from+3*stride).v,1), 1)); +} + +template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from, Index stride) +{ + __m512i fromi = _mm512_castpd_si512(from.v); + double* tod = (double*)(void*)to; + _mm_storeu_pd(tod+0*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,0)) ); + _mm_storeu_pd(tod+2*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,1)) ); + _mm_storeu_pd(tod+4*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,2)) ); + _mm_storeu_pd(tod+6*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,3)) ); +} + +template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet4cd>(const Packet4cd& a) +{ + __m128d low = extract128<0>(a.v); + EIGEN_ALIGN16 double res[2]; + _mm_store_pd(res, low); + return std::complex<double>(res[0],res[1]); +} + +template<> EIGEN_STRONG_INLINE Packet4cd preverse(const Packet4cd& a) { + return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, EIGEN_SSE_SHUFFLE_MASK(3,2,1,0))); +} + +template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet4cd>(const Packet4cd& a) +{ + return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v,0)), + Packet2cd(_mm512_extractf64x4_pd(a.v,1)))); +} + +template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet4cd>(const Packet4cd& a) +{ + return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v,0)), + Packet2cd(_mm512_extractf64x4_pd(a.v,1)))); +} + +template<int Offset> +struct palign_impl<Offset,Packet4cd> +{ + static EIGEN_STRONG_INLINE void run(Packet4cd& first, const Packet4cd& second) + { + if (Offset==0) return; + palign_impl<Offset*2,Packet8d>::run(first.v, second.v); + } +}; + +template<> struct conj_helper<Packet4cd, Packet4cd, false,true> +{ + EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const + { + return internal::pmul(a, pconj(b)); + } +}; + +template<> struct conj_helper<Packet4cd, Packet4cd, true,false> +{ + EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const + { + return internal::pmul(pconj(a), b); + } +}; + +template<> struct conj_helper<Packet4cd, Packet4cd, true,true> +{ + EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const + { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd,Packet8d) + +template<> EIGEN_STRONG_INLINE Packet4cd pdiv<Packet4cd>(const Packet4cd& a, const Packet4cd& b) +{ + Packet4cd num = pmul(a, pconj(b)); + __m512d tmp = _mm512_mul_pd(b.v, b.v); + __m512d denom = padd(_mm512_permute_pd(tmp,0x55), tmp); + return Packet4cd(_mm512_div_pd(num.v, denom)); +} + +template<> EIGEN_STRONG_INLINE Packet4cd pcplxflip<Packet4cd>(const Packet4cd& x) +{ + return Packet4cd(_mm512_permute_pd(x.v,0x55)); +} + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock<Packet8cf,4>& kernel) { + PacketBlock<Packet8d,4> pb; + + pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v); + pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v); + pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v); + pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v); + ptranspose(pb); + kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]); + kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]); + kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]); + kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]); +} + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock<Packet8cf,8>& kernel) { + PacketBlock<Packet8d,8> pb; + + pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v); + pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v); + pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v); + pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v); + pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v); + pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v); + pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v); + pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v); + ptranspose(pb); + kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]); + kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]); + kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]); + kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]); + kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]); + kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]); + kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]); + kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]); +} + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock<Packet4cd,4>& kernel) { + __m512d T0 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, EIGEN_SSE_SHUFFLE_MASK(0,1,0,1)); // [a0 a1 b0 b1] + __m512d T1 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, EIGEN_SSE_SHUFFLE_MASK(2,3,2,3)); // [a2 a3 b2 b3] + __m512d T2 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, EIGEN_SSE_SHUFFLE_MASK(0,1,0,1)); // [c0 c1 d0 d1] + __m512d T3 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, EIGEN_SSE_SHUFFLE_MASK(2,3,2,3)); // [c2 c3 d2 d3] + + kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, EIGEN_SSE_SHUFFLE_MASK(1,3,1,3))); // [a3 b3 c3 d3] + kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, EIGEN_SSE_SHUFFLE_MASK(0,2,0,2))); // [a2 b2 c2 d2] + kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, EIGEN_SSE_SHUFFLE_MASK(1,3,1,3))); // [a1 b1 c1 d1] + kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, EIGEN_SSE_SHUFFLE_MASK(0,2,0,2))); // [a0 b0 c0 d0] +} + +template<> EIGEN_STRONG_INLINE Packet8cf pinsertfirst(const Packet8cf& a, std::complex<float> b) +{ + Packet2cf tmp = Packet2cf(_mm512_extractf32x4_ps(a.v,0)); + tmp = pinsertfirst(tmp, b); + return Packet8cf( _mm512_insertf32x4(a.v, tmp.v, 0) ); +} + +template<> EIGEN_STRONG_INLINE Packet4cd pinsertfirst(const Packet4cd& a, std::complex<double> b) +{ + return Packet4cd(_mm512_castsi512_pd( _mm512_inserti32x4(_mm512_castpd_si512(a.v), _mm_castpd_si128(pset1<Packet1cd>(b).v), 0) )); +} + +template<> EIGEN_STRONG_INLINE Packet8cf pinsertlast(const Packet8cf& a, std::complex<float> b) +{ + Packet2cf tmp = Packet2cf(_mm512_extractf32x4_ps(a.v,3) ); + tmp = pinsertlast(tmp, b); + return Packet8cf( _mm512_insertf32x4(a.v, tmp.v, 3) ); +} + +template<> EIGEN_STRONG_INLINE Packet4cd pinsertlast(const Packet4cd& a, std::complex<double> b) +{ + return Packet4cd(_mm512_castsi512_pd( _mm512_inserti32x4(_mm512_castpd_si512(a.v), _mm_castpd_si128(pset1<Packet1cd>(b).v), 3) )); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_COMPLEX_AVX512_H
diff --git a/Eigen/src/Core/arch/AVX512/MathFunctions.h b/Eigen/src/Core/arch/AVX512/MathFunctions.h new file mode 100644 index 0000000..c2158c5 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/MathFunctions.h
@@ -0,0 +1,400 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Pedro Gonnet (pedro.gonnet@gmail.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ +#define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ + +namespace Eigen { + +namespace internal { + +// Disable the code for older versions of gcc that don't support many of the required avx512 instrinsics. +#if EIGEN_GNUC_AT_LEAST(5, 3) || EIGEN_COMP_CLANG + +#define _EIGEN_DECLARE_CONST_Packet16f(NAME, X) \ + const Packet16f p16f_##NAME = pset1<Packet16f>(X) + +#define _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(NAME, X) \ + const Packet16f p16f_##NAME = (__m512)pset1<Packet16i>(X) + +#define _EIGEN_DECLARE_CONST_Packet8d(NAME, X) \ + const Packet8d p8d_##NAME = pset1<Packet8d>(X) + +#define _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(NAME, X) \ + const Packet8d p8d_##NAME = _mm512_castsi512_pd(_mm512_set1_epi64(X)) + +// Natural logarithm +// Computes log(x) as log(2^e * m) = C*e + log(m), where the constant C =log(2) +// and m is in the range [sqrt(1/2),sqrt(2)). In this range, the logarithm can +// be easily approximated by a polynomial centered on m=1 for stability. +#if defined(EIGEN_VECTORIZE_AVX512DQ) +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +plog<Packet16f>(const Packet16f& _x) { + Packet16f x = _x; + _EIGEN_DECLARE_CONST_Packet16f(1, 1.0f); + _EIGEN_DECLARE_CONST_Packet16f(half, 0.5f); + _EIGEN_DECLARE_CONST_Packet16f(126f, 126.0f); + + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(inv_mant_mask, ~0x7f800000); + + // The smallest non denormalized float number. + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(min_norm_pos, 0x00800000); + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(minus_inf, 0xff800000); + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(pos_inf, 0x7f800000); + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(nan, 0x7fc00000); + + // Polynomial coefficients. + _EIGEN_DECLARE_CONST_Packet16f(cephes_SQRTHF, 0.707106781186547524f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p0, 7.0376836292E-2f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p1, -1.1514610310E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p2, 1.1676998740E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p3, -1.2420140846E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p4, +1.4249322787E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p5, -1.6668057665E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p6, +2.0000714765E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p7, -2.4999993993E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_p8, +3.3333331174E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_q1, -2.12194440e-4f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_log_q2, 0.693359375f); + + // invalid_mask is set to true when x is NaN + __mmask16 invalid_mask = _mm512_cmp_ps_mask(x, _mm512_setzero_ps(), _CMP_NGE_UQ); + __mmask16 iszero_mask = _mm512_cmp_ps_mask(x, _mm512_setzero_ps(), _CMP_EQ_OQ); + + // Truncate input values to the minimum positive normal. + x = pmax(x, p16f_min_norm_pos); + + // Extract the shifted exponents. + Packet16f emm0 = _mm512_cvtepi32_ps(_mm512_srli_epi32((__m512i)x, 23)); + Packet16f e = _mm512_sub_ps(emm0, p16f_126f); + + // Set the exponents to -1, i.e. x are in the range [0.5,1). + x = _mm512_and_ps(x, p16f_inv_mant_mask); + x = _mm512_or_ps(x, p16f_half); + + // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) + // and shift by -1. The values are then centered around 0, which improves + // the stability of the polynomial evaluation. + // if( x < SQRTHF ) { + // e -= 1; + // x = x + x - 1.0; + // } else { x = x - 1.0; } + __mmask16 mask = _mm512_cmp_ps_mask(x, p16f_cephes_SQRTHF, _CMP_LT_OQ); + Packet16f tmp = _mm512_mask_blend_ps(mask, _mm512_setzero_ps(), x); + x = psub(x, p16f_1); + e = psub(e, _mm512_mask_blend_ps(mask, _mm512_setzero_ps(), p16f_1)); + x = padd(x, tmp); + + Packet16f x2 = pmul(x, x); + Packet16f x3 = pmul(x2, x); + + // Evaluate the polynomial approximant of degree 8 in three parts, probably + // to improve instruction-level parallelism. + Packet16f y, y1, y2; + y = pmadd(p16f_cephes_log_p0, x, p16f_cephes_log_p1); + y1 = pmadd(p16f_cephes_log_p3, x, p16f_cephes_log_p4); + y2 = pmadd(p16f_cephes_log_p6, x, p16f_cephes_log_p7); + y = pmadd(y, x, p16f_cephes_log_p2); + y1 = pmadd(y1, x, p16f_cephes_log_p5); + y2 = pmadd(y2, x, p16f_cephes_log_p8); + y = pmadd(y, x3, y1); + y = pmadd(y, x3, y2); + y = pmul(y, x3); + + // Add the logarithm of the exponent back to the result of the interpolation. + y1 = pmul(e, p16f_cephes_log_q1); + tmp = pmul(x2, p16f_half); + y = padd(y, y1); + x = psub(x, tmp); + y2 = pmul(e, p16f_cephes_log_q2); + x = padd(x, y); + x = padd(x, y2); + + __mmask16 pos_inf_mask = _mm512_cmp_ps_mask(_x,p16f_pos_inf,_CMP_EQ_OQ); + // Filter out invalid inputs, i.e.: + // - negative arg will be NAN, + // - 0 will be -INF. + // - +INF will be +INF + return _mm512_mask_blend_ps(iszero_mask, + _mm512_mask_blend_ps(invalid_mask, + _mm512_mask_blend_ps(pos_inf_mask,x,p16f_pos_inf), + p16f_nan), + p16f_minus_inf); +} +#endif + +// Exponential function. Works by writing "x = m*log(2) + r" where +// "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then +// "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +pexp<Packet16f>(const Packet16f& _x) { + _EIGEN_DECLARE_CONST_Packet16f(1, 1.0f); + _EIGEN_DECLARE_CONST_Packet16f(half, 0.5f); + _EIGEN_DECLARE_CONST_Packet16f(127, 127.0f); + + _EIGEN_DECLARE_CONST_Packet16f(exp_hi, 88.3762626647950f); + _EIGEN_DECLARE_CONST_Packet16f(exp_lo, -88.3762626647949f); + + _EIGEN_DECLARE_CONST_Packet16f(cephes_LOG2EF, 1.44269504088896341f); + + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p0, 1.9875691500E-4f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p1, 1.3981999507E-3f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p2, 8.3334519073E-3f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p3, 4.1665795894E-2f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p4, 1.6666665459E-1f); + _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p5, 5.0000001201E-1f); + + // Clamp x. + Packet16f x = pmax(pmin(_x, p16f_exp_hi), p16f_exp_lo); + + // Express exp(x) as exp(m*ln(2) + r), start by extracting + // m = floor(x/ln(2) + 0.5). + Packet16f m = _mm512_floor_ps(pmadd(x, p16f_cephes_LOG2EF, p16f_half)); + + // Get r = x - m*ln(2). Note that we can do this without losing more than one + // ulp precision due to the FMA instruction. + _EIGEN_DECLARE_CONST_Packet16f(nln2, -0.6931471805599453f); + Packet16f r = _mm512_fmadd_ps(m, p16f_nln2, x); + Packet16f r2 = pmul(r, r); + + // TODO(gonnet): Split into odd/even polynomials and try to exploit + // instruction-level parallelism. + Packet16f y = p16f_cephes_exp_p0; + y = pmadd(y, r, p16f_cephes_exp_p1); + y = pmadd(y, r, p16f_cephes_exp_p2); + y = pmadd(y, r, p16f_cephes_exp_p3); + y = pmadd(y, r, p16f_cephes_exp_p4); + y = pmadd(y, r, p16f_cephes_exp_p5); + y = pmadd(y, r2, r); + y = padd(y, p16f_1); + + // Build emm0 = 2^m. + Packet16i emm0 = _mm512_cvttps_epi32(padd(m, p16f_127)); + emm0 = _mm512_slli_epi32(emm0, 23); + + // Return 2^m * exp(r). + return pmax(pmul(y, _mm512_castsi512_ps(emm0)), _x); +} + +/*template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d +pexp<Packet8d>(const Packet8d& _x) { + Packet8d x = _x; + + _EIGEN_DECLARE_CONST_Packet8d(1, 1.0); + _EIGEN_DECLARE_CONST_Packet8d(2, 2.0); + + _EIGEN_DECLARE_CONST_Packet8d(exp_hi, 709.437); + _EIGEN_DECLARE_CONST_Packet8d(exp_lo, -709.436139303); + + _EIGEN_DECLARE_CONST_Packet8d(cephes_LOG2EF, 1.4426950408889634073599); + + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_p0, 1.26177193074810590878e-4); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_p1, 3.02994407707441961300e-2); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_p2, 9.99999999999999999910e-1); + + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_q0, 3.00198505138664455042e-6); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_q1, 2.52448340349684104192e-3); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_q2, 2.27265548208155028766e-1); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_q3, 2.00000000000000000009e0); + + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_C1, 0.693145751953125); + _EIGEN_DECLARE_CONST_Packet8d(cephes_exp_C2, 1.42860682030941723212e-6); + + // clamp x + x = pmax(pmin(x, p8d_exp_hi), p8d_exp_lo); + + // Express exp(x) as exp(g + n*log(2)). + const Packet8d n = + _mm512_mul_round_pd(p8d_cephes_LOG2EF, x, _MM_FROUND_TO_NEAREST_INT); + + // Get the remainder modulo log(2), i.e. the "g" described above. Subtract + // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last + // digits right. + const Packet8d nC1 = pmul(n, p8d_cephes_exp_C1); + const Packet8d nC2 = pmul(n, p8d_cephes_exp_C2); + x = psub(x, nC1); + x = psub(x, nC2); + + const Packet8d x2 = pmul(x, x); + + // Evaluate the numerator polynomial of the rational interpolant. + Packet8d px = p8d_cephes_exp_p0; + px = pmadd(px, x2, p8d_cephes_exp_p1); + px = pmadd(px, x2, p8d_cephes_exp_p2); + px = pmul(px, x); + + // Evaluate the denominator polynomial of the rational interpolant. + Packet8d qx = p8d_cephes_exp_q0; + qx = pmadd(qx, x2, p8d_cephes_exp_q1); + qx = pmadd(qx, x2, p8d_cephes_exp_q2); + qx = pmadd(qx, x2, p8d_cephes_exp_q3); + + // I don't really get this bit, copied from the SSE2 routines, so... + // TODO(gonnet): Figure out what is going on here, perhaps find a better + // rational interpolant? + x = _mm512_div_pd(px, psub(qx, px)); + x = pmadd(p8d_2, x, p8d_1); + + // Build e=2^n. + const Packet8d e = _mm512_castsi512_pd(_mm512_slli_epi64( + _mm512_add_epi64(_mm512_cvtpd_epi64(n), _mm512_set1_epi64(1023)), 52)); + + // Construct the result 2^n * exp(g) = e * x. The max is used to catch + // non-finite values in the input. + return pmax(pmul(x, e), _x); + }*/ + +// Functions for sqrt. +// The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step +// of Newton's method, at a cost of 1-2 bits of precision as opposed to the +// exact solution. The main advantage of this approach is not just speed, but +// also the fact that it can be inlined and pipelined with other computations, +// further reducing its effective latency. +#if EIGEN_FAST_MATH +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +psqrt<Packet16f>(const Packet16f& _x) { + Packet16f neg_half = pmul(_x, pset1<Packet16f>(-.5f)); + __mmask16 denormal_mask = _mm512_kand( + _mm512_cmp_ps_mask(_x, pset1<Packet16f>((std::numeric_limits<float>::min)()), + _CMP_LT_OQ), + _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_GE_OQ)); + + Packet16f x = _mm512_rsqrt14_ps(_x); + + // Do a single step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), pset1<Packet16f>(1.5f))); + + // Flush results for denormals to zero. + return _mm512_mask_blend_ps(denormal_mask, pmul(_x,x), _mm512_setzero_ps()); +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d +psqrt<Packet8d>(const Packet8d& _x) { + Packet8d neg_half = pmul(_x, pset1<Packet8d>(-.5)); + __mmask16 denormal_mask = _mm512_kand( + _mm512_cmp_pd_mask(_x, pset1<Packet8d>((std::numeric_limits<double>::min)()), + _CMP_LT_OQ), + _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_GE_OQ)); + + Packet8d x = _mm512_rsqrt14_pd(_x); + + // Do a single step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), pset1<Packet8d>(1.5))); + + // Do a second step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), pset1<Packet8d>(1.5))); + + return _mm512_mask_blend_pd(denormal_mask, pmul(_x,x), _mm512_setzero_pd()); +} +#else +template <> +EIGEN_STRONG_INLINE Packet16f psqrt<Packet16f>(const Packet16f& x) { + return _mm512_sqrt_ps(x); +} +template <> +EIGEN_STRONG_INLINE Packet8d psqrt<Packet8d>(const Packet8d& x) { + return _mm512_sqrt_pd(x); +} +#endif + +// Functions for rsqrt. +// Almost identical to the sqrt routine, just leave out the last multiplication +// and fill in NaN/Inf where needed. Note that this function only exists as an +// iterative version for doubles since there is no instruction for diretly +// computing the reciprocal square root in AVX-512. +#ifdef EIGEN_FAST_MATH +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +prsqrt<Packet16f>(const Packet16f& _x) { + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(inf, 0x7f800000); + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(nan, 0x7fc00000); + _EIGEN_DECLARE_CONST_Packet16f(one_point_five, 1.5f); + _EIGEN_DECLARE_CONST_Packet16f(minus_half, -0.5f); + _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(flt_min, 0x00800000); + + Packet16f neg_half = pmul(_x, p16f_minus_half); + + // select only the inverse sqrt of positive normal inputs (denormals are + // flushed to zero and cause infs as well). + __mmask16 le_zero_mask = _mm512_cmp_ps_mask(_x, p16f_flt_min, _CMP_LT_OQ); + Packet16f x = _mm512_mask_blend_ps(le_zero_mask, _mm512_rsqrt14_ps(_x), _mm512_setzero_ps()); + + // Fill in NaNs and Infs for the negative/zero entries. + __mmask16 neg_mask = _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_LT_OQ); + Packet16f infs_and_nans = _mm512_mask_blend_ps( + neg_mask, _mm512_mask_blend_ps(le_zero_mask, _mm512_setzero_ps(), p16f_inf), p16f_nan); + + // Do a single step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), p16f_one_point_five)); + + // Insert NaNs and Infs in all the right places. + return _mm512_mask_blend_ps(le_zero_mask, x, infs_and_nans); +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d +prsqrt<Packet8d>(const Packet8d& _x) { + _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(inf, 0x7ff0000000000000LL); + _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(nan, 0x7ff1000000000000LL); + _EIGEN_DECLARE_CONST_Packet8d(one_point_five, 1.5); + _EIGEN_DECLARE_CONST_Packet8d(minus_half, -0.5); + _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(dbl_min, 0x0010000000000000LL); + + Packet8d neg_half = pmul(_x, p8d_minus_half); + + // select only the inverse sqrt of positive normal inputs (denormals are + // flushed to zero and cause infs as well). + __mmask8 le_zero_mask = _mm512_cmp_pd_mask(_x, p8d_dbl_min, _CMP_LT_OQ); + Packet8d x = _mm512_mask_blend_pd(le_zero_mask, _mm512_rsqrt14_pd(_x), _mm512_setzero_pd()); + + // Fill in NaNs and Infs for the negative/zero entries. + __mmask8 neg_mask = _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_LT_OQ); + Packet8d infs_and_nans = _mm512_mask_blend_pd( + neg_mask, _mm512_mask_blend_pd(le_zero_mask, _mm512_setzero_pd(), p8d_inf), p8d_nan); + + // Do a first step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), p8d_one_point_five)); + + // Do a second step of Newton's iteration. + x = pmul(x, pmadd(neg_half, pmul(x, x), p8d_one_point_five)); + + // Insert NaNs and Infs in all the right places. + return _mm512_mask_blend_pd(le_zero_mask, x, infs_and_nans); +} +#elif defined(EIGEN_VECTORIZE_AVX512ER) +template <> +EIGEN_STRONG_INLINE Packet16f prsqrt<Packet16f>(const Packet16f& x) { + return _mm512_rsqrt28_ps(x); +} +#endif +#endif + + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +psin<Packet16f>(const Packet16f& _x) { + return psin_float(_x); +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f +pcos<Packet16f>(const Packet16f& _x) { + return pcos_float(_x); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h new file mode 100644 index 0000000..3842f57 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -0,0 +1,1374 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_AVX512_H +#define EIGEN_PACKET_MATH_AVX512_H + +namespace Eigen { + +namespace internal { + +#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD +#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 +#endif + +#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#endif + +#ifdef EIGEN_VECTORIZE_FMA +#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#endif +#endif + +typedef __m512 Packet16f; +typedef __m512i Packet16i; +typedef __m512d Packet8d; + +template <> +struct is_arithmetic<__m512> { + enum { value = true }; +}; +template <> +struct is_arithmetic<__m512i> { + enum { value = true }; +}; +template <> +struct is_arithmetic<__m512d> { + enum { value = true }; +}; + +template<> struct packet_traits<float> : default_packet_traits +{ + typedef Packet16f type; + typedef Packet8f half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 16, + HasHalfPacket = 1, + HasBlend = 0, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, +#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) +#ifdef EIGEN_VECTORIZE_AVX512DQ + HasLog = 1, +#endif + HasExp = 1, + HasSqrt = EIGEN_FAST_MATH, + HasRsqrt = EIGEN_FAST_MATH, +#endif + HasDiv = 1 + }; + }; +template<> struct packet_traits<double> : default_packet_traits +{ + typedef Packet8d type; + typedef Packet4d half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + HasHalfPacket = 1, +#if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) + HasSqrt = EIGEN_FAST_MATH, + HasRsqrt = EIGEN_FAST_MATH, +#endif + HasDiv = 1 + }; +}; + +/* TODO Implement AVX512 for integers +template<> struct packet_traits<int> : default_packet_traits +{ + typedef Packet16i type; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size=8 + }; +}; +*/ + +template <> +struct unpacket_traits<Packet16f> { + typedef float type; + typedef Packet8f half; + typedef Packet16i integer_packet; + enum { size = 16, alignment=Aligned64, vectorizable=true }; +}; +template <> +struct unpacket_traits<Packet8d> { + typedef double type; + typedef Packet4d half; + enum { size = 8, alignment=Aligned64, vectorizable=true }; +}; +template <> +struct unpacket_traits<Packet16i> { + typedef int type; + typedef Packet8i half; + enum { size = 16, alignment=Aligned64, vectorizable=false }; +}; + +template <> +EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) { + return _mm512_set1_ps(from); +} +template <> +EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) { + return _mm512_set1_pd(from); +} +template <> +EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) { + return _mm512_set1_epi32(from); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) { + return _mm512_castsi512_ps(_mm512_set1_epi32(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) { + return _mm512_broadcastss_ps(_mm_load_ps1(from)); +} +template <> +EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) { + return _mm512_set1_pd(*from); +} + +template <> +EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) { + return _mm512_add_ps( + _mm512_set1_ps(a), + _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f, + 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)); +} +template <> +EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) { + return _mm512_add_pd(_mm512_set1_pd(a), + _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0)); +} + +template <> +EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, + const Packet16f& b) { + return _mm512_add_ps(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, + const Packet8d& b) { + return _mm512_add_pd(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a, + const Packet16i& b) { + return _mm512_add_epi32(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, + const Packet16f& b) { + return _mm512_sub_ps(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a, + const Packet8d& b) { + return _mm512_sub_pd(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a, + const Packet16i& b) { + return _mm512_sub_epi32(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) { + return _mm512_sub_ps(_mm512_set1_ps(0.0), a); +} +template <> +EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) { + return _mm512_sub_pd(_mm512_set1_pd(0.0), a); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) { + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, + const Packet16f& b) { + return _mm512_mul_ps(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a, + const Packet8d& b) { + return _mm512_mul_pd(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a, + const Packet16i& b) { + return _mm512_mul_epi32(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a, + const Packet16f& b) { + return _mm512_div_ps(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a, + const Packet8d& b) { + return _mm512_div_pd(a, b); +} + +#ifdef EIGEN_VECTORIZE_FMA +template <> +EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, + const Packet16f& c) { + return _mm512_fmadd_ps(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, + const Packet8d& c) { + return _mm512_fmadd_pd(a, b, c); +} +#endif + +template <> +EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a, + const Packet16f& b) { + // Arguments are reversed to match NaN propagation behavior of std::min. + return _mm512_min_ps(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a, + const Packet8d& b) { + // Arguments are reversed to match NaN propagation behavior of std::min. + return _mm512_min_pd(b, a); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a, + const Packet16f& b) { + // Arguments are reversed to match NaN propagation behavior of std::max. + return _mm512_max_ps(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a, + const Packet8d& b) { + // Arguments are reversed to match NaN propagation behavior of std::max. + return _mm512_max_pd(b, a); +} + +#ifdef EIGEN_VECTORIZE_AVX512DQ +template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm512_extractf32x8_ps(x,I_); } +template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm512_extractf64x2_pd(x,I_); } +EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_insertf32x8(_mm512_castps256_ps512(a),b,1); } +#else +// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 +template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { + return _mm256_castsi256_ps(_mm512_extracti64x4_epi64( _mm512_castps_si512(x),I_)); +} + +// AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512 +template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { + return _mm_castsi128_pd(_mm512_extracti32x4_epi32( _mm512_castpd_si512(x),I_)); +} + +EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { + return _mm512_castsi512_ps(_mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), + _mm256_castps_si256(b),1)); +} +#endif + +template<> EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) { + __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ); + return _mm512_castsi512_ps( + _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); +} + +template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) { + __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ); + return _mm512_castsi512_ps( + _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); +} + +template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) { + __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGT_UQ); + return _mm512_castsi512_ps( + _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); +} + +template<> EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) { + __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _CMP_EQ_OQ); + return _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) { + __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ); + return _mm512_castsi512_ps( + _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); +} + +template <> +EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) { + __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ); + return _mm512_castsi512_pd( + _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); +} + +template <> +EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) { + return _mm512_set1_epi32(0xffffffffu); +} + +template <> +EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) { + return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) { + return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a, + const Packet16i& b) { + return _mm512_and_si512(a,b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a, + const Packet16f& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_and_ps(a, b); +#else + return _mm512_castsi512_ps(pand(_mm512_castps_si512(a),_mm512_castps_si512(b))); +#endif +} +template <> +EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a, + const Packet8d& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_and_pd(a, b); +#else + Packet8d res = _mm512_undefined_pd(); + Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); + Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); + res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0); + + Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); + Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); + res = _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1); + + return res; +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) { + return _mm512_or_si512(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_or_ps(a, b); +#else + return _mm512_castsi512_ps(por(_mm512_castps_si512(a),_mm512_castps_si512(b))); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a, + const Packet8d& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_or_pd(a, b); +#else + return _mm512_castsi512_pd(por(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) { + return _mm512_xor_si512(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_xor_ps(a, b); +#else + return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a),_mm512_castps_si512(b))); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_xor_pd(a, b); +#else + return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) { + return _mm512_andnot_si512(b, a); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_andnot_ps(b, a); +#else + return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a),_mm512_castps_si512(b))); +#endif +} +template <> +EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a,const Packet8d& b) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + return _mm512_andnot_pd(b, a); +#else + return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); +#endif +} + +template<int N> EIGEN_STRONG_INLINE Packet16i pshiftleft(Packet16i a) { + return _mm512_slli_epi32(a, N); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from); +} +template <> +EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from); +} +template <> +EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512( + reinterpret_cast<const __m512i*>(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from); +} +template <> +EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from); +} +template <> +EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512( + reinterpret_cast<const __m512i*>(from)); +} + +// Loads 8 floats from memory a returns the packet +// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} +template <> +EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) { + // an unaligned load is required here as there is no requirement + // on the alignment of input pointer 'from' + __m256i low_half = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from)); + __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half)); + __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0)); + return pairs; +} + +#ifdef EIGEN_VECTORIZE_AVX512DQ +// FIXME: this does not look optimal, better load a Packet4d and shuffle... +// Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3, +// a3} +template <> +EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) { + __m512d x = _mm512_setzero_pd(); + x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[0]), 0); + x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[1]), 1); + x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[2]), 2); + x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[3]), 3); + return x; +} +#else +template <> +EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) { + __m512d x = _mm512_setzero_pd(); + x = _mm512_mask_broadcastsd_pd(x, 0x3<<0, _mm_load_sd(from+0)); + x = _mm512_mask_broadcastsd_pd(x, 0x3<<2, _mm_load_sd(from+1)); + x = _mm512_mask_broadcastsd_pd(x, 0x3<<4, _mm_load_sd(from+2)); + x = _mm512_mask_broadcastsd_pd(x, 0x3<<6, _mm_load_sd(from+3)); + return x; +} +#endif + +// Loads 4 floats from memory a returns the packet +// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} +template <> +EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) { + Packet16f tmp = _mm512_undefined_ps(); + tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from), 0); + tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 1), 1); + tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 2), 2); + tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 3), 3); + return tmp; +} +// Loads 2 doubles from memory a returns the packet +// {a0, a0 a0, a0, a1, a1, a1, a1} +template <> +EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) { + __m256d lane0 = _mm256_set1_pd(*from); + __m256d lane1 = _mm256_set1_pd(*(from+1)); + __m512d tmp = _mm512_undefined_pd(); + tmp = _mm512_insertf64x4(tmp, lane0, 0); + return _mm512_insertf64x4(tmp, lane1, 1); +} + +template <> +EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from); +} +template <> +EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from); +} +template <> +EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to), + from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from); +} +template <> +EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from); +} +template <> +EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512( + reinterpret_cast<__m512i*>(to), from); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from, + Index stride) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + + return _mm512_i32gather_ps(indices, from, 4); +} +template <> +EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from, + Index stride) { + Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); + Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); + + return _mm512_i32gather_pd(indices, from, 8); +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, + const Packet16f& from, + Index stride) { + Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride)); + Packet16i stride_multiplier = + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); + _mm512_i32scatter_ps(to, indices, from, 4); +} +template <> +EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, + const Packet8d& from, + Index stride) { + Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride)); + Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); + _mm512_i32scatter_pd(to, indices, from, 8); +} + +template <> +EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) { + Packet16f pa = pset1<Packet16f>(a); + pstore(to, pa); +} +template <> +EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) { + Packet8d pa = pset1<Packet8d>(a); + pstore(to, pa); +} +template <> +EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) { + Packet16i pa = pset1<Packet16i>(a); + pstore(to, pa); +} + +template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } + +template <> +EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) { + return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0)); +} +template <> +EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) { + return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0)); +} +template <> +EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) { + return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0); +} + +template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) +{ + return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a); +} + +template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) +{ + return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a); +} + +template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) +{ + // _mm512_abs_ps intrinsic not found, so hack around it + return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff))); +} +template <> +EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) { + // _mm512_abs_ps intrinsic not found, so hack around it + return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), + _mm512_set1_epi64(0x7fffffffffffffff))); +} + +#ifdef EIGEN_VECTORIZE_AVX512DQ +// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 +#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ + __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \ + __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1) +#else +#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ + __m256 OUTPUT##_0 = _mm256_insertf128_ps( \ + _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \ + _mm512_extractf32x4_ps(INPUT, 1), 1); \ + __m256 OUTPUT##_1 = _mm256_insertf128_ps( \ + _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \ + _mm512_extractf32x4_ps(INPUT, 3), 1); +#endif + +#ifdef EIGEN_VECTORIZE_AVX512DQ +#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ + OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1); +#else +#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ + OUTPUT = _mm512_undefined_ps(); \ + OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \ + OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \ + OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \ + OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3); +#endif +template<> EIGEN_STRONG_INLINE Packet16f preduxp<Packet16f>(const Packet16f* +vecs) +{ + EIGEN_EXTRACT_8f_FROM_16f(vecs[0], vecs0); + EIGEN_EXTRACT_8f_FROM_16f(vecs[1], vecs1); + EIGEN_EXTRACT_8f_FROM_16f(vecs[2], vecs2); + EIGEN_EXTRACT_8f_FROM_16f(vecs[3], vecs3); + EIGEN_EXTRACT_8f_FROM_16f(vecs[4], vecs4); + EIGEN_EXTRACT_8f_FROM_16f(vecs[5], vecs5); + EIGEN_EXTRACT_8f_FROM_16f(vecs[6], vecs6); + EIGEN_EXTRACT_8f_FROM_16f(vecs[7], vecs7); + EIGEN_EXTRACT_8f_FROM_16f(vecs[8], vecs8); + EIGEN_EXTRACT_8f_FROM_16f(vecs[9], vecs9); + EIGEN_EXTRACT_8f_FROM_16f(vecs[10], vecs10); + EIGEN_EXTRACT_8f_FROM_16f(vecs[11], vecs11); + EIGEN_EXTRACT_8f_FROM_16f(vecs[12], vecs12); + EIGEN_EXTRACT_8f_FROM_16f(vecs[13], vecs13); + EIGEN_EXTRACT_8f_FROM_16f(vecs[14], vecs14); + EIGEN_EXTRACT_8f_FROM_16f(vecs[15], vecs15); + + __m256 hsum1 = _mm256_hadd_ps(vecs0_0, vecs1_0); + __m256 hsum2 = _mm256_hadd_ps(vecs2_0, vecs3_0); + __m256 hsum3 = _mm256_hadd_ps(vecs4_0, vecs5_0); + __m256 hsum4 = _mm256_hadd_ps(vecs6_0, vecs7_0); + + __m256 hsum5 = _mm256_hadd_ps(hsum1, hsum1); + __m256 hsum6 = _mm256_hadd_ps(hsum2, hsum2); + __m256 hsum7 = _mm256_hadd_ps(hsum3, hsum3); + __m256 hsum8 = _mm256_hadd_ps(hsum4, hsum4); + + __m256 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); + __m256 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); + __m256 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); + __m256 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); + + __m256 sum1 = _mm256_add_ps(perm1, hsum5); + __m256 sum2 = _mm256_add_ps(perm2, hsum6); + __m256 sum3 = _mm256_add_ps(perm3, hsum7); + __m256 sum4 = _mm256_add_ps(perm4, hsum8); + + __m256 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); + __m256 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); + + __m256 final = _mm256_blend_ps(blend1, blend2, 0xf0); + + hsum1 = _mm256_hadd_ps(vecs0_1, vecs1_1); + hsum2 = _mm256_hadd_ps(vecs2_1, vecs3_1); + hsum3 = _mm256_hadd_ps(vecs4_1, vecs5_1); + hsum4 = _mm256_hadd_ps(vecs6_1, vecs7_1); + + hsum5 = _mm256_hadd_ps(hsum1, hsum1); + hsum6 = _mm256_hadd_ps(hsum2, hsum2); + hsum7 = _mm256_hadd_ps(hsum3, hsum3); + hsum8 = _mm256_hadd_ps(hsum4, hsum4); + + perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); + perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); + perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); + perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); + + sum1 = _mm256_add_ps(perm1, hsum5); + sum2 = _mm256_add_ps(perm2, hsum6); + sum3 = _mm256_add_ps(perm3, hsum7); + sum4 = _mm256_add_ps(perm4, hsum8); + + blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); + blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); + + final = _mm256_add_ps(final, _mm256_blend_ps(blend1, blend2, 0xf0)); + + hsum1 = _mm256_hadd_ps(vecs8_0, vecs9_0); + hsum2 = _mm256_hadd_ps(vecs10_0, vecs11_0); + hsum3 = _mm256_hadd_ps(vecs12_0, vecs13_0); + hsum4 = _mm256_hadd_ps(vecs14_0, vecs15_0); + + hsum5 = _mm256_hadd_ps(hsum1, hsum1); + hsum6 = _mm256_hadd_ps(hsum2, hsum2); + hsum7 = _mm256_hadd_ps(hsum3, hsum3); + hsum8 = _mm256_hadd_ps(hsum4, hsum4); + + perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); + perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); + perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); + perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); + + sum1 = _mm256_add_ps(perm1, hsum5); + sum2 = _mm256_add_ps(perm2, hsum6); + sum3 = _mm256_add_ps(perm3, hsum7); + sum4 = _mm256_add_ps(perm4, hsum8); + + blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); + blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); + + __m256 final_1 = _mm256_blend_ps(blend1, blend2, 0xf0); + + hsum1 = _mm256_hadd_ps(vecs8_1, vecs9_1); + hsum2 = _mm256_hadd_ps(vecs10_1, vecs11_1); + hsum3 = _mm256_hadd_ps(vecs12_1, vecs13_1); + hsum4 = _mm256_hadd_ps(vecs14_1, vecs15_1); + + hsum5 = _mm256_hadd_ps(hsum1, hsum1); + hsum6 = _mm256_hadd_ps(hsum2, hsum2); + hsum7 = _mm256_hadd_ps(hsum3, hsum3); + hsum8 = _mm256_hadd_ps(hsum4, hsum4); + + perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23); + perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23); + perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23); + perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23); + + sum1 = _mm256_add_ps(perm1, hsum5); + sum2 = _mm256_add_ps(perm2, hsum6); + sum3 = _mm256_add_ps(perm3, hsum7); + sum4 = _mm256_add_ps(perm4, hsum8); + + blend1 = _mm256_blend_ps(sum1, sum2, 0xcc); + blend2 = _mm256_blend_ps(sum3, sum4, 0xcc); + + final_1 = _mm256_add_ps(final_1, _mm256_blend_ps(blend1, blend2, 0xf0)); + + __m512 final_output; + + EIGEN_INSERT_8f_INTO_16f(final_output, final, final_1); + return final_output; +} + +template<> EIGEN_STRONG_INLINE Packet8d preduxp<Packet8d>(const Packet8d* vecs) +{ + Packet4d vecs0_0 = _mm512_extractf64x4_pd(vecs[0], 0); + Packet4d vecs0_1 = _mm512_extractf64x4_pd(vecs[0], 1); + + Packet4d vecs1_0 = _mm512_extractf64x4_pd(vecs[1], 0); + Packet4d vecs1_1 = _mm512_extractf64x4_pd(vecs[1], 1); + + Packet4d vecs2_0 = _mm512_extractf64x4_pd(vecs[2], 0); + Packet4d vecs2_1 = _mm512_extractf64x4_pd(vecs[2], 1); + + Packet4d vecs3_0 = _mm512_extractf64x4_pd(vecs[3], 0); + Packet4d vecs3_1 = _mm512_extractf64x4_pd(vecs[3], 1); + + Packet4d vecs4_0 = _mm512_extractf64x4_pd(vecs[4], 0); + Packet4d vecs4_1 = _mm512_extractf64x4_pd(vecs[4], 1); + + Packet4d vecs5_0 = _mm512_extractf64x4_pd(vecs[5], 0); + Packet4d vecs5_1 = _mm512_extractf64x4_pd(vecs[5], 1); + + Packet4d vecs6_0 = _mm512_extractf64x4_pd(vecs[6], 0); + Packet4d vecs6_1 = _mm512_extractf64x4_pd(vecs[6], 1); + + Packet4d vecs7_0 = _mm512_extractf64x4_pd(vecs[7], 0); + Packet4d vecs7_1 = _mm512_extractf64x4_pd(vecs[7], 1); + + Packet4d tmp0, tmp1; + + tmp0 = _mm256_hadd_pd(vecs0_0, vecs1_0); + tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); + + tmp1 = _mm256_hadd_pd(vecs2_0, vecs3_0); + tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); + + __m256d final_0 = _mm256_blend_pd(tmp0, tmp1, 0xC); + + tmp0 = _mm256_hadd_pd(vecs0_1, vecs1_1); + tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); + + tmp1 = _mm256_hadd_pd(vecs2_1, vecs3_1); + tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); + + final_0 = _mm256_add_pd(final_0, _mm256_blend_pd(tmp0, tmp1, 0xC)); + + tmp0 = _mm256_hadd_pd(vecs4_0, vecs5_0); + tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); + + tmp1 = _mm256_hadd_pd(vecs6_0, vecs7_0); + tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); + + __m256d final_1 = _mm256_blend_pd(tmp0, tmp1, 0xC); + + tmp0 = _mm256_hadd_pd(vecs4_1, vecs5_1); + tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1)); + + tmp1 = _mm256_hadd_pd(vecs6_1, vecs7_1); + tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1)); + + final_1 = _mm256_add_pd(final_1, _mm256_blend_pd(tmp0, tmp1, 0xC)); + + __m512d final_output = _mm512_castpd256_pd512(final_0); + + return _mm512_insertf64x4(final_output, final_1, 1); +} + +template <> +EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + __m256 lane0 = _mm512_extractf32x8_ps(a, 0); + __m256 lane1 = _mm512_extractf32x8_ps(a, 1); + Packet8f x = _mm256_add_ps(lane0, lane1); + return predux<Packet8f>(x); +#else + __m128 lane0 = _mm512_extractf32x4_ps(a, 0); + __m128 lane1 = _mm512_extractf32x4_ps(a, 1); + __m128 lane2 = _mm512_extractf32x4_ps(a, 2); + __m128 lane3 = _mm512_extractf32x4_ps(a, 3); + __m128 sum = _mm_add_ps(_mm_add_ps(lane0, lane1), _mm_add_ps(lane2, lane3)); + sum = _mm_hadd_ps(sum, sum); + sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1)); + return _mm_cvtss_f32(sum); +#endif +} +template <> +EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) { + __m256d lane0 = _mm512_extractf64x4_pd(a, 0); + __m256d lane1 = _mm512_extractf64x4_pd(a, 1); + __m256d sum = _mm256_add_pd(lane0, lane1); + __m256d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1)); + return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_hadd_pd(tmp0, tmp0))); +} + +template <> +EIGEN_STRONG_INLINE Packet8f predux_half_dowto4<Packet16f>(const Packet16f& a) { +#ifdef EIGEN_VECTORIZE_AVX512DQ + __m256 lane0 = _mm512_extractf32x8_ps(a, 0); + __m256 lane1 = _mm512_extractf32x8_ps(a, 1); + return _mm256_add_ps(lane0, lane1); +#else + __m128 lane0 = _mm512_extractf32x4_ps(a, 0); + __m128 lane1 = _mm512_extractf32x4_ps(a, 1); + __m128 lane2 = _mm512_extractf32x4_ps(a, 2); + __m128 lane3 = _mm512_extractf32x4_ps(a, 3); + __m128 sum0 = _mm_add_ps(lane0, lane2); + __m128 sum1 = _mm_add_ps(lane1, lane3); + return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1); +#endif +} +template <> +EIGEN_STRONG_INLINE Packet4d predux_half_dowto4<Packet8d>(const Packet8d& a) { + __m256d lane0 = _mm512_extractf64x4_pd(a, 0); + __m256d lane1 = _mm512_extractf64x4_pd(a, 1); + __m256d res = _mm256_add_pd(lane0, lane1); + return res; +} + +template <> +EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) { +//#ifdef EIGEN_VECTORIZE_AVX512DQ +#if 0 + Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); + Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); + Packet8f res = pmul(lane0, lane1); + res = pmul(res, _mm256_permute2f128_ps(res, res, 1)); + res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); + return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); +#else + __m128 lane0 = _mm512_extractf32x4_ps(a, 0); + __m128 lane1 = _mm512_extractf32x4_ps(a, 1); + __m128 lane2 = _mm512_extractf32x4_ps(a, 2); + __m128 lane3 = _mm512_extractf32x4_ps(a, 3); + __m128 res = pmul(pmul(lane0, lane1), pmul(lane2, lane3)); + res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); + return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); +#endif +} +template <> +EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) { + __m256d lane0 = _mm512_extractf64x4_pd(a, 0); + __m256d lane1 = _mm512_extractf64x4_pd(a, 1); + __m256d res = pmul(lane0, lane1); + res = pmul(res, _mm256_permute2f128_pd(res, res, 1)); + return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1))); +} + +template <> +EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) { + __m128 lane0 = _mm512_extractf32x4_ps(a, 0); + __m128 lane1 = _mm512_extractf32x4_ps(a, 1); + __m128 lane2 = _mm512_extractf32x4_ps(a, 2); + __m128 lane3 = _mm512_extractf32x4_ps(a, 3); + __m128 res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3)); + res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); + return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); +} +template <> +EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) { + __m256d lane0 = _mm512_extractf64x4_pd(a, 0); + __m256d lane1 = _mm512_extractf64x4_pd(a, 1); + __m256d res = _mm256_min_pd(lane0, lane1); + res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1)); + return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1))); +} + +template <> +EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) { + __m128 lane0 = _mm512_extractf32x4_ps(a, 0); + __m128 lane1 = _mm512_extractf32x4_ps(a, 1); + __m128 lane2 = _mm512_extractf32x4_ps(a, 2); + __m128 lane3 = _mm512_extractf32x4_ps(a, 3); + __m128 res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3)); + res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); + return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); +} + +template <> +EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) { + __m256d lane0 = _mm512_extractf64x4_pd(a, 0); + __m256d lane1 = _mm512_extractf64x4_pd(a, 1); + __m256d res = _mm256_max_pd(lane0, lane1); + res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1)); + return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1))); +} + +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& x) +{ + Packet16i xi = _mm512_castps_si512(x); + __mmask16 tmp = _mm512_test_epi32_mask(xi,xi); + return !_mm512_kortestz(tmp,tmp); +} + +template <int Offset> +struct palign_impl<Offset, Packet16f> { + static EIGEN_STRONG_INLINE void run(Packet16f& first, + const Packet16f& second) { + if (Offset != 0) { + __m512i first_idx = _mm512_set_epi32( + Offset + 15, Offset + 14, Offset + 13, Offset + 12, Offset + 11, + Offset + 10, Offset + 9, Offset + 8, Offset + 7, Offset + 6, + Offset + 5, Offset + 4, Offset + 3, Offset + 2, Offset + 1, Offset); + + __m512i second_idx = + _mm512_set_epi32(Offset - 1, Offset - 2, Offset - 3, Offset - 4, + Offset - 5, Offset - 6, Offset - 7, Offset - 8, + Offset - 9, Offset - 10, Offset - 11, Offset - 12, + Offset - 13, Offset - 14, Offset - 15, Offset - 16); + + unsigned short mask = 0xFFFF; + mask <<= (16 - Offset); + + first = _mm512_permutexvar_ps(first_idx, first); + Packet16f tmp = _mm512_permutexvar_ps(second_idx, second); + first = _mm512_mask_blend_ps(mask, first, tmp); + } + } +}; +template <int Offset> +struct palign_impl<Offset, Packet8d> { + static EIGEN_STRONG_INLINE void run(Packet8d& first, const Packet8d& second) { + if (Offset != 0) { + __m512i first_idx = _mm512_set_epi32( + 0, Offset + 7, 0, Offset + 6, 0, Offset + 5, 0, Offset + 4, 0, + Offset + 3, 0, Offset + 2, 0, Offset + 1, 0, Offset); + + __m512i second_idx = _mm512_set_epi32( + 0, Offset - 1, 0, Offset - 2, 0, Offset - 3, 0, Offset - 4, 0, + Offset - 5, 0, Offset - 6, 0, Offset - 7, 0, Offset - 8); + + unsigned char mask = 0xFF; + mask <<= (8 - Offset); + + first = _mm512_permutexvar_pd(first_idx, first); + Packet8d tmp = _mm512_permutexvar_pd(second_idx, second); + first = _mm512_mask_blend_pd(mask, first, tmp); + } + } +}; + + +#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \ + EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]); + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) { + __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); + __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); + __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); + __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); + __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]); + __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]); + __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]); + __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]); + __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]); + __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]); + __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]); + __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]); + __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]); + __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]); + __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]); + __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]); + __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2)); + + EIGEN_EXTRACT_8f_FROM_16f(S0, S0); + EIGEN_EXTRACT_8f_FROM_16f(S1, S1); + EIGEN_EXTRACT_8f_FROM_16f(S2, S2); + EIGEN_EXTRACT_8f_FROM_16f(S3, S3); + EIGEN_EXTRACT_8f_FROM_16f(S4, S4); + EIGEN_EXTRACT_8f_FROM_16f(S5, S5); + EIGEN_EXTRACT_8f_FROM_16f(S6, S6); + EIGEN_EXTRACT_8f_FROM_16f(S7, S7); + EIGEN_EXTRACT_8f_FROM_16f(S8, S8); + EIGEN_EXTRACT_8f_FROM_16f(S9, S9); + EIGEN_EXTRACT_8f_FROM_16f(S10, S10); + EIGEN_EXTRACT_8f_FROM_16f(S11, S11); + EIGEN_EXTRACT_8f_FROM_16f(S12, S12); + EIGEN_EXTRACT_8f_FROM_16f(S13, S13); + EIGEN_EXTRACT_8f_FROM_16f(S14, S14); + EIGEN_EXTRACT_8f_FROM_16f(S15, S15); + + PacketBlock<Packet8f, 32> tmp; + + tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20); + tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20); + tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20); + tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20); + tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31); + tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31); + tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31); + tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31); + + tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20); + tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20); + tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20); + tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20); + tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31); + tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31); + tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31); + tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31); + + // Second set of _m256 outputs + tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20); + tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20); + tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20); + tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20); + tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31); + tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31); + tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31); + tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31); + + tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20); + tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20); + tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20); + tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20); + tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31); + tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31); + tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31); + tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31); + + // Pack them into the output + PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16); + + PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16); + + PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16); + + PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16); + PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16); +} +#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \ + EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \ + INPUT[2 * INDEX + STRIDE]); + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) { + __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); + __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); + __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); + __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); + + __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); + __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); + + EIGEN_EXTRACT_8f_FROM_16f(S0, S0); + EIGEN_EXTRACT_8f_FROM_16f(S1, S1); + EIGEN_EXTRACT_8f_FROM_16f(S2, S2); + EIGEN_EXTRACT_8f_FROM_16f(S3, S3); + + PacketBlock<Packet8f, 8> tmp; + + tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20); + tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20); + tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31); + tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31); + + tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20); + tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20); + tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31); + tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31); + + PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1); + PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1); + PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1); + PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1); +} + +#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \ + OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \ + OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1); + +#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \ + OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \ + OUTPUT[INDEX] = \ + _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1); + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) { + __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0); + __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff); + __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0); + __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff); + + PacketBlock<Packet4d, 8> tmp; + + tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), + _mm512_extractf64x4_pd(T2, 0), 0x20); + tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), + _mm512_extractf64x4_pd(T3, 0), 0x20); + tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), + _mm512_extractf64x4_pd(T2, 0), 0x31); + tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), + _mm512_extractf64x4_pd(T3, 0), 0x31); + + tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), + _mm512_extractf64x4_pd(T2, 1), 0x20); + tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), + _mm512_extractf64x4_pd(T3, 1), 0x20); + tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), + _mm512_extractf64x4_pd(T2, 1), 0x31); + tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), + _mm512_extractf64x4_pd(T3, 1), 0x31); + + PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1); + PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1); + PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1); + PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1); +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) { + __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]); + __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]); + __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]); + __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]); + __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]); + __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]); + __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]); + __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]); + + PacketBlock<Packet4d, 16> tmp; + + tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), + _mm512_extractf64x4_pd(T2, 0), 0x20); + tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), + _mm512_extractf64x4_pd(T3, 0), 0x20); + tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), + _mm512_extractf64x4_pd(T2, 0), 0x31); + tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), + _mm512_extractf64x4_pd(T3, 0), 0x31); + + tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), + _mm512_extractf64x4_pd(T2, 1), 0x20); + tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), + _mm512_extractf64x4_pd(T3, 1), 0x20); + tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), + _mm512_extractf64x4_pd(T2, 1), 0x31); + tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), + _mm512_extractf64x4_pd(T3, 1), 0x31); + + tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), + _mm512_extractf64x4_pd(T6, 0), 0x20); + tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), + _mm512_extractf64x4_pd(T7, 0), 0x20); + tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), + _mm512_extractf64x4_pd(T6, 0), 0x31); + tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), + _mm512_extractf64x4_pd(T7, 0), 0x31); + + tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), + _mm512_extractf64x4_pd(T6, 1), 0x20); + tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), + _mm512_extractf64x4_pd(T7, 1), 0x20); + tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), + _mm512_extractf64x4_pd(T6, 1), 0x31); + tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), + _mm512_extractf64x4_pd(T7, 1), 0x31); + + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8); + + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8); + PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8); +} +template <> +EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/, + const Packet16f& /*thenPacket*/, + const Packet16f& /*elsePacket*/) { + assert(false && "To be implemented"); + return Packet16f(); +} +template <> +EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, + const Packet8d& thenPacket, + const Packet8d& elsePacket) { + __mmask8 m = (ifPacket.select[0] ) + | (ifPacket.select[1]<<1) + | (ifPacket.select[2]<<2) + | (ifPacket.select[3]<<3) + | (ifPacket.select[4]<<4) + | (ifPacket.select[5]<<5) + | (ifPacket.select[6]<<6) + | (ifPacket.select[7]<<7); + return _mm512_mask_blend_pd(m, elsePacket, thenPacket); +} + +template<> EIGEN_STRONG_INLINE Packet16f pinsertfirst(const Packet16f& a, float b) +{ + return _mm512_mask_broadcastss_ps(a, (1), _mm_load_ss(&b)); +} + +template<> EIGEN_STRONG_INLINE Packet8d pinsertfirst(const Packet8d& a, double b) +{ + return _mm512_mask_broadcastsd_pd(a, (1), _mm_load_sd(&b)); +} + +template<> EIGEN_STRONG_INLINE Packet16f pinsertlast(const Packet16f& a, float b) +{ + return _mm512_mask_broadcastss_ps(a, (1<<15), _mm_load_ss(&b)); +} + +template<> EIGEN_STRONG_INLINE Packet8d pinsertlast(const Packet8d& a, double b) +{ + return _mm512_mask_broadcastsd_pd(a, (1<<7), _mm_load_sd(&b)); +} + +template<> EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) { + return _mm512_cvttps_epi32(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f pcast<Packet16i, Packet16f>(const Packet16i& a) { + return _mm512_cvtepi32_ps(a); +} + +template<> EIGEN_STRONG_INLINE Packet16i preinterpret<Packet16i,Packet16f>(const Packet16f& a) { + return _mm512_castps_si512(a); +} + +template<> EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f,Packet16i>(const Packet16i& a) { + return _mm512_castsi512_ps(a); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_AVX512_H
diff --git a/Eigen/src/Core/arch/AltiVec/CMakeLists.txt b/Eigen/src/Core/arch/AltiVec/CMakeLists.txt deleted file mode 100644 index 9f8d2e9..0000000 --- a/Eigen/src/Core/arch/AltiVec/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_AltiVec_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_AltiVec_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/AltiVec COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/AltiVec/Complex.h b/Eigen/src/Core/arch/AltiVec/Complex.h index 58c2961..440d058 100644 --- a/Eigen/src/Core/arch/AltiVec/Complex.h +++ b/Eigen/src/Core/arch/AltiVec/Complex.h
@@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2010-2016 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -14,19 +15,21 @@ namespace internal { -static Packet4ui p4ui_CONJ_XOR = vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_ZERO_);//{ 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; -#ifdef _BIG_ENDIAN -static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; -static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_ZERO_, 8);//{ 0x8000000000000000, 0x0000000000000000 }; +static Packet4ui p4ui_CONJ_XOR = vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO);//{ 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; +#ifdef __VSX__ +#if defined(_BIG_ENDIAN) +static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_MZERO, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; +static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_MZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; #else -static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_ZERO_, 8);//{ 0x8000000000000000, 0x0000000000000000 }; -static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; +static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_MZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; +static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2d_MZERO, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; +#endif #endif //---------- float ---------- struct Packet2cf { - EIGEN_STRONG_INLINE Packet2cf() {} + EIGEN_STRONG_INLINE explicit Packet2cf() : v(p4f_ZERO) {} EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {} Packet4f v; }; @@ -39,6 +42,7 @@ Vectorizable = 1, AlignedOnScalar = 1, size = 2, + HasHalfPacket = 0, HasAdd = 1, HasSub = 1, @@ -49,17 +53,19 @@ HasAbs2 = 0, HasMin = 0, HasMax = 0, +#ifdef __VSX__ + HasBlend = 1, +#endif HasSetLinear = 0 }; }; -template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16}; typedef Packet2cf half; }; +template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2cf half; }; template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) { Packet2cf res; - /* On AltiVec we cannot load 64-bit registers, so wa have to take care of alignment */ - if((ptrdiff_t(&from) % 16) == 0) + if((std::ptrdiff_t(&from) % 16) == 0) res.v = pload<Packet4f>((const float *)&from); else res.v = ploadu<Packet4f>((const float *)&from); @@ -67,26 +73,32 @@ return res; } +template<> EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) { return Packet2cf(pload<Packet4f>((const float *) from)); } +template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { return Packet2cf(ploadu<Packet4f>((const float*) from)); } +template<> EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) { return pset1<Packet2cf>(*from); } + +template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { pstore((float*)to, from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { pstoreu((float*)to, from.v); } + template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from, Index stride) { - std::complex<float> EIGEN_ALIGN16 af[2]; + EIGEN_ALIGN16 std::complex<float> af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; - return Packet2cf(vec_ld(0, (const float*)af)); + return pload<Packet2cf>(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from, Index stride) { - std::complex<float> EIGEN_ALIGN16 af[2]; - vec_st(from.v, 0, (float*)af); + EIGEN_ALIGN16 std::complex<float> af[2]; + pstore<std::complex<float> >((std::complex<float> *) af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } - -template<> EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_add(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_sub(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(a.v + b.v); } +template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(a.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf((Packet4f)vec_xor((Packet4ui)a.v, p4ui_CONJ_XOR)); } +template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf(pxor<Packet4f>(a.v, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR))); } template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { @@ -100,34 +112,23 @@ v1 = vec_madd(v1, b.v, p4f_ZERO); // multiply a_im * b and get the conjugate result v2 = vec_madd(v2, b.v, p4f_ZERO); - v2 = (Packet4f) vec_xor((Packet4ui)v2, p4ui_CONJ_XOR); + v2 = reinterpret_cast<Packet4f>(pxor(v2, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR))); // permute back to a proper order v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV); - return Packet2cf(vec_add(v1, v2)); + return Packet2cf(padd<Packet4f>(v1, v2)); } -template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_and(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_or(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_xor(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vec_and(a.v, vec_nor(b.v,b.v))); } +template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pand<Packet4f>(a.v, b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(por<Packet4f>(a.v, b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pxor<Packet4f>(a.v, b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pandnot<Packet4f>(a.v, b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>((const float*)from)); } -template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>((const float*)from)); } - -template<> EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) -{ - return pset1<Packet2cf>(*from); -} - -template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } -template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v); } - -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { vec_dstt((float *)addr, DST_CTRL(2,2,32), DST_CHAN); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) { - std::complex<float> EIGEN_ALIGN16 res[2]; + EIGEN_ALIGN16 std::complex<float> res[2]; pstore((float *)&res, a.v); return res[0]; @@ -143,23 +144,23 @@ template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) { Packet4f b; - b = (Packet4f) vec_sld(a.v, a.v, 8); - b = padd(a.v, b); - return pfirst(Packet2cf(b)); + b = vec_sld(a.v, a.v, 8); + b = padd<Packet4f>(a.v, b); + return pfirst<Packet2cf>(Packet2cf(b)); } template<> EIGEN_STRONG_INLINE Packet2cf preduxp<Packet2cf>(const Packet2cf* vecs) { Packet4f b1, b2; #ifdef _BIG_ENDIAN - b1 = (Packet4f) vec_sld(vecs[0].v, vecs[1].v, 8); - b2 = (Packet4f) vec_sld(vecs[1].v, vecs[0].v, 8); + b1 = vec_sld(vecs[0].v, vecs[1].v, 8); + b2 = vec_sld(vecs[1].v, vecs[0].v, 8); #else - b1 = (Packet4f) vec_sld(vecs[1].v, vecs[0].v, 8); - b2 = (Packet4f) vec_sld(vecs[0].v, vecs[1].v, 8); + b1 = vec_sld(vecs[1].v, vecs[0].v, 8); + b2 = vec_sld(vecs[0].v, vecs[1].v, 8); #endif - b2 = (Packet4f) vec_sld(b2, b2, 8); - b2 = padd(b1, b2); + b2 = vec_sld(b2, b2, 8); + b2 = padd<Packet4f>(b1, b2); return Packet2cf(b2); } @@ -168,10 +169,10 @@ { Packet4f b; Packet2cf prod; - b = (Packet4f) vec_sld(a.v, a.v, 8); - prod = pmul(a, Packet2cf(b)); + b = vec_sld(a.v, a.v, 8); + prod = pmul<Packet2cf>(a, Packet2cf(b)); - return pfirst(prod); + return pfirst<Packet2cf>(prod); } template<int Offset> @@ -223,12 +224,14 @@ } }; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) + template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for AltiVec - Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b); - Packet4f s = vec_madd(b.v, b.v, p4f_ZERO); - return Packet2cf(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_COMPLEX32_REV)))); + Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a, b); + Packet4f s = pmul<Packet4f>(b.v, b.v); + return Packet2cf(pdiv(res.v, padd<Packet4f>(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& x) @@ -243,6 +246,14 @@ kernel.packet[0].v = tmp; } +#ifdef __VSX__ +template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { + Packet2cf result; + result.v = reinterpret_cast<Packet4f>(pblend<Packet2d>(ifPacket, reinterpret_cast<Packet2d>(thenPacket.v), reinterpret_cast<Packet2d>(elsePacket.v))); + return result; +} +#endif + //---------- double ---------- #ifdef __VSX__ struct Packet1cd @@ -275,35 +286,35 @@ }; }; -template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16}; typedef Packet1cd half; }; +template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16, vectorizable=true}; typedef Packet1cd half; }; -template<> EIGEN_STRONG_INLINE Packet1cd pload <Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from)); } -template<> EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from)); } -template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } -template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } +template<> EIGEN_STRONG_INLINE Packet1cd pload <Packet1cd>(const std::complex<double>* from) { return Packet1cd(pload<Packet2d>((const double*)from)); } +template<> EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) { return Packet1cd(ploadu<Packet2d>((const double*)from)); } +template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { pstore((double*)to, from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { pstoreu((double*)to, from.v); } template<> EIGEN_STRONG_INLINE Packet1cd pset1<Packet1cd>(const std::complex<double>& from) { /* here we really have to use unaligned loads :( */ return ploadu<Packet1cd>(&from); } template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index stride) { - std::complex<double> EIGEN_ALIGN16 af[2]; + EIGEN_ALIGN16 std::complex<double> af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload<Packet1cd>(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from, Index stride) { - std::complex<double> EIGEN_ALIGN16 af[2]; + EIGEN_ALIGN16 std::complex<double> af[2]; pstore<std::complex<double> >(af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } -template<> EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_add(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_sub(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v + b.v); } +template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); } -template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd((Packet2d)vec_xor((Packet2d)a.v, (Packet2d)p2ul_CONJ_XOR2)); } +template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR2))); } template<> EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { @@ -317,27 +328,24 @@ v1 = vec_madd(a_re, b.v, p2d_ZERO); // multiply a_im * b and get the conjugate result v2 = vec_madd(a_im, b.v, p2d_ZERO); - v2 = (Packet2d) vec_sld((Packet4ui)v2, (Packet4ui)v2, 8); - v2 = (Packet2d) vec_xor((Packet2d)v2, (Packet2d) p2ul_CONJ_XOR1); + v2 = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(v2), reinterpret_cast<Packet4ui>(v2), 8)); + v2 = pxor(v2, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR1)); - return Packet1cd(vec_add(v1, v2)); + return Packet1cd(padd<Packet2d>(v1, v2)); } -template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_or(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_xor(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v, vec_nor(b.v,b.v))); } +template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pand(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(por(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pxor(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pandnot(a.v, b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) -{ - return pset1<Packet1cd>(*from); -} +template<> EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) { return pset1<Packet1cd>(*from); } -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { vec_dstt((long *)addr, DST_CTRL(2,2,32), DST_CHAN); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) { - std::complex<double> EIGEN_ALIGN16 res[2]; + EIGEN_ALIGN16 std::complex<double> res[2]; pstore<std::complex<double> >(res, a); return res[0]; @@ -345,20 +353,10 @@ template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } -template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) -{ - return pfirst(a); -} +template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) { return pfirst(a); } +template<> EIGEN_STRONG_INLINE Packet1cd preduxp<Packet1cd>(const Packet1cd* vecs) { return vecs[0]; } -template<> EIGEN_STRONG_INLINE Packet1cd preduxp<Packet1cd>(const Packet1cd* vecs) -{ - return vecs[0]; -} - -template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) -{ - return pfirst(a); -} +template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) { return pfirst(a); } template<int Offset> struct palign_impl<Offset,Packet1cd> @@ -403,12 +401,14 @@ } }; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) + template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for AltiVec Packet1cd res = conj_helper<Packet1cd,Packet1cd,false,true>().pmul(a,b); - Packet2d s = vec_madd(b.v, b.v, p2d_ZERO_); - return Packet1cd(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_REVERSE64)))); + Packet2d s = pmul<Packet2d>(b.v, b.v); + return Packet1cd(pdiv(res.v, padd<Packet2d>(s, vec_perm(s, s, p16uc_REVERSE64)))); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/*<Packet1cd>*/(const Packet1cd& x)
diff --git a/Eigen/src/Core/arch/AltiVec/MathFunctions.h b/Eigen/src/Core/arch/AltiVec/MathFunctions.h index 9e37e93..81097e6 100644 --- a/Eigen/src/Core/arch/AltiVec/MathFunctions.h +++ b/Eigen/src/Core/arch/AltiVec/MathFunctions.h
@@ -3,18 +3,17 @@ // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -/* The sin, cos, exp, and log functions of this file come from - * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ - */ - #ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H #define EIGEN_MATH_FUNCTIONS_ALTIVEC_H +#include "../Default/GenericPacketMathFunctions.h" + namespace Eigen { namespace internal { @@ -22,264 +21,60 @@ template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); - _EIGEN_DECLARE_CONST_Packet4i(23, 23); - - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); - - /* the smallest non denormalized float number */ - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000); // -1.f/0.f - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_nan, 0xffffffff); - - /* natural logarithm computed for 4 simultaneous float - return NaN for x <= 0 - */ - _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); - - - Packet4i emm0; - - /* isvalid_mask is 0 if x < 0 or x is NaN. */ - Packet4ui isvalid_mask = reinterpret_cast<Packet4ui>(vec_cmpge(x, p4f_ZERO)); - Packet4ui iszero_mask = reinterpret_cast<Packet4ui>(vec_cmpeq(x, p4f_ZERO)); - - x = pmax(x, p4f_min_norm_pos); /* cut off denormalized stuff */ - emm0 = vec_sr(reinterpret_cast<Packet4i>(x), - reinterpret_cast<Packet4ui>(p4i_23)); - - /* keep only the fractional part */ - x = pand(x, p4f_inv_mant_mask); - x = por(x, p4f_half); - - emm0 = psub(emm0, p4i_0x7f); - Packet4f e = padd(vec_ctf(emm0, 0), p4f_1); - - /* part2: - if( x < SQRTHF ) { - e -= 1; - x = x + x - 1.0; - } else { x = x - 1.0; } - */ - Packet4f mask = reinterpret_cast<Packet4f>(vec_cmplt(x, p4f_cephes_SQRTHF)); - Packet4f tmp = pand(x, mask); - x = psub(x, p4f_1); - e = psub(e, pand(p4f_1, mask)); - x = padd(x, tmp); - - Packet4f x2 = pmul(x,x); - Packet4f x3 = pmul(x2,x); - - Packet4f y, y1, y2; - y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1); - y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4); - y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7); - y = pmadd(y , x, p4f_cephes_log_p2); - y1 = pmadd(y1, x, p4f_cephes_log_p5); - y2 = pmadd(y2, x, p4f_cephes_log_p8); - y = pmadd(y, x3, y1); - y = pmadd(y, x3, y2); - y = pmul(y, x3); - - y1 = pmul(e, p4f_cephes_log_q1); - tmp = pmul(x2, p4f_half); - y = padd(y, y1); - x = psub(x, tmp); - y2 = pmul(e, p4f_cephes_log_q2); - x = padd(x, y); - x = padd(x, y2); - // negative arg will be NAN, 0 will be -INF - x = vec_sel(x, p4f_minus_inf, iszero_mask); - x = vec_sel(p4f_minus_nan, x, isvalid_mask); - return x; + return plog_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); - _EIGEN_DECLARE_CONST_Packet4i(23, 23); - - - _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); - _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); - - _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); - - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); - - Packet4f tmp, fx; - Packet4i emm0; - - // clamp x - x = vec_max(vec_min(x, p4f_exp_hi), p4f_exp_lo); - - /* express exp(x) as exp(g + n*log(2)) */ - fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half); - - fx = vec_floor(fx); - - tmp = pmul(fx, p4f_cephes_exp_C1); - Packet4f z = pmul(fx, p4f_cephes_exp_C2); - x = psub(x, tmp); - x = psub(x, z); - - z = pmul(x,x); - - Packet4f y = p4f_cephes_exp_p0; - y = pmadd(y, x, p4f_cephes_exp_p1); - y = pmadd(y, x, p4f_cephes_exp_p2); - y = pmadd(y, x, p4f_cephes_exp_p3); - y = pmadd(y, x, p4f_cephes_exp_p4); - y = pmadd(y, x, p4f_cephes_exp_p5); - y = pmadd(y, z, x); - y = padd(y, p4f_1); - - // build 2^n - emm0 = vec_cts(fx, 0); - emm0 = vec_add(emm0, p4i_0x7f); - emm0 = vec_sl(emm0, reinterpret_cast<Packet4ui>(p4i_23)); - - // Altivec's max & min operators just drop silent NaNs. Check NaNs in - // inputs and return them unmodified. - Packet4ui isnumber_mask = reinterpret_cast<Packet4ui>(vec_cmpeq(_x, _x)); - return vec_sel(_x, pmax(pmul(y, reinterpret_cast<Packet4f>(emm0)), _x), - isnumber_mask); + return pexp_float(_x); } -#ifdef __VSX__ -// VSX support varies between different compilers and even different -// versions of the same compiler. For gcc version >= 4.9.3, we can use -// vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use -// a slow version that works with older compilers. -static inline Packet2l ConvertToPacket2l(const Packet2d& x) { -#if EIGEN_GNUC_AT_LEAST(5, 0) || \ - (EIGEN_GNUC_AT(4, 9) && __GNUC_PATCHLEVEL__ >= 3) - return vec_cts(x, 0); // TODO: check clang version. -#else - double tmp[2]; - memcpy(tmp, &x, sizeof(tmp)); - Packet2l l = { static_cast<long long>(tmp[0]), - static_cast<long long>(tmp[1]) }; - return l; +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f psin<Packet4f>(const Packet4f& _x) +{ + return psin_float(_x); +} + +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f pcos<Packet4f>(const Packet4f& _x) +{ + return pcos_float(_x); +} + +#ifndef EIGEN_COMP_CLANG +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f prsqrt<Packet4f>(const Packet4f& x) +{ + return vec_rsqrt(x); +} #endif + +#ifdef __VSX__ +#ifndef EIGEN_COMP_CLANG +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet2d prsqrt<Packet2d>(const Packet2d& x) +{ + return vec_rsqrt(x); +} +#endif + +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f psqrt<Packet4f>(const Packet4f& x) +{ + return vec_sqrt(x); +} + +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet2d psqrt<Packet2d>(const Packet2d& x) +{ + return vec_sqrt(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp<Packet2d>(const Packet2d& _x) { - Packet2d x = _x; - - _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); - _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); - _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); - - _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); - _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); - - Packet2d tmp, fx; - Packet2l emm0; - - // clamp x - x = pmax(pmin(x, p2d_exp_hi), p2d_exp_lo); - /* express exp(x) as exp(g + n*log(2)) */ - fx = pmadd(p2d_cephes_LOG2EF, x, p2d_half); - - fx = vec_floor(fx); - - tmp = pmul(fx, p2d_cephes_exp_C1); - Packet2d z = pmul(fx, p2d_cephes_exp_C2); - x = psub(x, tmp); - x = psub(x, z); - - Packet2d x2 = pmul(x,x); - - Packet2d px = p2d_cephes_exp_p0; - px = pmadd(px, x2, p2d_cephes_exp_p1); - px = pmadd(px, x2, p2d_cephes_exp_p2); - px = pmul (px, x); - - Packet2d qx = p2d_cephes_exp_q0; - qx = pmadd(qx, x2, p2d_cephes_exp_q1); - qx = pmadd(qx, x2, p2d_cephes_exp_q2); - qx = pmadd(qx, x2, p2d_cephes_exp_q3); - - x = pdiv(px,psub(qx,px)); - x = pmadd(p2d_2,x,p2d_1); - - // build 2^n - emm0 = ConvertToPacket2l(fx); - -#ifdef __POWER8_VECTOR__ - static const Packet2l p2l_1023 = { 1023, 1023 }; - static const Packet2ul p2ul_52 = { 52, 52 }; - - emm0 = vec_add(emm0, p2l_1023); - emm0 = vec_sl(emm0, p2ul_52); -#else - // Code is a bit complex for POWER7. There is actually a - // vec_xxsldi intrinsic but it is not supported by some gcc versions. - // So we shift (52-32) bits and do a word swap with zeros. - _EIGEN_DECLARE_CONST_Packet4i(1023, 1023); - _EIGEN_DECLARE_CONST_Packet4i(20, 20); // 52 - 32 - - Packet4i emm04i = reinterpret_cast<Packet4i>(emm0); - emm04i = vec_add(emm04i, p4i_1023); - emm04i = vec_sl(emm04i, reinterpret_cast<Packet4ui>(p4i_20)); - static const Packet16uc perm = { - 0x14, 0x15, 0x16, 0x17, 0x00, 0x01, 0x02, 0x03, - 0x1c, 0x1d, 0x1e, 0x1f, 0x08, 0x09, 0x0a, 0x0b }; -#ifdef _BIG_ENDIAN - emm0 = reinterpret_cast<Packet2l>(vec_perm(p4i_ZERO, emm04i, perm)); -#else - emm0 = reinterpret_cast<Packet2l>(vec_perm(emm04i, p4i_ZERO, perm)); -#endif - -#endif - - // Altivec's max & min operators just drop silent NaNs. Check NaNs in - // inputs and return them unmodified. - Packet2ul isnumber_mask = reinterpret_cast<Packet2ul>(vec_cmpeq(_x, _x)); - return vec_sel(_x, pmax(pmul(x, reinterpret_cast<Packet2d>(emm0)), _x), - isnumber_mask); + return pexp_double(_x); } #endif
diff --git a/Eigen/src/Core/arch/AltiVec/PacketMath.h b/Eigen/src/Core/arch/AltiVec/PacketMath.h index 0dbbc2e..9535724 100755 --- a/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/Eigen/src/Core/arch/AltiVec/PacketMath.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2014 Konstantinos Margaritis <markos@freevec.org> +// Copyright (C) 2008-2016 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -42,7 +42,7 @@ // and it doesn't really work to declare them global, so we define macros instead #define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ - Packet4f p4f_##NAME = (Packet4f) vec_splat_s32(X) + Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X)) #define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = vec_splat_s32(X) @@ -69,13 +69,13 @@ // These constants are endian-agnostic static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} -#ifndef __VSX__ static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1); //{ 1, 1, 1, 1} -static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0} -#endif static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16); //{ -16, -16, -16, -16} static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} -static Packet4f p4f_ZERO_ = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000} +static Packet4f p4f_MZERO = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000} +#ifndef __VSX__ +static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0} +#endif static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 }; static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 }; @@ -90,18 +90,20 @@ #define _EIGEN_MASK_ALIGNMENT 0xfffffff0 #endif -#define _EIGEN_ALIGNED_PTR(x) ((ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) +#define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) // Handle endianness properly while loading constants // Define global static constants: #ifdef _BIG_ENDIAN -static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0); +static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0); +#ifdef __VSX__ static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; +#endif static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; #else -static Packet16uc p16uc_FORWARD = p16uc_REVERSE32; +static Packet16uc p16uc_FORWARD = p16uc_REVERSE32; static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; @@ -110,8 +112,8 @@ static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 }; -static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16); //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; -static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16); //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31}; +static Packet16uc p16uc_TRANSPOSE64_HI = p16uc_PSET64_HI + p16uc_HALF64_0_16; //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; +static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16; //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31}; static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; @@ -121,6 +123,12 @@ static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #endif // _BIG_ENDIAN +#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC + #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR); +#else + #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); +#endif + template<> struct packet_traits<float> : default_packet_traits { typedef Packet4f type; @@ -129,15 +137,35 @@ Vectorizable = 1, AlignedOnScalar = 1, size=4, - HasHalfPacket=0, + HasHalfPacket = 1, - // FIXME check the Has* + HasAdd = 1, + HasSub = 1, + HasMul = 1, HasDiv = 1, - HasSin = 0, - HasCos = 0, + HasMin = 1, + HasMax = 1, + HasAbs = 1, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, - HasSqrt = 0 +#ifdef __VSX__ + HasSqrt = 1, +#if !EIGEN_COMP_CLANG + HasRsqrt = 1, +#else + HasRsqrt = 0, +#endif +#else + HasSqrt = 0, + HasRsqrt = 0, +#endif + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasNegate = 1, + HasBlend = 1 }; }; template<> struct packet_traits<int> : default_packet_traits @@ -145,16 +173,33 @@ typedef Packet4i type; typedef Packet4i half; enum { - // FIXME check the Has* Vectorizable = 1, AlignedOnScalar = 1, - size=4 + size = 4, + HasHalfPacket = 0, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 0, + HasBlend = 1 }; }; -template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; }; -template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; +template<> struct unpacket_traits<Packet4f> +{ + typedef float type; + typedef Packet4f half; + typedef Packet4i integer_packet; + enum {size=4, alignment=Aligned16, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet4i> +{ + typedef int type; + typedef Packet4i half; + enum {size=4, alignment=Aligned16, vectorizable=false}; +}; inline std::ostream & operator <<(std::ostream & s, const Packet16uc & v) { @@ -200,42 +245,62 @@ s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; return s; } -/* -inline std::ostream & operator <<(std::ostream & s, const Packetbi & v) -{ - union { - Packet4bi v; - unsigned int n[4]; - } vt; - vt.v = v; - s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; - return s; -}*/ - // Need to define them first or we get specialization after instantiation errors -template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vec_ld(0, from); } -template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return vec_ld(0, from); } +template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) +{ + EIGEN_DEBUG_ALIGNED_LOAD +#ifdef __VSX__ + return vec_vsx_ld(0, from); +#else + return vec_ld(0, from); +#endif +} -template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vec_st(from, 0, to); } -template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vec_st(from, 0, to); } +template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) +{ + EIGEN_DEBUG_ALIGNED_LOAD +#ifdef __VSX__ + return vec_vsx_ld(0, from); +#else + return vec_ld(0, from); +#endif +} + +template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) +{ + EIGEN_DEBUG_ALIGNED_STORE +#ifdef __VSX__ + vec_vsx_st(from, 0, to); +#else + vec_st(from, 0, to); +#endif +} + +template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) +{ + EIGEN_DEBUG_ALIGNED_STORE +#ifdef __VSX__ + vec_vsx_st(from, 0, to); +#else + vec_st(from, 0, to); +#endif +} template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { - // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html - float EIGEN_ALIGN16 af[4]; - af[0] = from; - Packet4f vc = pload<Packet4f>(af); - vc = vec_splat(vc, 0); - return vc; + Packet4f v = {from, from, from, from}; + return v; } template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { - int EIGEN_ALIGN16 ai[4]; - ai[0] = from; - Packet4i vc = pload<Packet4i>(ai); - vc = vec_splat(vc, 0); - return vc; + Packet4i v = {from, from, from, from}; + return v; } + +template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) { + return reinterpret_cast<Packet4f>(pset1<Packet4i>(from)); +} + template<> EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float *a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) @@ -259,7 +324,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) { - float EIGEN_ALIGN16 af[4]; + EIGEN_ALIGN16 float af[4]; af[0] = from[0*stride]; af[1] = from[1*stride]; af[2] = from[2*stride]; @@ -268,7 +333,7 @@ } template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride) { - int EIGEN_ALIGN16 ai[4]; + EIGEN_ALIGN16 int ai[4]; ai[0] = from[0*stride]; ai[1] = from[1*stride]; ai[2] = from[2*stride]; @@ -277,7 +342,7 @@ } template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) { - float EIGEN_ALIGN16 af[4]; + EIGEN_ALIGN16 float af[4]; pstore<float>(af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; @@ -286,7 +351,7 @@ } template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) { - int EIGEN_ALIGN16 ai[4]; + EIGEN_ALIGN16 int ai[4]; pstore<int>((int *)ai, from); to[0*stride] = ai[0]; to[1*stride] = ai[1]; @@ -294,58 +359,24 @@ to[3*stride] = ai[3]; } -template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return vec_add(pset1<Packet4f>(a), p4f_COUNTDOWN); } -template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return vec_add(pset1<Packet4i>(a), p4i_COUNTDOWN); } +template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return pset1<Packet4f>(a) + p4f_COUNTDOWN; } +template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return pset1<Packet4i>(a) + p4i_COUNTDOWN; } -template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_add(a,b); } -template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_add(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return a + b; } +template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return a + b; } -template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_sub(a,b); } -template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_sub(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return a - b; } +template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return a - b; } -template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return psub<Packet4f>(p4f_ZERO, a); } -template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return psub<Packet4i>(p4i_ZERO, a); } +template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return p4f_ZERO - a; } +template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return p4i_ZERO - a; } template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } -template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_madd(a,b,p4f_ZERO); } -/* Commented out: it's actually slower than processing it scalar - * -template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) -{ - // Detailed in: http://freevec.org/content/32bit_signed_integer_multiplication_altivec - //Set up constants, variables - Packet4i a1, b1, bswap, low_prod, high_prod, prod, prod_, v1sel; +template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_madd(a,b, p4f_MZERO); } +template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return a * b; } - // Get the absolute values - a1 = vec_abs(a); - b1 = vec_abs(b); - - // Get the signs using xor - Packet4bi sgn = (Packet4bi) vec_cmplt(vec_xor(a, b), p4i_ZERO); - - // Do the multiplication for the asbolute values. - bswap = (Packet4i) vec_rl((Packet4ui) b1, (Packet4ui) p4i_MINUS16 ); - low_prod = vec_mulo((Packet8i) a1, (Packet8i)b1); - high_prod = vec_msum((Packet8i) a1, (Packet8i) bswap, p4i_ZERO); - high_prod = (Packet4i) vec_sl((Packet4ui) high_prod, (Packet4ui) p4i_MINUS16); - prod = vec_add( low_prod, high_prod ); - - // NOR the product and select only the negative elements according to the sign mask - prod_ = vec_nor(prod, prod); - prod_ = vec_sel(p4i_ZERO, prod_, sgn); - - // Add 1 to the result to get the negative numbers - v1sel = vec_sel(p4i_ZERO, p4i_ONE, sgn); - prod_ = vec_add(prod_, v1sel); - - // Merge the results back to the final vector. - prod = vec_sel(prod, prod_, sgn); - - return prod; -} -*/ template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { #ifndef __VSX__ // VSX actually provides a div instruction @@ -358,7 +389,7 @@ t = vec_nmsub(y_0, b, p4f_ONE); y_1 = vec_madd(y_0, t, y_0); - return vec_madd(a, y_1, p4f_ZERO); + return vec_madd(a, y_1, p4f_MZERO); #else return vec_div(a, b); #endif @@ -370,15 +401,44 @@ } // for some weird raisons, it has to be overloaded for packet of integers -template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a, b, c); } -template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); } +template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a,b,c); } +template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return a*b + c; } -template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_min(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + #ifdef __VSX__ + // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN + Packet4f ret; + __asm__ ("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); + return ret; + #else + return vec_min(a, b); + #endif +} template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); } -template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_max(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + #ifdef __VSX__ + // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN + Packet4f ret; + __asm__ ("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); + return ret; + #else + return vec_max(a, b); + #endif +} template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return reinterpret_cast<Packet4f>(vec_cmple(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return reinterpret_cast<Packet4f>(vec_cmplt(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return reinterpret_cast<Packet4f>(vec_cmpeq(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { + Packet4f c = reinterpret_cast<Packet4f>(vec_cmpge(a,b)); + return vec_nor(c,c); +} +template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return reinterpret_cast<Packet4i>(vec_cmpeq(a,b)); } + template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); } @@ -391,6 +451,14 @@ template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, vec_nor(b, b)); } +template<> EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { + return vec_sel(b, a, mask); +} + +template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { return vec_round(a); } +template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return vec_ceil(a); } +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return vec_floor(a); } + #ifdef _BIG_ENDIAN template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { @@ -415,15 +483,15 @@ return (Packet4i) vec_perm(MSQ, LSQ, mask); // align the data } #else -// We also need ot redefine little endian loading of Packet4i/Packet4f using VSX +// We also need to redefine little endian loading of Packet4i/Packet4f using VSX template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { - EIGEN_DEBUG_ALIGNED_LOAD + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4i) vec_vsx_ld((long)from & 15, (const int*) _EIGEN_ALIGNED_PTR(from)); } template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { - EIGEN_DEBUG_ALIGNED_LOAD + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f) vec_vsx_ld((long)from & 15, (const float*) _EIGEN_ALIGNED_PTR(from)); } #endif @@ -431,15 +499,15 @@ template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) { Packet4f p; - if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from); - else p = ploadu<Packet4f>(from); + if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from); + else p = ploadu<Packet4f>(from); return vec_perm(p, p, p16uc_DUPLICATE32_HI); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) { Packet4i p; - if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from); - else p = ploadu<Packet4i>(from); + if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from); + else p = ploadu<Packet4i>(from); return vec_perm(p, p, p16uc_DUPLICATE32_HI); } @@ -481,7 +549,7 @@ vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part } #else -// We also need ot redefine little endian loading of Packet4i/Packet4f using VSX +// We also need to redefine little endian loading of Packet4i/Packet4f using VSX template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE @@ -494,27 +562,43 @@ } #endif -#ifndef __VSX__ -template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); } -template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); } -#endif +template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_PPC_PREFETCH(addr); } +template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { EIGEN_PPC_PREFETCH(addr); } -template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; } -template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; } +template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { EIGEN_ALIGN16 float x; vec_ste(a, 0, &x); return x; } +template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { EIGEN_ALIGN16 int x; vec_ste(a, 0, &x); return x; } -template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return (Packet4f)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE32); } -template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return (Packet4i)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE32); } +template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) +{ + return reinterpret_cast<Packet4f>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32)); +} +template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) +{ + return reinterpret_cast<Packet4i>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32)); } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); } +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftright(Packet4i a) +{ return vec_sr(a,reinterpret_cast<Packet4ui>(pset1<Packet4i>(N))); } +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftleft(Packet4i a) +{ return vec_sl(a,reinterpret_cast<Packet4ui>(pset1<Packet4i>(N))); } + +template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) { + return pfrexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) { + return pldexp_float(a,exponent); +} + template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { Packet4f b, sum; - b = (Packet4f) vec_sld(a, a, 8); - sum = vec_add(a, b); - b = (Packet4f) vec_sld(sum, sum, 4); - sum = vec_add(sum, b); + b = vec_sld(a, a, 8); + sum = a + b; + b = vec_sld(sum, sum, 4); + sum += b; return pfirst(sum); } @@ -537,11 +621,11 @@ // Now do the summation: // Lines 0+1 - sum[0] = vec_add(sum[0], sum[1]); + sum[0] = sum[0] + sum[1]; // Lines 2+3 - sum[1] = vec_add(sum[2], sum[3]); + sum[1] = sum[2] + sum[3]; // Add the results - sum[0] = vec_add(sum[0], sum[1]); + sum[0] = sum[0] + sum[1]; return sum[0]; } @@ -577,11 +661,11 @@ // Now do the summation: // Lines 0+1 - sum[0] = vec_add(sum[0], sum[1]); + sum[0] = sum[0] + sum[1]; // Lines 2+3 - sum[1] = vec_add(sum[2], sum[3]); + sum[1] = sum[2] + sum[3]; // Add the results - sum[0] = vec_add(sum[0], sum[1]); + sum[0] = sum[0] + sum[1]; return sum[0]; } @@ -591,8 +675,8 @@ template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) { Packet4f prod; - prod = pmul(a, (Packet4f)vec_sld(a, a, 8)); - return pfirst(pmul(prod, (Packet4f)vec_sld(prod, prod, 4))); + prod = pmul(a, vec_sld(a, a, 8)); + return pfirst(pmul(prod, vec_sld(prod, prod, 4))); } template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) @@ -636,6 +720,11 @@ return pfirst(res); } +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) +{ + return vec_any_ne(x, pzero(x)); +} + template<int Offset> struct palign_impl<Offset,Packet4f> { @@ -716,33 +805,89 @@ kernel.packet[3] = vec_mergel(t1, t3); } +template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { + Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; + Packet4ui mask = reinterpret_cast<Packet4ui>(vec_cmpeq(reinterpret_cast<Packet4ui>(select), reinterpret_cast<Packet4ui>(p4i_ONE))); + return vec_sel(elsePacket, thenPacket, mask); +} + +template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { + Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; + Packet4ui mask = reinterpret_cast<Packet4ui>(vec_cmpeq(reinterpret_cast<Packet4ui>(select), reinterpret_cast<Packet4ui>(p4i_ONE))); + return vec_sel(elsePacket, thenPacket, mask); +} + + +template <> +struct type_casting_traits<float, int> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template <> +struct type_casting_traits<int, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + + +template<> EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) { + return vec_cts(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) { + return vec_ctf(a,0); +} + +template<> EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i,Packet4f>(const Packet4f& a) { + return reinterpret_cast<Packet4i>(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f,Packet4i>(const Packet4i& a) { + return reinterpret_cast<Packet4f>(a); +} + + //---------- double ---------- #ifdef __VSX__ typedef __vector double Packet2d; typedef __vector unsigned long long Packet2ul; typedef __vector long long Packet2l; - -static Packet2l p2l_ZERO = (Packet2l) p4i_ZERO; -static Packet2d p2d_ONE = { 1.0, 1.0 }; -static Packet2d p2d_ZERO = (Packet2d) p4f_ZERO; -static Packet2d p2d_ZERO_ = { -0.0, -0.0 }; - -#ifdef _BIG_ENDIAN -static Packet2d p2d_COUNTDOWN = (Packet2d) vec_sld((Packet16uc) p2d_ZERO, (Packet16uc) p2d_ONE, 8); +#if EIGEN_COMP_CLANG +typedef Packet2ul Packet2bl; #else -static Packet2d p2d_COUNTDOWN = (Packet2d) vec_sld((Packet16uc) p2d_ONE, (Packet16uc) p2d_ZERO, 8); +typedef __vector __bool long Packet2bl; #endif -static EIGEN_STRONG_INLINE Packet2d vec_splat_dbl(Packet2d& a, int index) +static Packet2l p2l_ONE = { 1, 1 }; +static Packet2l p2l_ZERO = reinterpret_cast<Packet2l>(p4i_ZERO); +static Packet2d p2d_ONE = { 1.0, 1.0 }; +static Packet2d p2d_ZERO = reinterpret_cast<Packet2d>(p4f_ZERO); +static Packet2d p2d_MZERO = { -0.0, -0.0 }; + +#ifdef _BIG_ENDIAN +static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ZERO), reinterpret_cast<Packet4f>(p2d_ONE), 8)); +#else +static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ONE), reinterpret_cast<Packet4f>(p2d_ZERO), 8)); +#endif + +template<int index> Packet2d vec_splat_dbl(Packet2d& a); + +template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<0>(Packet2d& a) { - switch (index) { - case 0: - return (Packet2d) vec_perm(a, a, p16uc_PSET64_HI); - case 1: - return (Packet2d) vec_perm(a, a, p16uc_PSET64_LO); - } - return a; + return reinterpret_cast<Packet2d>(vec_perm(a, a, p16uc_PSET64_HI)); +} + +template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<1>(Packet2d& a) +{ + return reinterpret_cast<Packet2d>(vec_perm(a, a, p16uc_PSET64_LO)); } template<> struct packet_traits<double> : default_packet_traits @@ -753,16 +898,41 @@ Vectorizable = 1, AlignedOnScalar = 1, size=2, - HasHalfPacket = 0, + HasHalfPacket = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, HasDiv = 1, + HasMin = 1, + HasMax = 1, + HasAbs = 1, + HasSin = 0, + HasCos = 0, + HasLog = 0, HasExp = 1, - HasSqrt = 0 + HasSqrt = 1, + HasRsqrt = 1, + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasNegate = 1, + HasBlend = 1 }; }; -template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; +template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2d half; }; +inline std::ostream & operator <<(std::ostream & s, const Packet2l & v) +{ + union { + Packet2l v; + int64_t n[2]; + } vt; + vt.v = v; + s << vt.n[0] << ", " << vt.n[1]; + return s; +} inline std::ostream & operator <<(std::ostream & s, const Packet2d & v) { @@ -776,61 +946,89 @@ } // Need to define them first or we get specialization after instantiation errors -template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d) vec_ld(0, (const float *) from); } //FIXME +template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) +{ + EIGEN_DEBUG_ALIGNED_LOAD +#ifdef __VSX__ + return vec_vsx_ld(0, from); +#else + return vec_ld(0, from); +#endif +} -template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE vec_st((Packet4f)from, 0, (float *)to); } +template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) +{ + EIGEN_DEBUG_ALIGNED_STORE +#ifdef __VSX__ + vec_vsx_st(from, 0, to); +#else + vec_st(from, 0, to); +#endif +} template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { - double EIGEN_ALIGN16 af[2]; - af[0] = from; - Packet2d vc = pload<Packet2d>(af); - vc = vec_splat_dbl(vc, 0); - return vc; + Packet2d v = {from, from}; + return v; } + template<> EIGEN_STRONG_INLINE void pbroadcast4<Packet2d>(const double *a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { a1 = pload<Packet2d>(a); - a0 = vec_splat_dbl(a1, 0); - a1 = vec_splat_dbl(a1, 1); + a0 = vec_splat_dbl<0>(a1); + a1 = vec_splat_dbl<1>(a1); a3 = pload<Packet2d>(a+2); - a2 = vec_splat_dbl(a3, 0); - a3 = vec_splat_dbl(a3, 1); + a2 = vec_splat_dbl<0>(a3); + a3 = vec_splat_dbl<1>(a3); } + template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) { - double EIGEN_ALIGN16 af[2]; + EIGEN_ALIGN16 double af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload<Packet2d>(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) { - double EIGEN_ALIGN16 af[2]; + EIGEN_ALIGN16 double af[2]; pstore<double>(af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } -template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return vec_add(pset1<Packet2d>(a), p2d_COUNTDOWN); } -template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_add(a,b); } +template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return pset1<Packet2d>(a) + p2d_COUNTDOWN; } -template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_sub(a,b); } +template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return a + b; } -template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return psub<Packet2d>(p2d_ZERO, a); } +template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return a - b; } + +template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return p2d_ZERO - a; } template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } -template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_madd(a,b,p2d_ZERO); } +template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_madd(a,b,p2d_MZERO); } template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_div(a,b); } // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); } -template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_min(a, b); } +template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) +{ + // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN + Packet2d ret; + __asm__ ("xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); + return ret; + } -template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_max(a, b); } +template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) +{ + // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN + Packet2d ret; + __asm__ ("xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); + return ret; +} template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); } @@ -840,17 +1038,22 @@ template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); } +template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return vec_round(a); } +template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return vec_ceil(a); } +template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return vec_floor(a); } + template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD - return (Packet2d) vec_vsx_ld((long)from & 15, (const float*) _EIGEN_ALIGNED_PTR(from)); + return (Packet2d) vec_vsx_ld((long)from & 15, (const double*) _EIGEN_ALIGNED_PTR(from)); } + template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) { Packet2d p; - if((ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from); - else p = ploadu<Packet2d>(from); - return vec_perm(p, p, p16uc_PSET64_HI); + if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from); + else p = ploadu<Packet2d>(from); + return vec_splat_dbl<0>(p); } template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) @@ -859,32 +1062,87 @@ vec_vsx_st((Packet4f)from, (long)to & 15, (float*) _EIGEN_ALIGNED_PTR(to)); } -template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { vec_dstt((const float *) addr, DST_CTRL(2,2,32), DST_CHAN); } +template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_PPC_PREFETCH(addr); } -template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; } +template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { EIGEN_ALIGN16 double x[2]; pstore<double>(x, a); return x[0]; } -template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return (Packet2d)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE64); } - +template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) +{ + return reinterpret_cast<Packet2d>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64)); +} template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } +// VSX support varies between different compilers and even different +// versions of the same compiler. For gcc version >= 4.9.3, we can use +// vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use +// a slow version that works with older compilers. +// Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles +// are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963 +static inline Packet2l ConvertToPacket2l(const Packet2d& x) { +#if EIGEN_GNUC_AT_LEAST(5, 4) || \ + (EIGEN_GNUC_AT(6, 1) && __GNUC_PATCHLEVEL__ >= 1) + return vec_cts(x, 0); // TODO: check clang version. +#else + double tmp[2]; + memcpy(tmp, &x, sizeof(tmp)); + Packet2l l = { static_cast<long long>(tmp[0]), + static_cast<long long>(tmp[1]) }; + return l; +#endif +} + +template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { + + // build 2^n + Packet2l emm0 = ConvertToPacket2l(exponent); + +#ifdef __POWER8_VECTOR__ + const Packet2l p2l_1023 = { 1023, 1023 }; + const Packet2ul p2ul_52 = { 52, 52 }; + emm0 = vec_add(emm0, p2l_1023); + emm0 = vec_sl(emm0, p2ul_52); +#else + // Code is a bit complex for POWER7. There is actually a + // vec_xxsldi intrinsic but it is not supported by some gcc versions. + // So we shift (52-32) bits and do a word swap with zeros. + const Packet4i p4i_1023 = pset1<Packet4i>(1023); + const Packet4i p4i_20 = pset1<Packet4i>(20); // 52 - 32 + + Packet4i emm04i = reinterpret_cast<Packet4i>(emm0); + emm04i = vec_add(emm04i, p4i_1023); + emm04i = vec_sl(emm04i, reinterpret_cast<Packet4ui>(p4i_20)); + static const Packet16uc perm = { + 0x14, 0x15, 0x16, 0x17, 0x00, 0x01, 0x02, 0x03, + 0x1c, 0x1d, 0x1e, 0x1f, 0x08, 0x09, 0x0a, 0x0b }; +#ifdef _BIG_ENDIAN + emm0 = reinterpret_cast<Packet2l>(vec_perm(p4i_ZERO, emm04i, perm)); +#else + emm0 = reinterpret_cast<Packet2l>(vec_perm(emm04i, p4i_ZERO, perm)); +#endif + +#endif + + return pmul(a, reinterpret_cast<Packet2d>(emm0)); +} + template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { Packet2d b, sum; - b = (Packet2d) vec_sld((Packet4ui) a, (Packet4ui)a, 8); - sum = vec_add(a, b); - return pfirst(sum); + b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(a), reinterpret_cast<Packet4f>(a), 8)); + sum = a + b; + return pfirst<Packet2d>(sum); } template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) { Packet2d v[2], sum; - v[0] = vec_add(vecs[0], (Packet2d) vec_sld((Packet4ui) vecs[0], (Packet4ui) vecs[0], 8)); - v[1] = vec_add(vecs[1], (Packet2d) vec_sld((Packet4ui) vecs[1], (Packet4ui) vecs[1], 8)); - + v[0] = vecs[0] + reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(vecs[0]), reinterpret_cast<Packet4f>(vecs[0]), 8)); + v[1] = vecs[1] + reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(vecs[1]), reinterpret_cast<Packet4f>(vecs[1]), 8)); + #ifdef _BIG_ENDIAN - sum = (Packet2d) vec_sld((Packet4ui) v[0], (Packet4ui) v[1], 8); + sum = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(v[0]), reinterpret_cast<Packet4f>(v[1]), 8)); #else - sum = (Packet2d) vec_sld((Packet4ui) v[1], (Packet4ui) v[0], 8); + sum = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(v[1]), reinterpret_cast<Packet4f>(v[0]), 8)); #endif return sum; @@ -893,19 +1151,19 @@ // mul template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) { - return pfirst(pmul(a, (Packet2d)vec_sld((Packet4ui) a, (Packet4ui) a, 8))); + return pfirst(pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8)))); } // min template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) { - return pfirst(vec_min(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8))); + return pfirst(pmin(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8)))); } // max template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) { - return pfirst(vec_max(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8))); + return pfirst(pmax(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8)))); } template<int Offset> @@ -915,9 +1173,9 @@ { if (Offset == 1) #ifdef _BIG_ENDIAN - first = (Packet2d) vec_sld((Packet4ui) first, (Packet4ui) second, 8); + first = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(first), reinterpret_cast<Packet4ui>(second), 8)); #else - first = (Packet2d) vec_sld((Packet4ui) second, (Packet4ui) first, 8); + first = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(second), reinterpret_cast<Packet4ui>(first), 8)); #endif } }; @@ -931,6 +1189,11 @@ kernel.packet[1] = t1; } +template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { + Packet2l select = { ifPacket.select[0], ifPacket.select[1] }; + Packet2bl mask = reinterpret_cast<Packet2bl>( vec_cmpeq(reinterpret_cast<Packet2d>(select), reinterpret_cast<Packet2d>(p2l_ONE)) ); + return vec_sel(elsePacket, thenPacket, mask); +} #endif // __VSX__ } // end namespace internal
diff --git a/Eigen/src/Core/arch/CMakeLists.txt b/Eigen/src/Core/arch/CMakeLists.txt deleted file mode 100644 index 42b0b48..0000000 --- a/Eigen/src/Core/arch/CMakeLists.txt +++ /dev/null
@@ -1,9 +0,0 @@ -ADD_SUBDIRECTORY(AltiVec) -ADD_SUBDIRECTORY(AVX) -ADD_SUBDIRECTORY(CUDA) -ADD_SUBDIRECTORY(Default) -ADD_SUBDIRECTORY(NEON) -ADD_SUBDIRECTORY(SSE) - - -
diff --git a/Eigen/src/Core/arch/CUDA/CMakeLists.txt b/Eigen/src/Core/arch/CUDA/CMakeLists.txt deleted file mode 100644 index 7ba28da..0000000 --- a/Eigen/src/Core/arch/CUDA/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_CUDA_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_CUDA_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/CUDA COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/CUDA/Complex.h b/Eigen/src/Core/arch/CUDA/Complex.h new file mode 100644 index 0000000..57d1201 --- /dev/null +++ b/Eigen/src/Core/arch/CUDA/Complex.h
@@ -0,0 +1,103 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_COMPLEX_CUDA_H +#define EIGEN_COMPLEX_CUDA_H + +// clang-format off + +namespace Eigen { + +namespace internal { + +#if defined(EIGEN_CUDACC) && defined(EIGEN_USE_GPU) + +// Many std::complex methods such as operator+, operator-, operator* and +// operator/ are not constexpr. Due to this, clang does not treat them as device +// functions and thus Eigen functors making use of these operators fail to +// compile. Here, we manually specialize these functors for complex types when +// building for CUDA to avoid non-constexpr methods. + +// Sum +template<typename T> struct scalar_sum_op<const std::complex<T>, const std::complex<T> > : binary_op_base<const std::complex<T>, const std::complex<T> > { + typedef typename std::complex<T> result_type; + + EIGEN_EMPTY_STRUCT_CTOR(scalar_sum_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> operator() (const std::complex<T>& a, const std::complex<T>& b) const { + return std::complex<T>(numext::real(a) + numext::real(b), + numext::imag(a) + numext::imag(b)); + } +}; + +template<typename T> struct scalar_sum_op<std::complex<T>, std::complex<T> > : scalar_sum_op<const std::complex<T>, const std::complex<T> > {}; + + +// Difference +template<typename T> struct scalar_difference_op<const std::complex<T>, const std::complex<T> > : binary_op_base<const std::complex<T>, const std::complex<T> > { + typedef typename std::complex<T> result_type; + + EIGEN_EMPTY_STRUCT_CTOR(scalar_difference_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> operator() (const std::complex<T>& a, const std::complex<T>& b) const { + return std::complex<T>(numext::real(a) - numext::real(b), + numext::imag(a) - numext::imag(b)); + } +}; + +template<typename T> struct scalar_difference_op<std::complex<T>, std::complex<T> > : scalar_difference_op<const std::complex<T>, const std::complex<T> > {}; + + +// Product +template<typename T> struct scalar_product_op<const std::complex<T>, const std::complex<T> > : binary_op_base<const std::complex<T>, const std::complex<T> > { + enum { + Vectorizable = packet_traits<std::complex<T> >::HasMul + }; + typedef typename std::complex<T> result_type; + + EIGEN_EMPTY_STRUCT_CTOR(scalar_product_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> operator() (const std::complex<T>& a, const std::complex<T>& b) const { + const T a_real = numext::real(a); + const T a_imag = numext::imag(a); + const T b_real = numext::real(b); + const T b_imag = numext::imag(b); + return std::complex<T>(a_real * b_real - a_imag * b_imag, + a_real * b_imag + a_imag * b_real); + } +}; + +template<typename T> struct scalar_product_op<std::complex<T>, std::complex<T> > : scalar_product_op<const std::complex<T>, const std::complex<T> > {}; + + +// Quotient +template<typename T> struct scalar_quotient_op<const std::complex<T>, const std::complex<T> > : binary_op_base<const std::complex<T>, const std::complex<T> > { + enum { + Vectorizable = packet_traits<std::complex<T> >::HasDiv + }; + typedef typename std::complex<T> result_type; + + EIGEN_EMPTY_STRUCT_CTOR(scalar_quotient_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> operator() (const std::complex<T>& a, const std::complex<T>& b) const { + const T a_real = numext::real(a); + const T a_imag = numext::imag(a); + const T b_real = numext::real(b); + const T b_imag = numext::imag(b); + const T norm = T(1) / (b_real * b_real + b_imag * b_imag); + return std::complex<T>((a_real * b_real + a_imag * b_imag) * norm, + (a_imag * b_real - a_real * b_imag) * norm); + } +}; + +template<typename T> struct scalar_quotient_op<std::complex<T>, std::complex<T> > : scalar_quotient_op<const std::complex<T>, const std::complex<T> > {}; + +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_COMPLEX_CUDA_H
diff --git a/Eigen/src/Core/arch/CUDA/Half.h b/Eigen/src/Core/arch/CUDA/Half.h deleted file mode 100644 index 9ecc4fd..0000000 --- a/Eigen/src/Core/arch/CUDA/Half.h +++ /dev/null
@@ -1,546 +0,0 @@ -// Standard 16-bit float type, mostly useful for GPUs. Defines a new -// class Eigen::half (inheriting from CUDA's __half struct) with -// operator overloads such that it behaves basically as an arithmetic -// type. It will be quite slow on CPUs (so it is recommended to stay -// in fp32 for CPUs, except for simple parameter conversions, I/O -// to disk and the likes), but fast on GPUs. -// -// -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// The conversion routines are Copyright (c) Fabian Giesen, 2016. -// The original license follows: -// -// Copyright (c) Fabian Giesen, 2016 -// All rights reserved. -// Redistribution and use in source and binary forms, with or without -// modification, are permitted. -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef EIGEN_HALF_CUDA_H -#define EIGEN_HALF_CUDA_H - -#if __cplusplus > 199711L -#define EIGEN_EXPLICIT_CAST(tgt_type) explicit operator tgt_type() -#else -#define EIGEN_EXPLICIT_CAST(tgt_type) operator tgt_type() -#endif - - -#if !defined(EIGEN_HAS_CUDA_FP16) - -// Make our own __half definition that is similar to CUDA's. -struct __half { - unsigned short x; -}; - -#endif - -namespace Eigen { - -namespace internal { - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half raw_uint16_to_half(unsigned short x); -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half float_to_half_rtne(float ff); -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half h); - -} // end namespace internal - -// Class definition. -struct half : public __half { - EIGEN_DEVICE_FUNC half() {} - - EIGEN_DEVICE_FUNC half(const __half& h) : __half(h) {} - EIGEN_DEVICE_FUNC half(const half& h) : __half(h) {} - - explicit EIGEN_DEVICE_FUNC half(bool b) - : __half(internal::raw_uint16_to_half(b ? 0x3c00 : 0)) {} - explicit EIGEN_DEVICE_FUNC half(unsigned int ui) - : __half(internal::float_to_half_rtne(static_cast<float>(ui))) {} - explicit EIGEN_DEVICE_FUNC half(int i) - : __half(internal::float_to_half_rtne(static_cast<float>(i))) {} - explicit EIGEN_DEVICE_FUNC half(unsigned long ul) - : __half(internal::float_to_half_rtne(static_cast<float>(ul))) {} - explicit EIGEN_DEVICE_FUNC half(long l) - : __half(internal::float_to_half_rtne(static_cast<float>(l))) {} - explicit EIGEN_DEVICE_FUNC half(long long ll) - : __half(internal::float_to_half_rtne(static_cast<float>(ll))) {} - explicit EIGEN_DEVICE_FUNC half(unsigned long long ull) - : __half(internal::float_to_half_rtne(static_cast<float>(ull))) {} - explicit EIGEN_DEVICE_FUNC half(float f) - : __half(internal::float_to_half_rtne(f)) {} - explicit EIGEN_DEVICE_FUNC half(double d) - : __half(internal::float_to_half_rtne(static_cast<float>(d))) {} - - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(bool) const { - // +0.0 and -0.0 become false, everything else becomes true. - return (x & 0x7fff) != 0; - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(signed char) const { - return static_cast<signed char>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned char) const { - return static_cast<unsigned char>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(short) const { - return static_cast<short>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned short) const { - return static_cast<unsigned short>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(int) const { - return static_cast<int>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned int) const { - return static_cast<unsigned int>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(long) const { - return static_cast<long>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned long) const { - return static_cast<unsigned long>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(long long) const { - return static_cast<long long>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned long long) const { - return static_cast<unsigned long long>(internal::half_to_float(*this)); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(float) const { - return internal::half_to_float(*this); - } - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(double) const { - return static_cast<double>(internal::half_to_float(*this)); - } - - EIGEN_DEVICE_FUNC half& operator=(const half& other) { - x = other.x; - return *this; - } -}; - -#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 - -// Intrinsics for native fp16 support. Note that on current hardware, -// these are no faster than fp32 arithmetic (you need to use the half2 -// versions to get the ALU speed increased), but you do save the -// conversion steps back and forth. - -__device__ half operator + (const half& a, const half& b) { - return __hadd(a, b); -} -__device__ half operator * (const half& a, const half& b) { - return __hmul(a, b); -} -__device__ half operator - (const half& a, const half& b) { - return __hsub(a, b); -} -__device__ half operator / (const half& a, const half& b) { - float num = __half2float(a); - float denom = __half2float(b); - return __float2half(num / denom); -} -__device__ half operator - (const half& a) { - return __hneg(a); -} -__device__ half& operator += (half& a, const half& b) { - a = a + b; - return a; -} -__device__ half& operator *= (half& a, const half& b) { - a = a * b; - return a; -} -__device__ half& operator -= (half& a, const half& b) { - a = a - b; - return a; -} -__device__ half& operator /= (half& a, const half& b) { - a = a / b; - return a; -} -__device__ bool operator == (const half& a, const half& b) { - return __heq(a, b); -} -__device__ bool operator != (const half& a, const half& b) { - return __hne(a, b); -} -__device__ bool operator < (const half& a, const half& b) { - return __hlt(a, b); -} -__device__ bool operator <= (const half& a, const half& b) { - return __hle(a, b); -} -__device__ bool operator > (const half& a, const half& b) { - return __hgt(a, b); -} -__device__ bool operator >= (const half& a, const half& b) { - return __hge(a, b); -} - -#else // Emulate support for half floats - -// Definitions for CPUs and older CUDA, mostly working through conversion -// to/from fp32. - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (const half& a, const half& b) { - return half(float(a) + float(b)); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator * (const half& a, const half& b) { - return half(float(a) * float(b)); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a, const half& b) { - return half(float(a) - float(b)); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, const half& b) { - return half(float(a) / float(b)); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a) { - half result; - result.x = a.x ^ 0x8000; - return result; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator += (half& a, const half& b) { - a = half(float(a) + float(b)); - return a; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator *= (half& a, const half& b) { - a = half(float(a) * float(b)); - return a; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator -= (half& a, const half& b) { - a = half(float(a) - float(b)); - return a; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator /= (half& a, const half& b) { - a = half(float(a) / float(b)); - return a; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator == (const half& a, const half& b) { - return float(a) == float(b); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator != (const half& a, const half& b) { - return float(a) != float(b); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator < (const half& a, const half& b) { - return float(a) < float(b); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator <= (const half& a, const half& b) { - return float(a) <= float(b); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator > (const half& a, const half& b) { - return float(a) > float(b); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const half& b) { - return float(a) >= float(b); -} - -#endif // Emulate support for half floats - -// Division by an index. Do it in full float precision to avoid accuracy -// issues in converting the denominator to half. -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, Index b) { - return Eigen::half(static_cast<float>(a) / static_cast<float>(b)); -} - -// Conversion routines, including fallbacks for the host or older CUDA. -// Note that newer Intel CPUs (Haswell or newer) have vectorized versions of -// these in hardware. If we need more performance on older/other CPUs, they are -// also possible to vectorize directly. - -namespace internal { - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half raw_uint16_to_half(unsigned short x) { - __half h; - h.x = x; - return h; -} - -union FP32 { - unsigned int u; - float f; -}; - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half float_to_half_rtne(float ff) { -#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 - return __float2half(ff); - -#elif defined(EIGEN_HAS_FP16_C) - __half h; - h.x = _cvtss_sh(ff, 0); - return h; - -#else - FP32 f; f.f = ff; - - const FP32 f32infty = { 255 << 23 }; - const FP32 f16max = { (127 + 16) << 23 }; - const FP32 denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 }; - unsigned int sign_mask = 0x80000000u; - __half o = { 0 }; - - unsigned int sign = f.u & sign_mask; - f.u ^= sign; - - // NOTE all the integer compares in this function can be safely - // compiled into signed compares since all operands are below - // 0x80000000. Important if you want fast straight SSE2 code - // (since there's no unsigned PCMPGTD). - - if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) - o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf - } else { // (De)normalized number or zero - if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero - // use a magic value to align our 10 mantissa bits at the bottom of - // the float. as long as FP addition is round-to-nearest-even this - // just works. - f.f += denorm_magic.f; - - // and one integer subtract of the bias later, we have our final float! - o.x = static_cast<unsigned short>(f.u - denorm_magic.u); - } else { - unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd - - // update exponent, rounding bias part 1 - f.u += ((unsigned int)(15 - 127) << 23) + 0xfff; - // rounding bias part 2 - f.u += mant_odd; - // take the bits! - o.x = static_cast<unsigned short>(f.u >> 13); - } - } - - o.x |= static_cast<unsigned short>(sign >> 16); - return o; -#endif -} - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half h) { -#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 - return __half2float(h); - -#elif defined(EIGEN_HAS_FP16_C) - return _cvtsh_ss(h.x); - -#else - const FP32 magic = { 113 << 23 }; - const unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift - FP32 o; - - o.u = (h.x & 0x7fff) << 13; // exponent/mantissa bits - unsigned int exp = shifted_exp & o.u; // just the exponent - o.u += (127 - 15) << 23; // exponent adjust - - // handle exponent special cases - if (exp == shifted_exp) { // Inf/NaN? - o.u += (128 - 16) << 23; // extra exp adjust - } else if (exp == 0) { // Zero/Denormal? - o.u += 1 << 23; // extra exp adjust - o.f -= magic.f; // renormalize - } - - o.u |= (h.x & 0x8000) << 16; // sign bit - return o.f; -#endif -} - -} // end namespace internal - -// Traits. - -namespace internal { - -template<> struct is_arithmetic<half> { enum { value = true }; }; - -} // end namespace internal - -template<> struct NumTraits<Eigen::half> - : GenericNumTraits<Eigen::half> -{ - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half epsilon() { - return internal::raw_uint16_to_half(0x0800); - } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half dummy_precision() { return half(1e-3f); } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half highest() { - return internal::raw_uint16_to_half(0x7bff); - } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half lowest() { - return internal::raw_uint16_to_half(0xfbff); - } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half infinity() { - return internal::raw_uint16_to_half(0x7c00); - } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half quiet_NaN() { - return internal::raw_uint16_to_half(0x7c01); - } -}; - -// Infinity/NaN checks. - -namespace numext { - -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(const Eigen::half& a) { - return (a.x & 0x7fff) == 0x7c00; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(const Eigen::half& a) { -#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 - return __hisnan(a); -#else - return (a.x & 0x7fff) > 0x7c00; -#endif -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(const Eigen::half& a) { - return !(Eigen::numext::isinf)(a) && !(Eigen::numext::isnan)(a); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half abs(const Eigen::half& a) { - Eigen::half result; - result.x = a.x & 0x7FFF; - return result; -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half exp(const Eigen::half& a) { - return Eigen::half(::expf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half log(const Eigen::half& a) { - return Eigen::half(::logf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half sqrt(const Eigen::half& a) { - return Eigen::half(::sqrtf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half pow(const Eigen::half& a, const Eigen::half& b) { - return Eigen::half(::powf(float(a), float(b))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half sin(const Eigen::half& a) { - return Eigen::half(::sinf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half cos(const Eigen::half& a) { - return Eigen::half(::cosf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half tan(const Eigen::half& a) { - return Eigen::half(::tanf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half tanh(const Eigen::half& a) { - return Eigen::half(::tanhf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half floor(const Eigen::half& a) { - return Eigen::half(::floorf(float(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half ceil(const Eigen::half& a) { - return Eigen::half(::ceilf(float(a))); -} - -#ifdef EIGEN_HAS_C99_MATH -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half lgamma(const Eigen::half& a) { - return Eigen::half(Eigen::numext::lgamma(static_cast<float>(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half digamma(const Eigen::half& a) { - return Eigen::half(Eigen::numext::digamma(static_cast<float>(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half zeta(const Eigen::half& x, const Eigen::half& q) { - return Eigen::half(Eigen::numext::zeta(static_cast<float>(x), static_cast<float>(q))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half polygamma(const Eigen::half& n, const Eigen::half& x) { - return Eigen::half(Eigen::numext::polygamma(static_cast<float>(n), static_cast<float>(x))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half erf(const Eigen::half& a) { - return Eigen::half(Eigen::numext::erf(static_cast<float>(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half erfc(const Eigen::half& a) { - return Eigen::half(Eigen::numext::erfc(static_cast<float>(a))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half igamma(const Eigen::half& a, const Eigen::half& x) { - return Eigen::half(Eigen::numext::igamma(static_cast<float>(a), static_cast<float>(x))); -} -template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half igammac(const Eigen::half& a, const Eigen::half& x) { - return Eigen::half(Eigen::numext::igammac(static_cast<float>(a), static_cast<float>(x))); -} -#endif -} // end namespace numext - -} // end namespace Eigen - -// Standard mathematical functions and trancendentals. -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half fabsh(const Eigen::half& a) { - Eigen::half result; - result.x = a.x & 0x7FFF; - return result; -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half exph(const Eigen::half& a) { - return Eigen::half(::expf(float(a))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half logh(const Eigen::half& a) { - return Eigen::half(::logf(float(a))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half sqrth(const Eigen::half& a) { - return Eigen::half(::sqrtf(float(a))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half powh(const Eigen::half& a, const Eigen::half& b) { - return Eigen::half(::powf(float(a), float(b))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half floorh(const Eigen::half& a) { - return Eigen::half(::floorf(float(a))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half ceilh(const Eigen::half& a) { - return Eigen::half(::ceilf(float(a))); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int (isnan)(const Eigen::half& a) { - return (Eigen::numext::isnan)(a); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int (isinf)(const Eigen::half& a) { - return (Eigen::numext::isinf)(a); -} -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int (isfinite)(const Eigen::half& a) { - return !(Eigen::numext::isinf)(a) && !(Eigen::numext::isnan)(a); -} - - -namespace std { - -EIGEN_ALWAYS_INLINE ostream& operator << (ostream& os, const Eigen::half& v) { - os << static_cast<float>(v); - return os; -} - -#if __cplusplus > 199711L -template <> -struct hash<Eigen::half> { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::half& a) const { - return static_cast<std::size_t>(a.x); - } -}; -#endif - -} // end namespace std - - -// Add the missing shfl_xor intrinsic -#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 -__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor(Eigen::half var, int laneMask, int width=warpSize) { - return static_cast<Eigen::half>(__shfl_xor(static_cast<float>(var), laneMask, width)); -} -#endif - -// ldg() has an overload for __half, but we also need one for Eigen::half. -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 320 -static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half __ldg(const Eigen::half* ptr) { - return Eigen::internal::raw_uint16_to_half( - __ldg(reinterpret_cast<const unsigned short*>(ptr))); -} -#endif - - -#endif // EIGEN_HALF_CUDA_H
diff --git a/Eigen/src/Core/arch/CUDA/MathFunctions.h b/Eigen/src/Core/arch/CUDA/MathFunctions.h deleted file mode 100644 index 317499b..0000000 --- a/Eigen/src/Core/arch/CUDA/MathFunctions.h +++ /dev/null
@@ -1,190 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_MATH_FUNCTIONS_CUDA_H -#define EIGEN_MATH_FUNCTIONS_CUDA_H - -namespace Eigen { - -namespace internal { - -// Make sure this is only available when targeting a GPU: we don't want to -// introduce conflicts between these packet_traits definitions and the ones -// we'll use on the host side (SSE, AVX, ...) -#if defined(__CUDACC__) && defined(EIGEN_USE_GPU) -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 plog<float4>(const float4& a) -{ - return make_float4(logf(a.x), logf(a.y), logf(a.z), logf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 plog<double2>(const double2& a) -{ - return make_double2(log(a.x), log(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 pexp<float4>(const float4& a) -{ - return make_float4(expf(a.x), expf(a.y), expf(a.z), expf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 pexp<double2>(const double2& a) -{ - return make_double2(exp(a.x), exp(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 psqrt<float4>(const float4& a) -{ - return make_float4(sqrtf(a.x), sqrtf(a.y), sqrtf(a.z), sqrtf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 psqrt<double2>(const double2& a) -{ - return make_double2(sqrt(a.x), sqrt(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 prsqrt<float4>(const float4& a) -{ - return make_float4(rsqrtf(a.x), rsqrtf(a.y), rsqrtf(a.z), rsqrtf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 prsqrt<double2>(const double2& a) -{ - return make_double2(rsqrt(a.x), rsqrt(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 plgamma<float4>(const float4& a) -{ - return make_float4(lgammaf(a.x), lgammaf(a.y), lgammaf(a.z), lgammaf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 plgamma<double2>(const double2& a) -{ - return make_double2(lgamma(a.x), lgamma(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 pdigamma<float4>(const float4& a) -{ - using numext::digamma; - return make_float4(digamma(a.x), digamma(a.y), digamma(a.z), digamma(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 pdigamma<double2>(const double2& a) -{ - using numext::digamma; - return make_double2(digamma(a.x), digamma(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 pzeta<float4>(const float4& x, const float4& q) -{ - using numext::zeta; - return make_float4(zeta(x.x, q.x), zeta(x.y, q.y), zeta(x.z, q.z), zeta(x.w, q.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 pzeta<double2>(const double2& x, const double2& q) -{ - using numext::zeta; - return make_double2(zeta(x.x, q.x), zeta(x.y, q.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 ppolygamma<float4>(const float4& n, const float4& x) -{ - using numext::polygamma; - return make_float4(polygamma(n.x, x.x), polygamma(n.y, x.y), polygamma(n.z, x.z), polygamma(n.w, x.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 ppolygamma<double2>(const double2& n, const double2& x) -{ - using numext::polygamma; - return make_double2(polygamma(n.x, x.x), polygamma(n.y, x.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 perf<float4>(const float4& a) -{ - return make_float4(erf(a.x), erf(a.y), erf(a.z), erf(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 perf<double2>(const double2& a) -{ - return make_double2(erf(a.x), erf(a.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 perfc<float4>(const float4& a) -{ - return make_float4(erfc(a.x), erfc(a.y), erfc(a.z), erfc(a.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 perfc<double2>(const double2& a) -{ - return make_double2(erfc(a.x), erfc(a.y)); -} - - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 pigamma<float4>(const float4& a, const float4& x) -{ - using numext::igamma; - return make_float4( - igamma(a.x, x.x), - igamma(a.y, x.y), - igamma(a.z, x.z), - igamma(a.w, x.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 pigamma<double2>(const double2& a, const double2& x) -{ - using numext::igamma; - return make_double2(igamma(a.x, x.x), igamma(a.y, x.y)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -float4 pigammac<float4>(const float4& a, const float4& x) -{ - using numext::igammac; - return make_float4( - igammac(a.x, x.x), - igammac(a.y, x.y), - igammac(a.z, x.z), - igammac(a.w, x.w)); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -double2 pigammac<double2>(const double2& a, const double2& x) -{ - using numext::igammac; - return make_double2(igammac(a.x, x.x), igammac(a.y, x.y)); -} - -#endif - -} // end namespace internal - -} // end namespace Eigen - -#endif // EIGEN_MATH_FUNCTIONS_CUDA_H
diff --git a/Eigen/src/Core/arch/CUDA/PacketMathHalf.h b/Eigen/src/Core/arch/CUDA/PacketMathHalf.h deleted file mode 100644 index 61d532e..0000000 --- a/Eigen/src/Core/arch/CUDA/PacketMathHalf.h +++ /dev/null
@@ -1,192 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_PACKET_MATH_HALF_CUDA_H -#define EIGEN_PACKET_MATH_HALF_CUDA_H - -#if defined(EIGEN_HAS_CUDA_FP16) - -// Make sure this is only available when targeting a GPU: we don't want to -// introduce conflicts between these packet_traits definitions and the ones -// we'll use on the host side (SSE, AVX, ...) -#if defined(__CUDACC__) && defined(EIGEN_USE_GPU) - -// Most of the following operations require arch >= 5.3 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 - -namespace Eigen { -namespace internal { - -template<> struct is_arithmetic<half2> { enum { value = true }; }; - -template<> struct packet_traits<half> : default_packet_traits -{ - typedef half2 type; - typedef half2 half; - enum { - Vectorizable = 1, - AlignedOnScalar = 1, - size=2, - HasHalfPacket = 0, - HasDiv = 1 - }; -}; - - -template<> struct unpacket_traits<half2> { typedef half type; enum {size=2, alignment=Aligned16}; typedef half2 half; }; - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pset1<half2>(const half& from) { - return __half2half2(from); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pload<half2>(const half* from) { - return *reinterpret_cast<const half2*>(from); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploadu<half2>(const half* from) { - return __halves2half2(from[0], from[1]); -} - -template<> EIGEN_STRONG_INLINE half2 ploaddup<half2>(const half* from) { - return __halves2half2(from[0], from[0]); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<half>(half* to, const half2& from) { - *reinterpret_cast<half2*>(to) = from; -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<half>(half* to, const half2& from) { - to[0] = __low2half(from); - to[1] = __high2half(from); -} - -template<> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(const half* from) { - return __ldg((const half2*)from); -} - -template<> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(const half* from) { - return __halves2half2(__ldg(from+0), __ldg(from+1)); -} - -template<> EIGEN_DEVICE_FUNC inline half2 pgather<half, half2>(const half* from, Index stride) { - return __halves2half2(from[0*stride], from[1*stride]); -} - -template<> EIGEN_DEVICE_FUNC inline void pscatter<half, half2>(half* to, const half2& from, Index stride) { - to[stride*0] = __low2half(from); - to[stride*1] = __high2half(from); -} - -template<> EIGEN_DEVICE_FUNC inline half pfirst<half2>(const half2& a) { - return __low2half(a); -} - -template<> EIGEN_DEVICE_FUNC inline half2 pabs<half2>(const half2& a) { - half2 result; - result.x = a.x & 0x7FFF7FFF; - return result; -} - - -EIGEN_DEVICE_FUNC inline void -ptranspose(PacketBlock<half2,2>& kernel) { - half a1 = __low2half(kernel.packet[0]); - half a2 = __high2half(kernel.packet[0]); - half b1 = __low2half(kernel.packet[1]); - half b2 = __high2half(kernel.packet[1]); - kernel.packet[0] = __halves2half2(a1, b1); - kernel.packet[1] = __halves2half2(a2, b2); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plset<half2>(const half& a) { - return __halves2half2(a, __hadd(a, __float2half(1.0f))); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd<half2>(const half2& a, const half2& b) { - return __hadd2(a, b); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psub<half2>(const half2& a, const half2& b) { - return __hsub2(a, b); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pnegate(const half2& a) { - return __hneg2(a); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pconj(const half2& a) { return a; } - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul<half2>(const half2& a, const half2& b) { - return __hmul2(a, b); -} - - template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmadd<half2>(const half2& a, const half2& b, const half2& c) { - return __hfma2(a, b, c); - } - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv<half2>(const half2& a, const half2& b) { - float a1 = __low2float(a); - float a2 = __high2float(a); - float b1 = __low2float(b); - float b2 = __high2float(b); - float r1 = a1 / b1; - float r2 = a2 / b2; - return __floats2half2_rn(r1, r2); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin<half2>(const half2& a, const half2& b) { - float a1 = __low2float(a); - float a2 = __high2float(a); - float b1 = __low2float(b); - float b2 = __high2float(b); - half r1 = a1 < b1 ? __low2half(a) : __low2half(b); - half r2 = a2 < b2 ? __high2half(a) : __high2half(b); - return __halves2half2(r1, r2); -} - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax<half2>(const half2& a, const half2& b) { - float a1 = __low2float(a); - float a2 = __high2float(a); - float b1 = __low2float(b); - float b2 = __high2float(b); - half r1 = a1 > b1 ? __low2half(a) : __low2half(b); - half r2 = a2 > b2 ? __high2half(a) : __high2half(b); - return __halves2half2(r1, r2); -} - -template<> EIGEN_DEVICE_FUNC inline half predux<half2>(const half2& a) { - return __hadd(__low2half(a), __high2half(a)); -} - -template<> EIGEN_DEVICE_FUNC inline half predux_max<half2>(const half2& a) { - half first = __low2half(a); - half second = __high2half(a); - return __hgt(first, second) ? first : second; -} - -template<> EIGEN_DEVICE_FUNC inline half predux_min<half2>(const half2& a) { - half first = __low2half(a); - half second = __high2half(a); - return __hlt(first, second) ? first : second; -} - -template<> EIGEN_DEVICE_FUNC inline half predux_mul<half2>(const half2& a) { - return __hmul(__low2half(a), __high2half(a)); -} - -} // end namespace internal - -} // end namespace Eigen - -#endif -#endif -#endif -#endif // EIGEN_PACKET_MATH_HALF_CUDA_H
diff --git a/Eigen/src/Core/arch/CUDA/TypeCasting.h b/Eigen/src/Core/arch/CUDA/TypeCasting.h deleted file mode 100644 index 396b38e..0000000 --- a/Eigen/src/Core/arch/CUDA/TypeCasting.h +++ /dev/null
@@ -1,112 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_TYPE_CASTING_CUDA_H -#define EIGEN_TYPE_CASTING_CUDA_H - -namespace Eigen { - -namespace internal { - -#if defined(EIGEN_HAS_CUDA_FP16) - -template<> -struct scalar_cast_op<float, half> { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef half result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half operator() (const float& a) const { - #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 - return __float2half(a); - #else - return half(a); - #endif - } -}; - -template<> -struct functor_traits<scalar_cast_op<float, half> > -{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op<int, half> { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef half result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half operator() (const int& a) const { - #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 - return __float2half(static_cast<float>(a)); - #else - return half(static_cast<float>(a)); - #endif - } -}; - -template<> -struct functor_traits<scalar_cast_op<int, half> > -{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; - - -template<> -struct scalar_cast_op<half, float> { - EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) - typedef float result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const half& a) const { - #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 - return __half2float(a); - #else - return static_cast<float>(a); - #endif - } -}; - -template<> -struct functor_traits<scalar_cast_op<half, float> > -{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; - - - -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 - -template <> -struct type_casting_traits<half, float> { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 2, - TgtCoeffRatio = 1 - }; -}; - -template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast<half2, float4>(const half2& a, const half2& b) { - float2 r1 = __half22float2(a); - float2 r2 = __half22float2(b); - return make_float4(r1.x, r1.y, r2.x, r2.y); -} - -template <> -struct type_casting_traits<float, half> { - enum { - VectorizedCast = 1, - SrcCoeffRatio = 1, - TgtCoeffRatio = 2 - }; -}; - -template<> EIGEN_STRONG_INLINE half2 pcast<float4, half2>(const float4& a) { - // Simply discard the second half of the input - return __float22half2_rn(make_float2(a.x, a.y)); -} - -#endif -#endif - -} // end namespace internal - -} // end namespace Eigen - -#endif // EIGEN_TYPE_CASTING_CUDA_H
diff --git a/Eigen/src/Core/arch/Default/CMakeLists.txt b/Eigen/src/Core/arch/Default/CMakeLists.txt deleted file mode 100644 index 339c091..0000000 --- a/Eigen/src/Core/arch/Default/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_Default_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_Default_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/Default COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/Default/ConjHelper.h b/Eigen/src/Core/arch/Default/ConjHelper.h new file mode 100644 index 0000000..4cfe34e --- /dev/null +++ b/Eigen/src/Core/arch/Default/ConjHelper.h
@@ -0,0 +1,29 @@ + +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_ARCH_CONJ_HELPER_H +#define EIGEN_ARCH_CONJ_HELPER_H + +#define EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(PACKET_CPLX, PACKET_REAL) \ + template<> struct conj_helper<PACKET_REAL, PACKET_CPLX, false,false> { \ + EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_REAL& x, const PACKET_CPLX& y, const PACKET_CPLX& c) const \ + { return padd(c, pmul(x,y)); } \ + EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_REAL& x, const PACKET_CPLX& y) const \ + { return PACKET_CPLX(Eigen::internal::pmul<PACKET_REAL>(x, y.v)); } \ + }; \ + \ + template<> struct conj_helper<PACKET_CPLX, PACKET_REAL, false,false> { \ + EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_CPLX& x, const PACKET_REAL& y, const PACKET_CPLX& c) const \ + { return padd(c, pmul(x,y)); } \ + EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_CPLX& x, const PACKET_REAL& y) const \ + { return PACKET_CPLX(Eigen::internal::pmul<PACKET_REAL>(x.v, y)); } \ + }; + +#endif // EIGEN_ARCH_CONJ_HELPER_H
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h new file mode 100644 index 0000000..693dd55 --- /dev/null +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
@@ -0,0 +1,458 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2007 Julien Pommier +// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) +// Copyright (C) 2009-2019 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/* The exp and log functions of this file initially come from + * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ + */ + +namespace Eigen { +namespace internal { + +template<typename Packet> EIGEN_STRONG_INLINE Packet +pfrexp_float(const Packet& a, Packet& exponent) { + typedef typename unpacket_traits<Packet>::integer_packet PacketI; + const Packet cst_126f = pset1<Packet>(126.0f); + const Packet cst_half = pset1<Packet>(0.5f); + const Packet cst_inv_mant_mask = pset1frombits<Packet>(~0x7f800000u); + exponent = psub(pcast<PacketI,Packet>(pshiftright<23>(preinterpret<PacketI>(a))), cst_126f); + return por(pand(a, cst_inv_mant_mask), cst_half); +} + +template<typename Packet> EIGEN_STRONG_INLINE Packet +pldexp_float(Packet a, Packet exponent) +{ + typedef typename unpacket_traits<Packet>::integer_packet PacketI; + const Packet cst_127 = pset1<Packet>(127.f); + // return a * 2^exponent + PacketI ei = pcast<Packet,PacketI>(padd(exponent, cst_127)); + return pmul(a, preinterpret<Packet>(pshiftleft<23>(ei))); +} + +// Natural logarithm +// Computes log(x) as log(2^e * m) = C*e + log(m), where the constant C =log(2) +// and m is in the range [sqrt(1/2),sqrt(2)). In this range, the logarithm can +// be easily approximated by a polynomial centered on m=1 for stability. +// TODO(gonnet): Further reduce the interval allowing for lower-degree +// polynomial interpolants -> ... -> profit! +template <typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +Packet plog_float(const Packet _x) +{ + Packet x = _x; + + const Packet cst_1 = pset1<Packet>(1.0f); + const Packet cst_half = pset1<Packet>(0.5f); + // The smallest non denormalized float number. + const Packet cst_min_norm_pos = pset1frombits<Packet>( 0x00800000u); + const Packet cst_minus_inf = pset1frombits<Packet>( 0xff800000u); + const Packet cst_pos_inf = pset1frombits<Packet>( 0x7f800000u); + + // Polynomial coefficients. + const Packet cst_cephes_SQRTHF = pset1<Packet>(0.707106781186547524f); + const Packet cst_cephes_log_p0 = pset1<Packet>(7.0376836292E-2f); + const Packet cst_cephes_log_p1 = pset1<Packet>(-1.1514610310E-1f); + const Packet cst_cephes_log_p2 = pset1<Packet>(1.1676998740E-1f); + const Packet cst_cephes_log_p3 = pset1<Packet>(-1.2420140846E-1f); + const Packet cst_cephes_log_p4 = pset1<Packet>(+1.4249322787E-1f); + const Packet cst_cephes_log_p5 = pset1<Packet>(-1.6668057665E-1f); + const Packet cst_cephes_log_p6 = pset1<Packet>(+2.0000714765E-1f); + const Packet cst_cephes_log_p7 = pset1<Packet>(-2.4999993993E-1f); + const Packet cst_cephes_log_p8 = pset1<Packet>(+3.3333331174E-1f); + const Packet cst_cephes_log_q1 = pset1<Packet>(-2.12194440e-4f); + const Packet cst_cephes_log_q2 = pset1<Packet>(0.693359375f); + + // Truncate input values to the minimum positive normal. + x = pmax(x, cst_min_norm_pos); + + Packet e; + // extract significant in the range [0.5,1) and exponent + x = pfrexp(x,e); + + // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) + // and shift by -1. The values are then centered around 0, which improves + // the stability of the polynomial evaluation. + // if( x < SQRTHF ) { + // e -= 1; + // x = x + x - 1.0; + // } else { x = x - 1.0; } + Packet mask = pcmp_lt(x, cst_cephes_SQRTHF); + Packet tmp = pand(x, mask); + x = psub(x, cst_1); + e = psub(e, pand(cst_1, mask)); + x = padd(x, tmp); + + Packet x2 = pmul(x, x); + Packet x3 = pmul(x2, x); + + // Evaluate the polynomial approximant of degree 8 in three parts, probably + // to improve instruction-level parallelism. + Packet y, y1, y2; + y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1); + y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4); + y2 = pmadd(cst_cephes_log_p6, x, cst_cephes_log_p7); + y = pmadd(y, x, cst_cephes_log_p2); + y1 = pmadd(y1, x, cst_cephes_log_p5); + y2 = pmadd(y2, x, cst_cephes_log_p8); + y = pmadd(y, x3, y1); + y = pmadd(y, x3, y2); + y = pmul(y, x3); + + // Add the logarithm of the exponent back to the result of the interpolation. + y1 = pmul(e, cst_cephes_log_q1); + tmp = pmul(x2, cst_half); + y = padd(y, y1); + x = psub(x, tmp); + y2 = pmul(e, cst_cephes_log_q2); + x = padd(x, y); + x = padd(x, y2); + + Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x)); + Packet iszero_mask = pcmp_eq(_x,pzero(_x)); + Packet pos_inf_mask = pcmp_eq(_x,cst_pos_inf); + // Filter out invalid inputs, i.e.: + // - negative arg will be NAN + // - 0 will be -INF + // - +INF will be +INF + return pselect(iszero_mask, cst_minus_inf, + por(pselect(pos_inf_mask,cst_pos_inf,x), invalid_mask)); +} + +// Exponential function. Works by writing "x = m*log(2) + r" where +// "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then +// "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). +template <typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +Packet pexp_float(const Packet _x) +{ + const Packet cst_1 = pset1<Packet>(1.0f); + const Packet cst_half = pset1<Packet>(0.5f); + const Packet cst_exp_hi = pset1<Packet>( 88.3762626647950f); + const Packet cst_exp_lo = pset1<Packet>(-88.3762626647949f); + + const Packet cst_cephes_LOG2EF = pset1<Packet>(1.44269504088896341f); + const Packet cst_cephes_exp_p0 = pset1<Packet>(1.9875691500E-4f); + const Packet cst_cephes_exp_p1 = pset1<Packet>(1.3981999507E-3f); + const Packet cst_cephes_exp_p2 = pset1<Packet>(8.3334519073E-3f); + const Packet cst_cephes_exp_p3 = pset1<Packet>(4.1665795894E-2f); + const Packet cst_cephes_exp_p4 = pset1<Packet>(1.6666665459E-1f); + const Packet cst_cephes_exp_p5 = pset1<Packet>(5.0000001201E-1f); + + // Clamp x. + Packet x = pmax(pmin(_x, cst_exp_hi), cst_exp_lo); + + // Express exp(x) as exp(m*ln(2) + r), start by extracting + // m = floor(x/ln(2) + 0.5). + Packet m = pfloor(pmadd(x, cst_cephes_LOG2EF, cst_half)); + + // Get r = x - m*ln(2). If no FMA instructions are available, m*ln(2) is + // subtracted out in two parts, m*C1+m*C2 = m*ln(2), to avoid accumulating + // truncation errors. + Packet r; +#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD + const Packet cst_nln2 = pset1<Packet>(-0.6931471805599453f); + r = pmadd(m, cst_nln2, x); +#else + const Packet cst_cephes_exp_C1 = pset1<Packet>(0.693359375f); + const Packet cst_cephes_exp_C2 = pset1<Packet>(-2.12194440e-4f); + r = psub(x, pmul(m, cst_cephes_exp_C1)); + r = psub(r, pmul(m, cst_cephes_exp_C2)); +#endif + + Packet r2 = pmul(r, r); + + // TODO(gonnet): Split into odd/even polynomials and try to exploit + // instruction-level parallelism. + Packet y = cst_cephes_exp_p0; + y = pmadd(y, r, cst_cephes_exp_p1); + y = pmadd(y, r, cst_cephes_exp_p2); + y = pmadd(y, r, cst_cephes_exp_p3); + y = pmadd(y, r, cst_cephes_exp_p4); + y = pmadd(y, r, cst_cephes_exp_p5); + y = pmadd(y, r2, r); + y = padd(y, cst_1); + + // Return 2^m * exp(r). + return pmax(pldexp(y,m), _x); +} + +template <typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +Packet pexp_double(const Packet _x) +{ + Packet x = _x; + + const Packet cst_1 = pset1<Packet>(1.0); + const Packet cst_2 = pset1<Packet>(2.0); + const Packet cst_half = pset1<Packet>(0.5); + + const Packet cst_exp_hi = pset1<Packet>(709.437); + const Packet cst_exp_lo = pset1<Packet>(-709.436139303); + + const Packet cst_cephes_LOG2EF = pset1<Packet>(1.4426950408889634073599); + const Packet cst_cephes_exp_p0 = pset1<Packet>(1.26177193074810590878e-4); + const Packet cst_cephes_exp_p1 = pset1<Packet>(3.02994407707441961300e-2); + const Packet cst_cephes_exp_p2 = pset1<Packet>(9.99999999999999999910e-1); + const Packet cst_cephes_exp_q0 = pset1<Packet>(3.00198505138664455042e-6); + const Packet cst_cephes_exp_q1 = pset1<Packet>(2.52448340349684104192e-3); + const Packet cst_cephes_exp_q2 = pset1<Packet>(2.27265548208155028766e-1); + const Packet cst_cephes_exp_q3 = pset1<Packet>(2.00000000000000000009e0); + const Packet cst_cephes_exp_C1 = pset1<Packet>(0.693145751953125); + const Packet cst_cephes_exp_C2 = pset1<Packet>(1.42860682030941723212e-6); + + Packet tmp, fx; + + // clamp x + x = pmax(pmin(x, cst_exp_hi), cst_exp_lo); + // Express exp(x) as exp(g + n*log(2)). + fx = pmadd(cst_cephes_LOG2EF, x, cst_half); + + // Get the integer modulus of log(2), i.e. the "n" described above. + fx = pfloor(fx); + + // Get the remainder modulo log(2), i.e. the "g" described above. Subtract + // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last + // digits right. + tmp = pmul(fx, cst_cephes_exp_C1); + Packet z = pmul(fx, cst_cephes_exp_C2); + x = psub(x, tmp); + x = psub(x, z); + + Packet x2 = pmul(x, x); + + // Evaluate the numerator polynomial of the rational interpolant. + Packet px = cst_cephes_exp_p0; + px = pmadd(px, x2, cst_cephes_exp_p1); + px = pmadd(px, x2, cst_cephes_exp_p2); + px = pmul(px, x); + + // Evaluate the denominator polynomial of the rational interpolant. + Packet qx = cst_cephes_exp_q0; + qx = pmadd(qx, x2, cst_cephes_exp_q1); + qx = pmadd(qx, x2, cst_cephes_exp_q2); + qx = pmadd(qx, x2, cst_cephes_exp_q3); + + // I don't really get this bit, copied from the SSE2 routines, so... + // TODO(gonnet): Figure out what is going on here, perhaps find a better + // rational interpolant? + x = pdiv(px, psub(qx, px)); + x = pmadd(cst_2, x, cst_1); + + // Construct the result 2^n * exp(g) = e * x. The max is used to catch + // non-finite values in the input. + return pmax(pldexp(x,fx), _x); +} + +// The following code is inspired by the following stack-overflow answer: +// https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751 +// It has been largely optimized: +// - By-pass calls to frexp. +// - Aligned loads of required 96 bits of 2/pi. This is accomplished by +// (1) balancing the mantissa and exponent to the required bits of 2/pi are +// aligned on 8-bits, and (2) replicating the storage of the bits of 2/pi. +// - Avoid a branch in rounding and extraction of the remaining fractional part. +// Overall, I measured a speed up higher than x2 on x86-64. +inline float trig_reduce_huge (float xf, int *quadrant) +{ + using Eigen::numext::int32_t; + using Eigen::numext::uint32_t; + using Eigen::numext::int64_t; + using Eigen::numext::uint64_t; + + const double pio2_62 = 3.4061215800865545e-19; // pi/2 * 2^-62 + const uint64_t zero_dot_five = uint64_t(1) << 61; // 0.5 in 2.62-bit fixed-point foramt + + // 192 bits of 2/pi for Payne-Hanek reduction + // Bits are introduced by packet of 8 to enable aligned reads. + static const uint32_t two_over_pi [] = + { + 0x00000028, 0x000028be, 0x0028be60, 0x28be60db, + 0xbe60db93, 0x60db9391, 0xdb939105, 0x9391054a, + 0x91054a7f, 0x054a7f09, 0x4a7f09d5, 0x7f09d5f4, + 0x09d5f47d, 0xd5f47d4d, 0xf47d4d37, 0x7d4d3770, + 0x4d377036, 0x377036d8, 0x7036d8a5, 0x36d8a566, + 0xd8a5664f, 0xa5664f10, 0x664f10e4, 0x4f10e410, + 0x10e41000, 0xe4100000 + }; + + uint32_t xi = numext::as_uint(xf); + // Below, -118 = -126 + 8. + // -126 is to get the exponent, + // +8 is to enable alignment of 2/pi's bits on 8 bits. + // This is possible because the fractional part of x as only 24 meaningful bits. + uint32_t e = (xi >> 23) - 118; + // Extract the mantissa and shift it to align it wrt the exponent + xi = ((xi & 0x007fffffu)| 0x00800000u) << (e & 0x7); + + uint32_t i = e >> 3; + uint32_t twoopi_1 = two_over_pi[i-1]; + uint32_t twoopi_2 = two_over_pi[i+3]; + uint32_t twoopi_3 = two_over_pi[i+7]; + + // Compute x * 2/pi in 2.62-bit fixed-point format. + uint64_t p; + p = uint64_t(xi) * twoopi_3; + p = uint64_t(xi) * twoopi_2 + (p >> 32); + p = (uint64_t(xi * twoopi_1) << 32) + p; + + // Round to nearest: add 0.5 and extract integral part. + uint64_t q = (p + zero_dot_five) >> 62; + *quadrant = int(q); + // Now it remains to compute "r = x - q*pi/2" with high accuracy, + // since we have p=x/(pi/2) with high accuracy, we can more efficiently compute r as: + // r = (p-q)*pi/2, + // where the product can be be carried out with sufficient accuracy using double precision. + p -= q<<62; + return float(double(int64_t(p)) * pio2_62); +} + +template<bool ComputeSine,typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +#if EIGEN_GNUC_AT_LEAST(4,4) +__attribute__((optimize("-fno-unsafe-math-optimizations"))) +#endif +Packet psincos_float(const Packet& _x) +{ + typedef typename unpacket_traits<Packet>::integer_packet PacketI; + + const Packet cst_2oPI = pset1<Packet>(0.636619746685028076171875f); // 2/PI + const Packet cst_rounding_magic = pset1<Packet>(12582912); // 2^23 for rounding + const PacketI csti_1 = pset1<PacketI>(1); + const Packet cst_sign_mask = pset1frombits<Packet>(0x80000000u); + + Packet x = pabs(_x); + + // Scale x by 2/Pi to find x's octant. + Packet y = pmul(x, cst_2oPI); + + // Rounding trick: + Packet y_round = padd(y, cst_rounding_magic); + PacketI y_int = preinterpret<PacketI>(y_round); // last 23 digits represent integer (if abs(x)<2^24) + y = psub(y_round, cst_rounding_magic); // nearest integer to x*4/pi + + // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4 + // using "Extended precision modular arithmetic" + #if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) + // This version requires true FMA for high accuracy + // It provides a max error of 1ULP up to (with absolute_error < 5.9605e-08): + const float huge_th = ComputeSine ? 117435.992f : 71476.0625f; + x = pmadd(y, pset1<Packet>(-1.57079601287841796875f), x); + x = pmadd(y, pset1<Packet>(-3.1391647326017846353352069854736328125e-07f), x); + x = pmadd(y, pset1<Packet>(-5.390302529957764765544681040410068817436695098876953125e-15f), x); + #else + // Without true FMA, the previous set of coefficients maintain 1ULP accuracy + // up to x<15.7 (for sin), but accuracy is immediately lost for x>15.7. + // We thus use one more iteration to maintain 2ULPs up to reasonably large inputs. + + // The following set of coefficients maintain 1ULP up to 9.43 and 14.16 for sin and cos respectively. + // and 2 ULP up to: + const float huge_th = ComputeSine ? 25966.f : 18838.f; + x = pmadd(y, pset1<Packet>(-1.5703125), x); // = 0xbfc90000 + x = pmadd(y, pset1<Packet>(-0.000483989715576171875), x); // = 0xb9fdc000 + x = pmadd(y, pset1<Packet>(1.62865035235881805419921875e-07), x); // = 0x342ee000 + x = pmadd(y, pset1<Packet>(5.5644315544167710640977020375430583953857421875e-11), x); // = 0x2e74b9ee + + // For the record, the following set of coefficients maintain 2ULP up + // to a slightly larger range: + // const float huge_th = ComputeSine ? 51981.f : 39086.125f; + // but it slightly fails to maintain 1ULP for two values of sin below pi. + // x = pmadd(y, pset1<Packet>(-3.140625/2.), x); + // x = pmadd(y, pset1<Packet>(-0.00048351287841796875), x); + // x = pmadd(y, pset1<Packet>(-3.13855707645416259765625e-07), x); + // x = pmadd(y, pset1<Packet>(-6.0771006282767103812147979624569416046142578125e-11), x); + + // For the record, with only 3 iterations it is possible to maintain + // 1 ULP up to 3PI (maybe more) and 2ULP up to 255. + // The coefficients are: 0xbfc90f80, 0xb7354480, 0x2e74b9ee + #endif + + if(predux_any(pcmp_le(pset1<Packet>(huge_th),pabs(_x)))) + { + const int PacketSize = unpacket_traits<Packet>::size; + EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float vals[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float x_cpy[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) int y_int2[PacketSize]; + pstoreu(vals, pabs(_x)); + pstoreu(x_cpy, x); + pstoreu(y_int2, y_int); + for(int k=0; k<PacketSize;++k) + { + float val = vals[k]; + if(val>=huge_th && (numext::isfinite)(val)) + x_cpy[k] = trig_reduce_huge(val,&y_int2[k]); + } + x = ploadu<Packet>(x_cpy); + y_int = ploadu<PacketI>(y_int2); + } + + // Compute the sign to apply to the polynomial. + // sin: sign = second_bit(y_int) xor signbit(_x) + // cos: sign = second_bit(y_int+1) + Packet sign_bit = ComputeSine ? pxor(_x, preinterpret<Packet>(pshiftleft<30>(y_int))) + : preinterpret<Packet>(pshiftleft<30>(padd(y_int,csti_1))); + sign_bit = pand(sign_bit, cst_sign_mask); // clear all but left most bit + + // Get the polynomial selection mask from the second bit of y_int + // We'll calculate both (sin and cos) polynomials and then select from the two. + Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(y_int, csti_1), pzero(y_int))); + + Packet x2 = pmul(x,x); + + // Evaluate the cos(x) polynomial. (-Pi/4 <= x <= Pi/4) + Packet y1 = pset1<Packet>(2.4372266125283204019069671630859375e-05f); + y1 = pmadd(y1, x2, pset1<Packet>(-0.00138865201734006404876708984375f )); + y1 = pmadd(y1, x2, pset1<Packet>(0.041666619479656219482421875f )); + y1 = pmadd(y1, x2, pset1<Packet>(-0.5f)); + y1 = pmadd(y1, x2, pset1<Packet>(1.f)); + + // Evaluate the sin(x) polynomial. (Pi/4 <= x <= Pi/4) + // octave/matlab code to compute those coefficients: + // x = (0:0.0001:pi/4)'; + // A = [x.^3 x.^5 x.^7]; + // w = ((1.-(x/(pi/4)).^2).^5)*2000+1; # weights trading relative accuracy + // c = (A'*diag(w)*A)\(A'*diag(w)*(sin(x)-x)); # weighted LS, linear coeff forced to 1 + // printf('%.64f\n %.64f\n%.64f\n', c(3), c(2), c(1)) + // + Packet y2 = pset1<Packet>(-0.0001959234114083702898469196984621021329076029360294342041015625f); + y2 = pmadd(y2, x2, pset1<Packet>( 0.0083326873655616851693794799871284340042620897293090820312500000f)); + y2 = pmadd(y2, x2, pset1<Packet>(-0.1666666203982298255503735617821803316473960876464843750000000000f)); + y2 = pmul(y2, x2); + y2 = pmadd(y2, x, x); + + // Select the correct result from the two polynomials. + y = ComputeSine ? pselect(poly_mask,y2,y1) + : pselect(poly_mask,y1,y2); + + // Update the sign and filter huge inputs + return pxor(y, sign_bit); +} + +template<typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +Packet psin_float(const Packet& x) +{ + return psincos_float<true>(x); +} + +template<typename Packet> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +EIGEN_UNUSED +Packet pcos_float(const Packet& x) +{ + return psincos_float<false>(x); +} + +} // end namespace internal +} // end namespace Eigen
diff --git a/Eigen/src/Core/arch/Default/Settings.h b/Eigen/src/Core/arch/Default/Settings.h index 097373c..a5c3ada 100644 --- a/Eigen/src/Core/arch/Default/Settings.h +++ b/Eigen/src/Core/arch/Default/Settings.h
@@ -21,7 +21,7 @@ * it does not correspond to the number of iterations or the number of instructions */ #ifndef EIGEN_UNROLLING_LIMIT -#define EIGEN_UNROLLING_LIMIT 100 +#define EIGEN_UNROLLING_LIMIT 110 #endif /** Defines the threshold between a "small" and a "large" matrix.
diff --git a/Eigen/src/Core/arch/GPU/Half.h b/Eigen/src/Core/arch/GPU/Half.h new file mode 100644 index 0000000..f87d8a1 --- /dev/null +++ b/Eigen/src/Core/arch/GPU/Half.h
@@ -0,0 +1,765 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// The conversion routines are Copyright (c) Fabian Giesen, 2016. +// The original license follows: +// +// Copyright (c) Fabian Giesen, 2016 +// All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Standard 16-bit float type, mostly useful for GPUs. Defines a new +// type Eigen::half (inheriting either from CUDA's or HIP's __half struct) with +// operator overloads such that it behaves basically as an arithmetic +// type. It will be quite slow on CPUs (so it is recommended to stay +// in fp32 for CPUs, except for simple parameter conversions, I/O +// to disk and the likes), but fast on GPUs. + + +#ifndef EIGEN_HALF_GPU_H +#define EIGEN_HALF_GPU_H + +#if __cplusplus > 199711L +#define EIGEN_EXPLICIT_CAST(tgt_type) explicit operator tgt_type() +#else +#define EIGEN_EXPLICIT_CAST(tgt_type) operator tgt_type() +#endif + + +namespace Eigen { + +struct half; + +namespace half_impl { + +#if !defined(EIGEN_HAS_GPU_FP16) +// Make our own __half_raw definition that is similar to CUDA's. +struct __half_raw { + EIGEN_DEVICE_FUNC __half_raw() : x(0) {} + explicit EIGEN_DEVICE_FUNC __half_raw(unsigned short raw) : x(raw) {} + unsigned short x; +}; +#elif defined(EIGEN_HAS_HIP_FP16) + #if defined(EIGEN_HAS_OLD_HIP_FP16) +// Make a __half_raw definition that is +// ++ compatible with that of Eigen and +// ++ add an implicit conversion to the native __half of the old HIP implementation. +// +// Keeping ".x" as "unsigned short" keeps the interface the same between the Eigen and HIP implementation. +// +// In the old HIP implementation, +// ++ __half is a typedef of __fp16 +// ++ the "__h*" routines take "__half" arguments +// so we need to implicitly convert "__half_raw" to "__half" to avoid having to explicitly make +// that conversiion in each call to a "__h*" routine...that is why we have "operator __half" routine +struct __half_raw { + EIGEN_DEVICE_FUNC __half_raw() : x(0) {} + explicit EIGEN_DEVICE_FUNC __half_raw(unsigned short raw) : x(raw) {} + union { + unsigned short x; + __half data; + }; + operator __half(void) const { return data; } +}; + #endif +#elif defined(EIGEN_HAS_CUDA_FP16) + #if defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 +// In CUDA < 9.0, __half is the equivalent of CUDA 9's __half_raw + typedef __half __half_raw; + #endif // defined(EIGEN_HAS_CUDA_FP16) + +#elif defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) +typedef cl::sycl::half __half_raw; + +#endif + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw raw_uint16_to_half(unsigned short x); +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff); +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h); + +struct half_base : public __half_raw { + EIGEN_DEVICE_FUNC half_base() {} + EIGEN_DEVICE_FUNC half_base(const half_base& h) : __half_raw(h) {} + EIGEN_DEVICE_FUNC half_base(const __half_raw& h) : __half_raw(h) {} + +#if defined(EIGEN_HAS_GPU_FP16) + #if defined(EIGEN_HAS_HIP_FP16) + #if defined(EIGEN_HAS_OLD_HIP_FP16) + EIGEN_DEVICE_FUNC half_base(const __half& h) : __half_raw(__half_as_ushort(h)) {} + #else + EIGEN_DEVICE_FUNC half_base(const __half& h) { x = __half_as_ushort(h); } + #endif + #elif defined(EIGEN_HAS_CUDA_FP16) + #if (defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER >= 90000) + EIGEN_DEVICE_FUNC half_base(const __half& h) : __half_raw(*(__half_raw*)&h) {} + #endif + #endif +#endif +}; + +} // namespace half_impl + +// Class definition. +struct half : public half_impl::half_base { + + // Writing this out as separate #if-else blocks to make the code easier to follow + // The same applies to most #if-else blocks in this file +#if !defined(EIGEN_HAS_GPU_FP16) + typedef half_impl::__half_raw __half_raw; +#elif defined(EIGEN_HAS_HIP_FP16) + #if defined(EIGEN_HAS_OLD_HIP_FP16) + typedef half_impl::__half_raw __half_raw; + #endif +#elif defined(EIGEN_HAS_CUDA_FP16) + // Note that EIGEN_CUDACC_VER is set to 0 even when compiling with HIP, so (EIGEN_CUDACC_VER < 90000) is true even for HIP! + // So keeping this within #if defined(EIGEN_HAS_CUDA_FP16) is needed + #if defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 + typedef half_impl::__half_raw __half_raw; + #endif +#endif + + EIGEN_DEVICE_FUNC half() {} + + EIGEN_DEVICE_FUNC half(const __half_raw& h) : half_impl::half_base(h) {} + EIGEN_DEVICE_FUNC half(const half& h) : half_impl::half_base(h) {} + +#if defined(EIGEN_HAS_GPU_FP16) + #if defined(EIGEN_HAS_HIP_FP16) + EIGEN_DEVICE_FUNC half(const __half& h) : half_impl::half_base(h) {} + #elif defined(EIGEN_HAS_CUDA_FP16) + #if defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER >= 90000 + EIGEN_DEVICE_FUNC half(const __half& h) : half_impl::half_base(h) {} + #endif + #endif +#endif + + + explicit EIGEN_DEVICE_FUNC half(bool b) + : half_impl::half_base(half_impl::raw_uint16_to_half(b ? 0x3c00 : 0)) {} + template<class T> + explicit EIGEN_DEVICE_FUNC half(const T& val) + : half_impl::half_base(half_impl::float_to_half_rtne(static_cast<float>(val))) {} + explicit EIGEN_DEVICE_FUNC half(float f) + : half_impl::half_base(half_impl::float_to_half_rtne(f)) {} + + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(bool) const { + // +0.0 and -0.0 become false, everything else becomes true. + return (x & 0x7fff) != 0; + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(signed char) const { + return static_cast<signed char>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned char) const { + return static_cast<unsigned char>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(short) const { + return static_cast<short>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned short) const { + return static_cast<unsigned short>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(int) const { + return static_cast<int>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned int) const { + return static_cast<unsigned int>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(long) const { + return static_cast<long>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned long) const { + return static_cast<unsigned long>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(long long) const { + return static_cast<long long>(half_impl::half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(unsigned long long) const { + return static_cast<unsigned long long>(half_to_float(*this)); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(float) const { + return half_impl::half_to_float(*this); + } + EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(double) const { + return static_cast<double>(half_impl::half_to_float(*this)); + } + + EIGEN_DEVICE_FUNC half& operator=(const half& other) { + x = other.x; + return *this; + } + +}; + +} // end namespace Eigen + +namespace std { +template<> +struct numeric_limits<Eigen::half> { + static const bool is_specialized = true; + static const bool is_signed = true; + static const bool is_integer = false; + static const bool is_exact = false; + static const bool has_infinity = true; + static const bool has_quiet_NaN = true; + static const bool has_signaling_NaN = true; + static const float_denorm_style has_denorm = denorm_present; + static const bool has_denorm_loss = false; + static const std::float_round_style round_style = std::round_to_nearest; + static const bool is_iec559 = false; + static const bool is_bounded = false; + static const bool is_modulo = false; + static const int digits = 11; + static const int digits10 = 3; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html + static const int max_digits10 = 5; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html + static const int radix = 2; + static const int min_exponent = -13; + static const int min_exponent10 = -4; + static const int max_exponent = 16; + static const int max_exponent10 = 4; + static const bool traps = true; + static const bool tinyness_before = false; + + static Eigen::half (min)() { return Eigen::half_impl::raw_uint16_to_half(0x400); } + static Eigen::half lowest() { return Eigen::half_impl::raw_uint16_to_half(0xfbff); } + static Eigen::half (max)() { return Eigen::half_impl::raw_uint16_to_half(0x7bff); } + static Eigen::half epsilon() { return Eigen::half_impl::raw_uint16_to_half(0x0800); } + static Eigen::half round_error() { return Eigen::half(0.5); } + static Eigen::half infinity() { return Eigen::half_impl::raw_uint16_to_half(0x7c00); } + static Eigen::half quiet_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); } + static Eigen::half signaling_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); } + static Eigen::half denorm_min() { return Eigen::half_impl::raw_uint16_to_half(0x1); } +}; + +// If std::numeric_limits<T> is specialized, should also specialize +// std::numeric_limits<const T>, std::numeric_limits<volatile T>, and +// std::numeric_limits<const volatile T> +// https://stackoverflow.com/a/16519653/ +template<> +struct numeric_limits<const Eigen::half> : numeric_limits<Eigen::half> {}; +template<> +struct numeric_limits<volatile Eigen::half> : numeric_limits<Eigen::half> {}; +template<> +struct numeric_limits<const volatile Eigen::half> : numeric_limits<Eigen::half> {}; +} // end namespace std + +namespace Eigen { + +namespace half_impl { + +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(HIP_DEVICE_COMPILE)) + +// Intrinsics for native fp16 support. Note that on current hardware, +// these are no faster than fp32 arithmetic (you need to use the half2 +// versions to get the ALU speed increased), but you do save the +// conversion steps back and forth. + +EIGEN_STRONG_INLINE __device__ half operator + (const half& a, const half& b) { +#if defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER >= 90000 + return __hadd(::__half(a), ::__half(b)); +#else + return __hadd(a, b); +#endif +} +EIGEN_STRONG_INLINE __device__ half operator * (const half& a, const half& b) { + return __hmul(a, b); +} +EIGEN_STRONG_INLINE __device__ half operator - (const half& a, const half& b) { + return __hsub(a, b); +} +EIGEN_STRONG_INLINE __device__ half operator / (const half& a, const half& b) { +#if defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER >= 90000 + return __hdiv(a, b); +#else + float num = __half2float(a); + float denom = __half2float(b); + return __float2half(num / denom); +#endif +} +EIGEN_STRONG_INLINE __device__ half operator - (const half& a) { + return __hneg(a); +} +EIGEN_STRONG_INLINE __device__ half& operator += (half& a, const half& b) { + a = a + b; + return a; +} +EIGEN_STRONG_INLINE __device__ half& operator *= (half& a, const half& b) { + a = a * b; + return a; +} +EIGEN_STRONG_INLINE __device__ half& operator -= (half& a, const half& b) { + a = a - b; + return a; +} +EIGEN_STRONG_INLINE __device__ half& operator /= (half& a, const half& b) { + a = a / b; + return a; +} +EIGEN_STRONG_INLINE __device__ bool operator == (const half& a, const half& b) { + return __heq(a, b); +} +EIGEN_STRONG_INLINE __device__ bool operator != (const half& a, const half& b) { + return __hne(a, b); +} +EIGEN_STRONG_INLINE __device__ bool operator < (const half& a, const half& b) { + return __hlt(a, b); +} +EIGEN_STRONG_INLINE __device__ bool operator <= (const half& a, const half& b) { + return __hle(a, b); +} +EIGEN_STRONG_INLINE __device__ bool operator > (const half& a, const half& b) { + return __hgt(a, b); +} +EIGEN_STRONG_INLINE __device__ bool operator >= (const half& a, const half& b) { + return __hge(a, b); +} + +#else // Emulate support for half floats + +// Definitions for CPUs and older HIP+CUDA, mostly working through conversion +// to/from fp32. + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (const half& a, const half& b) { + return half(float(a) + float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator * (const half& a, const half& b) { + return half(float(a) * float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a, const half& b) { + return half(float(a) - float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, const half& b) { + return half(float(a) / float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a) { + half result; + result.x = a.x ^ 0x8000; + return result; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator += (half& a, const half& b) { + a = half(float(a) + float(b)); + return a; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator *= (half& a, const half& b) { + a = half(float(a) * float(b)); + return a; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator -= (half& a, const half& b) { + a = half(float(a) - float(b)); + return a; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator /= (half& a, const half& b) { + a = half(float(a) / float(b)); + return a; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator == (const half& a, const half& b) { + return numext::equal_strict(float(a),float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator != (const half& a, const half& b) { + return numext::not_equal_strict(float(a), float(b)); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator < (const half& a, const half& b) { + return float(a) < float(b); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator <= (const half& a, const half& b) { + return float(a) <= float(b); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator > (const half& a, const half& b) { + return float(a) > float(b); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const half& b) { + return float(a) >= float(b); +} + +#endif // Emulate support for half floats + +// Division by an index. Do it in full float precision to avoid accuracy +// issues in converting the denominator to half. +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, Index b) { + return half(static_cast<float>(a) / static_cast<float>(b)); +} + +// Conversion routines, including fallbacks for the host or older CUDA. +// Note that newer Intel CPUs (Haswell or newer) have vectorized versions of +// these in hardware. If we need more performance on older/other CPUs, they are +// also possible to vectorize directly. + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw raw_uint16_to_half(unsigned short x) { + __half_raw h; + h.x = x; + return h; +} + +union float32_bits { + unsigned int u; + float f; +}; + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + __half tmp_ff = __float2half(ff); + return *(__half_raw*)&tmp_ff; + +#elif defined(EIGEN_HAS_FP16_C) + __half_raw h; + h.x = _cvtss_sh(ff, 0); + return h; + +#else + float32_bits f; f.f = ff; + + const float32_bits f32infty = { 255 << 23 }; + const float32_bits f16max = { (127 + 16) << 23 }; + const float32_bits denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 }; + unsigned int sign_mask = 0x80000000u; + __half_raw o; + o.x = static_cast<unsigned short>(0x0u); + + unsigned int sign = f.u & sign_mask; + f.u ^= sign; + + // NOTE all the integer compares in this function can be safely + // compiled into signed compares since all operands are below + // 0x80000000. Important if you want fast straight SSE2 code + // (since there's no unsigned PCMPGTD). + + if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) + o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf + } else { // (De)normalized number or zero + if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero + // use a magic value to align our 10 mantissa bits at the bottom of + // the float. as long as FP addition is round-to-nearest-even this + // just works. + f.f += denorm_magic.f; + + // and one integer subtract of the bias later, we have our final float! + o.x = static_cast<unsigned short>(f.u - denorm_magic.u); + } else { + unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd + + // update exponent, rounding bias part 1 + f.u += ((unsigned int)(15 - 127) << 23) + 0xfff; + // rounding bias part 2 + f.u += mant_odd; + // take the bits! + o.x = static_cast<unsigned short>(f.u >> 13); + } + } + + o.x |= static_cast<unsigned short>(sign >> 16); + return o; +#endif +} + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __half2float(h); + +#elif defined(EIGEN_HAS_FP16_C) + return _cvtsh_ss(h.x); + +#else + const float32_bits magic = { 113 << 23 }; + const unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift + float32_bits o; + + o.u = (h.x & 0x7fff) << 13; // exponent/mantissa bits + unsigned int exp = shifted_exp & o.u; // just the exponent + o.u += (127 - 15) << 23; // exponent adjust + + // handle exponent special cases + if (exp == shifted_exp) { // Inf/NaN? + o.u += (128 - 16) << 23; // extra exp adjust + } else if (exp == 0) { // Zero/Denormal? + o.u += 1 << 23; // extra exp adjust + o.f -= magic.f; // renormalize + } + + o.u |= (h.x & 0x8000) << 16; // sign bit + return o.f; +#endif +} + +// --- standard functions --- + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(const half& a) { + return (a.x & 0x7fff) == 0x7c00; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(const half& a) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __hisnan(a); +#else + return (a.x & 0x7fff) > 0x7c00; +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(const half& a) { + return !(isinf EIGEN_NOT_A_MACRO (a)) && !(isnan EIGEN_NOT_A_MACRO (a)); +} + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half abs(const half& a) { + half result; + result.x = a.x & 0x7FFF; + return result; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half exp(const half& a) { +#if (EIGEN_CUDACC_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 530) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + return half(hexp(a)); +#else + return half(::expf(float(a))); +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half expm1(const half& a) { + return half(numext::expm1(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log(const half& a) { +#if (defined(EIGEN_HAS_CUDA_FP16) && EIGEN_CUDACC_VER >= 80000 && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return half(::hlog(a)); +#else + return half(::logf(float(a))); +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log1p(const half& a) { + return half(numext::log1p(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log10(const half& a) { + return half(::log10f(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sqrt(const half& a) { +#if (EIGEN_CUDACC_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 530) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + return half(hsqrt(a)); +#else + return half(::sqrtf(float(a))); +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half pow(const half& a, const half& b) { + return half(::powf(float(a), float(b))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sin(const half& a) { + return half(::sinf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half cos(const half& a) { + return half(::cosf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tan(const half& a) { + return half(::tanf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tanh(const half& a) { + return half(::tanhf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half floor(const half& a) { +#if (EIGEN_CUDACC_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 300) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + return half(hfloor(a)); +#else + return half(::floorf(float(a))); +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ceil(const half& a) { +#if (EIGEN_CUDACC_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 300) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + return half(hceil(a)); +#else + return half(::ceilf(float(a))); +#endif +} + +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (min)(const half& a, const half& b) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __hlt(b, a) ? b : a; +#else + const float f1 = static_cast<float>(a); + const float f2 = static_cast<float>(b); + return f2 < f1 ? b : a; +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (max)(const half& a, const half& b) { +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __hlt(a, b) ? b : a; +#else + const float f1 = static_cast<float>(a); + const float f2 = static_cast<float>(b); + return f1 < f2 ? b : a; +#endif +} + +#ifndef EIGEN_NO_IO +EIGEN_ALWAYS_INLINE std::ostream& operator << (std::ostream& os, const half& v) { + os << static_cast<float>(v); + return os; +} +#endif + +} // end namespace half_impl + +// import Eigen::half_impl::half into Eigen namespace +// using half_impl::half; + +namespace internal { + +template<> +struct random_default_impl<half, false, false> +{ + static inline half run(const half& x, const half& y) + { + return x + (y-x) * half(float(std::rand()) / float(RAND_MAX)); + } + static inline half run() + { + return run(half(-1.f), half(1.f)); + } +}; + +template<> struct is_arithmetic<half> { enum { value = true }; }; + +} // end namespace internal + +template<> struct NumTraits<Eigen::half> + : GenericNumTraits<Eigen::half> +{ + enum { + IsSigned = true, + IsInteger = false, + IsComplex = false, + RequireInitialization = false + }; + + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half epsilon() { + return half_impl::raw_uint16_to_half(0x0800); + } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half dummy_precision() { return Eigen::half(1e-2f); } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half highest() { + return half_impl::raw_uint16_to_half(0x7bff); + } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half lowest() { + return half_impl::raw_uint16_to_half(0xfbff); + } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half infinity() { + return half_impl::raw_uint16_to_half(0x7c00); + } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Eigen::half quiet_NaN() { + return half_impl::raw_uint16_to_half(0x7c01); + } +}; + +} // end namespace Eigen + +// C-like standard mathematical functions and trancendentals. +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half fabsh(const Eigen::half& a) { + Eigen::half result; + result.x = a.x & 0x7FFF; + return result; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half exph(const Eigen::half& a) { + return Eigen::half(::expf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half logh(const Eigen::half& a) { +#if (EIGEN_CUDACC_VER >= 80000 && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + return Eigen::half(::hlog(a)); +#else + return Eigen::half(::logf(float(a))); +#endif +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half sqrth(const Eigen::half& a) { + return Eigen::half(::sqrtf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half powh(const Eigen::half& a, const Eigen::half& b) { + return Eigen::half(::powf(float(a), float(b))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half floorh(const Eigen::half& a) { + return Eigen::half(::floorf(float(a))); +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half ceilh(const Eigen::half& a) { + return Eigen::half(::ceilf(float(a))); +} + +namespace std { + +#if __cplusplus > 199711L +template <> +struct hash<Eigen::half> { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::half& a) const { + return static_cast<std::size_t>(a.x); + } +}; +#endif + +} // end namespace std + + +// Add the missing shfl_xor intrinsic +#if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + +__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor(Eigen::half var, int laneMask, int width=warpSize) { + #if (EIGEN_CUDACC_VER < 90000) || \ + defined(EIGEN_HAS_HIP_FP16) + return static_cast<Eigen::half>(__shfl_xor(static_cast<float>(var), laneMask, width)); + #else + return static_cast<Eigen::half>(__shfl_xor_sync(0xFFFFFFFF, static_cast<float>(var), laneMask, width)); + #endif +} +#endif + +// ldg() has an overload for __half_raw, but we also need one for Eigen::half. +#if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half __ldg(const Eigen::half* ptr) { + return Eigen::half_impl::raw_uint16_to_half( + __ldg(reinterpret_cast<const unsigned short*>(ptr))); +} +#endif + + +#if defined(EIGEN_GPU_COMPILE_PHASE) +namespace Eigen { +namespace numext { + +template<> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +bool (isnan)(const Eigen::half& h) { + return (half_impl::isnan)(h); +} + +template<> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +bool (isinf)(const Eigen::half& h) { + return (half_impl::isinf)(h); +} + +template<> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +bool (isfinite)(const Eigen::half& h) { + return (half_impl::isfinite)(h); +} + +} // namespace Eigen +} // namespace numext +#endif + +#endif // EIGEN_HALF_GPU_H
diff --git a/Eigen/src/Core/arch/GPU/MathFunctions.h b/Eigen/src/Core/arch/GPU/MathFunctions.h new file mode 100644 index 0000000..d2b3a25 --- /dev/null +++ b/Eigen/src/Core/arch/GPU/MathFunctions.h
@@ -0,0 +1,103 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATH_FUNCTIONS_GPU_H +#define EIGEN_MATH_FUNCTIONS_GPU_H + +namespace Eigen { + +namespace internal { + +// Make sure this is only available when targeting a GPU: we don't want to +// introduce conflicts between these packet_traits definitions and the ones +// we'll use on the host side (SSE, AVX, ...) +#if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 plog<float4>(const float4& a) +{ + return make_float4(logf(a.x), logf(a.y), logf(a.z), logf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 plog<double2>(const double2& a) +{ + using ::log; + return make_double2(log(a.x), log(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 plog1p<float4>(const float4& a) +{ + return make_float4(log1pf(a.x), log1pf(a.y), log1pf(a.z), log1pf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 plog1p<double2>(const double2& a) +{ + return make_double2(log1p(a.x), log1p(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pexp<float4>(const float4& a) +{ + return make_float4(expf(a.x), expf(a.y), expf(a.z), expf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pexp<double2>(const double2& a) +{ + using ::exp; + return make_double2(exp(a.x), exp(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pexpm1<float4>(const float4& a) +{ + return make_float4(expm1f(a.x), expm1f(a.y), expm1f(a.z), expm1f(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pexpm1<double2>(const double2& a) +{ + return make_double2(expm1(a.x), expm1(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 psqrt<float4>(const float4& a) +{ + return make_float4(sqrtf(a.x), sqrtf(a.y), sqrtf(a.z), sqrtf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 psqrt<double2>(const double2& a) +{ + using ::sqrt; + return make_double2(sqrt(a.x), sqrt(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 prsqrt<float4>(const float4& a) +{ + return make_float4(rsqrtf(a.x), rsqrtf(a.y), rsqrtf(a.z), rsqrtf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 prsqrt<double2>(const double2& a) +{ + return make_double2(rsqrt(a.x), rsqrt(a.y)); +} + + +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MATH_FUNCTIONS_GPU_H
diff --git a/Eigen/src/Core/arch/CUDA/PacketMath.h b/Eigen/src/Core/arch/GPU/PacketMath.h similarity index 62% rename from Eigen/src/Core/arch/CUDA/PacketMath.h rename to Eigen/src/Core/arch/GPU/PacketMath.h index 932df10..c1b097f 100644 --- a/Eigen/src/Core/arch/CUDA/PacketMath.h +++ b/Eigen/src/Core/arch/GPU/PacketMath.h
@@ -7,8 +7,8 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_PACKET_MATH_CUDA_H -#define EIGEN_PACKET_MATH_CUDA_H +#ifndef EIGEN_PACKET_MATH_GPU_H +#define EIGEN_PACKET_MATH_GPU_H namespace Eigen { @@ -17,7 +17,7 @@ // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) -#if defined(__CUDACC__) && defined(EIGEN_USE_GPU) +#if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) template<> struct is_arithmetic<float4> { enum { value = true }; }; template<> struct is_arithmetic<double2> { enum { value = true }; }; @@ -44,8 +44,13 @@ HasPolygamma = 1, HasErf = 1, HasErfc = 1, - HasIgamma = 1, + HasI0e = 1, + HasI1e = 1, + HasIGamma = 1, + HasIGammaDerA = 1, + HasGammaSampleDerAlpha = 1, HasIGammac = 1, + HasBetaInc = 1, HasBlend = 0, }; @@ -68,18 +73,25 @@ HasRsqrt = 1, HasLGamma = 1, HasDiGamma = 1, + HasZeta = 1, + HasPolygamma = 1, HasErf = 1, HasErfc = 1, + HasI0e = 1, + HasI1e = 1, HasIGamma = 1, + HasIGammaDerA = 1, + HasGammaSampleDerAlpha = 1, HasIGammac = 1, + HasBetaInc = 1, HasBlend = 0, }; }; -template<> struct unpacket_traits<float4> { typedef float type; enum {size=4, alignment=Aligned16}; typedef float4 half; }; -template<> struct unpacket_traits<double2> { typedef double type; enum {size=2, alignment=Aligned16}; typedef double2 half; }; +template<> struct unpacket_traits<float4> { typedef float type; enum {size=4, alignment=Aligned16, vectorizable=true}; typedef float4 half; }; +template<> struct unpacket_traits<double2> { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef double2 half; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pset1<float4>(const float& from) { return make_float4(from, from, from, from); @@ -88,6 +100,117 @@ return make_double2(from, from); } +namespace { + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_and(const float& a, + const float& b) { + return __int_as_float(__float_as_int(a) & __float_as_int(b)); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_and(const double& a, + const double& b) { + return __longlong_as_double(__double_as_longlong(a) & + __double_as_longlong(b)); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_or(const float& a, + const float& b) { + return __int_as_float(__float_as_int(a) | __float_as_int(b)); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_or(const double& a, + const double& b) { + return __longlong_as_double(__double_as_longlong(a) | + __double_as_longlong(b)); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_xor(const float& a, + const float& b) { + return __int_as_float(__float_as_int(a) ^ __float_as_int(b)); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_xor(const double& a, + const double& b) { + return __longlong_as_double(__double_as_longlong(a) ^ + __double_as_longlong(b)); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_andnot(const float& a, + const float& b) { + return __int_as_float(__float_as_int(a) & ~__float_as_int(b)); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_andnot(const double& a, + const double& b) { + return __longlong_as_double(__double_as_longlong(a) & + ~__double_as_longlong(b)); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float eq_mask(const float& a, + const float& b) { + return __int_as_float(a == b ? 0xffffffffu : 0u); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double eq_mask(const double& a, + const double& b) { + return __longlong_as_double(a == b ? 0xffffffffffffffffull : 0ull); +} + +} // namespace + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pand<float4>(const float4& a, + const float4& b) { + return make_float4(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y), + bitwise_and(a.z, b.z), bitwise_and(a.w, b.w)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pand<double2>(const double2& a, + const double2& b) { + return make_double2(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 por<float4>(const float4& a, + const float4& b) { + return make_float4(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y), + bitwise_or(a.z, b.z), bitwise_or(a.w, b.w)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 por<double2>(const double2& a, + const double2& b) { + return make_double2(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pxor<float4>(const float4& a, + const float4& b) { + return make_float4(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y), + bitwise_xor(a.z, b.z), bitwise_xor(a.w, b.w)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pxor<double2>(const double2& a, + const double2& b) { + return make_double2(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pandnot<float4>(const float4& a, + const float4& b) { + return make_float4(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y), + bitwise_andnot(a.z, b.z), bitwise_andnot(a.w, b.w)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pandnot<double2>(const double2& a, const double2& b) { + return make_double2(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_eq<float4>(const float4& a, + const float4& b) { + return make_float4(eq_mask(a.x, b.x), eq_mask(a.y, b.y), eq_mask(a.z, b.z), + eq_mask(a.w, b.w)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pcmp_eq<double2>(const double2& a, const double2& b) { + return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y)); +} template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset<float4>(const float& a) { return make_float4(a, a+1, a+2, a+3); @@ -163,10 +286,10 @@ return make_double2(from[0], from[1]); } -template<> EIGEN_STRONG_INLINE float4 ploaddup<float4>(const float* from) { +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploaddup<float4>(const float* from) { return make_float4(from[0], from[0], from[1], from[1]); } -template<> EIGEN_STRONG_INLINE double2 ploaddup<double2>(const double* from) { +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploaddup<double2>(const double* from) { return make_double2(from[0], from[0]); } @@ -192,7 +315,7 @@ template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Aligned>(const float* from) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350 +#if defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350 return __ldg((const float4*)from); #else return make_float4(from[0], from[1], from[2], from[3]); @@ -200,7 +323,7 @@ } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Aligned>(const double* from) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350 +#if defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350 return __ldg((const double2*)from); #else return make_double2(from[0], from[1]); @@ -209,7 +332,7 @@ template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Unaligned>(const float* from) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350 +#if defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350 return make_float4(__ldg(from+0), __ldg(from+1), __ldg(from+2), __ldg(from+3)); #else return make_float4(from[0], from[1], from[2], from[3]); @@ -217,7 +340,7 @@ } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Unaligned>(const double* from) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350 +#if defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350 return make_double2(__ldg(from+0), __ldg(from+1)); #else return make_double2(from[0], from[1]); @@ -278,35 +401,6 @@ return a.x * a.y; } -template<size_t offset> -struct protate_impl<offset, float4> -{ - static float4 run(const float4& a) { - if (offset == 0) { - return make_float4(a.x, a.y, a.z, a.w); - } - if (offset == 1) { - return make_float4(a.w, a.x, a.y, a.z); - } - if (offset == 2) { - return make_float4(a.z, a.w, a.x, a.y); - } - return make_float4(a.y, a.z, a.w, a.x); - } -}; - -template<size_t offset> -struct protate_impl<offset, double2> -{ - static double2 run(const double2& a) { - if (offset == 0) { - return make_double2(a.x, a.y); - } - return make_double2(a.y, a.x); - } -}; - - template<> EIGEN_DEVICE_FUNC inline float4 pabs<float4>(const float4& a) { return make_float4(fabsf(a.x), fabsf(a.y), fabsf(a.z), fabsf(a.w)); } @@ -316,7 +410,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<float4,4>& kernel) { - double tmp = kernel.packet[0].y; + float tmp = kernel.packet[0].y; kernel.packet[0].y = kernel.packet[1].x; kernel.packet[1].x = tmp; @@ -355,4 +449,4 @@ } // end namespace Eigen -#endif // EIGEN_PACKET_MATH_CUDA_H +#endif // EIGEN_PACKET_MATH_GPU_H
diff --git a/Eigen/src/Core/arch/GPU/PacketMathHalf.h b/Eigen/src/Core/arch/GPU/PacketMathHalf.h new file mode 100644 index 0000000..316ac02 --- /dev/null +++ b/Eigen/src/Core/arch/GPU/PacketMathHalf.h
@@ -0,0 +1,1483 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_HALF_GPU_H +#define EIGEN_PACKET_MATH_HALF_GPU_H + + +namespace Eigen { +namespace internal { + +// Most of the following operations require arch >= 3.0 +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDACC) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIPCC) && defined(EIGEN_HIP_DEVICE_COMPILE)) + +template<> struct is_arithmetic<half2> { enum { value = true }; }; + +template<> struct packet_traits<Eigen::half> : default_packet_traits +{ + typedef half2 type; + typedef half2 half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size=2, + HasHalfPacket = 0, + HasAdd = 1, + HasMul = 1, + HasDiv = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasExp = 1, + HasExpm1 = 1, + HasLog = 1, + HasLog1p = 1 + }; +}; + +template<> struct unpacket_traits<half2> { typedef Eigen::half type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef half2 half; }; + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pset1<half2>(const Eigen::half& from) { + +#if defined(EIGEN_HIP_DEVICE_COMPILE) + +#if defined(EIGEN_HAS_OLD_HIP_FP16) + return half2half2(from); +#else + return __half2half2(from); +#endif + +#else // EIGEN_CUDA_ARCH + return __half2half2(from); +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pload<half2>(const Eigen::half* from) { + return *reinterpret_cast<const half2*>(from); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploadu<half2>(const Eigen::half* from) { + return __halves2half2(from[0], from[1]); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploaddup<half2>(const Eigen::half* from) { + return __halves2half2(from[0], from[0]); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const half2& from) { + *reinterpret_cast<half2*>(to) = from; +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const half2& from) { + to[0] = __low2half(from); + to[1] = __high2half(from); +} + +template<> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(const Eigen::half* from) { + +#if defined(EIGEN_HIP_DEVICE_COMPILE) + +#if defined(EIGEN_HAS_OLD_HIP_FP16) + return __halves2half2((*(from+0)), (*(from+1))); +#else + return __ldg((const half2*)from); +#endif + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 350 + return __ldg((const half2*)from); +#else + return __halves2half2(*(from+0), *(from+1)); +#endif + +#endif +} + +template<> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(const Eigen::half* from) { + +#if defined(EIGEN_HIP_DEVICE_COMPILE) + +#if defined(EIGEN_HAS_OLD_HIP_FP16) + return __halves2half2((*(from+0)), (*(from+1))); +#else + return __halves2half2(__ldg(from+0), __ldg(from+1)); +#endif + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 350 + return __halves2half2(__ldg(from+0), __ldg(from+1)); +#else + return __halves2half2(*(from+0), *(from+1)); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pgather<Eigen::half, half2>(const Eigen::half* from, Index stride) { + return __halves2half2(from[0*stride], from[1*stride]); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, half2>(Eigen::half* to, const half2& from, Index stride) { + to[stride*0] = __low2half(from); + to[stride*1] = __high2half(from); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst<half2>(const half2& a) { + return __low2half(a); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pabs<half2>(const half2& a) { + half2 result; + unsigned temp = *(reinterpret_cast<const unsigned*>(&(a))); + *(reinterpret_cast<unsigned*>(&(result))) = temp & 0x7FFF7FFF; + return result; +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ptrue<half2>(const half2& a) { + half2 result; + *(reinterpret_cast<unsigned*>(&(result))) = 0xffffffffu; +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<half2,2>& kernel) { + __half a1 = __low2half(kernel.packet[0]); + __half a2 = __high2half(kernel.packet[0]); + __half b1 = __low2half(kernel.packet[1]); + __half b2 = __high2half(kernel.packet[1]); + kernel.packet[0] = __halves2half2(a1, b1); + kernel.packet[1] = __halves2half2(a2, b2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plset<half2>(const Eigen::half& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __halves2half2(a, __hadd(a, __float2half(1.0f))); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __halves2half2(a, __hadd(a, __float2half(1.0f))); +#else + float f = __half2float(a) + 1.0f; + return __halves2half2(a, __float2half(f)); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd<half2>(const half2& a, const half2& b) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hadd2(a, b); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hadd2(a, b); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + float r1 = a1 + b1; + float r2 = a2 + b2; + return __floats2half2_rn(r1, r2); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psub<half2>(const half2& a, const half2& b) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hsub2(a, b); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hsub2(a, b); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + float r1 = a1 - b1; + float r2 = a2 - b2; + return __floats2half2_rn(r1, r2); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pnegate(const half2& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hneg2(a); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hneg2(a); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + return __floats2half2_rn(-a1, -a2); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pconj(const half2& a) { return a; } + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul<half2>(const half2& a, const half2& b) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hmul2(a, b); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hmul2(a, b); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + float r1 = a1 * b1; + float r2 = a2 * b2; + return __floats2half2_rn(r1, r2); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmadd<half2>(const half2& a, const half2& b, const half2& c) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hfma2(a, b, c); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hfma2(a, b, c); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + float c1 = __low2float(c); + float c2 = __high2float(c); + float r1 = a1 * b1 + c1; + float r2 = a2 * b2 + c2; + return __floats2half2_rn(r1, r2); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv<half2>(const half2& a, const half2& b) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + +#if defined(EIGEN_HAS_OLD_HIP_FP16) + return h2div(a, b); +#else + return __h2div(a, b); +#endif + +#else // EIGEN_CUDA_ARCH + + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + float r1 = a1 / b1; + float r2 = a2 / b2; + return __floats2half2_rn(r1, r2); + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin<half2>(const half2& a, const half2& b) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + __half r1 = a1 < b1 ? __low2half(a) : __low2half(b); + __half r2 = a2 < b2 ? __high2half(a) : __high2half(b); + return __halves2half2(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax<half2>(const half2& a, const half2& b) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float b1 = __low2float(b); + float b2 = __high2float(b); + __half r1 = a1 > b1 ? __low2half(a) : __low2half(b); + __half r2 = a2 > b2 ? __high2half(a) : __high2half(b); + return __halves2half2(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux<half2>(const half2& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hadd(__low2half(a), __high2half(a)); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hadd(__low2half(a), __high2half(a)); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + return Eigen::half(__float2half(a1 + a2)); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<half2>(const half2& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + __half first = __low2half(a); + __half second = __high2half(a); + return __hgt(first, second) ? first : second; + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + __half first = __low2half(a); + __half second = __high2half(a); + return __hgt(first, second) ? first : second; +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + return a1 > a2 ? __low2half(a) : __high2half(a); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min<half2>(const half2& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + __half first = __low2half(a); + __half second = __high2half(a); + return __hlt(first, second) ? first : second; + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + __half first = __low2half(a); + __half second = __high2half(a); + return __hlt(first, second) ? first : second; +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + return a1 < a2 ? __low2half(a) : __high2half(a); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul<half2>(const half2& a) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + return __hmul(__low2half(a), __high2half(a)); + +#else // EIGEN_CUDA_ARCH + +#if EIGEN_CUDA_ARCH >= 530 + return __hmul(__low2half(a), __high2half(a)); +#else + float a1 = __low2float(a); + float a2 = __high2float(a); + return Eigen::half(__float2half(a1 * a2)); +#endif + +#endif +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog1p<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = log1pf(a1); + float r2 = log1pf(a2); + return __floats2half2_rn(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexpm1<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = expm1f(a1); + float r2 = expm1f(a2); + return __floats2half2_rn(r1, r2); +} + +#if (EIGEN_CUDACC_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 530) || \ + defined(EIGEN_HIP_DEVICE_COMPILE) + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +half2 plog<half2>(const half2& a) { + return h2log(a); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +half2 pexp<half2>(const half2& a) { + return h2exp(a); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +half2 psqrt<half2>(const half2& a) { + return h2sqrt(a); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +half2 prsqrt<half2>(const half2& a) { + return h2rsqrt(a); +} + +#else + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = logf(a1); + float r2 = logf(a2); + return __floats2half2_rn(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexp<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = expf(a1); + float r2 = expf(a2); + return __floats2half2_rn(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psqrt<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = sqrtf(a1); + float r2 = sqrtf(a2); + return __floats2half2_rn(r1, r2); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 prsqrt<half2>(const half2& a) { + float a1 = __low2float(a); + float a2 = __high2float(a); + float r1 = rsqrtf(a1); + float r2 = rsqrtf(a2); + return __floats2half2_rn(r1, r2); +} + +#endif + +#elif defined EIGEN_VECTORIZE_AVX512 + +typedef struct { + __m256i x; +} Packet16h; + + +template<> struct is_arithmetic<Packet16h> { enum { value = true }; }; + +template <> +struct packet_traits<half> : default_packet_traits { + typedef Packet16h type; + // There is no half-size packet for Packet16h. + typedef Packet16h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 16, + HasHalfPacket = 0, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasConj = 0, + HasSetLinear = 0, + HasDiv = 0, + HasSqrt = 0, + HasRsqrt = 0, + HasExp = 0, + HasLog = 0, + HasBlend = 0 + }; +}; + + +template<> struct unpacket_traits<Packet16h> { typedef Eigen::half type; enum {size=16, alignment=Aligned32, vectorizable=true}; typedef Packet16h half; }; + +template<> EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) { + Packet16h result; + result.x = _mm256_set1_epi16(from.x); + return result; +} + +template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) { + return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from.x, 0))); +} + +template<> EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) { + Packet16h result; + result.x = _mm256_load_si256(reinterpret_cast<const __m256i*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) { + Packet16h result; + result.x = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) { + // (void*) -> workaround clang warning: + // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32 + _mm256_store_si256((__m256i*)(void*)to, from.x); +} + +template<> EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) { + // (void*) -> workaround clang warning: + // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32 + _mm256_storeu_si256((__m256i*)(void*)to, from.x); +} + +template<> EIGEN_STRONG_INLINE Packet16h +ploaddup<Packet16h>(const Eigen::half* from) { + Packet16h result; + unsigned short a = from[0].x; + unsigned short b = from[1].x; + unsigned short c = from[2].x; + unsigned short d = from[3].x; + unsigned short e = from[4].x; + unsigned short f = from[5].x; + unsigned short g = from[6].x; + unsigned short h = from[7].x; + result.x = _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet16h +ploadquad(const Eigen::half* from) { + Packet16h result; + unsigned short a = from[0].x; + unsigned short b = from[1].x; + unsigned short c = from[2].x; + unsigned short d = from[3].x; + result.x = _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a); + return result; +} + +EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { +#ifdef EIGEN_HAS_FP16_C + return _mm512_cvtph_ps(a.x); +#else + EIGEN_ALIGN64 half aux[16]; + pstore(aux, a); + float f0(aux[0]); + float f1(aux[1]); + float f2(aux[2]); + float f3(aux[3]); + float f4(aux[4]); + float f5(aux[5]); + float f6(aux[6]); + float f7(aux[7]); + float f8(aux[8]); + float f9(aux[9]); + float fa(aux[10]); + float fb(aux[11]); + float fc(aux[12]); + float fd(aux[13]); + float fe(aux[14]); + float ff(aux[15]); + + return _mm512_set_ps( + ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0); +#endif +} + +EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { +#ifdef EIGEN_HAS_FP16_C + Packet16h result; + result.x = _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); + return result; +#else + EIGEN_ALIGN64 float aux[16]; + pstore(aux, a); + half h0(aux[0]); + half h1(aux[1]); + half h2(aux[2]); + half h3(aux[3]); + half h4(aux[4]); + half h5(aux[5]); + half h6(aux[6]); + half h7(aux[7]); + half h8(aux[8]); + half h9(aux[9]); + half ha(aux[10]); + half hb(aux[11]); + half hc(aux[12]); + half hd(aux[13]); + half he(aux[14]); + half hf(aux[15]); + + Packet16h result; + result.x = _mm256_set_epi16( + hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x, + h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); + return result; +#endif +} + +template<> EIGEN_STRONG_INLINE Packet16h pnot(const Packet16h& a) { + Packet16h r; r.x = _mm256_xor_si256(a.x, pcmp_eq(a.x, a.x)); return r; +} + +template<> EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) { + Packet16h r; r.x = Packet8i(ptrue(a.x)); return r; +} + +template<> EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a,const Packet16h& b) { + // in some cases Packet8i is a wrapper around __m256i, so we need to + // cast to Packet8i to call the correct overload. + Packet16h r; r.x = por(Packet8i(a.x),Packet8i(b.x)); return r; +} +template<> EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a,const Packet16h& b) { + Packet16h r; r.x = pxor(Packet8i(a.x),Packet8i(b.x)); return r; +} +template<> EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a,const Packet16h& b) { + Packet16h r; r.x = pand(Packet8i(a.x),Packet8i(b.x)); return r; +} +template<> EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a,const Packet16h& b) { + Packet16h r; r.x = pandnot(Packet8i(a.x),Packet8i(b.x)); return r; +} + +template<> EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a,const Packet16h& b) { + Packet16f af = half2float(a); + Packet16f bf = half2float(b); + Packet16f rf = pcmp_eq(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) { + // FIXME we could do that with bit manipulation + Packet16f af = half2float(a); + Packet16f rf = pnegate(af); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) { + Packet16f af = half2float(a); + Packet16f bf = half2float(b); + Packet16f rf = padd(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) { + Packet16f af = half2float(a); + Packet16f bf = half2float(b); + Packet16f rf = psub(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) { + Packet16f af = half2float(a); + Packet16f bf = half2float(b); + Packet16f rf = pmul(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& from) { + Packet16f from_float = half2float(from); + return half(predux(from_float)); +} + +template<> EIGEN_STRONG_INLINE half predux_mul<Packet16h>(const Packet16h& from) { + Packet16f from_float = half2float(from); + return half(predux_mul(from_float)); +} + +template<> EIGEN_STRONG_INLINE Packet16h preduxp<Packet16h>(const Packet16h* p) { + Packet16f pf[16]; + pf[0] = half2float(p[0]); + pf[1] = half2float(p[1]); + pf[2] = half2float(p[2]); + pf[3] = half2float(p[3]); + pf[4] = half2float(p[4]); + pf[5] = half2float(p[5]); + pf[6] = half2float(p[6]); + pf[7] = half2float(p[7]); + pf[8] = half2float(p[8]); + pf[9] = half2float(p[9]); + pf[10] = half2float(p[10]); + pf[11] = half2float(p[11]); + pf[12] = half2float(p[12]); + pf[13] = half2float(p[13]); + pf[14] = half2float(p[14]); + pf[15] = half2float(p[15]); + Packet16f reduced = preduxp<Packet16f>(pf); + return float2half(reduced); +} + +template<> EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) +{ + __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); + Packet16h res; + res.x = _mm256_insertf128_si256( + _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a.x,1),m)), + _mm_shuffle_epi8(_mm256_extractf128_si256(a.x,0),m), 1); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet16h pinsertfirst(const Packet16h& a, Eigen::half b) +{ + Packet16h res; + res.x = _mm256_insert_epi16(a.x,b.x,0); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet16h pinsertlast(const Packet16h& a, Eigen::half b) +{ + Packet16h res; + res.x = _mm256_insert_epi16(a.x,b.x,15); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) +{ + Packet16h result; + result.x = _mm256_set_epi16( + from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x, + from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x, + from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, + from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); + return result; +} + +template<> EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) +{ + EIGEN_ALIGN64 half aux[16]; + pstore(aux, from); + to[stride*0].x = aux[0].x; + to[stride*1].x = aux[1].x; + to[stride*2].x = aux[2].x; + to[stride*3].x = aux[3].x; + to[stride*4].x = aux[4].x; + to[stride*5].x = aux[5].x; + to[stride*6].x = aux[6].x; + to[stride*7].x = aux[7].x; + to[stride*8].x = aux[8].x; + to[stride*9].x = aux[9].x; + to[stride*10].x = aux[10].x; + to[stride*11].x = aux[11].x; + to[stride*12].x = aux[12].x; + to[stride*13].x = aux[13].x; + to[stride*14].x = aux[14].x; + to[stride*15].x = aux[15].x; +} + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet16h,16>& kernel) { + __m256i a = kernel.packet[0].x; + __m256i b = kernel.packet[1].x; + __m256i c = kernel.packet[2].x; + __m256i d = kernel.packet[3].x; + __m256i e = kernel.packet[4].x; + __m256i f = kernel.packet[5].x; + __m256i g = kernel.packet[6].x; + __m256i h = kernel.packet[7].x; + __m256i i = kernel.packet[8].x; + __m256i j = kernel.packet[9].x; + __m256i k = kernel.packet[10].x; + __m256i l = kernel.packet[11].x; + __m256i m = kernel.packet[12].x; + __m256i n = kernel.packet[13].x; + __m256i o = kernel.packet[14].x; + __m256i p = kernel.packet[15].x; + + __m256i ab_07 = _mm256_unpacklo_epi16(a, b); + __m256i cd_07 = _mm256_unpacklo_epi16(c, d); + __m256i ef_07 = _mm256_unpacklo_epi16(e, f); + __m256i gh_07 = _mm256_unpacklo_epi16(g, h); + __m256i ij_07 = _mm256_unpacklo_epi16(i, j); + __m256i kl_07 = _mm256_unpacklo_epi16(k, l); + __m256i mn_07 = _mm256_unpacklo_epi16(m, n); + __m256i op_07 = _mm256_unpacklo_epi16(o, p); + + __m256i ab_8f = _mm256_unpackhi_epi16(a, b); + __m256i cd_8f = _mm256_unpackhi_epi16(c, d); + __m256i ef_8f = _mm256_unpackhi_epi16(e, f); + __m256i gh_8f = _mm256_unpackhi_epi16(g, h); + __m256i ij_8f = _mm256_unpackhi_epi16(i, j); + __m256i kl_8f = _mm256_unpackhi_epi16(k, l); + __m256i mn_8f = _mm256_unpackhi_epi16(m, n); + __m256i op_8f = _mm256_unpackhi_epi16(o, p); + + __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); + __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); + __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); + __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); + __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); + __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); + __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); + __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); + + __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); + __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); + __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); + __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); + __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); + __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); + __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); + __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); + + __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); + __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); + __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); + __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); + __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); + __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); + __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); + __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); + __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); + __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); + __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); + __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); + __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); + __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); + __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); + __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); + + // NOTE: no unpacklo/hi instr in this case, so using permute instr. + __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); + __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); + __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); + __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); + __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); + __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); + __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); + __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); + __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); + __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); + __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); + __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); + __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); + __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); + __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); + __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); + + kernel.packet[0].x = a_p_0; + kernel.packet[1].x = a_p_1; + kernel.packet[2].x = a_p_2; + kernel.packet[3].x = a_p_3; + kernel.packet[4].x = a_p_4; + kernel.packet[5].x = a_p_5; + kernel.packet[6].x = a_p_6; + kernel.packet[7].x = a_p_7; + kernel.packet[8].x = a_p_8; + kernel.packet[9].x = a_p_9; + kernel.packet[10].x = a_p_a; + kernel.packet[11].x = a_p_b; + kernel.packet[12].x = a_p_c; + kernel.packet[13].x = a_p_d; + kernel.packet[14].x = a_p_e; + kernel.packet[15].x = a_p_f; +} + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet16h,8>& kernel) { + EIGEN_ALIGN64 half in[8][16]; + pstore<half>(in[0], kernel.packet[0]); + pstore<half>(in[1], kernel.packet[1]); + pstore<half>(in[2], kernel.packet[2]); + pstore<half>(in[3], kernel.packet[3]); + pstore<half>(in[4], kernel.packet[4]); + pstore<half>(in[5], kernel.packet[5]); + pstore<half>(in[6], kernel.packet[6]); + pstore<half>(in[7], kernel.packet[7]); + + EIGEN_ALIGN64 half out[8][16]; + + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + out[i][j] = in[j][2*i]; + } + for (int j = 0; j < 8; ++j) { + out[i][j+8] = in[j][2*i+1]; + } + } + + kernel.packet[0] = pload<Packet16h>(out[0]); + kernel.packet[1] = pload<Packet16h>(out[1]); + kernel.packet[2] = pload<Packet16h>(out[2]); + kernel.packet[3] = pload<Packet16h>(out[3]); + kernel.packet[4] = pload<Packet16h>(out[4]); + kernel.packet[5] = pload<Packet16h>(out[5]); + kernel.packet[6] = pload<Packet16h>(out[6]); + kernel.packet[7] = pload<Packet16h>(out[7]); +} + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet16h,4>& kernel) { + EIGEN_ALIGN64 half in[4][16]; + pstore<half>(in[0], kernel.packet[0]); + pstore<half>(in[1], kernel.packet[1]); + pstore<half>(in[2], kernel.packet[2]); + pstore<half>(in[3], kernel.packet[3]); + + EIGEN_ALIGN64 half out[4][16]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][4*i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j+4] = in[j][4*i+1]; + } + for (int j = 0; j < 4; ++j) { + out[i][j+8] = in[j][4*i+2]; + } + for (int j = 0; j < 4; ++j) { + out[i][j+12] = in[j][4*i+3]; + } + } + + kernel.packet[0] = pload<Packet16h>(out[0]); + kernel.packet[1] = pload<Packet16h>(out[1]); + kernel.packet[2] = pload<Packet16h>(out[2]); + kernel.packet[3] = pload<Packet16h>(out[3]); +} + + +#elif defined EIGEN_VECTORIZE_AVX + +typedef struct { + __m128i x; +} Packet8h; + + +template<> struct is_arithmetic<Packet8h> { enum { value = true }; }; + +template <> +struct packet_traits<Eigen::half> : default_packet_traits { + typedef Packet8h type; + // There is no half-size packet for Packet8h. + typedef Packet8h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + HasHalfPacket = 0, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasConj = 0, + HasSetLinear = 0, + HasDiv = 0, + HasSqrt = 0, + HasRsqrt = 0, + HasExp = 0, + HasLog = 0, + HasBlend = 0 + }; +}; + + +template<> struct unpacket_traits<Packet8h> { typedef Eigen::half type; enum {size=8, alignment=Aligned16, vectorizable=true}; typedef Packet8h half; }; + +template<> EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) { + Packet8h result; + result.x = _mm_set1_epi16(from.x); + return result; +} + +template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) { + return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_extract_epi16(from.x, 0))); +} + +template<> EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) { + Packet8h result; + result.x = _mm_load_si128(reinterpret_cast<const __m128i*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) { + Packet8h result; + result.x = _mm_loadu_si128(reinterpret_cast<const __m128i*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8h& from) { + _mm_store_si128(reinterpret_cast<__m128i*>(to), from.x); +} + +template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8h& from) { + _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from.x); +} + +template<> EIGEN_STRONG_INLINE Packet8h +ploaddup<Packet8h>(const Eigen::half* from) { + Packet8h result; + unsigned short a = from[0].x; + unsigned short b = from[1].x; + unsigned short c = from[2].x; + unsigned short d = from[3].x; + result.x = _mm_set_epi16(d, d, c, c, b, b, a, a); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet8h +ploadquad<Packet8h>(const Eigen::half* from) { + Packet8h result; + unsigned short a = from[0].x; + unsigned short b = from[1].x; + result.x = _mm_set_epi16(b, b, b, b, a, a, a, a); + return result; +} + +EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { +#ifdef EIGEN_HAS_FP16_C + return _mm256_cvtph_ps(a.x); +#else + EIGEN_ALIGN32 Eigen::half aux[8]; + pstore(aux, a); + float f0(aux[0]); + float f1(aux[1]); + float f2(aux[2]); + float f3(aux[3]); + float f4(aux[4]); + float f5(aux[5]); + float f6(aux[6]); + float f7(aux[7]); + + return _mm256_set_ps(f7, f6, f5, f4, f3, f2, f1, f0); +#endif +} + +EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { +#ifdef EIGEN_HAS_FP16_C + Packet8h result; + result.x = _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); + return result; +#else + EIGEN_ALIGN32 float aux[8]; + pstore(aux, a); + Eigen::half h0(aux[0]); + Eigen::half h1(aux[1]); + Eigen::half h2(aux[2]); + Eigen::half h3(aux[3]); + Eigen::half h4(aux[4]); + Eigen::half h5(aux[5]); + Eigen::half h6(aux[6]); + Eigen::half h7(aux[7]); + + Packet8h result; + result.x = _mm_set_epi16(h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); + return result; +#endif +} + +template<> EIGEN_STRONG_INLINE Packet8h ptrue(const Packet8h& a) { + Packet8h r; r.x = _mm_cmpeq_epi32(a.x, a.x); return r; +} + +template<> EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a,const Packet8h& b) { + // in some cases Packet4i is a wrapper around __m128i, so we either need to + // cast to Packet4i to directly call the intrinsics as below: + Packet8h r; r.x = _mm_or_si128(a.x,b.x); return r; +} +template<> EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a,const Packet8h& b) { + Packet8h r; r.x = _mm_xor_si128(a.x,b.x); return r; +} +template<> EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a,const Packet8h& b) { + Packet8h r; r.x = _mm_and_si128(a.x,b.x); return r; +} +template<> EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a,const Packet8h& b) { + Packet8h r; r.x = _mm_andnot_si128(b.x,a.x); return r; +} + +template<> EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a,const Packet8h& b) { + Packet8f af = half2float(a); + Packet8f bf = half2float(b); + Packet8f rf = pcmp_eq(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet8h pconj(const Packet8h& a) { return a; } + +template<> EIGEN_STRONG_INLINE Packet8h pnegate(const Packet8h& a) { + // FIXME we could do that with bit manipulation + Packet8f af = half2float(a); + Packet8f rf = pnegate(af); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) { + Packet8f af = half2float(a); + Packet8f bf = half2float(b); + Packet8f rf = padd(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet8h psub<Packet8h>(const Packet8h& a, const Packet8h& b) { + Packet8f af = half2float(a); + Packet8f bf = half2float(b); + Packet8f rf = psub(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) { + Packet8f af = half2float(a); + Packet8f bf = half2float(b); + Packet8f rf = pmul(af, bf); + return float2half(rf); +} + +template<> EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride) +{ + Packet8h result; + result.x = _mm_set_epi16(from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); + return result; +} + +template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride) +{ + EIGEN_ALIGN32 Eigen::half aux[8]; + pstore(aux, from); + to[stride*0].x = aux[0].x; + to[stride*1].x = aux[1].x; + to[stride*2].x = aux[2].x; + to[stride*3].x = aux[3].x; + to[stride*4].x = aux[4].x; + to[stride*5].x = aux[5].x; + to[stride*6].x = aux[6].x; + to[stride*7].x = aux[7].x; +} + +template<> EIGEN_STRONG_INLINE Eigen::half predux<Packet8h>(const Packet8h& a) { + Packet8f af = half2float(a); + float reduced = predux<Packet8f>(af); + return Eigen::half(reduced); +} + +template<> EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8h>(const Packet8h& a) { + Packet8f af = half2float(a); + float reduced = predux_max<Packet8f>(af); + return Eigen::half(reduced); +} + +template<> EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8h>(const Packet8h& a) { + Packet8f af = half2float(a); + float reduced = predux_min<Packet8f>(af); + return Eigen::half(reduced); +} + +template<> EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8h>(const Packet8h& a) { + Packet8f af = half2float(a); + float reduced = predux_mul<Packet8f>(af); + return Eigen::half(reduced); +} + +template<> EIGEN_STRONG_INLINE Packet8h preduxp<Packet8h>(const Packet8h* p) { + Packet8f pf[8]; + pf[0] = half2float(p[0]); + pf[1] = half2float(p[1]); + pf[2] = half2float(p[2]); + pf[3] = half2float(p[3]); + pf[4] = half2float(p[4]); + pf[5] = half2float(p[5]); + pf[6] = half2float(p[6]); + pf[7] = half2float(p[7]); + Packet8f reduced = preduxp<Packet8f>(pf); + return float2half(reduced); +} + +template<> EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) +{ + __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); + Packet8h res; + res.x = _mm_shuffle_epi8(a.x,m); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet8h pinsertfirst(const Packet8h& a, Eigen::half b) +{ + Packet8h res; + res.x = _mm_insert_epi16(a.x,int(b.x),0); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet8h pinsertlast(const Packet8h& a, Eigen::half b) +{ + Packet8h res; + res.x = _mm_insert_epi16(a.x,int(b.x),7); + return res; +} + +template<int Offset> +struct palign_impl<Offset,Packet8h> +{ + static EIGEN_STRONG_INLINE void run(Packet8h& first, const Packet8h& second) + { + if (Offset!=0) + first.x = _mm_alignr_epi8(second.x,first.x, Offset*2); + } +}; + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet8h,8>& kernel) { + __m128i a = kernel.packet[0].x; + __m128i b = kernel.packet[1].x; + __m128i c = kernel.packet[2].x; + __m128i d = kernel.packet[3].x; + __m128i e = kernel.packet[4].x; + __m128i f = kernel.packet[5].x; + __m128i g = kernel.packet[6].x; + __m128i h = kernel.packet[7].x; + + __m128i a03b03 = _mm_unpacklo_epi16(a, b); + __m128i c03d03 = _mm_unpacklo_epi16(c, d); + __m128i e03f03 = _mm_unpacklo_epi16(e, f); + __m128i g03h03 = _mm_unpacklo_epi16(g, h); + __m128i a47b47 = _mm_unpackhi_epi16(a, b); + __m128i c47d47 = _mm_unpackhi_epi16(c, d); + __m128i e47f47 = _mm_unpackhi_epi16(e, f); + __m128i g47h47 = _mm_unpackhi_epi16(g, h); + + __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); + __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); + __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); + __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); + __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); + __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); + __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); + __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); + + __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); + __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); + __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); + __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); + __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); + __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); + __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); + __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); + + kernel.packet[0].x = a0b0c0d0e0f0g0h0; + kernel.packet[1].x = a1b1c1d1e1f1g1h1; + kernel.packet[2].x = a2b2c2d2e2f2g2h2; + kernel.packet[3].x = a3b3c3d3e3f3g3h3; + kernel.packet[4].x = a4b4c4d4e4f4g4h4; + kernel.packet[5].x = a5b5c5d5e5f5g5h5; + kernel.packet[6].x = a6b6c6d6e6f6g6h6; + kernel.packet[7].x = a7b7c7d7e7f7g7h7; +} + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet8h,4>& kernel) { + EIGEN_ALIGN32 Eigen::half in[4][8]; + pstore<Eigen::half>(in[0], kernel.packet[0]); + pstore<Eigen::half>(in[1], kernel.packet[1]); + pstore<Eigen::half>(in[2], kernel.packet[2]); + pstore<Eigen::half>(in[3], kernel.packet[3]); + + EIGEN_ALIGN32 Eigen::half out[4][8]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][2*i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j+4] = in[j][2*i+1]; + } + } + + kernel.packet[0] = pload<Packet8h>(out[0]); + kernel.packet[1] = pload<Packet8h>(out[1]); + kernel.packet[2] = pload<Packet8h>(out[2]); + kernel.packet[3] = pload<Packet8h>(out[3]); +} + + +// Disable the following code since it's broken on too many platforms / compilers. +//#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) +#elif 0 + +typedef struct { + __m64 x; +} Packet4h; + + +template<> struct is_arithmetic<Packet4h> { enum { value = true }; }; + +template <> +struct packet_traits<Eigen::half> : default_packet_traits { + typedef Packet4h type; + // There is no half-size packet for Packet4h. + typedef Packet4h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + HasHalfPacket = 0, + HasAdd = 0, + HasSub = 0, + HasMul = 0, + HasNegate = 0, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasConj = 0, + HasSetLinear = 0, + HasDiv = 0, + HasSqrt = 0, + HasRsqrt = 0, + HasExp = 0, + HasLog = 0, + HasBlend = 0 + }; +}; + + +template<> struct unpacket_traits<Packet4h> { typedef Eigen::half type; enum {size=4, alignment=Aligned16, vectorizable=true}; typedef Packet4h half; }; + +template<> EIGEN_STRONG_INLINE Packet4h pset1<Packet4h>(const Eigen::half& from) { + Packet4h result; + result.x = _mm_set1_pi16(from.x); + return result; +} + +template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h>(const Packet4h& from) { + return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_cvtsi64_si32(from.x))); +} + +template<> EIGEN_STRONG_INLINE Packet4h pconj(const Packet4h& a) { return a; } + +template<> EIGEN_STRONG_INLINE Packet4h padd<Packet4h>(const Packet4h& a, const Packet4h& b) { + __int64_t a64 = _mm_cvtm64_si64(a.x); + __int64_t b64 = _mm_cvtm64_si64(b.x); + + Eigen::half h[4]; + + Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64)); + Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64)); + h[0] = ha + hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16)); + h[1] = ha + hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32)); + h[2] = ha + hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48)); + h[3] = ha + hb; + Packet4h result; + result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet4h pmul<Packet4h>(const Packet4h& a, const Packet4h& b) { + __int64_t a64 = _mm_cvtm64_si64(a.x); + __int64_t b64 = _mm_cvtm64_si64(b.x); + + Eigen::half h[4]; + + Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64)); + Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64)); + h[0] = ha * hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16)); + h[1] = ha * hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32)); + h[2] = ha * hb; + ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48)); + hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48)); + h[3] = ha * hb; + Packet4h result; + result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet4h pload<Packet4h>(const Eigen::half* from) { + Packet4h result; + result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE Packet4h ploadu<Packet4h>(const Eigen::half* from) { + Packet4h result; + result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from)); + return result; +} + +template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4h& from) { + __int64_t r = _mm_cvtm64_si64(from.x); + *(reinterpret_cast<__int64_t*>(to)) = r; +} + +template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4h& from) { + __int64_t r = _mm_cvtm64_si64(from.x); + *(reinterpret_cast<__int64_t*>(to)) = r; +} + +template<> EIGEN_STRONG_INLINE Packet4h +ploadquad<Packet4h>(const Eigen::half* from) { + return pset1<Packet4h>(*from); +} + +template<> EIGEN_STRONG_INLINE Packet4h pgather<Eigen::half, Packet4h>(const Eigen::half* from, Index stride) +{ + Packet4h result; + result.x = _mm_set_pi16(from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); + return result; +} + +template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4h>(Eigen::half* to, const Packet4h& from, Index stride) +{ + __int64_t a = _mm_cvtm64_si64(from.x); + to[stride*0].x = static_cast<unsigned short>(a); + to[stride*1].x = static_cast<unsigned short>(a >> 16); + to[stride*2].x = static_cast<unsigned short>(a >> 32); + to[stride*3].x = static_cast<unsigned short>(a >> 48); +} + +EIGEN_STRONG_INLINE void +ptranspose(PacketBlock<Packet4h,4>& kernel) { + __m64 T0 = _mm_unpacklo_pi16(kernel.packet[0].x, kernel.packet[1].x); + __m64 T1 = _mm_unpacklo_pi16(kernel.packet[2].x, kernel.packet[3].x); + __m64 T2 = _mm_unpackhi_pi16(kernel.packet[0].x, kernel.packet[1].x); + __m64 T3 = _mm_unpackhi_pi16(kernel.packet[2].x, kernel.packet[3].x); + + kernel.packet[0].x = _mm_unpacklo_pi32(T0, T1); + kernel.packet[1].x = _mm_unpackhi_pi32(T0, T1); + kernel.packet[2].x = _mm_unpacklo_pi32(T2, T3); + kernel.packet[3].x = _mm_unpackhi_pi32(T2, T3); +} + +#endif + +} +} + +#endif // EIGEN_PACKET_MATH_HALF_GPU_H
diff --git a/Eigen/src/Core/arch/GPU/TypeCasting.h b/Eigen/src/Core/arch/GPU/TypeCasting.h new file mode 100644 index 0000000..57a55d0 --- /dev/null +++ b/Eigen/src/Core/arch/GPU/TypeCasting.h
@@ -0,0 +1,216 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_GPU_H +#define EIGEN_TYPE_CASTING_GPU_H + +namespace Eigen { + +namespace internal { + +template<> +struct scalar_cast_op<float, Eigen::half> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) + typedef Eigen::half result_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const float& a) const { + #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __float2half(a); + #else + return Eigen::half(a); + #endif + } +}; + +template<> +struct functor_traits<scalar_cast_op<float, Eigen::half> > +{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; + + +template<> +struct scalar_cast_op<int, Eigen::half> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) + typedef Eigen::half result_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const int& a) const { + #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __float2half(static_cast<float>(a)); + #else + return Eigen::half(static_cast<float>(a)); + #endif + } +}; + +template<> +struct functor_traits<scalar_cast_op<int, Eigen::half> > +{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; + + +template<> +struct scalar_cast_op<Eigen::half, float> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) + typedef float result_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const Eigen::half& a) const { + #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + return __half2float(a); + #else + return static_cast<float>(a); + #endif + } +}; + +template<> +struct functor_traits<scalar_cast_op<Eigen::half, float> > +{ enum { Cost = NumTraits<float>::AddCost, PacketAccess = false }; }; + + + +#if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ + (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) + +template <> +struct type_casting_traits<Eigen::half, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 2, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast<half2, float4>(const half2& a, const half2& b) { + float2 r1 = __half22float2(a); + float2 r2 = __half22float2(b); + return make_float4(r1.x, r1.y, r2.x, r2.y); +} + +template <> +struct type_casting_traits<float, Eigen::half> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 2 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcast<float4, half2>(const float4& a) { + // Simply discard the second half of the input + return __floats2half2_rn(a.x, a.y); +} + +#elif defined EIGEN_VECTORIZE_AVX512 +template <> +struct type_casting_traits<half, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet16f pcast<Packet16h, Packet16f>(const Packet16h& a) { + return half2float(a); +} + +template <> +struct type_casting_traits<float, half> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet16h pcast<Packet16f, Packet16h>(const Packet16f& a) { + return float2half(a); +} + +#elif defined EIGEN_VECTORIZE_AVX + +template <> +struct type_casting_traits<Eigen::half, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet8f pcast<Packet8h, Packet8f>(const Packet8h& a) { + return half2float(a); +} + +template <> +struct type_casting_traits<float, Eigen::half> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(const Packet8f& a) { + return float2half(a); +} + +// Disable the following code since it's broken on too many platforms / compilers. +//#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) +#elif 0 + +template <> +struct type_casting_traits<Eigen::half, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet4h, Packet4f>(const Packet4h& a) { + __int64_t a64 = _mm_cvtm64_si64(a.x); + Eigen::half h = raw_uint16_to_half(static_cast<unsigned short>(a64)); + float f1 = static_cast<float>(h); + h = raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16)); + float f2 = static_cast<float>(h); + h = raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32)); + float f3 = static_cast<float>(h); + h = raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48)); + float f4 = static_cast<float>(h); + return _mm_set_ps(f4, f3, f2, f1); +} + +template <> +struct type_casting_traits<float, Eigen::half> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_STRONG_INLINE Packet4h pcast<Packet4f, Packet4h>(const Packet4f& a) { + EIGEN_ALIGN16 float aux[4]; + pstore(aux, a); + Eigen::half h0(aux[0]); + Eigen::half h1(aux[1]); + Eigen::half h2(aux[2]); + Eigen::half h3(aux[3]); + + Packet4h result; + result.x = _mm_set_pi16(h3.x, h2.x, h1.x, h0.x); + return result; +} + +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_GPU_H
diff --git a/Eigen/src/Core/arch/HIP/hcc/math_constants.h b/Eigen/src/Core/arch/HIP/hcc/math_constants.h new file mode 100644 index 0000000..25375a0 --- /dev/null +++ b/Eigen/src/Core/arch/HIP/hcc/math_constants.h
@@ -0,0 +1,23 @@ +/* + * math_constants.h - + * HIP equivalent of the CUDA header of the same name + */ + +#ifndef __MATH_CONSTANTS_H__ +#define __MATH_CONSTANTS_H__ + +/* single precision constants */ + +#define HIPRT_INF_F __int_as_float(0x7f800000) +#define HIPRT_NAN_F __int_as_float(0x7fffffff) +#define HIPRT_MIN_DENORM_F __int_as_float(0x00000001) +#define HIPRT_MAX_NORMAL_F __int_as_float(0x7f7fffff) +#define HIPRT_NEG_ZERO_F __int_as_float(0x80000000) +#define HIPRT_ZERO_F 0.0f +#define HIPRT_ONE_F 1.0f + +/* double precision constants */ +#define HIPRT_INF __hiloint2double(0x7ff00000, 0x00000000) +#define HIPRT_NAN __hiloint2double(0xfff80000, 0x00000000) + +#endif
diff --git a/Eigen/src/Core/arch/MSA/Complex.h b/Eigen/src/Core/arch/MSA/Complex.h new file mode 100644 index 0000000..fa64d35 --- /dev/null +++ b/Eigen/src/Core/arch/MSA/Complex.h
@@ -0,0 +1,759 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Wave Computing, Inc. +// Written by: +// Chris Larsen +// Alexey Frunze (afrunze@wavecomp.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_COMPLEX_MSA_H +#define EIGEN_COMPLEX_MSA_H + +#include <iostream> + +namespace Eigen { + +namespace internal { + +//---------- float ---------- +struct Packet2cf { + EIGEN_STRONG_INLINE Packet2cf() { + } + EIGEN_STRONG_INLINE explicit Packet2cf(const std::complex<float>& a, + const std::complex<float>& b) { + Packet4f t = { std::real(a), std::imag(a), std::real(b), std::imag(b) }; + v = t; + } + EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) { + } + EIGEN_STRONG_INLINE Packet2cf(const Packet2cf& a) : v(a.v) { + } + EIGEN_STRONG_INLINE Packet2cf& operator=(const Packet2cf& b) { + v = b.v; + return *this; + } + EIGEN_STRONG_INLINE Packet2cf conjugate(void) const { + return Packet2cf((Packet4f)__builtin_msa_bnegi_d((v2u64)v, 63)); + } + EIGEN_STRONG_INLINE Packet2cf& operator*=(const Packet2cf& b) { + Packet4f v1, v2; + + // Get the real values of a | a1_re | a1_re | a2_re | a2_re | + v1 = (Packet4f)__builtin_msa_ilvev_w((v4i32)v, (v4i32)v); + // Get the imag values of a | a1_im | a1_im | a2_im | a2_im | + v2 = (Packet4f)__builtin_msa_ilvod_w((v4i32)v, (v4i32)v); + // Multiply the real a with b + v1 = pmul(v1, b.v); + // Multiply the imag a with b + v2 = pmul(v2, b.v); + // Conjugate v2 + v2 = Packet2cf(v2).conjugate().v; + // Swap real/imag elements in v2. + v2 = (Packet4f)__builtin_msa_shf_w((v4i32)v2, EIGEN_MSA_SHF_I8(1, 0, 3, 2)); + // Add and return the result + v = padd(v1, v2); + return *this; + } + EIGEN_STRONG_INLINE Packet2cf operator*(const Packet2cf& b) const { + return Packet2cf(*this) *= b; + } + EIGEN_STRONG_INLINE Packet2cf& operator+=(const Packet2cf& b) { + v = padd(v, b.v); + return *this; + } + EIGEN_STRONG_INLINE Packet2cf operator+(const Packet2cf& b) const { + return Packet2cf(*this) += b; + } + EIGEN_STRONG_INLINE Packet2cf& operator-=(const Packet2cf& b) { + v = psub(v, b.v); + return *this; + } + EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const { + return Packet2cf(*this) -= b; + } + EIGEN_STRONG_INLINE Packet2cf& operator/=(const Packet2cf& b) { + *this *= b.conjugate(); + Packet4f s = pmul<Packet4f>(b.v, b.v); + s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + v = pdiv(v, s); + return *this; + } + EIGEN_STRONG_INLINE Packet2cf operator/(const Packet2cf& b) const { + return Packet2cf(*this) /= b; + } + EIGEN_STRONG_INLINE Packet2cf operator-(void) const { + return Packet2cf(pnegate(v)); + } + + Packet4f v; +}; + +inline std::ostream& operator<<(std::ostream& os, const Packet2cf& value) { + os << "[ (" << value.v[0] << ", " << value.v[1] + << "i)," + " (" + << value.v[2] << ", " << value.v[3] << "i) ]"; + return os; +} + +template <> +struct packet_traits<std::complex<float> > : default_packet_traits { + typedef Packet2cf type; + typedef Packet2cf half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + HasHalfPacket = 0, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0, + HasBlend = 1 + }; +}; + +template <> +struct unpacket_traits<Packet2cf> { + typedef std::complex<float> type; + enum { size = 2, alignment = Aligned16, vectorizable=true }; + typedef Packet2cf half; +}; + +template <> +EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) { + EIGEN_MSA_DEBUG; + + float f0 = from.real(), f1 = from.imag(); + Packet4f v0 = { f0, f0, f0, f0 }; + Packet4f v1 = { f1, f1, f1, f1 }; + return Packet2cf((Packet4f)__builtin_msa_ilvr_w((Packet4i)v1, (Packet4i)v0)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return a + b; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return a - b; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return -a; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return a.conjugate(); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return a * b; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pand<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return Packet2cf(pand(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf por<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return Packet2cf(por(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pxor<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return Packet2cf(pxor(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return Packet2cf(pandnot(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>((const float*)from)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>((const float*)from)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) { + EIGEN_MSA_DEBUG; + + return pset1<Packet2cf>(*from); +} + +template <> +EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, + const Packet2cf& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_STORE pstore<float>((float*)to, from.v); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, + const Packet2cf& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_STORE pstoreu<float>((float*)to, from.v); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>( + const std::complex<float>* from, Index stride) { + EIGEN_MSA_DEBUG; + + return Packet2cf(from[0 * stride], from[1 * stride]); +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, + const Packet2cf& from, + Index stride) { + EIGEN_MSA_DEBUG; + + *to = std::complex<float>(from.v[0], from.v[1]); + to += stride; + *to = std::complex<float>(from.v[2], from.v[3]); +} + +template <> +EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float>* addr) { + EIGEN_MSA_DEBUG; + + prefetch(reinterpret_cast<const float*>(addr)); +} + +template <> +EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return std::complex<float>(a.v[0], a.v[1]); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return Packet2cf((Packet4f)__builtin_msa_shf_w((v4i32)a.v, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return Packet2cf((Packet4f)__builtin_msa_shf_w((v4i32)a.v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); +} + +template <> +EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + Packet4f value = (Packet4f)preverse((Packet2d)a.v); + value += a.v; + return std::complex<float>(value[0], value[1]); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf preduxp<Packet2cf>(const Packet2cf* vecs) { + EIGEN_MSA_DEBUG; + + Packet4f sum1, sum2, sum; + + // Add the first two 64-bit float32x2_t of vecs[0] + sum1 = (Packet4f)__builtin_msa_ilvr_d((v2i64)vecs[1].v, (v2i64)vecs[0].v); + sum2 = (Packet4f)__builtin_msa_ilvl_d((v2i64)vecs[1].v, (v2i64)vecs[0].v); + sum = padd(sum1, sum2); + + return Packet2cf(sum); +} + +template <> +EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) { + EIGEN_MSA_DEBUG; + + return std::complex<float>((a.v[0] * a.v[2]) - (a.v[1] * a.v[3]), + (a.v[0] * a.v[3]) + (a.v[1] * a.v[2])); +} + +template <int Offset> +struct palign_impl<Offset, Packet2cf> { + EIGEN_STRONG_INLINE static void run(Packet2cf& first, const Packet2cf& second) { + if (Offset == 1) { + first.v = (Packet4f)__builtin_msa_sldi_b((v16i8)second.v, (v16i8)first.v, Offset * 8); + } + } +}; + +template <> +struct conj_helper<Packet2cf, Packet2cf, false, true> { + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, + const Packet2cf& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const { + return internal::pmul(a, pconj(b)); + } +}; + +template <> +struct conj_helper<Packet2cf, Packet2cf, true, false> { + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, + const Packet2cf& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const { + return internal::pmul(pconj(a), b); + } +}; + +template <> +struct conj_helper<Packet2cf, Packet2cf, true, true> { + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, + const Packet2cf& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f) + +template <> +EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { + EIGEN_MSA_DEBUG; + + return a / b; +} + +inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet2cf, 2>& value) { + os << "[ " << value.packet[0] << ", " << std::endl << " " << value.packet[1] << " ]"; + return os; +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2cf, 2>& kernel) { + EIGEN_MSA_DEBUG; + + Packet4f tmp = + (Packet4f)__builtin_msa_ilvl_d((v2i64)kernel.packet[1].v, (v2i64)kernel.packet[0].v); + kernel.packet[0].v = + (Packet4f)__builtin_msa_ilvr_d((v2i64)kernel.packet[1].v, (v2i64)kernel.packet[0].v); + kernel.packet[1].v = tmp; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, + const Packet2cf& elsePacket) { + return (Packet2cf)(Packet4f)pblend<Packet2d>(ifPacket, (Packet2d)thenPacket.v, + (Packet2d)elsePacket.v); +} + +//---------- double ---------- + +struct Packet1cd { + EIGEN_STRONG_INLINE Packet1cd() { + } + EIGEN_STRONG_INLINE explicit Packet1cd(const std::complex<double>& a) { + v[0] = std::real(a); + v[1] = std::imag(a); + } + EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) { + } + EIGEN_STRONG_INLINE Packet1cd(const Packet1cd& a) : v(a.v) { + } + EIGEN_STRONG_INLINE Packet1cd& operator=(const Packet1cd& b) { + v = b.v; + return *this; + } + EIGEN_STRONG_INLINE Packet1cd conjugate(void) const { + static const v2u64 p2ul_CONJ_XOR = { 0x0, 0x8000000000000000 }; + return (Packet1cd)pxor(v, (Packet2d)p2ul_CONJ_XOR); + } + EIGEN_STRONG_INLINE Packet1cd& operator*=(const Packet1cd& b) { + Packet2d v1, v2; + + // Get the real values of a | a1_re | a1_re + v1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)v, (v2i64)v); + // Get the imag values of a | a1_im | a1_im + v2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)v, (v2i64)v); + // Multiply the real a with b + v1 = pmul(v1, b.v); + // Multiply the imag a with b + v2 = pmul(v2, b.v); + // Conjugate v2 + v2 = Packet1cd(v2).conjugate().v; + // Swap real/imag elements in v2. + v2 = (Packet2d)__builtin_msa_shf_w((v4i32)v2, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); + // Add and return the result + v = padd(v1, v2); + return *this; + } + EIGEN_STRONG_INLINE Packet1cd operator*(const Packet1cd& b) const { + return Packet1cd(*this) *= b; + } + EIGEN_STRONG_INLINE Packet1cd& operator+=(const Packet1cd& b) { + v = padd(v, b.v); + return *this; + } + EIGEN_STRONG_INLINE Packet1cd operator+(const Packet1cd& b) const { + return Packet1cd(*this) += b; + } + EIGEN_STRONG_INLINE Packet1cd& operator-=(const Packet1cd& b) { + v = psub(v, b.v); + return *this; + } + EIGEN_STRONG_INLINE Packet1cd operator-(const Packet1cd& b) const { + return Packet1cd(*this) -= b; + } + EIGEN_STRONG_INLINE Packet1cd& operator/=(const Packet1cd& b) { + *this *= b.conjugate(); + Packet2d s = pmul<Packet2d>(b.v, b.v); + s = padd(s, preverse<Packet2d>(s)); + v = pdiv(v, s); + return *this; + } + EIGEN_STRONG_INLINE Packet1cd operator/(const Packet1cd& b) const { + return Packet1cd(*this) /= b; + } + EIGEN_STRONG_INLINE Packet1cd operator-(void) const { + return Packet1cd(pnegate(v)); + } + + Packet2d v; +}; + +inline std::ostream& operator<<(std::ostream& os, const Packet1cd& value) { + os << "[ (" << value.v[0] << ", " << value.v[1] << "i) ]"; + return os; +} + +template <> +struct packet_traits<std::complex<double> > : default_packet_traits { + typedef Packet1cd type; + typedef Packet1cd half; + enum { + Vectorizable = 1, + AlignedOnScalar = 0, + size = 1, + HasHalfPacket = 0, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0 + }; +}; + +template <> +struct unpacket_traits<Packet1cd> { + typedef std::complex<double> type; + enum { size = 1, alignment = Aligned16, vectorizable=true }; + typedef Packet1cd half; +}; + +template <> +EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pset1<Packet1cd>(const std::complex<double>& from) { + EIGEN_MSA_DEBUG; + + return Packet1cd(from); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return a + b; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return a - b; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return -a; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return a.conjugate(); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return a * b; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pand<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return Packet1cd(pand(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd por<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return Packet1cd(por(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pxor<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return Packet1cd(pxor(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return Packet1cd(pandnot(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) { + EIGEN_MSA_DEBUG; + + return pset1<Packet1cd>(*from); +} + +template <> +EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, + const Packet1cd& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_STORE pstore<double>((double*)to, from.v); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, + const Packet1cd& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_STORE pstoreu<double>((double*)to, from.v); +} + +template <> +EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double>* addr) { + EIGEN_MSA_DEBUG; + + prefetch(reinterpret_cast<const double*>(addr)); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>( + const std::complex<double>* from, Index stride __attribute__((unused))) { + EIGEN_MSA_DEBUG; + + Packet1cd res; + res.v[0] = std::real(from[0]); + res.v[1] = std::imag(from[0]); + return res; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, + const Packet1cd& from, + Index stride + __attribute__((unused))) { + EIGEN_MSA_DEBUG; + + pstore(to, from); +} + +template <> +EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return std::complex<double>(a.v[0], a.v[1]); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return a; +} + +template <> +EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return pfirst(a); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd preduxp<Packet1cd>(const Packet1cd* vecs) { + EIGEN_MSA_DEBUG; + + return vecs[0]; +} + +template <> +EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) { + EIGEN_MSA_DEBUG; + + return pfirst(a); +} + +template <int Offset> +struct palign_impl<Offset, Packet1cd> { + static EIGEN_STRONG_INLINE void run(Packet1cd& /*first*/, const Packet1cd& /*second*/) { + // FIXME is it sure we never have to align a Packet1cd? + // Even though a std::complex<double> has 16 bytes, it is not necessarily aligned on a 16 bytes + // boundary... + } +}; + +template <> +struct conj_helper<Packet1cd, Packet1cd, false, true> { + EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, + const Packet1cd& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const { + return internal::pmul(a, pconj(b)); + } +}; + +template <> +struct conj_helper<Packet1cd, Packet1cd, true, false> { + EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, + const Packet1cd& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const { + return internal::pmul(pconj(a), b); + } +}; + +template <> +struct conj_helper<Packet1cd, Packet1cd, true, true> { + EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet1cd& y, + const Packet1cd& c) const { + return padd(pmul(x, y), c); + } + + EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) const { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d) + +template <> +EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { + EIGEN_MSA_DEBUG; + + return a / b; +} + +EIGEN_STRONG_INLINE Packet1cd pcplxflip /*<Packet1cd>*/ (const Packet1cd& x) { + EIGEN_MSA_DEBUG; + + return Packet1cd(preverse(Packet2d(x.v))); +} + +inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet1cd, 2>& value) { + os << "[ " << value.packet[0] << ", " << std::endl << " " << value.packet[1] << " ]"; + return os; +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd, 2>& kernel) { + EIGEN_MSA_DEBUG; + + Packet2d v1, v2; + + v1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[0].v, (v2i64)kernel.packet[1].v); + // Get the imag values of a + v2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[0].v, (v2i64)kernel.packet[1].v); + + kernel.packet[0].v = v1; + kernel.packet[1].v = v2; +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_COMPLEX_MSA_H
diff --git a/Eigen/src/Core/arch/MSA/MathFunctions.h b/Eigen/src/Core/arch/MSA/MathFunctions.h new file mode 100644 index 0000000..f5181b9 --- /dev/null +++ b/Eigen/src/Core/arch/MSA/MathFunctions.h
@@ -0,0 +1,387 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2007 Julien Pommier +// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// Copyright (C) 2018 Wave Computing, Inc. +// Written by: +// Chris Larsen +// Alexey Frunze (afrunze@wavecomp.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/* The sin, cos, exp, and log functions of this file come from + * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ + */ + +/* The tanh function of this file is an adaptation of + * template<typename T> T generic_fast_tanh_float(const T&) + * from MathFunctionsImpl.h. + */ + +#ifndef EIGEN_MATH_FUNCTIONS_MSA_H +#define EIGEN_MATH_FUNCTIONS_MSA_H + +namespace Eigen { + +namespace internal { + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +plog<Packet4f>(const Packet4f& _x) { + static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292e-2f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, -1.1514610310e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, -1.2420140846e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, +1.4249322787e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, -1.6668057665e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, +2.0000714765e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, -2.4999993993e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, +3.3333331174e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); + static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + + // Convert negative argument into NAN (quiet negative, to be specific). + Packet4f zero = (Packet4f)__builtin_msa_ldi_w(0); + Packet4i neg_mask = __builtin_msa_fclt_w(_x, zero); + Packet4i zero_mask = __builtin_msa_fceq_w(_x, zero); + Packet4f non_neg_x_or_nan = padd(_x, (Packet4f)neg_mask); // Add 0.0 or NAN. + Packet4f x = non_neg_x_or_nan; + + // Extract exponent from x = mantissa * 2**exponent, where 1.0 <= mantissa < 2.0. + // N.B. the exponent is one less of what frexpf() would return. + Packet4i e_int = __builtin_msa_ftint_s_w(__builtin_msa_flog2_w(x)); + // Multiply x by 2**(-exponent-1) to get 0.5 <= x < 1.0 as from frexpf(). + x = __builtin_msa_fexp2_w(x, (Packet4i)__builtin_msa_nori_b((v16u8)e_int, 0)); + + /* + if (x < SQRTHF) { + x = x + x - 1.0; + } else { + e += 1; + x = x - 1.0; + } + */ + Packet4f xx = padd(x, x); + Packet4i ge_mask = __builtin_msa_fcle_w(p4f_cephes_SQRTHF, x); + e_int = psub(e_int, ge_mask); + x = (Packet4f)__builtin_msa_bsel_v((v16u8)ge_mask, (v16u8)xx, (v16u8)x); + x = psub(x, p4f_1); + Packet4f e = __builtin_msa_ffint_s_w(e_int); + + Packet4f x2 = pmul(x, x); + Packet4f x3 = pmul(x2, x); + + Packet4f y, y1, y2; + y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1); + y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4); + y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7); + y = pmadd(y, x, p4f_cephes_log_p2); + y1 = pmadd(y1, x, p4f_cephes_log_p5); + y2 = pmadd(y2, x, p4f_cephes_log_p8); + y = pmadd(y, x3, y1); + y = pmadd(y, x3, y2); + y = pmul(y, x3); + + y = pmadd(e, p4f_cephes_log_q1, y); + x = __builtin_msa_fmsub_w(x, x2, p4f_half); + x = padd(x, y); + x = pmadd(e, p4f_cephes_log_q2, x); + + // x is now the logarithm result candidate. We still need to handle the + // extreme arguments of zero and positive infinity, though. + // N.B. if the argument is +INFINITY, x is NAN because the polynomial terms + // contain infinities of both signs (see the coefficients and code above). + // INFINITY - INFINITY is NAN. + + // If the argument is +INFINITY, make it the new result candidate. + // To achieve that we choose the smaller of the result candidate and the + // argument. + // This is correct for all finite pairs of values (the logarithm is smaller + // than the argument). + // This is also correct in the special case when the argument is +INFINITY + // and the result candidate is NAN. This is because the fmin.df instruction + // prefers non-NANs to NANs. + x = __builtin_msa_fmin_w(x, non_neg_x_or_nan); + + // If the argument is zero (including -0.0), the result becomes -INFINITY. + Packet4i neg_infs = __builtin_msa_slli_w(zero_mask, 23); + x = (Packet4f)__builtin_msa_bsel_v((v16u8)zero_mask, (v16u8)x, (v16u8)neg_infs); + + return x; +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +pexp<Packet4f>(const Packet4f& _x) { + // Limiting single-precision pexp's argument to [-128, +128] lets pexp + // reach 0 and INFINITY naturally. + static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -128.0f); + static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, +128.0f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894e-2f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + + Packet4f x = _x; + + // Clamp x. + x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(x, p4f_exp_lo), (v16u8)x, + (v16u8)p4f_exp_lo); + x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_exp_hi, x), (v16u8)x, + (v16u8)p4f_exp_hi); + + // Round to nearest integer by adding 0.5 (with x's sign) and truncating. + Packet4f x2_add = (Packet4f)__builtin_msa_binsli_w((v4u32)p4f_half, (v4u32)x, 0); + Packet4f x2 = pmadd(x, p4f_cephes_LOG2EF, x2_add); + Packet4i x2_int = __builtin_msa_ftrunc_s_w(x2); + Packet4f x2_int_f = __builtin_msa_ffint_s_w(x2_int); + + x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C1); + x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C2); + + Packet4f z = pmul(x, x); + + Packet4f y = p4f_cephes_exp_p0; + y = pmadd(y, x, p4f_cephes_exp_p1); + y = pmadd(y, x, p4f_cephes_exp_p2); + y = pmadd(y, x, p4f_cephes_exp_p3); + y = pmadd(y, x, p4f_cephes_exp_p4); + y = pmadd(y, x, p4f_cephes_exp_p5); + y = pmadd(y, z, x); + y = padd(y, p4f_1); + + // y *= 2**exponent. + y = __builtin_msa_fexp2_w(y, x2_int); + + return y; +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +ptanh<Packet4f>(const Packet4f& _x) { + static _EIGEN_DECLARE_CONST_Packet4f(tanh_tiny, 1e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(tanh_hi, 9.0f); + // The monomial coefficients of the numerator polynomial (odd). + static _EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-5f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-8f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f); + static _EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f); + // The monomial coefficients of the denominator polynomial (even). + static _EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-6f); + + Packet4f x = pabs(_x); + Packet4i tiny_mask = __builtin_msa_fclt_w(x, p4f_tanh_tiny); + + // Clamp the inputs to the range [-9, 9] since anything outside + // this range is -/+1.0f in single-precision. + x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_tanh_hi, x), (v16u8)x, + (v16u8)p4f_tanh_hi); + + // Since the polynomials are odd/even, we need x**2. + Packet4f x2 = pmul(x, x); + + // Evaluate the numerator polynomial p. + Packet4f p = pmadd(x2, p4f_alpha_13, p4f_alpha_11); + p = pmadd(x2, p, p4f_alpha_9); + p = pmadd(x2, p, p4f_alpha_7); + p = pmadd(x2, p, p4f_alpha_5); + p = pmadd(x2, p, p4f_alpha_3); + p = pmadd(x2, p, p4f_alpha_1); + p = pmul(x, p); + + // Evaluate the denominator polynomial q. + Packet4f q = pmadd(x2, p4f_beta_6, p4f_beta_4); + q = pmadd(x2, q, p4f_beta_2); + q = pmadd(x2, q, p4f_beta_0); + + // Divide the numerator by the denominator. + p = pdiv(p, q); + + // Reinstate the sign. + p = (Packet4f)__builtin_msa_binsli_w((v4u32)p, (v4u32)_x, 0); + + // When the argument is very small in magnitude it's more accurate to just return it. + p = (Packet4f)__builtin_msa_bsel_v((v16u8)tiny_mask, (v16u8)p, (v16u8)_x); + + return p; +} + +template <bool sine> +Packet4f psincos_inner_msa_float(const Packet4f& _x) { + static _EIGEN_DECLARE_CONST_Packet4f(sincos_max_arg, 13176795.0f); // Approx. (2**24) / (4/Pi). + static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1, -0.78515625f); + static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); + static _EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891e-4f); + static _EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611e-1f); + static _EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948e-5f); + static _EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765e-3f); + static _EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827e-2f); + static _EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4/Pi. + static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); + static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); + + Packet4f x = pabs(_x); + + // Translate infinite arguments into NANs. + Packet4f zero_or_nan_if_inf = psub(_x, _x); + x = padd(x, zero_or_nan_if_inf); + // Prevent sin/cos from generating values larger than 1.0 in magnitude + // for very large arguments by setting x to 0.0. + Packet4i small_or_nan_mask = __builtin_msa_fcult_w(x, p4f_sincos_max_arg); + x = pand(x, (Packet4f)small_or_nan_mask); + + // Scale x by 4/Pi to find x's octant. + Packet4f y = pmul(x, p4f_cephes_FOPI); + // Get the octant. We'll reduce x by this number of octants or by one more than it. + Packet4i y_int = __builtin_msa_ftrunc_s_w(y); + // x's from even-numbered octants will translate to octant 0: [0, +Pi/4]. + // x's from odd-numbered octants will translate to octant -1: [-Pi/4, 0]. + // Adjustment for odd-numbered octants: octant = (octant + 1) & (~1). + Packet4i y_int1 = __builtin_msa_addvi_w(y_int, 1); + Packet4i y_int2 = (Packet4i)__builtin_msa_bclri_w((Packet4ui)y_int1, 0); // bclri = bit-clear + y = __builtin_msa_ffint_s_w(y_int2); + + // Compute the sign to apply to the polynomial. + Packet4i sign_mask = sine ? pxor(__builtin_msa_slli_w(y_int1, 29), (Packet4i)_x) + : __builtin_msa_slli_w(__builtin_msa_addvi_w(y_int, 3), 29); + + // Get the polynomial selection mask. + // We'll calculate both (sin and cos) polynomials and then select from the two. + Packet4i poly_mask = __builtin_msa_ceqi_w(__builtin_msa_slli_w(y_int2, 30), 0); + + // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4. + // The magic pass: "Extended precision modular arithmetic" + // x = ((x - y * DP1) - y * DP2) - y * DP3 + Packet4f tmp1 = pmul(y, p4f_minus_cephes_DP1); + Packet4f tmp2 = pmul(y, p4f_minus_cephes_DP2); + Packet4f tmp3 = pmul(y, p4f_minus_cephes_DP3); + x = padd(x, tmp1); + x = padd(x, tmp2); + x = padd(x, tmp3); + + // Evaluate the cos(x) polynomial. + y = p4f_coscof_p0; + Packet4f z = pmul(x, x); + y = pmadd(y, z, p4f_coscof_p1); + y = pmadd(y, z, p4f_coscof_p2); + y = pmul(y, z); + y = pmul(y, z); + y = __builtin_msa_fmsub_w(y, z, p4f_half); + y = padd(y, p4f_1); + + // Evaluate the sin(x) polynomial. + Packet4f y2 = p4f_sincof_p0; + y2 = pmadd(y2, z, p4f_sincof_p1); + y2 = pmadd(y2, z, p4f_sincof_p2); + y2 = pmul(y2, z); + y2 = pmadd(y2, x, x); + + // Select the correct result from the two polynomials. + y = sine ? (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y, (v16u8)y2) + : (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y2, (v16u8)y); + + // Update the sign. + sign_mask = pxor(sign_mask, (Packet4i)y); + y = (Packet4f)__builtin_msa_binsli_w((v4u32)y, (v4u32)sign_mask, 0); // binsli = bit-insert-left + return y; +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +psin<Packet4f>(const Packet4f& x) { + return psincos_inner_msa_float</* sine */ true>(x); +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f +pcos<Packet4f>(const Packet4f& x) { + return psincos_inner_msa_float</* sine */ false>(x); +} + +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d +pexp<Packet2d>(const Packet2d& _x) { + // Limiting double-precision pexp's argument to [-1024, +1024] lets pexp + // reach 0 and INFINITY naturally. + static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -1024.0); + static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, +1024.0); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); + static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); + static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); + static _EIGEN_DECLARE_CONST_Packet2d(1, 1.0); + static _EIGEN_DECLARE_CONST_Packet2d(2, 2.0); + + Packet2d x = _x; + + // Clamp x. + x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(x, p2d_exp_lo), (v16u8)x, + (v16u8)p2d_exp_lo); + x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(p2d_exp_hi, x), (v16u8)x, + (v16u8)p2d_exp_hi); + + // Round to nearest integer by adding 0.5 (with x's sign) and truncating. + Packet2d x2_add = (Packet2d)__builtin_msa_binsli_d((v2u64)p2d_half, (v2u64)x, 0); + Packet2d x2 = pmadd(x, p2d_cephes_LOG2EF, x2_add); + Packet2l x2_long = __builtin_msa_ftrunc_s_d(x2); + Packet2d x2_long_d = __builtin_msa_ffint_s_d(x2_long); + + x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C1); + x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C2); + + x2 = pmul(x, x); + + Packet2d px = p2d_cephes_exp_p0; + px = pmadd(px, x2, p2d_cephes_exp_p1); + px = pmadd(px, x2, p2d_cephes_exp_p2); + px = pmul(px, x); + + Packet2d qx = p2d_cephes_exp_q0; + qx = pmadd(qx, x2, p2d_cephes_exp_q1); + qx = pmadd(qx, x2, p2d_cephes_exp_q2); + qx = pmadd(qx, x2, p2d_cephes_exp_q3); + + x = pdiv(px, psub(qx, px)); + x = pmadd(p2d_2, x, p2d_1); + + // x *= 2**exponent. + x = __builtin_msa_fexp2_d(x, x2_long); + + return x; +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MATH_FUNCTIONS_MSA_H
diff --git a/Eigen/src/Core/arch/MSA/PacketMath.h b/Eigen/src/Core/arch/MSA/PacketMath.h new file mode 100644 index 0000000..a97156a --- /dev/null +++ b/Eigen/src/Core/arch/MSA/PacketMath.h
@@ -0,0 +1,1317 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Wave Computing, Inc. +// Written by: +// Chris Larsen +// Alexey Frunze (afrunze@wavecomp.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_MSA_H +#define EIGEN_PACKET_MATH_MSA_H + +#include <iostream> +#include <string> + +namespace Eigen { + +namespace internal { + +#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD +#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 +#endif + +#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#endif + +#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD +#define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD +#endif + +#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#endif + +#if 0 +#define EIGEN_MSA_DEBUG \ + static bool firstTime = true; \ + do { \ + if (firstTime) { \ + std::cout << __FILE__ << ':' << __LINE__ << ':' << __FUNCTION__ << std::endl; \ + firstTime = false; \ + } \ + } while (0) +#else +#define EIGEN_MSA_DEBUG +#endif + +#define EIGEN_MSA_SHF_I8(a, b, c, d) (((d) << 6) | ((c) << 4) | ((b) << 2) | (a)) + +typedef v4f32 Packet4f; +typedef v4i32 Packet4i; +typedef v4u32 Packet4ui; + +#define _EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = { X, X, X, X } +#define _EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = { X, X, X, X } +#define _EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = { X, X, X, X } + +inline std::ostream& operator<<(std::ostream& os, const Packet4f& value) { + os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const Packet4i& value) { + os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const Packet4ui& value) { + os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; + return os; +} + +template <> +struct packet_traits<float> : default_packet_traits { + typedef Packet4f type; + typedef Packet4f half; // Packet2f intrinsics not implemented yet + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + HasHalfPacket = 0, // Packet2f intrinsics not implemented yet + // FIXME check the Has* + HasDiv = 1, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasTanh = EIGEN_FAST_MATH, + HasLog = 1, + HasExp = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasBlend = 1 + }; +}; + +template <> +struct packet_traits<int32_t> : default_packet_traits { + typedef Packet4i type; + typedef Packet4i half; // Packet2i intrinsics not implemented yet + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + HasHalfPacket = 0, // Packet2i intrinsics not implemented yet + // FIXME check the Has* + HasDiv = 1, + HasBlend = 1 + }; +}; + +template <> +struct unpacket_traits<Packet4f> { + typedef float type; + enum { size = 4, alignment = Aligned16, vectorizable=true }; + typedef Packet4f half; +}; + +template <> +struct unpacket_traits<Packet4i> { + typedef int32_t type; + enum { size = 4, alignment = Aligned16, vectorizable=true }; + typedef Packet4i half; +}; + +template <> +EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { + EIGEN_MSA_DEBUG; + + Packet4f v = { from, from, from, from }; + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fill_w(from); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(const float* from) { + EIGEN_MSA_DEBUG; + + float f = *from; + Packet4f v = { f, f, f, f }; + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4i pload1<Packet4i>(const int32_t* from) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fill_w(*from); +} + +template <> +EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fadd_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_addv_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { + EIGEN_MSA_DEBUG; + + static const Packet4f countdown = { 0.0f, 1.0f, 2.0f, 3.0f }; + return padd(pset1<Packet4f>(a), countdown); +} + +template <> +EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) { + EIGEN_MSA_DEBUG; + + static const Packet4i countdown = { 0, 1, 2, 3 }; + return padd(pset1<Packet4i>(a), countdown); +} + +template <> +EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fsub_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_subv_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_bnegi_w((v4u32)a, 31); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_addvi_w((v4i32)__builtin_msa_nori_b((v16u8)a, 0), 1); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fmul_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_mulv_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fdiv_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_div_s_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fmadd_w(c, a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { + EIGEN_MSA_DEBUG; + + // Use "asm" construct to avoid __builtin_msa_maddv_w GNU C bug. + Packet4i value = c; + __asm__("maddv.w %w[value], %w[a], %w[b]\n" + // Outputs + : [value] "+f"(value) + // Inputs + : [a] "f"(a), [b] "f"(b)); + return value; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_and_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return (Packet4i)__builtin_msa_and_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_or_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return (Packet4i)__builtin_msa_or_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_xor_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return (Packet4i)__builtin_msa_xor_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + + return pand(a, (Packet4f)__builtin_msa_xori_b((v16u8)b, 255)); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return pand(a, (Packet4i)__builtin_msa_xori_b((v16u8)b, 255)); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + // This prefers numbers to NaNs. + return __builtin_msa_fmin_w(a, b); +#else + // This prefers NaNs to numbers. + Packet4i aNaN = __builtin_msa_fcun_w(a, a); + Packet4i aMinOrNaN = por(__builtin_msa_fclt_w(a, b), aNaN); + return (Packet4f)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_min_s_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + // This prefers numbers to NaNs. + return __builtin_msa_fmax_w(a, b); +#else + // This prefers NaNs to numbers. + Packet4i aNaN = __builtin_msa_fcun_w(a, a); + Packet4i aMaxOrNaN = por(__builtin_msa_fclt_w(b, a), aNaN); + return (Packet4f)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_max_s_w(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast<float*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_LOAD return __builtin_msa_ld_w(const_cast<int32_t*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast<float*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4i)__builtin_msa_ld_w(const_cast<int32_t*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) { + EIGEN_MSA_DEBUG; + + float f0 = from[0], f1 = from[1]; + Packet4f v0 = { f0, f0, f0, f0 }; + Packet4f v1 = { f1, f1, f1, f1 }; + return (Packet4f)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0); +} + +template <> +EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) { + EIGEN_MSA_DEBUG; + + int32_t i0 = from[0], i1 = from[1]; + Packet4i v0 = { i0, i0, i0, i0 }; + Packet4i v1 = { i1, i1, i1, i1 }; + return (Packet4i)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0); +} + +template <> +EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w(from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w(from, to, 0); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) { + EIGEN_MSA_DEBUG; + + float f = *from; + Packet4f v = { f, f, f, f }; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + return v; +} + +template <> +EIGEN_DEVICE_FUNC inline Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) { + EIGEN_MSA_DEBUG; + + int32_t i = *from; + Packet4i v = { i, i, i, i }; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + return v; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, + Index stride) { + EIGEN_MSA_DEBUG; + + *to = from[0]; + to += stride; + *to = from[1]; + to += stride; + *to = from[2]; + to += stride; + *to = from[3]; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, + Index stride) { + EIGEN_MSA_DEBUG; + + *to = from[0]; + to += stride; + *to = from[1]; + to += stride; + *to = from[2]; + to += stride; + *to = from[3]; +} + +template <> +EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { + EIGEN_MSA_DEBUG; + + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) { + EIGEN_MSA_DEBUG; + + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return a[0]; +} + +template <> +EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + return a[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return (Packet4f)__builtin_msa_bclri_w((v4u32)a, 31); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + Packet4i zero = __builtin_msa_ldi_w(0); + return __builtin_msa_add_a_w(zero, a); +} + +template <> +EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + Packet4f s = padd(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return s[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) { + EIGEN_MSA_DEBUG; + + v4i32 tmp1, tmp2, tmp3, tmp4; + Packet4f sum; + + tmp1 = __builtin_msa_ilvr_w((v4i32)vecs[1], (v4i32)vecs[0]); + tmp2 = __builtin_msa_ilvr_w((v4i32)vecs[3], (v4i32)vecs[2]); + tmp3 = __builtin_msa_ilvl_w((v4i32)vecs[1], (v4i32)vecs[0]); + tmp4 = __builtin_msa_ilvl_w((v4i32)vecs[3], (v4i32)vecs[2]); + + sum = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); + sum = padd(sum, (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1)); + sum = padd(sum, (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3)); + sum = padd(sum, (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3)); + + return sum; +} + +template <> +EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) { + EIGEN_MSA_DEBUG; + + v4i32 tmp1, tmp2, tmp3, tmp4; + Packet4i sum; + + tmp1 = __builtin_msa_ilvr_w((v4i32)vecs[1], (v4i32)vecs[0]); + tmp2 = __builtin_msa_ilvr_w((v4i32)vecs[3], (v4i32)vecs[2]); + tmp3 = __builtin_msa_ilvl_w((v4i32)vecs[1], (v4i32)vecs[0]); + tmp4 = __builtin_msa_ilvl_w((v4i32)vecs[3], (v4i32)vecs[2]); + + sum = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); + sum = padd(sum, (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1)); + sum = padd(sum, (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3)); + sum = padd(sum, (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3)); + + return sum; +} + +template <> +EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + Packet4i s = padd(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + s = padd(s, __builtin_msa_shf_w(s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return s[0]; +} + +// Other reduction functions: +// mul +template <> +EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + Packet4f p = pmul(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + p = pmul(p, (Packet4f)__builtin_msa_shf_w((v4i32)p, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return p[0]; +} + +template <> +EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + Packet4i p = pmul(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + p = pmul(p, __builtin_msa_shf_w(p, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return p[0]; +} + +// min +template <> +EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + // Swap 64-bit halves of a. + Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); +#if !EIGEN_FAST_MATH + // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit + // masks of all zeroes/ones in low 64 bits. + v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped); + // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes. + unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0); +#endif + // Continue with min computation. + Packet4f v = __builtin_msa_fmin_w(a, swapped); + v = __builtin_msa_fmin_w( + v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); +#if !EIGEN_FAST_MATH + // Based on the mask select between v and 4 qNaNs. + v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000); + v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v); +#endif + return v[0]; +} + +template <> +EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + Packet4i m = pmin(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + m = pmin(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return m[0]; +} + +// max +template <> +EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + // Swap 64-bit halves of a. + Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); +#if !EIGEN_FAST_MATH + // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit + // masks of all zeroes/ones in low 64 bits. + v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped); + // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes. + unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0); +#endif + // Continue with max computation. + Packet4f v = __builtin_msa_fmax_w(a, swapped); + v = __builtin_msa_fmax_w( + v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); +#if !EIGEN_FAST_MATH + // Based on the mask select between v and 4 qNaNs. + v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000); + v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v); +#endif + return v[0]; +} + +template <> +EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) { + EIGEN_MSA_DEBUG; + + Packet4i m = pmax(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); + m = pmax(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); + return m[0]; +} + +#define PALIGN_MSA(Offset, Type, Command) \ + template <> \ + struct palign_impl<Offset, Type> { \ + EIGEN_STRONG_INLINE static void run(Type& first, const Type& second) { \ + if (Offset != 0) first = (Type)(Command((v16i8)second, (v16i8)first, Offset * 4)); \ + } \ + }; + +PALIGN_MSA(0, Packet4f, __builtin_msa_sldi_b) +PALIGN_MSA(1, Packet4f, __builtin_msa_sldi_b) +PALIGN_MSA(2, Packet4f, __builtin_msa_sldi_b) +PALIGN_MSA(3, Packet4f, __builtin_msa_sldi_b) +PALIGN_MSA(0, Packet4i, __builtin_msa_sldi_b) +PALIGN_MSA(1, Packet4i, __builtin_msa_sldi_b) +PALIGN_MSA(2, Packet4i, __builtin_msa_sldi_b) +PALIGN_MSA(3, Packet4i, __builtin_msa_sldi_b) + +#undef PALIGN_MSA + +inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet4f, 4>& value) { + os << "[ " << value.packet[0] << "," << std::endl + << " " << value.packet[1] << "," << std::endl + << " " << value.packet[2] << "," << std::endl + << " " << value.packet[3] << " ]"; + return os; +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) { + EIGEN_MSA_DEBUG; + + v4i32 tmp1, tmp2, tmp3, tmp4; + + tmp1 = __builtin_msa_ilvr_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]); + tmp2 = __builtin_msa_ilvr_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]); + tmp3 = __builtin_msa_ilvl_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]); + tmp4 = __builtin_msa_ilvl_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]); + + kernel.packet[0] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); + kernel.packet[1] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1); + kernel.packet[2] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3); + kernel.packet[3] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3); +} + +inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet4i, 4>& value) { + os << "[ " << value.packet[0] << "," << std::endl + << " " << value.packet[1] << "," << std::endl + << " " << value.packet[2] << "," << std::endl + << " " << value.packet[3] << " ]"; + return os; +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) { + EIGEN_MSA_DEBUG; + + v4i32 tmp1, tmp2, tmp3, tmp4; + + tmp1 = __builtin_msa_ilvr_w(kernel.packet[1], kernel.packet[0]); + tmp2 = __builtin_msa_ilvr_w(kernel.packet[3], kernel.packet[2]); + tmp3 = __builtin_msa_ilvl_w(kernel.packet[1], kernel.packet[0]); + tmp4 = __builtin_msa_ilvl_w(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); + kernel.packet[1] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1); + kernel.packet[2] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3); + kernel.packet[3] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3); +} + +template <> +EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fsqrt_w(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + return __builtin_msa_frsqrt_w(a); +#else + Packet4f ones = __builtin_msa_ffint_s_w(__builtin_msa_ldi_w(1)); + return pdiv(ones, psqrt(a)); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { + Packet4f v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY. + "ctcmsa $1, %[new_mode]\n" + "frint.w %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { + Packet4f v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" + "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY. + "ctcmsa $1, %[new_mode]\n" + "frint.w %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { + Packet4f v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" + "xori %[new_mode], %[new_mode], 3\n" // 0 = round to nearest, ties to even. + "ctcmsa $1, %[new_mode]\n" + "frint.w %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, + const Packet4f& elsePacket) { + Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], + ifPacket.select[3] }; + Packet4i mask = __builtin_msa_ceqi_w((Packet4i)select, 0); + return (Packet4f)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, + const Packet4i& elsePacket) { + Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], + ifPacket.select[3] }; + Packet4i mask = __builtin_msa_ceqi_w((Packet4i)select, 0); + return (Packet4i)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); +} + +//---------- double ---------- + +typedef v2f64 Packet2d; +typedef v2i64 Packet2l; +typedef v2u64 Packet2ul; + +#define _EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = { X, X } +#define _EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = { X, X } +#define _EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = { X, X } + +inline std::ostream& operator<<(std::ostream& os, const Packet2d& value) { + os << "[ " << value[0] << ", " << value[1] << " ]"; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const Packet2l& value) { + os << "[ " << value[0] << ", " << value[1] << " ]"; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const Packet2ul& value) { + os << "[ " << value[0] << ", " << value[1] << " ]"; + return os; +} + +template <> +struct packet_traits<double> : default_packet_traits { + typedef Packet2d type; + typedef Packet2d half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + HasHalfPacket = 0, + // FIXME check the Has* + HasDiv = 1, + HasExp = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasBlend = 1 + }; +}; + +template <> +struct unpacket_traits<Packet2d> { + typedef double type; + enum { size = 2, alignment = Aligned16, vectorizable=true }; + typedef Packet2d half; +}; + +template <> +EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { + EIGEN_MSA_DEBUG; + + Packet2d value = { from, from }; + return value; +} + +template <> +EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fadd_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { + EIGEN_MSA_DEBUG; + + static const Packet2d countdown = { 0.0, 1.0 }; + return padd(pset1<Packet2d>(a), countdown); +} + +template <> +EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fsub_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_bnegi_d((v2u64)a, 63); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fmul_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fdiv_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fmadd_d(c, a, b); +} + +// Logical Operations are not supported for float, so we have to reinterpret casts using MSA +// intrinsics +template <> +EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_and_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_or_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_xor_v((v16u8)a, (v16u8)b); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + + return pand(a, (Packet2d)__builtin_msa_xori_b((v16u8)b, 255)); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast<double*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + // This prefers numbers to NaNs. + return __builtin_msa_fmin_d(a, b); +#else + // This prefers NaNs to numbers. + v2i64 aNaN = __builtin_msa_fcun_d(a, a); + v2i64 aMinOrNaN = por(__builtin_msa_fclt_d(a, b), aNaN); + return (Packet2d)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + // This prefers numbers to NaNs. + return __builtin_msa_fmax_d(a, b); +#else + // This prefers NaNs to numbers. + v2i64 aNaN = __builtin_msa_fcun_d(a, a); + v2i64 aMaxOrNaN = por(__builtin_msa_fclt_d(b, a), aNaN); + return (Packet2d)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a); +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast<double*>(from), 0); +} + +template <> +EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) { + EIGEN_MSA_DEBUG; + + Packet2d value = { *from, *from }; + return value; +} + +template <> +EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) { + EIGEN_MSA_DEBUG; + + EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) { + EIGEN_MSA_DEBUG; + + Packet2d value; + value[0] = *from; + from += stride; + value[1] = *from; + return value; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, + Index stride) { + EIGEN_MSA_DEBUG; + + *to = from[0]; + to += stride; + *to = from[1]; +} + +template <> +EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { + EIGEN_MSA_DEBUG; + + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return a[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return (Packet2d)__builtin_msa_bclri_d((v2u64)a, 63); +} + +template <> +EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + Packet2d s = padd(a, preverse(a)); + return s[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) { + EIGEN_MSA_DEBUG; + + Packet2d v0 = (Packet2d)__builtin_msa_ilvev_d((v2i64)vecs[1], (v2i64)vecs[0]); + Packet2d v1 = (Packet2d)__builtin_msa_ilvod_d((v2i64)vecs[1], (v2i64)vecs[0]); + + return padd(v0, v1); +} + +// Other reduction functions: +// mul +template <> +EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + Packet2d p = pmul(a, preverse(a)); + return p[0]; +} + +// min +template <> +EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); + Packet2d v = __builtin_msa_fmin_d(a, swapped); + return v[0]; +#else + double a0 = a[0], a1 = a[1]; + return ((numext::isnan)(a0) || a0 < a1) ? a0 : a1; +#endif +} + +// max +template <> +EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); + Packet2d v = __builtin_msa_fmax_d(a, swapped); + return v[0]; +#else + double a0 = a[0], a1 = a[1]; + return ((numext::isnan)(a0) || a0 > a1) ? a0 : a1; +#endif +} + +template <> +EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) { + EIGEN_MSA_DEBUG; + + return __builtin_msa_fsqrt_d(a); +} + +template <> +EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) { + EIGEN_MSA_DEBUG; + +#if EIGEN_FAST_MATH + return __builtin_msa_frsqrt_d(a); +#else + Packet2d ones = __builtin_msa_ffint_s_d(__builtin_msa_ldi_d(1)); + return pdiv(ones, psqrt(a)); +#endif +} + +#define PALIGN_MSA(Offset, Type, Command) \ + template <> \ + struct palign_impl<Offset, Type> { \ + EIGEN_STRONG_INLINE static void run(Type& first, const Type& second) { \ + if (Offset != 0) first = (Type)(Command((v16i8)second, (v16i8)first, Offset * 8)); \ + } \ + }; + +PALIGN_MSA(0, Packet2d, __builtin_msa_sldi_b) +PALIGN_MSA(1, Packet2d, __builtin_msa_sldi_b) + +#undef PALIGN_MSA + +inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet2d, 2>& value) { + os << "[ " << value.packet[0] << "," << std::endl << " " << value.packet[1] << " ]"; + return os; +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) { + EIGEN_MSA_DEBUG; + + Packet2d trn1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]); + Packet2d trn2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]); + kernel.packet[0] = trn1; + kernel.packet[1] = trn2; +} + +template <> +EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { + Packet2d v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY. + "ctcmsa $1, %[new_mode]\n" + "frint.d %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { + Packet2d v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" + "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY. + "ctcmsa $1, %[new_mode]\n" + "frint.d %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { + Packet2d v = a; + int32_t old_mode, new_mode; + asm volatile( + "cfcmsa %[old_mode], $1\n" + "ori %[new_mode], %[old_mode], 3\n" + "xori %[new_mode], %[new_mode], 3\n" // 0 = round to nearest, ties to even. + "ctcmsa $1, %[new_mode]\n" + "frint.d %w[v], %w[v]\n" + "ctcmsa $1, %[old_mode]\n" + : // outputs + [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), + [v] "+f"(v) + : // inputs + : // clobbers + ); + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, + const Packet2d& elsePacket) { + Packet2ul select = { ifPacket.select[0], ifPacket.select[1] }; + Packet2l mask = __builtin_msa_ceqi_d((Packet2l)select, 0); + return (Packet2d)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_MSA_H
diff --git a/Eigen/src/Core/arch/NEON/CMakeLists.txt b/Eigen/src/Core/arch/NEON/CMakeLists.txt deleted file mode 100644 index fd4d4af..0000000 --- a/Eigen/src/Core/arch/NEON/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_NEON_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_NEON_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/NEON COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/NEON/Complex.h b/Eigen/src/Core/arch/NEON/Complex.h index d2d4679..d149275 100644 --- a/Eigen/src/Core/arch/NEON/Complex.h +++ b/Eigen/src/Core/arch/NEON/Complex.h
@@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -14,8 +15,21 @@ namespace internal { -static uint32x4_t p4ui_CONJ_XOR = EIGEN_INIT_NEON_PACKET4(0x00000000, 0x80000000, 0x00000000, 0x80000000); -static uint32x2_t p2ui_CONJ_XOR = EIGEN_INIT_NEON_PACKET2(0x00000000, 0x80000000); +inline uint32x4_t p4ui_CONJ_XOR() { +// See bug 1325, clang fails to call vld1q_u64. +#if EIGEN_COMP_CLANG + uint32x4_t ret = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; + return ret; +#else + static const uint32_t conj_XOR_DATA[] = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; + return vld1q_u32( conj_XOR_DATA ); +#endif +} + +inline uint32x2_t p2ui_CONJ_XOR() { + static const uint32_t conj_XOR_DATA[] = { 0x00000000, 0x80000000 }; + return vld1_u32( conj_XOR_DATA ); +} //---------- float ---------- struct Packet2cf @@ -48,12 +62,12 @@ }; }; -template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16}; typedef Packet2cf half; }; +template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2cf half; }; template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) { float32x2_t r64; - r64 = vld1_f32((float *)&from); + r64 = vld1_f32((const float *)&from); return Packet2cf(vcombine_f32(r64, r64)); } @@ -64,7 +78,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { Packet4ui b = vreinterpretq_u32_f32(a.v); - return Packet2cf(vreinterpretq_f32_u32(veorq_u32(b, p4ui_CONJ_XOR))); + return Packet2cf(vreinterpretq_f32_u32(veorq_u32(b, p4ui_CONJ_XOR()))); } template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) @@ -80,7 +94,7 @@ // Multiply the imag a with b v2 = vmulq_f32(v2, b.v); // Conjugate v2 - v2 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v2), p4ui_CONJ_XOR)); + v2 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v2), p4ui_CONJ_XOR())); // Swap real/imag elements in v2. v2 = vrev64q_f32(v2); // Add and return the result @@ -128,11 +142,11 @@ to[stride*1] = std::complex<float>(vgetq_lane_f32(from.v, 2), vgetq_lane_f32(from.v, 3)); } -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { EIGEN_ARM_PREFETCH((float *)addr); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { EIGEN_ARM_PREFETCH((const float *)addr); } template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) { - std::complex<float> EIGEN_ALIGN16 x[2]; + EIGEN_ALIGN16 std::complex<float> x[2]; vst1q_f32((float *)x, a.v); return x[0]; } @@ -195,7 +209,7 @@ // Multiply the imag a with b v2 = vmul_f32(v2, a2); // Conjugate v2 - v2 = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(v2), p2ui_CONJ_XOR)); + v2 = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(v2), p2ui_CONJ_XOR())); // Swap real/imag elements in v2. v2 = vrev64_f32(v2); // Add v1, v2 @@ -251,6 +265,8 @@ } }; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) + template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for NEON @@ -261,7 +277,7 @@ s = vmulq_f32(b.v, b.v); rev_s = vrev64q_f32(s); - return Packet2cf(pdiv(res.v, vaddq_f32(s,rev_s))); + return Packet2cf(pdiv<Packet4f>(res.v, vaddq_f32(s,rev_s))); } EIGEN_DEVICE_FUNC inline void @@ -274,7 +290,13 @@ //---------- double ---------- #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG -static uint64x2_t p2ul_CONJ_XOR = EIGEN_INIT_NEON_PACKET2(0x0, 0x8000000000000000); +// See bug 1325, clang fails to call vld1q_u64. +#if EIGEN_COMP_CLANG + static uint64x2_t p2ul_CONJ_XOR = {0x0, 0x8000000000000000}; +#else + const uint64_t p2ul_conj_XOR_DATA[] = { 0x0, 0x8000000000000000 }; + static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA ); +#endif struct Packet1cd { @@ -306,7 +328,7 @@ }; }; -template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16}; typedef Packet1cd half; }; +template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16, vectorizable=true}; typedef Packet1cd half; }; template<> EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from)); } @@ -361,7 +383,7 @@ template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { EIGEN_ARM_PREFETCH((double *)addr); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { EIGEN_ARM_PREFETCH((const double *)addr); } template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index stride) { @@ -379,7 +401,7 @@ template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) { - std::complex<double> EIGEN_ALIGN16 res; + EIGEN_ALIGN16 std::complex<double> res; pstore<std::complex<double> >(&res, a); return res; @@ -436,6 +458,8 @@ } }; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) + template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for NEON
diff --git a/Eigen/src/Core/arch/NEON/MathFunctions.h b/Eigen/src/Core/arch/NEON/MathFunctions.h index 6bb05bb..2e7d0e9 100644 --- a/Eigen/src/Core/arch/NEON/MathFunctions.h +++ b/Eigen/src/Core/arch/NEON/MathFunctions.h
@@ -5,83 +5,37 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -/* The sin, cos, exp, and log functions of this file come from - * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ - */ - #ifndef EIGEN_MATH_FUNCTIONS_NEON_H #define EIGEN_MATH_FUNCTIONS_NEON_H +#include "../Default/GenericPacketMathFunctions.h" + namespace Eigen { namespace internal { template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet4f pexp<Packet4f>(const Packet4f& _x) +Packet4f pexp<Packet4f>(const Packet4f& x) { - Packet4f x = _x; - Packet4f tmp, fx; + return pexp_float(x); +} - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); - _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); - _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f plog<Packet4f>(const Packet4f& x) +{ + return plog_float(x); +} - x = vminq_f32(x, p4f_exp_hi); - x = vmaxq_f32(x, p4f_exp_lo); +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f psin<Packet4f>(const Packet4f& x) +{ + return psin_float(x); +} - /* express exp(x) as exp(g + n*log(2)) */ - fx = vmlaq_f32(p4f_half, x, p4f_cephes_LOG2EF); - - /* perform a floorf */ - tmp = vcvtq_f32_s32(vcvtq_s32_f32(fx)); - - /* if greater, substract 1 */ - Packet4ui mask = vcgtq_f32(tmp, fx); - mask = vandq_u32(mask, vreinterpretq_u32_f32(p4f_1)); - - fx = vsubq_f32(tmp, vreinterpretq_f32_u32(mask)); - - tmp = vmulq_f32(fx, p4f_cephes_exp_C1); - Packet4f z = vmulq_f32(fx, p4f_cephes_exp_C2); - x = vsubq_f32(x, tmp); - x = vsubq_f32(x, z); - - Packet4f y = vmulq_f32(p4f_cephes_exp_p0, x); - z = vmulq_f32(x, x); - y = vaddq_f32(y, p4f_cephes_exp_p1); - y = vmulq_f32(y, x); - y = vaddq_f32(y, p4f_cephes_exp_p2); - y = vmulq_f32(y, x); - y = vaddq_f32(y, p4f_cephes_exp_p3); - y = vmulq_f32(y, x); - y = vaddq_f32(y, p4f_cephes_exp_p4); - y = vmulq_f32(y, x); - y = vaddq_f32(y, p4f_cephes_exp_p5); - - y = vmulq_f32(y, z); - y = vaddq_f32(y, x); - y = vaddq_f32(y, p4f_1); - - /* build 2^n */ - int32x4_t mm; - mm = vcvtq_s32_f32(fx); - mm = vaddq_s32(mm, p4i_0x7f); - mm = vshlq_n_s32(mm, 23); - Packet4f pow2n = vreinterpretq_f32_s32(mm); - - y = vmulq_f32(y, pow2n); - return y; +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f pcos<Packet4f>(const Packet4f& x) +{ + return pcos_float(x); } } // end namespace internal
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 3224c36..e8b3518 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -2,7 +2,7 @@ // for linear algebra. // // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> -// Copyright (C) 2010 Konstantinos Margaritis <markos@codex.gr> +// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org> // Heavily based on Gael's SSE version. // // This Source Code Form is subject to the terms of the Mozilla @@ -28,11 +28,42 @@ #define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD #endif -// FIXME NEON has 16 quad registers, but since the current register allocator -// is so bad, it is much better to reduce it to 8 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#if EIGEN_ARCH_ARM64 +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#else #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 #endif +#endif + +#if EIGEN_COMP_MSVC + +// In MSVC's arm_neon.h header file, all NEON vector types +// are aliases to the same underlying type __n128. +// We thus have to wrap them to make them different C++ types. +// (See also bug 1428) + +template<typename T,int unique_id> +struct eigen_packet_wrapper +{ + operator T&() { return m_val; } + operator const T&() const { return m_val; } + eigen_packet_wrapper() {} + eigen_packet_wrapper(const T &v) : m_val(v) {} + eigen_packet_wrapper& operator=(const T &v) { + m_val = v; + return *this; + } + + T m_val; +}; +typedef eigen_packet_wrapper<float32x2_t,0> Packet2f; +typedef eigen_packet_wrapper<float32x4_t,1> Packet4f; +typedef eigen_packet_wrapper<int32x4_t ,2> Packet4i; +typedef eigen_packet_wrapper<int32x2_t ,3> Packet2i; +typedef eigen_packet_wrapper<uint32x4_t ,4> Packet4ui; + +#else typedef float32x2_t Packet2f; typedef float32x4_t Packet4f; @@ -40,34 +71,28 @@ typedef int32x2_t Packet2i; typedef uint32x4_t Packet4ui; +#endif // EIGEN_COMP_MSVC + #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1<Packet4f>(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ - const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int>(X)) + const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X)) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1<Packet4i>(X) -#if EIGEN_COMP_LLVM && !EIGEN_COMP_CLANG - //Special treatment for Apple's llvm-gcc, its NEON packet types are unions - #define EIGEN_INIT_NEON_PACKET2(X, Y) {{X, Y}} - #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {{X, Y, Z, W}} -#else - //Default initializer for packets - #define EIGEN_INIT_NEON_PACKET2(X, Y) {X, Y} - #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {X, Y, Z, W} -#endif - - -// arm64 does have the pld instruction. If available, let's trust the __builtin_prefetch built-in function -// which available on LLVM and GCC (at least) -#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC +#if EIGEN_ARCH_ARM64 + // __builtin_prefetch tends to do nothing on ARM64 compilers because the + // prefetch instructions there are too detailed for __builtin_prefetch to map + // meaningfully to them. + #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : ); +#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR); #elif defined __pld #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR) -#elif !EIGEN_ARCH_ARM64 - #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ( " pld [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); +#elif EIGEN_ARCH_ARM32 + #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : ); #else // by default no explicit prefetching #define EIGEN_ARM_PREFETCH(ADDR) @@ -83,16 +108,17 @@ size = 4, HasHalfPacket=0, // Packet2f intrinsics not implemented yet - HasDiv = 1, + HasDiv = 1, + HasFloor = 1, // FIXME check the Has* - HasSin = 0, - HasCos = 0, - HasLog = 0, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasLog = 1, HasExp = 1, HasSqrt = 0 }; }; -template<> struct packet_traits<int> : default_packet_traits +template<> struct packet_traits<int32_t> : default_packet_traits { typedef Packet4i type; typedef Packet4i half; // Packet2i intrinsics not implemented yet @@ -114,20 +140,35 @@ EIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); } #endif -template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; }; -template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; +template<> struct unpacket_traits<Packet4f> +{ + typedef float type; + typedef Packet4f half; + typedef Packet4i integer_packet; + enum {size=4, alignment=Aligned16, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet4i> +{ + typedef int32_t type; + typedef Packet4i half; + enum {size=4, alignment=Aligned16, vectorizable=true}; +}; template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return vdupq_n_f32(from); } -template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return vdupq_n_s32(from); } +template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) { return vdupq_n_s32(from); } + +template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) { return vreinterpretq_f32_u32(vdupq_n_u32(from)); } template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { - Packet4f countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3); + const float f[] = {0, 1, 2, 3}; + Packet4f countdown = vld1q_f32(f); return vaddq_f32(pset1<Packet4f>(a), countdown); } -template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) +template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) { - Packet4i countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3); + const int32_t i[] = {0, 1, 2, 3}; + Packet4i countdown = vld1q_s32(i); return vaddq_s32(pset1<Packet4i>(a), countdown); } @@ -222,6 +263,25 @@ template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vcleq_f32(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vcltq_f32(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vceqq_f32(a,b)); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a,b))); } + +template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return vreinterpretq_s32_u32(vceqq_s32(a,b)); } + +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) +{ + const Packet4f cst_1 = pset1<Packet4f>(1.0f); + /* perform a floorf */ + Packet4f tmp = vcvtq_f32_s32(vcvtq_s32_f32(a)); + + /* if greater, substract 1 */ + Packet4ui mask = vcgtq_f32(tmp, a); + mask = vandq_u32(mask, vreinterpretq_u32_f32(cst_1)); + return vsubq_f32(tmp, vreinterpretq_f32_u32(mask)); +} + // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { @@ -247,20 +307,23 @@ } template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vbicq_s32(a,b); } -template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); } -template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); } +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftright(Packet4i a) { return vshrq_n_s32(a,N); } +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftleft(Packet4i a) { return vshlq_n_s32(a,N); } -template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); } -template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); } +template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); } +template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); } -template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) +template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); } +template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); } + +template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) { float32x2_t lo, hi; lo = vld1_dup_f32(from); hi = vld1_dup_f32(from+1); return vcombine_f32(lo, hi); } -template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) +template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) { int32x2_t lo, hi; lo = vld1_dup_s32(from); @@ -268,11 +331,11 @@ return vcombine_s32(lo, hi); } -template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); } -template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); } +template<> EIGEN_STRONG_INLINE void pstore<float> (float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); } +template<> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); } -template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); } -template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); } +template<> EIGEN_STRONG_INLINE void pstoreu<float> (float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); } +template<> EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); } template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) { @@ -283,7 +346,7 @@ res = vsetq_lane_f32(from[3*stride], res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride) +template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) { Packet4i res = pset1<Packet4i>(0); res = vsetq_lane_s32(from[0*stride], res, 0); @@ -300,7 +363,7 @@ to[stride*2] = vgetq_lane_f32(from, 2); to[stride*3] = vgetq_lane_f32(from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) +template<> EIGEN_DEVICE_FUNC inline void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, Index stride) { to[stride*0] = vgetq_lane_s32(from, 0); to[stride*1] = vgetq_lane_s32(from, 1); @@ -308,12 +371,12 @@ to[stride*3] = vgetq_lane_s32(from, 3); } -template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ARM_PREFETCH(addr); } -template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { EIGEN_ARM_PREFETCH(addr); } +template<> EIGEN_STRONG_INLINE void prefetch<float> (const float* addr) { EIGEN_ARM_PREFETCH(addr); } +template<> EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); } // FIXME only store the 2 first elements ? -template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a); return x[0]; } -template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; vst1q_s32(x, a); return x[0]; } +template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { EIGEN_ALIGN16 float x[4]; vst1q_f32(x, a); return x[0]; } +template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) { EIGEN_ALIGN16 int32_t x[4]; vst1q_s32(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { float32x2_t a_lo, a_hi; @@ -334,25 +397,17 @@ return vcombine_s32(a_hi, a_lo); } -template<size_t offset> -struct protate_impl<offset, Packet4f> -{ - static Packet4f run(const Packet4f& a) { - return vextq_f32(a, a, offset); - } -}; - -template<size_t offset> -struct protate_impl<offset, Packet4i> -{ - static Packet4i run(const Packet4i& a) { - return vextq_s32(a, a, offset); - } -}; - template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); } +template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) { + return pfrexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) { + return pldexp_float(a,exponent); +} + template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { float32x2_t a_lo, a_hi, sum; @@ -384,7 +439,7 @@ return sum; } -template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) +template<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, sum; @@ -431,7 +486,7 @@ return vget_lane_f32(prod, 0); } -template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) +template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, prod; @@ -459,7 +514,7 @@ return vget_lane_f32(min, 0); } -template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) +template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, min; @@ -484,7 +539,7 @@ return vget_lane_f32(max, 0); } -template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) +template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, max; @@ -496,6 +551,13 @@ return vget_lane_s32(max, 0); } +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) +{ + uint32x2_t tmp = vorr_u32(vget_low_u32( vreinterpretq_u32_f32(x)), + vget_high_u32(vreinterpretq_u32_f32(x))); + return vget_lane_u32(vpmax_u32(tmp,tmp),0); +} + // this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors, // see bug 347 and this LLVM bug: http://llvm.org/bugs/show_bug.cgi?id=11074 #define PALIGN_NEON(Offset,Type,Command) \ @@ -595,13 +657,14 @@ }; }; -template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; +template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2d half; }; template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return vdupq_n_f64(from); } template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { - Packet2d countdown = EIGEN_INIT_NEON_PACKET2(0, 1); + const double countdown_raw[] = {0.0,1.0}; + const Packet2d countdown = vld1q_f64(countdown_raw); return vaddq_f64(pset1<Packet2d>(a), countdown); } template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return vaddq_f64(a,b); } @@ -648,6 +711,8 @@ return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); } +template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vceqq_f64(a,b)); } + template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); } template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from); } @@ -679,14 +744,6 @@ template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); } -template<size_t offset> -struct protate_impl<offset, Packet2d> -{ - static Packet2d run(const Packet2d& a) { - return vextq_f64(a, a, offset); - } -}; - template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); } #if EIGEN_COMP_CLANG && defined(__apple_build_version__)
diff --git a/Eigen/src/Core/arch/NEON/TypeCasting.h b/Eigen/src/Core/arch/NEON/TypeCasting.h new file mode 100644 index 0000000..20dbe13 --- /dev/null +++ b/Eigen/src/Core/arch/NEON/TypeCasting.h
@@ -0,0 +1,56 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_NEON_H +#define EIGEN_TYPE_CASTING_NEON_H + +namespace Eigen { + +namespace internal { + +template <> +struct type_casting_traits<float, int> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template <> +struct type_casting_traits<int, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + + +template<> EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) { + return vcvtq_s32_f32(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) { + return vcvtq_f32_s32(a); +} + +template<> EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i,Packet4f>(const Packet4f& a) { + return vreinterpretq_s32_f32(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f,Packet4i>(const Packet4i& a) { + return vreinterpretq_f32_s32(a); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_NEON_H
diff --git a/Eigen/src/Core/arch/SSE/CMakeLists.txt b/Eigen/src/Core/arch/SSE/CMakeLists.txt deleted file mode 100644 index 46ea7cc..0000000 --- a/Eigen/src/Core/arch/SSE/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_SSE_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_SSE_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/SSE COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/SSE/Complex.h b/Eigen/src/Core/arch/SSE/Complex.h index fd7f4d7..f39988e 100644 --- a/Eigen/src/Core/arch/SSE/Complex.h +++ b/Eigen/src/Core/arch/SSE/Complex.h
@@ -50,7 +50,7 @@ }; #endif -template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16}; typedef Packet2cf half; }; +template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2cf half; }; template<> EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_add_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_sub_ps(a.v,b.v)); } @@ -82,10 +82,13 @@ #endif } +template<> EIGEN_STRONG_INLINE Packet2cf ptrue <Packet2cf>(const Packet2cf& a) { return Packet2cf(ptrue(Packet4f(a.v))); } +template<> EIGEN_STRONG_INLINE Packet2cf pnot <Packet2cf>(const Packet2cf& a) { return Packet2cf(pnot(Packet4f(a.v))); } + template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_and_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_or_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_xor_ps(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&numext::real_ref(*from))); } @@ -128,7 +131,7 @@ _mm_cvtss_f32(_mm_shuffle_ps(from.v, from.v, 3))); } -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) { @@ -229,23 +232,7 @@ } }; -template<> struct conj_helper<Packet4f, Packet2cf, false,false> -{ - EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet4f& x, const Packet2cf& y, const Packet2cf& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet2cf pmul(const Packet4f& x, const Packet2cf& y) const - { return Packet2cf(Eigen::internal::pmul<Packet4f>(x, y.v)); } -}; - -template<> struct conj_helper<Packet2cf, Packet4f, false,false> -{ - EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet4f& y, const Packet2cf& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& x, const Packet4f& y) const - { return Packet2cf(Eigen::internal::pmul<Packet4f>(x.v, y)); } -}; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { @@ -296,7 +283,7 @@ }; #endif -template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16}; typedef Packet1cd half; }; +template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16, vectorizable=true}; typedef Packet1cd half; }; template<> EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_add_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_sub_pd(a.v,b.v)); } @@ -321,10 +308,12 @@ #endif } +template<> EIGEN_STRONG_INLINE Packet1cd ptrue <Packet1cd>(const Packet1cd& a) { return Packet1cd(ptrue(Packet2d(a.v))); } +template<> EIGEN_STRONG_INLINE Packet1cd pnot <Packet1cd>(const Packet1cd& a) { return Packet1cd(pnot(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_and_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_or_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_xor_pd(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_andnot_pd(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_andnot_pd(b.v,a.v)); } // FIXME force unaligned load, this is a temporary fix template<> EIGEN_STRONG_INLINE Packet1cd pload <Packet1cd>(const std::complex<double>* from) @@ -340,7 +329,7 @@ template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v)); } template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v)); } -template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) { @@ -430,23 +419,7 @@ } }; -template<> struct conj_helper<Packet2d, Packet1cd, false,false> -{ - EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet2d& x, const Packet1cd& y, const Packet1cd& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet1cd pmul(const Packet2d& x, const Packet1cd& y) const - { return Packet1cd(Eigen::internal::pmul<Packet2d>(x, y.v)); } -}; - -template<> struct conj_helper<Packet1cd, Packet2d, false,false> -{ - EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& x, const Packet2d& y, const Packet1cd& c) const - { return padd(c, pmul(x,y)); } - - EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& x, const Packet2d& y) const - { return Packet1cd(Eigen::internal::pmul<Packet2d>(x.v, y)); } -}; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { @@ -471,11 +444,43 @@ kernel.packet[1].v = tmp; } +template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) +{ + __m128 eq = _mm_cmpeq_ps(a.v, b.v); + return Packet2cf(pand<Packet4f>(eq, vec4f_swizzle1(eq, 1, 0, 3, 2))); +} + +template<> EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) +{ + __m128d eq = _mm_cmpeq_pd(a.v, b.v); + return Packet1cd(pand<Packet2d>(eq, vec2d_swizzle1(eq, 1, 0))); +} + template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { __m128d result = pblend<Packet2d>(ifPacket, _mm_castps_pd(thenPacket.v), _mm_castps_pd(elsePacket.v)); return Packet2cf(_mm_castpd_ps(result)); } +template<> EIGEN_STRONG_INLINE Packet2cf pinsertfirst(const Packet2cf& a, std::complex<float> b) +{ + return Packet2cf(_mm_loadl_pi(a.v, reinterpret_cast<const __m64*>(&b))); +} + +template<> EIGEN_STRONG_INLINE Packet1cd pinsertfirst(const Packet1cd&, std::complex<double> b) +{ + return pset1<Packet1cd>(b); +} + +template<> EIGEN_STRONG_INLINE Packet2cf pinsertlast(const Packet2cf& a, std::complex<float> b) +{ + return Packet2cf(_mm_loadh_pi(a.v, reinterpret_cast<const __m64*>(&b))); +} + +template<> EIGEN_STRONG_INLINE Packet1cd pinsertlast(const Packet1cd&, std::complex<double> b) +{ + return pset1<Packet1cd>(b); +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/SSE/MathFunctions.h b/Eigen/src/Core/arch/SSE/MathFunctions.h index 28f103e..0d491ab 100644 --- a/Eigen/src/Core/arch/SSE/MathFunctions.h +++ b/Eigen/src/Core/arch/SSE/MathFunctions.h
@@ -8,13 +8,15 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -/* The sin, cos, exp, and log functions of this file come from +/* The sin and cos and functions of this file come from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ #ifndef EIGEN_MATH_FUNCTIONS_SSE_H #define EIGEN_MATH_FUNCTIONS_SSE_H +#include "../Default/GenericPacketMathFunctions.h" + namespace Eigen { namespace internal { @@ -22,447 +24,62 @@ template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); - - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); - - /* the smallest non denormalized float number */ - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000);//-1.f/0.f); - - /* natural logarithm computed for 4 simultaneous float - return NaN for x <= 0 - */ - _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); - - - Packet4i emm0; - - Packet4f invalid_mask = _mm_cmpnge_ps(x, _mm_setzero_ps()); // not greater equal is true if x is NaN - Packet4f iszero_mask = _mm_cmpeq_ps(x, _mm_setzero_ps()); - - x = pmax(x, p4f_min_norm_pos); /* cut off denormalized stuff */ - emm0 = _mm_srli_epi32(_mm_castps_si128(x), 23); - - /* keep only the fractional part */ - x = _mm_and_ps(x, p4f_inv_mant_mask); - x = _mm_or_ps(x, p4f_half); - - emm0 = _mm_sub_epi32(emm0, p4i_0x7f); - Packet4f e = padd(Packet4f(_mm_cvtepi32_ps(emm0)), p4f_1); - - /* part2: - if( x < SQRTHF ) { - e -= 1; - x = x + x - 1.0; - } else { x = x - 1.0; } - */ - Packet4f mask = _mm_cmplt_ps(x, p4f_cephes_SQRTHF); - Packet4f tmp = pand(x, mask); - x = psub(x, p4f_1); - e = psub(e, pand(p4f_1, mask)); - x = padd(x, tmp); - - Packet4f x2 = pmul(x,x); - Packet4f x3 = pmul(x2,x); - - Packet4f y, y1, y2; - y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1); - y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4); - y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7); - y = pmadd(y , x, p4f_cephes_log_p2); - y1 = pmadd(y1, x, p4f_cephes_log_p5); - y2 = pmadd(y2, x, p4f_cephes_log_p8); - y = pmadd(y, x3, y1); - y = pmadd(y, x3, y2); - y = pmul(y, x3); - - y1 = pmul(e, p4f_cephes_log_q1); - tmp = pmul(x2, p4f_half); - y = padd(y, y1); - x = psub(x, tmp); - y2 = pmul(e, p4f_cephes_log_q2); - x = padd(x, y); - x = padd(x, y2); - // negative arg will be NAN, 0 will be -INF - return _mm_or_ps(_mm_andnot_ps(iszero_mask, _mm_or_ps(x, invalid_mask)), - _mm_and_ps(iszero_mask, p4f_minus_inf)); + return plog_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); - - - _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); - _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); - - _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); - - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); - - Packet4f tmp, fx; - Packet4i emm0; - - // clamp x - x = pmax(pmin(x, p4f_exp_hi), p4f_exp_lo); - - /* express exp(x) as exp(g + n*log(2)) */ - fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half); - -#ifdef EIGEN_VECTORIZE_SSE4_1 - fx = _mm_floor_ps(fx); -#else - emm0 = _mm_cvttps_epi32(fx); - tmp = _mm_cvtepi32_ps(emm0); - /* if greater, substract 1 */ - Packet4f mask = _mm_cmpgt_ps(tmp, fx); - mask = _mm_and_ps(mask, p4f_1); - fx = psub(tmp, mask); -#endif - - tmp = pmul(fx, p4f_cephes_exp_C1); - Packet4f z = pmul(fx, p4f_cephes_exp_C2); - x = psub(x, tmp); - x = psub(x, z); - - z = pmul(x,x); - - Packet4f y = p4f_cephes_exp_p0; - y = pmadd(y, x, p4f_cephes_exp_p1); - y = pmadd(y, x, p4f_cephes_exp_p2); - y = pmadd(y, x, p4f_cephes_exp_p3); - y = pmadd(y, x, p4f_cephes_exp_p4); - y = pmadd(y, x, p4f_cephes_exp_p5); - y = pmadd(y, z, x); - y = padd(y, p4f_1); - - // build 2^n - emm0 = _mm_cvttps_epi32(fx); - emm0 = _mm_add_epi32(emm0, p4i_0x7f); - emm0 = _mm_slli_epi32(emm0, 23); - return pmax(pmul(y, Packet4f(_mm_castsi128_ps(emm0))), _x); + return pexp_float(_x); } + template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED -Packet2d pexp<Packet2d>(const Packet2d& _x) +Packet2d pexp<Packet2d>(const Packet2d& x) { - Packet2d x = _x; - - _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); - _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); - _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); - - _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); - _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); - static const __m128i p4i_1023_0 = _mm_setr_epi32(1023, 1023, 0, 0); - - Packet2d tmp, fx; - Packet4i emm0; - - // clamp x - x = pmax(pmin(x, p2d_exp_hi), p2d_exp_lo); - /* express exp(x) as exp(g + n*log(2)) */ - fx = pmadd(p2d_cephes_LOG2EF, x, p2d_half); - -#ifdef EIGEN_VECTORIZE_SSE4_1 - fx = _mm_floor_pd(fx); -#else - emm0 = _mm_cvttpd_epi32(fx); - tmp = _mm_cvtepi32_pd(emm0); - /* if greater, substract 1 */ - Packet2d mask = _mm_cmpgt_pd(tmp, fx); - mask = _mm_and_pd(mask, p2d_1); - fx = psub(tmp, mask); -#endif - - tmp = pmul(fx, p2d_cephes_exp_C1); - Packet2d z = pmul(fx, p2d_cephes_exp_C2); - x = psub(x, tmp); - x = psub(x, z); - - Packet2d x2 = pmul(x,x); - - Packet2d px = p2d_cephes_exp_p0; - px = pmadd(px, x2, p2d_cephes_exp_p1); - px = pmadd(px, x2, p2d_cephes_exp_p2); - px = pmul (px, x); - - Packet2d qx = p2d_cephes_exp_q0; - qx = pmadd(qx, x2, p2d_cephes_exp_q1); - qx = pmadd(qx, x2, p2d_cephes_exp_q2); - qx = pmadd(qx, x2, p2d_cephes_exp_q3); - - x = pdiv(px,psub(qx,px)); - x = pmadd(p2d_2,x,p2d_1); - - // build 2^n - emm0 = _mm_cvttpd_epi32(fx); - emm0 = _mm_add_epi32(emm0, p4i_1023_0); - emm0 = _mm_slli_epi32(emm0, 20); - emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(1,2,0,3)); - return pmax(pmul(x, Packet2d(_mm_castsi128_pd(emm0))), _x); + return pexp_double(x); } -/* evaluation of 4 sines at onces, using SSE2 intrinsics. - - The code is the exact rewriting of the cephes sinf function. - Precision is excellent as long as x < 8192 (I did not bother to - take into account the special handling they have for greater values - -- it does not return garbage for arguments over 8192, though, but - the extra precision is missing). - - Note that it is such that sinf((float)M_PI) = 8.74e-8, which is the - surprising but correct result. -*/ - template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - - _EIGEN_DECLARE_CONST_Packet4i(1, 1); - _EIGEN_DECLARE_CONST_Packet4i(not1, ~1); - _EIGEN_DECLARE_CONST_Packet4i(2, 2); - _EIGEN_DECLARE_CONST_Packet4i(4, 4); - - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(sign_mask, 0x80000000); - - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1,-0.78515625f); - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891E-4f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736E-3f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611E-1f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948E-005f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765E-003f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827E-002f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4 / M_PI - - Packet4f xmm1, xmm2, xmm3, sign_bit, y; - - Packet4i emm0, emm2; - sign_bit = x; - /* take the absolute value */ - x = pabs(x); - - /* take the modulo */ - - /* extract the sign bit (upper one) */ - sign_bit = _mm_and_ps(sign_bit, p4f_sign_mask); - - /* scale by 4/Pi */ - y = pmul(x, p4f_cephes_FOPI); - - /* store the integer part of y in mm0 */ - emm2 = _mm_cvttps_epi32(y); - /* j=(j+1) & (~1) (see the cephes sources) */ - emm2 = _mm_add_epi32(emm2, p4i_1); - emm2 = _mm_and_si128(emm2, p4i_not1); - y = _mm_cvtepi32_ps(emm2); - /* get the swap sign flag */ - emm0 = _mm_and_si128(emm2, p4i_4); - emm0 = _mm_slli_epi32(emm0, 29); - /* get the polynom selection mask - there is one polynom for 0 <= x <= Pi/4 - and another one for Pi/4<x<=Pi/2 - - Both branches will be computed. - */ - emm2 = _mm_and_si128(emm2, p4i_2); - emm2 = _mm_cmpeq_epi32(emm2, _mm_setzero_si128()); - - Packet4f swap_sign_bit = _mm_castsi128_ps(emm0); - Packet4f poly_mask = _mm_castsi128_ps(emm2); - sign_bit = _mm_xor_ps(sign_bit, swap_sign_bit); - - /* The magic pass: "Extended precision modular arithmetic" - x = ((x - y * DP1) - y * DP2) - y * DP3; */ - xmm1 = pmul(y, p4f_minus_cephes_DP1); - xmm2 = pmul(y, p4f_minus_cephes_DP2); - xmm3 = pmul(y, p4f_minus_cephes_DP3); - x = padd(x, xmm1); - x = padd(x, xmm2); - x = padd(x, xmm3); - - /* Evaluate the first polynom (0 <= x <= Pi/4) */ - y = p4f_coscof_p0; - Packet4f z = _mm_mul_ps(x,x); - - y = pmadd(y, z, p4f_coscof_p1); - y = pmadd(y, z, p4f_coscof_p2); - y = pmul(y, z); - y = pmul(y, z); - Packet4f tmp = pmul(z, p4f_half); - y = psub(y, tmp); - y = padd(y, p4f_1); - - /* Evaluate the second polynom (Pi/4 <= x <= 0) */ - - Packet4f y2 = p4f_sincof_p0; - y2 = pmadd(y2, z, p4f_sincof_p1); - y2 = pmadd(y2, z, p4f_sincof_p2); - y2 = pmul(y2, z); - y2 = pmul(y2, x); - y2 = padd(y2, x); - - /* select the correct result from the two polynoms */ - y2 = _mm_and_ps(poly_mask, y2); - y = _mm_andnot_ps(poly_mask, y); - y = _mm_or_ps(y,y2); - /* update the sign */ - return _mm_xor_ps(y, sign_bit); + return psin_float(_x); } -/* almost the same as psin */ template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos<Packet4f>(const Packet4f& _x) { - Packet4f x = _x; - _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); - _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); - - _EIGEN_DECLARE_CONST_Packet4i(1, 1); - _EIGEN_DECLARE_CONST_Packet4i(not1, ~1); - _EIGEN_DECLARE_CONST_Packet4i(2, 2); - _EIGEN_DECLARE_CONST_Packet4i(4, 4); - - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1,-0.78515625f); - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); - _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891E-4f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736E-3f); - _EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611E-1f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948E-005f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765E-003f); - _EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827E-002f); - _EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4 / M_PI - - Packet4f xmm1, xmm2, xmm3, y; - Packet4i emm0, emm2; - - x = pabs(x); - - /* scale by 4/Pi */ - y = pmul(x, p4f_cephes_FOPI); - - /* get the integer part of y */ - emm2 = _mm_cvttps_epi32(y); - /* j=(j+1) & (~1) (see the cephes sources) */ - emm2 = _mm_add_epi32(emm2, p4i_1); - emm2 = _mm_and_si128(emm2, p4i_not1); - y = _mm_cvtepi32_ps(emm2); - - emm2 = _mm_sub_epi32(emm2, p4i_2); - - /* get the swap sign flag */ - emm0 = _mm_andnot_si128(emm2, p4i_4); - emm0 = _mm_slli_epi32(emm0, 29); - /* get the polynom selection mask */ - emm2 = _mm_and_si128(emm2, p4i_2); - emm2 = _mm_cmpeq_epi32(emm2, _mm_setzero_si128()); - - Packet4f sign_bit = _mm_castsi128_ps(emm0); - Packet4f poly_mask = _mm_castsi128_ps(emm2); - - /* The magic pass: "Extended precision modular arithmetic" - x = ((x - y * DP1) - y * DP2) - y * DP3; */ - xmm1 = pmul(y, p4f_minus_cephes_DP1); - xmm2 = pmul(y, p4f_minus_cephes_DP2); - xmm3 = pmul(y, p4f_minus_cephes_DP3); - x = padd(x, xmm1); - x = padd(x, xmm2); - x = padd(x, xmm3); - - /* Evaluate the first polynom (0 <= x <= Pi/4) */ - y = p4f_coscof_p0; - Packet4f z = pmul(x,x); - - y = pmadd(y,z,p4f_coscof_p1); - y = pmadd(y,z,p4f_coscof_p2); - y = pmul(y, z); - y = pmul(y, z); - Packet4f tmp = _mm_mul_ps(z, p4f_half); - y = psub(y, tmp); - y = padd(y, p4f_1); - - /* Evaluate the second polynom (Pi/4 <= x <= 0) */ - Packet4f y2 = p4f_sincof_p0; - y2 = pmadd(y2, z, p4f_sincof_p1); - y2 = pmadd(y2, z, p4f_sincof_p2); - y2 = pmul(y2, z); - y2 = pmadd(y2, x, x); - - /* select the correct result from the two polynoms */ - y2 = _mm_and_ps(poly_mask, y2); - y = _mm_andnot_ps(poly_mask, y); - y = _mm_or_ps(y,y2); - - /* update the sign */ - return _mm_xor_ps(y, sign_bit); + return pcos_float(_x); } #if EIGEN_FAST_MATH -// This is based on Quake3's fast inverse square root. +// Functions for sqrt. +// The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step +// of Newton's method, at a cost of 1-2 bits of precision as opposed to the +// exact solution. It does not handle +inf, or denormalized numbers correctly. +// The main advantage of this approach is not just speed, but also the fact that +// it can be inlined and pipelined with other computations, further reducing its +// effective latency. This is similar to Quake3's fast inverse square root. // For detail see here: http://www.beyond3d.com/content/articles/8/ -// It lacks 1 (or 2 bits in some rare cases) of precision, and does not handle negative, +inf, or denormalized numbers correctly. template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt<Packet4f>(const Packet4f& _x) { Packet4f half = pmul(_x, pset1<Packet4f>(.5f)); + Packet4f denormal_mask = _mm_and_ps( + _mm_cmpge_ps(_x, _mm_setzero_ps()), + _mm_cmplt_ps(_x, pset1<Packet4f>((std::numeric_limits<float>::min)()))); - /* select only the inverse sqrt of non-zero inputs */ - Packet4f non_zero_mask = _mm_cmpge_ps(_x, pset1<Packet4f>((std::numeric_limits<float>::min)())); - Packet4f x = _mm_and_ps(non_zero_mask, _mm_rsqrt_ps(_x)); - + // Compute approximate reciprocal sqrt. + Packet4f x = _mm_rsqrt_ps(_x); + // Do a single step of Newton's iteration. x = pmul(x, psub(pset1<Packet4f>(1.5f), pmul(half, pmul(x,x)))); - return pmul(_x,x); + // Flush results for denormals to zero. + return _mm_andnot_ps(denormal_mask, pmul(_x,x)); } #else -template<>EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +template<>EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt<Packet4f>(const Packet4f& x) { return _mm_sqrt_ps(x); } #endif @@ -474,11 +91,11 @@ template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt<Packet4f>(const Packet4f& _x) { - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inf, 0x7f800000); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(nan, 0x7fc00000); + _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inf, 0x7f800000u); + _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(nan, 0x7fc00000u); _EIGEN_DECLARE_CONST_Packet4f(one_point_five, 1.5f); _EIGEN_DECLARE_CONST_Packet4f(minus_half, -0.5f); - _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(flt_min, 0x00800000); + _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(flt_min, 0x00800000u); Packet4f neg_half = pmul(_x, p4f_minus_half); @@ -491,7 +108,7 @@ Packet4f neg_mask = _mm_cmplt_ps(_x, _mm_setzero_ps()); Packet4f zero_mask = _mm_andnot_ps(neg_mask, le_zero_mask); Packet4f infs_and_nans = _mm_or_ps(_mm_and_ps(neg_mask, p4f_nan), - _mm_and_ps(zero_mask, p4f_inf)); + _mm_and_ps(zero_mask, p4f_inf)); // Do a single step of Newton's iteration. x = pmul(x, pmadd(neg_half, pmul(x, x), p4f_one_point_five)); @@ -517,52 +134,10 @@ } // Hyperbolic Tangent function. -// Doesn't do anything fancy, just a 13/6-degree rational interpolant which -// is accurate up to a couple of ulp in the range [-9, 9], outside of which the -// fl(tanh(x)) = +/-1. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f -ptanh<Packet4f>(const Packet4f& _x) { - // Clamp the inputs to the range [-9, 9] since anything outside - // this range is +/-1.0f in single-precision. - _EIGEN_DECLARE_CONST_Packet4f(plus_9, 9.0f); - _EIGEN_DECLARE_CONST_Packet4f(minus_9, -9.0f); - const Packet4f x = pmax(p4f_minus_9, pmin(p4f_plus_9, _x)); - - // The monomial coefficients of the numerator polynomial (odd). - _EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-03f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-04f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-05f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-08f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f); - _EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f); - - // The monomial coefficients of the denominator polynomial (even). - _EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-03f); - _EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-03f); - _EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-04f); - _EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-06f); - - // Since the polynomials are odd/even, we need x^2. - const Packet4f x2 = pmul(x, x); - - // Evaluate the numerator polynomial p. - Packet4f p = pmadd(x2, p4f_alpha_13, p4f_alpha_11); - p = pmadd(x2, p, p4f_alpha_9); - p = pmadd(x2, p, p4f_alpha_7); - p = pmadd(x2, p, p4f_alpha_5); - p = pmadd(x2, p, p4f_alpha_3); - p = pmadd(x2, p, p4f_alpha_1); - p = pmul(x, p); - - // Evaluate the denominator polynomial p. - Packet4f q = pmadd(x2, p4f_beta_6, p4f_beta_4); - q = pmadd(x2, q, p4f_beta_2); - q = pmadd(x2, q, p4f_beta_0); - - // Divide the numerator by the denominator. - return pdiv(p, q); +ptanh<Packet4f>(const Packet4f& x) { + return internal::generic_fast_tanh_float(x); } } // end namespace internal
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index 4510345..9c3750a 100755 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -18,17 +18,19 @@ #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif -#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#if !defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) +// 32 bits => 8 registers +// 64 bits => 16 registers #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*)) #endif -#ifdef __FMA__ +#ifdef EIGEN_VECTORIZE_FMA #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 1 #endif #endif -#if (defined EIGEN_VECTORIZE_AVX) && EIGEN_COMP_GNUC_STRICT && (__GXX_ABI_VERSION < 1004) +#if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW) && (__GXX_ABI_VERSION < 1004)) || EIGEN_OS_QNX // With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot // have overloads for both types without linking error. // One solution is to increase ABI version using -fabi-version=4 (or greater). @@ -45,7 +47,7 @@ m_val = v; return *this; } - + T m_val; }; typedef eigen_packet_wrapper<__m128> Packet4f; @@ -61,20 +63,22 @@ template<> struct is_arithmetic<__m128i> { enum { value = true }; }; template<> struct is_arithmetic<__m128d> { enum { value = true }; }; +#define EIGEN_SSE_SHUFFLE_MASK(p,q,r,s) ((s)<<6|(r)<<4|(q)<<2|(p)) + #define vec4f_swizzle1(v,p,q,r,s) \ - (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p))))) + (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s)))) #define vec4i_swizzle1(v,p,q,r,s) \ - (_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p)))) + (_mm_shuffle_epi32( v, EIGEN_SSE_SHUFFLE_MASK(p,q,r,s))) #define vec2d_swizzle1(v,p,q) \ - (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2))))) - + (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), EIGEN_SSE_SHUFFLE_MASK(2*p,2*p+1,2*q,2*q+1)))) + #define vec4f_swizzle2(a,b,p,q,r,s) \ - (_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p)))) + (_mm_shuffle_ps( (a), (b), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s))) #define vec4i_swizzle2(a,b,p,q,r,s) \ - (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p)))))) + (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s))))) #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1<Packet4f>(X) @@ -83,7 +87,7 @@ const Packet2d p2d_##NAME = pset1<Packet2d>(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ - const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X)) + const Packet4f p4f_##NAME = pset1frombits<Packet4f>(X) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1<Packet4i>(X) @@ -110,12 +114,12 @@ HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, - HasBlend = 1 + HasBlend = 1, + HasFloor = 1 #ifdef EIGEN_VECTORIZE_SSE4_1 , HasRound = 1, - HasFloor = 1, HasCeil = 1 #endif }; @@ -158,9 +162,27 @@ }; }; -template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; }; -template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; -template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; +template<> struct unpacket_traits<Packet4f> { + typedef float type; + typedef Packet4f half; + typedef Packet4i integer_packet; + enum {size=4, alignment=Aligned16, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet2d> { + typedef double type; + typedef Packet2d half; + enum {size=2, alignment=Aligned16, vectorizable=true}; +}; +template<> struct unpacket_traits<Packet4i> { + typedef int type; + typedef Packet4i half; + enum {size=4, alignment=Aligned16, vectorizable=false}; +}; + +#ifndef EIGEN_VECTORIZE_AVX +template<> struct scalar_div_cost<float,true> { enum { value = 7 }; }; +template<> struct scalar_div_cost<double,true> { enum { value = 8 }; }; +#endif #if EIGEN_COMP_MSVC==1500 // Workaround MSVC 9 internal compiler error. @@ -175,6 +197,12 @@ template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); } #endif +template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) { return _mm_castsi128_ps(pset1<Packet4i>(from)); } + +template<> EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) { return _mm_setzero_ps(); } +template<> EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /*a*/) { return _mm_setzero_pd(); } +template<> EIGEN_STRONG_INLINE Packet4i pzero(const Packet4i& /*a*/) { return _mm_setzero_si128(); } + // GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction. // However, using inrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203) // Using inline assembly is also not an option because then gcc fails to reorder properly the instructions. @@ -185,7 +213,7 @@ return vec4f_swizzle1(_mm_load_ss(from),0,0,0,0); } #endif - + template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); } template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); } template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); } @@ -240,13 +268,49 @@ // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); } -#ifdef __FMA__ +#ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ps(a,b,c); } template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_pd(a,b,c); } #endif -template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { +#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 + // There appears to be a bug in GCC, by which the optimizer may + // flip the argument order in calls to _mm_min_ps, so we have to + // resort to inline ASM here. This is supposed to be fixed in gcc6.3, + // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 + #ifdef EIGEN_VECTORIZE_AVX + Packet4f res; + asm("vminps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); + #else + Packet4f res = b; + asm("minps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); + #endif + return res; +#else + // Arguments are reversed to match NaN propagation behavior of std::min. + return _mm_min_ps(b, a); +#endif +} +template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { +#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 + // There appears to be a bug in GCC, by which the optimizer may + // flip the argument order in calls to _mm_min_pd, so we have to + // resort to inline ASM here. This is supposed to be fixed in gcc6.3, + // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 + #ifdef EIGEN_VECTORIZE_AVX + Packet2d res; + asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); + #else + Packet2d res = b; + asm("minpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); + #endif + return res; +#else + // Arguments are reversed to match NaN propagation behavior of std::min. + return _mm_min_pd(b, a); +#endif +} template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { #ifdef EIGEN_VECTORIZE_SSE4_1 @@ -258,8 +322,44 @@ #endif } -template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { +#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 + // There appears to be a bug in GCC, by which the optimizer may + // flip the argument order in calls to _mm_max_ps, so we have to + // resort to inline ASM here. This is supposed to be fixed in gcc6.3, + // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 + #ifdef EIGEN_VECTORIZE_AVX + Packet4f res; + asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); + #else + Packet4f res = b; + asm("maxps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); + #endif + return res; +#else + // Arguments are reversed to match NaN propagation behavior of std::max. + return _mm_max_ps(b, a); +#endif +} +template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { +#if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 + // There appears to be a bug in GCC, by which the optimizer may + // flip the argument order in calls to _mm_max_pd, so we have to + // resort to inline ASM here. This is supposed to be fixed in gcc6.3, + // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 + #ifdef EIGEN_VECTORIZE_AVX + Packet2d res; + asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); + #else + Packet2d res = b; + asm("maxpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); + #endif + return res; +#else + // Arguments are reversed to match NaN propagation behavior of std::max. + return _mm_max_pd(b, a); +#endif +} template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { #ifdef EIGEN_VECTORIZE_SSE4_1 @@ -271,16 +371,24 @@ #endif } -#ifdef EIGEN_VECTORIZE_SSE4_1 -template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { return _mm_round_ps(a, 0); } -template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return _mm_round_pd(a, 0); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return _mm_cmple_ps(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return _mm_cmplt_ps(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return _mm_cmpeq_ps(a,b); } +template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return _mm_cmpeq_epi32(a,b); } +template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return _mm_cmpeq_pd(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { return _mm_cmpnge_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return _mm_ceil_ps(a); } -template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return _mm_ceil_pd(a); } - -template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return _mm_floor_ps(a); } -template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return _mm_floor_pd(a); } -#endif +template<> EIGEN_STRONG_INLINE Packet4i ptrue<Packet4i>(const Packet4i& a) { return _mm_cmpeq_epi32(a, a); } +template<> EIGEN_STRONG_INLINE Packet4f +ptrue<Packet4f>(const Packet4f& a) { + Packet4i b = _mm_castps_si128(a); + return _mm_castsi128_ps(_mm_cmpeq_epi32(b, b)); +} +template<> EIGEN_STRONG_INLINE Packet2d +ptrue<Packet2d>(const Packet2d& a) { + Packet4i b = _mm_castpd_si128(a); + return _mm_castsi128_pd(_mm_cmpeq_epi32(b, b)); +} template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); } @@ -294,9 +402,47 @@ template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); } -template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); } -template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); } -template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); } +template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(b,a); } +template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(b,a); } +template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(b,a); } + +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftright(Packet4i a) { return _mm_srli_epi32(a,N); } +template<int N> EIGEN_STRONG_INLINE Packet4i pshiftleft(Packet4i a) { return _mm_slli_epi32(a,N); } + +#ifdef EIGEN_VECTORIZE_SSE4_1 +template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { return _mm_round_ps(a, 0); } +template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return _mm_round_pd(a, 0); } + +template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return _mm_ceil_ps(a); } +template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return _mm_ceil_pd(a); } + +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return _mm_floor_ps(a); } +template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return _mm_floor_pd(a); } +#else +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) +{ + const Packet4f cst_1 = pset1<Packet4f>(1.0f); + Packet4i emm0 = _mm_cvttps_epi32(a); + Packet4f tmp = _mm_cvtepi32_ps(emm0); + /* if greater, substract 1 */ + Packet4f mask = _mm_cmpgt_ps(tmp, a); + mask = pand(mask, cst_1); + return psub(tmp, mask); +} + +// WARNING: this pfloor implementation makes sense for small inputs only, +// It is currently only used by pexp and not exposed through HasFloor. +template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) +{ + const Packet2d cst_1 = pset1<Packet2d>(1.0); + Packet4i emm0 = _mm_cvttpd_epi32(a); + Packet2d tmp = _mm_cvtepi32_pd(emm0); + /* if greater, substract 1 */ + Packet2d mask = _mm_cmpgt_pd(tmp, a); + mask = pand(mask, cst_1); + return psub(tmp, mask); +} +#endif template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); } @@ -404,10 +550,16 @@ pstore(to, Packet2d(vec2d_swizzle1(pa,0,0))); } +#if EIGEN_COMP_PGI +typedef const void * SsePrefetchPtrType; +#else +typedef const char * SsePrefetchPtrType; +#endif + #ifndef EIGEN_VECTORIZE_AVX -template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } -template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } -template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } +template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } #endif #if EIGEN_COMP_MSVC_STRICT && EIGEN_OS_WIN64 @@ -434,30 +586,6 @@ template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return _mm_shuffle_epi32(a,0x1B); } -template<size_t offset> -struct protate_impl<offset, Packet4f> -{ - static Packet4f run(const Packet4f& a) { - return vec4f_swizzle1(a, offset, (offset + 1) % 4, (offset + 2) % 4, (offset + 3) % 4); - } -}; - -template<size_t offset> -struct protate_impl<offset, Packet4i> -{ - static Packet4i run(const Packet4i& a) { - return vec4i_swizzle1(a, offset, (offset + 1) % 4, (offset + 2) % 4, (offset + 3) % 4); - } -}; - -template<size_t offset> -struct protate_impl<offset, Packet2d> -{ - static Packet2d run(const Packet2d& a) { - return vec2d_swizzle1(a, offset, (offset + 1) % 2); - } -}; - template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF)); @@ -478,6 +606,23 @@ #endif } +template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) { + return pfrexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) { + return pldexp_float(a,exponent); +} + +template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { + const Packet4i cst_1023_0 = _mm_setr_epi32(1023, 1023, 0, 0); + Packet4i emm0 = _mm_cvttpd_epi32(exponent); + emm0 = padd(emm0, cst_1023_0); + emm0 = _mm_slli_epi32(emm0, 20); + emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(1,2,0,3)); + return pmul(a, Packet2d(_mm_castsi128_pd(emm0))); +} + // with AVX, the default implementations based on pload1 are faster #ifndef __AVX__ template<> EIGEN_STRONG_INLINE void @@ -523,30 +668,13 @@ { return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3])); } + template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) { return _mm_hadd_pd(vecs[0], vecs[1]); } -template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) -{ - Packet4f tmp0 = _mm_hadd_ps(a,a); - return pfirst<Packet4f>(_mm_hadd_ps(tmp0, tmp0)); -} - -template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst<Packet2d>(_mm_hadd_pd(a, a)); } #else -// SSE2 versions -template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) -{ - Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); - return pfirst<Packet4f>(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); -} -template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) -{ - return pfirst<Packet2d>(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); -} - template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) { Packet4f tmp0, tmp1, tmp2; @@ -567,6 +695,29 @@ } #endif // SSE3 +template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) +{ + // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures + // (from Nehalem to Haswell) +// #ifdef EIGEN_VECTORIZE_SSE3 +// Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3)); +// return pfirst<Packet4f>(_mm_hadd_ps(tmp, tmp)); +// #else + Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); + return pfirst<Packet4f>(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); +// #endif +} + +template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) +{ + // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures + // (from Nehalem to Haswell) +// #ifdef EIGEN_VECTORIZE_SSE3 +// return pfirst<Packet2d>(_mm_hadd_pd(a, a)); +// #else + return pfirst<Packet2d>(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); +// #endif +} #ifdef EIGEN_VECTORIZE_SSSE3 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) @@ -618,7 +769,7 @@ // TODO try to call _mm_mul_epu32 directly EIGEN_ALIGN16 int aux[4]; pstore(aux, a); - return (aux[0] * aux[1]) * (aux[2] * aux[3]);; + return (aux[0] * aux[1]) * (aux[2] * aux[3]); } // min @@ -673,6 +824,17 @@ #endif // EIGEN_VECTORIZE_SSE4_1 } +// not needed yet +// template<> EIGEN_STRONG_INLINE bool predux_all(const Packet4f& x) +// { +// return _mm_movemask_ps(x) == 0xF; +// } + +template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) +{ + return _mm_movemask_ps(x) != 0x0; +} + #if EIGEN_COMP_GNUC // template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) // { @@ -837,8 +999,66 @@ #endif } +template<> EIGEN_STRONG_INLINE Packet4f pinsertfirst(const Packet4f& a, float b) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_blend_ps(a,pset1<Packet4f>(b),1); +#else + return _mm_move_ss(a, _mm_load_ss(&b)); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet2d pinsertfirst(const Packet2d& a, double b) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_blend_pd(a,pset1<Packet2d>(b),1); +#else + return _mm_move_sd(a, _mm_load_sd(&b)); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet4f pinsertlast(const Packet4f& a, float b) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_blend_ps(a,pset1<Packet4f>(b),(1<<3)); +#else + const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x0,0x0,0x0,0xFFFFFFFF)); + return _mm_or_ps(_mm_andnot_ps(mask, a), _mm_and_ps(mask, pset1<Packet4f>(b))); +#endif +} + +template<> EIGEN_STRONG_INLINE Packet2d pinsertlast(const Packet2d& a, double b) +{ +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_blend_pd(a,pset1<Packet2d>(b),(1<<1)); +#else + const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x0,0xFFFFFFFF,0xFFFFFFFF)); + return _mm_or_pd(_mm_andnot_pd(mask, a), _mm_and_pd(mask, pset1<Packet2d>(b))); +#endif +} + +// Scalar path for pmadd with FMA to ensure consistency with vectorized path. +#ifdef EIGEN_VECTORIZE_FMA +template<> EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const float& c) { + return ::fmaf(a,b,c); +} +template<> EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) { + return ::fma(a,b,c); +} +#endif + } // end namespace internal } // end namespace Eigen +#if EIGEN_COMP_PGI +// PGI++ does not define the following intrinsics in C++ mode. +static inline __m128 _mm_castpd_ps (__m128d x) { return reinterpret_cast<__m128&>(x); } +static inline __m128i _mm_castpd_si128(__m128d x) { return reinterpret_cast<__m128i&>(x); } +static inline __m128d _mm_castps_pd (__m128 x) { return reinterpret_cast<__m128d&>(x); } +static inline __m128i _mm_castps_si128(__m128 x) { return reinterpret_cast<__m128i&>(x); } +static inline __m128 _mm_castsi128_ps(__m128i x) { return reinterpret_cast<__m128&>(x); } +static inline __m128d _mm_castsi128_pd(__m128i x) { return reinterpret_cast<__m128d&>(x); } +#endif + #endif // EIGEN_PACKET_MATH_SSE_H
diff --git a/Eigen/src/Core/arch/SSE/TypeCasting.h b/Eigen/src/Core/arch/SSE/TypeCasting.h index c848932..f607366 100644 --- a/Eigen/src/Core/arch/SSE/TypeCasting.h +++ b/Eigen/src/Core/arch/SSE/TypeCasting.h
@@ -14,6 +14,7 @@ namespace internal { +#ifndef EIGEN_VECTORIZE_AVX template <> struct type_casting_traits<float, int> { enum { @@ -23,11 +24,6 @@ }; }; -template<> EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) { - return _mm_cvttps_epi32(a); -} - - template <> struct type_casting_traits<int, float> { enum { @@ -37,11 +33,6 @@ }; }; -template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) { - return _mm_cvtepi32_ps(a); -} - - template <> struct type_casting_traits<double, float> { enum { @@ -51,10 +42,6 @@ }; }; -template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) { - return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6)); -} - template <> struct type_casting_traits<float, double> { enum { @@ -63,12 +50,32 @@ TgtCoeffRatio = 2 }; }; +#endif + +template<> EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) { + return _mm_cvttps_epi32(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) { + return _mm_cvtepi32_ps(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) { + return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6)); +} template<> EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) { // Simply discard the second half of the input return _mm_cvtps_pd(a); } +template<> EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i,Packet4f>(const Packet4f& a) { + return _mm_castps_si128(a); +} + +template<> EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f,Packet4i>(const Packet4i& a) { + return _mm_castsi128_ps(a); +} } // end namespace internal
diff --git a/Eigen/src/Core/arch/SYCL/InteropHeaders.h b/Eigen/src/Core/arch/SYCL/InteropHeaders.h new file mode 100644 index 0000000..294cb10 --- /dev/null +++ b/Eigen/src/Core/arch/SYCL/InteropHeaders.h
@@ -0,0 +1,104 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * InteropHeaders.h + * + * \brief: + * InteropHeaders + * +*****************************************************************/ + +#ifndef EIGEN_INTEROP_HEADERS_SYCL_H +#define EIGEN_INTEROP_HEADERS_SYCL_H +#if defined EIGEN_USE_SYCL +namespace Eigen { + +namespace internal { +#define SYCL_PACKET_TRAITS(packet_type, val, unpacket_type, lengths)\ + template<> struct packet_traits<unpacket_type> : default_packet_traits\ + {\ + typedef packet_type type;\ + typedef packet_type half;\ + enum {\ + Vectorizable = 1,\ + AlignedOnScalar = 1,\ + size=lengths,\ + HasHalfPacket = 0,\ + HasDiv = 1,\ + HasLog = 1,\ + HasExp = 1,\ + HasSqrt = 1,\ + HasRsqrt = 1,\ + HasSin = 1,\ + HasCos = 1,\ + HasTan = 1,\ + HasASin = 1,\ + HasACos = 1,\ + HasATan = 1,\ + HasSinh = 1,\ + HasCosh = 1,\ + HasTanh = 1,\ + HasLGamma = 0,\ + HasDiGamma = 0,\ + HasZeta = 0,\ + HasPolygamma = 0,\ + HasErf = 0,\ + HasErfc = 0,\ + HasIGamma = 0,\ + HasIGammac = 0,\ + HasBetaInc = 0,\ + HasBlend = val,\ + HasMax=1,\ + HasMin=1,\ + HasMul=1,\ + HasAdd=1,\ + HasFloor=1,\ + HasRound=1,\ + HasLog1p=1,\ + HasExpm1=1,\ + HasCeil=1,\ + };\ + }; + +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, const float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, double, 2) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, const double, 2) +#undef SYCL_PACKET_TRAITS + + +// Make sure this is only available when targeting a GPU: we don't want to +// introduce conflicts between these packet_traits definitions and the ones +// we'll use on the host side (SSE, AVX, ...) +#define SYCL_ARITHMETIC(packet_type) template<> struct is_arithmetic<packet_type> { enum { value = true }; }; +SYCL_ARITHMETIC(cl::sycl::cl_float4) +SYCL_ARITHMETIC(cl::sycl::cl_double2) +#undef SYCL_ARITHMETIC + +#define SYCL_UNPACKET_TRAITS(packet_type, unpacket_type, lengths)\ +template<> struct unpacket_traits<packet_type> {\ + typedef unpacket_type type;\ + enum {size=lengths, alignment=Aligned16, vectorizable=true};\ + typedef packet_type half;\ +}; +SYCL_UNPACKET_TRAITS(cl::sycl::cl_float4, float, 4) +SYCL_UNPACKET_TRAITS(cl::sycl::cl_double2, double, 2) + +#undef SYCL_UNPACKET_TRAITS + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_USE_SYCL +#endif // EIGEN_INTEROP_HEADERS_SYCL_H
diff --git a/Eigen/src/Core/arch/SYCL/MathFunctions.h b/Eigen/src/Core/arch/SYCL/MathFunctions.h new file mode 100644 index 0000000..422839c --- /dev/null +++ b/Eigen/src/Core/arch/SYCL/MathFunctions.h
@@ -0,0 +1,221 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * MathFunctions.h + * + * \brief: + * MathFunctions + * +*****************************************************************/ + +#ifndef EIGEN_MATH_FUNCTIONS_SYCL_H +#define EIGEN_MATH_FUNCTIONS_SYCL_H + +namespace Eigen { + +namespace internal { + +// Make sure this is only available when targeting a GPU: we don't want to +// introduce conflicts between these packet_traits definitions and the ones +// we'll use on the host side (SSE, AVX, ...) +//#if defined(__SYCL_DEVICE_ONLY__) && defined(EIGEN_USE_SYCL) +#define SYCL_PLOG(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type plog<packet_type>(const packet_type& a) { return cl::sycl::log(a); } + +SYCL_PLOG(cl::sycl::cl_float4) +SYCL_PLOG(cl::sycl::cl_double2) +#undef SYCL_PLOG + +#define SYCL_PLOG1P(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type plog1p<packet_type>(const packet_type& a) { return cl::sycl::log1p(a); } + +SYCL_PLOG1P(cl::sycl::cl_float4) +SYCL_PLOG1P(cl::sycl::cl_double2) +#undef SYCL_PLOG1P + +#define SYCL_PLOG10(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type plog10<packet_type>(const packet_type& a) { return cl::sycl::log10(a); } + +SYCL_PLOG10(cl::sycl::cl_float4) +SYCL_PLOG10(cl::sycl::cl_double2) +#undef SYCL_PLOG10 + +#define SYCL_PEXP(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pexp<packet_type>(const packet_type& a) { return cl::sycl::exp(a); } + +SYCL_PEXP(cl::sycl::cl_float4) +SYCL_PEXP(cl::sycl::cl_double2) +#undef SYCL_PEXP + +#define SYCL_PEXPM1(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pexpm1<packet_type>(const packet_type& a) { return cl::sycl::expm1(a); } + +SYCL_PEXPM1(cl::sycl::cl_float4) +SYCL_PEXPM1(cl::sycl::cl_double2) +#undef SYCL_PEXPM1 + +#define SYCL_PSQRT(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type psqrt<packet_type>(const packet_type& a) { return cl::sycl::sqrt(a); } + +SYCL_PSQRT(cl::sycl::cl_float4) +SYCL_PSQRT(cl::sycl::cl_double2) +#undef SYCL_PSQRT + + +#define SYCL_PRSQRT(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type prsqrt<packet_type>(const packet_type& a) { return cl::sycl::rsqrt(a); } + +SYCL_PRSQRT(cl::sycl::cl_float4) +SYCL_PRSQRT(cl::sycl::cl_double2) +#undef SYCL_PRSQRT + + +/** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ +#define SYCL_PSIN(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type psin<packet_type>(const packet_type& a) { return cl::sycl::sin(a); } + +SYCL_PSIN(cl::sycl::cl_float4) +SYCL_PSIN(cl::sycl::cl_double2) +#undef SYCL_PSIN + + +/** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ +#define SYCL_PCOS(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pcos<packet_type>(const packet_type& a) { return cl::sycl::cos(a); } + +SYCL_PCOS(cl::sycl::cl_float4) +SYCL_PCOS(cl::sycl::cl_double2) +#undef SYCL_PCOS + +/** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ +#define SYCL_PTAN(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type ptan<packet_type>(const packet_type& a) { return cl::sycl::tan(a); } + +SYCL_PTAN(cl::sycl::cl_float4) +SYCL_PTAN(cl::sycl::cl_double2) +#undef SYCL_PTAN + +/** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ +#define SYCL_PASIN(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pasin<packet_type>(const packet_type& a) { return cl::sycl::asin(a); } + +SYCL_PASIN(cl::sycl::cl_float4) +SYCL_PASIN(cl::sycl::cl_double2) +#undef SYCL_PASIN + + +/** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ +#define SYCL_PACOS(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pacos<packet_type>(const packet_type& a) { return cl::sycl::acos(a); } + +SYCL_PACOS(cl::sycl::cl_float4) +SYCL_PACOS(cl::sycl::cl_double2) +#undef SYCL_PACOS + +/** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ +#define SYCL_PATAN(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type patan<packet_type>(const packet_type& a) { return cl::sycl::atan(a); } + +SYCL_PATAN(cl::sycl::cl_float4) +SYCL_PATAN(cl::sycl::cl_double2) +#undef SYCL_PATAN + +/** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ +#define SYCL_PSINH(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type psinh<packet_type>(const packet_type& a) { return cl::sycl::sinh(a); } + +SYCL_PSINH(cl::sycl::cl_float4) +SYCL_PSINH(cl::sycl::cl_double2) +#undef SYCL_PSINH + +/** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ +#define SYCL_PCOSH(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pcosh<packet_type>(const packet_type& a) { return cl::sycl::cosh(a); } + +SYCL_PCOSH(cl::sycl::cl_float4) +SYCL_PCOSH(cl::sycl::cl_double2) +#undef SYCL_PCOSH + +/** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ +#define SYCL_PTANH(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type ptanh<packet_type>(const packet_type& a) { return cl::sycl::tanh(a); } + +SYCL_PTANH(cl::sycl::cl_float4) +SYCL_PTANH(cl::sycl::cl_double2) +#undef SYCL_PTANH + +#define SYCL_PCEIL(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pceil<packet_type>(const packet_type& a) { return cl::sycl::ceil(a); } + +SYCL_PCEIL(cl::sycl::cl_float4) +SYCL_PCEIL(cl::sycl::cl_double2) +#undef SYCL_PCEIL + + +#define SYCL_PROUND(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pround<packet_type>(const packet_type& a) { return cl::sycl::round(a); } + +SYCL_PROUND(cl::sycl::cl_float4) +SYCL_PROUND(cl::sycl::cl_double2) +#undef SYCL_PROUND + +#define SYCL_FLOOR(packet_type) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pfloor<packet_type>(const packet_type& a) { return cl::sycl::floor(a); } + +SYCL_FLOOR(cl::sycl::cl_float4) +SYCL_FLOOR(cl::sycl::cl_double2) +#undef SYCL_FLOOR + + +#define SYCL_PMIN(packet_type, expr) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pmin<packet_type>(const packet_type& a, const packet_type& b) { return expr; } + +SYCL_PMIN(cl::sycl::cl_float4, cl::sycl::fmin(a, b)) +SYCL_PMIN(cl::sycl::cl_double2, cl::sycl::fmin(a, b)) +#undef SYCL_PMIN + +#define SYCL_PMAX(packet_type, expr) \ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +packet_type pmax<packet_type>(const packet_type& a, const packet_type& b) { return expr; } + +SYCL_PMAX(cl::sycl::cl_float4, cl::sycl::fmax(a, b)) +SYCL_PMAX(cl::sycl::cl_double2, cl::sycl::fmax(a, b)) +#undef SYCL_PMAX + +//#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MATH_FUNCTIONS_CUDA_H
diff --git a/Eigen/src/Core/arch/SYCL/PacketMath.h b/Eigen/src/Core/arch/SYCL/PacketMath.h new file mode 100644 index 0000000..820a833 --- /dev/null +++ b/Eigen/src/Core/arch/SYCL/PacketMath.h
@@ -0,0 +1,458 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * PacketMath.h + * + * \brief: + * PacketMath + * +*****************************************************************/ + +#ifndef EIGEN_PACKET_MATH_SYCL_H +#define EIGEN_PACKET_MATH_SYCL_H +#include <type_traits> +#if defined EIGEN_USE_SYCL +namespace Eigen { + +namespace internal { + +#define SYCL_PLOADT_RO(address_space_target)\ +template<typename packet_type, int Alignment>\ + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type\ + ploadt_ro(typename cl::sycl::multi_ptr<const typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t from) {\ + typedef typename unpacket_traits<packet_type>::type scalar;\ + typedef cl::sycl::multi_ptr<scalar, cl::sycl::access::address_space::address_space_target> multi_ptr;\ + auto res=packet_type(static_cast<typename unpacket_traits<packet_type>::type>(0));\ + res.load(0, multi_ptr(const_cast<typename multi_ptr::pointer_t>(from)));\ + return res;\ +} + +SYCL_PLOADT_RO(global_space) +SYCL_PLOADT_RO(local_space) + +#undef SYCL_PLOADT_RO + + +#define SYCL_PLOAD(address_space_target, Alignment, AlignedType)\ +template<typename packet_type> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type\ + pload##AlignedType(typename cl::sycl::multi_ptr<const typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t from) {\ + return ploadt_ro<packet_type, Alignment>(from);\ + } + +// global space +SYCL_PLOAD(global_space, Unaligned, u) +SYCL_PLOAD(global_space, Aligned, ) + +// local space +SYCL_PLOAD(local_space, Unaligned, u) +SYCL_PLOAD(local_space, Aligned, ) + +// private space +//SYCL_PLOAD(private_space, Unaligned, u) +//SYCL_PLOAD(private_space, Aligned, ) + +#undef SYCL_PLOAD + + +/** \internal \returns a packet version of \a *from. + * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ +#define SYCL_PLOADT(address_space_target)\ +template<typename packet_type, int Alignment>\ +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt(\ + typename cl::sycl::multi_ptr<const typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t from)\ +{\ + if(Alignment >= unpacket_traits<packet_type>::alignment)\ + return pload<packet_type>(from);\ + else\ + return ploadu<packet_type>(from);\ +} + +// global space +SYCL_PLOADT(global_space) +// local space +SYCL_PLOADT(local_space) + +//private_space +// There is no need to specialise it for private space as it can use the GenericPacketMath version + +#define SYCL_PLOADT_RO_SPECIAL(packet_type, Alignment)\ + template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type\ + ploadt_ro<packet_type, Alignment>(const typename unpacket_traits<packet_type>::type * from) { \ + typedef typename unpacket_traits<packet_type>::type scalar;\ + auto res=packet_type(static_cast<scalar>(0));\ + res. template load<cl::sycl::access::address_space::private_space>(0, const_cast<scalar*>(from));\ + return res;\ + } + +SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Aligned) +SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Aligned) +SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Unaligned) +SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Unaligned) + + +#define SYCL_PLOAD_SPECIAL(packet_type, alignment_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type\ + pload##alignment_type(const typename unpacket_traits<packet_type>::type * from) { \ + typedef typename unpacket_traits<packet_type>::type scalar;\ + auto res=packet_type(static_cast<scalar>(0));\ + res. template load<cl::sycl::access::address_space::private_space>(0, const_cast<scalar*>(from));\ + return res;\ + } +SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4,) +SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2,) +SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4, u) +SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2, u) + +#undef SYCL_PLOAD_SPECIAL + +#define SYCL_PSTORE(scalar, packet_type, address_space_target, alignment)\ +template<>\ + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ + typename cl::sycl::multi_ptr<scalar, cl::sycl::access::address_space::address_space_target>::pointer_t to, \ + const packet_type& from) {\ + typedef cl::sycl::multi_ptr<scalar, cl::sycl::access::address_space::address_space_target> multi_ptr;\ + from.store(0, multi_ptr(to));\ +} + +// global space +SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, ) +SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, u) +SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, ) +SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, u) + +SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, ) +SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, u) +SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, ) +SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, u) + +SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, ) +SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, u) +SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, ) +SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, u) + + +#define SYCL_PSTORE_T(scalar, packet_type, Alignment)\ +template<>\ +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret<scalar, packet_type, Alignment>(\ + scalar* to,\ + const packet_type& from) {\ + if(Alignment)\ + pstore(to, from);\ + else\ + pstoreu(to,from);\ +} + + +SYCL_PSTORE_T(float, cl::sycl::cl_float4, Aligned) + +SYCL_PSTORE_T(float, cl::sycl::cl_float4, Unaligned) + +SYCL_PSTORE_T(double, cl::sycl::cl_double2, Aligned) + +SYCL_PSTORE_T(double, cl::sycl::cl_double2, Unaligned) + + +#undef SYCL_PSTORE_T + +#define SYCL_PSET1(packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pset1<packet_type>(\ + const typename unpacket_traits<packet_type>::type& from) {\ + return packet_type(from);\ +} + +// global space +SYCL_PSET1(cl::sycl::cl_float4) +SYCL_PSET1(cl::sycl::cl_double2) + +#undef SYCL_PSET1 + + +template <typename packet_type> struct get_base_packet { +template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type get_ploaddup(sycl_multi_pointer ) {} + + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type get_pgather(sycl_multi_pointer , Index ) {} +}; + +template <> struct get_base_packet <cl::sycl::cl_float4> { + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 get_ploaddup(sycl_multi_pointer from) { + return cl::sycl::cl_float4(from[0], from[0], from[1], from[1]); + } + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 get_pgather(sycl_multi_pointer from, Index stride) { + return cl::sycl::cl_float4(from[0*stride], from[1*stride], from[2*stride], from[3*stride]); + } + + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_pscatter(sycl_multi_pointer to , const cl::sycl::cl_float4& from, Index stride) { + auto tmp = stride; + to[0] = from.x(); + to[tmp] = from.y(); + to[tmp += stride] = from.z(); + to[tmp += stride] = from.w(); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 set_plset(const float& a) { + return cl::sycl::cl_float4(static_cast<float>(a), static_cast<float>(a+1), static_cast<float>(a+2), static_cast<float>(a+3)); + } +}; + +template <> struct get_base_packet <cl::sycl::cl_double2> { + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 get_ploaddup(const sycl_multi_pointer from) { + return cl::sycl::cl_double2(from[0], from[0]); + } + + template <typename sycl_multi_pointer, typename Index> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 get_pgather(const sycl_multi_pointer from, Index stride) { + return cl::sycl::cl_double2(from[0*stride], from[1*stride]); + } + + template <typename sycl_multi_pointer> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_pscatter(sycl_multi_pointer to , const cl::sycl::cl_double2& from, Index stride) { + to[0] = from.x(); + to[stride] = from.y(); + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 set_plset(const double& a) { + return cl::sycl::cl_double2(static_cast<double>(a), static_cast<double>(a + 1)); + } +}; + +#define SYCL_PLOAD_DUP(address_space_target)\ +template<typename packet_type> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type \ +ploaddup(typename cl::sycl::multi_ptr<const typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t from)\ +{\ + return get_base_packet<packet_type>::get_ploaddup(from); \ +} + +// global space +SYCL_PLOAD_DUP(global_space) +// local_space +SYCL_PLOAD_DUP(local_space) +// private_space +//SYCL_PLOAD_DUP(private_space) +#undef SYCL_PLOAD_DUP + +#define SYCL_PLOAD_DUP_SPECILIZE(packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type \ +ploaddup<packet_type>(const typename unpacket_traits<packet_type>::type * from)\ +{ \ + return get_base_packet<packet_type>::get_ploaddup(from); \ +} + +SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_float4) +SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_double2) + +#undef SYCL_PLOAD_DUP_SPECILIZE + +#define SYCL_PLSET(packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type plset<packet_type>(const typename unpacket_traits<packet_type>::type& a) {\ + return get_base_packet<packet_type>::set_plset(a);\ +} + +SYCL_PLSET(cl::sycl::cl_float4) +SYCL_PLSET(cl::sycl::cl_double2) + +#undef SYCL_PLSET + + +#define SYCL_PGATHER(address_space_target)\ +template<typename Scalar, typename packet_type> EIGEN_DEVICE_FUNC inline packet_type pgather(\ + typename cl::sycl::multi_ptr<const typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t from, Index stride) {\ + return get_base_packet<packet_type>::get_pgather(from, stride); \ +} + +// global space +SYCL_PGATHER(global_space) +// local space +SYCL_PGATHER(local_space) +// private space +//SYCL_PGATHER(private_space) + +#undef SYCL_PGATHER + + +#define SYCL_PGATHER_SPECILIZE(scalar, packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type \ +pgather<scalar, packet_type>(const typename unpacket_traits<packet_type>::type * from, Index stride)\ +{ \ + return get_base_packet<packet_type>::get_pgather(from, stride); \ +} + +SYCL_PGATHER_SPECILIZE(float, cl::sycl::cl_float4) +SYCL_PGATHER_SPECILIZE(double, cl::sycl::cl_double2) + +#undef SYCL_PGATHER_SPECILIZE + +#define SYCL_PSCATTER(address_space_target)\ +template<typename Scalar, typename packet_type> EIGEN_DEVICE_FUNC inline void pscatter(\ + typename cl::sycl::multi_ptr<typename unpacket_traits<packet_type>::type,\ + cl::sycl::access::address_space::address_space_target>::pointer_t to,\ + const packet_type& from, Index stride) {\ + get_base_packet<packet_type>::set_pscatter(to, from, stride);\ +} + +// global space +SYCL_PSCATTER(global_space) +// local space +SYCL_PSCATTER(local_space) +// private space +//SYCL_PSCATTER(private_space) + +#undef SYCL_PSCATTER + + + +#define SYCL_PSCATTER_SPECILIZE(scalar, packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void \ +pscatter<scalar, packet_type>(typename unpacket_traits<packet_type>::type * to, const packet_type& from, Index stride)\ +{ \ + get_base_packet<packet_type>::set_pscatter(to, from, stride);\ +} + +SYCL_PSCATTER_SPECILIZE(float, cl::sycl::cl_float4) +SYCL_PSCATTER_SPECILIZE(double, cl::sycl::cl_double2) + +#undef SYCL_PSCATTER_SPECILIZE + + +#define SYCL_PMAD(packet_type)\ +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pmadd( const packet_type& a,\ + const packet_type& b, const packet_type& c){\ + return cl::sycl::mad(a,b,c);\ +} + +SYCL_PMAD(cl::sycl::cl_float4) +SYCL_PMAD(cl::sycl::cl_double2) +#undef SYCL_PMAD + + + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float pfirst<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return a.x(); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double pfirst<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return a.x(); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return a.x() + a.y() + a.z() + a.w(); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return a.x() + a.y(); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_max<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return cl::sycl::fmax(cl::sycl::fmax(a.x(), a.y()), cl::sycl::fmax(a.z(), a.w())); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_max<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return cl::sycl::fmax(a.x(), a.y()); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_min<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return cl::sycl::fmin(cl::sycl::fmin(a.x(), a.y()), cl::sycl::fmin(a.z(), a.w())); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_min<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return cl::sycl::fmin(a.x(), a.y()); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_mul<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return a.x() * a.y() * a.z() * a.w(); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_mul<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return a.x() * a.y(); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pabs<cl::sycl::cl_float4>(const cl::sycl::cl_float4& a) { + return cl::sycl::cl_float4(cl::sycl::fabs(a.x()), cl::sycl::fabs(a.y()), cl::sycl::fabs(a.z()), cl::sycl::fabs(a.w())); +} +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double2 pabs<cl::sycl::cl_double2>(const cl::sycl::cl_double2& a) { + return cl::sycl::cl_double2(cl::sycl::fabs(a.x()), cl::sycl::fabs(a.y())); +} + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void +ptranspose(PacketBlock<cl::sycl::cl_float4,4>& kernel) { + float tmp = kernel.packet[0].y(); + kernel.packet[0].y() = kernel.packet[1].x(); + kernel.packet[1].x() = tmp; +// std::swap(kernel.packet[0].y(), kernel.packet[1].x()); + + tmp = kernel.packet[0].z(); + kernel.packet[0].z() = kernel.packet[2].x(); + kernel.packet[2].x() = tmp; + //std::swap(kernel.packet[0].z(), kernel.packet[2].x()); + + tmp = kernel.packet[0].w(); + kernel.packet[0].w() = kernel.packet[3].x(); + kernel.packet[3].x() = tmp; + + //std::swap(kernel.packet[0].w(), kernel.packet[3].x()); + + tmp = kernel.packet[1].z(); + kernel.packet[1].z() = kernel.packet[2].y(); + kernel.packet[2].y() = tmp; +// std::swap(kernel.packet[1].z(), kernel.packet[2].y()); + + tmp = kernel.packet[1].w(); + kernel.packet[1].w() = kernel.packet[3].y(); + kernel.packet[3].y() = tmp; +// std::swap(kernel.packet[1].w(), kernel.packet[3].y()); + + tmp = kernel.packet[2].w(); + kernel.packet[2].w() = kernel.packet[3].z(); + kernel.packet[3].z() = tmp; +// std::swap(kernel.packet[2].w(), kernel.packet[3].z()); + +} + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void +ptranspose(PacketBlock<cl::sycl::cl_double2,2>& kernel) { + double tmp = kernel.packet[0].y(); + kernel.packet[0].y() = kernel.packet[1].x(); + kernel.packet[1].x() = tmp; +//std::swap(kernel.packet[0].y(), kernel.packet[1].x()); +} + + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 +pblend(const Selector<unpacket_traits<cl::sycl::cl_float4>::size>& ifPacket, + const cl::sycl::cl_float4& thenPacket, const cl::sycl::cl_float4& elsePacket) { + cl::sycl::cl_int4 condition(ifPacket.select[0] ? 0 : -1, + ifPacket.select[1] ? 0 : -1, + ifPacket.select[2] ? 0 : -1, + ifPacket.select[3] ? 0 : -1); + return cl::sycl::select(thenPacket, elsePacket, condition); +} + +template<> inline cl::sycl::cl_double2 +pblend(const Selector<unpacket_traits<cl::sycl::cl_double2>::size>& ifPacket, + const cl::sycl::cl_double2& thenPacket, const cl::sycl::cl_double2& elsePacket) { + cl::sycl::cl_long2 condition(ifPacket.select[0] ? 0 : -1, + ifPacket.select[1] ? 0 : -1); + return cl::sycl::select(thenPacket, elsePacket, condition); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_USE_SYCL +#endif // EIGEN_PACKET_MATH_SYCL_H
diff --git a/Eigen/src/Core/arch/SYCL/TypeCasting.h b/Eigen/src/Core/arch/SYCL/TypeCasting.h new file mode 100644 index 0000000..dedd5c8 --- /dev/null +++ b/Eigen/src/Core/arch/SYCL/TypeCasting.h
@@ -0,0 +1,89 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TypeCasting.h + * + * \brief: + * TypeCasting + * +*****************************************************************/ + +#ifndef EIGEN_TYPE_CASTING_SYCL_H +#define EIGEN_TYPE_CASTING_SYCL_H + +namespace Eigen { + +namespace internal { +#ifdef __SYCL_DEVICE_ONLY__ +template <> +struct type_casting_traits<float, int> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_int4 pcast<cl::sycl::cl_float4, cl::sycl::cl_int4>(const cl::sycl::cl_float4& a) { + return a. template convert<cl::sycl::cl_int, cl::sycl::rounding_mode::automatic>(); +} + + +template <> +struct type_casting_traits<int, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pcast<cl::sycl::cl_int4, cl::sycl::cl_float4>(const cl::sycl::cl_int4& a) { + return a. template convert<cl::sycl::cl_float, cl::sycl::rounding_mode::automatic>(); +} + +template <> +struct type_casting_traits<double, float> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 2, + TgtCoeffRatio = 1 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pcast<cl::sycl::cl_double2, cl::sycl::cl_float4>(const cl::sycl::cl_double2& a, const cl::sycl::cl_double2& b) { + auto a1=a. template convert<cl::sycl::cl_float, cl::sycl::rounding_mode::automatic>(); + auto b1=b. template convert<cl::sycl::cl_float, cl::sycl::rounding_mode::automatic>(); + return cl::sycl::float4(a1.x(), a1.y(), b1.x(), b1.y()); +} + +template <> +struct type_casting_traits<float, double> { + enum { + VectorizedCast = 1, + SrcCoeffRatio = 1, + TgtCoeffRatio = 2 + }; +}; + +template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double2 pcast<cl::sycl::cl_float4, cl::sycl::cl_double2>(const cl::sycl::cl_float4& a) { + // Simply discard the second half of the input + return cl::sycl::cl_double2(a.x(), a.y()); +} + +#endif +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_SYCL_H
diff --git a/Eigen/src/Core/arch/ZVector/CMakeLists.txt b/Eigen/src/Core/arch/ZVector/CMakeLists.txt deleted file mode 100644 index 5eb0957..0000000 --- a/Eigen/src/Core/arch/ZVector/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_arch_ZVector_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_arch_ZVector_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/arch/ZVector COMPONENT Devel -)
diff --git a/Eigen/src/Core/arch/ZVector/Complex.h b/Eigen/src/Core/arch/ZVector/Complex.h index 9a8735a..167c3ee 100644 --- a/Eigen/src/Core/arch/ZVector/Complex.h +++ b/Eigen/src/Core/arch/ZVector/Complex.h
@@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -14,6 +15,10 @@ namespace internal { +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +static Packet4ui p4ui_CONJ_XOR = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; //vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO); +#endif + static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_ZERO_, 8);//{ 0x8000000000000000, 0x0000000000000000 }; @@ -24,13 +29,52 @@ Packet2d v; }; +struct Packet2cf +{ + EIGEN_STRONG_INLINE Packet2cf() {} + EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {} +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) + union { + Packet4f v; + Packet1cd cd[2]; + }; +#else + Packet4f v; +#endif +}; + +template<> struct packet_traits<std::complex<float> > : default_packet_traits +{ + typedef Packet2cf type; + typedef Packet2cf half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + HasHalfPacket = 0, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasBlend = 1, + HasSetLinear = 0 + }; +}; + + template<> struct packet_traits<std::complex<double> > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; enum { Vectorizable = 1, - AlignedOnScalar = 0, + AlignedOnScalar = 1, size = 1, HasHalfPacket = 0, @@ -47,8 +91,13 @@ }; }; -template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16}; typedef Packet1cd half; }; +template<> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2cf half; }; +template<> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; enum {size=1, alignment=Aligned16, vectorizable=true}; typedef Packet1cd half; }; +/* Forward declaration */ +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf,2>& kernel); + +/* complex<double> first */ template<> EIGEN_STRONG_INLINE Packet1cd pload <Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from)); } template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } @@ -57,26 +106,18 @@ template<> EIGEN_STRONG_INLINE Packet1cd pset1<Packet1cd>(const std::complex<double>& from) { /* here we really have to use unaligned loads :( */ return ploadu<Packet1cd>(&from); } -template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index stride) +template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index stride EIGEN_UNUSED) { - std::complex<double> EIGEN_ALIGN16 af[2]; - af[0] = from[0*stride]; - af[1] = from[1*stride]; - return pload<Packet1cd>(af); + return pload<Packet1cd>(from); } -template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from, Index stride) +template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from, Index stride EIGEN_UNUSED) { - std::complex<double> EIGEN_ALIGN16 af[2]; - pstore<std::complex<double> >(af, from); - to[0*stride] = af[0]; - to[1*stride] = af[1]; + pstore<std::complex<double> >(to, from); } - template<> EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v + b.v); } template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd((Packet2d)vec_xor((Packet2d)a.v, (Packet2d)p2ul_CONJ_XOR2)); } - template<> EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { Packet2d a_re, a_im, v1, v2; @@ -94,44 +135,35 @@ return Packet1cd(v1 + v2); } - -template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_or(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_xor(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v, vec_nor(b.v,b.v))); } - -template<> EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) -{ - return pset1<Packet1cd>(*from); -} +template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_or(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_xor(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet1cd pandnot <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v, vec_nor(b.v,b.v))); } +template<> EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) { return pset1<Packet1cd>(*from); } template<> EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double> * addr) { EIGEN_ZVECTOR_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) { - std::complex<double> EIGEN_ALIGN16 res[2]; - pstore<std::complex<double> >(res, a); + std::complex<double> EIGEN_ALIGN16 res; + pstore<std::complex<double> >(&res, a); - return res[0]; + return res; } template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } - template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) { return pfirst(a); } - template<> EIGEN_STRONG_INLINE Packet1cd preduxp<Packet1cd>(const Packet1cd* vecs) { return vecs[0]; } - template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) { return pfirst(a); } - template<int Offset> struct palign_impl<Offset,Packet1cd> { @@ -175,6 +207,8 @@ } }; +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) + template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for AltiVec @@ -194,6 +228,334 @@ kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); kernel.packet[0].v = tmp; } + +/* complex<float> follows */ +template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>((const float*)from)); } +template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>((const float*)from)); } +template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v); } + +template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) +{ + std::complex<float> EIGEN_ALIGN16 res[2]; + pstore<std::complex<float> >(res, a); + + return res[0]; +} + + +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) +template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) +{ + Packet2cf res; + res.cd[0] = Packet1cd(vec_ld2f((const float *)&from)); + res.cd[1] = res.cd[0]; + return res; +} +#else +template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) +{ + Packet2cf res; + if((std::ptrdiff_t(&from) % 16) == 0) + res.v = pload<Packet4f>((const float *)&from); + else + res.v = ploadu<Packet4f>((const float *)&from); + res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI); + return res; +} +#endif + +template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from, Index stride) +{ + std::complex<float> EIGEN_ALIGN16 af[2]; + af[0] = from[0*stride]; + af[1] = from[1*stride]; + return pload<Packet2cf>(af); +} +template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from, Index stride) +{ + std::complex<float> EIGEN_ALIGN16 af[2]; + pstore<std::complex<float> >((std::complex<float> *) af, from); + to[0*stride] = af[0]; + to[1*stride] = af[1]; +} + +template<> EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(padd<Packet4f>(a.v, b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(psub<Packet4f>(a.v, b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(Packet4f(a.v))); } + +template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pand<Packet4f>(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(por<Packet4f>(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pxor<Packet4f>(a.v,b.v)); } +template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pandnot<Packet4f>(a.v,b.v)); } + +template<> EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) { return pset1<Packet2cf>(*from); } + +template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { EIGEN_ZVECTOR_PREFETCH(addr); } + + +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) +template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) +{ + Packet2cf res; + res.v.v4f[0] = pconj(Packet1cd(reinterpret_cast<Packet2d>(a.v.v4f[0]))).v; + res.v.v4f[1] = pconj(Packet1cd(reinterpret_cast<Packet2d>(a.v.v4f[1]))).v; + return res; +} + +template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) +{ + Packet2cf res; + res.v.v4f[0] = pmul(Packet1cd(reinterpret_cast<Packet2d>(a.v.v4f[0])), Packet1cd(reinterpret_cast<Packet2d>(b.v.v4f[0]))).v; + res.v.v4f[1] = pmul(Packet1cd(reinterpret_cast<Packet2d>(a.v.v4f[1])), Packet1cd(reinterpret_cast<Packet2d>(b.v.v4f[1]))).v; + return res; +} + +template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) +{ + Packet2cf res; + res.cd[0] = a.cd[1]; + res.cd[1] = a.cd[0]; + return res; +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) +{ + std::complex<float> res; + Packet1cd b = padd<Packet1cd>(a.cd[0], a.cd[1]); + vec_st2f(b.v, (float*)&res); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet2cf preduxp<Packet2cf>(const Packet2cf* vecs) +{ + PacketBlock<Packet2cf,2> transpose; + transpose.packet[0] = vecs[0]; + transpose.packet[1] = vecs[1]; + ptranspose(transpose); + + return padd<Packet2cf>(transpose.packet[0], transpose.packet[1]); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) +{ + std::complex<float> res; + Packet1cd b = pmul<Packet1cd>(a.cd[0], a.cd[1]); + vec_st2f(b.v, (float*)&res); + return res; +} + +template<int Offset> +struct palign_impl<Offset,Packet2cf> +{ + static EIGEN_STRONG_INLINE void run(Packet2cf& first, const Packet2cf& second) + { + if (Offset == 1) { + first.cd[0] = first.cd[1]; + first.cd[1] = second.cd[0]; + } + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, false,true> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return internal::pmul(a, pconj(b)); + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, true,false> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return internal::pmul(pconj(a), b); + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, true,true> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) + +template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) +{ + // TODO optimize it for AltiVec + Packet2cf res; + res.cd[0] = pdiv<Packet1cd>(a.cd[0], b.cd[0]); + res.cd[1] = pdiv<Packet1cd>(a.cd[1], b.cd[1]); + return res; +} + +EIGEN_STRONG_INLINE Packet2cf pcplxflip/*<Packet2cf>*/(const Packet2cf& x) +{ + Packet2cf res; + res.cd[0] = pcplxflip(x.cd[0]); + res.cd[1] = pcplxflip(x.cd[1]); + return res; +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf,2>& kernel) +{ + Packet1cd tmp = kernel.packet[0].cd[1]; + kernel.packet[0].cd[1] = kernel.packet[1].cd[0]; + kernel.packet[1].cd[0] = tmp; +} + +template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { + Packet2cf result; + const Selector<4> ifPacket4 = { ifPacket.select[0], ifPacket.select[0], ifPacket.select[1], ifPacket.select[1] }; + result.v = pblend<Packet4f>(ifPacket4, thenPacket.v, elsePacket.v); + return result; +} +#else +template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf(pxor<Packet4f>(a.v, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR))); } +template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) +{ + Packet4f a_re, a_im, prod, prod_im; + + // Permute and multiply the real parts of a and b + a_re = vec_perm(a.v, a.v, p16uc_PSET32_WODD); + + // Get the imaginary parts of a + a_im = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN); + + // multiply a_im * b and get the conjugate result + prod_im = a_im * b.v; + prod_im = pxor<Packet4f>(prod_im, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)); + // permute back to a proper order + prod_im = vec_perm(prod_im, prod_im, p16uc_COMPLEX32_REV); + + // multiply a_re * b, add prod_im + prod = pmadd<Packet4f>(a_re, b.v, prod_im); + + return Packet2cf(prod); +} + +template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) +{ + Packet4f rev_a; + rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2); + return Packet2cf(rev_a); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) +{ + Packet4f b; + b = vec_sld(a.v, a.v, 8); + b = padd<Packet4f>(a.v, b); + return pfirst<Packet2cf>(Packet2cf(b)); +} + +template<> EIGEN_STRONG_INLINE Packet2cf preduxp<Packet2cf>(const Packet2cf* vecs) +{ + Packet4f b1, b2; + b1 = vec_sld(vecs[0].v, vecs[1].v, 8); + b2 = vec_sld(vecs[1].v, vecs[0].v, 8); + b2 = vec_sld(b2, b2, 8); + b2 = padd<Packet4f>(b1, b2); + + return Packet2cf(b2); +} + +template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) +{ + Packet4f b; + Packet2cf prod; + b = vec_sld(a.v, a.v, 8); + prod = pmul<Packet2cf>(a, Packet2cf(b)); + + return pfirst<Packet2cf>(prod); +} + +template<int Offset> +struct palign_impl<Offset,Packet2cf> +{ + static EIGEN_STRONG_INLINE void run(Packet2cf& first, const Packet2cf& second) + { + if (Offset==1) + { + first.v = vec_sld(first.v, second.v, 8); + } + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, false,true> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return internal::pmul(a, pconj(b)); + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, true,false> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return internal::pmul(pconj(a), b); + } +}; + +template<> struct conj_helper<Packet2cf, Packet2cf, true,true> +{ + EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const + { return padd(pmul(x,y),c); } + + EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const + { + return pconj(internal::pmul(a, b)); + } +}; + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) + +template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) +{ + // TODO optimize it for AltiVec + Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a, b); + Packet4f s = pmul<Packet4f>(b.v, b.v); + return Packet2cf(pdiv(res.v, padd<Packet4f>(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); +} + +template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& x) +{ + return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV)); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf,2>& kernel) +{ + Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); + kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); + kernel.packet[0].v = tmp; +} + +template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { + Packet2cf result; + result.v = reinterpret_cast<Packet4f>(pblend<Packet2d>(ifPacket, reinterpret_cast<Packet2d>(thenPacket.v), reinterpret_cast<Packet2d>(elsePacket.v))); + return result; +} +#endif + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/ZVector/MathFunctions.h b/Eigen/src/Core/arch/ZVector/MathFunctions.h index 6fff852..ff33a97 100644 --- a/Eigen/src/Core/arch/ZVector/MathFunctions.h +++ b/Eigen/src/Core/arch/ZVector/MathFunctions.h
@@ -3,6 +3,7 @@ // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -19,32 +20,76 @@ namespace internal { +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +static _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); +static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); +static _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); +static _EIGEN_DECLARE_CONST_Packet4i(23, 23); + +static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); + +/* the smallest non denormalized float number */ +static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); +static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000); // -1.f/0.f +static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_nan, 0xffffffff); + +/* natural logarithm computed for 4 simultaneous float + return NaN for x <= 0 +*/ +static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); + +static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); +static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); + +static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); + +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); +static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); +#endif + +static _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); +static _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); +static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); + +static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); +static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); + +static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); + +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); + +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); + +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); +static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); + template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp<Packet2d>(const Packet2d& _x) { Packet2d x = _x; - _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); - _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); - _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); - - _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); - _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); - - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); - _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); - Packet2d tmp, fx; Packet2l emm0; @@ -92,17 +137,94 @@ } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f pexp<Packet4f>(const Packet4f& _x) +{ +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +/* + Packet4f x = _x; + + Packet4f tmp, fx; + Packet4i emm0; + + // clamp x + x = pmax(pmin(x, p4f_exp_hi), p4f_exp_lo); + + // express exp(x) as exp(g + n*log(2)) + fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half); + + fx = pfloor(fx); + + tmp = pmul(fx, p4f_cephes_exp_C1); + Packet4f z = pmul(fx, p4f_cephes_exp_C2); + x = psub(x, tmp); + x = psub(x, z); + + z = pmul(x,x); + + Packet4f y = p4f_cephes_exp_p0; + y = pmadd(y, x, p4f_cephes_exp_p1); + y = pmadd(y, x, p4f_cephes_exp_p2); + y = pmadd(y, x, p4f_cephes_exp_p3); + y = pmadd(y, x, p4f_cephes_exp_p4); + y = pmadd(y, x, p4f_cephes_exp_p5); + y = pmadd(y, z, x); + y = padd(y, p4f_1); + + // build 2^n + emm0 = vec_cts(fx, 0); + emm0 = emm0 + p4i_0x7f; + emm0 = emm0 << reinterpret_cast<Packet4i>(p4i_23); + + // Altivec's max & min operators just drop silent NaNs. Check NaNs in + // inputs and return them unmodified. + Packet4ui isnumber_mask = reinterpret_cast<Packet4ui>(vec_cmpeq(_x, _x)); + return vec_sel(_x, pmax(pmul(y, reinterpret_cast<Packet4f>(emm0)), _x), + isnumber_mask);*/ + return _x; +#else + Packet4f res; + res.v4f[0] = pexp<Packet2d>(_x.v4f[0]); + res.v4f[1] = pexp<Packet2d>(_x.v4f[1]); + return res; +#endif +} + +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d psqrt<Packet2d>(const Packet2d& x) { - return __builtin_s390_vfsqdb(x); + return vec_sqrt(x); +} + +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f psqrt<Packet4f>(const Packet4f& x) +{ + Packet4f res; +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) + res = vec_sqrt(x); +#else + res.v4f[0] = psqrt<Packet2d>(x.v4f[0]); + res.v4f[1] = psqrt<Packet2d>(x.v4f[1]); +#endif + return res; } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d prsqrt<Packet2d>(const Packet2d& x) { - // Unfortunately we can't use the much faster mm_rqsrt_pd since it only provides an approximation. return pset1<Packet2d>(1.0) / psqrt<Packet2d>(x); } +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet4f prsqrt<Packet4f>(const Packet4f& x) { + Packet4f res; +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) + res = pset1<Packet4f>(1.0) / psqrt<Packet4f>(x); +#else + res.v4f[0] = prsqrt<Packet2d>(x.v4f[0]); + res.v4f[1] = prsqrt<Packet2d>(x.v4f[1]); +#endif + return res; +} + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/ZVector/PacketMath.h b/Eigen/src/Core/arch/ZVector/PacketMath.h index 5a7226b..c8e90f1 100755 --- a/Eigen/src/Core/arch/ZVector/PacketMath.h +++ b/Eigen/src/Core/arch/ZVector/PacketMath.h
@@ -17,7 +17,7 @@ namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD -#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4 +#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD @@ -28,7 +28,6 @@ #define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD #endif -// NOTE Altivec has 32 registers, but Eigen only accepts a value of 8 or 16 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #endif @@ -42,17 +41,30 @@ typedef __vector unsigned long long Packet2ul; typedef __vector long long Packet2l; +// Z14 has builtin support for float vectors +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +typedef __vector float Packet4f; +#else +typedef struct { + Packet2d v4f[2]; +} Packet4f; +#endif + typedef union { int32_t i[4]; uint32_t ui[4]; int64_t l[2]; uint64_t ul[2]; double d[2]; + float f[4]; Packet4i v4i; Packet4ui v4ui; Packet2l v2l; Packet2ul v2ul; Packet2d v2d; +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) + Packet4f v4f; +#endif } Packet; // We don't want to write the same code all the time, but we need to reuse the constants @@ -77,7 +89,7 @@ Packet2l p2l_##NAME = pset1<Packet2l>(X) // These constants are endian-agnostic -//static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} +static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1} static _EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0); @@ -87,7 +99,23 @@ static Packet2d p2d_ONE = { 1.0, 1.0 }; static Packet2d p2d_ZERO_ = { -0.0, -0.0 }; +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +#define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ + Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X)) + +#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ + Packet4f p4f_##NAME = pset1<Packet4f>(X) + +#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ + const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X)) + +static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} +static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} +static Packet4f p4f_MZERO = { 0x80000000, 0x80000000, 0x80000000, 0x80000000}; +#endif + static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 }; +static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 }; static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8)); static Packet16uc p16uc_PSET64_HI = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 }; @@ -96,7 +124,7 @@ // Mask alignment #define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0 -#define _EIGEN_ALIGNED_PTR(x) ((ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) +#define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) // Handle endianness properly while loading constants // Define global static constants: @@ -116,9 +144,9 @@ static Packet16uc p16uc_TRANSPOSE64_HI = { 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; static Packet16uc p16uc_TRANSPOSE64_LO = { 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31}; -//static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; +static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; -//static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; +static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC @@ -132,13 +160,11 @@ typedef Packet4i type; typedef Packet4i half; enum { - // FIXME check the Has* Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, - // FIXME check the Has* HasAdd = 1, HasSub = 1, HasMul = 1, @@ -147,6 +173,41 @@ }; }; +template<> struct packet_traits<float> : default_packet_traits +{ + typedef Packet4f type; + typedef Packet4f half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size=4, + HasHalfPacket = 0, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasMin = 1, + HasMax = 1, + HasAbs = 1, + HasSin = 0, + HasCos = 0, + HasLog = 0, +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) + HasExp = 0, +#else + HasExp = 1, +#endif + HasSqrt = 1, + HasRsqrt = 1, + HasRound = 1, + HasFloor = 1, + HasCeil = 1, + HasNegate = 1, + HasBlend = 1 + }; +}; + template<> struct packet_traits<double> : default_packet_traits { typedef Packet2d type; @@ -157,7 +218,6 @@ size=2, HasHalfPacket = 1, - // FIXME check the Has* HasAdd = 1, HasSub = 1, HasMul = 1, @@ -179,9 +239,13 @@ }; }; -template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; -template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; +template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16, vectorizable=true}; typedef Packet4i half; }; +template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16, vectorizable=true}; typedef Packet4f half; }; +template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true}; typedef Packet2d half; }; +/* Forward declaration */ +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f,4>& kernel); + inline std::ostream & operator <<(std::ostream & s, const Packet4i & v) { Packet vt; @@ -222,6 +286,17 @@ return s; } +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) +inline std::ostream & operator <<(std::ostream & s, const Packet4f & v) +{ + Packet vt; + vt.v4f = v; + s << vt.f[0] << ", " << vt.f[1] << ", " << vt.f[2] << ", " << vt.f[3]; + return s; +} +#endif + + template<int Offset> struct palign_impl<Offset,Packet4i> { @@ -288,7 +363,6 @@ { return vec_splats(from); } - template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return vec_splats(from); } @@ -433,8 +507,8 @@ return reinterpret_cast<Packet2d>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64)); } -template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); } -template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } +template<> EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(const Packet4i& a) { return vec_abs(a); } +template<> EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(const Packet2d& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) { @@ -562,12 +636,554 @@ return vec_sel(elsePacket, thenPacket, mask); } + template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { Packet2ul select = { ifPacket.select[0], ifPacket.select[1] }; Packet2ul mask = vec_cmpeq(select, reinterpret_cast<Packet2ul>(p2l_ONE)); return vec_sel(elsePacket, thenPacket, mask); } +/* z13 has no vector float support so we emulate that with double + z14 has proper vector float support. +*/ +#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) +/* Helper function to simulate a vec_splat_packet4f + */ +template<int element> EIGEN_STRONG_INLINE Packet4f vec_splat_packet4f(const Packet4f& from) +{ + Packet4f splat; + switch (element) { + case 0: + splat.v4f[0] = vec_splat(from.v4f[0], 0); + splat.v4f[1] = splat.v4f[0]; + break; + case 1: + splat.v4f[0] = vec_splat(from.v4f[0], 1); + splat.v4f[1] = splat.v4f[0]; + break; + case 2: + splat.v4f[0] = vec_splat(from.v4f[1], 0); + splat.v4f[1] = splat.v4f[0]; + break; + case 3: + splat.v4f[0] = vec_splat(from.v4f[1], 1); + splat.v4f[1] = splat.v4f[0]; + break; + } + return splat; +} + +/* This is a tricky one, we have to translate float alignment to vector elements of sizeof double + */ +template<int Offset> +struct palign_impl<Offset,Packet4f> +{ + static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second) + { + switch (Offset % 4) { + case 1: + first.v4f[0] = vec_sld(first.v4f[0], first.v4f[1], 8); + first.v4f[1] = vec_sld(first.v4f[1], second.v4f[0], 8); + break; + case 2: + first.v4f[0] = first.v4f[1]; + first.v4f[1] = second.v4f[0]; + break; + case 3: + first.v4f[0] = vec_sld(first.v4f[1], second.v4f[0], 8); + first.v4f[1] = vec_sld(second.v4f[0], second.v4f[1], 8); + break; + } + } +}; + +template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) +{ + // FIXME: No intrinsic yet + EIGEN_DEBUG_ALIGNED_LOAD + Packet4f vfrom; + vfrom.v4f[0] = vec_ld2f(&from[0]); + vfrom.v4f[1] = vec_ld2f(&from[2]); + return vfrom; +} + +template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) +{ + // FIXME: No intrinsic yet + EIGEN_DEBUG_ALIGNED_STORE + vec_st2f(from.v4f[0], &to[0]); + vec_st2f(from.v4f[1], &to[2]); +} + +template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) +{ + Packet4f to; + to.v4f[0] = pset1<Packet2d>(static_cast<const double&>(from)); + to.v4f[1] = to.v4f[0]; + return to; +} + +template<> EIGEN_STRONG_INLINE void +pbroadcast4<Packet4f>(const float *a, + Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) +{ + a3 = pload<Packet4f>(a); + a0 = vec_splat_packet4f<0>(a3); + a1 = vec_splat_packet4f<1>(a3); + a2 = vec_splat_packet4f<2>(a3); + a3 = vec_splat_packet4f<3>(a3); +} + +template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) +{ + float EIGEN_ALIGN16 ai[4]; + ai[0] = from[0*stride]; + ai[1] = from[1*stride]; + ai[2] = from[2*stride]; + ai[3] = from[3*stride]; + return pload<Packet4f>(ai); +} + +template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) +{ + float EIGEN_ALIGN16 ai[4]; + pstore<float>((float *)ai, from); + to[0*stride] = ai[0]; + to[1*stride] = ai[1]; + to[2*stride] = ai[2]; + to[3*stride] = ai[3]; +} + +template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f c; + c.v4f[0] = a.v4f[0] + b.v4f[0]; + c.v4f[1] = a.v4f[1] + b.v4f[1]; + return c; +} + +template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f c; + c.v4f[0] = a.v4f[0] - b.v4f[0]; + c.v4f[1] = a.v4f[1] - b.v4f[1]; + return c; +} + +template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f c; + c.v4f[0] = a.v4f[0] * b.v4f[0]; + c.v4f[1] = a.v4f[1] * b.v4f[1]; + return c; +} + +template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f c; + c.v4f[0] = a.v4f[0] / b.v4f[0]; + c.v4f[1] = a.v4f[1] / b.v4f[1]; + return c; +} + +template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) +{ + Packet4f c; + c.v4f[0] = -a.v4f[0]; + c.v4f[1] = -a.v4f[1]; + return c; +} + +template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) +{ + Packet4f res; + res.v4f[0] = vec_madd(a.v4f[0], b.v4f[0], c.v4f[0]); + res.v4f[1] = vec_madd(a.v4f[1], b.v4f[1], c.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pmin(a.v4f[0], b.v4f[0]); + res.v4f[1] = pmin(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pmax(a.v4f[0], b.v4f[0]); + res.v4f[1] = pmax(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pand(a.v4f[0], b.v4f[0]); + res.v4f[1] = pand(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pand(a.v4f[0], b.v4f[0]); + res.v4f[1] = pand(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pand(a.v4f[0], b.v4f[0]); + res.v4f[1] = pand(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) +{ + Packet4f res; + res.v4f[0] = pandnot(a.v4f[0], b.v4f[0]); + res.v4f[1] = pandnot(a.v4f[1], b.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) +{ + Packet4f res; + res.v4f[0] = vec_round(a.v4f[0]); + res.v4f[1] = vec_round(a.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) +{ + Packet4f res; + res.v4f[0] = vec_ceil(a.v4f[0]); + res.v4f[1] = vec_ceil(a.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) +{ + Packet4f res; + res.v4f[0] = vec_floor(a.v4f[0]); + res.v4f[1] = vec_floor(a.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) +{ + Packet4f p = pload<Packet4f>(from); + p.v4f[1] = vec_splat(p.v4f[0], 1); + p.v4f[0] = vec_splat(p.v4f[0], 0); + return p; +} + +template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[2]; vec_st2f(a.v4f[0], &x[0]); return x[0]; } + +template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) +{ + Packet4f rev; + rev.v4f[0] = preverse<Packet2d>(a.v4f[1]); + rev.v4f[1] = preverse<Packet2d>(a.v4f[0]); + return rev; +} + +template<> EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a) +{ + Packet4f res; + res.v4f[0] = pabs(a.v4f[0]); + res.v4f[1] = pabs(a.v4f[1]); + return res; +} + +template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) +{ + Packet2d sum; + sum = padd<Packet2d>(a.v4f[0], a.v4f[1]); + double first = predux<Packet2d>(sum); + return static_cast<float>(first); +} + +template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) +{ + PacketBlock<Packet4f,4> transpose; + transpose.packet[0] = vecs[0]; + transpose.packet[1] = vecs[1]; + transpose.packet[2] = vecs[2]; + transpose.packet[3] = vecs[3]; + ptranspose(transpose); + + Packet4f sum = padd(transpose.packet[0], transpose.packet[1]); + sum = padd(sum, transpose.packet[2]); + sum = padd(sum, transpose.packet[3]); + return sum; +} + +template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) +{ + // Return predux_mul<Packet2d> of the subvectors product + return static_cast<float>(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1])))); +} + +template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) +{ + Packet2d b, res; + b = pmin<Packet2d>(a.v4f[0], a.v4f[1]); + res = pmin<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8))); + return static_cast<float>(pfirst(res)); +} + +template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) +{ + Packet2d b, res; + b = pmax<Packet2d>(a.v4f[0], a.v4f[1]); + res = pmax<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8))); + return static_cast<float>(pfirst(res)); +} + +/* Split the Packet4f PacketBlock into 4 Packet2d PacketBlocks and transpose each one + */ +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock<Packet4f,4>& kernel) { + PacketBlock<Packet2d,2> t0,t1,t2,t3; + // copy top-left 2x2 Packet2d block + t0.packet[0] = kernel.packet[0].v4f[0]; + t0.packet[1] = kernel.packet[1].v4f[0]; + + // copy top-right 2x2 Packet2d block + t1.packet[0] = kernel.packet[0].v4f[1]; + t1.packet[1] = kernel.packet[1].v4f[1]; + + // copy bottom-left 2x2 Packet2d block + t2.packet[0] = kernel.packet[2].v4f[0]; + t2.packet[1] = kernel.packet[3].v4f[0]; + + // copy bottom-right 2x2 Packet2d block + t3.packet[0] = kernel.packet[2].v4f[1]; + t3.packet[1] = kernel.packet[3].v4f[1]; + + // Transpose all 2x2 blocks + ptranspose(t0); + ptranspose(t1); + ptranspose(t2); + ptranspose(t3); + + // Copy back transposed blocks, but exchange t1 and t2 due to transposition + kernel.packet[0].v4f[0] = t0.packet[0]; + kernel.packet[0].v4f[1] = t2.packet[0]; + kernel.packet[1].v4f[0] = t0.packet[1]; + kernel.packet[1].v4f[1] = t2.packet[1]; + kernel.packet[2].v4f[0] = t1.packet[0]; + kernel.packet[2].v4f[1] = t3.packet[0]; + kernel.packet[3].v4f[0] = t1.packet[1]; + kernel.packet[3].v4f[1] = t3.packet[1]; +} + +template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { + Packet2ul select_hi = { ifPacket.select[0], ifPacket.select[1] }; + Packet2ul select_lo = { ifPacket.select[2], ifPacket.select[3] }; + Packet2ul mask_hi = vec_cmpeq(select_hi, reinterpret_cast<Packet2ul>(p2l_ONE)); + Packet2ul mask_lo = vec_cmpeq(select_lo, reinterpret_cast<Packet2ul>(p2l_ONE)); + Packet4f result; + result.v4f[0] = vec_sel(elsePacket.v4f[0], thenPacket.v4f[0], mask_hi); + result.v4f[1] = vec_sel(elsePacket.v4f[1], thenPacket.v4f[1], mask_lo); + return result; +} +#else +template<int Offset> +struct palign_impl<Offset,Packet4f> +{ + static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second) + { + switch (Offset % 4) { + case 1: + first = vec_sld(first, second, 4); break; + case 2: + first = vec_sld(first, second, 8); break; + case 3: + first = vec_sld(first, second, 12); break; + } + } +}; + +template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) +{ + // FIXME: No intrinsic yet + EIGEN_DEBUG_ALIGNED_LOAD + Packet *vfrom; + vfrom = (Packet *) from; + return vfrom->v4f; +} + +template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) +{ + // FIXME: No intrinsic yet + EIGEN_DEBUG_ALIGNED_STORE + Packet *vto; + vto = (Packet *) to; + vto->v4f = from; +} + +template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) +{ + return vec_splats(from); +} + +template<> EIGEN_STRONG_INLINE void +pbroadcast4<Packet4f>(const float *a, + Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) +{ + a3 = pload<Packet4f>(a); + a0 = vec_splat(a3, 0); + a1 = vec_splat(a3, 1); + a2 = vec_splat(a3, 2); + a3 = vec_splat(a3, 3); +} + +template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) +{ + float EIGEN_ALIGN16 af[4]; + af[0] = from[0*stride]; + af[1] = from[1*stride]; + af[2] = from[2*stride]; + af[3] = from[3*stride]; + return pload<Packet4f>(af); +} + +template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) +{ + float EIGEN_ALIGN16 af[4]; + pstore<float>((float*)af, from); + to[0*stride] = af[0]; + to[1*stride] = af[1]; + to[2*stride] = af[2]; + to[3*stride] = af[3]; +} + +template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a + b); } +template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a - b); } +template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a * b); } +template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a / b); } +template<> EIGEN_STRONG_INLINE Packet4f pnegate<Packet4f>(const Packet4f& a) { return (-a); } +template<> EIGEN_STRONG_INLINE Packet4f pconj<Packet4f> (const Packet4f& a) { return a; } +template<> EIGEN_STRONG_INLINE Packet4f pmadd<Packet4f> (const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a, b, c); } +template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f> (const Packet4f& a, const Packet4f& b) { return vec_min(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f> (const Packet4f& a, const Packet4f& b) { return vec_max(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f> (const Packet4f& a, const Packet4f& b) { return vec_and(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f> (const Packet4f& a, const Packet4f& b) { return vec_or(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f> (const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); } +template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, vec_nor(b, b)); } +template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f> (const Packet4f& a) { return vec_round(a); } +template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f> (const Packet4f& a) { return vec_ceil(a); } +template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f> (const Packet4f& a) { return vec_floor(a); } +template<> EIGEN_STRONG_INLINE Packet4f pabs<Packet4f> (const Packet4f& a) { return vec_abs(a); } +template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; } + +template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) +{ + Packet4f p = pload<Packet4f>(from); + return vec_perm(p, p, p16uc_DUPLICATE32_HI); +} + +template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) +{ + return reinterpret_cast<Packet4f>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32)); +} + +template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) +{ + Packet4f b, sum; + b = vec_sld(a, a, 8); + sum = padd<Packet4f>(a, b); + b = vec_sld(sum, sum, 4); + sum = padd<Packet4f>(sum, b); + return pfirst(sum); +} + +template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) +{ + Packet4f v[4], sum[4]; + + // It's easier and faster to transpose then add as columns + // Check: http://www.freevec.org/function/matrix_4x4_transpose_floats for explanation + // Do the transpose, first set of moves + v[0] = vec_mergeh(vecs[0], vecs[2]); + v[1] = vec_mergel(vecs[0], vecs[2]); + v[2] = vec_mergeh(vecs[1], vecs[3]); + v[3] = vec_mergel(vecs[1], vecs[3]); + // Get the resulting vectors + sum[0] = vec_mergeh(v[0], v[2]); + sum[1] = vec_mergel(v[0], v[2]); + sum[2] = vec_mergeh(v[1], v[3]); + sum[3] = vec_mergel(v[1], v[3]); + + // Now do the summation: + // Lines 0+1 + sum[0] = padd<Packet4f>(sum[0], sum[1]); + // Lines 2+3 + sum[1] = padd<Packet4f>(sum[2], sum[3]); + // Add the results + sum[0] = padd<Packet4f>(sum[0], sum[1]); + + return sum[0]; +} + +// Other reduction functions: +// mul +template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) +{ + Packet4f prod; + prod = pmul(a, vec_sld(a, a, 8)); + return pfirst(pmul(prod, vec_sld(prod, prod, 4))); +} + +// min +template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) +{ + Packet4f b, res; + b = pmin<Packet4f>(a, vec_sld(a, a, 8)); + res = pmin<Packet4f>(b, vec_sld(b, b, 4)); + return pfirst(res); +} + +// max +template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) +{ + Packet4f b, res; + b = pmax<Packet4f>(a, vec_sld(a, a, 8)); + res = pmax<Packet4f>(b, vec_sld(b, b, 4)); + return pfirst(res); +} + +EIGEN_DEVICE_FUNC inline void +ptranspose(PacketBlock<Packet4f,4>& kernel) { + Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); + Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); + Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); + Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); + kernel.packet[0] = vec_mergeh(t0, t2); + kernel.packet[1] = vec_mergel(t0, t2); + kernel.packet[2] = vec_mergeh(t1, t3); + kernel.packet[3] = vec_mergel(t1, t3); +} + +template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { + Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; + Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE)); + return vec_sel(elsePacket, thenPacket, mask); +} + +#endif + +template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ZVECTOR_PREFETCH(addr); } +template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f> (const float* from) { return pload<Packet4f>(from); } +template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { pstore<float>(to, from); } +template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f> (const float& a) { return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN); } + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/functors/AssignmentFunctors.h b/Eigen/src/Core/functors/AssignmentFunctors.h index d55ae60..9765cc7 100644 --- a/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Eigen/src/Core/functors/AssignmentFunctors.h
@@ -18,20 +18,24 @@ * \brief Template functor for scalar/packet assignment * */ -template<typename Scalar> struct assign_op { +template<typename DstScalar,typename SrcScalar> struct assign_op { EIGEN_EMPTY_STRUCT_CTOR(assign_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { a = b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } template<int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(Scalar* a, const Packet& b) const - { internal::pstoret<Scalar,Packet,Alignment>(a,b); } + EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const + { internal::pstoret<DstScalar,Packet,Alignment>(a,b); } }; -template<typename Scalar> -struct functor_traits<assign_op<Scalar> > { + +// Empty overload for void type (used by PermutationMatrix) +template<typename DstScalar> struct assign_op<DstScalar,void> {}; + +template<typename DstScalar,typename SrcScalar> +struct functor_traits<assign_op<DstScalar,SrcScalar> > { enum { - Cost = NumTraits<Scalar>::ReadCost, - PacketAccess = packet_traits<Scalar>::Vectorizable + Cost = NumTraits<DstScalar>::ReadCost, + PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::Vectorizable && packet_traits<SrcScalar>::Vectorizable }; }; @@ -39,20 +43,20 @@ * \brief Template functor for scalar/packet assignment with addition * */ -template<typename Scalar> struct add_assign_op { +template<typename DstScalar,typename SrcScalar> struct add_assign_op { EIGEN_EMPTY_STRUCT_CTOR(add_assign_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { a += b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a += b; } template<int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(Scalar* a, const Packet& b) const - { internal::pstoret<Scalar,Packet,Alignment>(a,internal::padd(internal::ploadt<Packet,Alignment>(a),b)); } + EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const + { internal::pstoret<DstScalar,Packet,Alignment>(a,internal::padd(internal::ploadt<Packet,Alignment>(a),b)); } }; -template<typename Scalar> -struct functor_traits<add_assign_op<Scalar> > { +template<typename DstScalar,typename SrcScalar> +struct functor_traits<add_assign_op<DstScalar,SrcScalar> > { enum { - Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasAdd + Cost = NumTraits<DstScalar>::ReadCost + NumTraits<DstScalar>::AddCost, + PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::HasAdd }; }; @@ -60,20 +64,20 @@ * \brief Template functor for scalar/packet assignment with subtraction * */ -template<typename Scalar> struct sub_assign_op { +template<typename DstScalar,typename SrcScalar> struct sub_assign_op { EIGEN_EMPTY_STRUCT_CTOR(sub_assign_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { a -= b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a -= b; } template<int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(Scalar* a, const Packet& b) const - { internal::pstoret<Scalar,Packet,Alignment>(a,internal::psub(internal::ploadt<Packet,Alignment>(a),b)); } + EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const + { internal::pstoret<DstScalar,Packet,Alignment>(a,internal::psub(internal::ploadt<Packet,Alignment>(a),b)); } }; -template<typename Scalar> -struct functor_traits<sub_assign_op<Scalar> > { +template<typename DstScalar,typename SrcScalar> +struct functor_traits<sub_assign_op<DstScalar,SrcScalar> > { enum { - Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasSub + Cost = NumTraits<DstScalar>::ReadCost + NumTraits<DstScalar>::AddCost, + PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::HasSub }; }; @@ -98,30 +102,28 @@ PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::HasMul }; }; -template<typename DstScalar,typename SrcScalar> struct functor_is_product_like<mul_assign_op<DstScalar,SrcScalar> > { enum { ret = 1 }; }; /** \internal * \brief Template functor for scalar/packet assignment with diviving * */ -template<typename Scalar> struct div_assign_op { +template<typename DstScalar, typename SrcScalar=DstScalar> struct div_assign_op { EIGEN_EMPTY_STRUCT_CTOR(div_assign_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { a /= b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a /= b; } template<int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(Scalar* a, const Packet& b) const - { internal::pstoret<Scalar,Packet,Alignment>(a,internal::pdiv(internal::ploadt<Packet,Alignment>(a),b)); } + EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const + { internal::pstoret<DstScalar,Packet,Alignment>(a,internal::pdiv(internal::ploadt<Packet,Alignment>(a),b)); } }; -template<typename Scalar> -struct functor_traits<div_assign_op<Scalar> > { +template<typename DstScalar, typename SrcScalar> +struct functor_traits<div_assign_op<DstScalar,SrcScalar> > { enum { - Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasDiv + Cost = NumTraits<DstScalar>::ReadCost + NumTraits<DstScalar>::MulCost, + PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::HasDiv }; }; - /** \internal * \brief Template functor for scalar/packet assignment with swapping * @@ -142,7 +144,7 @@ EIGEN_EMPTY_STRUCT_CTOR(swap_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { -#ifdef __CUDACC__ +#ifdef EIGEN_GPUCC // FIXME is there some kind of cuda::swap? Scalar t=b; const_cast<Scalar&>(b)=a; a=t; #else
diff --git a/Eigen/src/Core/functors/BinaryFunctors.h b/Eigen/src/Core/functors/BinaryFunctors.h index 5cd8ca9..401d597 100644 --- a/Eigen/src/Core/functors/BinaryFunctors.h +++ b/Eigen/src/Core/functors/BinaryFunctors.h
@@ -16,27 +16,43 @@ //---------- associative binary functors ---------- +template<typename Arg1, typename Arg2> +struct binary_op_base +{ + typedef Arg1 first_argument_type; + typedef Arg2 second_argument_type; +}; + /** \internal * \brief Template functor to compute the sum of two scalars * * \sa class CwiseBinaryOp, MatrixBase::operator+, class VectorwiseOp, DenseBase::sum() */ -template<typename Scalar> struct scalar_sum_op { -// typedef Scalar result_type; +template<typename LhsScalar,typename RhsScalar> +struct scalar_sum_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_sum_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_sum_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a + b; } +#else + scalar_sum_op() { + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a + b; } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::padd(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar predux(const Packet& a) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const { return internal::predux(a); } }; -template<typename Scalar> -struct functor_traits<scalar_sum_op<Scalar> > { +template<typename LhsScalar,typename RhsScalar> +struct functor_traits<scalar_sum_op<LhsScalar,RhsScalar> > { enum { - Cost = NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasAdd + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, // rough estimate! + PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasAdd && packet_traits<RhsScalar>::HasAdd + // TODO vectorize mixed sum }; }; @@ -45,7 +61,7 @@ * This is required to solve Bug 426. * \sa DenseBase::count(), DenseBase::any(), ArrayBase::cast(), MatrixBase::cast() */ -template<> struct scalar_sum_op<bool> : scalar_sum_op<int> { +template<> struct scalar_sum_op<bool,bool> : scalar_sum_op<int,int> { EIGEN_DEPRECATED scalar_sum_op() {} }; @@ -56,13 +72,17 @@ * * \sa class CwiseBinaryOp, Cwise::operator*(), class VectorwiseOp, MatrixBase::redux() */ -template<typename LhsScalar,typename RhsScalar> struct scalar_product_op { - enum { - // TODO vectorize mixed product - Vectorizable = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasMul && packet_traits<RhsScalar>::HasMul - }; - typedef typename scalar_product_traits<LhsScalar,RhsScalar>::ReturnType result_type; +template<typename LhsScalar,typename RhsScalar> +struct scalar_product_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_product_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_product_op) +#else + scalar_product_op() { + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a * b; } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const @@ -75,7 +95,8 @@ struct functor_traits<scalar_product_op<LhsScalar,RhsScalar> > { enum { Cost = (NumTraits<LhsScalar>::MulCost + NumTraits<RhsScalar>::MulCost)/2, // rough estimate! - PacketAccess = scalar_product_op<LhsScalar,RhsScalar>::Vectorizable + PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasMul && packet_traits<RhsScalar>::HasMul + // TODO vectorize mixed product }; }; @@ -84,13 +105,15 @@ * * This is a short cut for conj(x) * y which is needed for optimization purpose; in Eigen2 support mode, this becomes x * conj(y) */ -template<typename LhsScalar,typename RhsScalar> struct scalar_conj_product_op { +template<typename LhsScalar,typename RhsScalar> +struct scalar_conj_product_op : binary_op_base<LhsScalar,RhsScalar> +{ enum { Conj = NumTraits<LhsScalar>::IsComplex }; - typedef typename scalar_product_traits<LhsScalar,RhsScalar>::ReturnType result_type; + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_conj_product_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_conj_product_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const @@ -113,21 +136,24 @@ * * \sa class CwiseBinaryOp, MatrixBase::cwiseMin, class VectorwiseOp, MatrixBase::minCoeff() */ -template<typename Scalar> struct scalar_min_op { +template<typename LhsScalar,typename RhsScalar> +struct scalar_min_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_min_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_min_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return numext::mini(a, b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return numext::mini(a, b); } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmin(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar predux(const Packet& a) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const { return internal::predux_min(a); } }; -template<typename Scalar> -struct functor_traits<scalar_min_op<Scalar> > { +template<typename LhsScalar,typename RhsScalar> +struct functor_traits<scalar_min_op<LhsScalar,RhsScalar> > { enum { - Cost = NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasMin + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, + PacketAccess = internal::is_same<LhsScalar, RhsScalar>::value && packet_traits<LhsScalar>::HasMin }; }; @@ -136,21 +162,24 @@ * * \sa class CwiseBinaryOp, MatrixBase::cwiseMax, class VectorwiseOp, MatrixBase::maxCoeff() */ -template<typename Scalar> struct scalar_max_op { +template<typename LhsScalar,typename RhsScalar> +struct scalar_max_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_max_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_max_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return numext::maxi(a, b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return numext::maxi(a, b); } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmax(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar predux(const Packet& a) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const { return internal::predux_max(a); } }; -template<typename Scalar> -struct functor_traits<scalar_max_op<Scalar> > { +template<typename LhsScalar,typename RhsScalar> +struct functor_traits<scalar_max_op<LhsScalar,RhsScalar> > { enum { - Cost = NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasMax + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, + PacketAccess = internal::is_same<LhsScalar, RhsScalar>::value && packet_traits<LhsScalar>::HasMax }; }; @@ -158,91 +187,100 @@ * \brief Template functors for comparison of two scalars * \todo Implement packet-comparisons */ -template<typename Scalar, ComparisonName cmp> struct scalar_cmp_op; +template<typename LhsScalar, typename RhsScalar, ComparisonName cmp> struct scalar_cmp_op; -template<typename Scalar, ComparisonName cmp> -struct functor_traits<scalar_cmp_op<Scalar, cmp> > { +template<typename LhsScalar, typename RhsScalar, ComparisonName cmp> +struct functor_traits<scalar_cmp_op<LhsScalar,RhsScalar, cmp> > { enum { - Cost = NumTraits<Scalar>::AddCost, + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, PacketAccess = false }; }; -template<ComparisonName Cmp, typename Scalar> -struct result_of<scalar_cmp_op<Scalar, Cmp>(Scalar,Scalar)> { +template<ComparisonName Cmp, typename LhsScalar, typename RhsScalar> +struct result_of<scalar_cmp_op<LhsScalar, RhsScalar, Cmp>(LhsScalar,RhsScalar)> { typedef bool type; }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_EQ> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_EQ> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a==b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a==b;} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_LT> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_LT> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a<b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a<b;} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_LE> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_LE> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a<=b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a<=b;} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_GT> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_GT> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a>b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>b;} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_GE> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_GE> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a>=b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>=b;} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_UNORD> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_UNORD> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return !(a<=b || b<=a);} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return !(a<=b || b<=a);} }; -template<typename Scalar> struct scalar_cmp_op<Scalar, cmp_NEQ> { +template<typename LhsScalar, typename RhsScalar> +struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_NEQ> : binary_op_base<LhsScalar,RhsScalar> +{ typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& a, const Scalar& b) const {return a!=b;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a!=b;} }; /** \internal - * \brief Template functor to compute the hypot of two scalars + * \brief Template functor to compute the hypot of two \b positive \b and \b real scalars * * \sa MatrixBase::stableNorm(), class Redux */ -template<typename Scalar> struct scalar_hypot_op { +template<typename Scalar> +struct scalar_hypot_op<Scalar,Scalar> : binary_op_base<Scalar,Scalar> +{ EIGEN_EMPTY_STRUCT_CTOR(scalar_hypot_op) -// typedef typename NumTraits<Scalar>::Real result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& _x, const Scalar& _y) const + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar &x, const Scalar &y) const { - using std::sqrt; - Scalar p, qp; - if(_x>_y) - { - p = _x; - qp = _y / p; - } - else - { - p = _y; - qp = _x / p; - } - return p * sqrt(Scalar(1) + qp*qp); + // This functor is used by hypotNorm only for which it is faster to first apply abs + // on all coefficients prior to reduction through hypot. + // This way we avoid calling abs on positive and real entries, and this also permits + // to seamlessly handle complexes. Otherwise we would have to handle both real and complexes + // through the same functor... + return internal::positive_real_hypot(x,y); } }; template<typename Scalar> -struct functor_traits<scalar_hypot_op<Scalar> > { +struct functor_traits<scalar_hypot_op<Scalar,Scalar> > { enum { Cost = 3 * NumTraits<Scalar>::AddCost + 2 * NumTraits<Scalar>::MulCost + - 2 * NumTraits<Scalar>::template Div<false>::Cost, + 2 * scalar_div_cost<Scalar,false>::value, PacketAccess = false }; }; @@ -250,13 +288,24 @@ /** \internal * \brief Template functor to compute the pow of two scalars */ -template<typename Scalar, typename OtherScalar> struct scalar_binary_pow_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op) +template<typename Scalar, typename Exponent> +struct scalar_pow_op : binary_op_base<Scalar,Exponent> +{ + typedef typename ScalarBinaryOpTraits<Scalar,Exponent,scalar_pow_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN + EIGEN_EMPTY_STRUCT_CTOR(scalar_pow_op) +#else + scalar_pow_op() { + typedef Scalar LhsScalar; + typedef Exponent RhsScalar; + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif EIGEN_DEVICE_FUNC - inline Scalar operator() (const Scalar& a, const OtherScalar& b) const { return numext::pow(a, b); } + inline result_type operator() (const Scalar& a, const Exponent& b) const { return numext::pow(a, b); } }; -template<typename Scalar, typename OtherScalar> -struct functor_traits<scalar_binary_pow_op<Scalar,OtherScalar> > { +template<typename Scalar, typename Exponent> +struct functor_traits<scalar_pow_op<Scalar,Exponent> > { enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; }; @@ -269,18 +318,27 @@ * * \sa class CwiseBinaryOp, MatrixBase::operator- */ -template<typename Scalar> struct scalar_difference_op { +template<typename LhsScalar,typename RhsScalar> +struct scalar_difference_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_difference_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_difference_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a - b; } +#else + scalar_difference_op() { + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a - b; } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::psub(a,b); } }; -template<typename Scalar> -struct functor_traits<scalar_difference_op<Scalar> > { +template<typename LhsScalar,typename RhsScalar> +struct functor_traits<scalar_difference_op<LhsScalar,RhsScalar> > { enum { - Cost = NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasSub + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, + PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasSub && packet_traits<RhsScalar>::HasSub }; }; @@ -289,13 +347,17 @@ * * \sa class CwiseBinaryOp, Cwise::operator/() */ -template<typename LhsScalar,typename RhsScalar> struct scalar_quotient_op { - enum { - // TODO vectorize mixed product - Vectorizable = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasDiv && packet_traits<RhsScalar>::HasDiv - }; - typedef typename scalar_product_traits<LhsScalar,RhsScalar>::ReturnType result_type; +template<typename LhsScalar,typename RhsScalar> +struct scalar_quotient_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_quotient_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_quotient_op) +#else + scalar_quotient_op() { + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a / b; } template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const @@ -305,8 +367,8 @@ struct functor_traits<scalar_quotient_op<LhsScalar,RhsScalar> > { typedef typename scalar_quotient_op<LhsScalar,RhsScalar>::result_type result_type; enum { - PacketAccess = scalar_quotient_op<LhsScalar,RhsScalar>::Vectorizable, - Cost = NumTraits<result_type>::template Div<PacketAccess>::Cost + PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasDiv && packet_traits<RhsScalar>::HasDiv, + Cost = scalar_div_cost<result_type,PacketAccess>::value }; }; @@ -360,236 +422,50 @@ }; }; -/** \internal - * \brief Template functor to compute the incomplete gamma function igamma(a, x) - * - * \sa class CwiseBinaryOp, Cwise::igamma - */ -template<typename Scalar> struct scalar_igamma_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_igamma_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& x) const { - using numext::igamma; return igamma(a, x); - } - template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& x) const { - return internal::pigammac(a, x); - } -}; -template<typename Scalar> -struct functor_traits<scalar_igamma_op<Scalar> > { - enum { - // Guesstimate - Cost = 20 * NumTraits<Scalar>::MulCost + 10 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasIGamma - }; -}; - - -/** \internal - * \brief Template functor to compute the complementary incomplete gamma function igammac(a, x) - * - * \sa class CwiseBinaryOp, Cwise::igammac - */ -template<typename Scalar> struct scalar_igammac_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_igammac_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& x) const { - using numext::igammac; return igammac(a, x); - } - template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& x) const - { - return internal::pigammac(a, x); - } -}; -template<typename Scalar> -struct functor_traits<scalar_igammac_op<Scalar> > { - enum { - // Guesstimate - Cost = 20 * NumTraits<Scalar>::MulCost + 10 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasIGammac - }; -}; //---------- binary functors bound to a constant, thus appearing as a unary functor ---------- -/** \internal - * \brief Template functor to multiply a scalar by a fixed other one - * - * \sa class CwiseUnaryOp, MatrixBase::operator*, MatrixBase::operator/ - */ -/* NOTE why doing the pset1() in packetOp *is* an optimization ? - * indeed it seems better to declare m_other as a Packet and do the pset1() once - * in the constructor. However, in practice: - * - GCC does not like m_other as a Packet and generate a load every time it needs it - * - on the other hand GCC is able to moves the pset1() outside the loop :) - * - simpler code ;) - * (ICC and gcc 4.4 seems to perform well in both cases, the issue is visible with y = a*x + b*y) - */ -template<typename Scalar> -struct scalar_multiple_op { - // FIXME default copy constructors seems bugged with std::complex<> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE scalar_multiple_op(const scalar_multiple_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE scalar_multiple_op(const Scalar& other) : m_other(other) { } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a * m_other; } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const - { return internal::pmul(a, pset1<Packet>(m_other)); } - typename add_const_on_value_type<typename NumTraits<Scalar>::Nested>::type m_other; -}; -template<typename Scalar> -struct functor_traits<scalar_multiple_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasMul }; }; +// The following two classes permits to turn any binary functor into a unary one with one argument bound to a constant value. +// They are analogues to std::binder1st/binder2nd but with the following differences: +// - they are compatible with packetOp +// - they are portable across C++ versions (the std::binder* are deprecated in C++11) +template<typename BinaryOp> struct bind1st_op : BinaryOp { -template<typename Scalar1, typename Scalar2> -struct scalar_multiple2_op { - typedef typename scalar_product_traits<Scalar1,Scalar2>::ReturnType result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_multiple2_op(const scalar_multiple2_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_multiple2_op(const Scalar2& other) : m_other(other) { } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar1& a) const { return a * m_other; } - typename add_const_on_value_type<typename NumTraits<Scalar2>::Nested>::type m_other; -}; -template<typename Scalar1,typename Scalar2> -struct functor_traits<scalar_multiple2_op<Scalar1,Scalar2> > -{ enum { Cost = NumTraits<Scalar1>::MulCost, PacketAccess = false }; }; + typedef typename BinaryOp::first_argument_type first_argument_type; + typedef typename BinaryOp::second_argument_type second_argument_type; + typedef typename BinaryOp::result_type result_type; -/** \internal - * \brief Template functor to divide a scalar by a fixed other one - * - * This functor is used to implement the quotient of a matrix by - * a scalar where the scalar type is not necessarily a floating point type. - * - * \sa class CwiseUnaryOp, MatrixBase::operator/ - */ -template<typename Scalar> -struct scalar_quotient1_op { - // FIXME default copy constructors seems bugged with std::complex<> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_quotient1_op(const scalar_quotient1_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_quotient1_op(const Scalar& other) : m_other(other) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a / m_other; } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const - { return internal::pdiv(a, pset1<Packet>(m_other)); } - typename add_const_on_value_type<typename NumTraits<Scalar>::Nested>::type m_other; -}; -template<typename Scalar> -struct functor_traits<scalar_quotient1_op<Scalar> > -{ enum { Cost = 2 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasDiv }; }; + EIGEN_DEVICE_FUNC explicit bind1st_op(const first_argument_type &val) : m_value(val) {} -template<typename Scalar1, typename Scalar2> -struct scalar_quotient2_op { - typedef typename scalar_product_traits<Scalar1,Scalar2>::ReturnType result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_quotient2_op(const scalar_quotient2_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_quotient2_op(const Scalar2& other) : m_other(other) { } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar1& a) const { return a / m_other; } - typename add_const_on_value_type<typename NumTraits<Scalar2>::Nested>::type m_other; -}; -template<typename Scalar1,typename Scalar2> -struct functor_traits<scalar_quotient2_op<Scalar1,Scalar2> > -{ enum { Cost = 2 * NumTraits<Scalar1>::MulCost, PacketAccess = false }; }; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const second_argument_type& b) const { return BinaryOp::operator()(m_value,b); } -// In Eigen, any binary op (Product, CwiseBinaryOp) require the Lhs and Rhs to have the same scalar type, except for multiplication -// where the mixing of different types is handled by scalar_product_traits -// In particular, real * complex<real> is allowed. -// FIXME move this to functor_traits adding a functor_default -template<typename Functor> struct functor_is_product_like { enum { ret = 0 }; }; -template<typename LhsScalar,typename RhsScalar> struct functor_is_product_like<scalar_product_op<LhsScalar,RhsScalar> > { enum { ret = 1 }; }; -template<typename LhsScalar,typename RhsScalar> struct functor_is_product_like<scalar_conj_product_op<LhsScalar,RhsScalar> > { enum { ret = 1 }; }; -template<typename LhsScalar,typename RhsScalar> struct functor_is_product_like<scalar_quotient_op<LhsScalar,RhsScalar> > { enum { ret = 1 }; }; - - -/** \internal - * \brief Template functor to add a scalar to a fixed other one - * \sa class CwiseUnaryOp, Array::operator+ - */ -/* If you wonder why doing the pset1() in packetOp() is an optimization check scalar_multiple_op */ -template<typename Scalar> -struct scalar_add_op { - // FIXME default copy constructors seems bugged with std::complex<> - EIGEN_DEVICE_FUNC inline scalar_add_op(const scalar_add_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC inline scalar_add_op(const Scalar& other) : m_other(other) { } - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a + m_other; } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const - { return internal::padd(a, pset1<Packet>(m_other)); } - const Scalar m_other; -}; -template<typename Scalar> -struct functor_traits<scalar_add_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasAdd }; }; - -/** \internal - * \brief Template functor to subtract a fixed scalar to another one - * \sa class CwiseUnaryOp, Array::operator-, struct scalar_add_op, struct scalar_rsub_op - */ -template<typename Scalar> -struct scalar_sub_op { - EIGEN_DEVICE_FUNC inline scalar_sub_op(const scalar_sub_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC inline scalar_sub_op(const Scalar& other) : m_other(other) { } - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a - m_other; } - template <typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const - { return internal::psub(a, pset1<Packet>(m_other)); } - const Scalar m_other; -}; -template<typename Scalar> -struct functor_traits<scalar_sub_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasAdd }; }; - -/** \internal - * \brief Template functor to subtract a scalar to fixed another one - * \sa class CwiseUnaryOp, Array::operator-, struct scalar_add_op, struct scalar_sub_op - */ -template<typename Scalar> -struct scalar_rsub_op { - EIGEN_DEVICE_FUNC inline scalar_rsub_op(const scalar_rsub_op& other) : m_other(other.m_other) { } - EIGEN_DEVICE_FUNC inline scalar_rsub_op(const Scalar& other) : m_other(other) { } - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return m_other - a; } - template <typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const - { return internal::psub(pset1<Packet>(m_other), a); } - const Scalar m_other; -}; -template<typename Scalar> -struct functor_traits<scalar_rsub_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasAdd }; }; - -/** \internal - * \brief Template functor to raise a scalar to a power - * \sa class CwiseUnaryOp, Cwise::pow - */ -template<typename Scalar> -struct scalar_pow_op { - // FIXME default copy constructors seems bugged with std::complex<> - EIGEN_DEVICE_FUNC inline scalar_pow_op(const scalar_pow_op& other) : m_exponent(other.m_exponent) { } - EIGEN_DEVICE_FUNC inline scalar_pow_op(const Scalar& exponent) : m_exponent(exponent) {} - EIGEN_DEVICE_FUNC - inline Scalar operator() (const Scalar& a) const { return numext::pow(a, m_exponent); } - const Scalar m_exponent; -}; -template<typename Scalar> -struct functor_traits<scalar_pow_op<Scalar> > -{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; }; - -/** \internal - * \brief Template functor to compute the quotient between a scalar and array entries. - * \sa class CwiseUnaryOp, Cwise::inverse() - */ -template<typename Scalar> -struct scalar_inverse_mult_op { - EIGEN_DEVICE_FUNC scalar_inverse_mult_op(const Scalar& other) : m_other(other) {} - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return m_other / a; } template<typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const - { return internal::pdiv(pset1<Packet>(m_other),a); } - Scalar m_other; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& b) const + { return BinaryOp::packetOp(internal::pset1<Packet>(m_value), b); } + + first_argument_type m_value; }; -template<typename Scalar> -struct functor_traits<scalar_inverse_mult_op<Scalar> > -{ enum { PacketAccess = packet_traits<Scalar>::HasDiv, Cost = NumTraits<Scalar>::template Div<PacketAccess>::Cost }; }; +template<typename BinaryOp> struct functor_traits<bind1st_op<BinaryOp> > : functor_traits<BinaryOp> {}; + + +template<typename BinaryOp> struct bind2nd_op : BinaryOp { + + typedef typename BinaryOp::first_argument_type first_argument_type; + typedef typename BinaryOp::second_argument_type second_argument_type; + typedef typename BinaryOp::result_type result_type; + + EIGEN_DEVICE_FUNC explicit bind2nd_op(const second_argument_type &val) : m_value(val) {} + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const first_argument_type& a) const { return BinaryOp::operator()(a,m_value); } + + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return BinaryOp::packetOp(a,internal::pset1<Packet>(m_value)); } + + second_argument_type m_value; +}; +template<typename BinaryOp> struct functor_traits<bind2nd_op<BinaryOp> > : functor_traits<BinaryOp> {}; } // end namespace internal
diff --git a/Eigen/src/Core/functors/CMakeLists.txt b/Eigen/src/Core/functors/CMakeLists.txt deleted file mode 100644 index f4b99a9..0000000 --- a/Eigen/src/Core/functors/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_Functor_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_Functor_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/functors COMPONENT Devel - )
diff --git a/Eigen/src/Core/functors/NullaryFunctors.h b/Eigen/src/Core/functors/NullaryFunctors.h index c5836d0..16b645f 100644 --- a/Eigen/src/Core/functors/NullaryFunctors.h +++ b/Eigen/src/Core/functors/NullaryFunctors.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -18,106 +18,97 @@ struct scalar_constant_op { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_constant_op(const scalar_constant_op& other) : m_other(other.m_other) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_constant_op(const Scalar& other) : m_other(other) { } - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index, Index = 0) const { return m_other; } - template<typename Index, typename PacketType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packetOp(Index, Index = 0) const { return internal::pset1<PacketType>(m_other); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() () const { return m_other; } + template<typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packetOp() const { return internal::pset1<PacketType>(m_other); } const Scalar m_other; }; template<typename Scalar> struct functor_traits<scalar_constant_op<Scalar> > -{ enum { Cost = 1, PacketAccess = packet_traits<Scalar>::Vectorizable, IsRepeatable = true }; }; +{ enum { Cost = 0 /* as the constant value should be loaded in register only once for the whole expression */, + PacketAccess = packet_traits<Scalar>::Vectorizable, IsRepeatable = true }; }; template<typename Scalar> struct scalar_identity_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_identity_op) - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index row, Index col) const { return row==col ? Scalar(1) : Scalar(0); } + template<typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType row, IndexType col) const { return row==col ? Scalar(1) : Scalar(0); } }; template<typename Scalar> struct functor_traits<scalar_identity_op<Scalar> > { enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = false, IsRepeatable = true }; }; -template <typename Scalar, typename Packet, bool RandomAccess, bool IsInteger> struct linspaced_op_impl; +template <typename Scalar, bool IsInteger> struct linspaced_op_impl; -// linear access for packet ops: -// 1) initialization -// base = [low, ..., low] + ([step, ..., step] * [-size, ..., 0]) -// 2) each step (where size is 1 for coeff access or PacketSize for packet access) -// base += [size*step, ..., size*step] -// -// TODO: Perhaps it's better to initialize lazily (so not in the constructor but in packetOp) -// in order to avoid the padd() in operator() ? -template <typename Scalar, typename Packet> -struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/false,/*IsInteger*/false> +template <typename Scalar> +struct linspaced_op_impl<Scalar,/*IsInteger*/false> { + typedef typename NumTraits<Scalar>::Real RealScalar; + linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : - m_low(low), m_step(num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1)), - m_packetStep(pset1<Packet>(unpacket_traits<Packet>::size*m_step)), - m_base(padd(pset1<Packet>(low), pmul(pset1<Packet>(m_step),plset<Packet>(-unpacket_traits<Packet>::size)))) {} - - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index i) const - { - m_base = padd(m_base, pset1<Packet>(m_step)); - return m_low+Scalar(i)*m_step; - } - - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(Index) const { return m_base = padd(m_base,m_packetStep); } - - const Scalar m_low; - const Scalar m_step; - const Packet m_packetStep; - mutable Packet m_base; -}; - -// random access for packet ops: -// 1) each step -// [low, ..., low] + ( [step, ..., step] * ( [i, ..., i] + [0, ..., size] ) ) -template <typename Scalar, typename Packet> -struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/true,/*IsInteger*/false> -{ - linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : - m_low(low), m_step(num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1)), - m_lowPacket(pset1<Packet>(m_low)), m_stepPacket(pset1<Packet>(m_step)), m_interPacket(plset<Packet>(0)) {} - - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index i) const { return m_low+i*m_step; } - - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(Index i) const - { return internal::padd(m_lowPacket, pmul(m_stepPacket, padd(pset1<Packet>(Scalar(i)),m_interPacket))); } - - const Scalar m_low; - const Scalar m_step; - const Packet m_lowPacket; - const Packet m_stepPacket; - const Packet m_interPacket; -}; - -template <typename Scalar, typename Packet> -struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/true,/*IsInteger*/true> -{ - linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : - m_low(low), m_length(high-low), m_divisor(num_steps==1?1:num_steps-1), m_interPacket(plset<Packet>(0)) + m_low(low), m_high(high), m_size1(num_steps==1 ? 1 : num_steps-1), m_step(num_steps==1 ? Scalar() : (high-low)/RealScalar(num_steps-1)), + m_flip(numext::abs(high)<numext::abs(low)) {} - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const Scalar operator() (Index i) const { - return m_low + (m_length*Scalar(i))/m_divisor; + template<typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { + if(m_flip) + return (i==0)? m_low : (m_high - RealScalar(m_size1-i)*m_step); + else + return (i==m_size1)? m_high : (m_low + RealScalar(i)*m_step); } - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const Packet packetOp(Index i) const { - return internal::padd(pset1<Packet>(m_low), pdiv(pmul(pset1<Packet>(m_length), padd(pset1<Packet>(Scalar(i)),m_interPacket)), - pset1<Packet>(m_divisor))); } + template<typename Packet, typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const + { + // Principle: + // [low, ..., low] + ( [step, ..., step] * ( [i, ..., i] + [0, ..., size] ) ) + if(m_flip) + { + Packet pi = plset<Packet>(Scalar(i-m_size1)); + Packet res = padd(pset1<Packet>(m_high), pmul(pset1<Packet>(m_step), pi)); + if(i==0) + res = pinsertfirst(res, m_low); + return res; + } + else + { + Packet pi = plset<Packet>(Scalar(i)); + Packet res = padd(pset1<Packet>(m_low), pmul(pset1<Packet>(m_step), pi)); + if(i==m_size1-unpacket_traits<Packet>::size+1) + res = pinsertlast(res, m_high); + return res; + } + } const Scalar m_low; - const Scalar m_length; - const Index m_divisor; - const Packet m_interPacket; + const Scalar m_high; + const Index m_size1; + const Scalar m_step; + const bool m_flip; +}; + +template <typename Scalar> +struct linspaced_op_impl<Scalar,/*IsInteger*/true> +{ + linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : + m_low(low), + m_multiplier((high-low)/convert_index<Scalar>(num_steps<=1 ? 1 : num_steps-1)), + m_divisor(convert_index<Scalar>((high>=low?num_steps:-num_steps)+(high-low))/((numext::abs(high-low)+1)==0?1:(numext::abs(high-low)+1))), + m_use_divisor(num_steps>1 && (numext::abs(high-low)+1)<num_steps) + {} + + template<typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Scalar operator() (IndexType i) const + { + if(m_use_divisor) return m_low + convert_index<Scalar>(i)/m_divisor; + else return m_low + convert_index<Scalar>(i)*m_multiplier; + } + + const Scalar m_low; + const Scalar m_multiplier; + const Scalar m_divisor; + const bool m_use_divisor; }; // ----- Linspace functor ---------------------------------------------------------------- @@ -125,60 +116,71 @@ // Forward declaration (we default to random access which does not really give // us a speed gain when using packet access but it allows to use the functor in // nested expressions). -template <typename Scalar, typename PacketType, bool RandomAccess = true> struct linspaced_op; -template <typename Scalar, typename PacketType, bool RandomAccess> struct functor_traits< linspaced_op<Scalar,PacketType,RandomAccess> > +template <typename Scalar> struct linspaced_op; +template <typename Scalar> struct functor_traits< linspaced_op<Scalar> > { enum { Cost = 1, - PacketAccess = packet_traits<Scalar>::HasSetLinear - && ((!NumTraits<Scalar>::IsInteger) || packet_traits<Scalar>::HasDiv), + PacketAccess = (!NumTraits<Scalar>::IsInteger) && packet_traits<Scalar>::HasSetLinear && packet_traits<Scalar>::HasBlend, + /*&& ((!NumTraits<Scalar>::IsInteger) || packet_traits<Scalar>::HasDiv),*/ // <- vectorization for integer is currently disabled IsRepeatable = true }; }; -template <typename Scalar, typename PacketType, bool RandomAccess> struct linspaced_op +template <typename Scalar> struct linspaced_op { linspaced_op(const Scalar& low, const Scalar& high, Index num_steps) : impl((num_steps==1 ? high : low),high,num_steps) {} - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index i) const { return impl(i); } + template<typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { return impl(i); } - // We need this function when assigning e.g. a RowVectorXd to a MatrixXd since - // there row==0 and col is used for the actual iteration. - template<typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index row, Index col) const - { - eigen_assert(col==0 || row==0); - return impl(col + row); - } + template<typename Packet,typename IndexType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { return impl.template packetOp<Packet>(i); } - template<typename Index, typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(Index i) const { return impl.packetOp(i); } - - // We need this function when assigning e.g. a RowVectorXd to a MatrixXd since - // there row==0 and col is used for the actual iteration. - template<typename Index, typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(Index row, Index col) const - { - eigen_assert(col==0 || row==0); - return impl.packetOp(col + row); - } - - // This proxy object handles the actual required temporaries, the different - // implementations (random vs. sequential access) as well as the - // correct piping to size 2/4 packet operations. - // As long as we don't have a Bresenham-like implementation for linear-access and integer types, - // we have to by-pass RandomAccess for integer types. See bug 698. - const linspaced_op_impl<Scalar,PacketType,(NumTraits<Scalar>::IsInteger?true:RandomAccess),NumTraits<Scalar>::IsInteger> impl; + // This proxy object handles the actual required temporaries and the different + // implementations (integer vs. floating point). + const linspaced_op_impl<Scalar,NumTraits<Scalar>::IsInteger> impl; }; -// all functors allow linear access, except scalar_identity_op. So we fix here a quick meta -// to indicate whether a functor allows linear access, just always answering 'yes' except for -// scalar_identity_op. -template<typename Functor> struct functor_has_linear_access { enum { ret = 1 }; }; -template<typename Scalar> struct functor_has_linear_access<scalar_identity_op<Scalar> > { enum { ret = 0 }; }; +// Linear access is automatically determined from the operator() prototypes available for the given functor. +// If it exposes an operator()(i,j), then we assume the i and j coefficients are required independently +// and linear access is not possible. In all other cases, linear access is enabled. +// Users should not have to deal with this structure. +template<typename Functor> struct functor_has_linear_access { enum { ret = !has_binary_operator<Functor>::value }; }; + +// For unreliable compilers, let's specialize the has_*ary_operator +// helpers so that at least built-in nullary functors work fine. +#if !( (EIGEN_COMP_MSVC>1600) || (EIGEN_GNUC_AT_LEAST(4,8)) || (EIGEN_COMP_ICC>=1600)) +template<typename Scalar,typename IndexType> +struct has_nullary_operator<scalar_constant_op<Scalar>,IndexType> { enum { value = 1}; }; +template<typename Scalar,typename IndexType> +struct has_unary_operator<scalar_constant_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_binary_operator<scalar_constant_op<Scalar>,IndexType> { enum { value = 0}; }; + +template<typename Scalar,typename IndexType> +struct has_nullary_operator<scalar_identity_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_unary_operator<scalar_identity_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_binary_operator<scalar_identity_op<Scalar>,IndexType> { enum { value = 1}; }; + +template<typename Scalar,typename IndexType> +struct has_nullary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_unary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 1}; }; +template<typename Scalar,typename IndexType> +struct has_binary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 0}; }; + +template<typename Scalar,typename IndexType> +struct has_nullary_operator<scalar_random_op<Scalar>,IndexType> { enum { value = 1}; }; +template<typename Scalar,typename IndexType> +struct has_unary_operator<scalar_random_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_binary_operator<scalar_random_op<Scalar>,IndexType> { enum { value = 0}; }; +#endif } // end namespace internal
diff --git a/Eigen/src/Core/functors/StlFunctors.h b/Eigen/src/Core/functors/StlFunctors.h index 0b4e5a2..9c1d758 100644 --- a/Eigen/src/Core/functors/StlFunctors.h +++ b/Eigen/src/Core/functors/StlFunctors.h
@@ -72,7 +72,7 @@ struct functor_traits<std::not_equal_to<T> > { enum { Cost = 1, PacketAccess = false }; }; -#if(__cplusplus < 201103L) +#if (__cplusplus < 201103L) && (EIGEN_COMP_MSVC <= 1900) // std::binder* are deprecated since c++11 and will be removed in c++17 template<typename T> struct functor_traits<std::binder2nd<T> > @@ -83,13 +83,17 @@ { enum { Cost = functor_traits<T>::Cost, PacketAccess = false }; }; #endif +#if (__cplusplus < 201703L) && (EIGEN_COMP_MSVC < 1910) +// std::unary_negate is deprecated since c++17 and will be removed in c++20 template<typename T> struct functor_traits<std::unary_negate<T> > { enum { Cost = 1 + functor_traits<T>::Cost, PacketAccess = false }; }; +// std::binary_negate is deprecated since c++17 and will be removed in c++20 template<typename T> struct functor_traits<std::binary_negate<T> > { enum { Cost = 1 + functor_traits<T>::Cost, PacketAccess = false }; }; +#endif #ifdef EIGEN_STDEXT_SUPPORT
diff --git a/Eigen/src/Core/functors/TernaryFunctors.h b/Eigen/src/Core/functors/TernaryFunctors.h new file mode 100644 index 0000000..b254e96 --- /dev/null +++ b/Eigen/src/Core/functors/TernaryFunctors.h
@@ -0,0 +1,25 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Eugene Brevdo <ebrevdo@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TERNARY_FUNCTORS_H +#define EIGEN_TERNARY_FUNCTORS_H + +namespace Eigen { + +namespace internal { + +//---------- associative ternary functors ---------- + + + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TERNARY_FUNCTORS_H
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index 5baba14..1d5eb36 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -117,7 +117,15 @@ struct functor_traits<scalar_conjugate_op<Scalar> > { enum { - Cost = NumTraits<Scalar>::IsComplex ? NumTraits<Scalar>::AddCost : 0, + Cost = 0, + // Yes the cost is zero even for complexes because in most cases for which + // the cost is used, conjugation turns to be a no-op. Some examples: + // cost(a*conj(b)) == cost(a*b) + // cost(a+conj(b)) == cost(a+b) + // <etc. + // If we don't set it to zero, then: + // A.conjugate().lazyProduct(B.conjugate()) + // will bake its operands. We definitely don't want that! PacketAccess = packet_traits<Scalar>::HasConj }; }; @@ -248,7 +256,7 @@ // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other : (14 * NumTraits<Scalar>::AddCost + 6 * NumTraits<Scalar>::MulCost + - NumTraits<Scalar>::template Div<packet_traits<Scalar>::HasDiv>::Cost)) + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value)) #else Cost = (sizeof(Scalar) == 4 @@ -257,16 +265,36 @@ // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other : (23 * NumTraits<Scalar>::AddCost + 12 * NumTraits<Scalar>::MulCost + - NumTraits<Scalar>::template Div<packet_traits<Scalar>::HasDiv>::Cost)) + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value)) #endif }; }; /** \internal * + * \brief Template functor to compute the exponential of a scalar - 1. + * + * \sa class CwiseUnaryOp, ArrayBase::expm1() + */ +template<typename Scalar> struct scalar_expm1_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_expm1_op) + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::expm1(a); } + template <typename Packet> + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexpm1(a); } +}; +template <typename Scalar> +struct functor_traits<scalar_expm1_op<Scalar> > { + enum { + PacketAccess = packet_traits<Scalar>::HasExpm1, + Cost = functor_traits<scalar_exp_op<Scalar> >::Cost // TODO measure cost of expm1 + }; +}; + +/** \internal + * * \brief Template functor to compute the logarithm of a scalar * - * \sa class CwiseUnaryOp, Cwise::log() + * \sa class CwiseUnaryOp, ArrayBase::log() */ template<typename Scalar> struct scalar_log_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log_op) @@ -295,13 +323,33 @@ /** \internal * + * \brief Template functor to compute the logarithm of 1 plus a scalar value + * + * \sa class CwiseUnaryOp, ArrayBase::log1p() + */ +template<typename Scalar> struct scalar_log1p_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_log1p_op) + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log1p(a); } + template <typename Packet> + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog1p(a); } +}; +template <typename Scalar> +struct functor_traits<scalar_log1p_op<Scalar> > { + enum { + PacketAccess = packet_traits<Scalar>::HasLog1p, + Cost = functor_traits<scalar_log_op<Scalar> >::Cost // TODO measure cost of log1p + }; +}; + +/** \internal + * * \brief Template functor to compute the base-10 logarithm of a scalar * * \sa class CwiseUnaryOp, Cwise::log10() */ template<typename Scalar> struct scalar_log10_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log10_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { using std::log10; return log10(a); } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { EIGEN_USING_STD_MATH(log10) return log10(a); } template <typename Packet> EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog10(a); } }; @@ -453,142 +501,6 @@ /** \internal - * \brief Template functor to compute the natural log of the absolute - * value of Gamma of a scalar - * \sa class CwiseUnaryOp, Cwise::lgamma() - */ -template<typename Scalar> struct scalar_lgamma_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_lgamma_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { - using numext::lgamma; return lgamma(a); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plgamma(a); } -}; -template<typename Scalar> -struct functor_traits<scalar_lgamma_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasLGamma - }; -}; - -/** \internal - * \brief Template functor to compute psi, the derivative of lgamma of a scalar. - * \sa class CwiseUnaryOp, Cwise::digamma() - */ -template<typename Scalar> struct scalar_digamma_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_digamma_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { - using numext::digamma; return digamma(a); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pdigamma(a); } -}; -template<typename Scalar> -struct functor_traits<scalar_digamma_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasDiGamma - }; -}; - -/** \internal - * \brief Template functor to compute the Riemann Zeta function of two arguments. - * \sa class CwiseUnaryOp, Cwise::zeta() - */ -template<typename Scalar> struct scalar_zeta_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_zeta_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& x, const Scalar& q) const { - using numext::zeta; return zeta(x, q); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x, const Packet& q) const { return internal::pzeta(x, q); } -}; -template<typename Scalar> -struct functor_traits<scalar_zeta_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasZeta - }; -}; - -/** \internal - * \brief Template functor to compute the polygamma function. - * \sa class CwiseUnaryOp, Cwise::polygamma() - */ -template<typename Scalar> struct scalar_polygamma_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_polygamma_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& n, const Scalar& x) const { - using numext::polygamma; return polygamma(n, x); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& n, const Packet& x) const { return internal::ppolygamma(n, x); } -}; -template<typename Scalar> -struct functor_traits<scalar_polygamma_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasPolygamma - }; -}; - -/** \internal - * \brief Template functor to compute the Gauss error function of a - * scalar - * \sa class CwiseUnaryOp, Cwise::erf() - */ -template<typename Scalar> struct scalar_erf_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_erf_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { - using numext::erf; return erf(a); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::perf(a); } -}; -template<typename Scalar> -struct functor_traits<scalar_erf_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasErf - }; -}; - -/** \internal - * \brief Template functor to compute the Complementary Error Function - * of a scalar - * \sa class CwiseUnaryOp, Cwise::erfc() - */ -template<typename Scalar> struct scalar_erfc_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_erfc_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { - using numext::erfc; return erfc(a); - } - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::perfc(a); } -}; -template<typename Scalar> -struct functor_traits<scalar_erfc_op<Scalar> > -{ - enum { - // Guesstimate - Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasErfc - }; -}; - - -/** \internal * \brief Template functor to compute the atan of a scalar * \sa class CwiseUnaryOp, ArrayBase::atan() */ @@ -607,42 +519,60 @@ }; }; - /** \internal * \brief Template functor to compute the tanh of a scalar * \sa class CwiseUnaryOp, ArrayBase::tanh() */ -template<typename Scalar> struct scalar_tanh_op { +template <typename Scalar> +struct scalar_tanh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_op) - EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::tanh(a); } + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::tanh(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::ptanh(a); } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { return ptanh(x); } }; -template<typename Scalar> -struct functor_traits<scalar_tanh_op<Scalar> > -{ + +template <typename Scalar> +struct functor_traits<scalar_tanh_op<Scalar> > { enum { PacketAccess = packet_traits<Scalar>::HasTanh, - Cost = - (PacketAccess - // The following numbers are based on the AVX implementation, + Cost = ( (EIGEN_FAST_MATH && is_same<Scalar,float>::value) +// The following numbers are based on the AVX implementation, #ifdef EIGEN_VECTORIZE_FMA - // Haswell can issue 2 add/mul/madd per cycle. - // 9 pmadd, 2 pmul, 1 div, 2 other - ? (2 * NumTraits<Scalar>::AddCost + 6 * NumTraits<Scalar>::MulCost + - NumTraits<Scalar>::template Div<packet_traits<Scalar>::HasDiv>::Cost) + // Haswell can issue 2 add/mul/madd per cycle. + // 9 pmadd, 2 pmul, 1 div, 2 other + ? (2 * NumTraits<Scalar>::AddCost + + 6 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value) #else - ? (11 * NumTraits<Scalar>::AddCost + - 11 * NumTraits<Scalar>::MulCost + - NumTraits<Scalar>::template Div<packet_traits<Scalar>::HasDiv>::Cost) + ? (11 * NumTraits<Scalar>::AddCost + + 11 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value) #endif - // This number assumes a naive implementation of tanh - : (6 * NumTraits<Scalar>::AddCost + 3 * NumTraits<Scalar>::MulCost + - 2 * NumTraits<Scalar>::template Div<packet_traits<Scalar>::HasDiv>::Cost + - functor_traits<scalar_exp_op<Scalar> >::Cost)) + // This number assumes a naive implementation of tanh + : (6 * NumTraits<Scalar>::AddCost + + 3 * NumTraits<Scalar>::MulCost + + 2 * scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value + + functor_traits<scalar_exp_op<Scalar> >::Cost)) }; }; +#if EIGEN_HAS_CXX11_MATH +/** \internal + * \brief Template functor to compute the atanh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::atanh() + */ +template <typename Scalar> +struct scalar_atanh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::atanh(a); } +}; + +template <typename Scalar> +struct functor_traits<scalar_atanh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + /** \internal * \brief Template functor to compute the sinh of a scalar * \sa class CwiseUnaryOp, ArrayBase::sinh() @@ -662,6 +592,23 @@ }; }; +#if EIGEN_HAS_CXX11_MATH +/** \internal + * \brief Template functor to compute the asinh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::asinh() + */ +template <typename Scalar> +struct scalar_asinh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::asinh(a); } +}; + +template <typename Scalar> +struct functor_traits<scalar_asinh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + /** \internal * \brief Template functor to compute the cosh of a scalar * \sa class CwiseUnaryOp, ArrayBase::cosh() @@ -681,6 +628,23 @@ }; }; +#if EIGEN_HAS_CXX11_MATH +/** \internal + * \brief Template functor to compute the acosh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::acosh() + */ +template <typename Scalar> +struct scalar_acosh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::acosh(a); } +}; + +template <typename Scalar> +struct functor_traits<scalar_acosh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + /** \internal * \brief Template functor to compute the inverse of a scalar * \sa class CwiseUnaryOp, Cwise::inverse() @@ -693,9 +657,13 @@ EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return internal::pdiv(pset1<Packet>(Scalar(1)),a); } }; -template<typename Scalar> -struct functor_traits<scalar_inverse_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasDiv }; }; +template <typename Scalar> +struct functor_traits<scalar_inverse_op<Scalar> > { + enum { + PacketAccess = packet_traits<Scalar>::HasDiv, + Cost = scalar_div_cost<Scalar, PacketAccess>::value + }; +}; /** \internal * \brief Template functor to compute the square of a scalar @@ -793,7 +761,13 @@ template<typename Scalar> struct scalar_isnan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isnan_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return (numext::isnan)(a); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(__SYCL_DEVICE_ONLY__) + return numext::isnan(a); +#else + return (numext::isnan)(a); +#endif + } }; template<typename Scalar> struct functor_traits<scalar_isnan_op<Scalar> > @@ -811,7 +785,13 @@ template<typename Scalar> struct scalar_isinf_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isinf_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return (numext::isinf)(a); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(__SYCL_DEVICE_ONLY__) + return numext::isinf(a); +#else + return (numext::isinf)(a); +#endif + } }; template<typename Scalar> struct functor_traits<scalar_isinf_op<Scalar> > @@ -829,7 +809,13 @@ template<typename Scalar> struct scalar_isfinite_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isfinite_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return (numext::isfinite)(a); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(__SYCL_DEVICE_ONLY__) + return numext::isfinite(a); +#else + return (numext::isfinite)(a); +#endif + } }; template<typename Scalar> struct functor_traits<scalar_isfinite_op<Scalar> > @@ -880,9 +866,9 @@ { typedef typename NumTraits<Scalar>::Real real_type; real_type aa = numext::abs(a); - if (aa==0) + if (aa==real_type(0)) return Scalar(0); - aa = 1./aa; + aa = real_type(1)/aa; return Scalar(real(a)*aa, imag(a)*aa ); } //TODO @@ -892,7 +878,7 @@ template<typename Scalar> struct functor_traits<scalar_sign_op<Scalar> > { enum { - Cost = + Cost = NumTraits<Scalar>::IsComplex ? ( 8*NumTraits<Scalar>::MulCost ) // roughly : ( 3*NumTraits<Scalar>::AddCost), @@ -900,6 +886,95 @@ }; }; +/** \internal + * \brief Template functor to compute the logistic function of a scalar + * \sa class CwiseUnaryOp, ArrayBase::logistic() + */ +template <typename T> +struct scalar_logistic_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { + const T one = T(1); + return one / (one + numext::exp(-x)); + } + + template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& x) const { + const Packet one = pset1<Packet>(T(1)); + return pdiv(one, padd(one, pexp(pnegate(x)))); + } +}; +template <typename T> +struct functor_traits<scalar_logistic_op<T> > { + enum { + Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost * 6, + PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasDiv && + packet_traits<T>::HasNegate && packet_traits<T>::HasExp + }; +}; + +/** \internal + * \brief Template specialization of the logistic function for float. + * + * Uses just a 9/10-degree rational interpolant which + * interpolates 1/(1+exp(-x)) - 0.5 up to a couple of ulp in the range + * [-18, 18], outside of which the fl(logistic(x)) = {0|1}. The shifted + * logistic is interpolated because it was easier to make the fit converge. + * + */ + +template <> +struct scalar_logistic_op<float> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(const float& x) const { + const float one = 1.0f; + return one / (one + numext::exp(-x)); + } + + template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& _x) const { + // Clamp the inputs to the range [-18, 18] since anything outside + // this range is 0.0f or 1.0f in single-precision. + const Packet x = pmax(pmin(_x, pset1<Packet>(18.0)), pset1<Packet>(-18.0)); + + // The monomial coefficients of the numerator polynomial (odd). + const Packet alpha_1 = pset1<Packet>(2.48287947061529e-01); + const Packet alpha_3 = pset1<Packet>(8.51377133304701e-03); + const Packet alpha_5 = pset1<Packet>(6.08574864600143e-05); + const Packet alpha_7 = pset1<Packet>(1.15627324459942e-07); + const Packet alpha_9 = pset1<Packet>(4.37031012579801e-11); + + // The monomial coefficients of the denominator polynomial (even). + const Packet beta_0 = pset1<Packet>(9.93151921023180e-01); + const Packet beta_2 = pset1<Packet>(1.16817656904453e-01); + const Packet beta_4 = pset1<Packet>(1.70198817374094e-03); + const Packet beta_6 = pset1<Packet>(6.29106785017040e-06); + const Packet beta_8 = pset1<Packet>(5.76102136993427e-09); + const Packet beta_10 = pset1<Packet>(6.10247389755681e-13); + + // Since the polynomials are odd/even, we need x^2. + const Packet x2 = pmul(x, x); + + // Evaluate the numerator polynomial p. + Packet p = pmadd(x2, alpha_9, alpha_7); + p = pmadd(x2, p, alpha_5); + p = pmadd(x2, p, alpha_3); + p = pmadd(x2, p, alpha_1); + p = pmul(x, p); + + // Evaluate the denominator polynomial p. + Packet q = pmadd(x2, beta_10, beta_8); + q = pmadd(x2, q, beta_6); + q = pmadd(x2, q, beta_4); + q = pmadd(x2, q, beta_2); + q = pmadd(x2, q, beta_0); + + // Divide the numerator by the denominator and shift it up. + return pmax(pmin(padd(pdiv(p, q), pset1<Packet>(0.5)), pset1<Packet>(1.0)), + pset1<Packet>(0.0)); + } +}; + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/products/CMakeLists.txt b/Eigen/src/Core/products/CMakeLists.txt deleted file mode 100644 index 21fc94a..0000000 --- a/Eigen/src/Core/products/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_Product_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_Product_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/products COMPONENT Devel - )
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index bd559dc..fdd0ec0 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -15,7 +15,13 @@ namespace internal { -template<typename _LhsScalar, typename _RhsScalar, bool _ConjLhs=false, bool _ConjRhs=false> +enum PacketSizeType { + PacketFull = 0, + PacketHalf, + PacketQuarter +}; + +template<typename _LhsScalar, typename _RhsScalar, bool _ConjLhs=false, bool _ConjRhs=false, int Arch=Architecture::Target, int _PacketSize=PacketFull> class gebp_traits; @@ -89,7 +95,7 @@ * * \sa setCpuCacheSizes */ -template<typename LhsScalar, typename RhsScalar, int KcFactor> +template<typename LhsScalar, typename RhsScalar, int KcFactor, typename Index> void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index num_threads = 1) { typedef gebp_traits<LhsScalar,RhsScalar> Traits; @@ -101,6 +107,16 @@ // at the register level. This small horizontal panel has to stay within L1 cache. std::ptrdiff_t l1, l2, l3; manage_caching_sizes(GetAction, &l1, &l2, &l3); + #ifdef EIGEN_VECTORIZE_AVX512 + // We need to find a rationale for that, but without this adjustment, + // performance with AVX512 is pretty bad, like -20% slower. + // One reason is that with increasing packet-size, the blocking size k + // has to become pretty small if we want that 1 lhs panel fit within L1. + // For instance, with the 3pX4 kernel and double, the size of the lhs+rhs panels are: + // k*(3*64 + 4*8) Bytes, with l1=32kBytes, and k%8=0, we have k=144. + // This is quite small for a good reuse of the accumulation registers. + l1 *= 4; + #endif if (num_threads > 1) { typedef typename Traits::ResScalar ResScalar; @@ -115,7 +131,7 @@ // registers. However once the latency is hidden there is no point in // increasing the value of k, so we'll cap it at 320 (value determined // experimentally). - const Index k_cache = (std::min<Index>)((l1-ksub)/kdiv, 320); + const Index k_cache = (numext::mini<Index>)((l1-ksub)/kdiv, 320); if (k_cache < k) { k = k_cache - (k_cache % kr); eigen_internal_assert(k > 0); @@ -129,7 +145,7 @@ n = n_cache - (n_cache % nr); eigen_internal_assert(n > 0); } else { - n = (std::min<Index>)(n, (n_per_thread + nr - 1) - ((n_per_thread + nr - 1) % nr)); + n = (numext::mini<Index>)(n, (n_per_thread + nr - 1) - ((n_per_thread + nr - 1) % nr)); } if (l3 > l2) { @@ -140,7 +156,7 @@ m = m_cache - (m_cache % mr); eigen_internal_assert(m > 0); } else { - m = (std::min<Index>)(m, (m_per_thread + mr - 1) - ((m_per_thread + mr - 1) % mr)); + m = (numext::mini<Index>)(m, (m_per_thread + mr - 1) - ((m_per_thread + mr - 1) % mr)); } } } @@ -157,7 +173,7 @@ // Perhaps it would make more sense to consider k*n*m?? // Note that for very tiny problem, this function should be bypassed anyway // because we use the coefficient-based implementation for them. - if((std::max)(k,(std::max)(m,n))<48) + if((numext::maxi)(k,(numext::maxi)(m,n))<48) return; typedef typename Traits::ResScalar ResScalar; @@ -174,7 +190,7 @@ // We also include a register-level block of the result (mx x nr). // (In an ideal world only the lhs panel would stay in L1) // Moreover, kc has to be a multiple of 8 to be compatible with loop peeling, leading to a maximum blocking size of: - const Index max_kc = std::max<Index>(((l1-k_sub)/k_div) & (~(k_peeling-1)),1); + const Index max_kc = numext::maxi<Index>(((l1-k_sub)/k_div) & (~(k_peeling-1)),1); const Index old_k = k; if(k>max_kc) { @@ -219,7 +235,7 @@ max_nc = (3*actual_l2)/(2*2*max_kc*sizeof(RhsScalar)); } // WARNING Below, we assume that Traits::nr is a power of two. - Index nc = std::min<Index>(actual_l2/(2*k*sizeof(RhsScalar)), max_nc) & (~(Traits::nr-1)); + Index nc = numext::mini<Index>(actual_l2/(2*k*sizeof(RhsScalar)), max_nc) & (~(Traits::nr-1)); if(n>nc) { // We are really blocking over the columns: @@ -248,9 +264,9 @@ // we have both L2 and L3, and problem is small enough to be kept in L2 // Let's choose m such that lhs's block fit in 1/3 of L2 actual_lm = l2; - max_mc = (std::min<Index>)(576,max_mc); + max_mc = (numext::mini<Index>)(576,max_mc); } - Index mc = (std::min<Index>)(actual_lm/(3*k*sizeof(LhsScalar)), max_mc); + Index mc = (numext::mini<Index>)(actual_lm/(3*k*sizeof(LhsScalar)), max_mc); if (mc > Traits::mr) mc -= mc % Traits::mr; else if (mc==0) return; m = (m%mc)==0 ? mc @@ -259,13 +275,14 @@ } } +template <typename Index> inline bool useSpecificBlockingSizes(Index& k, Index& m, Index& n) { #ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) { - k = std::min<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K); - m = std::min<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M); - n = std::min<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N); + k = numext::mini<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K); + m = numext::mini<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M); + n = numext::mini<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N); return true; } #else @@ -292,28 +309,18 @@ * * \sa setCpuCacheSizes */ -template<typename LhsScalar, typename RhsScalar, int KcFactor> +template<typename LhsScalar, typename RhsScalar, int KcFactor, typename Index> void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) { if (!useSpecificBlockingSizes(k, m, n)) { - evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor>(k, m, n, num_threads); + evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor, Index>(k, m, n, num_threads); } - - typedef gebp_traits<LhsScalar,RhsScalar> Traits; - enum { - kr = 8, - mr = Traits::mr, - nr = Traits::nr - }; - if (k > kr) k -= k % kr; - if (m > mr) m -= m % mr; - if (n > nr) n -= n % nr; } -template<typename LhsScalar, typename RhsScalar> +template<typename LhsScalar, typename RhsScalar, typename Index> inline void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) { - computeProductBlockingSizes<LhsScalar,RhsScalar,1>(k, m, n, num_threads); + computeProductBlockingSizes<LhsScalar,RhsScalar,1,Index>(k, m, n, num_threads); } #ifdef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD @@ -346,6 +353,61 @@ // #define CJMADD(CJ,A,B,C,T) T = B; T = CJ.pmul(A,T); C = padd(C,T); #endif +template <typename RhsPacket, typename RhsPacketx4, int registers_taken> +struct RhsPanelHelper { + private: + static const int remaining_registers = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS - registers_taken; + public: + typedef typename conditional<remaining_registers>=4, RhsPacketx4, RhsPacket>::type type; +}; + +template <typename Packet> +struct QuadPacket +{ + Packet B_0, B1, B2, B3; + const Packet& get(const FixedInt<0>&) const { return B_0; } + const Packet& get(const FixedInt<1>&) const { return B1; } + const Packet& get(const FixedInt<2>&) const { return B2; } + const Packet& get(const FixedInt<3>&) const { return B3; } +}; + +template <int N, typename T1, typename T2, typename T3> +struct packet_conditional { typedef T3 type; }; + +template <typename T1, typename T2, typename T3> +struct packet_conditional<PacketFull, T1, T2, T3> { typedef T1 type; }; + +template <typename T1, typename T2, typename T3> +struct packet_conditional<PacketHalf, T1, T2, T3> { typedef T2 type; }; + +#define PACKET_DECL_COND_PREFIX(prefix, name, packet_size) \ + typedef typename packet_conditional<packet_size, \ + typename packet_traits<name ## Scalar>::type, \ + typename packet_traits<name ## Scalar>::half, \ + typename unpacket_traits<typename packet_traits<name ## Scalar>::half>::half>::type \ + prefix ## name ## Packet + +#define PACKET_DECL_COND(name, packet_size) \ + typedef typename packet_conditional<packet_size, \ + typename packet_traits<name ## Scalar>::type, \ + typename packet_traits<name ## Scalar>::half, \ + typename unpacket_traits<typename packet_traits<name ## Scalar>::half>::half>::type \ + name ## Packet + +#define PACKET_DECL_COND_SCALAR_PREFIX(prefix, packet_size) \ + typedef typename packet_conditional<packet_size, \ + typename packet_traits<Scalar>::type, \ + typename packet_traits<Scalar>::half, \ + typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type \ + prefix ## ScalarPacket + +#define PACKET_DECL_COND_SCALAR(packet_size) \ + typedef typename packet_conditional<packet_size, \ + typename packet_traits<Scalar>::type, \ + typename packet_traits<Scalar>::half, \ + typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type \ + ScalarPacket + /* Vectorization logic * real*real: unpack rhs to constant packets, ... * @@ -356,21 +418,25 @@ * cplx*real : unpack rhs to constant packets, ... * real*cplx : load lhs as (a0,a0,a1,a1), and mul as usual */ -template<typename _LhsScalar, typename _RhsScalar, bool _ConjLhs, bool _ConjRhs> +template<typename _LhsScalar, typename _RhsScalar, bool _ConjLhs, bool _ConjRhs, int Arch, int _PacketSize> class gebp_traits { public: typedef _LhsScalar LhsScalar; typedef _RhsScalar RhsScalar; - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; + + PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); enum { ConjLhs = _ConjLhs, ConjRhs = _ConjRhs, - Vectorizable = packet_traits<LhsScalar>::Vectorizable && packet_traits<RhsScalar>::Vectorizable, - LhsPacketSize = Vectorizable ? packet_traits<LhsScalar>::size : 1, - RhsPacketSize = Vectorizable ? packet_traits<RhsScalar>::size : 1, - ResPacketSize = Vectorizable ? packet_traits<ResScalar>::size : 1, + Vectorizable = unpacket_traits<_LhsPacket>::vectorizable && unpacket_traits<_RhsPacket>::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, @@ -379,10 +445,12 @@ // register block size along the M direction (currently, this one cannot be modified) default_mr = (EIGEN_PLAIN_ENUM_MIN(16,NumberOfRegisters)/2/nr)*LhsPacketSize, -#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX) - // we assume 16 registers +#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX) \ + && ((!EIGEN_COMP_MSVC) || (EIGEN_COMP_MSVC>=1914)) + // we assume 16 registers or more // See bug 992, if the scalar type is not vectorizable but that EIGEN_HAS_SINGLE_INSTRUCTION_MADD is defined, // then using 3*LhsPacketSize triggers non-implemented paths in syrk. + // Bug 1515: MSVC prior to v19.14 yields to register spilling. mr = Vectorizable ? 3*LhsPacketSize : default_mr, #else mr = default_mr, @@ -392,37 +460,41 @@ RhsProgress = 1 }; - typedef typename packet_traits<LhsScalar>::type _LhsPacket; - typedef typename packet_traits<RhsScalar>::type _RhsPacket; - typedef typename packet_traits<ResScalar>::type _ResPacket; typedef typename conditional<Vectorizable,_LhsPacket,LhsScalar>::type LhsPacket; typedef typename conditional<Vectorizable,_RhsPacket,RhsScalar>::type RhsPacket; typedef typename conditional<Vectorizable,_ResPacket,ResScalar>::type ResPacket; + typedef LhsPacket LhsPacket4Packing; + typedef QuadPacket<RhsPacket> RhsPacketx4; typedef ResPacket AccPacket; EIGEN_STRONG_INLINE void initAcc(AccPacket& p) { p = pset1<ResPacket>(ResScalar(0)); } - - EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) - { - pbroadcast4(b, b0, b1, b2, b3); - } - -// EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1) -// { -// pbroadcast2(b, b0, b1); -// } - + template<typename RhsPacketType> EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const { dest = pset1<RhsPacketType>(*b); } - + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const + { + pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3); + } + + template<typename RhsPacketType> + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const + { + loadRhs(b, dest); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const + { + } + EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); @@ -440,21 +512,28 @@ dest = ploadu<LhsPacketType>(a); } - template<typename LhsPacketType, typename RhsPacketType, typename AccPacketType> - EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, AccPacketType& tmp) const + template<typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const LaneIdType&) const { + conj_helper<LhsPacketType,RhsPacketType,ConjLhs,ConjRhs> cj; // It would be a lot cleaner to call pmadd all the time. Unfortunately if we // let gcc allocate the register in which to store the result of the pmul // (in the case where there is no FMA) gcc fails to figure out how to avoid // spilling register. #ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD EIGEN_UNUSED_VARIABLE(tmp); - c = pmadd(a,b,c); + c = cj.pmadd(a,b,c); #else - tmp = b; tmp = pmul(a,tmp); c = padd(c,tmp); + tmp = b; tmp = cj.pmul(a,tmp); c = padd(c,tmp); #endif } + template<typename LhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp, const LaneIdType& lane) const + { + madd(a, b.get(lane), c, tmp, lane); + } + EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const { r = pmadd(c,alpha,r); @@ -466,26 +545,27 @@ r = pmadd(c,alpha,r); } -protected: -// conj_helper<LhsScalar,RhsScalar,ConjLhs,ConjRhs> cj; -// conj_helper<LhsPacket,RhsPacket,ConjLhs,ConjRhs> pcj; }; -template<typename RealScalar, bool _ConjLhs> -class gebp_traits<std::complex<RealScalar>, RealScalar, _ConjLhs, false> +template<typename RealScalar, bool _ConjLhs, int Arch, int _PacketSize> +class gebp_traits<std::complex<RealScalar>, RealScalar, _ConjLhs, false, Arch, _PacketSize> { public: typedef std::complex<RealScalar> LhsScalar; typedef RealScalar RhsScalar; - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; + + PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); enum { ConjLhs = _ConjLhs, ConjRhs = false, - Vectorizable = packet_traits<LhsScalar>::Vectorizable && packet_traits<RhsScalar>::Vectorizable, - LhsPacketSize = Vectorizable ? packet_traits<LhsScalar>::size : 1, - RhsPacketSize = Vectorizable ? packet_traits<RhsScalar>::size : 1, - ResPacketSize = Vectorizable ? packet_traits<ResScalar>::size : 1, + Vectorizable = unpacket_traits<_LhsPacket>::vectorizable && unpacket_traits<_RhsPacket>::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, nr = 4, @@ -500,13 +580,12 @@ RhsProgress = 1 }; - typedef typename packet_traits<LhsScalar>::type _LhsPacket; - typedef typename packet_traits<RhsScalar>::type _RhsPacket; - typedef typename packet_traits<ResScalar>::type _ResPacket; - typedef typename conditional<Vectorizable,_LhsPacket,LhsScalar>::type LhsPacket; typedef typename conditional<Vectorizable,_RhsPacket,RhsScalar>::type RhsPacket; typedef typename conditional<Vectorizable,_ResPacket,ResScalar>::type ResPacket; + typedef LhsPacket LhsPacket4Packing; + + typedef QuadPacket<RhsPacket> RhsPacketx4; typedef ResPacket AccPacket; @@ -515,13 +594,42 @@ p = pset1<ResPacket>(ResScalar(0)); } - EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const + template<typename RhsPacketType> + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const { - dest = pset1<RhsPacket>(*b); + dest = pset1<RhsPacketType>(*b); } + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const + { + pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3); + } + + template<typename RhsPacketType> + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const + { + loadRhs(b, dest); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const + {} EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { + loadRhsQuad_impl(b,dest, typename conditional<RhsPacketSize==16,true_type,false_type>::type()); + } + + EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const true_type&) const + { + // FIXME we can do better! + // what we want here is a ploadheight + RhsScalar tmp[4] = {b[0],b[0],b[1],b[1]}; + dest = ploadquad<RhsPacket>(tmp); + } + + EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const false_type&) const + { + eigen_internal_assert(RhsPacketSize<=8); dest = pset1<RhsPacket>(*b); } @@ -530,27 +638,20 @@ dest = pload<LhsPacket>(a); } - EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacket& dest) const + template<typename LhsPacketType> + EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const { - dest = ploadu<LhsPacket>(a); + dest = ploadu<LhsPacketType>(a); } - EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) - { - pbroadcast4(b, b0, b1, b2, b3); - } - -// EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1) -// { -// pbroadcast2(b, b0, b1); -// } - - EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& tmp) const + template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const LaneIdType&) const { madd_impl(a, b, c, tmp, typename conditional<Vectorizable,true_type,false_type>::type()); } - EIGEN_STRONG_INLINE void madd_impl(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& tmp, const true_type&) const + template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType> + EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const true_type&) const { #ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD EIGEN_UNUSED_VARIABLE(tmp); @@ -565,13 +666,20 @@ c += a * b; } - EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const + template<typename LhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp, const LaneIdType& lane) const { + madd(a, b.get(lane), c, tmp, lane); + } + + template <typename ResPacketType, typename AccPacketType> + EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const + { + conj_helper<ResPacketType,ResPacketType,ConjLhs,false> cj; r = cj.pmadd(c,alpha,r); } protected: - conj_helper<ResPacket,ResPacket,ConjLhs,false> cj; }; template<typename Packet> @@ -590,13 +698,57 @@ return res; } +// note that for DoublePacket<RealPacket> the "4" in "downto4" +// corresponds to the number of complexes, so it means "8" +// it terms of real coefficients. + template<typename Packet> -const DoublePacket<Packet>& predux4(const DoublePacket<Packet> &a) +const DoublePacket<Packet>& +predux_half_dowto4(const DoublePacket<Packet> &a, + typename enable_if<unpacket_traits<Packet>::size<=8>::type* = 0) { return a; } -template<typename Packet> struct unpacket_traits<DoublePacket<Packet> > { typedef DoublePacket<Packet> half; }; +template<typename Packet> +DoublePacket<typename unpacket_traits<Packet>::half> +predux_half_dowto4(const DoublePacket<Packet> &a, + typename enable_if<unpacket_traits<Packet>::size==16>::type* = 0) +{ + // yes, that's pretty hackish :( + DoublePacket<typename unpacket_traits<Packet>::half> res; + typedef std::complex<typename unpacket_traits<Packet>::type> Cplx; + typedef typename packet_traits<Cplx>::type CplxPacket; + res.first = predux_half_dowto4(CplxPacket(a.first)).v; + res.second = predux_half_dowto4(CplxPacket(a.second)).v; + return res; +} + +// same here, "quad" actually means "8" in terms of real coefficients +template<typename Scalar, typename RealPacket> +void loadQuadToDoublePacket(const Scalar* b, DoublePacket<RealPacket>& dest, + typename enable_if<unpacket_traits<RealPacket>::size<=8>::type* = 0) +{ + dest.first = pset1<RealPacket>(real(*b)); + dest.second = pset1<RealPacket>(imag(*b)); +} + +template<typename Scalar, typename RealPacket> +void loadQuadToDoublePacket(const Scalar* b, DoublePacket<RealPacket>& dest, + typename enable_if<unpacket_traits<RealPacket>::size==16>::type* = 0) +{ + // yes, that's pretty hackish too :( + typedef typename NumTraits<Scalar>::Real RealScalar; + RealScalar r[4] = {real(b[0]), real(b[0]), real(b[1]), real(b[1])}; + RealScalar i[4] = {imag(b[0]), imag(b[0]), imag(b[1]), imag(b[1])}; + dest.first = ploadquad<RealPacket>(r); + dest.second = ploadquad<RealPacket>(i); +} + + +template<typename Packet> struct unpacket_traits<DoublePacket<Packet> > { + typedef DoublePacket<typename unpacket_traits<Packet>::half> half; +}; // template<typename Packet> // DoublePacket<Packet> pmadd(const DoublePacket<Packet> &a, const DoublePacket<Packet> &b) // { @@ -606,8 +758,8 @@ // return res; // } -template<typename RealScalar, bool _ConjLhs, bool _ConjRhs> -class gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, _ConjLhs, _ConjRhs > +template<typename RealScalar, bool _ConjLhs, bool _ConjRhs, int Arch, int _PacketSize> +class gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, _ConjLhs, _ConjRhs, Arch, _PacketSize > { public: typedef std::complex<RealScalar> Scalar; @@ -615,15 +767,21 @@ typedef std::complex<RealScalar> RhsScalar; typedef std::complex<RealScalar> ResScalar; + PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); + PACKET_DECL_COND(Real, _PacketSize); + PACKET_DECL_COND_SCALAR(_PacketSize); + enum { ConjLhs = _ConjLhs, ConjRhs = _ConjRhs, - Vectorizable = packet_traits<RealScalar>::Vectorizable - && packet_traits<Scalar>::Vectorizable, - RealPacketSize = Vectorizable ? packet_traits<RealScalar>::size : 1, - ResPacketSize = Vectorizable ? packet_traits<ResScalar>::size : 1, - LhsPacketSize = Vectorizable ? packet_traits<LhsScalar>::size : 1, - RhsPacketSize = Vectorizable ? packet_traits<RhsScalar>::size : 1, + Vectorizable = unpacket_traits<RealPacket>::vectorizable + && unpacket_traits<ScalarPacket>::vectorizable, + ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, + LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits<RhsScalar>::size : 1, + RealPacketSize = Vectorizable ? unpacket_traits<RealPacket>::size : 1, // FIXME: should depend on NumberOfRegisters nr = 4, @@ -633,14 +791,16 @@ RhsProgress = 1 }; - typedef typename packet_traits<RealScalar>::type RealPacket; - typedef typename packet_traits<Scalar>::type ScalarPacket; - typedef DoublePacket<RealPacket> DoublePacketType; + typedef DoublePacket<RealPacket> DoublePacketType; + typedef typename conditional<Vectorizable,ScalarPacket,Scalar>::type LhsPacket4Packing; typedef typename conditional<Vectorizable,RealPacket, Scalar>::type LhsPacket; typedef typename conditional<Vectorizable,DoublePacketType,Scalar>::type RhsPacket; typedef typename conditional<Vectorizable,ScalarPacket,Scalar>::type ResPacket; typedef typename conditional<Vectorizable,DoublePacketType,Scalar>::type AccPacket; + + // this actualy holds 8 packets! + typedef QuadPacket<RhsPacket> RhsPacketx4; EIGEN_STRONG_INLINE void initAcc(Scalar& p) { p = Scalar(0); } @@ -651,17 +811,41 @@ } // Scalar path - EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, ResPacket& dest) const + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, ScalarPacket& dest) const { - dest = pset1<ResPacket>(*b); + dest = pset1<ScalarPacket>(*b); } // Vectorized path - EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, DoublePacketType& dest) const + template<typename RealPacketType> + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const { - dest.first = pset1<RealPacket>(real(*b)); - dest.second = pset1<RealPacket>(imag(*b)); + dest.first = pset1<RealPacketType>(real(*b)); + dest.second = pset1<RealPacketType>(imag(*b)); } + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const + { + loadRhs(b, dest.B_0); + loadRhs(b + 1, dest.B1); + loadRhs(b + 2, dest.B2); + loadRhs(b + 3, dest.B3); + } + + // Scalar path + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, ScalarPacket& dest) const + { + loadRhs(b, dest); + } + + // Vectorized path + template<typename RealPacketType> + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const + { + loadRhs(b, dest); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {} EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, ResPacket& dest) const { @@ -669,33 +853,7 @@ } EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, DoublePacketType& dest) const { - eigen_internal_assert(unpacket_traits<ScalarPacket>::size<=4); - loadRhs(b,dest); - } - - EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) - { - // FIXME not sure that's the best way to implement it! - loadRhs(b+0, b0); - loadRhs(b+1, b1); - loadRhs(b+2, b2); - loadRhs(b+3, b3); - } - - // Vectorized path - EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, DoublePacketType& b0, DoublePacketType& b1) - { - // FIXME not sure that's the best way to implement it! - loadRhs(b+0, b0); - loadRhs(b+1, b1); - } - - // Scalar path - EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsScalar& b0, RhsScalar& b1) - { - // FIXME not sure that's the best way to implement it! - loadRhs(b+0, b0); - loadRhs(b+1, b1); + loadQuadToDoublePacket(b,dest); } // nothing special here @@ -704,47 +862,59 @@ dest = pload<LhsPacket>((const typename unpacket_traits<LhsPacket>::type*)(a)); } - EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacket& dest) const + template<typename LhsPacketType> + EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const { - dest = ploadu<LhsPacket>((const typename unpacket_traits<LhsPacket>::type*)(a)); + dest = ploadu<LhsPacketType>((const typename unpacket_traits<LhsPacketType>::type*)(a)); } - EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, DoublePacketType& c, RhsPacket& /*tmp*/) const + template<typename LhsPacketType, typename RhsPacketType, typename ResPacketType, typename TmpType, typename LaneIdType> + EIGEN_STRONG_INLINE + typename enable_if<!is_same<RhsPacketType,RhsPacketx4>::value>::type + madd(const LhsPacketType& a, const RhsPacketType& b, DoublePacket<ResPacketType>& c, TmpType& /*tmp*/, const LaneIdType&) const { c.first = padd(pmul(a,b.first), c.first); c.second = padd(pmul(a,b.second),c.second); } - EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, ResPacket& c, RhsPacket& /*tmp*/) const + template<typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, ResPacket& c, RhsPacket& /*tmp*/, const LaneIdType&) const { c = cj.pmadd(a,b,c); } + + template<typename LhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp, const LaneIdType& lane) const + { + madd(a, b.get(lane), c, tmp, lane); + } EIGEN_STRONG_INLINE void acc(const Scalar& c, const Scalar& alpha, Scalar& r) const { r += alpha * c; } - EIGEN_STRONG_INLINE void acc(const DoublePacketType& c, const ResPacket& alpha, ResPacket& r) const + template<typename RealPacketType, typename ResPacketType> + EIGEN_STRONG_INLINE void acc(const DoublePacket<RealPacketType>& c, const ResPacketType& alpha, ResPacketType& r) const { // assemble c - ResPacket tmp; + ResPacketType tmp; if((!ConjLhs)&&(!ConjRhs)) { - tmp = pcplxflip(pconj(ResPacket(c.second))); - tmp = padd(ResPacket(c.first),tmp); + tmp = pcplxflip(pconj(ResPacketType(c.second))); + tmp = padd(ResPacketType(c.first),tmp); } else if((!ConjLhs)&&(ConjRhs)) { - tmp = pconj(pcplxflip(ResPacket(c.second))); - tmp = padd(ResPacket(c.first),tmp); + tmp = pconj(pcplxflip(ResPacketType(c.second))); + tmp = padd(ResPacketType(c.first),tmp); } else if((ConjLhs)&&(!ConjRhs)) { - tmp = pcplxflip(ResPacket(c.second)); - tmp = padd(pconj(ResPacket(c.first)),tmp); + tmp = pcplxflip(ResPacketType(c.second)); + tmp = padd(pconj(ResPacketType(c.first)),tmp); } else if((ConjLhs)&&(ConjRhs)) { - tmp = pcplxflip(ResPacket(c.second)); - tmp = psub(pconj(ResPacket(c.first)),tmp); + tmp = pcplxflip(ResPacketType(c.second)); + tmp = psub(pconj(ResPacketType(c.first)),tmp); } r = pmadd(tmp,alpha,r); @@ -754,8 +924,8 @@ conj_helper<LhsScalar,RhsScalar,ConjLhs,ConjRhs> cj; }; -template<typename RealScalar, bool _ConjRhs> -class gebp_traits<RealScalar, std::complex<RealScalar>, false, _ConjRhs > +template<typename RealScalar, bool _ConjRhs, int Arch, int _PacketSize> +class gebp_traits<RealScalar, std::complex<RealScalar>, false, _ConjRhs, Arch, _PacketSize > { public: typedef std::complex<RealScalar> Scalar; @@ -763,14 +933,25 @@ typedef Scalar RhsScalar; typedef Scalar ResScalar; + PACKET_DECL_COND_PREFIX(_, Lhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Rhs, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Res, _PacketSize); + PACKET_DECL_COND_PREFIX(_, Real, _PacketSize); + PACKET_DECL_COND_SCALAR_PREFIX(_, _PacketSize); + +#undef PACKET_DECL_COND_SCALAR_PREFIX +#undef PACKET_DECL_COND_PREFIX +#undef PACKET_DECL_COND_SCALAR +#undef PACKET_DECL_COND + enum { ConjLhs = false, ConjRhs = _ConjRhs, - Vectorizable = packet_traits<RealScalar>::Vectorizable - && packet_traits<Scalar>::Vectorizable, - LhsPacketSize = Vectorizable ? packet_traits<LhsScalar>::size : 1, - RhsPacketSize = Vectorizable ? packet_traits<RhsScalar>::size : 1, - ResPacketSize = Vectorizable ? packet_traits<ResScalar>::size : 1, + Vectorizable = unpacket_traits<_RealPacket>::vectorizable + && unpacket_traits<_ScalarPacket>::vectorizable, + LhsPacketSize = Vectorizable ? unpacket_traits<_LhsPacket>::size : 1, + RhsPacketSize = Vectorizable ? unpacket_traits<_RhsPacket>::size : 1, + ResPacketSize = Vectorizable ? unpacket_traits<_ResPacket>::size : 1, NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS, // FIXME: should depend on NumberOfRegisters @@ -781,14 +962,11 @@ RhsProgress = 1 }; - typedef typename packet_traits<LhsScalar>::type _LhsPacket; - typedef typename packet_traits<RhsScalar>::type _RhsPacket; - typedef typename packet_traits<ResScalar>::type _ResPacket; - typedef typename conditional<Vectorizable,_LhsPacket,LhsScalar>::type LhsPacket; typedef typename conditional<Vectorizable,_RhsPacket,RhsScalar>::type RhsPacket; typedef typename conditional<Vectorizable,_ResPacket,ResScalar>::type ResPacket; - + typedef LhsPacket LhsPacket4Packing; + typedef QuadPacket<RhsPacket> RhsPacketx4; typedef ResPacket AccPacket; EIGEN_STRONG_INLINE void initAcc(AccPacket& p) @@ -796,22 +974,25 @@ p = pset1<ResPacket>(ResScalar(0)); } - EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const + template<typename RhsPacketType> + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const { - dest = pset1<RhsPacket>(*b); + dest = pset1<RhsPacketType>(*b); } - - void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) + + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { - pbroadcast4(b, b0, b1, b2, b3); + pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3); } - -// EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1) -// { -// // FIXME not sure that's the best way to implement it! -// b0 = pload1<RhsPacket>(b+0); -// b1 = pload1<RhsPacket>(b+1); -// } + + template<typename RhsPacketType> + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const + { + loadRhs(b, dest); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const + {} EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const { @@ -820,21 +1001,23 @@ EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { - eigen_internal_assert(unpacket_traits<RhsPacket>::size<=4); - loadRhs(b,dest); + dest = ploadquad<RhsPacket>(b); } - EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacket& dest) const + template<typename LhsPacketType> + EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const { - dest = ploaddup<LhsPacket>(a); + dest = ploaddup<LhsPacketType>(a); } - EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& tmp) const + template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const LaneIdType&) const { madd_impl(a, b, c, tmp, typename conditional<Vectorizable,true_type,false_type>::type()); } - EIGEN_STRONG_INLINE void madd_impl(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& tmp, const true_type&) const + template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType> + EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp, const true_type&) const { #ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD EIGEN_UNUSED_VARIABLE(tmp); @@ -850,90 +1033,166 @@ c += a * b; } - EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const + template<typename LhsPacketType, typename AccPacketType, typename LaneIdType> + EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp, const LaneIdType& lane) const { + madd(a, b.get(lane), c, tmp, lane); + } + + template <typename ResPacketType, typename AccPacketType> + EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const + { + conj_helper<ResPacketType,ResPacketType,false,ConjRhs> cj; r = cj.pmadd(alpha,c,r); } protected: - conj_helper<ResPacket,ResPacket,false,ConjRhs> cj; + }; -// helper for the rotating kernel below -template <typename GebpKernel, bool UseRotatingKernel = GebpKernel::UseRotatingKernel> -struct PossiblyRotatingKernelHelper + +#if EIGEN_ARCH_ARM64 && defined EIGEN_VECTORIZE_NEON + +template<> +struct gebp_traits <float, float, false, false,Architecture::NEON,PacketFull> + : gebp_traits<float,float,false,false,Architecture::Generic,PacketFull> { - // default implementation, not rotating + typedef float RhsPacket; - typedef typename GebpKernel::Traits Traits; - typedef typename Traits::RhsScalar RhsScalar; - typedef typename Traits::RhsPacket RhsPacket; - typedef typename Traits::AccPacket AccPacket; + typedef float32x4_t RhsPacketx4; - const Traits& traits; - PossiblyRotatingKernelHelper(const Traits& t) : traits(t) {} - - - template <size_t K, size_t Index> - void loadOrRotateRhs(RhsPacket& to, const RhsScalar* from) const + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { - traits.loadRhs(from + (Index+4*K)*Traits::RhsProgress, to); + dest = *b; } - void unrotateResult(AccPacket&, - AccPacket&, - AccPacket&, - AccPacket&) + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { + dest = vld1q_f32(b); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const + { + dest = *b; + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketx4& dest) const + {} + + EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const + { + loadRhs(b,dest); + } + + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { + c = vfmaq_n_f32(c, a, b); + } + + // NOTE: Template parameter inference failed when compiled with Android NDK: + // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>". + + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { madd_helper<0>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<1>&) const + { madd_helper<1>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<2>&) const + { madd_helper<2>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<3>&) const + { madd_helper<3>(a, b, c); } + + private: + template<int LaneID> + EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const + { + #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) + // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 + // vfmaq_laneq_f32 is implemented through a costly dup + if(LaneID==0) asm("fmla %0.4s, %1.4s, %2.s[0]\n" : "+w" (c) : "w" (a), "w" (b) : ); + else if(LaneID==1) asm("fmla %0.4s, %1.4s, %2.s[1]\n" : "+w" (c) : "w" (a), "w" (b) : ); + else if(LaneID==2) asm("fmla %0.4s, %1.4s, %2.s[2]\n" : "+w" (c) : "w" (a), "w" (b) : ); + else if(LaneID==3) asm("fmla %0.4s, %1.4s, %2.s[3]\n" : "+w" (c) : "w" (a), "w" (b) : ); + #else + c = vfmaq_laneq_f32(c, a, b, LaneID); + #endif } }; -// rotating implementation -template <typename GebpKernel> -struct PossiblyRotatingKernelHelper<GebpKernel, true> + +template<> +struct gebp_traits <double, double, false, false,Architecture::NEON> + : gebp_traits<double,double,false,false,Architecture::Generic> { - typedef typename GebpKernel::Traits Traits; - typedef typename Traits::RhsScalar RhsScalar; - typedef typename Traits::RhsPacket RhsPacket; - typedef typename Traits::AccPacket AccPacket; + typedef double RhsPacket; - const Traits& traits; - PossiblyRotatingKernelHelper(const Traits& t) : traits(t) {} + struct RhsPacketx4 { + float64x2_t B_0, B_1; + }; - template <size_t K, size_t Index> - void loadOrRotateRhs(RhsPacket& to, const RhsScalar* from) const + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { - if (Index == 0) { - to = pload<RhsPacket>(from + 4*K*Traits::RhsProgress); - } else { - EIGEN_ASM_COMMENT("Do not reorder code, we're very tight on registers"); - to = protate<1>(to); - } + dest = *b; } - void unrotateResult(AccPacket& res0, - AccPacket& res1, - AccPacket& res2, - AccPacket& res3) + EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { - PacketBlock<AccPacket> resblock; - resblock.packet[0] = res0; - resblock.packet[1] = res1; - resblock.packet[2] = res2; - resblock.packet[3] = res3; - ptranspose(resblock); - resblock.packet[3] = protate<1>(resblock.packet[3]); - resblock.packet[2] = protate<2>(resblock.packet[2]); - resblock.packet[1] = protate<3>(resblock.packet[1]); - ptranspose(resblock); - res0 = resblock.packet[0]; - res1 = resblock.packet[1]; - res2 = resblock.packet[2]; - res3 = resblock.packet[3]; + dest.B_0 = vld1q_f64(b); + dest.B_1 = vld1q_f64(b+2); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const + { + loadRhs(b,dest); + } + + EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketx4& dest) const + {} + + EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const + { + loadRhs(b,dest); + } + + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { + c = vfmaq_n_f64(c, a, b); + } + + // NOTE: Template parameter inference failed when compiled with Android NDK: + // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>". + + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const + { madd_helper<0>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<1>&) const + { madd_helper<1>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<2>&) const + { madd_helper<2>(a, b, c); } + EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<3>&) const + { madd_helper<3>(a, b, c); } + + private: + template <int LaneID> + EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const + { + #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) + // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 + // vfmaq_laneq_f64 is implemented through a costly dup + if(LaneID==0) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); + else if(LaneID==1) asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); + else if(LaneID==2) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_1) : ); + else if(LaneID==3) asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w" (c) : "w" (a), "w" (b.B_1) : ); + #else + if(LaneID==0) c = vfmaq_laneq_f64(c, a, b.B_0, 0); + else if(LaneID==1) c = vfmaq_laneq_f64(c, a, b.B_0, 1); + else if(LaneID==2) c = vfmaq_laneq_f64(c, a, b.B_1, 0); + else if(LaneID==3) c = vfmaq_laneq_f64(c, a, b.B_1, 1); + #endif } }; -/* optimized GEneral packed Block * packed Panel product kernel +#endif + +/* optimized General packed Block * packed Panel product kernel * * Mixing type logic: C += A * B * | A | B | comments @@ -943,45 +1202,349 @@ template<typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs> struct gebp_kernel { - typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs> Traits; + typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs,Architecture::Target> Traits; + typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs,Architecture::Target,PacketHalf> HalfTraits; + typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs,Architecture::Target,PacketQuarter> QuarterTraits; + typedef typename Traits::ResScalar ResScalar; typedef typename Traits::LhsPacket LhsPacket; typedef typename Traits::RhsPacket RhsPacket; typedef typename Traits::ResPacket ResPacket; typedef typename Traits::AccPacket AccPacket; + typedef typename Traits::RhsPacketx4 RhsPacketx4; - typedef gebp_traits<RhsScalar,LhsScalar,ConjugateRhs,ConjugateLhs> SwappedTraits; + typedef typename RhsPanelHelper<RhsPacket, RhsPacketx4, 15>::type RhsPanel15; + + typedef gebp_traits<RhsScalar,LhsScalar,ConjugateRhs,ConjugateLhs,Architecture::Target> SwappedTraits; + typedef typename SwappedTraits::ResScalar SResScalar; typedef typename SwappedTraits::LhsPacket SLhsPacket; typedef typename SwappedTraits::RhsPacket SRhsPacket; typedef typename SwappedTraits::ResPacket SResPacket; typedef typename SwappedTraits::AccPacket SAccPacket; + typedef typename HalfTraits::LhsPacket LhsPacketHalf; + typedef typename HalfTraits::RhsPacket RhsPacketHalf; + typedef typename HalfTraits::ResPacket ResPacketHalf; + typedef typename HalfTraits::AccPacket AccPacketHalf; + + typedef typename QuarterTraits::LhsPacket LhsPacketQuarter; + typedef typename QuarterTraits::RhsPacket RhsPacketQuarter; + typedef typename QuarterTraits::ResPacket ResPacketQuarter; + typedef typename QuarterTraits::AccPacket AccPacketQuarter; + typedef typename DataMapper::LinearMapper LinearMapper; enum { Vectorizable = Traits::Vectorizable, LhsProgress = Traits::LhsProgress, + LhsProgressHalf = HalfTraits::LhsProgress, + LhsProgressQuarter = QuarterTraits::LhsProgress, RhsProgress = Traits::RhsProgress, + RhsProgressHalf = HalfTraits::RhsProgress, + RhsProgressQuarter = QuarterTraits::RhsProgress, ResPacketSize = Traits::ResPacketSize }; - - static const bool UseRotatingKernel = - EIGEN_ARCH_ARM && - internal::is_same<LhsScalar, float>::value && - internal::is_same<RhsScalar, float>::value && - internal::is_same<ResScalar, float>::value && - Traits::LhsPacketSize == 4 && - Traits::RhsPacketSize == 4 && - Traits::ResPacketSize == 4; - EIGEN_DONT_INLINE void operator()(const DataMapper& res, const LhsScalar* blockA, const RhsScalar* blockB, Index rows, Index depth, Index cols, ResScalar alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; +template<typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs, +int SwappedLhsProgress = gebp_traits<RhsScalar,LhsScalar,ConjugateRhs,ConjugateLhs,Architecture::Target>::LhsProgress> +struct last_row_process_16_packets +{ + typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs,Architecture::Target> Traits; + typedef gebp_traits<RhsScalar,LhsScalar,ConjugateRhs,ConjugateLhs,Architecture::Target> SwappedTraits; + + typedef typename Traits::ResScalar ResScalar; + typedef typename SwappedTraits::LhsPacket SLhsPacket; + typedef typename SwappedTraits::RhsPacket SRhsPacket; + typedef typename SwappedTraits::ResPacket SResPacket; + typedef typename SwappedTraits::AccPacket SAccPacket; + + EIGEN_STRONG_INLINE void operator()(const DataMapper& res, SwappedTraits &straits, const LhsScalar* blA, + const RhsScalar* blB, Index depth, const Index endk, Index i, Index j2, + ResScalar alpha, SAccPacket &C0) + { + EIGEN_UNUSED_VARIABLE(res); + EIGEN_UNUSED_VARIABLE(straits); + EIGEN_UNUSED_VARIABLE(blA); + EIGEN_UNUSED_VARIABLE(blB); + EIGEN_UNUSED_VARIABLE(depth); + EIGEN_UNUSED_VARIABLE(endk); + EIGEN_UNUSED_VARIABLE(i); + EIGEN_UNUSED_VARIABLE(j2); + EIGEN_UNUSED_VARIABLE(alpha); + EIGEN_UNUSED_VARIABLE(C0); + } +}; + + +template<typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs> +struct last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs, 16> { + typedef gebp_traits<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs,Architecture::Target> Traits; + typedef gebp_traits<RhsScalar,LhsScalar,ConjugateRhs,ConjugateLhs,Architecture::Target> SwappedTraits; + + typedef typename Traits::ResScalar ResScalar; + typedef typename SwappedTraits::LhsPacket SLhsPacket; + typedef typename SwappedTraits::RhsPacket SRhsPacket; + typedef typename SwappedTraits::ResPacket SResPacket; + typedef typename SwappedTraits::AccPacket SAccPacket; + + EIGEN_STRONG_INLINE void operator()(const DataMapper& res, SwappedTraits &straits, const LhsScalar* blA, + const RhsScalar* blB, Index depth, const Index endk, Index i, Index j2, + ResScalar alpha, SAccPacket &C0) + { + typedef typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half SResPacketQuarter; + typedef typename unpacket_traits<typename unpacket_traits<SLhsPacket>::half>::half SLhsPacketQuarter; + typedef typename unpacket_traits<typename unpacket_traits<SRhsPacket>::half>::half SRhsPacketQuarter; + typedef typename unpacket_traits<typename unpacket_traits<SAccPacket>::half>::half SAccPacketQuarter; + + SResPacketQuarter R = res.template gatherPacket<SResPacketQuarter>(i, j2); + SResPacketQuarter alphav = pset1<SResPacketQuarter>(alpha); + + if (depth - endk > 0) + { + // We have to handle the last row(s) of the rhs, which + // correspond to a half-packet + SAccPacketQuarter c0 = predux_half_dowto4(predux_half_dowto4(C0)); + + for (Index kk = endk; kk < depth; kk++) + { + SLhsPacketQuarter a0; + SRhsPacketQuarter b0; + straits.loadLhsUnaligned(blB, a0); + straits.loadRhs(blA, b0); + straits.madd(a0,b0,c0,b0, fix<0>); + blB += SwappedTraits::LhsProgress/4; + blA += 1; + } + straits.acc(c0, alphav, R); + } + else + { + straits.acc(predux_half_dowto4(predux_half_dowto4(C0)), alphav, R); + } + res.scatterPacket(i, j2, R); + } +}; + +template<int nr, Index LhsProgress, Index RhsProgress, typename LhsScalar, typename RhsScalar, typename ResScalar, typename AccPacket, typename LhsPacket, typename RhsPacket, typename ResPacket, typename GEBPTraits, typename LinearMapper, typename DataMapper> +struct lhs_process_one_packet +{ + typedef typename GEBPTraits::RhsPacketx4 RhsPacketx4; + + EIGEN_STRONG_INLINE void peeled_kc_onestep(Index K, const LhsScalar* blA, const RhsScalar* blB, GEBPTraits traits, LhsPacket *A0, RhsPacketx4 *rhs_panel, RhsPacket *T0, AccPacket *C0, AccPacket *C1, AccPacket *C2, AccPacket *C3) + { + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 1X4"); + EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); + traits.loadLhs(&blA[(0+1*K)*LhsProgress], *A0); + traits.loadRhs(&blB[(0+4*K)*RhsProgress], *rhs_panel); + traits.madd(*A0, *rhs_panel, *C0, *T0, fix<0>); + traits.madd(*A0, *rhs_panel, *C1, *T0, fix<1>); + traits.madd(*A0, *rhs_panel, *C2, *T0, fix<2>); + traits.madd(*A0, *rhs_panel, *C3, *T0, fix<3>); + #if EIGEN_GNUC_AT_LEAST(6,0) && defined(EIGEN_VECTORIZE_SSE) + __asm__ ("" : "+x,m" (*A0)); + #endif + EIGEN_ASM_COMMENT("end step of gebp micro kernel 1X4"); + } + + EIGEN_STRONG_INLINE void operator()( + const DataMapper& res, const LhsScalar* blockA, const RhsScalar* blockB, ResScalar alpha, + Index peelStart, Index peelEnd, Index strideA, Index strideB, Index offsetA, Index offsetB, + int prefetch_res_offset, Index peeled_kc, Index pk, Index cols, Index depth, Index packet_cols4) + { + GEBPTraits traits; + + // loops on each largest micro horizontal panel of lhs + // (LhsProgress x depth) + for(Index i=peelStart; i<peelEnd; i+=LhsProgress) + { + // loops on each largest micro vertical panel of rhs (depth * nr) + for(Index j2=0; j2<packet_cols4; j2+=nr) + { + // We select a LhsProgress x nr micro block of res + // which is entirely stored into 1 x nr registers. + + const LhsScalar* blA = &blockA[i*strideA+offsetA*(LhsProgress)]; + prefetch(&blA[0]); + + // gets res block as register + AccPacket C0, C1, C2, C3; + traits.initAcc(C0); + traits.initAcc(C1); + traits.initAcc(C2); + traits.initAcc(C3); + // To improve instruction pipelining, let's double the accumulation registers: + // even k will accumulate in C*, while odd k will accumulate in D*. + // This trick is crutial to get good performance with FMA, otherwise it is + // actually faster to perform separated MUL+ADD because of a naturally + // better instruction-level parallelism. + AccPacket D0, D1, D2, D3; + traits.initAcc(D0); + traits.initAcc(D1); + traits.initAcc(D2); + traits.initAcc(D3); + + LinearMapper r0 = res.getLinearMapper(i, j2 + 0); + LinearMapper r1 = res.getLinearMapper(i, j2 + 1); + LinearMapper r2 = res.getLinearMapper(i, j2 + 2); + LinearMapper r3 = res.getLinearMapper(i, j2 + 3); + + r0.prefetch(prefetch_res_offset); + r1.prefetch(prefetch_res_offset); + r2.prefetch(prefetch_res_offset); + r3.prefetch(prefetch_res_offset); + + // performs "inner" products + const RhsScalar* blB = &blockB[j2*strideB+offsetB*nr]; + prefetch(&blB[0]); + LhsPacket A0, A1; + + for(Index k=0; k<peeled_kc; k+=pk) + { + EIGEN_ASM_COMMENT("begin gebp micro kernel 1/half/quarterX4"); + RhsPacketx4 rhs_panel; + RhsPacket T0; + + internal::prefetch(blB+(48+0)); + peeled_kc_onestep(0, blA, blB, traits, &A0, &rhs_panel, &T0, &C0, &C1, &C2, &C3); + peeled_kc_onestep(1, blA, blB, traits, &A1, &rhs_panel, &T0, &D0, &D1, &D2, &D3); + peeled_kc_onestep(2, blA, blB, traits, &A0, &rhs_panel, &T0, &C0, &C1, &C2, &C3); + peeled_kc_onestep(3, blA, blB, traits, &A1, &rhs_panel, &T0, &D0, &D1, &D2, &D3); + internal::prefetch(blB+(48+16)); + peeled_kc_onestep(4, blA, blB, traits, &A0, &rhs_panel, &T0, &C0, &C1, &C2, &C3); + peeled_kc_onestep(5, blA, blB, traits, &A1, &rhs_panel, &T0, &D0, &D1, &D2, &D3); + peeled_kc_onestep(6, blA, blB, traits, &A0, &rhs_panel, &T0, &C0, &C1, &C2, &C3); + peeled_kc_onestep(7, blA, blB, traits, &A1, &rhs_panel, &T0, &D0, &D1, &D2, &D3); + + blB += pk*4*RhsProgress; + blA += pk*LhsProgress; + + EIGEN_ASM_COMMENT("end gebp micro kernel 1/half/quarterX4"); + } + C0 = padd(C0,D0); + C1 = padd(C1,D1); + C2 = padd(C2,D2); + C3 = padd(C3,D3); + + // process remaining peeled loop + for(Index k=peeled_kc; k<depth; k++) + { + RhsPacketx4 rhs_panel; + RhsPacket T0; + peeled_kc_onestep(0, blA, blB, traits, &A0, &rhs_panel, &T0, &C0, &C1, &C2, &C3); + blB += 4*RhsProgress; + blA += LhsProgress; + } + + ResPacket R0, R1; + ResPacket alphav = pset1<ResPacket>(alpha); + + R0 = r0.template loadPacket<ResPacket>(0); + R1 = r1.template loadPacket<ResPacket>(0); + traits.acc(C0, alphav, R0); + traits.acc(C1, alphav, R1); + r0.storePacket(0, R0); + r1.storePacket(0, R1); + + R0 = r2.template loadPacket<ResPacket>(0); + R1 = r3.template loadPacket<ResPacket>(0); + traits.acc(C2, alphav, R0); + traits.acc(C3, alphav, R1); + r2.storePacket(0, R0); + r3.storePacket(0, R1); + } + + // Deal with remaining columns of the rhs + for(Index j2=packet_cols4; j2<cols; j2++) + { + // One column at a time + const LhsScalar* blA = &blockA[i*strideA+offsetA*(LhsProgress)]; + prefetch(&blA[0]); + + // gets res block as register + AccPacket C0; + traits.initAcc(C0); + + LinearMapper r0 = res.getLinearMapper(i, j2); + + // performs "inner" products + const RhsScalar* blB = &blockB[j2*strideB+offsetB]; + LhsPacket A0; + + for(Index k= 0; k<peeled_kc; k+=pk) + { + EIGEN_ASM_COMMENT("begin gebp micro kernel 1/half/quarterX1"); + RhsPacket B_0; + +#define EIGEN_GEBGP_ONESTEP(K) \ + do { \ + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 1/half/quarterX1"); \ + EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ + /* FIXME: why unaligned???? */ \ + traits.loadLhsUnaligned(&blA[(0+1*K)*LhsProgress], A0); \ + traits.loadRhs(&blB[(0+K)*RhsProgress], B_0); \ + traits.madd(A0, B_0, C0, B_0, fix<0>); \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 1/half/quarterX1"); \ + } while(false); + + EIGEN_GEBGP_ONESTEP(0); + EIGEN_GEBGP_ONESTEP(1); + EIGEN_GEBGP_ONESTEP(2); + EIGEN_GEBGP_ONESTEP(3); + EIGEN_GEBGP_ONESTEP(4); + EIGEN_GEBGP_ONESTEP(5); + EIGEN_GEBGP_ONESTEP(6); + EIGEN_GEBGP_ONESTEP(7); + + blB += pk*RhsProgress; + blA += pk*LhsProgress; + + EIGEN_ASM_COMMENT("end gebp micro kernel 1/half/quarterX1"); + } + + // process remaining peeled loop + for(Index k=peeled_kc; k<depth; k++) + { + RhsPacket B_0; + EIGEN_GEBGP_ONESTEP(0); + blB += RhsProgress; + blA += LhsProgress; + } +#undef EIGEN_GEBGP_ONESTEP + ResPacket R0; + ResPacket alphav = pset1<ResPacket>(alpha); + R0 = r0.template loadPacket<ResPacket>(0); + traits.acc(C0, alphav, R0); + r0.storePacket(0, R0); + } + } + } +}; + +template<int nr, Index LhsProgress, Index RhsProgress, typename LhsScalar, typename RhsScalar, typename ResScalar, typename AccPacket, typename LhsPacket, typename RhsPacket, typename ResPacket, typename GEBPTraits, typename LinearMapper, typename DataMapper> +struct lhs_process_fraction_of_packet : lhs_process_one_packet<nr, LhsProgress, RhsProgress, LhsScalar, RhsScalar, ResScalar, AccPacket, LhsPacket, RhsPacket, ResPacket, GEBPTraits, LinearMapper, DataMapper> +{ + +EIGEN_STRONG_INLINE void peeled_kc_onestep(Index K, const LhsScalar* blA, const RhsScalar* blB, GEBPTraits traits, LhsPacket *A0, RhsPacket *B_0, RhsPacket *B1, RhsPacket *B2, RhsPacket *B3, AccPacket *C0, AccPacket *C1, AccPacket *C2, AccPacket *C3) + { + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 1X4"); + EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); + traits.loadLhsUnaligned(&blA[(0+1*K)*(LhsProgress)], *A0); + traits.broadcastRhs(&blB[(0+4*K)*RhsProgress], *B_0, *B1, *B2, *B3); + traits.madd(*A0, *B_0, *C0, *B_0); + traits.madd(*A0, *B1, *C1, *B1); + traits.madd(*A0, *B2, *C2, *B2); + traits.madd(*A0, *B3, *C3, *B3); + EIGEN_ASM_COMMENT("end step of gebp micro kernel 1X4"); + } +}; + template<typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs> EIGEN_DONT_INLINE void gebp_kernel<LhsScalar,RhsScalar,Index,DataMapper,mr,nr,ConjugateLhs,ConjugateRhs> @@ -998,19 +1561,19 @@ Index packet_cols4 = nr>=4 ? (cols/4) * 4 : 0; const Index peeled_mc3 = mr>=3*Traits::LhsProgress ? (rows/(3*LhsProgress))*(3*LhsProgress) : 0; const Index peeled_mc2 = mr>=2*Traits::LhsProgress ? peeled_mc3+((rows-peeled_mc3)/(2*LhsProgress))*(2*LhsProgress) : 0; - const Index peeled_mc1 = mr>=1*Traits::LhsProgress ? (rows/(1*LhsProgress))*(1*LhsProgress) : 0; + const Index peeled_mc1 = mr>=1*Traits::LhsProgress ? peeled_mc2+((rows-peeled_mc2)/(1*LhsProgress))*(1*LhsProgress) : 0; + const Index peeled_mc_half = mr>=LhsProgressHalf ? peeled_mc1+((rows-peeled_mc1)/(LhsProgressHalf))*(LhsProgressHalf) : 0; + const Index peeled_mc_quarter = mr>=LhsProgressQuarter ? peeled_mc_half+((rows-peeled_mc_half)/(LhsProgressQuarter))*(LhsProgressQuarter) : 0; enum { pk = 8 }; // NOTE Such a large peeling factor is important for large matrices (~ +5% when >1000 on Haswell) const Index peeled_kc = depth & ~(pk-1); - const Index prefetch_res_offset = 32/sizeof(ResScalar); + const int prefetch_res_offset = 32/sizeof(ResScalar); // const Index depth2 = depth & ~1; //---------- Process 3 * LhsProgress rows at once ---------- // This corresponds to 3*LhsProgress x nr register blocks. // Usually, make sense only with FMA if(mr>=3*Traits::LhsProgress) - { - PossiblyRotatingKernelHelper<gebp_kernel> possiblyRotatingKernelHelper(traits); - + { // Here, the general idea is to loop on each largest micro horizontal panel of the lhs (3*Traits::LhsProgress x depth) // and on each largest micro vertical panel of the rhs (depth * nr). // Blocking sizes, i.e., 'depth' has been computed so that the micro horizontal panel of the lhs fit in L1. @@ -1061,36 +1624,48 @@ for(Index k=0; k<peeled_kc; k+=pk) { EIGEN_ASM_COMMENT("begin gebp micro kernel 3pX4"); - RhsPacket B_0, T0; + // 15 registers are taken (12 for acc, 2 for lhs). + RhsPanel15 rhs_panel; + RhsPacket T0; LhsPacket A2; - -#define EIGEN_GEBP_ONESTEP(K) \ - do { \ - EIGEN_ASM_COMMENT("begin step of gebp micro kernel 3pX4"); \ + #if EIGEN_COMP_GNUC_STRICT && EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && !(EIGEN_GNUC_AT_LEAST(9,0)) + // see http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1633 + // without this workaround A0, A1, and A2 are loaded in the same register, + // which is not good for pipelining + #define EIGEN_GEBP_3PX4_REGISTER_ALLOC_WORKAROUND __asm__ ("" : "+w,m" (A0), "+w,m" (A1), "+w,m" (A2)); + #else + #define EIGEN_GEBP_3PX4_REGISTER_ALLOC_WORKAROUND + #endif +#define EIGEN_GEBP_ONESTEP(K) \ + do { \ + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 3pX4"); \ EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ - internal::prefetch(blA+(3*K+16)*LhsProgress); \ - if (EIGEN_ARCH_ARM) internal::prefetch(blB+(4*K+16)*RhsProgress); /* Bug 953 */ \ - traits.loadLhs(&blA[(0+3*K)*LhsProgress], A0); \ - traits.loadLhs(&blA[(1+3*K)*LhsProgress], A1); \ - traits.loadLhs(&blA[(2+3*K)*LhsProgress], A2); \ - possiblyRotatingKernelHelper.template loadOrRotateRhs<K, 0>(B_0, blB); \ - traits.madd(A0, B_0, C0, T0); \ - traits.madd(A1, B_0, C4, T0); \ - traits.madd(A2, B_0, C8, B_0); \ - possiblyRotatingKernelHelper.template loadOrRotateRhs<K, 1>(B_0, blB); \ - traits.madd(A0, B_0, C1, T0); \ - traits.madd(A1, B_0, C5, T0); \ - traits.madd(A2, B_0, C9, B_0); \ - possiblyRotatingKernelHelper.template loadOrRotateRhs<K, 2>(B_0, blB); \ - traits.madd(A0, B_0, C2, T0); \ - traits.madd(A1, B_0, C6, T0); \ - traits.madd(A2, B_0, C10, B_0); \ - possiblyRotatingKernelHelper.template loadOrRotateRhs<K, 3>(B_0, blB); \ - traits.madd(A0, B_0, C3 , T0); \ - traits.madd(A1, B_0, C7, T0); \ - traits.madd(A2, B_0, C11, B_0); \ - EIGEN_ASM_COMMENT("end step of gebp micro kernel 3pX4"); \ - } while(false) + internal::prefetch(blA + (3 * K + 16) * LhsProgress); \ + if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \ + internal::prefetch(blB + (4 * K + 16) * RhsProgress); \ + } /* Bug 953 */ \ + traits.loadLhs(&blA[(0 + 3 * K) * LhsProgress], A0); \ + traits.loadLhs(&blA[(1 + 3 * K) * LhsProgress], A1); \ + traits.loadLhs(&blA[(2 + 3 * K) * LhsProgress], A2); \ + EIGEN_GEBP_3PX4_REGISTER_ALLOC_WORKAROUND \ + traits.loadRhs(blB + (0+4*K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C0, T0, fix<0>); \ + traits.madd(A1, rhs_panel, C4, T0, fix<0>); \ + traits.madd(A2, rhs_panel, C8, T0, fix<0>); \ + traits.updateRhs(blB + (1+4*K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C1, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C5, T0, fix<1>); \ + traits.madd(A2, rhs_panel, C9, T0, fix<1>); \ + traits.updateRhs(blB + (2+4*K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C2, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C6, T0, fix<2>); \ + traits.madd(A2, rhs_panel, C10, T0, fix<2>); \ + traits.updateRhs(blB + (3+4*K) * Traits::RhsProgress, rhs_panel); \ + traits.madd(A0, rhs_panel, C3, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C7, T0, fix<3>); \ + traits.madd(A2, rhs_panel, C11, T0, fix<3>); \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 3pX4"); \ + } while (false) internal::prefetch(blB); EIGEN_GEBP_ONESTEP(0); @@ -1110,7 +1685,8 @@ // process remaining peeled loop for(Index k=peeled_kc; k<depth; k++) { - RhsPacket B_0, T0; + RhsPanel15 rhs_panel; + RhsPacket T0; LhsPacket A2; EIGEN_GEBP_ONESTEP(0); blB += 4*RhsProgress; @@ -1119,16 +1695,12 @@ #undef EIGEN_GEBP_ONESTEP - possiblyRotatingKernelHelper.unrotateResult(C0, C1, C2, C3); - possiblyRotatingKernelHelper.unrotateResult(C4, C5, C6, C7); - possiblyRotatingKernelHelper.unrotateResult(C8, C9, C10, C11); - ResPacket R0, R1, R2; ResPacket alphav = pset1<ResPacket>(alpha); - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - R1 = r0.loadPacket(1 * Traits::ResPacketSize); - R2 = r0.loadPacket(2 * Traits::ResPacketSize); + R0 = r0.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r0.template loadPacket<ResPacket>(2 * Traits::ResPacketSize); traits.acc(C0, alphav, R0); traits.acc(C4, alphav, R1); traits.acc(C8, alphav, R2); @@ -1136,9 +1708,9 @@ r0.storePacket(1 * Traits::ResPacketSize, R1); r0.storePacket(2 * Traits::ResPacketSize, R2); - R0 = r1.loadPacket(0 * Traits::ResPacketSize); - R1 = r1.loadPacket(1 * Traits::ResPacketSize); - R2 = r1.loadPacket(2 * Traits::ResPacketSize); + R0 = r1.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r1.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r1.template loadPacket<ResPacket>(2 * Traits::ResPacketSize); traits.acc(C1, alphav, R0); traits.acc(C5, alphav, R1); traits.acc(C9, alphav, R2); @@ -1146,9 +1718,9 @@ r1.storePacket(1 * Traits::ResPacketSize, R1); r1.storePacket(2 * Traits::ResPacketSize, R2); - R0 = r2.loadPacket(0 * Traits::ResPacketSize); - R1 = r2.loadPacket(1 * Traits::ResPacketSize); - R2 = r2.loadPacket(2 * Traits::ResPacketSize); + R0 = r2.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r2.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r2.template loadPacket<ResPacket>(2 * Traits::ResPacketSize); traits.acc(C2, alphav, R0); traits.acc(C6, alphav, R1); traits.acc(C10, alphav, R2); @@ -1156,9 +1728,9 @@ r2.storePacket(1 * Traits::ResPacketSize, R1); r2.storePacket(2 * Traits::ResPacketSize, R2); - R0 = r3.loadPacket(0 * Traits::ResPacketSize); - R1 = r3.loadPacket(1 * Traits::ResPacketSize); - R2 = r3.loadPacket(2 * Traits::ResPacketSize); + R0 = r3.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r3.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r3.template loadPacket<ResPacket>(2 * Traits::ResPacketSize); traits.acc(C3, alphav, R0); traits.acc(C7, alphav, R1); traits.acc(C11, alphav, R2); @@ -1194,20 +1766,20 @@ { EIGEN_ASM_COMMENT("begin gebp micro kernel 3pX1"); RhsPacket B_0; -#define EIGEN_GEBGP_ONESTEP(K) \ - do { \ - EIGEN_ASM_COMMENT("begin step of gebp micro kernel 3pX1"); \ +#define EIGEN_GEBGP_ONESTEP(K) \ + do { \ + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 3pX1"); \ EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ - traits.loadLhs(&blA[(0+3*K)*LhsProgress], A0); \ - traits.loadLhs(&blA[(1+3*K)*LhsProgress], A1); \ - traits.loadLhs(&blA[(2+3*K)*LhsProgress], A2); \ - traits.loadRhs(&blB[(0+K)*RhsProgress], B_0); \ - traits.madd(A0, B_0, C0, B_0); \ - traits.madd(A1, B_0, C4, B_0); \ - traits.madd(A2, B_0, C8, B_0); \ - EIGEN_ASM_COMMENT("end step of gebp micro kernel 3pX1"); \ - } while(false) - + traits.loadLhs(&blA[(0 + 3 * K) * LhsProgress], A0); \ + traits.loadLhs(&blA[(1 + 3 * K) * LhsProgress], A1); \ + traits.loadLhs(&blA[(2 + 3 * K) * LhsProgress], A2); \ + traits.loadRhs(&blB[(0 + K) * RhsProgress], B_0); \ + traits.madd(A0, B_0, C0, B_0, fix<0>); \ + traits.madd(A1, B_0, C4, B_0, fix<0>); \ + traits.madd(A2, B_0, C8, B_0, fix<0>); \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 3pX1"); \ + } while (false) + EIGEN_GEBGP_ONESTEP(0); EIGEN_GEBGP_ONESTEP(1); EIGEN_GEBGP_ONESTEP(2); @@ -1235,9 +1807,9 @@ ResPacket R0, R1, R2; ResPacket alphav = pset1<ResPacket>(alpha); - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - R1 = r0.loadPacket(1 * Traits::ResPacketSize); - R2 = r0.loadPacket(2 * Traits::ResPacketSize); + R0 = r0.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r0.template loadPacket<ResPacket>(2 * Traits::ResPacketSize); traits.acc(C0, alphav, R0); traits.acc(C4, alphav, R1); traits.acc(C8, alphav, R2); @@ -1296,26 +1868,34 @@ for(Index k=0; k<peeled_kc; k+=pk) { EIGEN_ASM_COMMENT("begin gebp micro kernel 2pX4"); - RhsPacket B_0, B1, B2, B3, T0; + RhsPacketx4 rhs_panel; + RhsPacket T0; - #define EIGEN_GEBGP_ONESTEP(K) \ - do { \ - EIGEN_ASM_COMMENT("begin step of gebp micro kernel 2pX4"); \ - EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ - traits.loadLhs(&blA[(0+2*K)*LhsProgress], A0); \ - traits.loadLhs(&blA[(1+2*K)*LhsProgress], A1); \ - traits.broadcastRhs(&blB[(0+4*K)*RhsProgress], B_0, B1, B2, B3); \ - traits.madd(A0, B_0, C0, T0); \ - traits.madd(A1, B_0, C4, B_0); \ - traits.madd(A0, B1, C1, T0); \ - traits.madd(A1, B1, C5, B1); \ - traits.madd(A0, B2, C2, T0); \ - traits.madd(A1, B2, C6, B2); \ - traits.madd(A0, B3, C3, T0); \ - traits.madd(A1, B3, C7, B3); \ - EIGEN_ASM_COMMENT("end step of gebp micro kernel 2pX4"); \ - } while(false) - + // NOTE: the begin/end asm comments below work around bug 935! + // but they are not enough for gcc>=6 without FMA (bug 1637) + #if EIGEN_GNUC_AT_LEAST(6,0) && defined(EIGEN_VECTORIZE_SSE) + #define EIGEN_GEBP_2PX4_SPILLING_WORKAROUND __asm__ ("" : [a0] "+x,m" (A0),[a1] "+x,m" (A1)); + #else + #define EIGEN_GEBP_2PX4_SPILLING_WORKAROUND + #endif +#define EIGEN_GEBGP_ONESTEP(K) \ + do { \ + EIGEN_ASM_COMMENT("begin step of gebp micro kernel 2pX4"); \ + traits.loadLhs(&blA[(0 + 2 * K) * LhsProgress], A0); \ + traits.loadLhs(&blA[(1 + 2 * K) * LhsProgress], A1); \ + traits.loadRhs(&blB[(0 + 4 * K) * RhsProgress], rhs_panel); \ + traits.madd(A0, rhs_panel, C0, T0, fix<0>); \ + traits.madd(A1, rhs_panel, C4, T0, fix<0>); \ + traits.madd(A0, rhs_panel, C1, T0, fix<1>); \ + traits.madd(A1, rhs_panel, C5, T0, fix<1>); \ + traits.madd(A0, rhs_panel, C2, T0, fix<2>); \ + traits.madd(A1, rhs_panel, C6, T0, fix<2>); \ + traits.madd(A0, rhs_panel, C3, T0, fix<3>); \ + traits.madd(A1, rhs_panel, C7, T0, fix<3>); \ + EIGEN_GEBP_2PX4_SPILLING_WORKAROUND \ + EIGEN_ASM_COMMENT("end step of gebp micro kernel 2pX4"); \ + } while (false) + internal::prefetch(blB+(48+0)); EIGEN_GEBGP_ONESTEP(0); EIGEN_GEBGP_ONESTEP(1); @@ -1335,7 +1915,8 @@ // process remaining peeled loop for(Index k=peeled_kc; k<depth; k++) { - RhsPacket B_0, B1, B2, B3, T0; + RhsPacketx4 rhs_panel; + RhsPacket T0; EIGEN_GEBGP_ONESTEP(0); blB += 4*RhsProgress; blA += 2*Traits::LhsProgress; @@ -1345,10 +1926,10 @@ ResPacket R0, R1, R2, R3; ResPacket alphav = pset1<ResPacket>(alpha); - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - R1 = r0.loadPacket(1 * Traits::ResPacketSize); - R2 = r1.loadPacket(0 * Traits::ResPacketSize); - R3 = r1.loadPacket(1 * Traits::ResPacketSize); + R0 = r0.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r1.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R3 = r1.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); traits.acc(C0, alphav, R0); traits.acc(C4, alphav, R1); traits.acc(C1, alphav, R2); @@ -1358,10 +1939,10 @@ r1.storePacket(0 * Traits::ResPacketSize, R2); r1.storePacket(1 * Traits::ResPacketSize, R3); - R0 = r2.loadPacket(0 * Traits::ResPacketSize); - R1 = r2.loadPacket(1 * Traits::ResPacketSize); - R2 = r3.loadPacket(0 * Traits::ResPacketSize); - R3 = r3.loadPacket(1 * Traits::ResPacketSize); + R0 = r2.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r2.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); + R2 = r3.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R3 = r3.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); traits.acc(C2, alphav, R0); traits.acc(C6, alphav, R1); traits.acc(C3, alphav, R2); @@ -1406,8 +1987,8 @@ traits.loadLhs(&blA[(0+2*K)*LhsProgress], A0); \ traits.loadLhs(&blA[(1+2*K)*LhsProgress], A1); \ traits.loadRhs(&blB[(0+K)*RhsProgress], B_0); \ - traits.madd(A0, B_0, C0, B1); \ - traits.madd(A1, B_0, C4, B_0); \ + traits.madd(A0, B_0, C0, B1, fix<0>); \ + traits.madd(A1, B_0, C4, B_0, fix<0>); \ EIGEN_ASM_COMMENT("end step of gebp micro kernel 2pX1"); \ } while(false) @@ -1438,8 +2019,8 @@ ResPacket R0, R1; ResPacket alphav = pset1<ResPacket>(alpha); - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - R1 = r0.loadPacket(1 * Traits::ResPacketSize); + R0 = r0.template loadPacket<ResPacket>(0 * Traits::ResPacketSize); + R1 = r0.template loadPacket<ResPacket>(1 * Traits::ResPacketSize); traits.acc(C0, alphav, R0); traits.acc(C4, alphav, R1); r0.storePacket(0 * Traits::ResPacketSize, R0); @@ -1451,182 +2032,44 @@ //---------- Process 1 * LhsProgress rows at once ---------- if(mr>=1*Traits::LhsProgress) { - // loops on each largest micro horizontal panel of lhs (1*LhsProgress x depth) - for(Index i=peeled_mc2; i<peeled_mc1; i+=1*LhsProgress) - { - // loops on each largest micro vertical panel of rhs (depth * nr) - for(Index j2=0; j2<packet_cols4; j2+=nr) - { - // We select a 1*Traits::LhsProgress x nr micro block of res which is entirely - // stored into 1 x nr registers. - - const LhsScalar* blA = &blockA[i*strideA+offsetA*(1*Traits::LhsProgress)]; - prefetch(&blA[0]); - - // gets res block as register - AccPacket C0, C1, C2, C3; - traits.initAcc(C0); - traits.initAcc(C1); - traits.initAcc(C2); - traits.initAcc(C3); - - LinearMapper r0 = res.getLinearMapper(i, j2 + 0); - LinearMapper r1 = res.getLinearMapper(i, j2 + 1); - LinearMapper r2 = res.getLinearMapper(i, j2 + 2); - LinearMapper r3 = res.getLinearMapper(i, j2 + 3); - - r0.prefetch(prefetch_res_offset); - r1.prefetch(prefetch_res_offset); - r2.prefetch(prefetch_res_offset); - r3.prefetch(prefetch_res_offset); - - // performs "inner" products - const RhsScalar* blB = &blockB[j2*strideB+offsetB*nr]; - prefetch(&blB[0]); - LhsPacket A0; - - for(Index k=0; k<peeled_kc; k+=pk) - { - EIGEN_ASM_COMMENT("begin gebp micro kernel 1pX4"); - RhsPacket B_0, B1, B2, B3; - -#define EIGEN_GEBGP_ONESTEP(K) \ - do { \ - EIGEN_ASM_COMMENT("begin step of gebp micro kernel 1pX4"); \ - EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ - traits.loadLhs(&blA[(0+1*K)*LhsProgress], A0); \ - traits.broadcastRhs(&blB[(0+4*K)*RhsProgress], B_0, B1, B2, B3); \ - traits.madd(A0, B_0, C0, B_0); \ - traits.madd(A0, B1, C1, B1); \ - traits.madd(A0, B2, C2, B2); \ - traits.madd(A0, B3, C3, B3); \ - EIGEN_ASM_COMMENT("end step of gebp micro kernel 1pX4"); \ - } while(false) - - internal::prefetch(blB+(48+0)); - EIGEN_GEBGP_ONESTEP(0); - EIGEN_GEBGP_ONESTEP(1); - EIGEN_GEBGP_ONESTEP(2); - EIGEN_GEBGP_ONESTEP(3); - internal::prefetch(blB+(48+16)); - EIGEN_GEBGP_ONESTEP(4); - EIGEN_GEBGP_ONESTEP(5); - EIGEN_GEBGP_ONESTEP(6); - EIGEN_GEBGP_ONESTEP(7); - - blB += pk*4*RhsProgress; - blA += pk*1*LhsProgress; - - EIGEN_ASM_COMMENT("end gebp micro kernel 1pX4"); - } - // process remaining peeled loop - for(Index k=peeled_kc; k<depth; k++) - { - RhsPacket B_0, B1, B2, B3; - EIGEN_GEBGP_ONESTEP(0); - blB += 4*RhsProgress; - blA += 1*LhsProgress; - } -#undef EIGEN_GEBGP_ONESTEP - - ResPacket R0, R1; - ResPacket alphav = pset1<ResPacket>(alpha); - - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - R1 = r1.loadPacket(0 * Traits::ResPacketSize); - traits.acc(C0, alphav, R0); - traits.acc(C1, alphav, R1); - r0.storePacket(0 * Traits::ResPacketSize, R0); - r1.storePacket(0 * Traits::ResPacketSize, R1); - - R0 = r2.loadPacket(0 * Traits::ResPacketSize); - R1 = r3.loadPacket(0 * Traits::ResPacketSize); - traits.acc(C2, alphav, R0); - traits.acc(C3, alphav, R1); - r2.storePacket(0 * Traits::ResPacketSize, R0); - r3.storePacket(0 * Traits::ResPacketSize, R1); - } - - // Deal with remaining columns of the rhs - for(Index j2=packet_cols4; j2<cols; j2++) - { - // One column at a time - const LhsScalar* blA = &blockA[i*strideA+offsetA*(1*Traits::LhsProgress)]; - prefetch(&blA[0]); - - // gets res block as register - AccPacket C0; - traits.initAcc(C0); - - LinearMapper r0 = res.getLinearMapper(i, j2); - - // performs "inner" products - const RhsScalar* blB = &blockB[j2*strideB+offsetB]; - LhsPacket A0; - - for(Index k=0; k<peeled_kc; k+=pk) - { - EIGEN_ASM_COMMENT("begin gebp micro kernel 1pX1"); - RhsPacket B_0; - -#define EIGEN_GEBGP_ONESTEP(K) \ - do { \ - EIGEN_ASM_COMMENT("begin step of gebp micro kernel 1pX1"); \ - EIGEN_ASM_COMMENT("Note: these asm comments work around bug 935!"); \ - traits.loadLhs(&blA[(0+1*K)*LhsProgress], A0); \ - traits.loadRhs(&blB[(0+K)*RhsProgress], B_0); \ - traits.madd(A0, B_0, C0, B_0); \ - EIGEN_ASM_COMMENT("end step of gebp micro kernel 1pX1"); \ - } while(false); - - EIGEN_GEBGP_ONESTEP(0); - EIGEN_GEBGP_ONESTEP(1); - EIGEN_GEBGP_ONESTEP(2); - EIGEN_GEBGP_ONESTEP(3); - EIGEN_GEBGP_ONESTEP(4); - EIGEN_GEBGP_ONESTEP(5); - EIGEN_GEBGP_ONESTEP(6); - EIGEN_GEBGP_ONESTEP(7); - - blB += pk*RhsProgress; - blA += pk*1*Traits::LhsProgress; - - EIGEN_ASM_COMMENT("end gebp micro kernel 1pX1"); - } - - // process remaining peeled loop - for(Index k=peeled_kc; k<depth; k++) - { - RhsPacket B_0; - EIGEN_GEBGP_ONESTEP(0); - blB += RhsProgress; - blA += 1*Traits::LhsProgress; - } -#undef EIGEN_GEBGP_ONESTEP - ResPacket R0; - ResPacket alphav = pset1<ResPacket>(alpha); - R0 = r0.loadPacket(0 * Traits::ResPacketSize); - traits.acc(C0, alphav, R0); - r0.storePacket(0 * Traits::ResPacketSize, R0); - } - } + lhs_process_one_packet<nr, LhsProgress, RhsProgress, LhsScalar, RhsScalar, ResScalar, AccPacket, LhsPacket, RhsPacket, ResPacket, Traits, LinearMapper, DataMapper> p; + p(res, blockA, blockB, alpha, peeled_mc2, peeled_mc1, strideA, strideB, offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, cols, depth, packet_cols4); + } + //---------- Process LhsProgressHalf rows at once ---------- + if((LhsProgressHalf < LhsProgress) && mr>=LhsProgressHalf) + { + lhs_process_fraction_of_packet<nr, LhsProgressHalf, RhsProgressHalf, LhsScalar, RhsScalar, ResScalar, AccPacketHalf, LhsPacketHalf, RhsPacketHalf, ResPacketHalf, HalfTraits, LinearMapper, DataMapper> p; + p(res, blockA, blockB, alpha, peeled_mc1, peeled_mc_half, strideA, strideB, offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, cols, depth, packet_cols4); + } + //---------- Process LhsProgressQuarter rows at once ---------- + if((LhsProgressQuarter < LhsProgressHalf) && mr>=LhsProgressQuarter) + { + lhs_process_fraction_of_packet<nr, LhsProgressQuarter, RhsProgressQuarter, LhsScalar, RhsScalar, ResScalar, AccPacketQuarter, LhsPacketQuarter, RhsPacketQuarter, ResPacketQuarter, QuarterTraits, LinearMapper, DataMapper> p; + p(res, blockA, blockB, alpha, peeled_mc_half, peeled_mc_quarter, strideA, strideB, offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, cols, depth, packet_cols4); } //---------- Process remaining rows, 1 at once ---------- - if(peeled_mc1<rows) + if(peeled_mc_quarter<rows) { // loop on each panel of the rhs for(Index j2=0; j2<packet_cols4; j2+=nr) { // loop on each row of the lhs (1*LhsProgress x depth) - for(Index i=peeled_mc1; i<rows; i+=1) + for(Index i=peeled_mc_quarter; i<rows; i+=1) { const LhsScalar* blA = &blockA[i*strideA+offsetA]; prefetch(&blA[0]); const RhsScalar* blB = &blockB[j2*strideB+offsetB*nr]; - if( (SwappedTraits::LhsProgress % 4)==0 ) + // If LhsProgress is 8 or 16, it assumes that there is a + // half or quarter packet, respectively, of the same size as + // nr (which is currently 4) for the return type. + const int SResPacketHalfSize = unpacket_traits<typename unpacket_traits<SResPacket>::half>::size; + const int SResPacketQuarterSize = unpacket_traits<typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half>::size; + if ((SwappedTraits::LhsProgress % 4) == 0 && + (SwappedTraits::LhsProgress<=16) && + (SwappedTraits::LhsProgress!=8 || SResPacketHalfSize==nr) && + (SwappedTraits::LhsProgress!=16 || SResPacketQuarterSize==nr)) { - // NOTE The following piece of code wont work for 512 bit registers SAccPacket C0, C1, C2, C3; straits.initAcc(C0); straits.initAcc(C1); @@ -1648,15 +2091,15 @@ straits.loadRhsQuad(blA+0*spk, B_0); straits.loadRhsQuad(blA+1*spk, B_1); - straits.madd(A0,B_0,C0,B_0); - straits.madd(A1,B_1,C1,B_1); + straits.madd(A0,B_0,C0,B_0, fix<0>); + straits.madd(A1,B_1,C1,B_1, fix<0>); straits.loadLhsUnaligned(blB+2*SwappedTraits::LhsProgress, A0); straits.loadLhsUnaligned(blB+3*SwappedTraits::LhsProgress, A1); straits.loadRhsQuad(blA+2*spk, B_0); straits.loadRhsQuad(blA+3*spk, B_1); - straits.madd(A0,B_0,C2,B_0); - straits.madd(A1,B_1,C3,B_1); + straits.madd(A0,B_0,C2,B_0, fix<0>); + straits.madd(A1,B_1,C3,B_1, fix<0>); blB += 4*SwappedTraits::LhsProgress; blA += 4*spk; @@ -1669,7 +2112,7 @@ straits.loadLhsUnaligned(blB, A0); straits.loadRhsQuad(blA, B_0); - straits.madd(A0,B_0,C0,B_0); + straits.madd(A0,B_0,C0,B_0, fix<0>); blB += SwappedTraits::LhsProgress; blA += spk; @@ -1677,10 +2120,10 @@ if(SwappedTraits::LhsProgress==8) { // Special case where we have to first reduce the accumulation register C0 - typedef typename conditional<SwappedTraits::LhsProgress==8,typename unpacket_traits<SResPacket>::half,SResPacket>::type SResPacketHalf; - typedef typename conditional<SwappedTraits::LhsProgress==8,typename unpacket_traits<SLhsPacket>::half,SLhsPacket>::type SLhsPacketHalf; - typedef typename conditional<SwappedTraits::LhsProgress==8,typename unpacket_traits<SLhsPacket>::half,SRhsPacket>::type SRhsPacketHalf; - typedef typename conditional<SwappedTraits::LhsProgress==8,typename unpacket_traits<SAccPacket>::half,SAccPacket>::type SAccPacketHalf; + typedef typename conditional<SwappedTraits::LhsProgress>=8,typename unpacket_traits<SResPacket>::half,SResPacket>::type SResPacketHalf; + typedef typename conditional<SwappedTraits::LhsProgress>=8,typename unpacket_traits<SLhsPacket>::half,SLhsPacket>::type SLhsPacketHalf; + typedef typename conditional<SwappedTraits::LhsProgress>=8,typename unpacket_traits<SRhsPacket>::half,SRhsPacket>::type SRhsPacketHalf; + typedef typename conditional<SwappedTraits::LhsProgress>=8,typename unpacket_traits<SAccPacket>::half,SAccPacket>::type SAccPacketHalf; SResPacketHalf R = res.template gatherPacket<SResPacketHalf>(i, j2); SResPacketHalf alphav = pset1<SResPacketHalf>(alpha); @@ -1692,16 +2135,25 @@ SRhsPacketHalf b0; straits.loadLhsUnaligned(blB, a0); straits.loadRhs(blA, b0); - SAccPacketHalf c0 = predux4(C0); - straits.madd(a0,b0,c0,b0); + SAccPacketHalf c0 = predux_half_dowto4(C0); + straits.madd(a0,b0,c0,b0, fix<0>); straits.acc(c0, alphav, R); } else { - straits.acc(predux4(C0), alphav, R); + straits.acc(predux_half_dowto4(C0), alphav, R); } res.scatterPacket(i, j2, R); } + else if (SwappedTraits::LhsProgress==16) + { + // Special case where we have to first reduce the + // accumulation register C0. We specialize the block in + // template form, so that LhsProgress < 16 paths don't + // fail to compile + last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> p; + p(res, straits, blA, blB, depth, endk, i, j2,alpha, C0); + } else { SResPacket R = res.template gatherPacket<SResPacket>(i, j2); @@ -1745,7 +2197,7 @@ for(Index j2=packet_cols4; j2<cols; j2++) { // loop on each row of the lhs (1*LhsProgress x depth) - for(Index i=peeled_mc1; i<rows; i+=1) + for(Index i=peeled_mc_quarter; i<rows; i+=1) { const LhsScalar* blA = &blockA[i*strideA+offsetA]; prefetch(&blA[0]); @@ -1781,19 +2233,24 @@ // // 32 33 34 35 ... // 36 36 38 39 ... -template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, bool Conjugate, bool PanelMode> -struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, ColMajor, Conjugate, PanelMode> +template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode> +struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> { typedef typename DataMapper::LinearMapper LinearMapper; EIGEN_DONT_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; -template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, bool Conjugate, bool PanelMode> -EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, ColMajor, Conjugate, PanelMode> +template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode> +EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> ::operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - typedef typename packet_traits<Scalar>::type Packet; - enum { PacketSize = packet_traits<Scalar>::size }; + typedef typename unpacket_traits<Packet>::half HalfPacket; + typedef typename unpacket_traits<typename unpacket_traits<Packet>::half>::half QuarterPacket; + enum { PacketSize = unpacket_traits<Packet>::size, + HalfPacketSize = unpacket_traits<HalfPacket>::size, + QuarterPacketSize = unpacket_traits<QuarterPacket>::size, + HasHalf = (int)HalfPacketSize < (int)PacketSize, + HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize}; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK LHS"); EIGEN_UNUSED_VARIABLE(stride); @@ -1805,9 +2262,12 @@ const Index peeled_mc3 = Pack1>=3*PacketSize ? (rows/(3*PacketSize))*(3*PacketSize) : 0; const Index peeled_mc2 = Pack1>=2*PacketSize ? peeled_mc3+((rows-peeled_mc3)/(2*PacketSize))*(2*PacketSize) : 0; - const Index peeled_mc1 = Pack1>=1*PacketSize ? (rows/(1*PacketSize))*(1*PacketSize) : 0; - const Index peeled_mc0 = Pack2>=1*PacketSize ? peeled_mc1 - : Pack2>1 ? (rows/Pack2)*Pack2 : 0; + const Index peeled_mc1 = Pack1>=1*PacketSize ? peeled_mc2+((rows-peeled_mc2)/(1*PacketSize))*(1*PacketSize) : 0; + const Index peeled_mc_half = Pack1>=HalfPacketSize ? peeled_mc1+((rows-peeled_mc1)/(HalfPacketSize))*(HalfPacketSize) : 0; + const Index peeled_mc_quarter = Pack1>=QuarterPacketSize ? (rows/(QuarterPacketSize))*(QuarterPacketSize) : 0; + const Index last_lhs_progress = rows > peeled_mc_quarter ? (rows - peeled_mc_quarter) & ~1 : 0; + const Index peeled_mc0 = Pack2>=PacketSize ? peeled_mc_quarter + : Pack2>1 && last_lhs_progress ? (rows/last_lhs_progress)*last_lhs_progress : 0; Index i=0; @@ -1821,9 +2281,9 @@ for(Index k=0; k<depth; k++) { Packet A, B, C; - A = lhs.loadPacket(i+0*PacketSize, k); - B = lhs.loadPacket(i+1*PacketSize, k); - C = lhs.loadPacket(i+2*PacketSize, k); + A = lhs.template loadPacket<Packet>(i+0*PacketSize, k); + B = lhs.template loadPacket<Packet>(i+1*PacketSize, k); + C = lhs.template loadPacket<Packet>(i+2*PacketSize, k); pstore(blockA+count, cj.pconj(A)); count+=PacketSize; pstore(blockA+count, cj.pconj(B)); count+=PacketSize; pstore(blockA+count, cj.pconj(C)); count+=PacketSize; @@ -1841,8 +2301,8 @@ for(Index k=0; k<depth; k++) { Packet A, B; - A = lhs.loadPacket(i+0*PacketSize, k); - B = lhs.loadPacket(i+1*PacketSize, k); + A = lhs.template loadPacket<Packet>(i+0*PacketSize, k); + B = lhs.template loadPacket<Packet>(i+1*PacketSize, k); pstore(blockA+count, cj.pconj(A)); count+=PacketSize; pstore(blockA+count, cj.pconj(B)); count+=PacketSize; } @@ -1859,27 +2319,67 @@ for(Index k=0; k<depth; k++) { Packet A; - A = lhs.loadPacket(i+0*PacketSize, k); + A = lhs.template loadPacket<Packet>(i+0*PacketSize, k); pstore(blockA+count, cj.pconj(A)); count+=PacketSize; } if(PanelMode) count += (1*PacketSize) * (stride-offset-depth); } } - // Pack scalars - if(Pack2<PacketSize && Pack2>1) + // Pack half packets + if(HasHalf && Pack1>=HalfPacketSize) { - for(; i<peeled_mc0; i+=Pack2) + for(; i<peeled_mc_half; i+=HalfPacketSize) { - if(PanelMode) count += Pack2 * offset; + if(PanelMode) count += (HalfPacketSize) * offset; for(Index k=0; k<depth; k++) - for(Index w=0; w<Pack2; w++) - blockA[count++] = cj(lhs(i+w, k)); - - if(PanelMode) count += Pack2 * (stride-offset-depth); + { + HalfPacket A; + A = lhs.template loadPacket<HalfPacket>(i+0*(HalfPacketSize), k); + pstoreu(blockA+count, cj.pconj(A)); + count+=HalfPacketSize; + } + if(PanelMode) count += (HalfPacketSize) * (stride-offset-depth); } } + // Pack quarter packets + if(HasQuarter && Pack1>=QuarterPacketSize) + { + for(; i<peeled_mc_quarter; i+=QuarterPacketSize) + { + if(PanelMode) count += (QuarterPacketSize) * offset; + + for(Index k=0; k<depth; k++) + { + QuarterPacket A; + A = lhs.template loadPacket<QuarterPacket>(i+0*(QuarterPacketSize), k); + pstoreu(blockA+count, cj.pconj(A)); + count+=QuarterPacketSize; + } + if(PanelMode) count += (QuarterPacketSize) * (stride-offset-depth); + } + } + // Pack2 may be *smaller* than PacketSize—that happens for + // products like real * complex, where we have to go half the + // progress on the lhs in order to duplicate those operands to + // address both real & imaginary parts on the rhs. This portion will + // pack those half ones until they match the number expected on the + // last peeling loop at this point (for the rhs). + if(Pack2<PacketSize && Pack2>1) + { + for(; i<peeled_mc0; i+=last_lhs_progress) + { + if(PanelMode) count += last_lhs_progress * offset; + + for(Index k=0; k<depth; k++) + for(Index w=0; w<last_lhs_progress; w++) + blockA[count++] = cj(lhs(i+w, k)); + + if(PanelMode) count += last_lhs_progress * (stride-offset-depth); + } + } + // Pack scalars for(; i<rows; i++) { if(PanelMode) count += offset; @@ -1889,19 +2389,24 @@ } } -template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, bool Conjugate, bool PanelMode> -struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, RowMajor, Conjugate, PanelMode> +template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode> +struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> { typedef typename DataMapper::LinearMapper LinearMapper; EIGEN_DONT_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; -template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, bool Conjugate, bool PanelMode> -EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, RowMajor, Conjugate, PanelMode> +template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode> +EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> ::operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { - typedef typename packet_traits<Scalar>::type Packet; - enum { PacketSize = packet_traits<Scalar>::size }; + typedef typename unpacket_traits<Packet>::half HalfPacket; + typedef typename unpacket_traits<typename unpacket_traits<Packet>::half>::half QuarterPacket; + enum { PacketSize = unpacket_traits<Packet>::size, + HalfPacketSize = unpacket_traits<HalfPacket>::size, + QuarterPacketSize = unpacket_traits<QuarterPacket>::size, + HasHalf = (int)HalfPacketSize < (int)PacketSize, + HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize}; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK LHS"); EIGEN_UNUSED_VARIABLE(stride); @@ -1909,37 +2414,51 @@ eigen_assert(((!PanelMode) && stride==0 && offset==0) || (PanelMode && stride>=depth && offset<=stride)); conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj; Index count = 0; + bool gone_half = false, gone_quarter = false, gone_last = false; -// const Index peeled_mc3 = Pack1>=3*PacketSize ? (rows/(3*PacketSize))*(3*PacketSize) : 0; -// const Index peeled_mc2 = Pack1>=2*PacketSize ? peeled_mc3+((rows-peeled_mc3)/(2*PacketSize))*(2*PacketSize) : 0; -// const Index peeled_mc1 = Pack1>=1*PacketSize ? (rows/(1*PacketSize))*(1*PacketSize) : 0; - - int pack = Pack1; Index i = 0; + int pack = Pack1; + int psize = PacketSize; while(pack>0) { Index remaining_rows = rows-i; - Index peeled_mc = i+(remaining_rows/pack)*pack; + Index peeled_mc = gone_last ? Pack2>1 ? (rows/pack)*pack : 0 : i+(remaining_rows/pack)*pack; + Index starting_pos = i; for(; i<peeled_mc; i+=pack) { if(PanelMode) count += pack * offset; - const Index peeled_k = (depth/PacketSize)*PacketSize; Index k=0; - if(pack>=PacketSize) + if(pack>=psize && psize >= QuarterPacketSize) { - for(; k<peeled_k; k+=PacketSize) + const Index peeled_k = (depth/psize)*psize; + for(; k<peeled_k; k+=psize) { - for (Index m = 0; m < pack; m += PacketSize) + for (Index m = 0; m < pack; m += psize) { - PacketBlock<Packet> kernel; - for (int p = 0; p < PacketSize; ++p) kernel.packet[p] = lhs.loadPacket(i+p+m, k); - ptranspose(kernel); - for (int p = 0; p < PacketSize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel.packet[p])); + if (psize == PacketSize) { + PacketBlock<Packet> kernel; + for (int p = 0; p < psize; ++p) kernel.packet[p] = lhs.template loadPacket<Packet>(i+p+m, k); + ptranspose(kernel); + for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel.packet[p])); + } else if (HasHalf && psize == HalfPacketSize) { + gone_half = true; + PacketBlock<HalfPacket> kernel_half; + for (int p = 0; p < psize; ++p) kernel_half.packet[p] = lhs.template loadPacket<HalfPacket>(i+p+m, k); + ptranspose(kernel_half); + for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_half.packet[p])); + } else if (HasQuarter && psize == QuarterPacketSize) { + gone_quarter = true; + PacketBlock<QuarterPacket> kernel_quarter; + for (int p = 0; p < psize; ++p) kernel_quarter.packet[p] = lhs.template loadPacket<QuarterPacket>(i+p+m, k); + ptranspose(kernel_quarter); + for (int p = 0; p < psize; ++p) pstore(blockA+count+m+(pack)*p, cj.pconj(kernel_quarter.packet[p])); + } } - count += PacketSize*pack; + count += psize*pack; } } + for(; k<depth; k++) { Index w=0; @@ -1962,9 +2481,28 @@ if(PanelMode) count += pack * (stride-offset-depth); } - pack -= PacketSize; - if(pack<Pack2 && (pack+PacketSize)!=Pack2) - pack = Pack2; + pack -= psize; + Index left = rows - i; + if (pack <= 0) { + if (!gone_last && + (starting_pos == i || left >= psize/2 || left >= psize/4) && + ((psize/2 == HalfPacketSize && HasHalf && !gone_half) || + (psize/2 == QuarterPacketSize && HasQuarter && !gone_quarter))) { + psize /= 2; + pack = psize; + continue; + } + // Pack2 may be *smaller* than PacketSize—that happens for + // products like real * complex, where we have to go half the + // progress on the lhs in order to duplicate those operands to + // address both real & imaginary parts on the rhs. This portion will + // pack those half ones until they match the number expected on the + // last peeling loop at this point (for the rhs). + if (Pack2 < PacketSize && !gone_last) { + gone_last = true; + psize = pack = left & ~1; + } + } } for(; i<rows; i++) @@ -2020,7 +2558,7 @@ // const Scalar* b6 = &rhs[(j2+6)*rhsStride]; // const Scalar* b7 = &rhs[(j2+7)*rhsStride]; // Index k=0; -// if(PacketSize==8) // TODO enbale vectorized transposition for PacketSize==4 +// if(PacketSize==8) // TODO enable vectorized transposition for PacketSize==4 // { // for(; k<peeled_k; k+=PacketSize) { // PacketBlock<Packet> kernel; @@ -2067,10 +2605,10 @@ { for(; k<peeled_k; k+=PacketSize) { PacketBlock<Packet,(PacketSize%4)==0?4:PacketSize> kernel; - kernel.packet[0] = dm0.loadPacket(k); - kernel.packet[1%PacketSize] = dm1.loadPacket(k); - kernel.packet[2%PacketSize] = dm2.loadPacket(k); - kernel.packet[3%PacketSize] = dm3.loadPacket(k); + kernel.packet[0 ] = dm0.template loadPacket<Packet>(k); + kernel.packet[1%PacketSize] = dm1.template loadPacket<Packet>(k); + kernel.packet[2%PacketSize] = dm2.template loadPacket<Packet>(k); + kernel.packet[3%PacketSize] = dm3.template loadPacket<Packet>(k); ptranspose(kernel); pstoreu(blockB+count+0*PacketSize, cj.pconj(kernel.packet[0])); pstoreu(blockB+count+1*PacketSize, cj.pconj(kernel.packet[1%PacketSize])); @@ -2171,7 +2709,7 @@ for(Index k=0; k<depth; k++) { if (PacketSize==4) { - Packet A = rhs.loadPacket(k, j2); + Packet A = rhs.template loadPacket<Packet>(k, j2); pstoreu(blockB+count, cj.pconj(A)); count += PacketSize; } else {
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrix.h b/Eigen/src/Core/products/GeneralMatrixMatrix.h index a39c780..90c9c46 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrix.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrix.h
@@ -10,7 +10,7 @@ #ifndef EIGEN_GENERAL_MATRIX_MATRIX_H #define EIGEN_GENERAL_MATRIX_MATRIX_H -namespace Eigen { +namespace Eigen { namespace internal { @@ -24,8 +24,8 @@ struct general_matrix_matrix_product<Index,LhsScalar,LhsStorageOrder,ConjugateLhs,RhsScalar,RhsStorageOrder,ConjugateRhs,RowMajor> { typedef gebp_traits<RhsScalar,LhsScalar> Traits; - - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; static EIGEN_STRONG_INLINE void run( Index rows, Index cols, Index depth, const LhsScalar* lhs, Index lhsStride, @@ -54,8 +54,8 @@ { typedef gebp_traits<LhsScalar,RhsScalar> Traits; - -typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + +typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; static void run(Index rows, Index cols, Index depth, const LhsScalar* _lhs, Index lhsStride, const RhsScalar* _rhs, Index rhsStride, @@ -75,7 +75,7 @@ Index mc = (std::min)(rows,blocking.mc()); // cache block size along the M direction Index nc = (std::min)(cols,blocking.nc()); // cache block size along the N direction - gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; + gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder> pack_lhs; gemm_pack_rhs<RhsScalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs; gebp_kernel<LhsScalar, RhsScalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp; @@ -83,15 +83,15 @@ if(info) { // this is the parallel version! - Index tid = omp_get_thread_num(); - Index threads = omp_get_num_threads(); - + int tid = omp_get_thread_num(); + int threads = omp_get_num_threads(); + LhsScalar* blockA = blocking.blockA(); eigen_internal_assert(blockA!=0); - + std::size_t sizeB = kc*nc; ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, 0); - + // For each horizontal panel of the rhs, and corresponding vertical panel of the lhs... for(Index k=0; k<depth; k+=kc) { @@ -108,17 +108,17 @@ // i.e., we test that info[tid].users equals 0. // Then, we set info[tid].users to the number of threads to mark that all other threads are going to use it. while(info[tid].users!=0) {} - info[tid].users += threads; + info[tid].users = threads; pack_lhs(blockA+info[tid].lhs_start*actual_kc, lhs.getSubMapper(info[tid].lhs_start,k), actual_kc, info[tid].lhs_length); // Notify the other threads that the part A'_i is ready to go. info[tid].sync = k; - + // Computes C_i += A' * B' per A'_i - for(Index shift=0; shift<threads; ++shift) + for(int shift=0; shift<threads; ++shift) { - Index i = (tid+shift)%threads; + int i = (tid+shift)%threads; // At this point we have to make sure that A'_i has been updated by the thread i, // we use testAndSetOrdered to mimic a volatile access. @@ -146,7 +146,9 @@ // Release all the sub blocks A'_i of A' for the current thread, // i.e., we simply decrement the number of users by 1 for(Index i=0; i<threads; ++i) +#if !EIGEN_HAS_CXX11_ATOMIC #pragma omp atomic +#endif info[i].users -= 1; } } @@ -161,7 +163,7 @@ ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, blocking.blockA()); ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, blocking.blockB()); - + const bool pack_rhs_once = mc!=rows && kc==depth && nc==cols; // For each horizontal panel of the rhs, and corresponding panel of the lhs... @@ -172,24 +174,24 @@ for(Index k2=0; k2<depth; k2+=kc) { const Index actual_kc = (std::min)(k2+kc,depth)-k2; - + // OK, here we have selected one horizontal panel of rhs and one vertical panel of lhs. // => Pack lhs's panel into a sequential chunk of memory (L2/L3 caching) // Note that this panel will be read as many times as the number of blocks in the rhs's // horizontal panel which is, in practice, a very low number. pack_lhs(blockA, lhs.getSubMapper(i2,k2), actual_kc, actual_mc); - + // For each kc x nc block of the rhs's horizontal panel... for(Index j2=0; j2<cols; j2+=nc) { const Index actual_nc = (std::min)(j2+nc,cols)-j2; - + // We pack the rhs's block into a sequential chunk of memory (L2 caching) // Note that this block will be read a very high number of times, which is equal to the number of // micro horizontal panel of the large rhs's panel (e.g., rows/12 times). if((!pack_rhs_once) || i2==0) pack_rhs(blockB, rhs.getSubMapper(k2,j2), actual_kc, actual_nc); - + // Everything is packed, we can now call the panel * block kernel: gebp(res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha); } @@ -229,7 +231,7 @@ (Scalar*)&(m_dest.coeffRef(row,col)), m_dest.outerStride(), m_actualAlpha, m_blocking, info); } - + typedef typename Gemm::Traits Traits; protected: @@ -309,11 +311,11 @@ this->m_blockA = m_staticA; this->m_blockB = m_staticB; #else - this->m_blockA = reinterpret_cast<LhsScalar*>((std::size_t(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); - this->m_blockB = reinterpret_cast<RhsScalar*>((std::size_t(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); + this->m_blockA = reinterpret_cast<LhsScalar*>((internal::UIntPtr(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); + this->m_blockB = reinterpret_cast<RhsScalar*>((internal::UIntPtr(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES-1)) & ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)); #endif } - + void initParallel(Index, Index, Index, Index) {} @@ -359,14 +361,14 @@ m_sizeA = this->m_mc * this->m_kc; m_sizeB = this->m_kc * this->m_nc; } - + void initParallel(Index rows, Index cols, Index depth, Index num_threads) { this->m_mc = Transpose ? cols : rows; this->m_nc = Transpose ? rows : cols; this->m_kc = depth; - - eigen_internal_assert(this->m_blockA==0 && this->m_blockB==0); + + eigen_internal_assert(this->m_blockA==0 && this->m_blockB==0); Index m = this->m_mc; computeProductBlockingSizes<LhsScalar,RhsScalar,KcFactor>(this->m_kc, m, this->m_nc, num_threads); m_sizeA = this->m_mc * this->m_kc; @@ -401,7 +403,7 @@ } // end namespace internal namespace internal { - + template<typename Lhs, typename Rhs> struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> > @@ -409,26 +411,32 @@ typedef typename Product<Lhs,Rhs>::Scalar Scalar; typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; - + typedef internal::blas_traits<Lhs> LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef typename internal::remove_all<ActualLhsType>::type ActualLhsTypeCleaned; - + typedef internal::blas_traits<Rhs> RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; - + enum { MaxDepthAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(Lhs::MaxColsAtCompileTime,Rhs::MaxRowsAtCompileTime) }; - + typedef generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> lazyproduct; - + template<typename Dst> static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) - lazyproduct::evalTo(dst, lhs, rhs); + // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=404 for a discussion and helper program + // to determine the following heuristic. + // EIGEN_GEMM_TO_COEFFBASED_THRESHOLD is typically defined to 20 in GeneralProduct.h, + // unless it has been specialized by the user or for a given architecture. + // Note that the condition rhs.rows()>0 was required because lazy product is (was?) not happy with empty inputs. + // I'm not sure it is still required. + if((rhs.rows()+dst.rows()+dst.cols())<EIGEN_GEMM_TO_COEFFBASED_THRESHOLD && rhs.rows()>0) + lazyproduct::eval_dynamic(dst, lhs, rhs, internal::assign_op<typename Dst::Scalar,Scalar>()); else { dst.setZero(); @@ -439,8 +447,8 @@ template<typename Dst> static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) - lazyproduct::addTo(dst, lhs, rhs); + if((rhs.rows()+dst.rows()+dst.cols())<EIGEN_GEMM_TO_COEFFBASED_THRESHOLD && rhs.rows()>0) + lazyproduct::eval_dynamic(dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar,Scalar>()); else scaleAndAddTo(dst,lhs, rhs, Scalar(1)); } @@ -448,12 +456,12 @@ template<typename Dst> static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) - lazyproduct::subTo(dst, lhs, rhs); + if((rhs.rows()+dst.rows()+dst.cols())<EIGEN_GEMM_TO_COEFFBASED_THRESHOLD && rhs.rows()>0) + lazyproduct::eval_dynamic(dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar,Scalar>()); else scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); } - + template<typename Dest> static void scaleAndAddTo(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha) { @@ -461,6 +469,20 @@ if(a_lhs.cols()==0 || a_lhs.rows()==0 || a_rhs.cols()==0) return; + // Fallback to GEMV if either the lhs or rhs is a runtime vector + if (dst.cols() == 1) + { + typename Dest::ColXpr dst_vec(dst.col(0)); + return internal::generic_product_impl<Lhs,typename Rhs::ConstColXpr,DenseShape,DenseShape,GemvProduct> + ::scaleAndAddTo(dst_vec, a_lhs, a_rhs.col(0), alpha); + } + else if (dst.rows() == 1) + { + typename Dest::RowXpr dst_vec(dst.row(0)); + return internal::generic_product_impl<typename Lhs::ConstRowXpr,Rhs,DenseShape,DenseShape,GemvProduct> + ::scaleAndAddTo(dst_vec, a_lhs.row(0), a_rhs, alpha); + } + typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); @@ -481,7 +503,7 @@ BlockingType blocking(dst.rows(), dst.cols(), lhs.cols(), 1, true); internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime>32 || Dest::MaxRowsAtCompileTime==Dynamic)> - (GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), a_lhs.rows(), a_rhs.cols(), Dest::Flags&RowMajorBit); + (GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), a_lhs.rows(), a_rhs.cols(), a_lhs.cols(), Dest::Flags&RowMajorBit); } };
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h index 80ba894..ec2825b 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h
@@ -40,7 +40,7 @@ typename RhsScalar, int RhsStorageOrder, bool ConjugateRhs, int UpLo, int Version> struct general_matrix_matrix_triangular_product<Index,LhsScalar,LhsStorageOrder,ConjugateLhs,RhsScalar,RhsStorageOrder,ConjugateRhs,RowMajor,UpLo,Version> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; static EIGEN_STRONG_INLINE void run(Index size, Index depth,const LhsScalar* lhs, Index lhsStride, const RhsScalar* rhs, Index rhsStride, ResScalar* res, Index resStride, const ResScalar& alpha, level3_blocking<RhsScalar,LhsScalar>& blocking) @@ -57,7 +57,7 @@ typename RhsScalar, int RhsStorageOrder, bool ConjugateRhs, int UpLo, int Version> struct general_matrix_matrix_triangular_product<Index,LhsScalar,LhsStorageOrder,ConjugateLhs,RhsScalar,RhsStorageOrder,ConjugateRhs,ColMajor,UpLo,Version> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; static EIGEN_STRONG_INLINE void run(Index size, Index depth,const LhsScalar* _lhs, Index lhsStride, const RhsScalar* _rhs, Index rhsStride, ResScalar* _res, Index resStride, const ResScalar& alpha, level3_blocking<LhsScalar,RhsScalar>& blocking) @@ -84,7 +84,7 @@ ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, blocking.blockA()); ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, blocking.blockB()); - gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; + gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder> pack_lhs; gemm_pack_rhs<RhsScalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs; gebp_kernel<LhsScalar, RhsScalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp; tribb_kernel<LhsScalar, RhsScalar, Index, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs, UpLo> sybb; @@ -110,7 +110,6 @@ gebp(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, (std::min)(size,i2), alpha, -1, -1, 0, 0); - sybb(_res+resStride*i2 + i2, resStride, blockA, blockB + actual_kc*i2, actual_mc, actual_kc, alpha); if (UpLo==Upper) @@ -148,7 +147,7 @@ ResMapper res(_res, resStride); gebp_kernel<LhsScalar, RhsScalar, Index, ResMapper, mr, nr, ConjLhs, ConjRhs> gebp_kernel; - Matrix<ResScalar,BlockSize,BlockSize,ColMajor> buffer; + Matrix<ResScalar,BlockSize,BlockSize,ColMajor> buffer((internal::constructor_without_unaligned_array_assert())); // let's process the block per panel of actual_mc x BlockSize, // again, each is split into three parts, etc. @@ -160,7 +159,7 @@ if(UpLo==Upper) gebp_kernel(res.getSubMapper(0, j), blockA, actual_b, j, depth, actualBlockSize, alpha, -1, -1, 0, 0); - + // selfadjoint micro block { Index i = j; @@ -168,6 +167,7 @@ // 1 - apply the kernel on the temporary buffer gebp_kernel(ResMapper(buffer.data(), BlockSize), blockA+depth*i, actual_b, actualBlockSize, depth, actualBlockSize, alpha, -1, -1, 0, 0); + // 2 - triangular accumulation for(Index j1=0; j1<actualBlockSize; ++j1) { @@ -199,7 +199,7 @@ template<typename MatrixType, typename ProductType, int UpLo> struct general_product_to_triangular_selector<MatrixType,ProductType,UpLo,true> { - static void run(MatrixType& mat, const ProductType& prod, const typename MatrixType::Scalar& alpha) + static void run(MatrixType& mat, const ProductType& prod, const typename MatrixType::Scalar& alpha, bool beta) { typedef typename MatrixType::Scalar Scalar; @@ -217,6 +217,9 @@ Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs().derived()) * RhsBlasTraits::extractScalarFactor(prod.rhs().derived()); + if(!beta) + mat.template triangularView<UpLo>().setZero(); + enum { StorageOrder = (internal::traits<MatrixType>::Flags&RowMajorBit) ? RowMajor : ColMajor, UseLhsDirectly = _ActualLhs::InnerStrideAtCompileTime==1, @@ -244,7 +247,7 @@ template<typename MatrixType, typename ProductType, int UpLo> struct general_product_to_triangular_selector<MatrixType,ProductType,UpLo,false> { - static void run(MatrixType& mat, const ProductType& prod, const typename MatrixType::Scalar& alpha) + static void run(MatrixType& mat, const ProductType& prod, const typename MatrixType::Scalar& alpha, bool beta) { typedef typename internal::remove_all<typename ProductType::LhsNested>::type Lhs; typedef internal::blas_traits<Lhs> LhsBlasTraits; @@ -260,13 +263,19 @@ typename ProductType::Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs().derived()) * RhsBlasTraits::extractScalarFactor(prod.rhs().derived()); + if(!beta) + mat.template triangularView<UpLo>().setZero(); + enum { IsRowMajor = (internal::traits<MatrixType>::Flags&RowMajorBit) ? 1 : 0, LhsIsRowMajor = _ActualLhs::Flags&RowMajorBit ? 1 : 0, - RhsIsRowMajor = _ActualRhs::Flags&RowMajorBit ? 1 : 0 + RhsIsRowMajor = _ActualRhs::Flags&RowMajorBit ? 1 : 0, + SkipDiag = (UpLo&(UnitDiag|ZeroDiag))!=0 }; Index size = mat.cols(); + if(SkipDiag) + size--; Index depth = actualLhs.cols(); typedef internal::gemm_blocking_space<IsRowMajor ? RowMajor : ColMajor,typename Lhs::Scalar,typename Rhs::Scalar, @@ -277,21 +286,23 @@ internal::general_matrix_matrix_triangular_product<Index, typename Lhs::Scalar, LhsIsRowMajor ? RowMajor : ColMajor, LhsBlasTraits::NeedToConjugate, typename Rhs::Scalar, RhsIsRowMajor ? RowMajor : ColMajor, RhsBlasTraits::NeedToConjugate, - IsRowMajor ? RowMajor : ColMajor, UpLo> + IsRowMajor ? RowMajor : ColMajor, UpLo&(Lower|Upper)> ::run(size, depth, - &actualLhs.coeffRef(0,0), actualLhs.outerStride(), &actualRhs.coeffRef(0,0), actualRhs.outerStride(), - mat.data(), mat.outerStride(), actualAlpha, blocking); + &actualLhs.coeffRef(SkipDiag&&(UpLo&Lower)==Lower ? 1 : 0,0), actualLhs.outerStride(), + &actualRhs.coeffRef(0,SkipDiag&&(UpLo&Upper)==Upper ? 1 : 0), actualRhs.outerStride(), + mat.data() + (SkipDiag ? (bool(IsRowMajor) != ((UpLo&Lower)==Lower) ? 1 : mat.outerStride() ) : 0), mat.outerStride(), actualAlpha, blocking); } }; template<typename MatrixType, unsigned int UpLo> template<typename ProductType> -TriangularView<MatrixType,UpLo>& TriangularViewImpl<MatrixType,UpLo,Dense>::_assignProduct(const ProductType& prod, const Scalar& alpha) +EIGEN_DEVICE_FUNC TriangularView<MatrixType,UpLo>& TriangularViewImpl<MatrixType,UpLo,Dense>::_assignProduct(const ProductType& prod, const Scalar& alpha, bool beta) { + EIGEN_STATIC_ASSERT((UpLo&UnitDiag)==0, WRITING_TO_TRIANGULAR_PART_WITH_UNIT_DIAGONAL_IS_NOT_SUPPORTED); eigen_assert(derived().nestedExpression().rows() == prod.rows() && derived().cols() == prod.cols()); - - general_product_to_triangular_selector<MatrixType, ProductType, UpLo, internal::traits<ProductType>::InnerSize==1>::run(derived().nestedExpression().const_cast_derived(), prod, alpha); - + + general_product_to_triangular_selector<MatrixType, ProductType, UpLo, internal::traits<ProductType>::InnerSize==1>::run(derived().nestedExpression().const_cast_derived(), prod, alpha, beta); + return derived(); }
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h index 911df8f..49565c0 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h
@@ -33,11 +33,11 @@ #ifndef EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H #define EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H -namespace Eigen { +namespace Eigen { namespace internal { -template <typename Index, typename Scalar, int AStorageOrder, bool ConjugateA, int ResStorageOrder, int UpLo> +template <typename Index, typename Scalar, int AStorageOrder, bool ConjugateA, int ResStorageOrder, int UpLo> struct general_matrix_matrix_rankupdate : general_matrix_matrix_triangular_product< Index,Scalar,AStorageOrder,ConjugateA,Scalar,AStorageOrder,ConjugateA,ResStorageOrder,UpLo,BuiltIn> {}; @@ -52,7 +52,7 @@ static EIGEN_STRONG_INLINE void run(Index size, Index depth,const Scalar* lhs, Index lhsStride, \ const Scalar* rhs, Index rhsStride, Scalar* res, Index resStride, Scalar alpha, level3_blocking<Scalar, Scalar>& blocking) \ { \ - if (lhs==rhs) { \ + if ( lhs==rhs && ((UpLo&(Lower|Upper))==UpLo) ) { \ general_matrix_matrix_rankupdate<Index,Scalar,LhsStorageOrder,ConjugateLhs,ColMajor,UpLo> \ ::run(size,depth,lhs,lhsStride,rhs,rhsStride,res,resStride,alpha,blocking); \ } else { \ @@ -86,9 +86,9 @@ /* typedef Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder> MatrixRhs;*/ \ \ BlasIndex lda=convert_index<BlasIndex>(lhsStride), ldc=convert_index<BlasIndex>(resStride), n=convert_index<BlasIndex>(size), k=convert_index<BlasIndex>(depth); \ - char uplo=(IsLower) ? 'L' : 'U', trans=(AStorageOrder==RowMajor) ? 'T':'N'; \ - EIGTYPE beta; \ - BLASFUNC(&uplo, &trans, &n, &k, &numext::real_ref(alpha), lhs, &lda, &numext::real_ref(beta), res, &ldc); \ + char uplo=((IsLower) ? 'L' : 'U'), trans=((AStorageOrder==RowMajor) ? 'T':'N'); \ + EIGTYPE beta(1); \ + BLASFUNC(&uplo, &trans, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), lhs, &lda, (const BLASTYPE*)&numext::real_ref(beta), res, &ldc); \ } \ }; @@ -107,7 +107,7 @@ typedef Matrix<EIGTYPE, Dynamic, Dynamic, AStorageOrder> MatrixType; \ \ BlasIndex lda=convert_index<BlasIndex>(lhsStride), ldc=convert_index<BlasIndex>(resStride), n=convert_index<BlasIndex>(size), k=convert_index<BlasIndex>(depth); \ - char uplo=(IsLower) ? 'L' : 'U', trans=(AStorageOrder==RowMajor) ? 'C':'N'; \ + char uplo=((IsLower) ? 'L' : 'U'), trans=((AStorageOrder==RowMajor) ? 'C':'N'); \ RTYPE alpha_, beta_; \ const EIGTYPE* a_ptr; \ \ @@ -125,9 +125,13 @@ } \ }; - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_RANKUPDATE_R(double, double, dsyrk) +EIGEN_BLAS_RANKUPDATE_R(float, float, ssyrk) +#else EIGEN_BLAS_RANKUPDATE_R(double, double, dsyrk_) EIGEN_BLAS_RANKUPDATE_R(float, float, ssyrk_) +#endif // TODO hanlde complex cases // EIGEN_BLAS_RANKUPDATE_C(dcomplex, double, double, zherk_)
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h b/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h index 7a3bdbf..b0f6b0d 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h
@@ -46,7 +46,7 @@ // gemm specialization -#define GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASPREFIX) \ +#define GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASFUNC) \ template< \ typename Index, \ int LhsStorageOrder, bool ConjugateLhs, \ @@ -100,13 +100,20 @@ ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } else b = _rhs; \ \ - BLASPREFIX##gemm_(&transa, &transb, &m, &n, &k, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, &numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ + BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ }}; -GEMM_SPECIALIZATION(double, d, double, d) -GEMM_SPECIALIZATION(float, f, float, s) -GEMM_SPECIALIZATION(dcomplex, cd, double, z) -GEMM_SPECIALIZATION(scomplex, cf, float, c) +#ifdef EIGEN_USE_MKL +GEMM_SPECIALIZATION(double, d, double, dgemm) +GEMM_SPECIALIZATION(float, f, float, sgemm) +GEMM_SPECIALIZATION(dcomplex, cd, MKL_Complex16, zgemm) +GEMM_SPECIALIZATION(scomplex, cf, MKL_Complex8, cgemm) +#else +GEMM_SPECIALIZATION(double, d, double, dgemm_) +GEMM_SPECIALIZATION(float, f, float, sgemm_) +GEMM_SPECIALIZATION(dcomplex, cd, double, zgemm_) +GEMM_SPECIALIZATION(scomplex, cf, float, cgemm_) +#endif } // end namespase internal
diff --git a/Eigen/src/Core/products/GeneralMatrixVector.h b/Eigen/src/Core/products/GeneralMatrixVector.h index 8b7dca4..767feb9 100644 --- a/Eigen/src/Core/products/GeneralMatrixVector.h +++ b/Eigen/src/Core/products/GeneralMatrixVector.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -15,10 +15,8 @@ namespace internal { /* Optimized col-major matrix * vector product: - * This algorithm processes 4 columns at onces that allows to both reduce - * the number of load/stores of the result by a factor 4 and to reduce - * the instruction dependency. Moreover, we know that all bands have the - * same alignment pattern. + * This algorithm processes the matrix per vertical panels, + * which are then processed horizontaly per chunck of 8*PacketSize x 1 vertical segments. * * Mixing type logic: C += alpha * A * B * | A | B |alpha| comments @@ -27,38 +25,12 @@ * |cplx |real |cplx | invalid, the caller has to do tmp: = A * B; C += alpha*tmp * |cplx |real |real | optimal case, vectorization possible via real-cplx mul * - * Accesses to the matrix coefficients follow the following logic: - * - * - if all columns have the same alignment then - * - if the columns have the same alignment as the result vector, then easy! (-> AllAligned case) - * - otherwise perform unaligned loads only (-> NoneAligned case) - * - otherwise - * - if even columns have the same alignment then - * // odd columns are guaranteed to have the same alignment too - * - if even or odd columns have the same alignment as the result, then - * // for a register size of 2 scalars, this is guarantee to be the case (e.g., SSE with double) - * - perform half aligned and half unaligned loads (-> EvenAligned case) - * - otherwise perform unaligned loads only (-> NoneAligned case) - * - otherwise, if the register size is 4 scalars (e.g., SSE with float) then - * - one over 4 consecutive columns is guaranteed to be aligned with the result vector, - * perform simple aligned loads for this column and aligned loads plus re-alignment for the other. (-> FirstAligned case) - * // this re-alignment is done by the palign function implemented for SSE in Eigen/src/Core/arch/SSE/PacketMath.h - * - otherwise, - * // if we get here, this means the register size is greater than 4 (e.g., AVX with floats), - * // we currently fall back to the NoneAligned case - * * The same reasoning apply for the transposed case. - * - * The last case (PacketSize>4) could probably be improved by generalizing the FirstAligned case, but since we do not support AVX yet... - * One might also wonder why in the EvenAligned case we perform unaligned loads instead of using the aligned-loads plus re-alignment - * strategy as in the FirstAligned case. The reason is that we observed that unaligned loads on a 8 byte boundary are not too slow - * compared to unaligned loads on a 4 byte boundary. - * */ template<typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version> struct general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { Vectorizable = packet_traits<LhsScalar>::Vectorizable && packet_traits<RhsScalar>::Vectorizable @@ -76,7 +48,7 @@ typedef typename conditional<Vectorizable,_RhsPacket,RhsScalar>::type RhsPacket; typedef typename conditional<Vectorizable,_ResPacket,ResScalar>::type ResPacket; -EIGEN_DONT_INLINE static void run( +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( Index rows, Index cols, const LhsMapper& lhs, const RhsMapper& rhs, @@ -85,244 +57,151 @@ }; template<typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version> -EIGEN_DONT_INLINE void general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version>::run( +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version>::run( Index rows, Index cols, - const LhsMapper& lhs, + const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res, Index resIncr, RhsScalar alpha) { EIGEN_UNUSED_VARIABLE(resIncr); eigen_internal_assert(resIncr==1); - #ifdef _EIGEN_ACCUMULATE_PACKETS - #error _EIGEN_ACCUMULATE_PACKETS has already been defined - #endif - #define _EIGEN_ACCUMULATE_PACKETS(Alignment0,Alignment13,Alignment2) \ - pstore(&res[j], \ - padd(pload<ResPacket>(&res[j]), \ - padd( \ - padd(pcj.pmul(lhs0.template load<LhsPacket, Alignment0>(j), ptmp0), \ - pcj.pmul(lhs1.template load<LhsPacket, Alignment13>(j), ptmp1)), \ - padd(pcj.pmul(lhs2.template load<LhsPacket, Alignment2>(j), ptmp2), \ - pcj.pmul(lhs3.template load<LhsPacket, Alignment13>(j), ptmp3)) ))) - typedef typename LhsMapper::VectorMapper LhsScalars; + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate propoer code. + LhsMapper lhs(alhs); conj_helper<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs> cj; conj_helper<LhsPacket,RhsPacket,ConjugateLhs,ConjugateRhs> pcj; - if(ConjugateRhs) - alpha = numext::conj(alpha); - - enum { AllAligned = 0, EvenAligned, FirstAligned, NoneAligned }; - const Index columnsAtOnce = 4; - const Index peels = 2; - const Index LhsPacketAlignedMask = LhsPacketSize-1; - const Index ResPacketAlignedMask = ResPacketSize-1; -// const Index PeelAlignedMask = ResPacketSize*peels-1; - const Index size = rows; - const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { LhsAlignment = Unaligned }; - // How many coeffs of the result do we have to skip to be aligned. - // Here we assume data are at least aligned on the base scalar type. - Index alignedStart = internal::first_default_aligned(res,size); - Index alignedSize = ResPacketSize>1 ? alignedStart + ((size-alignedStart) & ~ResPacketAlignedMask) : 0; - const Index peeledSize = alignedSize - RhsPacketSize*peels - RhsPacketSize + 1; + const Index n8 = rows-8*ResPacketSize+1; + const Index n4 = rows-4*ResPacketSize+1; + const Index n3 = rows-3*ResPacketSize+1; + const Index n2 = rows-2*ResPacketSize+1; + const Index n1 = rows-1*ResPacketSize+1; - const Index alignmentStep = LhsPacketSize>1 ? (LhsPacketSize - lhsStride % LhsPacketSize) & LhsPacketAlignedMask : 0; - Index alignmentPattern = alignmentStep==0 ? AllAligned - : alignmentStep==(LhsPacketSize/2) ? EvenAligned - : FirstAligned; + // TODO: improve the following heuristic: + const Index block_cols = cols<128 ? cols : (lhsStride*sizeof(LhsScalar)<32000?16:4); + ResPacket palpha = pset1<ResPacket>(alpha); - // we cannot assume the first element is aligned because of sub-matrices - const Index lhsAlignmentOffset = lhs.firstAligned(size); - - // find how many columns do we have to skip to be aligned with the result (if possible) - Index skipColumns = 0; - // if the data cannot be aligned (TODO add some compile time tests when possible, e.g. for floats) - if( (lhsAlignmentOffset < 0) || (lhsAlignmentOffset == size) || (size_t(res)%sizeof(ResScalar)) ) + for(Index j2=0; j2<cols; j2+=block_cols) { - alignedSize = 0; - alignedStart = 0; - alignmentPattern = NoneAligned; - } - else if(LhsPacketSize > 4) - { - // TODO: extend the code to support aligned loads whenever possible when LhsPacketSize > 4. - // Currently, it seems to be better to perform unaligned loads anyway - alignmentPattern = NoneAligned; - } - else if (LhsPacketSize>1) - { - // eigen_internal_assert(size_t(firstLhs+lhsAlignmentOffset)%sizeof(LhsPacket)==0 || size<LhsPacketSize); - - while (skipColumns<LhsPacketSize && - alignedStart != ((lhsAlignmentOffset + alignmentStep*skipColumns)%LhsPacketSize)) - ++skipColumns; - if (skipColumns==LhsPacketSize) + Index jend = numext::mini(j2+block_cols,cols); + Index i=0; + for(; i<n8; i+=ResPacketSize*8) { - // nothing can be aligned, no need to skip any column - alignmentPattern = NoneAligned; - skipColumns = 0; - } - else - { - skipColumns = (std::min)(skipColumns,cols); - // note that the skiped columns are processed later. - } + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)), + c2 = pset1<ResPacket>(ResScalar(0)), + c3 = pset1<ResPacket>(ResScalar(0)), + c4 = pset1<ResPacket>(ResScalar(0)), + c5 = pset1<ResPacket>(ResScalar(0)), + c6 = pset1<ResPacket>(ResScalar(0)), + c7 = pset1<ResPacket>(ResScalar(0)); - /* eigen_internal_assert( (alignmentPattern==NoneAligned) - || (skipColumns + columnsAtOnce >= cols) - || LhsPacketSize > size - || (size_t(firstLhs+alignedStart+lhsStride*skipColumns)%sizeof(LhsPacket))==0);*/ - } - else if(Vectorizable) - { - alignedStart = 0; - alignedSize = size; - alignmentPattern = AllAligned; - } - - const Index offset1 = (FirstAligned && alignmentStep==1?3:1); - const Index offset3 = (FirstAligned && alignmentStep==1?1:3); - - Index columnBound = ((cols-skipColumns)/columnsAtOnce)*columnsAtOnce + skipColumns; - for (Index i=skipColumns; i<columnBound; i+=columnsAtOnce) - { - RhsPacket ptmp0 = pset1<RhsPacket>(alpha*rhs(i, 0)), - ptmp1 = pset1<RhsPacket>(alpha*rhs(i+offset1, 0)), - ptmp2 = pset1<RhsPacket>(alpha*rhs(i+2, 0)), - ptmp3 = pset1<RhsPacket>(alpha*rhs(i+offset3, 0)); - - // this helps a lot generating better binary code - const LhsScalars lhs0 = lhs.getVectorMapper(0, i+0), lhs1 = lhs.getVectorMapper(0, i+offset1), - lhs2 = lhs.getVectorMapper(0, i+2), lhs3 = lhs.getVectorMapper(0, i+offset3); - - if (Vectorizable) - { - /* explicit vectorization */ - // process initial unaligned coeffs - for (Index j=0; j<alignedStart; ++j) + for(Index j=j2; j<jend; j+=1) { - res[j] = cj.pmadd(lhs0(j), pfirst(ptmp0), res[j]); - res[j] = cj.pmadd(lhs1(j), pfirst(ptmp1), res[j]); - res[j] = cj.pmadd(lhs2(j), pfirst(ptmp2), res[j]); - res[j] = cj.pmadd(lhs3(j), pfirst(ptmp3), res[j]); + RhsPacket b0 = pset1<RhsPacket>(rhs(j,0)); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1); + c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2); + c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*3,j),b0,c3); + c4 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*4,j),b0,c4); + c5 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*5,j),b0,c5); + c6 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*6,j),b0,c6); + c7 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*7,j),b0,c7); } - - if (alignedSize>alignedStart) - { - switch(alignmentPattern) - { - case AllAligned: - for (Index j = alignedStart; j<alignedSize; j+=ResPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Aligned,Aligned); - break; - case EvenAligned: - for (Index j = alignedStart; j<alignedSize; j+=ResPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Unaligned,Aligned); - break; - case FirstAligned: - { - Index j = alignedStart; - if(peels>1) - { - LhsPacket A00, A01, A02, A03, A10, A11, A12, A13; - ResPacket T0, T1; - - A01 = lhs1.template load<LhsPacket, Aligned>(alignedStart-1); - A02 = lhs2.template load<LhsPacket, Aligned>(alignedStart-2); - A03 = lhs3.template load<LhsPacket, Aligned>(alignedStart-3); - - for (; j<peeledSize; j+=peels*ResPacketSize) - { - A11 = lhs1.template load<LhsPacket, Aligned>(j-1+LhsPacketSize); palign<1>(A01,A11); - A12 = lhs2.template load<LhsPacket, Aligned>(j-2+LhsPacketSize); palign<2>(A02,A12); - A13 = lhs3.template load<LhsPacket, Aligned>(j-3+LhsPacketSize); palign<3>(A03,A13); - - A00 = lhs0.template load<LhsPacket, Aligned>(j); - A10 = lhs0.template load<LhsPacket, Aligned>(j+LhsPacketSize); - T0 = pcj.pmadd(A00, ptmp0, pload<ResPacket>(&res[j])); - T1 = pcj.pmadd(A10, ptmp0, pload<ResPacket>(&res[j+ResPacketSize])); - - T0 = pcj.pmadd(A01, ptmp1, T0); - A01 = lhs1.template load<LhsPacket, Aligned>(j-1+2*LhsPacketSize); palign<1>(A11,A01); - T0 = pcj.pmadd(A02, ptmp2, T0); - A02 = lhs2.template load<LhsPacket, Aligned>(j-2+2*LhsPacketSize); palign<2>(A12,A02); - T0 = pcj.pmadd(A03, ptmp3, T0); - pstore(&res[j],T0); - A03 = lhs3.template load<LhsPacket, Aligned>(j-3+2*LhsPacketSize); palign<3>(A13,A03); - T1 = pcj.pmadd(A11, ptmp1, T1); - T1 = pcj.pmadd(A12, ptmp2, T1); - T1 = pcj.pmadd(A13, ptmp3, T1); - pstore(&res[j+ResPacketSize],T1); - } - } - for (; j<alignedSize; j+=ResPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Unaligned,Unaligned); - break; - } - default: - for (Index j = alignedStart; j<alignedSize; j+=ResPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Unaligned,Unaligned,Unaligned); - break; - } - } - } // end explicit vectorization - - /* process remaining coeffs (or all if there is no explicit vectorization) */ - for (Index j=alignedSize; j<size; ++j) + pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0))); + pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1))); + pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2))); + pstoreu(res+i+ResPacketSize*3, pmadd(c3,palpha,ploadu<ResPacket>(res+i+ResPacketSize*3))); + pstoreu(res+i+ResPacketSize*4, pmadd(c4,palpha,ploadu<ResPacket>(res+i+ResPacketSize*4))); + pstoreu(res+i+ResPacketSize*5, pmadd(c5,palpha,ploadu<ResPacket>(res+i+ResPacketSize*5))); + pstoreu(res+i+ResPacketSize*6, pmadd(c6,palpha,ploadu<ResPacket>(res+i+ResPacketSize*6))); + pstoreu(res+i+ResPacketSize*7, pmadd(c7,palpha,ploadu<ResPacket>(res+i+ResPacketSize*7))); + } + if(i<n4) { - res[j] = cj.pmadd(lhs0(j), pfirst(ptmp0), res[j]); - res[j] = cj.pmadd(lhs1(j), pfirst(ptmp1), res[j]); - res[j] = cj.pmadd(lhs2(j), pfirst(ptmp2), res[j]); - res[j] = cj.pmadd(lhs3(j), pfirst(ptmp3), res[j]); + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)), + c2 = pset1<ResPacket>(ResScalar(0)), + c3 = pset1<ResPacket>(ResScalar(0)); + + for(Index j=j2; j<jend; j+=1) + { + RhsPacket b0 = pset1<RhsPacket>(rhs(j,0)); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1); + c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2); + c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*3,j),b0,c3); + } + pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0))); + pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1))); + pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2))); + pstoreu(res+i+ResPacketSize*3, pmadd(c3,palpha,ploadu<ResPacket>(res+i+ResPacketSize*3))); + + i+=ResPacketSize*4; + } + if(i<n3) + { + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)), + c2 = pset1<ResPacket>(ResScalar(0)); + + for(Index j=j2; j<jend; j+=1) + { + RhsPacket b0 = pset1<RhsPacket>(rhs(j,0)); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1); + c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2); + } + pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0))); + pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1))); + pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2))); + + i+=ResPacketSize*3; + } + if(i<n2) + { + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)); + + for(Index j=j2; j<jend; j+=1) + { + RhsPacket b0 = pset1<RhsPacket>(rhs(j,0)); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1); + } + pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0))); + pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1))); + i+=ResPacketSize*2; + } + if(i<n1) + { + ResPacket c0 = pset1<ResPacket>(ResScalar(0)); + for(Index j=j2; j<jend; j+=1) + { + RhsPacket b0 = pset1<RhsPacket>(rhs(j,0)); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+0,j),b0,c0); + } + pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0))); + i+=ResPacketSize; + } + for(;i<rows;++i) + { + ResScalar c0(0); + for(Index j=j2; j<jend; j+=1) + c0 += cj.pmul(lhs(i,j), rhs(j,0)); + res[i] += alpha*c0; } } - - // process remaining first and last columns (at most columnsAtOnce-1) - Index end = cols; - Index start = columnBound; - do - { - for (Index k=start; k<end; ++k) - { - RhsPacket ptmp0 = pset1<RhsPacket>(alpha*rhs(k, 0)); - const LhsScalars lhs0 = lhs.getVectorMapper(0, k); - - if (Vectorizable) - { - /* explicit vectorization */ - // process first unaligned result's coeffs - for (Index j=0; j<alignedStart; ++j) - res[j] += cj.pmul(lhs0(j), pfirst(ptmp0)); - // process aligned result's coeffs - if (lhs0.template aligned<LhsPacket>(alignedStart)) - for (Index i = alignedStart;i<alignedSize;i+=ResPacketSize) - pstore(&res[i], pcj.pmadd(lhs0.template load<LhsPacket, Aligned>(i), ptmp0, pload<ResPacket>(&res[i]))); - else - for (Index i = alignedStart;i<alignedSize;i+=ResPacketSize) - pstore(&res[i], pcj.pmadd(lhs0.template load<LhsPacket, Unaligned>(i), ptmp0, pload<ResPacket>(&res[i]))); - } - - // process remaining scalars (or all if no explicit vectorization) - for (Index i=alignedSize; i<size; ++i) - res[i] += cj.pmul(lhs0(i), pfirst(ptmp0)); - } - if (skipColumns) - { - start = 0; - end = skipColumns; - skipColumns = 0; - } - else - break; - } while(Vectorizable); - #undef _EIGEN_ACCUMULATE_PACKETS } /* Optimized row-major matrix * vector product: - * This algorithm processes 4 rows at onces that allows to both reduce + * This algorithm processes 4 rows at once that allows to both reduce * the number of load/stores of the result by a factor 4 and to reduce * the instruction dependency. Moreover, we know that all bands have the * same alignment pattern. @@ -334,7 +213,7 @@ template<typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version> struct general_matrix_vector_product<Index,LhsScalar,LhsMapper,RowMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version> { -typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; +typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { Vectorizable = packet_traits<LhsScalar>::Vectorizable && packet_traits<RhsScalar>::Vectorizable @@ -352,7 +231,7 @@ typedef typename conditional<Vectorizable,_RhsPacket,RhsScalar>::type RhsPacket; typedef typename conditional<Vectorizable,_ResPacket,ResScalar>::type ResPacket; -EIGEN_DONT_INLINE static void run( +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run( Index rows, Index cols, const LhsMapper& lhs, const RhsMapper& rhs, @@ -361,255 +240,162 @@ }; template<typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version> -EIGEN_DONT_INLINE void general_matrix_vector_product<Index,LhsScalar,LhsMapper,RowMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version>::run( +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product<Index,LhsScalar,LhsMapper,RowMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version>::run( Index rows, Index cols, - const LhsMapper& lhs, + const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res, Index resIncr, ResScalar alpha) { + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate propoer code. + LhsMapper lhs(alhs); + eigen_internal_assert(rhs.stride()==1); - - #ifdef _EIGEN_ACCUMULATE_PACKETS - #error _EIGEN_ACCUMULATE_PACKETS has already been defined - #endif - - #define _EIGEN_ACCUMULATE_PACKETS(Alignment0,Alignment13,Alignment2) {\ - RhsPacket b = rhs.getVectorMapper(j, 0).template load<RhsPacket, Aligned>(0); \ - ptmp0 = pcj.pmadd(lhs0.template load<LhsPacket, Alignment0>(j), b, ptmp0); \ - ptmp1 = pcj.pmadd(lhs1.template load<LhsPacket, Alignment13>(j), b, ptmp1); \ - ptmp2 = pcj.pmadd(lhs2.template load<LhsPacket, Alignment2>(j), b, ptmp2); \ - ptmp3 = pcj.pmadd(lhs3.template load<LhsPacket, Alignment13>(j), b, ptmp3); } - conj_helper<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs> cj; conj_helper<LhsPacket,RhsPacket,ConjugateLhs,ConjugateRhs> pcj; - typedef typename LhsMapper::VectorMapper LhsScalars; + // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large, + // processing 8 rows at once might be counter productive wrt cache. + const Index n8 = lhs.stride()*sizeof(LhsScalar)>32000 ? 0 : rows-7; + const Index n4 = rows-3; + const Index n2 = rows-1; - enum { AllAligned=0, EvenAligned=1, FirstAligned=2, NoneAligned=3 }; - const Index rowsAtOnce = 4; - const Index peels = 2; - const Index RhsPacketAlignedMask = RhsPacketSize-1; - const Index LhsPacketAlignedMask = LhsPacketSize-1; - const Index depth = cols; - const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { LhsAlignment = Unaligned }; - // How many coeffs of the result do we have to skip to be aligned. - // Here we assume data are at least aligned on the base scalar type - // if that's not the case then vectorization is discarded, see below. - Index alignedStart = rhs.firstAligned(depth); - Index alignedSize = RhsPacketSize>1 ? alignedStart + ((depth-alignedStart) & ~RhsPacketAlignedMask) : 0; - const Index peeledSize = alignedSize - RhsPacketSize*peels - RhsPacketSize + 1; - - const Index alignmentStep = LhsPacketSize>1 ? (LhsPacketSize - lhsStride % LhsPacketSize) & LhsPacketAlignedMask : 0; - Index alignmentPattern = alignmentStep==0 ? AllAligned - : alignmentStep==(LhsPacketSize/2) ? EvenAligned - : FirstAligned; - - // we cannot assume the first element is aligned because of sub-matrices - const Index lhsAlignmentOffset = lhs.firstAligned(depth); - const Index rhsAlignmentOffset = rhs.firstAligned(rows); - - // find how many rows do we have to skip to be aligned with rhs (if possible) - Index skipRows = 0; - // if the data cannot be aligned (TODO add some compile time tests when possible, e.g. for floats) - if( (sizeof(LhsScalar)!=sizeof(RhsScalar)) || - (lhsAlignmentOffset < 0) || (lhsAlignmentOffset == depth) || - (rhsAlignmentOffset < 0) || (rhsAlignmentOffset == rows) ) + Index i=0; + for(; i<n8; i+=8) { - alignedSize = 0; - alignedStart = 0; - alignmentPattern = NoneAligned; - } - else if(LhsPacketSize > 4) - { - // TODO: extend the code to support aligned loads whenever possible when LhsPacketSize > 4. - alignmentPattern = NoneAligned; - } - else if (LhsPacketSize>1) - { - // eigen_internal_assert(size_t(firstLhs+lhsAlignmentOffset)%sizeof(LhsPacket)==0 || depth<LhsPacketSize); + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)), + c2 = pset1<ResPacket>(ResScalar(0)), + c3 = pset1<ResPacket>(ResScalar(0)), + c4 = pset1<ResPacket>(ResScalar(0)), + c5 = pset1<ResPacket>(ResScalar(0)), + c6 = pset1<ResPacket>(ResScalar(0)), + c7 = pset1<ResPacket>(ResScalar(0)); - while (skipRows<LhsPacketSize && - alignedStart != ((lhsAlignmentOffset + alignmentStep*skipRows)%LhsPacketSize)) - ++skipRows; - if (skipRows==LhsPacketSize) + Index j=0; + for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) { - // nothing can be aligned, no need to skip any column - alignmentPattern = NoneAligned; - skipRows = 0; + RhsPacket b0 = rhs.template load<RhsPacket, Unaligned>(j,0); + + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+1,j),b0,c1); + c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+2,j),b0,c2); + c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+3,j),b0,c3); + c4 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+4,j),b0,c4); + c5 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+5,j),b0,c5); + c6 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+6,j),b0,c6); + c7 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+7,j),b0,c7); } - else + ResScalar cc0 = predux(c0); + ResScalar cc1 = predux(c1); + ResScalar cc2 = predux(c2); + ResScalar cc3 = predux(c3); + ResScalar cc4 = predux(c4); + ResScalar cc5 = predux(c5); + ResScalar cc6 = predux(c6); + ResScalar cc7 = predux(c7); + for(; j<cols; ++j) { - skipRows = (std::min)(skipRows,Index(rows)); - // note that the skiped columns are processed later. + RhsScalar b0 = rhs(j,0); + + cc0 += cj.pmul(lhs(i+0,j), b0); + cc1 += cj.pmul(lhs(i+1,j), b0); + cc2 += cj.pmul(lhs(i+2,j), b0); + cc3 += cj.pmul(lhs(i+3,j), b0); + cc4 += cj.pmul(lhs(i+4,j), b0); + cc5 += cj.pmul(lhs(i+5,j), b0); + cc6 += cj.pmul(lhs(i+6,j), b0); + cc7 += cj.pmul(lhs(i+7,j), b0); } - /* eigen_internal_assert( alignmentPattern==NoneAligned - || LhsPacketSize==1 - || (skipRows + rowsAtOnce >= rows) - || LhsPacketSize > depth - || (size_t(firstLhs+alignedStart+lhsStride*skipRows)%sizeof(LhsPacket))==0);*/ + res[(i+0)*resIncr] += alpha*cc0; + res[(i+1)*resIncr] += alpha*cc1; + res[(i+2)*resIncr] += alpha*cc2; + res[(i+3)*resIncr] += alpha*cc3; + res[(i+4)*resIncr] += alpha*cc4; + res[(i+5)*resIncr] += alpha*cc5; + res[(i+6)*resIncr] += alpha*cc6; + res[(i+7)*resIncr] += alpha*cc7; } - else if(Vectorizable) + for(; i<n4; i+=4) { - alignedStart = 0; - alignedSize = depth; - alignmentPattern = AllAligned; + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)), + c2 = pset1<ResPacket>(ResScalar(0)), + c3 = pset1<ResPacket>(ResScalar(0)); + + Index j=0; + for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) + { + RhsPacket b0 = rhs.template load<RhsPacket, Unaligned>(j,0); + + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+1,j),b0,c1); + c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+2,j),b0,c2); + c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+3,j),b0,c3); + } + ResScalar cc0 = predux(c0); + ResScalar cc1 = predux(c1); + ResScalar cc2 = predux(c2); + ResScalar cc3 = predux(c3); + for(; j<cols; ++j) + { + RhsScalar b0 = rhs(j,0); + + cc0 += cj.pmul(lhs(i+0,j), b0); + cc1 += cj.pmul(lhs(i+1,j), b0); + cc2 += cj.pmul(lhs(i+2,j), b0); + cc3 += cj.pmul(lhs(i+3,j), b0); + } + res[(i+0)*resIncr] += alpha*cc0; + res[(i+1)*resIncr] += alpha*cc1; + res[(i+2)*resIncr] += alpha*cc2; + res[(i+3)*resIncr] += alpha*cc3; } - - const Index offset1 = (FirstAligned && alignmentStep==1?3:1); - const Index offset3 = (FirstAligned && alignmentStep==1?1:3); - - Index rowBound = ((rows-skipRows)/rowsAtOnce)*rowsAtOnce + skipRows; - for (Index i=skipRows; i<rowBound; i+=rowsAtOnce) + for(; i<n2; i+=2) { - // FIXME: what is the purpose of this EIGEN_ALIGN_DEFAULT ?? - EIGEN_ALIGN_MAX ResScalar tmp0 = ResScalar(0); - ResScalar tmp1 = ResScalar(0), tmp2 = ResScalar(0), tmp3 = ResScalar(0); + ResPacket c0 = pset1<ResPacket>(ResScalar(0)), + c1 = pset1<ResPacket>(ResScalar(0)); - // this helps the compiler generating good binary code - const LhsScalars lhs0 = lhs.getVectorMapper(i+0, 0), lhs1 = lhs.getVectorMapper(i+offset1, 0), - lhs2 = lhs.getVectorMapper(i+2, 0), lhs3 = lhs.getVectorMapper(i+offset3, 0); - - if (Vectorizable) + Index j=0; + for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) { - /* explicit vectorization */ - ResPacket ptmp0 = pset1<ResPacket>(ResScalar(0)), ptmp1 = pset1<ResPacket>(ResScalar(0)), - ptmp2 = pset1<ResPacket>(ResScalar(0)), ptmp3 = pset1<ResPacket>(ResScalar(0)); + RhsPacket b0 = rhs.template load<RhsPacket, Unaligned>(j,0); - // process initial unaligned coeffs - // FIXME this loop get vectorized by the compiler ! - for (Index j=0; j<alignedStart; ++j) - { - RhsScalar b = rhs(j, 0); - tmp0 += cj.pmul(lhs0(j),b); tmp1 += cj.pmul(lhs1(j),b); - tmp2 += cj.pmul(lhs2(j),b); tmp3 += cj.pmul(lhs3(j),b); - } - - if (alignedSize>alignedStart) - { - switch(alignmentPattern) - { - case AllAligned: - for (Index j = alignedStart; j<alignedSize; j+=RhsPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Aligned,Aligned); - break; - case EvenAligned: - for (Index j = alignedStart; j<alignedSize; j+=RhsPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Unaligned,Aligned); - break; - case FirstAligned: - { - Index j = alignedStart; - if (peels>1) - { - /* Here we proccess 4 rows with with two peeled iterations to hide - * the overhead of unaligned loads. Moreover unaligned loads are handled - * using special shift/move operations between the two aligned packets - * overlaping the desired unaligned packet. This is *much* more efficient - * than basic unaligned loads. - */ - LhsPacket A01, A02, A03, A11, A12, A13; - A01 = lhs1.template load<LhsPacket, Aligned>(alignedStart-1); - A02 = lhs2.template load<LhsPacket, Aligned>(alignedStart-2); - A03 = lhs3.template load<LhsPacket, Aligned>(alignedStart-3); - - for (; j<peeledSize; j+=peels*RhsPacketSize) - { - RhsPacket b = rhs.getVectorMapper(j, 0).template load<RhsPacket, Aligned>(0); - A11 = lhs1.template load<LhsPacket, Aligned>(j-1+LhsPacketSize); palign<1>(A01,A11); - A12 = lhs2.template load<LhsPacket, Aligned>(j-2+LhsPacketSize); palign<2>(A02,A12); - A13 = lhs3.template load<LhsPacket, Aligned>(j-3+LhsPacketSize); palign<3>(A03,A13); - - ptmp0 = pcj.pmadd(lhs0.template load<LhsPacket, Aligned>(j), b, ptmp0); - ptmp1 = pcj.pmadd(A01, b, ptmp1); - A01 = lhs1.template load<LhsPacket, Aligned>(j-1+2*LhsPacketSize); palign<1>(A11,A01); - ptmp2 = pcj.pmadd(A02, b, ptmp2); - A02 = lhs2.template load<LhsPacket, Aligned>(j-2+2*LhsPacketSize); palign<2>(A12,A02); - ptmp3 = pcj.pmadd(A03, b, ptmp3); - A03 = lhs3.template load<LhsPacket, Aligned>(j-3+2*LhsPacketSize); palign<3>(A13,A03); - - b = rhs.getVectorMapper(j+RhsPacketSize, 0).template load<RhsPacket, Aligned>(0); - ptmp0 = pcj.pmadd(lhs0.template load<LhsPacket, Aligned>(j+LhsPacketSize), b, ptmp0); - ptmp1 = pcj.pmadd(A11, b, ptmp1); - ptmp2 = pcj.pmadd(A12, b, ptmp2); - ptmp3 = pcj.pmadd(A13, b, ptmp3); - } - } - for (; j<alignedSize; j+=RhsPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Aligned,Unaligned,Unaligned); - break; - } - default: - for (Index j = alignedStart; j<alignedSize; j+=RhsPacketSize) - _EIGEN_ACCUMULATE_PACKETS(Unaligned,Unaligned,Unaligned); - break; - } - tmp0 += predux(ptmp0); - tmp1 += predux(ptmp1); - tmp2 += predux(ptmp2); - tmp3 += predux(ptmp3); - } - } // end explicit vectorization - - // process remaining coeffs (or all if no explicit vectorization) - // FIXME this loop get vectorized by the compiler ! - for (Index j=alignedSize; j<depth; ++j) - { - RhsScalar b = rhs(j, 0); - tmp0 += cj.pmul(lhs0(j),b); tmp1 += cj.pmul(lhs1(j),b); - tmp2 += cj.pmul(lhs2(j),b); tmp3 += cj.pmul(lhs3(j),b); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+0,j),b0,c0); + c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+1,j),b0,c1); } - res[i*resIncr] += alpha*tmp0; - res[(i+offset1)*resIncr] += alpha*tmp1; - res[(i+2)*resIncr] += alpha*tmp2; - res[(i+offset3)*resIncr] += alpha*tmp3; + ResScalar cc0 = predux(c0); + ResScalar cc1 = predux(c1); + for(; j<cols; ++j) + { + RhsScalar b0 = rhs(j,0); + + cc0 += cj.pmul(lhs(i+0,j), b0); + cc1 += cj.pmul(lhs(i+1,j), b0); + } + res[(i+0)*resIncr] += alpha*cc0; + res[(i+1)*resIncr] += alpha*cc1; } - - // process remaining first and last rows (at most columnsAtOnce-1) - Index end = rows; - Index start = rowBound; - do + for(; i<rows; ++i) { - for (Index i=start; i<end; ++i) + ResPacket c0 = pset1<ResPacket>(ResScalar(0)); + Index j=0; + for(; j+LhsPacketSize<=cols; j+=LhsPacketSize) { - EIGEN_ALIGN_MAX ResScalar tmp0 = ResScalar(0); - ResPacket ptmp0 = pset1<ResPacket>(tmp0); - const LhsScalars lhs0 = lhs.getVectorMapper(i, 0); - // process first unaligned result's coeffs - // FIXME this loop get vectorized by the compiler ! - for (Index j=0; j<alignedStart; ++j) - tmp0 += cj.pmul(lhs0(j), rhs(j, 0)); - - if (alignedSize>alignedStart) - { - // process aligned rhs coeffs - if (lhs0.template aligned<LhsPacket>(alignedStart)) - for (Index j = alignedStart;j<alignedSize;j+=RhsPacketSize) - ptmp0 = pcj.pmadd(lhs0.template load<LhsPacket, Aligned>(j), rhs.getVectorMapper(j, 0).template load<RhsPacket, Aligned>(0), ptmp0); - else - for (Index j = alignedStart;j<alignedSize;j+=RhsPacketSize) - ptmp0 = pcj.pmadd(lhs0.template load<LhsPacket, Unaligned>(j), rhs.getVectorMapper(j, 0).template load<RhsPacket, Aligned>(0), ptmp0); - tmp0 += predux(ptmp0); - } - - // process remaining scalars - // FIXME this loop get vectorized by the compiler ! - for (Index j=alignedSize; j<depth; ++j) - tmp0 += cj.pmul(lhs0(j), rhs(j, 0)); - res[i*resIncr] += alpha*tmp0; + RhsPacket b0 = rhs.template load<RhsPacket,Unaligned>(j,0); + c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i,j),b0,c0); } - if (skipRows) + ResScalar cc0 = predux(c0); + for(; j<cols; ++j) { - start = 0; - end = skipRows; - skipRows = 0; + cc0 += cj.pmul(lhs(i,j), rhs(j,0)); } - else - break; - } while(Vectorizable); - - #undef _EIGEN_ACCUMULATE_PACKETS + res[i*resIncr] += alpha*cc0; + } } } // end namespace internal
diff --git a/Eigen/src/Core/products/GeneralMatrixVector_BLAS.h b/Eigen/src/Core/products/GeneralMatrixVector_BLAS.h index e3a5d58..6e36c2b 100644 --- a/Eigen/src/Core/products/GeneralMatrixVector_BLAS.h +++ b/Eigen/src/Core/products/GeneralMatrixVector_BLAS.h
@@ -85,7 +85,7 @@ EIGEN_BLAS_GEMV_SPECIALIZE(dcomplex) EIGEN_BLAS_GEMV_SPECIALIZE(scomplex) -#define EIGEN_BLAS_GEMV_SPECIALIZATION(EIGTYPE,BLASTYPE,BLASPREFIX) \ +#define EIGEN_BLAS_GEMV_SPECIALIZATION(EIGTYPE,BLASTYPE,BLASFUNC) \ template<typename Index, int LhsStorageOrder, bool ConjugateLhs, bool ConjugateRhs> \ struct general_matrix_vector_product_gemv<Index,EIGTYPE,LhsStorageOrder,ConjugateLhs,EIGTYPE,ConjugateRhs> \ { \ @@ -113,14 +113,21 @@ x_ptr=x_tmp.data(); \ incx=1; \ } else x_ptr=rhs; \ - BLASPREFIX##gemv_(&trans, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)lhs, &lda, (const BLASTYPE*)x_ptr, &incx, &numext::real_ref(beta), (BLASTYPE*)res, &incy); \ + BLASFUNC(&trans, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)lhs, &lda, (const BLASTYPE*)x_ptr, &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &incy); \ }\ }; -EIGEN_BLAS_GEMV_SPECIALIZATION(double, double, d) -EIGEN_BLAS_GEMV_SPECIALIZATION(float, float, s) -EIGEN_BLAS_GEMV_SPECIALIZATION(dcomplex, double, z) -EIGEN_BLAS_GEMV_SPECIALIZATION(scomplex, float, c) +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_GEMV_SPECIALIZATION(double, double, dgemv) +EIGEN_BLAS_GEMV_SPECIALIZATION(float, float, sgemv) +EIGEN_BLAS_GEMV_SPECIALIZATION(dcomplex, MKL_Complex16, zgemv) +EIGEN_BLAS_GEMV_SPECIALIZATION(scomplex, MKL_Complex8 , cgemv) +#else +EIGEN_BLAS_GEMV_SPECIALIZATION(double, double, dgemv_) +EIGEN_BLAS_GEMV_SPECIALIZATION(float, float, sgemv_) +EIGEN_BLAS_GEMV_SPECIALIZATION(dcomplex, double, zgemv_) +EIGEN_BLAS_GEMV_SPECIALIZATION(scomplex, float, cgemv_) +#endif } // end namespase internal
diff --git a/Eigen/src/Core/products/Parallelizer.h b/Eigen/src/Core/products/Parallelizer.h index e0bfcc3..92e9b0d 100644 --- a/Eigen/src/Core/products/Parallelizer.h +++ b/Eigen/src/Core/products/Parallelizer.h
@@ -10,7 +10,11 @@ #ifndef EIGEN_PARALLELIZER_H #define EIGEN_PARALLELIZER_H -namespace Eigen { +#if EIGEN_HAS_CXX11_ATOMIC +#include <atomic> +#endif + +namespace Eigen { namespace internal { @@ -75,23 +79,36 @@ { GemmParallelInfo() : sync(-1), users(0), lhs_start(0), lhs_length(0) {} - int volatile sync; + // volatile is not enough on all architectures (see bug 1572) + // to guarantee that when thread A says to thread B that it is + // done with packing a block, then all writes have been really + // carried out... C++11 memory model+atomic guarantees this. +#if EIGEN_HAS_CXX11_ATOMIC + std::atomic<Index> sync; + std::atomic<int> users; +#else + Index volatile sync; int volatile users; +#endif Index lhs_start; Index lhs_length; }; template<bool Condition, typename Functor, typename Index> -void parallelize_gemm(const Functor& func, Index rows, Index cols, bool transpose) +void parallelize_gemm(const Functor& func, Index rows, Index cols, Index depth, bool transpose) { // TODO when EIGEN_USE_BLAS is defined, // we should still enable OMP for other scalar types -#if !(defined (EIGEN_HAS_OPENMP)) || defined (EIGEN_USE_BLAS) + // Without C++11, we have to disable GEMM's parallelization on + // non x86 architectures because there volatile is not enough for our purpose. + // See bug 1572. +#if (! defined(EIGEN_HAS_OPENMP)) || defined(EIGEN_USE_BLAS) || ((!EIGEN_HAS_CXX11_ATOMIC) && !(EIGEN_ARCH_i386_OR_x86_64)) // FIXME the transpose variable is only needed to properly split // the matrix product when multithreading is enabled. This is a temporary // fix to support row-major destination matrices. This whole - // parallelizer mechanism has to be redisigned anyway. + // parallelizer mechanism has to be redesigned anyway. + EIGEN_UNUSED_VARIABLE(depth); EIGEN_UNUSED_VARIABLE(transpose); func(0,rows, 0,cols); #else @@ -103,13 +120,20 @@ // - the sizes are large enough // compute the maximal number of threads from the size of the product: - // FIXME this has to be fine tuned + // This first heuristic takes into account that the product kernel is fully optimized when working with nr columns at once. Index size = transpose ? rows : cols; - Index pb_max_threads = std::max<Index>(1,size / 32); + Index pb_max_threads = std::max<Index>(1,size / Functor::Traits::nr); + + // compute the maximal number of threads from the total amount of work: + double work = static_cast<double>(rows) * static_cast<double>(cols) * + static_cast<double>(depth); + double kMinTaskSize = 50000; // FIXME improve this heuristic. + pb_max_threads = std::max<Index>(1, std::min<Index>(pb_max_threads, work / kMinTaskSize)); + // compute the number of threads we are going to use Index threads = std::min<Index>(nbThreads(), pb_max_threads); - // if multi-threading is explicitely disabled, not useful, or if we already are in a parallel session, + // if multi-threading is explicitly disabled, not useful, or if we already are in a parallel session, // then abort multi-threading // FIXME omp_get_num_threads()>1 only works for openmp, what if the user does not use openmp? if((!Condition) || (threads==1) || (omp_get_num_threads()>1)) @@ -120,19 +144,19 @@ if(transpose) std::swap(rows,cols); - + ei_declare_aligned_stack_constructed_variable(GemmParallelInfo<Index>,info,threads,0); - + #pragma omp parallel num_threads(threads) { Index i = omp_get_thread_num(); // Note that the actual number of threads might be lower than the number of request ones. Index actual_threads = omp_get_num_threads(); - + Index blockCols = (cols / actual_threads) & ~Index(0x3); Index blockRows = (rows / actual_threads); blockRows = (blockRows/Functor::Traits::mr)*Functor::Traits::mr; - + Index r0 = i*blockRows; Index actualBlockRows = (i+1==actual_threads) ? rows-r0 : blockRows;
diff --git a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h index da6f82a..6730736 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h +++ b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h
@@ -45,14 +45,23 @@ } void operator()(Scalar* blockA, const Scalar* _lhs, Index lhsStride, Index cols, Index rows) { - enum { PacketSize = packet_traits<Scalar>::size }; + typedef typename unpacket_traits<typename packet_traits<Scalar>::type>::half HalfPacket; + typedef typename unpacket_traits<typename unpacket_traits<typename packet_traits<Scalar>::type>::half>::half QuarterPacket; + enum { PacketSize = packet_traits<Scalar>::size, + HalfPacketSize = unpacket_traits<HalfPacket>::size, + QuarterPacketSize = unpacket_traits<QuarterPacket>::size, + HasHalf = (int)HalfPacketSize < (int)PacketSize, + HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize}; + const_blas_data_mapper<Scalar,Index,StorageOrder> lhs(_lhs,lhsStride); Index count = 0; //Index peeled_mc3 = (rows/Pack1)*Pack1; const Index peeled_mc3 = Pack1>=3*PacketSize ? (rows/(3*PacketSize))*(3*PacketSize) : 0; const Index peeled_mc2 = Pack1>=2*PacketSize ? peeled_mc3+((rows-peeled_mc3)/(2*PacketSize))*(2*PacketSize) : 0; - const Index peeled_mc1 = Pack1>=1*PacketSize ? (rows/(1*PacketSize))*(1*PacketSize) : 0; + const Index peeled_mc1 = Pack1>=1*PacketSize ? peeled_mc2+((rows-peeled_mc2)/(1*PacketSize))*(1*PacketSize) : 0; + const Index peeled_mc_half = Pack1>=HalfPacketSize ? peeled_mc1+((rows-peeled_mc1)/(HalfPacketSize))*(HalfPacketSize) : 0; + const Index peeled_mc_quarter = Pack1>=QuarterPacketSize ? peeled_mc_half+((rows-peeled_mc_half)/(QuarterPacketSize))*(QuarterPacketSize) : 0; if(Pack1>=3*PacketSize) for(Index i=0; i<peeled_mc3; i+=3*PacketSize) @@ -66,8 +75,16 @@ for(Index i=peeled_mc2; i<peeled_mc1; i+=1*PacketSize) pack<1*PacketSize>(blockA, lhs, cols, i, count); + if(HasHalf && Pack1>=HalfPacketSize) + for(Index i=peeled_mc1; i<peeled_mc_half; i+=HalfPacketSize) + pack<HalfPacketSize>(blockA, lhs, cols, i, count); + + if(HasQuarter && Pack1>=QuarterPacketSize) + for(Index i=peeled_mc_half; i<peeled_mc_quarter; i+=QuarterPacketSize) + pack<QuarterPacketSize>(blockA, lhs, cols, i, count); + // do the same with mr==1 - for(Index i=peeled_mc1; i<rows; i++) + for(Index i=peeled_mc_quarter; i<rows; i++) { for(Index k=0; k<i; k++) blockA[count++] = lhs(i, k); // normal @@ -352,7 +369,7 @@ gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel; symm_pack_lhs<Scalar, Index, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr,RhsStorageOrder> pack_rhs; - gemm_pack_lhs<Scalar, Index, LhsTransposeMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder==RowMajor?ColMajor:RowMajor, true> pack_lhs_transposed; + gemm_pack_lhs<Scalar, Index, LhsTransposeMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder==RowMajor?ColMajor:RowMajor, true> pack_lhs_transposed; for(Index k2=0; k2<size; k2+=kc) { @@ -387,7 +404,7 @@ for(Index i2=k2+kc; i2<size; i2+=mc) { const Index actual_mc = (std::min)(i2+mc,size)-i2; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder,false>() + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder,false>() (blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc); gebp_kernel(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, cols, alpha); @@ -437,7 +454,7 @@ ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB()); gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder> pack_lhs; symm_pack_rhs<Scalar, Index, Traits::nr,RhsStorageOrder> pack_rhs; for(Index k2=0; k2<size; k2+=kc)
diff --git a/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h b/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h index c3e37b1..9a53185 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h +++ b/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h
@@ -40,7 +40,7 @@ /* Optimized selfadjoint matrix * matrix (?SYMM/?HEMM) product */ -#define EIGEN_BLAS_SYMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_SYMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -81,13 +81,13 @@ ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } else b = _rhs; \ \ - BLASPREFIX##symm_(&side, &uplo, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, &numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ + BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ \ } \ }; -#define EIGEN_BLAS_HEMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_HEMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -122,7 +122,7 @@ Map<const Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>, 0, OuterStride<> > lhs(_lhs,m,m,OuterStride<>(lhsStride)); \ a_tmp = lhs.conjugate(); \ a = a_tmp.data(); \ - lda = a_tmp.outerStride(); \ + lda = convert_index<BlasIndex>(a_tmp.outerStride()); \ } else a = _lhs; \ if (LhsStorageOrder==RowMajor) uplo='U'; \ \ @@ -144,20 +144,26 @@ ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } \ \ - BLASPREFIX##hemm_(&side, &uplo, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, &numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ + BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ \ } \ }; -EIGEN_BLAS_SYMM_L(double, double, d, d) -EIGEN_BLAS_SYMM_L(float, float, f, s) -EIGEN_BLAS_HEMM_L(dcomplex, double, cd, z) -EIGEN_BLAS_HEMM_L(scomplex, float, cf, c) - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_SYMM_L(double, double, d, dsymm) +EIGEN_BLAS_SYMM_L(float, float, f, ssymm) +EIGEN_BLAS_HEMM_L(dcomplex, MKL_Complex16, cd, zhemm) +EIGEN_BLAS_HEMM_L(scomplex, MKL_Complex8, cf, chemm) +#else +EIGEN_BLAS_SYMM_L(double, double, d, dsymm_) +EIGEN_BLAS_SYMM_L(float, float, f, ssymm_) +EIGEN_BLAS_HEMM_L(dcomplex, double, cd, zhemm_) +EIGEN_BLAS_HEMM_L(scomplex, float, cf, chemm_) +#endif /* Optimized matrix * selfadjoint matrix (?SYMM/?HEMM) product */ -#define EIGEN_BLAS_SYMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_SYMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -197,13 +203,13 @@ ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } else b = _lhs; \ \ - BLASPREFIX##symm_(&side, &uplo, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, &numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ + BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ \ } \ }; -#define EIGEN_BLAS_HEMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_HEMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -256,18 +262,24 @@ b_tmp = lhs.transpose(); \ } \ b = b_tmp.data(); \ - ldb = b_tmp.outerStride(); \ + ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } \ \ - BLASPREFIX##hemm_(&side, &uplo, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, &numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ + BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ } \ }; -EIGEN_BLAS_SYMM_R(double, double, d, d) -EIGEN_BLAS_SYMM_R(float, float, f, s) -EIGEN_BLAS_HEMM_R(dcomplex, double, cd, z) -EIGEN_BLAS_HEMM_R(scomplex, float, cf, c) - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_SYMM_R(double, double, d, dsymm) +EIGEN_BLAS_SYMM_R(float, float, f, ssymm) +EIGEN_BLAS_HEMM_R(dcomplex, MKL_Complex16, cd, zhemm) +EIGEN_BLAS_HEMM_R(scomplex, MKL_Complex8, cf, chemm) +#else +EIGEN_BLAS_SYMM_R(double, double, d, dsymm_) +EIGEN_BLAS_SYMM_R(float, float, f, ssymm_) +EIGEN_BLAS_HEMM_R(dcomplex, double, cd, zhemm_) +EIGEN_BLAS_HEMM_R(scomplex, float, cf, chemm_) +#endif } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector.h b/Eigen/src/Core/products/SelfadjointMatrixVector.h index d8d3026..d38fd72 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector.h
@@ -15,7 +15,7 @@ namespace internal { /* Optimized selfadjoint matrix * vector product: - * This algorithm processes 2 columns at onces that allows to both reduce + * This algorithm processes 2 columns at once that allows to both reduce * the number of load/stores of the result by a factor 2 and to reduce * the instruction dependency. */ @@ -27,7 +27,8 @@ struct selfadjoint_matrix_vector_product { -static EIGEN_DONT_INLINE void run( +static EIGEN_DONT_INLINE EIGEN_DEVICE_FUNC +void run( Index size, const Scalar* lhs, Index lhsStride, const Scalar* rhs, @@ -36,7 +37,8 @@ }; template<typename Scalar, typename Index, int StorageOrder, int UpLo, bool ConjugateLhs, bool ConjugateRhs, int Version> -EIGEN_DONT_INLINE void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrder,UpLo,ConjugateLhs,ConjugateRhs,Version>::run( +EIGEN_DONT_INLINE EIGEN_DEVICE_FUNC +void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrder,UpLo,ConjugateLhs,ConjugateRhs,Version>::run( Index size, const Scalar* lhs, Index lhsStride, const Scalar* rhs, @@ -62,8 +64,7 @@ Scalar cjAlpha = ConjugateRhs ? numext::conj(alpha) : alpha; - - Index bound = (std::max)(Index(0),size-8) & 0xfffffffe; + Index bound = numext::maxi(Index(0), size-8) & 0xfffffffe; if (FirstTriangular) bound = size - bound; @@ -83,10 +84,10 @@ Scalar t3(0); Packet ptmp3 = pset1<Packet>(t3); - size_t starti = FirstTriangular ? 0 : j+2; - size_t endi = FirstTriangular ? j : size; - size_t alignedStart = (starti) + internal::first_default_aligned(&res[starti], endi-starti); - size_t alignedEnd = alignedStart + ((endi-alignedStart)/(PacketSize))*(PacketSize); + Index starti = FirstTriangular ? 0 : j+2; + Index endi = FirstTriangular ? j : size; + Index alignedStart = (starti) + internal::first_default_aligned(&res[starti], endi-starti); + Index alignedEnd = alignedStart + ((endi-alignedStart)/(PacketSize))*(PacketSize); res[j] += cjd.pmul(numext::real(A0[j]), t0); res[j+1] += cjd.pmul(numext::real(A1[j+1]), t1); @@ -101,7 +102,7 @@ t2 += cj1.pmul(A0[j+1], rhs[j+1]); } - for (size_t i=starti; i<alignedStart; ++i) + for (Index i=starti; i<alignedStart; ++i) { res[i] += cj0.pmul(A0[i], t0) + cj0.pmul(A1[i],t1); t2 += cj1.pmul(A0[i], rhs[i]); @@ -113,7 +114,7 @@ const Scalar* EIGEN_RESTRICT a1It = A1 + alignedStart; const Scalar* EIGEN_RESTRICT rhsIt = rhs + alignedStart; Scalar* EIGEN_RESTRICT resIt = res + alignedStart; - for (size_t i=alignedStart; i<alignedEnd; i+=PacketSize) + for (Index i=alignedStart; i<alignedEnd; i+=PacketSize) { Packet A0i = ploadu<Packet>(a0It); a0It += PacketSize; Packet A1i = ploadu<Packet>(a1It); a1It += PacketSize; @@ -125,7 +126,7 @@ ptmp3 = pcj1.pmadd(A1i, Bi, ptmp3); pstore(resIt,Xi); resIt += PacketSize; } - for (size_t i=alignedEnd; i<endi; i++) + for (Index i=alignedEnd; i<endi; i++) { res[i] += cj0.pmul(A0[i], t0) + cj0.pmul(A1[i],t1); t2 += cj1.pmul(A0[i], rhs[i]); @@ -175,11 +176,12 @@ enum { LhsUpLo = LhsMode&(Upper|Lower) }; template<typename Dest> - static void run(Dest& dest, const Lhs &a_lhs, const Rhs &a_rhs, const Scalar& alpha) + static EIGEN_DEVICE_FUNC + void run(Dest& dest, const Lhs &a_lhs, const Rhs &a_rhs, const Scalar& alpha) { typedef typename Dest::Scalar ResScalar; typedef typename Rhs::Scalar RhsScalar; - typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; + typedef Map<Matrix<ResScalar,Dynamic,1>, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits<ResScalar>::size)> MappedDest; eigen_assert(dest.rows()==a_lhs.rows() && dest.cols()==a_rhs.cols());
diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h index 38f23ac..1238345 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h
@@ -95,14 +95,21 @@ x_tmp=map_x.conjugate(); \ x_ptr=x_tmp.data(); \ } else x_ptr=_rhs; \ - BLASFUNC(&uplo, &n, &numext::real_ref(alpha), (const BLASTYPE*)lhs, &lda, (const BLASTYPE*)x_ptr, &incx, &numext::real_ref(beta), (BLASTYPE*)res, &incy); \ + BLASFUNC(&uplo, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)lhs, &lda, (const BLASTYPE*)x_ptr, &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &incy); \ }\ }; +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_SYMV_SPECIALIZATION(double, double, dsymv) +EIGEN_BLAS_SYMV_SPECIALIZATION(float, float, ssymv) +EIGEN_BLAS_SYMV_SPECIALIZATION(dcomplex, MKL_Complex16, zhemv) +EIGEN_BLAS_SYMV_SPECIALIZATION(scomplex, MKL_Complex8, chemv) +#else EIGEN_BLAS_SYMV_SPECIALIZATION(double, double, dsymv_) EIGEN_BLAS_SYMV_SPECIALIZATION(float, float, ssymv_) EIGEN_BLAS_SYMV_SPECIALIZATION(dcomplex, double, zhemv_) EIGEN_BLAS_SYMV_SPECIALIZATION(scomplex, float, chemv_) +#endif } // end namespace internal
diff --git a/Eigen/src/Core/products/SelfadjointProduct.h b/Eigen/src/Core/products/SelfadjointProduct.h index f038d68..39c5b59 100644 --- a/Eigen/src/Core/products/SelfadjointProduct.h +++ b/Eigen/src/Core/products/SelfadjointProduct.h
@@ -120,7 +120,7 @@ template<typename MatrixType, unsigned int UpLo> template<typename DerivedU> -SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo> +EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo> ::rankUpdate(const MatrixBase<DerivedU>& u, const Scalar& alpha) { selfadjoint_product_selector<MatrixType,DerivedU,UpLo>::run(_expression().const_cast_derived(), u.derived(), alpha);
diff --git a/Eigen/src/Core/products/SelfadjointRank2Update.h b/Eigen/src/Core/products/SelfadjointRank2Update.h index 2ae3641..09209f7 100644 --- a/Eigen/src/Core/products/SelfadjointRank2Update.h +++ b/Eigen/src/Core/products/SelfadjointRank2Update.h
@@ -24,7 +24,8 @@ template<typename Scalar, typename Index, typename UType, typename VType> struct selfadjoint_rank2_update_selector<Scalar,Index,UType,VType,Lower> { - static void run(Scalar* mat, Index stride, const UType& u, const VType& v, const Scalar& alpha) + static EIGEN_DEVICE_FUNC + void run(Scalar* mat, Index stride, const UType& u, const VType& v, const Scalar& alpha) { const Index size = u.size(); for (Index i=0; i<size; ++i) @@ -57,7 +58,7 @@ template<typename MatrixType, unsigned int UpLo> template<typename DerivedU, typename DerivedV> -SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo> +EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo> ::rankUpdate(const MatrixBase<DerivedU>& u, const MatrixBase<DerivedV>& v, const Scalar& alpha) { typedef internal::blas_traits<DerivedU> UBlasTraits;
diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Eigen/src/Core/products/TriangularMatrixMatrix.h index 8a2f7cd..85fd744 100644 --- a/Eigen/src/Core/products/TriangularMatrixMatrix.h +++ b/Eigen/src/Core/products/TriangularMatrixMatrix.h
@@ -137,7 +137,13 @@ ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA()); ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB()); - Matrix<Scalar,SmallPanelWidth,SmallPanelWidth,LhsStorageOrder> triangularBuffer; + // To work around an "error: member reference base type 'Matrix<...> + // (Eigen::internal::constructor_without_unaligned_array_assert (*)())' is + // not a structure or union" compilation error in nvcc (tested V8.0.61), + // create a dummy internal::constructor_without_unaligned_array_assert + // object to pass to the Matrix constructor. + internal::constructor_without_unaligned_array_assert a; + Matrix<Scalar,SmallPanelWidth,SmallPanelWidth,LhsStorageOrder> triangularBuffer(a); triangularBuffer.setZero(); if((Mode&ZeroDiag)==ZeroDiag) triangularBuffer.diagonal().setZero(); @@ -145,7 +151,7 @@ triangularBuffer.diagonal().setOnes(); gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder> pack_lhs; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr,RhsStorageOrder> pack_rhs; for(Index k2=IsLower ? depth : 0; @@ -216,7 +222,7 @@ for(Index i2=start; i2<end; i2+=mc) { const Index actual_mc = (std::min)(i2+mc,end)-i2; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr,Traits::LhsProgress, LhsStorageOrder,false>() + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr,Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder,false>() (blockA, lhs.getSubMapper(i2, actual_k2), actual_kc, actual_mc); gebp_kernel(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, @@ -284,7 +290,8 @@ ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA()); ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB()); - Matrix<Scalar,SmallPanelWidth,SmallPanelWidth,RhsStorageOrder> triangularBuffer; + internal::constructor_without_unaligned_array_assert a; + Matrix<Scalar,SmallPanelWidth,SmallPanelWidth,RhsStorageOrder> triangularBuffer(a); triangularBuffer.setZero(); if((Mode&ZeroDiag)==ZeroDiag) triangularBuffer.diagonal().setZero(); @@ -292,7 +299,7 @@ triangularBuffer.diagonal().setOnes(); gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs; + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, LhsStorageOrder> pack_lhs; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr,RhsStorageOrder> pack_rhs; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr,RhsStorageOrder,false,true> pack_rhs_panel; @@ -393,7 +400,9 @@ { template<typename Dest> static void run(Dest& dst, const Lhs &a_lhs, const Rhs &a_rhs, const typename Dest::Scalar& alpha) { - typedef typename Dest::Scalar Scalar; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef typename Dest::Scalar Scalar; typedef internal::blas_traits<Lhs> LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; @@ -405,8 +414,9 @@ typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); - Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) - * RhsBlasTraits::extractScalarFactor(a_rhs); + LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(a_lhs); + RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(a_rhs); + Scalar actualAlpha = alpha * lhs_alpha * rhs_alpha; typedef internal::gemm_blocking_space<(Dest::Flags&RowMajorBit) ? RowMajor : ColMajor,Scalar,Scalar, Lhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime, Lhs::MaxColsAtCompileTime,4> BlockingType; @@ -431,6 +441,21 @@ &dst.coeffRef(0,0), dst.outerStride(), // result info actualAlpha, blocking ); + + // Apply correction if the diagonal is unit and a scalar factor was nested: + if ((Mode&UnitDiag)==UnitDiag) + { + if (LhsIsTriangular && lhs_alpha!=LhsScalar(1)) + { + Index diagSize = (std::min)(lhs.rows(),lhs.cols()); + dst.topRows(diagSize) -= ((lhs_alpha-LhsScalar(1))*a_rhs).topRows(diagSize); + } + else if ((!LhsIsTriangular) && rhs_alpha!=RhsScalar(1)) + { + Index diagSize = (std::min)(rhs.rows(),rhs.cols()); + dst.leftCols(diagSize) -= (rhs_alpha-RhsScalar(1))*a_lhs.leftCols(diagSize); + } + } } };
diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h b/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h index aecded6..a25197a 100644 --- a/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h +++ b/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h
@@ -75,7 +75,7 @@ EIGEN_BLAS_TRMM_SPECIALIZE(scomplex, false) // implements col-major += alpha * op(triangular) * op(general) -#define EIGEN_BLAS_TRMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_TRMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, int Mode, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -172,7 +172,7 @@ } \ /*std::cout << "TRMM_L: A is square! Go to BLAS TRMM implementation! \n";*/ \ /* call ?trmm*/ \ - BLASPREFIX##trmm_(&side, &uplo, &transa, &diag, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)b, &ldb); \ + BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)b, &ldb); \ \ /* Add op(a_triangular)*b into res*/ \ Map<MatrixX##EIGPREFIX, 0, OuterStride<> > res_tmp(res,rows,cols,OuterStride<>(resStride)); \ @@ -180,13 +180,20 @@ } \ }; -EIGEN_BLAS_TRMM_L(double, double, d, d) -EIGEN_BLAS_TRMM_L(dcomplex, double, cd, z) -EIGEN_BLAS_TRMM_L(float, float, f, s) -EIGEN_BLAS_TRMM_L(scomplex, float, cf, c) +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRMM_L(double, double, d, dtrmm) +EIGEN_BLAS_TRMM_L(dcomplex, MKL_Complex16, cd, ztrmm) +EIGEN_BLAS_TRMM_L(float, float, f, strmm) +EIGEN_BLAS_TRMM_L(scomplex, MKL_Complex8, cf, ctrmm) +#else +EIGEN_BLAS_TRMM_L(double, double, d, dtrmm_) +EIGEN_BLAS_TRMM_L(dcomplex, double, cd, ztrmm_) +EIGEN_BLAS_TRMM_L(float, float, f, strmm_) +EIGEN_BLAS_TRMM_L(scomplex, float, cf, ctrmm_) +#endif // implements col-major += alpha * op(general) * op(triangular) -#define EIGEN_BLAS_TRMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_TRMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \ template <typename Index, int Mode, \ int LhsStorageOrder, bool ConjugateLhs, \ int RhsStorageOrder, bool ConjugateRhs> \ @@ -282,7 +289,7 @@ } \ /*std::cout << "TRMM_R: A is square! Go to BLAS TRMM implementation! \n";*/ \ /* call ?trmm*/ \ - BLASPREFIX##trmm_(&side, &uplo, &transa, &diag, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)b, &ldb); \ + BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)b, &ldb); \ \ /* Add op(a_triangular)*b into res*/ \ Map<MatrixX##EIGPREFIX, 0, OuterStride<> > res_tmp(res,rows,cols,OuterStride<>(resStride)); \ @@ -290,11 +297,17 @@ } \ }; -EIGEN_BLAS_TRMM_R(double, double, d, d) -EIGEN_BLAS_TRMM_R(dcomplex, double, cd, z) -EIGEN_BLAS_TRMM_R(float, float, f, s) -EIGEN_BLAS_TRMM_R(scomplex, float, cf, c) - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRMM_R(double, double, d, dtrmm) +EIGEN_BLAS_TRMM_R(dcomplex, MKL_Complex16, cd, ztrmm) +EIGEN_BLAS_TRMM_R(float, float, f, strmm) +EIGEN_BLAS_TRMM_R(scomplex, MKL_Complex8, cf, ctrmm) +#else +EIGEN_BLAS_TRMM_R(double, double, d, dtrmm_) +EIGEN_BLAS_TRMM_R(dcomplex, double, cd, ztrmm_) +EIGEN_BLAS_TRMM_R(float, float, f, strmm_) +EIGEN_BLAS_TRMM_R(scomplex, float, cf, ctrmm_) +#endif } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/products/TriangularMatrixVector.h b/Eigen/src/Core/products/TriangularMatrixVector.h index f79840a..76bfa15 100644 --- a/Eigen/src/Core/products/TriangularMatrixVector.h +++ b/Eigen/src/Core/products/TriangularMatrixVector.h
@@ -20,7 +20,7 @@ template<typename Index, int Mode, typename LhsScalar, bool ConjLhs, typename RhsScalar, bool ConjRhs, int Version> struct triangular_matrix_vector_product<Index,Mode,LhsScalar,ConjLhs,RhsScalar,ConjRhs,ColMajor,Version> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { IsLower = ((Mode&Lower)==Lower), HasUnitDiag = (Mode & UnitDiag)==UnitDiag, @@ -91,7 +91,7 @@ template<typename Index, int Mode, typename LhsScalar, bool ConjLhs, typename RhsScalar, bool ConjRhs,int Version> struct triangular_matrix_vector_product<Index,Mode,LhsScalar,ConjLhs,RhsScalar,ConjRhs,RowMajor,Version> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { IsLower = ((Mode&Lower)==Lower), HasUnitDiag = (Mode & UnitDiag)==UnitDiag, @@ -216,13 +216,14 @@ typedef internal::blas_traits<Rhs> RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; - typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; + typedef Map<Matrix<ResScalar,Dynamic,1>, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits<ResScalar>::size)> MappedDest; typename internal::add_const_on_value_type<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(lhs); typename internal::add_const_on_value_type<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(rhs); - ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) - * RhsBlasTraits::extractScalarFactor(rhs); + LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(lhs); + RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(rhs); + ResScalar actualAlpha = alpha * lhs_alpha * rhs_alpha; enum { // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1 @@ -274,6 +275,12 @@ else dest = MappedDest(actualDestPtr, dest.size()); } + + if ( ((Mode&UnitDiag)==UnitDiag) && (lhs_alpha!=LhsScalar(1)) ) + { + Index diagSize = (std::min)(lhs.rows(),lhs.cols()); + dest.head(diagSize) -= (lhs_alpha-LhsScalar(1))*rhs.head(diagSize); + } } }; @@ -295,8 +302,9 @@ typename add_const<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(lhs); typename add_const<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(rhs); - ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) - * RhsBlasTraits::extractScalarFactor(rhs); + LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(lhs); + RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(rhs); + ResScalar actualAlpha = alpha * lhs_alpha * rhs_alpha; enum { DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 @@ -326,6 +334,12 @@ actualRhsPtr,1, dest.data(),dest.innerStride(), actualAlpha); + + if ( ((Mode&UnitDiag)==UnitDiag) && (lhs_alpha!=LhsScalar(1)) ) + { + Index diagSize = (std::min)(lhs.rows(),lhs.cols()); + dest.head(diagSize) -= (lhs_alpha-LhsScalar(1))*rhs.head(diagSize); + } } };
diff --git a/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h b/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h index 07bf26c..3d47a2b 100644 --- a/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h +++ b/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h
@@ -71,7 +71,7 @@ EIGEN_BLAS_TRMV_SPECIALIZE(scomplex) // implements col-major: res += alpha * op(triangular) * vector -#define EIGEN_BLAS_TRMV_CM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_TRMV_CM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX, BLASPOSTFIX) \ template<typename Index, int Mode, bool ConjLhs, bool ConjRhs> \ struct triangular_matrix_vector_product_trmv<Index,Mode,EIGTYPE,ConjLhs,EIGTYPE,ConjRhs,ColMajor> { \ enum { \ @@ -121,10 +121,10 @@ diag = IsUnitDiag ? 'U' : 'N'; \ \ /* call ?TRMV*/ \ - BLASPREFIX##trmv_(&uplo, &trans, &diag, &n, (const BLASTYPE*)_lhs, &lda, (BLASTYPE*)x, &incx); \ + BLASPREFIX##trmv##BLASPOSTFIX(&uplo, &trans, &diag, &n, (const BLASTYPE*)_lhs, &lda, (BLASTYPE*)x, &incx); \ \ /* Add op(a_tr)rhs into res*/ \ - BLASPREFIX##axpy_(&n, &numext::real_ref(alpha),(const BLASTYPE*)x, &incx, (BLASTYPE*)_res, &incy); \ + BLASPREFIX##axpy##BLASPOSTFIX(&n, (const BLASTYPE*)&numext::real_ref(alpha),(const BLASTYPE*)x, &incx, (BLASTYPE*)_res, &incy); \ /* Non-square case - doesn't fit to BLAS ?TRMV. Fall to default triangular product*/ \ if (size<(std::max)(rows,cols)) { \ if (ConjRhs) x_tmp = rhs.conjugate(); else x_tmp = rhs; \ @@ -142,18 +142,25 @@ m = convert_index<BlasIndex>(size); \ n = convert_index<BlasIndex>(cols-size); \ } \ - BLASPREFIX##gemv_(&trans, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, &incx, &numext::real_ref(beta), (BLASTYPE*)y, &incy); \ + BLASPREFIX##gemv##BLASPOSTFIX(&trans, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)y, &incy); \ } \ } \ }; -EIGEN_BLAS_TRMV_CM(double, double, d, d) -EIGEN_BLAS_TRMV_CM(dcomplex, double, cd, z) -EIGEN_BLAS_TRMV_CM(float, float, f, s) -EIGEN_BLAS_TRMV_CM(scomplex, float, cf, c) +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRMV_CM(double, double, d, d,) +EIGEN_BLAS_TRMV_CM(dcomplex, MKL_Complex16, cd, z,) +EIGEN_BLAS_TRMV_CM(float, float, f, s,) +EIGEN_BLAS_TRMV_CM(scomplex, MKL_Complex8, cf, c,) +#else +EIGEN_BLAS_TRMV_CM(double, double, d, d, _) +EIGEN_BLAS_TRMV_CM(dcomplex, double, cd, z, _) +EIGEN_BLAS_TRMV_CM(float, float, f, s, _) +EIGEN_BLAS_TRMV_CM(scomplex, float, cf, c, _) +#endif // implements row-major: res += alpha * op(triangular) * vector -#define EIGEN_BLAS_TRMV_RM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX) \ +#define EIGEN_BLAS_TRMV_RM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX, BLASPOSTFIX) \ template<typename Index, int Mode, bool ConjLhs, bool ConjRhs> \ struct triangular_matrix_vector_product_trmv<Index,Mode,EIGTYPE,ConjLhs,EIGTYPE,ConjRhs,RowMajor> { \ enum { \ @@ -203,10 +210,10 @@ diag = IsUnitDiag ? 'U' : 'N'; \ \ /* call ?TRMV*/ \ - BLASPREFIX##trmv_(&uplo, &trans, &diag, &n, (const BLASTYPE*)_lhs, &lda, (BLASTYPE*)x, &incx); \ + BLASPREFIX##trmv##BLASPOSTFIX(&uplo, &trans, &diag, &n, (const BLASTYPE*)_lhs, &lda, (BLASTYPE*)x, &incx); \ \ /* Add op(a_tr)rhs into res*/ \ - BLASPREFIX##axpy_(&n, &numext::real_ref(alpha),(const BLASTYPE*)x, &incx, (BLASTYPE*)_res, &incy); \ + BLASPREFIX##axpy##BLASPOSTFIX(&n, (const BLASTYPE*)&numext::real_ref(alpha),(const BLASTYPE*)x, &incx, (BLASTYPE*)_res, &incy); \ /* Non-square case - doesn't fit to BLAS ?TRMV. Fall to default triangular product*/ \ if (size<(std::max)(rows,cols)) { \ if (ConjRhs) x_tmp = rhs.conjugate(); else x_tmp = rhs; \ @@ -224,15 +231,22 @@ m = convert_index<BlasIndex>(size); \ n = convert_index<BlasIndex>(cols-size); \ } \ - BLASPREFIX##gemv_(&trans, &n, &m, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, &incx, &numext::real_ref(beta), (BLASTYPE*)y, &incy); \ + BLASPREFIX##gemv##BLASPOSTFIX(&trans, &n, &m, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)y, &incy); \ } \ } \ }; -EIGEN_BLAS_TRMV_RM(double, double, d, d) -EIGEN_BLAS_TRMV_RM(dcomplex, double, cd, z) -EIGEN_BLAS_TRMV_RM(float, float, f, s) -EIGEN_BLAS_TRMV_RM(scomplex, float, cf, c) +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRMV_RM(double, double, d, d,) +EIGEN_BLAS_TRMV_RM(dcomplex, MKL_Complex16, cd, z,) +EIGEN_BLAS_TRMV_RM(float, float, f, s,) +EIGEN_BLAS_TRMV_RM(scomplex, MKL_Complex8, cf, c,) +#else +EIGEN_BLAS_TRMV_RM(double, double, d, d,_) +EIGEN_BLAS_TRMV_RM(dcomplex, double, cd, z,_) +EIGEN_BLAS_TRMV_RM(float, float, f, s,_) +EIGEN_BLAS_TRMV_RM(scomplex, float, cf, c,_) +#endif } // end namespase internal
diff --git a/Eigen/src/Core/products/TriangularSolverMatrix.h b/Eigen/src/Core/products/TriangularSolverMatrix.h index 1bed66e..8ff2e9d 100644 --- a/Eigen/src/Core/products/TriangularSolverMatrix.h +++ b/Eigen/src/Core/products/TriangularSolverMatrix.h
@@ -76,7 +76,7 @@ conj_if<Conjugate> conj; gebp_kernel<Scalar, Scalar, Index, OtherMapper, Traits::mr, Traits::nr, Conjugate, false> gebp_kernel; - gemm_pack_lhs<Scalar, Index, TriMapper, Traits::mr, Traits::LhsProgress, TriStorageOrder> pack_lhs; + gemm_pack_lhs<Scalar, Index, TriMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, TriStorageOrder> pack_lhs; gemm_pack_rhs<Scalar, Index, OtherMapper, Traits::nr, ColMajor, false, true> pack_rhs; // the goal here is to subdivise the Rhs panels such that we keep some cache @@ -183,7 +183,7 @@ } } -/* Optimized triangular solver with multiple left hand sides and the trinagular matrix on the right +/* Optimized triangular solver with multiple left hand sides and the triangular matrix on the right */ template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder> struct triangular_solve_matrix<Scalar,Index,OnTheRight,Mode,Conjugate,TriStorageOrder,ColMajor> @@ -202,6 +202,7 @@ level3_blocking<Scalar,Scalar>& blocking) { Index rows = otherSize; + typedef typename NumTraits<Scalar>::Real RealScalar; typedef blas_data_mapper<Scalar, Index, ColMajor> LhsMapper; typedef const_blas_data_mapper<Scalar, Index, TriStorageOrder> RhsMapper; @@ -228,7 +229,7 @@ gebp_kernel<Scalar, Scalar, Index, LhsMapper, Traits::mr, Traits::nr, false, Conjugate> gebp_kernel; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs; gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder,false,true> pack_rhs_panel; - gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, ColMajor, false, true> pack_lhs_panel; + gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, ColMajor, false, true> pack_lhs_panel; for(Index k2=IsLower ? size : 0; IsLower ? k2>0 : k2<size; @@ -306,9 +307,9 @@ } if((Mode & UnitDiag)==0) { - Scalar b = conj(rhs(j,j)); + Scalar inv_rjj = RealScalar(1)/conj(rhs(j,j)); for (Index i=0; i<actual_mc; ++i) - r[i] /= b; + r[i] *= inv_rjj; } }
diff --git a/Eigen/src/Core/products/TriangularSolverMatrix_BLAS.h b/Eigen/src/Core/products/TriangularSolverMatrix_BLAS.h index 88c0fb7..f077511 100644 --- a/Eigen/src/Core/products/TriangularSolverMatrix_BLAS.h +++ b/Eigen/src/Core/products/TriangularSolverMatrix_BLAS.h
@@ -38,7 +38,7 @@ namespace internal { // implements LeftSide op(triangular)^-1 * general -#define EIGEN_BLAS_TRSM_L(EIGTYPE, BLASTYPE, BLASPREFIX) \ +#define EIGEN_BLAS_TRSM_L(EIGTYPE, BLASTYPE, BLASFUNC) \ template <typename Index, int Mode, bool Conjugate, int TriStorageOrder> \ struct triangular_solve_matrix<EIGTYPE,Index,OnTheLeft,Mode,Conjugate,TriStorageOrder,ColMajor> \ { \ @@ -80,18 +80,24 @@ } \ if (IsUnitDiag) diag='U'; \ /* call ?trsm*/ \ - BLASPREFIX##trsm_(&side, &uplo, &transa, &diag, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)_other, &ldb); \ + BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)_other, &ldb); \ } \ }; -EIGEN_BLAS_TRSM_L(double, double, d) -EIGEN_BLAS_TRSM_L(dcomplex, double, z) -EIGEN_BLAS_TRSM_L(float, float, s) -EIGEN_BLAS_TRSM_L(scomplex, float, c) - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRSM_L(double, double, dtrsm) +EIGEN_BLAS_TRSM_L(dcomplex, MKL_Complex16, ztrsm) +EIGEN_BLAS_TRSM_L(float, float, strsm) +EIGEN_BLAS_TRSM_L(scomplex, MKL_Complex8, ctrsm) +#else +EIGEN_BLAS_TRSM_L(double, double, dtrsm_) +EIGEN_BLAS_TRSM_L(dcomplex, double, ztrsm_) +EIGEN_BLAS_TRSM_L(float, float, strsm_) +EIGEN_BLAS_TRSM_L(scomplex, float, ctrsm_) +#endif // implements RightSide general * op(triangular)^-1 -#define EIGEN_BLAS_TRSM_R(EIGTYPE, BLASTYPE, BLASPREFIX) \ +#define EIGEN_BLAS_TRSM_R(EIGTYPE, BLASTYPE, BLASFUNC) \ template <typename Index, int Mode, bool Conjugate, int TriStorageOrder> \ struct triangular_solve_matrix<EIGTYPE,Index,OnTheRight,Mode,Conjugate,TriStorageOrder,ColMajor> \ { \ @@ -133,16 +139,22 @@ } \ if (IsUnitDiag) diag='U'; \ /* call ?trsm*/ \ - BLASPREFIX##trsm_(&side, &uplo, &transa, &diag, &m, &n, &numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)_other, &ldb); \ + BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (BLASTYPE*)_other, &ldb); \ /*std::cout << "TRMS_L specialization!\n";*/ \ } \ }; -EIGEN_BLAS_TRSM_R(double, double, d) -EIGEN_BLAS_TRSM_R(dcomplex, double, z) -EIGEN_BLAS_TRSM_R(float, float, s) -EIGEN_BLAS_TRSM_R(scomplex, float, c) - +#ifdef EIGEN_USE_MKL +EIGEN_BLAS_TRSM_R(double, double, dtrsm) +EIGEN_BLAS_TRSM_R(dcomplex, MKL_Complex16, ztrsm) +EIGEN_BLAS_TRSM_R(float, float, strsm) +EIGEN_BLAS_TRSM_R(scomplex, MKL_Complex8, ctrsm) +#else +EIGEN_BLAS_TRSM_R(double, double, dtrsm_) +EIGEN_BLAS_TRSM_R(dcomplex, double, ztrsm_) +EIGEN_BLAS_TRSM_R(float, float, strsm_) +EIGEN_BLAS_TRSM_R(scomplex, float, ctrsm_) +#endif } // end namespace internal
diff --git a/Eigen/src/Core/products/TriangularSolverVector.h b/Eigen/src/Core/products/TriangularSolverVector.h index b994759..6473170 100644 --- a/Eigen/src/Core/products/TriangularSolverVector.h +++ b/Eigen/src/Core/products/TriangularSolverVector.h
@@ -58,7 +58,7 @@ { // let's directly call the low level product function because: // 1 - it is faster to compile - // 2 - it is slighlty faster at runtime + // 2 - it is slightly faster at runtime Index startRow = IsLower ? pi : pi-actualPanelWidth; Index startCol = IsLower ? 0 : pi; @@ -77,7 +77,7 @@ if (k>0) rhs[i] -= (cjLhs.row(i).segment(s,k).transpose().cwiseProduct(Map<const Matrix<RhsScalar,Dynamic,1> >(rhs+s,k))).sum(); - if(!(Mode & UnitDiag)) + if((!(Mode & UnitDiag)) && numext::not_equal_strict(rhs[i],RhsScalar(0))) rhs[i] /= cjLhs(i,i); } } @@ -114,20 +114,23 @@ for(Index k=0; k<actualPanelWidth; ++k) { Index i = IsLower ? pi+k : pi-k-1; - if(!(Mode & UnitDiag)) - rhs[i] /= cjLhs.coeff(i,i); + if(numext::not_equal_strict(rhs[i],RhsScalar(0))) + { + if(!(Mode & UnitDiag)) + rhs[i] /= cjLhs.coeff(i,i); - Index r = actualPanelWidth - k - 1; // remaining size - Index s = IsLower ? i+1 : i-r; - if (r>0) - Map<Matrix<RhsScalar,Dynamic,1> >(rhs+s,r) -= rhs[i] * cjLhs.col(i).segment(s,r); + Index r = actualPanelWidth - k - 1; // remaining size + Index s = IsLower ? i+1 : i-r; + if (r>0) + Map<Matrix<RhsScalar,Dynamic,1> >(rhs+s,r) -= rhs[i] * cjLhs.col(i).segment(s,r); + } } Index r = IsLower ? size - endBlock : startBlock; // remaining size if (r > 0) { // let's directly call the low level product function because: // 1 - it is faster to compile - // 2 - it is slighlty faster at runtime + // 2 - it is slightly faster at runtime general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,Conjugate,RhsScalar,RhsMapper,false>::run( r, actualPanelWidth, LhsMapper(&lhs.coeffRef(endBlock,startBlock), lhsStride),
diff --git a/Eigen/src/Core/util/BlasUtil.h b/Eigen/src/Core/util/BlasUtil.h index 498db3a..e6689c6 100755 --- a/Eigen/src/Core/util/BlasUtil.h +++ b/Eigen/src/Core/util/BlasUtil.h
@@ -24,7 +24,7 @@ template<typename Scalar, typename Index, typename DataMapper, int nr, int StorageOrder, bool Conjugate = false, bool PanelMode=false> struct gemm_pack_rhs; -template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, int StorageOrder, bool Conjugate = false, bool PanelMode = false> +template<typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, int StorageOrder, bool Conjugate = false, bool PanelMode = false> struct gemm_pack_lhs; template< @@ -44,16 +44,29 @@ template<> struct conj_if<true> { template<typename T> - inline T operator()(const T& x) { return numext::conj(x); } + inline T operator()(const T& x) const { return numext::conj(x); } template<typename T> - inline T pconj(const T& x) { return internal::pconj(x); } + inline T pconj(const T& x) const { return internal::pconj(x); } }; template<> struct conj_if<false> { template<typename T> - inline const T& operator()(const T& x) { return x; } + inline const T& operator()(const T& x) const { return x; } template<typename T> - inline const T& pconj(const T& x) { return x; } + inline const T& pconj(const T& x) const { return x; } +}; + +// Generic implementation for custom complex types. +template<typename LhsScalar, typename RhsScalar, bool ConjLhs, bool ConjRhs> +struct conj_helper +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar>::ReturnType Scalar; + + EIGEN_STRONG_INLINE Scalar pmadd(const LhsScalar& x, const RhsScalar& y, const Scalar& c) const + { return padd(c, pmul(x,y)); } + + EIGEN_STRONG_INLINE Scalar pmul(const LhsScalar& x, const RhsScalar& y) const + { return conj_if<ConjLhs>()(x) * conj_if<ConjRhs>()(y); } }; template<typename Scalar> struct conj_helper<Scalar,Scalar,false,false> @@ -111,7 +124,7 @@ }; template<typename From,typename To> struct get_factor { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE To run(const From& x) { return x; } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE To run(const From& x) { return To(x); } }; template<typename Scalar> struct get_factor<Scalar,typename NumTraits<Scalar>::Real> { @@ -135,7 +148,7 @@ template <typename Packet> EIGEN_DEVICE_FUNC bool aligned(Index i) const { - return (size_t(m_data+i)%sizeof(Packet))==0; + return (UIntPtr(m_data+i)%sizeof(Packet))==0; } protected: @@ -143,11 +156,9 @@ }; template<typename Scalar, typename Index, int AlignmentType> -class BlasLinearMapper { - public: - typedef typename packet_traits<Scalar>::type Packet; - typedef typename packet_traits<Scalar>::half HalfPacket; - +class BlasLinearMapper +{ +public: EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar *data) : m_data(data) {} EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(int i) const { @@ -158,29 +169,25 @@ return m_data[i]; } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i) const { - return ploadt<Packet, AlignmentType>(m_data + i); + template<typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const { + return ploadt<PacketType, AlignmentType>(m_data + i); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE HalfPacket loadHalfPacket(Index i) const { - return ploadt<HalfPacket, AlignmentType>(m_data + i); + template<typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType &p) const { + pstoret<Scalar, PacketType, AlignmentType>(m_data + i, p); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const Packet &p) const { - pstoret<Scalar, Packet, AlignmentType>(m_data + i, p); - } - - protected: +protected: Scalar *m_data; }; // Lightweight helper class to access matrix coefficients. template<typename Scalar, typename Index, int StorageOrder, int AlignmentType = Unaligned> -class blas_data_mapper { - public: - typedef typename packet_traits<Scalar>::type Packet; - typedef typename packet_traits<Scalar>::half HalfPacket; - +class blas_data_mapper +{ +public: typedef BlasLinearMapper<Scalar, Index, AlignmentType> LinearMapper; typedef BlasVectorMapper<Scalar, Index> VectorMapper; @@ -205,12 +212,14 @@ return m_data[StorageOrder==RowMajor ? j + i*m_stride : i + j*m_stride]; } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i, Index j) const { - return ploadt<Packet, AlignmentType>(&operator()(i, j)); + template<typename PacketType> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const { + return ploadt<PacketType, AlignmentType>(&operator()(i, j)); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE HalfPacket loadHalfPacket(Index i, Index j) const { - return ploadt<HalfPacket, AlignmentType>(&operator()(i, j)); + template <typename PacketT, int AlignmentT> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const { + return ploadt<PacketT, AlignmentT>(&operator()(i, j)); } template<typename SubPacket> @@ -227,13 +236,13 @@ EIGEN_DEVICE_FUNC const Scalar* data() const { return m_data; } EIGEN_DEVICE_FUNC Index firstAligned(Index size) const { - if (size_t(m_data)%sizeof(Scalar)) { + if (UIntPtr(m_data)%sizeof(Scalar)) { return -1; } return internal::first_default_aligned(m_data, size); } - protected: +protected: Scalar* EIGEN_RESTRICT m_data; const Index m_stride; }; @@ -265,14 +274,15 @@ HasUsableDirectAccess = ( (int(XprType::Flags)&DirectAccessBit) && ( bool(XprType::IsVectorAtCompileTime) || int(inner_stride_at_compile_time<XprType>::ret) == 1) - ) ? 1 : 0 + ) ? 1 : 0, + HasScalarFactor = false }; typedef typename conditional<bool(HasUsableDirectAccess), ExtractType, typename _ExtractType::PlainObject >::type DirectLinearAccessType; - static inline ExtractType extract(const XprType& x) { return x; } - static inline const Scalar extractScalarFactor(const XprType&) { return Scalar(1); } + static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return x; } + static inline EIGEN_DEVICE_FUNC const Scalar extractScalarFactor(const XprType&) { return Scalar(1); } }; // pop conjugate @@ -293,23 +303,48 @@ }; // pop scalar multiple -template<typename Scalar, typename NestedXpr> -struct blas_traits<CwiseUnaryOp<scalar_multiple_op<Scalar>, NestedXpr> > +template<typename Scalar, typename NestedXpr, typename Plain> +struct blas_traits<CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain>, NestedXpr> > : blas_traits<NestedXpr> { + enum { + HasScalarFactor = true + }; typedef blas_traits<NestedXpr> Base; - typedef CwiseUnaryOp<scalar_multiple_op<Scalar>, NestedXpr> XprType; + typedef CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain>, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; - static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } - static inline Scalar extractScalarFactor(const XprType& x) - { return x.functor().m_other * Base::extractScalarFactor(x.nestedExpression()); } + static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return Base::extract(x.rhs()); } + static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(const XprType& x) + { return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs()); } }; +template<typename Scalar, typename NestedXpr, typename Plain> +struct blas_traits<CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr, const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain> > > + : blas_traits<NestedXpr> +{ + enum { + HasScalarFactor = true + }; + typedef blas_traits<NestedXpr> Base; + typedef CwiseBinaryOp<scalar_product_op<Scalar>, NestedXpr, const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain> > XprType; + typedef typename Base::ExtractType ExtractType; + static inline ExtractType extract(const XprType& x) { return Base::extract(x.lhs()); } + static inline Scalar extractScalarFactor(const XprType& x) + { return Base::extractScalarFactor(x.lhs()) * x.rhs().functor().m_other; } +}; +template<typename Scalar, typename Plain1, typename Plain2> +struct blas_traits<CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain1>, + const CwiseNullaryOp<scalar_constant_op<Scalar>,Plain2> > > + : blas_traits<CwiseNullaryOp<scalar_constant_op<Scalar>,Plain1> > +{}; // pop opposite template<typename Scalar, typename NestedXpr> struct blas_traits<CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr> > : blas_traits<NestedXpr> { + enum { + HasScalarFactor = true + }; typedef blas_traits<NestedXpr> Base; typedef CwiseUnaryOp<scalar_opposite_op<Scalar>, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType;
diff --git a/Eigen/src/Core/util/CMakeLists.txt b/Eigen/src/Core/util/CMakeLists.txt deleted file mode 100644 index a1e2e52..0000000 --- a/Eigen/src/Core/util/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Core_util_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Core_util_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Core/util COMPONENT Devel - )
diff --git a/Eigen/src/Core/util/ConfigureVectorization.h b/Eigen/src/Core/util/ConfigureVectorization.h new file mode 100644 index 0000000..b00d8b0 --- /dev/null +++ b/Eigen/src/Core/util/ConfigureVectorization.h
@@ -0,0 +1,449 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CONFIGURE_VECTORIZATION_H +#define EIGEN_CONFIGURE_VECTORIZATION_H + +//------------------------------------------------------------------------------------------ +// Static and dynamic alignment control +// +// The main purpose of this section is to define EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES +// as the maximal boundary in bytes on which dynamically and statically allocated data may be alignment respectively. +// The values of EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES can be specified by the user. If not, +// a default value is automatically computed based on architecture, compiler, and OS. +// +// This section also defines macros EIGEN_ALIGN_TO_BOUNDARY(N) and the shortcuts EIGEN_ALIGN{8,16,32,_MAX} +// to be used to declare statically aligned buffers. +//------------------------------------------------------------------------------------------ + + +/* EIGEN_ALIGN_TO_BOUNDARY(n) forces data to be n-byte aligned. This is used to satisfy SIMD requirements. + * However, we do that EVEN if vectorization (EIGEN_VECTORIZE) is disabled, + * so that vectorization doesn't affect binary compatibility. + * + * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link + * vectorized and non-vectorized code. + * + * FIXME: this code can be cleaned up once we switch to proper C++11 only. + */ +#if (defined EIGEN_CUDACC) + #define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n) + #define EIGEN_ALIGNOF(x) __alignof(x) +#elif EIGEN_HAS_ALIGNAS + #define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n) + #define EIGEN_ALIGNOF(x) alignof(x) +#elif EIGEN_COMP_GNUC || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM + #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) + #define EIGEN_ALIGNOF(x) __alignof(x) +#elif EIGEN_COMP_MSVC + #define EIGEN_ALIGN_TO_BOUNDARY(n) __declspec(align(n)) + #define EIGEN_ALIGNOF(x) __alignof(x) +#elif EIGEN_COMP_SUNCC + // FIXME not sure about this one: + #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) + #define EIGEN_ALIGNOF(x) __alignof(x) +#else + #error Please tell me what is the equivalent of alignas(n) and alignof(x) for your compiler +#endif + +// If the user explicitly disable vectorization, then we also disable alignment +#if defined(EIGEN_DONT_VECTORIZE) + #if defined(EIGEN_GPUCC) + // GPU code is always vectorized and requires memory alignment for + // statically allocated buffers. + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 + #else + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 0 + #endif +#elif defined(__AVX512F__) + // 64 bytes static alignment is preferred only if really required + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 64 +#elif defined(__AVX__) + // 32 bytes static alignment is preferred only if really required + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 32 +#else + #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 +#endif + + +// EIGEN_MIN_ALIGN_BYTES defines the minimal value for which the notion of explicit alignment makes sense +#define EIGEN_MIN_ALIGN_BYTES 16 + +// Defined the boundary (in bytes) on which the data needs to be aligned. Note +// that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be +// aligned at all regardless of the value of this #define. + +#if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && EIGEN_MAX_STATIC_ALIGN_BYTES>0 +#error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY. +#endif + +// EIGEN_DONT_ALIGN_STATICALLY and EIGEN_DONT_ALIGN are deprecated +// They imply EIGEN_MAX_STATIC_ALIGN_BYTES=0 +#if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN) + #ifdef EIGEN_MAX_STATIC_ALIGN_BYTES + #undef EIGEN_MAX_STATIC_ALIGN_BYTES + #endif + #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 +#endif + +#ifndef EIGEN_MAX_STATIC_ALIGN_BYTES + + // Try to automatically guess what is the best default value for EIGEN_MAX_STATIC_ALIGN_BYTES + + // 16 byte alignment is only useful for vectorization. Since it affects the ABI, we need to enable + // 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always + // enable alignment, but it can be a cause of problems on some platforms, so we just disable it in + // certain common platform (compiler+architecture combinations) to avoid these problems. + // Only static alignment is really problematic (relies on nonstandard compiler extensions), + // try to keep heap alignment even when we have to disable static alignment. + #if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || EIGEN_ARCH_MIPS) + #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 + #elif EIGEN_ARCH_ARM_OR_ARM64 && EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(4, 6) + // Old versions of GCC on ARM, at least 4.4, were once seen to have buggy static alignment support. + // Not sure which version fixed it, hopefully it doesn't affect 4.7, which is still somewhat in use. + // 4.8 and newer seem definitely unaffected. + #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 + #else + #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0 + #endif + + // static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX + #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT \ + && !EIGEN_GCC3_OR_OLDER \ + && !EIGEN_COMP_SUNCC \ + && !EIGEN_OS_QNX + #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1 + #else + #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0 + #endif + + #if EIGEN_ARCH_WANTS_STACK_ALIGNMENT + #define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES + #else + #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 + #endif + +#endif + +// If EIGEN_MAX_ALIGN_BYTES is defined, then it is considered as an upper bound for EIGEN_MAX_STATIC_ALIGN_BYTES +#if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES<EIGEN_MAX_STATIC_ALIGN_BYTES +#undef EIGEN_MAX_STATIC_ALIGN_BYTES +#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES +#endif + +#if EIGEN_MAX_STATIC_ALIGN_BYTES==0 && !defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT) + #define EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT +#endif + +// At this stage, EIGEN_MAX_STATIC_ALIGN_BYTES>0 is the true test whether we want to align arrays on the stack or not. +// It takes into account both the user choice to explicitly enable/disable alignment (by setting EIGEN_MAX_STATIC_ALIGN_BYTES) +// and the architecture config (EIGEN_ARCH_WANTS_STACK_ALIGNMENT). +// Henceforth, only EIGEN_MAX_STATIC_ALIGN_BYTES should be used. + + +// Shortcuts to EIGEN_ALIGN_TO_BOUNDARY +#define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8) +#define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16) +#define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32) +#define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64) +#if EIGEN_MAX_STATIC_ALIGN_BYTES>0 +#define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES) +#else +#define EIGEN_ALIGN_MAX +#endif + + +// Dynamic alignment control + +#if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES>0 +#error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN. +#endif + +#ifdef EIGEN_DONT_ALIGN + #ifdef EIGEN_MAX_ALIGN_BYTES + #undef EIGEN_MAX_ALIGN_BYTES + #endif + #define EIGEN_MAX_ALIGN_BYTES 0 +#elif !defined(EIGEN_MAX_ALIGN_BYTES) + #define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES +#endif + +#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES +#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES +#else +#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES +#endif + + +#ifndef EIGEN_UNALIGNED_VECTORIZE +#define EIGEN_UNALIGNED_VECTORIZE 1 +#endif + +//---------------------------------------------------------------------- + +// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into +// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks +#if EIGEN_MAX_ALIGN_BYTES==0 + #ifndef EIGEN_DONT_VECTORIZE + #define EIGEN_DONT_VECTORIZE + #endif +#endif + + +// The following (except #include <malloc.h> and _M_IX86_FP ??) can likely be +// removed as gcc 4.1 and msvc 2008 are not supported anyways. +#if EIGEN_COMP_MSVC + #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled + #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later + // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. + #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 + #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER + #endif + #endif +#else + #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) + #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC + #endif +#endif + + +#if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC)) + + #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) + + // Defines symbols for compile-time detection of which instructions are + // used. + // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_SSE + #define EIGEN_VECTORIZE_SSE2 + + // Detect sse3/ssse3/sse4: + // gcc and icc defines __SSE3__, ... + // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you + // want to force the use of those instructions with msvc. + #ifdef __SSE3__ + #define EIGEN_VECTORIZE_SSE3 + #endif + #ifdef __SSSE3__ + #define EIGEN_VECTORIZE_SSSE3 + #endif + #ifdef __SSE4_1__ + #define EIGEN_VECTORIZE_SSE4_1 + #endif + #ifdef __SSE4_2__ + #define EIGEN_VECTORIZE_SSE4_2 + #endif + #ifdef __AVX__ + #define EIGEN_VECTORIZE_AVX + #define EIGEN_VECTORIZE_SSE3 + #define EIGEN_VECTORIZE_SSSE3 + #define EIGEN_VECTORIZE_SSE4_1 + #define EIGEN_VECTORIZE_SSE4_2 + #endif + #ifdef __AVX2__ + #define EIGEN_VECTORIZE_AVX2 + #define EIGEN_VECTORIZE_AVX + #define EIGEN_VECTORIZE_SSE3 + #define EIGEN_VECTORIZE_SSSE3 + #define EIGEN_VECTORIZE_SSE4_1 + #define EIGEN_VECTORIZE_SSE4_2 + #endif + #if defined(__FMA__) || (EIGEN_COMP_MSVC && defined(__AVX2__)) + // MSVC does not expose a switch dedicated for FMA + // For MSVC, AVX2 => FMA + #define EIGEN_VECTORIZE_FMA + #endif + #if defined(__AVX512F__) + #ifndef EIGEN_VECTORIZE_FMA + #if EIGEN_COMP_GNUC + #error Please add -mfma to your compiler flags: compiling with -mavx512f alone without SSE/AVX FMA is not supported (bug 1638). + #else + #error Please enable FMA in your compiler flags (e.g. -mfma): compiling with AVX512 alone without SSE/AVX FMA is not supported (bug 1638). + #endif + #endif + #define EIGEN_VECTORIZE_AVX512 + #define EIGEN_VECTORIZE_AVX2 + #define EIGEN_VECTORIZE_AVX + #define EIGEN_VECTORIZE_FMA + #define EIGEN_VECTORIZE_SSE3 + #define EIGEN_VECTORIZE_SSSE3 + #define EIGEN_VECTORIZE_SSE4_1 + #define EIGEN_VECTORIZE_SSE4_2 + #ifdef __AVX512DQ__ + #define EIGEN_VECTORIZE_AVX512DQ + #endif + #ifdef __AVX512ER__ + #define EIGEN_VECTORIZE_AVX512ER + #endif + #endif + + // include files + + // This extern "C" works around a MINGW-w64 compilation issue + // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354 + // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do). + // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations + // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know; + // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too. + // notice that since these are C headers, the extern "C" is theoretically needed anyways. + extern "C" { + // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. + // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: + #if EIGEN_COMP_ICC >= 1110 + #include <immintrin.h> + #else + #include <mmintrin.h> + #include <emmintrin.h> + #include <xmmintrin.h> + #ifdef EIGEN_VECTORIZE_SSE3 + #include <pmmintrin.h> + #endif + #ifdef EIGEN_VECTORIZE_SSSE3 + #include <tmmintrin.h> + #endif + #ifdef EIGEN_VECTORIZE_SSE4_1 + #include <smmintrin.h> + #endif + #ifdef EIGEN_VECTORIZE_SSE4_2 + #include <nmmintrin.h> + #endif + #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) + #include <immintrin.h> + #endif + #endif + } // end extern "C" + + #elif defined __VSX__ + + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_VSX + #include <altivec.h> + // We need to #undef all these ugly tokens defined in <altivec.h> + // => use __vector instead of vector + #undef bool + #undef vector + #undef pixel + + #elif defined __ALTIVEC__ + + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_ALTIVEC + #include <altivec.h> + // We need to #undef all these ugly tokens defined in <altivec.h> + // => use __vector instead of vector + #undef bool + #undef vector + #undef pixel + + #elif (defined __ARM_NEON) || (defined __ARM_NEON__) + + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_NEON + #include <arm_neon.h> + + #elif (defined __s390x__ && defined __VEC__) + + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_ZVECTOR + #include <vecintrin.h> + + #elif defined __mips_msa + + // Limit MSA optimizations to little-endian CPUs for now. + // TODO: Perhaps, eventually support MSA optimizations on big-endian CPUs? + #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #if defined(__LP64__) + #define EIGEN_MIPS_64 + #else + #define EIGEN_MIPS_32 + #endif + #define EIGEN_VECTORIZE + #define EIGEN_VECTORIZE_MSA + #include <msa.h> + #endif + + #endif +#endif + +#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG) + // We can use the optimized fp16 to float and float to fp16 conversion routines + #define EIGEN_HAS_FP16_C +#endif + +#if defined EIGEN_CUDACC + #define EIGEN_VECTORIZE_GPU + #include <vector_types.h> + #if EIGEN_CUDACC_VER >= 70500 + #define EIGEN_HAS_CUDA_FP16 + #endif +#endif + +#if defined(EIGEN_HAS_CUDA_FP16) + #include <cuda_runtime_api.h> + #include <cuda_fp16.h> +#endif + +#if defined(EIGEN_HIPCC) + #define EIGEN_VECTORIZE_GPU + #include <hip/hip_vector_types.h> +#endif + +#if defined(EIGEN_HIP_DEVICE_COMPILE) + + #define EIGEN_HAS_HIP_FP16 + #include <hip/hip_fp16.h> + + #define HIP_PATCH_WITH_NEW_FP16 18215 + #if (HIP_VERSION_PATCH < HIP_PATCH_WITH_NEW_FP16) + #define EIGEN_HAS_OLD_HIP_FP16 + // Old HIP implementation does not have a explicit typedef for "half2" + typedef __half2 half2; + #endif + +#endif + + +/** \brief Namespace containing all symbols from the %Eigen library. */ +namespace Eigen { + +inline static const char *SimdInstructionSetsInUse(void) { +#if defined(EIGEN_VECTORIZE_AVX512) + return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; +#elif defined(EIGEN_VECTORIZE_AVX) + return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; +#elif defined(EIGEN_VECTORIZE_SSE4_2) + return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; +#elif defined(EIGEN_VECTORIZE_SSE4_1) + return "SSE, SSE2, SSE3, SSSE3, SSE4.1"; +#elif defined(EIGEN_VECTORIZE_SSSE3) + return "SSE, SSE2, SSE3, SSSE3"; +#elif defined(EIGEN_VECTORIZE_SSE3) + return "SSE, SSE2, SSE3"; +#elif defined(EIGEN_VECTORIZE_SSE2) + return "SSE, SSE2"; +#elif defined(EIGEN_VECTORIZE_ALTIVEC) + return "AltiVec"; +#elif defined(EIGEN_VECTORIZE_VSX) + return "VSX"; +#elif defined(EIGEN_VECTORIZE_NEON) + return "ARM NEON"; +#elif defined(EIGEN_VECTORIZE_ZVECTOR) + return "S390X ZVECTOR"; +#elif defined(EIGEN_VECTORIZE_MSA) + return "MIPS MSA"; +#else + return "None"; +#endif +} + +} // end namespace Eigen + + +#endif // EIGEN_CONFIGURE_VECTORIZATION_H
diff --git a/Eigen/src/Core/util/Constants.h b/Eigen/src/Core/util/Constants.h index 5f71ba3..a5f63a9 100644 --- a/Eigen/src/Core/util/Constants.h +++ b/Eigen/src/Core/util/Constants.h
@@ -25,6 +25,10 @@ */ const int DynamicIndex = 0xffffff; +/** This value means that the increment to go from one value to another in a sequence is not constant for each step. + */ +const int UndefinedIncr = 0xfffffe; + /** This value means +Infinity; it is currently used only as the p parameter to MatrixBase::lpNorm<int>(). * The value Infinity there means the L-infinity norm. */ @@ -199,7 +203,7 @@ /** \ingroup enums * Enum containing possible values for the \c Mode or \c UpLo parameter of * MatrixBase::selfadjointView() and MatrixBase::triangularView(), and selfadjoint solvers. */ -enum { +enum UpLoType { /** View matrix as a lower triangular matrix. */ Lower=0x1, /** View matrix as an upper triangular matrix. */ @@ -224,7 +228,7 @@ /** \ingroup enums * Enum for indicating whether a buffer is aligned or not. */ -enum { +enum AlignmentType { Unaligned=0, /**< Data pointer has no specific alignment. */ Aligned8=8, /**< Data pointer is aligned on a 8 bytes boundary. */ Aligned16=16, /**< Data pointer is aligned on a 16 bytes boundary. */ @@ -273,7 +277,7 @@ /** \internal \ingroup enums * Enum to specify how to traverse the entries of a matrix. */ -enum { +enum TraversalType { /** \internal Default traversal, no vectorization, no index-based access */ DefaultTraversal, /** \internal No vectorization, use index-based access to have only one for loop instead of 2 nested loops */ @@ -295,7 +299,7 @@ /** \internal \ingroup enums * Enum to specify whether to unroll loops when traversing over the entries of a matrix. */ -enum { +enum UnrollingType { /** \internal Do not unroll loops. */ NoUnrolling, /** \internal Unroll only the inner loop, but not the outer loop. */ @@ -307,7 +311,7 @@ /** \internal \ingroup enums * Enum to specify whether to use the default (built-in) implementation or the specialization. */ -enum { +enum SpecializedType { Specialized, BuiltIn }; @@ -315,7 +319,7 @@ /** \ingroup enums * Enum containing possible values for the \p _Options template parameter of * Matrix, Array and BandMatrix. */ -enum { +enum StorageOptions { /** Storage order is column major (see \ref TopicStorageOrders). */ ColMajor = 0, /** Storage order is row major (see \ref TopicStorageOrders). */ @@ -328,13 +332,15 @@ /** \ingroup enums * Enum for specifying whether to apply or solve on the left or right. */ -enum { +enum SideType { /** Apply transformation on the left. */ OnTheLeft = 1, /** Apply transformation on the right. */ OnTheRight = 2 }; + + /* the following used to be written as: * * struct NoChange_t {}; @@ -353,7 +359,7 @@ /** \internal \ingroup enums * Used in AmbiVector. */ -enum { +enum AmbiVectorMode { IsDense = 0, IsSparse }; @@ -464,6 +470,7 @@ AltiVec = 0x2, VSX = 0x3, NEON = 0x4, + MSA = 0x5, #if defined EIGEN_VECTORIZE_SSE Target = SSE #elif defined EIGEN_VECTORIZE_ALTIVEC @@ -472,6 +479,8 @@ Target = VSX #elif defined EIGEN_VECTORIZE_NEON Target = NEON +#elif defined EIGEN_VECTORIZE_MSA + Target = MSA #else Target = Generic #endif @@ -479,8 +488,9 @@ } /** \internal \ingroup enums - * Enum used as template parameter in Product and product evalautors. */ -enum { DefaultProduct=0, LazyProduct, AliasFreeProduct, CoeffBasedProductMode, LazyCoeffBasedProductMode, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; + * Enum used as template parameter in Product and product evaluators. */ +enum ProductImplType +{ DefaultProduct=0, LazyProduct, AliasFreeProduct, CoeffBasedProductMode, LazyCoeffBasedProductMode, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; /** \internal \ingroup enums * Enum used in experimental parallel implementation. */ @@ -492,7 +502,7 @@ /** The type used to identify a general sparse storage. */ struct Sparse {}; -/** The type used to identify a general solver (foctored) storage. */ +/** The type used to identify a general solver (factored) storage. */ struct SolverStorage {}; /** The type used to identify a permutation storage. */
diff --git a/Eigen/src/Core/util/DisableStupidWarnings.h b/Eigen/src/Core/util/DisableStupidWarnings.h index cb27acf..df6fcd6 100755 --- a/Eigen/src/Core/util/DisableStupidWarnings.h +++ b/Eigen/src/Core/util/DisableStupidWarnings.h
@@ -4,7 +4,6 @@ #ifdef _MSC_VER // 4100 - unreferenced formal parameter (occurred e.g. in aligned_allocator::destroy(pointer p)) // 4101 - unreferenced local variable - // 4127 - conditional expression is constant // 4181 - qualifier applied to reference type ignored // 4211 - nonstandard extension used : redefined extern to static // 4244 - 'argument' : conversion from 'type1' to 'type2', possible loss of data @@ -14,12 +13,13 @@ // 4512 - assignment operator could not be generated // 4522 - 'class' : multiple assignment operators specified // 4700 - uninitialized local variable 'xyz' used + // 4714 - function marked as __forceinline not inlined // 4717 - 'function' : recursive on all control paths, function will cause runtime stack overflow // 4800 - 'type' : forcing value to bool 'true' or 'false' (performance warning) #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS #pragma warning( push ) #endif - #pragma warning( disable : 4100 4101 4127 4181 4211 4244 4273 4324 4503 4512 4522 4700 4717 4800) + #pragma warning( disable : 4100 4101 4181 4211 4244 4273 4324 4503 4512 4522 4700 4714 4717 4800) #elif defined __INTEL_COMPILER // 2196 - routine is both "inline" and "noinline" ("noinline" assumed) @@ -41,18 +41,55 @@ #pragma clang diagnostic push #endif #pragma clang diagnostic ignored "-Wconstant-logical-operand" + #if __clang_major__ >= 3 && __clang_minor__ >= 5 + #pragma clang diagnostic ignored "-Wabsolute-value" + #endif + +#elif defined __GNUC__ + + #if (!defined(EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS)) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) + #pragma GCC diagnostic push + #endif + // g++ warns about local variables shadowing member functions, which is too strict + #pragma GCC diagnostic ignored "-Wshadow" + #if __GNUC__ == 4 && __GNUC_MINOR__ < 8 + // Until g++-4.7 there are warnings when comparing unsigned int vs 0, even in templated functions: + #pragma GCC diagnostic ignored "-Wtype-limits" + #endif + #if __GNUC__>=6 + #pragma GCC diagnostic ignored "-Wignored-attributes" + #endif + #endif #if defined __NVCC__ + #pragma diag_suppress boolean_controlling_expr_is_constant // Disable the "statement is unreachable" message #pragma diag_suppress code_is_unreachable // Disable the "dynamic initialization in unreachable code" message #pragma diag_suppress initialization_not_reachable - // Disable the "calling a __host__ function from a __host__ __device__ function is not allowed" messages (yes, there are 4 of them) + // Disable the "invalid error number" message that we get with older versions of nvcc + #pragma diag_suppress 1222 + // Disable the "calling a __host__ function from a __host__ __device__ function is not allowed" messages (yes, there are many of them and they seem to change with every version of the compiler) + #pragma diag_suppress 2527 + #pragma diag_suppress 2529 #pragma diag_suppress 2651 #pragma diag_suppress 2653 #pragma diag_suppress 2668 + #pragma diag_suppress 2669 #pragma diag_suppress 2670 + #pragma diag_suppress 2671 + #pragma diag_suppress 2735 + #pragma diag_suppress 2737 + #pragma diag_suppress 2739 #endif +#else +// warnings already disabled: +# ifndef EIGEN_WARNINGS_DISABLED_2 +# define EIGEN_WARNINGS_DISABLED_2 +# elif defined(EIGEN_INTERNAL_DEBUGGING) +# error "Do not include \"DisableStupidWarnings.h\" recursively more than twice!" +# endif + #endif // not EIGEN_WARNINGS_DISABLED
diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h index a102e54..050d15e 100644 --- a/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Eigen/src/Core/util/ForwardDeclarations.h
@@ -47,11 +47,7 @@ template<typename Derived> struct EigenBase; template<typename Derived> class DenseBase; template<typename Derived> class PlainObjectBase; - - -template<typename Derived, - int Level = internal::accessors_level<Derived>::value > -class DenseCoeffsBase; +template<typename Derived, int Level> class DenseCoeffsBase; template<typename _Scalar, int _Rows, int _Cols, int _Options = AutoAlign | @@ -83,6 +79,8 @@ template<typename ExpressionType> class SwapWrapper; template<typename XprType, int BlockRows=Dynamic, int BlockCols=Dynamic, bool InnerPanel = false> class Block; +template<typename XprType, typename RowIndices, typename ColIndices> class IndexedView; +template<typename XprType, int Rows=Dynamic, int Cols=Dynamic, int Order=0> class Reshaped; template<typename MatrixType, int Size=Dynamic> class VectorBlock; template<typename MatrixType> class Transpose; @@ -91,6 +89,7 @@ template<typename UnaryOp, typename MatrixType> class CwiseUnaryOp; template<typename ViewOp, typename MatrixType> class CwiseUnaryView; template<typename BinaryOp, typename Lhs, typename Rhs> class CwiseBinaryOp; +template<typename TernaryOp, typename Arg1, typename Arg2, typename Arg3> class CwiseTernaryOp; template<typename Decomposition, typename Rhstype> class Solve; template<typename XprType> class Inverse; @@ -132,6 +131,9 @@ template<typename XprType> class InnerIterator; namespace internal { +template<typename XprType> class generic_randaccess_stl_iterator; +template<typename XprType> class pointer_based_stl_iterator; +template<typename XprType, DirectionType Direction> class subvector_stl_iterator; template<typename DecompositionType> struct kernel_retval_base; template<typename DecompositionType> struct kernel_retval; template<typename DecompositionType> struct image_retval_base; @@ -174,9 +176,11 @@ // with optional conjugation of the arguments. template<typename LhsScalar, typename RhsScalar, bool ConjLhs=false, bool ConjRhs=false> struct conj_helper; -template<typename Scalar> struct scalar_sum_op; -template<typename Scalar> struct scalar_difference_op; -template<typename LhsScalar,typename RhsScalar> struct scalar_conj_product_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_sum_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_difference_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_conj_product_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_min_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_max_op; template<typename Scalar> struct scalar_opposite_op; template<typename Scalar> struct scalar_conjugate_op; template<typename Scalar> struct scalar_real_op; @@ -192,27 +196,28 @@ template<typename Scalar> struct scalar_acos_op; template<typename Scalar> struct scalar_asin_op; template<typename Scalar> struct scalar_tan_op; -template<typename Scalar> struct scalar_pow_op; template<typename Scalar> struct scalar_inverse_op; template<typename Scalar> struct scalar_square_op; template<typename Scalar> struct scalar_cube_op; template<typename Scalar, typename NewType> struct scalar_cast_op; -template<typename Scalar> struct scalar_multiple_op; -template<typename Scalar> struct scalar_quotient1_op; -template<typename Scalar> struct scalar_min_op; -template<typename Scalar> struct scalar_max_op; template<typename Scalar> struct scalar_random_op; -template<typename Scalar> struct scalar_add_op; template<typename Scalar> struct scalar_constant_op; template<typename Scalar> struct scalar_identity_op; template<typename Scalar,bool iscpx> struct scalar_sign_op; +template<typename Scalar,typename ScalarExponent> struct scalar_pow_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_hypot_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_product_op; +template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_quotient_op; + +// SpecialFunctions module +template<typename Scalar> struct scalar_lgamma_op; +template<typename Scalar> struct scalar_digamma_op; +template<typename Scalar> struct scalar_erf_op; +template<typename Scalar> struct scalar_erfc_op; template<typename Scalar> struct scalar_igamma_op; template<typename Scalar> struct scalar_igammac_op; - -template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_product_op; -template<typename LhsScalar,typename RhsScalar> struct scalar_multiple2_op; -template<typename LhsScalar,typename RhsScalar=LhsScalar> struct scalar_quotient_op; -template<typename LhsScalar,typename RhsScalar> struct scalar_quotient2_op; +template<typename Scalar> struct scalar_zeta_op; +template<typename Scalar> struct scalar_betainc_op; } // end namespace internal @@ -251,6 +256,7 @@ template<typename MatrixType> class ColPivHouseholderQR; template<typename MatrixType> class FullPivHouseholderQR; template<typename MatrixType> class CompleteOrthogonalDecomposition; +template<typename MatrixType> class SVDBase; template<typename MatrixType, int QRPreconditioner = ColPivHouseholderQRPreconditioner> class JacobiSVD; template<typename MatrixType> class BDCSVD; template<typename MatrixType, int UpLo = Lower> class LLT;
diff --git a/Eigen/src/Core/util/IndexedViewHelper.h b/Eigen/src/Core/util/IndexedViewHelper.h new file mode 100644 index 0000000..1cda850 --- /dev/null +++ b/Eigen/src/Core/util/IndexedViewHelper.h
@@ -0,0 +1,186 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#ifndef EIGEN_INDEXED_VIEW_HELPER_H +#define EIGEN_INDEXED_VIEW_HELPER_H + +namespace Eigen { + +namespace internal { +struct symbolic_last_tag {}; +} + +/** \var last + * \ingroup Core_Module + * + * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically reference the last element/row/columns + * of the underlying vector or matrix once passed to DenseBase::operator()(const RowIndices&, const ColIndices&). + * + * This symbolic placeholder supports standard arithmetic operations. + * + * A typical usage example would be: + * \code + * using namespace Eigen; + * using Eigen::last; + * VectorXd v(n); + * v(seq(2,last-2)).setOnes(); + * \endcode + * + * \sa end + */ +static const symbolic::SymbolExpr<internal::symbolic_last_tag> last; // PLEASE use Eigen::last instead of Eigen::placeholders::last + +/** \var lastp1 + * \ingroup Core_Module + * + * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically + * reference the last+1 element/row/columns of the underlying vector or matrix once + * passed to DenseBase::operator()(const RowIndices&, const ColIndices&). + * + * This symbolic placeholder supports standard arithmetic operations. + * It is essentially an alias to last+fix<1>. + * + * \sa last + */ +#ifdef EIGEN_PARSED_BY_DOXYGEN +static const auto lastp1 = last+fix<1>; +#else +// Using a FixedExpr<1> expression is important here to make sure the compiler +// can fully optimize the computation starting indices with zero overhead. +static const symbolic::AddExpr<symbolic::SymbolExpr<internal::symbolic_last_tag>,symbolic::ValueExpr<Eigen::internal::FixedInt<1> > > lastp1(last+fix<1>()); +#endif + +namespace internal { + + // Replace symbolic last/end "keywords" by their true runtime value +inline Index eval_expr_given_size(Index x, Index /* size */) { return x; } + +template<int N> +FixedInt<N> eval_expr_given_size(FixedInt<N> x, Index /*size*/) { return x; } + +template<typename Derived> +Index eval_expr_given_size(const symbolic::BaseExpr<Derived> &x, Index size) +{ + return x.derived().eval(last=size-1); +} + +// Extract increment/step at compile time +template<typename T, typename EnableIf = void> struct get_compile_time_incr { + enum { value = UndefinedIncr }; +}; + +// Analogue of std::get<0>(x), but tailored for our needs. +template<typename T> +Index first(const T& x) { return x.first(); } + +// IndexedViewCompatibleType/makeIndexedViewCompatible turn an arbitrary object of type T into something usable by MatrixSlice +// The generic implementation is a no-op +template<typename T,int XprSize,typename EnableIf=void> +struct IndexedViewCompatibleType { + typedef T type; +}; + +template<typename T,typename Q> +const T& makeIndexedViewCompatible(const T& x, Index /*size*/, Q) { return x; } + +//-------------------------------------------------------------------------------- +// Handling of a single Index +//-------------------------------------------------------------------------------- + +struct SingleRange { + enum { + SizeAtCompileTime = 1 + }; + SingleRange(Index val) : m_value(val) {} + Index operator[](Index) const { return m_value; } + Index size() const { return 1; } + Index first() const { return m_value; } + Index m_value; +}; + +template<> struct get_compile_time_incr<SingleRange> { + enum { value = 1 }; // 1 or 0 ?? +}; + +// Turn a single index into something that looks like an array (i.e., that exposes a .size(), and operator[](int) methods) +template<typename T, int XprSize> +struct IndexedViewCompatibleType<T,XprSize,typename internal::enable_if<internal::is_integral<T>::value>::type> { + // Here we could simply use Array, but maybe it's less work for the compiler to use + // a simpler wrapper as SingleRange + //typedef Eigen::Array<Index,1,1> type; + typedef SingleRange type; +}; + +template<typename T, int XprSize> +struct IndexedViewCompatibleType<T, XprSize, typename enable_if<symbolic::is_symbolic<T>::value>::type> { + typedef SingleRange type; +}; + + +template<typename T> +typename enable_if<symbolic::is_symbolic<T>::value,SingleRange>::type +makeIndexedViewCompatible(const T& id, Index size, SpecializedType) { + return eval_expr_given_size(id,size); +} + +//-------------------------------------------------------------------------------- +// Handling of all +//-------------------------------------------------------------------------------- + +struct all_t { all_t() {} }; + +// Convert a symbolic 'all' into a usable range type +template<int XprSize> +struct AllRange { + enum { SizeAtCompileTime = XprSize }; + AllRange(Index size = XprSize) : m_size(size) {} + Index operator[](Index i) const { return i; } + Index size() const { return m_size.value(); } + Index first() const { return 0; } + variable_if_dynamic<Index,XprSize> m_size; +}; + +template<int XprSize> +struct IndexedViewCompatibleType<all_t,XprSize> { + typedef AllRange<XprSize> type; +}; + +template<typename XprSizeType> +inline AllRange<get_fixed_value<XprSizeType>::value> makeIndexedViewCompatible(all_t , XprSizeType size, SpecializedType) { + return AllRange<get_fixed_value<XprSizeType>::value>(size); +} + +template<int Size> struct get_compile_time_incr<AllRange<Size> > { + enum { value = 1 }; +}; + +} // end namespace internal + + +/** \var all + * \ingroup Core_Module + * Can be used as a parameter to DenseBase::operator()(const RowIndices&, const ColIndices&) to index all rows or columns + */ +static const Eigen::internal::all_t all; // PLEASE use Eigen::all instead of Eigen::placeholders::all + + +namespace placeholders { + typedef symbolic::SymbolExpr<internal::symbolic_last_tag> last_t; + typedef symbolic::AddExpr<symbolic::SymbolExpr<internal::symbolic_last_tag>,symbolic::ValueExpr<Eigen::internal::FixedInt<1> > > end_t; + typedef Eigen::internal::all_t all_t; + + EIGEN_DEPRECATED static const all_t all = Eigen::all; // PLEASE use Eigen::all instead of Eigen::placeholders::all + EIGEN_DEPRECATED static const last_t last = Eigen::last; // PLEASE use Eigen::last instead of Eigen::placeholders::last + EIGEN_DEPRECATED static const end_t end = Eigen::lastp1; // PLEASE use Eigen::lastp1 instead of Eigen::placeholders::end +} + +} // end namespace Eigen + +#endif // EIGEN_INDEXED_VIEW_HELPER_H
diff --git a/Eigen/src/Core/util/IntegralConstant.h b/Eigen/src/Core/util/IntegralConstant.h new file mode 100644 index 0000000..caeea23 --- /dev/null +++ b/Eigen/src/Core/util/IntegralConstant.h
@@ -0,0 +1,272 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#ifndef EIGEN_INTEGRAL_CONSTANT_H +#define EIGEN_INTEGRAL_CONSTANT_H + +namespace Eigen { + +namespace internal { + +template<int N> class FixedInt; +template<int N> class VariableAndFixedInt; + +/** \internal + * \class FixedInt + * + * This class embeds a compile-time integer \c N. + * + * It is similar to c++11 std::integral_constant<int,N> but with some additional features + * such as: + * - implicit conversion to int + * - arithmetic and some bitwise operators: -, +, *, /, %, &, | + * - c++98/14 compatibility with fix<N> and fix<N>() syntax to define integral constants. + * + * It is strongly discouraged to directly deal with this class FixedInt. Instances are expcected to + * be created by the user using Eigen::fix<N> or Eigen::fix<N>(). In C++98-11, the former syntax does + * not create a FixedInt<N> instance but rather a point to function that needs to be \em cleaned-up + * using the generic helper: + * \code + * internal::cleanup_index_type<T>::type + * internal::cleanup_index_type<T,DynamicKey>::type + * \endcode + * where T can a FixedInt<N>, a pointer to function FixedInt<N> (*)(), or numerous other integer-like representations. + * \c DynamicKey is either Dynamic (default) or DynamicIndex and used to identify true compile-time values. + * + * For convenience, you can extract the compile-time value \c N in a generic way using the following helper: + * \code + * internal::get_fixed_value<T,DefaultVal>::value + * \endcode + * that will give you \c N if T equals FixedInt<N> or FixedInt<N> (*)(), and \c DefaultVal if T does not embed any compile-time value (e.g., T==int). + * + * \sa fix<N>, class VariableAndFixedInt + */ +template<int N> class FixedInt +{ +public: + static const int value = N; + operator int() const { return value; } + FixedInt() {} + FixedInt( VariableAndFixedInt<N> other) { + #ifndef EIGEN_INTERNAL_DEBUGGING + EIGEN_UNUSED_VARIABLE(other); + #endif + eigen_internal_assert(int(other)==N); + } + + FixedInt<-N> operator-() const { return FixedInt<-N>(); } + template<int M> + FixedInt<N+M> operator+( FixedInt<M>) const { return FixedInt<N+M>(); } + template<int M> + FixedInt<N-M> operator-( FixedInt<M>) const { return FixedInt<N-M>(); } + template<int M> + FixedInt<N*M> operator*( FixedInt<M>) const { return FixedInt<N*M>(); } + template<int M> + FixedInt<N/M> operator/( FixedInt<M>) const { return FixedInt<N/M>(); } + template<int M> + FixedInt<N%M> operator%( FixedInt<M>) const { return FixedInt<N%M>(); } + template<int M> + FixedInt<N|M> operator|( FixedInt<M>) const { return FixedInt<N|M>(); } + template<int M> + FixedInt<N&M> operator&( FixedInt<M>) const { return FixedInt<N&M>(); } + +#if EIGEN_HAS_CXX14 + // Needed in C++14 to allow fix<N>(): + FixedInt operator() () const { return *this; } + + VariableAndFixedInt<N> operator() (int val) const { return VariableAndFixedInt<N>(val); } +#else + FixedInt ( FixedInt<N> (*)() ) {} +#endif + +#if EIGEN_HAS_CXX11 + FixedInt(std::integral_constant<int,N>) {} +#endif +}; + +/** \internal + * \class VariableAndFixedInt + * + * This class embeds both a compile-time integer \c N and a runtime integer. + * Both values are supposed to be equal unless the compile-time value \c N has a special + * value meaning that the runtime-value should be used. Depending on the context, this special + * value can be either Eigen::Dynamic (for positive quantities) or Eigen::DynamicIndex (for + * quantities that can be negative). + * + * It is the return-type of the function Eigen::fix<N>(int), and most of the time this is the only + * way it is used. It is strongly discouraged to directly deal with instances of VariableAndFixedInt. + * Indeed, in order to write generic code, it is the responsibility of the callee to properly convert + * it to either a true compile-time quantity (i.e. a FixedInt<N>), or to a runtime quantity (e.g., an Index) + * using the following generic helper: + * \code + * internal::cleanup_index_type<T>::type + * internal::cleanup_index_type<T,DynamicKey>::type + * \endcode + * where T can be a template instantiation of VariableAndFixedInt or numerous other integer-like representations. + * \c DynamicKey is either Dynamic (default) or DynamicIndex and used to identify true compile-time values. + * + * For convenience, you can also extract the compile-time value \c N using the following helper: + * \code + * internal::get_fixed_value<T,DefaultVal>::value + * \endcode + * that will give you \c N if T equals VariableAndFixedInt<N>, and \c DefaultVal if T does not embed any compile-time value (e.g., T==int). + * + * \sa fix<N>(int), class FixedInt + */ +template<int N> class VariableAndFixedInt +{ +public: + static const int value = N; + operator int() const { return m_value; } + VariableAndFixedInt(int val) { m_value = val; } +protected: + int m_value; +}; + +template<typename T, int Default=Dynamic> struct get_fixed_value { + static const int value = Default; +}; + +template<int N,int Default> struct get_fixed_value<FixedInt<N>,Default> { + static const int value = N; +}; + +#if !EIGEN_HAS_CXX14 +template<int N,int Default> struct get_fixed_value<FixedInt<N> (*)(),Default> { + static const int value = N; +}; +#endif + +template<int N,int Default> struct get_fixed_value<VariableAndFixedInt<N>,Default> { + static const int value = N ; +}; + +template<typename T, int N, int Default> +struct get_fixed_value<variable_if_dynamic<T,N>,Default> { + static const int value = N; +}; + +template<typename T> EIGEN_DEVICE_FUNC Index get_runtime_value(const T &x) { return x; } +#if !EIGEN_HAS_CXX14 +template<int N> EIGEN_DEVICE_FUNC Index get_runtime_value(FixedInt<N> (*)()) { return N; } +#endif + +// Cleanup integer/FixedInt/VariableAndFixedInt/etc types: + +// By default, no cleanup: +template<typename T, int DynamicKey=Dynamic, typename EnableIf=void> struct cleanup_index_type { typedef T type; }; + +// Convert any integral type (e.g., short, int, unsigned int, etc.) to Eigen::Index +template<typename T, int DynamicKey> struct cleanup_index_type<T,DynamicKey,typename internal::enable_if<internal::is_integral<T>::value>::type> { typedef Index type; }; + +#if !EIGEN_HAS_CXX14 +// In c++98/c++11, fix<N> is a pointer to function that we better cleanup to a true FixedInt<N>: +template<int N, int DynamicKey> struct cleanup_index_type<FixedInt<N> (*)(), DynamicKey> { typedef FixedInt<N> type; }; +#endif + +// If VariableAndFixedInt does not match DynamicKey, then we turn it to a pure compile-time value: +template<int N, int DynamicKey> struct cleanup_index_type<VariableAndFixedInt<N>, DynamicKey> { typedef FixedInt<N> type; }; +// If VariableAndFixedInt matches DynamicKey, then we turn it to a pure runtime-value (aka Index): +template<int DynamicKey> struct cleanup_index_type<VariableAndFixedInt<DynamicKey>, DynamicKey> { typedef Index type; }; + +#if EIGEN_HAS_CXX11 +template<int N, int DynamicKey> struct cleanup_index_type<std::integral_constant<int,N>, DynamicKey> { typedef FixedInt<N> type; }; +#endif + +} // end namespace internal + +#ifndef EIGEN_PARSED_BY_DOXYGEN + +#if EIGEN_HAS_CXX14 +template<int N> +static const internal::FixedInt<N> fix{}; +#else +template<int N> +inline internal::FixedInt<N> fix() { return internal::FixedInt<N>(); } + +// The generic typename T is mandatory. Otherwise, a code like fix<N> could refer to either the function above or this next overload. +// This way a code like fix<N> can only refer to the previous function. +template<int N,typename T> +inline internal::VariableAndFixedInt<N> fix(T val) { return internal::VariableAndFixedInt<N>(internal::convert_index<int>(val)); } +#endif + +#else // EIGEN_PARSED_BY_DOXYGEN + +/** \var fix<N>() + * \ingroup Core_Module + * + * This \em identifier permits to construct an object embedding a compile-time integer \c N. + * + * \tparam N the compile-time integer value + * + * It is typically used in conjunction with the Eigen::seq and Eigen::seqN functions to pass compile-time values to them: + * \code + * seqN(10,fix<4>,fix<-3>) // <=> [10 7 4 1] + * \endcode + * + * See also the function fix(int) to pass both a compile-time and runtime value. + * + * In c++14, it is implemented as: + * \code + * template<int N> static const internal::FixedInt<N> fix{}; + * \endcode + * where internal::FixedInt<N> is an internal template class similar to + * <a href="http://en.cppreference.com/w/cpp/types/integral_constant">\c std::integral_constant </a><tt> <int,N> </tt> + * Here, \c fix<N> is thus an object of type \c internal::FixedInt<N>. + * + * In c++98/11, it is implemented as a function: + * \code + * template<int N> inline internal::FixedInt<N> fix(); + * \endcode + * Here internal::FixedInt<N> is thus a pointer to function. + * + * If for some reason you want a true object in c++98 then you can write: \code fix<N>() \endcode which is also valid in c++14. + * + * \sa fix<N>(int), seq, seqN + */ +template<int N> +static const auto fix(); + +/** \fn fix<N>(int) + * \ingroup Core_Module + * + * This function returns an object embedding both a compile-time integer \c N, and a fallback runtime value \a val. + * + * \tparam N the compile-time integer value + * \param val the fallback runtime integer value + * + * This function is a more general version of the \ref fix identifier/function that can be used in template code + * where the compile-time value could turn out to actually mean "undefined at compile-time". For positive integers + * such as a size or a dimension, this case is identified by Eigen::Dynamic, whereas runtime signed integers + * (e.g., an increment/stride) are identified as Eigen::DynamicIndex. In such a case, the runtime value \a val + * will be used as a fallback. + * + * A typical use case would be: + * \code + * template<typename Derived> void foo(const MatrixBase<Derived> &mat) { + * const int N = Derived::RowsAtCompileTime==Dynamic ? Dynamic : Derived::RowsAtCompileTime/2; + * const int n = mat.rows()/2; + * ... mat( seqN(0,fix<N>(n) ) ...; + * } + * \endcode + * In this example, the function Eigen::seqN knows that the second argument is expected to be a size. + * If the passed compile-time value N equals Eigen::Dynamic, then the proxy object returned by fix will be dissmissed, and converted to an Eigen::Index of value \c n. + * Otherwise, the runtime-value \c n will be dissmissed, and the returned ArithmeticSequence will be of the exact same type as <tt> seqN(0,fix<N>) </tt>. + * + * \sa fix, seqN, class ArithmeticSequence + */ +template<int N> +static const auto fix(int val); + +#endif // EIGEN_PARSED_BY_DOXYGEN + +} // end namespace Eigen + +#endif // EIGEN_INTEGRAL_CONSTANT_H
diff --git a/Eigen/src/Core/util/MKL_support.h b/Eigen/src/Core/util/MKL_support.h old mode 100644 new mode 100755 index 8c9239b..17963fa --- a/Eigen/src/Core/util/MKL_support.h +++ b/Eigen/src/Core/util/MKL_support.h
@@ -49,12 +49,17 @@ #define EIGEN_USE_LAPACKE #endif -#if defined(EIGEN_USE_LAPACKE) || defined(EIGEN_USE_MKL_VML) +#if defined(EIGEN_USE_MKL_VML) && !defined(EIGEN_USE_MKL) #define EIGEN_USE_MKL #endif + #if defined EIGEN_USE_MKL -# include <mkl.h> +# if (!defined MKL_DIRECT_CALL) && (!defined EIGEN_MKL_NO_DIRECT_CALL) +# define MKL_DIRECT_CALL +# define MKL_DIRECT_CALL_JUST_SET +# endif +# include <mkl.h> /*Check IMKL version for compatibility: < 10.3 is not usable with Eigen*/ # ifndef INTEL_MKL_VERSION # undef EIGEN_USE_MKL /* INTEL_MKL_VERSION is not even defined on older versions */ @@ -68,11 +73,14 @@ # undef EIGEN_USE_MKL_VML # undef EIGEN_USE_LAPACKE_STRICT # undef EIGEN_USE_LAPACKE +# ifdef MKL_DIRECT_CALL_JUST_SET +# undef MKL_DIRECT_CALL +# endif # endif #endif #if defined EIGEN_USE_MKL -#include <mkl_lapacke.h> + #define EIGEN_MKL_VML_THRESHOLD 128 /* MKL_DOMAIN_BLAS, etc are defined only in 10.3 update 7 */ @@ -108,6 +116,10 @@ #endif #endif +#if defined(EIGEN_USE_BLAS) && !defined(EIGEN_USE_MKL) +#include "../../misc/blas.h" +#endif + namespace Eigen { typedef std::complex<double> dcomplex; @@ -121,8 +133,5 @@ } // end namespace Eigen -#if defined(EIGEN_USE_BLAS) -#include "../../misc/blas.h" -#endif #endif // EIGEN_MKL_SUPPORT_H
diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index 97627d1..ffd6a00 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h
@@ -11,26 +11,63 @@ #ifndef EIGEN_MACROS_H #define EIGEN_MACROS_H +//------------------------------------------------------------------------------------------ +// Eigen version and basic defaults +//------------------------------------------------------------------------------------------ + #define EIGEN_WORLD_VERSION 3 -#define EIGEN_MAJOR_VERSION 2 -#define EIGEN_MINOR_VERSION 92 +#define EIGEN_MAJOR_VERSION 3 +#define EIGEN_MINOR_VERSION 90 #define EIGEN_VERSION_AT_LEAST(x,y,z) (EIGEN_WORLD_VERSION>x || (EIGEN_WORLD_VERSION>=x && \ (EIGEN_MAJOR_VERSION>y || (EIGEN_MAJOR_VERSION>=y && \ EIGEN_MINOR_VERSION>=z)))) +#ifdef EIGEN_DEFAULT_TO_ROW_MAJOR +#define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::RowMajor +#else +#define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::ColMajor +#endif + +#ifndef EIGEN_DEFAULT_DENSE_INDEX_TYPE +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE std::ptrdiff_t +#endif + +// Upperbound on the C++ version to use. +// Expected values are 03, 11, 14, 17, etc. +// By default, let's use an arbitrarily large C++ version. +#ifndef EIGEN_MAX_CPP_VER +#define EIGEN_MAX_CPP_VER 99 +#endif + +/** Allows to disable some optimizations which might affect the accuracy of the result. + * Such optimization are enabled by default, and set EIGEN_FAST_MATH to 0 to disable them. + * They currently include: + * - single precision ArrayBase::sin() and ArrayBase::cos() for SSE and AVX vectorization. + */ +#ifndef EIGEN_FAST_MATH +#define EIGEN_FAST_MATH 1 +#endif + +#ifndef EIGEN_STACK_ALLOCATION_LIMIT +// 131072 == 128 KB +#define EIGEN_STACK_ALLOCATION_LIMIT 131072 +#endif + +//------------------------------------------------------------------------------------------ // Compiler identification, EIGEN_COMP_* +//------------------------------------------------------------------------------------------ /// \internal EIGEN_COMP_GNUC set to 1 for all compilers compatible with GCC #ifdef __GNUC__ - #define EIGEN_COMP_GNUC 1 + #define EIGEN_COMP_GNUC (__GNUC__*10+__GNUC_MINOR__) #else #define EIGEN_COMP_GNUC 0 #endif -/// \internal EIGEN_COMP_CLANG set to 1 if the compiler is clang (alias for __clang__) +/// \internal EIGEN_COMP_CLANG set to major+minor version (e.g., 307 for clang 3.7) if the compiler is clang #if defined(__clang__) - #define EIGEN_COMP_CLANG 1 + #define EIGEN_COMP_CLANG (__clang_major__*100+__clang_minor__) #else #define EIGEN_COMP_CLANG 0 #endif @@ -71,23 +108,42 @@ #define EIGEN_COMP_MSVC 0 #endif -/// \internal EIGEN_COMP_MSVC_STRICT set to 1 if the compiler is really Microsoft Visual C++ and not ,e.g., ICC -#if EIGEN_COMP_MSVC && !(EIGEN_COMP_ICC) +// For the record, here is a table summarizing the possible values for EIGEN_COMP_MSVC: +// name ver MSC_VER +// 2008 9 1500 +// 2010 10 1600 +// 2012 11 1700 +// 2013 12 1800 +// 2015 14 1900 +// "15" 15 1900 +// 2017-14.1 15.0 1910 +// 2017-14.11 15.3 1911 +// 2017-14.12 15.5 1912 +// 2017-14.13 15.6 1913 +// 2017-14.14 15.7 1914 + +/// \internal EIGEN_COMP_MSVC_STRICT set to 1 if the compiler is really Microsoft Visual C++ and not ,e.g., ICC or clang-cl +#if EIGEN_COMP_MSVC && !(EIGEN_COMP_ICC || EIGEN_COMP_LLVM || EIGEN_COMP_CLANG) #define EIGEN_COMP_MSVC_STRICT _MSC_VER #else #define EIGEN_COMP_MSVC_STRICT 0 #endif -/// \internal EIGEN_COMP_IBM set to 1 if the compiler is IBM XL C++ -#if defined(__IBMCPP__) || defined(__xlc__) - #define EIGEN_COMP_IBM 1 +/// \internal EIGEN_COMP_IBM set to xlc version if the compiler is IBM XL C++ +// XLC version +// 3.1 0x0301 +// 4.5 0x0405 +// 5.0 0x0500 +// 12.1 0x0C01 +#if defined(__IBMCPP__) || defined(__xlc__) || defined(__ibmxl__) + #define EIGEN_COMP_IBM __xlC__ #else #define EIGEN_COMP_IBM 0 #endif -/// \internal EIGEN_COMP_PGI set to 1 if the compiler is Portland Group Compiler +/// \internal EIGEN_COMP_PGI set to PGI version if the compiler is Portland Group Compiler #if defined(__PGI) - #define EIGEN_COMP_PGI 1 + #define EIGEN_COMP_PGI (__PGIC__*100+__PGIC_MINOR__) #else #define EIGEN_COMP_PGI 0 #endif @@ -133,7 +189,11 @@ #endif + +//------------------------------------------------------------------------------------------ // Architecture identification, EIGEN_ARCH_* +//------------------------------------------------------------------------------------------ + #if defined(__x86_64__) || defined(_M_X64) || defined(__amd64) #define EIGEN_ARCH_x86_64 1 @@ -203,7 +263,9 @@ +//------------------------------------------------------------------------------------------ // Operating system identification, EIGEN_OS_* +//------------------------------------------------------------------------------------------ /// \internal EIGEN_OS_UNIX set to 1 if the OS is a unix variant #if defined(__unix__) || defined(__unix) @@ -290,9 +352,17 @@ #define EIGEN_OS_WIN_STRICT 0 #endif -/// \internal EIGEN_OS_SUN set to 1 if the OS is SUN +/// \internal EIGEN_OS_SUN set to __SUNPRO_C if the OS is SUN +// compiler solaris __SUNPRO_C +// version studio +// 5.7 10 0x570 +// 5.8 11 0x580 +// 5.9 12 0x590 +// 5.10 12.1 0x5100 +// 5.11 12.2 0x5110 +// 5.12 12.3 0x5120 #if (defined(sun) || defined(__sun)) && !(defined(__SVR4) || defined(__svr4__)) - #define EIGEN_OS_SUN 1 + #define EIGEN_OS_SUN __SUNPRO_C #else #define EIGEN_OS_SUN 0 #endif @@ -305,6 +375,108 @@ #endif +//------------------------------------------------------------------------------------------ +// Detect GPU compilers and architectures +//------------------------------------------------------------------------------------------ + +// NVCC is not supported as the target platform for HIPCC +// Note that this also makes EIGEN_CUDACC and EIGEN_HIPCC mutually exclusive +#if defined(__NVCC__) && defined(__HIPCC__) + #error "NVCC as the target platform for HIPCC is currently not supported." +#endif + +#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA) + // Means the compiler is either nvcc or clang with CUDA enabled + #define EIGEN_CUDACC __CUDACC__ +#endif + +#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA) + // Means we are generating code for the device + #define EIGEN_CUDA_ARCH __CUDA_ARCH__ +#endif + +// Starting with CUDA 9 the composite __CUDACC_VER__ is not available. +#if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9) + #define EIGEN_CUDACC_VER ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100)) +#elif defined(__CUDACC_VER__) + #define EIGEN_CUDACC_VER __CUDACC_VER__ +#else + #define EIGEN_CUDACC_VER 0 +#endif + +#if defined(__HIPCC__) && !defined(EIGEN_NO_HIP) + // Means the compiler is HIPCC (analogous to EIGEN_CUDACC, but for HIP) + #define EIGEN_HIPCC __HIPCC__ + + // We need to include hip_runtime.h here because it pulls in + // ++ hip_common.h which contains the define for __HIP_DEVICE_COMPILE__ + // ++ host_defines.h which contains the defines for the __host__ and __device__ macros + #include <hip/hip_runtime.h> + + #if defined(__HIP_DEVICE_COMPILE__) + // analogous to EIGEN_CUDA_ARCH, but for HIP + #define EIGEN_HIP_DEVICE_COMPILE __HIP_DEVICE_COMPILE__ + #endif +#endif + +// Unify CUDA/HIPCC + +#if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC) +// +// If either EIGEN_CUDACC or EIGEN_HIPCC is defined, then define EIGEN_GPUCC +// +#define EIGEN_GPUCC +// +// EIGEN_HIPCC implies the HIP compiler and is used to tweak Eigen code for use in HIP kernels +// EIGEN_CUDACC implies the CUDA compiler and is used to tweak Eigen code for use in CUDA kernels +// +// In most cases the same tweaks are required to the Eigen code to enable in both the HIP and CUDA kernels. +// For those cases, the corresponding code should be guarded with +// #if defined(EIGEN_GPUCC) +// instead of +// #if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC) +// +// For cases where the tweak is specific to HIP, the code should be guarded with +// #if defined(EIGEN_HIPCC) +// +// For cases where the tweak is specific to CUDA, the code should be guarded with +// #if defined(EIGEN_CUDACC) +// +#endif + +#if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIP_DEVICE_COMPILE) +// +// If either EIGEN_CUDA_ARCH or EIGEN_HIP_DEVICE_COMPILE is defined, then define EIGEN_GPU_COMPILE_PHASE +// +#define EIGEN_GPU_COMPILE_PHASE +// +// GPU compilers (HIPCC, NVCC) typically do two passes over the source code, +// + one to compile the source for the "host" (ie CPU) +// + another to compile the source for the "device" (ie. GPU) +// +// Code that needs to enabled only during the either the "host" or "device" compilation phase +// needs to be guarded with a macro that indicates the current compilation phase +// +// EIGEN_HIP_DEVICE_COMPILE implies the device compilation phase in HIP +// EIGEN_CUDA_ARCH implies the device compilation phase in CUDA +// +// In most cases, the "host" / "device" specific code is the same for both HIP and CUDA +// For those cases, the code should be guarded with +// #if defined(EIGEN_GPU_COMPILE_PHASE) +// instead of +// #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIP_DEVICE_COMPILE) +// +// For cases where the tweak is specific to HIP, the code should be guarded with +// #if defined(EIGEN_HIP_DEVICE_COMPILE) +// +// For cases where the tweak is specific to CUDA, the code should be guarded with +// #if defined(EIGEN_CUDA_ARCH) +// +#endif + +//------------------------------------------------------------------------------------------ +// Detect Compiler/Architecture/OS specific features +//------------------------------------------------------------------------------------------ #if EIGEN_GNUC_AT_MOST(4,3) && !EIGEN_COMP_CLANG // see bug 89 @@ -313,20 +485,6 @@ #define EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO 1 #endif -// This macro can be used to prevent from macro expansion, e.g.: -// std::max EIGEN_NOT_A_MACRO(a,b) -#define EIGEN_NOT_A_MACRO - -#ifdef EIGEN_DEFAULT_TO_ROW_MAJOR -#define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::RowMajor -#else -#define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::ColMajor -#endif - -#ifndef EIGEN_DEFAULT_DENSE_INDEX_TYPE -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE std::ptrdiff_t -#endif - // Cross compiler wrapper around LLVM's __has_builtin #ifdef __has_builtin # define EIGEN_HAS_BUILTIN(x) __has_builtin(x) @@ -340,50 +498,155 @@ # define __has_feature(x) 0 #endif +// Some old compilers do not support template specializations like: +// template<typename T,int N> void foo(const T x[N]); +#if !( EIGEN_COMP_CLANG && ( (EIGEN_COMP_CLANG<309) \ + || (defined(__apple_build_version__) && (__apple_build_version__ < 9000000))) \ + || EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<49) +#define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 1 +#else +#define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 0 +#endif + + +// The macro EIGEN_COMP_CXXVER defines the c++ verson expected by the compiler. +// For instance, if compiling with gcc and -std=c++17, then EIGEN_COMP_CXXVER +// is defined to 17. +#if (defined(__cplusplus) && (__cplusplus > 201402L) || EIGEN_COMP_MSVC >= 1914) +#define EIGEN_COMP_CXXVER 17 +#elif (defined(__cplusplus) && (__cplusplus > 201103L) || EIGEN_COMP_MSVC >= 1910) +#define EIGEN_COMP_CXXVER 14 +#elif (defined(__cplusplus) && (__cplusplus >= 201103L) || EIGEN_COMP_MSVC >= 1900) +#define EIGEN_COMP_CXXVER 11 +#else +#define EIGEN_COMP_CXXVER 03 +#endif + + +// The macros EIGEN_HAS_CXX?? defines a rough estimate of available c++ features +// but in practice we should not rely on them but rather on the availabilty of +// individual features as defined later. +// This is why there is no EIGEN_HAS_CXX17. +// FIXME: get rid of EIGEN_HAS_CXX14 and maybe even EIGEN_HAS_CXX11. +#if EIGEN_MAX_CPP_VER>=11 && EIGEN_COMP_CXXVER>=11 +#define EIGEN_HAS_CXX11 1 +#else +#define EIGEN_HAS_CXX11 0 +#endif + +#if EIGEN_MAX_CPP_VER>=14 && EIGEN_COMP_CXXVER>=14 +#define EIGEN_HAS_CXX14 1 +#else +#define EIGEN_HAS_CXX14 0 +#endif + // Do we support r-value references? -#if (__has_feature(cxx_rvalue_references) || \ +#ifndef EIGEN_HAS_RVALUE_REFERENCES +#if EIGEN_MAX_CPP_VER>=11 && \ + (__has_feature(cxx_rvalue_references) || \ (defined(__cplusplus) && __cplusplus >= 201103L) || \ (EIGEN_COMP_MSVC >= 1600)) - #define EIGEN_HAVE_RVALUE_REFERENCES + #define EIGEN_HAS_RVALUE_REFERENCES 1 +#else + #define EIGEN_HAS_RVALUE_REFERENCES 0 +#endif #endif // Does the compiler support C99? -#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) \ +// Need to include <cmath> to make sure _GLIBCXX_USE_C99 gets defined +#include <cmath> +#ifndef EIGEN_HAS_C99_MATH +#if EIGEN_MAX_CPP_VER>=11 && \ + ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) \ || (defined(__GNUC__) && defined(_GLIBCXX_USE_C99)) \ - || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) -#define EIGEN_HAS_C99_MATH 1 + || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) \ + || (EIGEN_COMP_MSVC >= 1900) || defined(__SYCL_DEVICE_ONLY__)) + #define EIGEN_HAS_C99_MATH 1 +#else + #define EIGEN_HAS_C99_MATH 0 +#endif #endif // Does the compiler support result_of? -#if (__has_feature(cxx_lambdas) || (defined(__cplusplus) && __cplusplus >= 201103L)) +// It's likely that MSVC 2013 supports result_of but I couldn't not find a good source for that, +// so let's be conservative. +#ifndef EIGEN_HAS_STD_RESULT_OF +#if EIGEN_MAX_CPP_VER>=11 && \ + (__has_feature(cxx_lambdas) || (defined(__cplusplus) && __cplusplus >= 201103L) || EIGEN_COMP_MSVC >= 1900) #define EIGEN_HAS_STD_RESULT_OF 1 +#else +#define EIGEN_HAS_STD_RESULT_OF 0 +#endif +#endif + +#ifndef EIGEN_HAS_ALIGNAS +#if EIGEN_MAX_CPP_VER>=11 && EIGEN_HAS_CXX11 && \ + ( __has_feature(cxx_alignas) \ + || EIGEN_HAS_CXX14 \ + || (EIGEN_COMP_MSVC >= 1800) \ + || (EIGEN_GNUC_AT_LEAST(4,8)) \ + || (EIGEN_COMP_CLANG>=305) \ + || (EIGEN_COMP_ICC>=1500) \ + || (EIGEN_COMP_PGI>=1500) \ + || (EIGEN_COMP_SUNCC>=0x5130)) +#define EIGEN_HAS_ALIGNAS 1 +#else +#define EIGEN_HAS_ALIGNAS 0 +#endif +#endif + +// Does the compiler support type_traits? +// - full support of type traits was added only to GCC 5.1.0. +// - 20150626 corresponds to the last release of 4.x libstdc++ +#ifndef EIGEN_HAS_TYPE_TRAITS +#if EIGEN_MAX_CPP_VER>=11 && (EIGEN_HAS_CXX11 || EIGEN_COMP_MSVC >= 1700) \ + && ((!EIGEN_COMP_GNUC_STRICT) || EIGEN_GNUC_AT_LEAST(5, 1)) \ + && ((!defined(__GLIBCXX__)) || __GLIBCXX__ > 20150626) +#define EIGEN_HAS_TYPE_TRAITS 1 +#define EIGEN_INCLUDE_TYPE_TRAITS +#else +#define EIGEN_HAS_TYPE_TRAITS 0 +#endif #endif // Does the compiler support variadic templates? -#if __cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900 -// Disable the use of variadic templates when compiling with nvcc on ARM devices: -// this prevents nvcc from crashing when compiling Eigen on Tegra X1 -#if !defined(__NVCC__) || !EIGEN_ARCH_ARM_OR_ARM64 +#ifndef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_MAX_CPP_VER>=11 && (__cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900) \ + && (!defined(__NVCC__) || !EIGEN_ARCH_ARM_OR_ARM64 || (EIGEN_CUDACC_VER >= 80000) ) + // ^^ Disable the use of variadic templates when compiling with versions of nvcc older than 8.0 on ARM devices: + // this prevents nvcc from crashing when compiling Eigen on Tegra X1 #define EIGEN_HAS_VARIADIC_TEMPLATES 1 +#elif EIGEN_MAX_CPP_VER>=11 && (__cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900) && defined(__SYCL_DEVICE_ONLY__) +#define EIGEN_HAS_VARIADIC_TEMPLATES 1 +#else +#define EIGEN_HAS_VARIADIC_TEMPLATES 0 #endif #endif -// Does the compiler support const expressions? -#ifdef __CUDACC__ -// Const expressions are supported provided that c++11 is enabled and we're using either clang or nvcc 7.5 or above -#if __cplusplus > 199711L && defined(__CUDACC_VER__) && (defined(__clang__) || __CUDACC_VER__ >= 70500) - #define EIGEN_HAS_CONSTEXPR 1 -#endif -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || \ - EIGEN_GNUC_AT_LEAST(4,8) -#define EIGEN_HAS_CONSTEXPR 1 -#endif +// Does the compiler fully support const expressions? (as in c++14) +#ifndef EIGEN_HAS_CONSTEXPR + #if defined(EIGEN_CUDACC) + // Const expressions are supported provided that c++11 is enabled and we're using either clang or nvcc 7.5 or above + #if EIGEN_MAX_CPP_VER>=14 && (__cplusplus > 199711L && (EIGEN_COMP_CLANG || EIGEN_CUDACC_VER >= 70500)) + #define EIGEN_HAS_CONSTEXPR 1 + #endif + #elif EIGEN_MAX_CPP_VER>=14 && (__has_feature(cxx_relaxed_constexpr) || (defined(__cplusplus) && __cplusplus >= 201402L) || \ + (EIGEN_GNUC_AT_LEAST(4,8) && (__cplusplus > 199711L)) || \ + (EIGEN_COMP_CLANG >= 306 && (__cplusplus > 199711L))) + #define EIGEN_HAS_CONSTEXPR 1 + #endif + + #ifndef EIGEN_HAS_CONSTEXPR + #define EIGEN_HAS_CONSTEXPR 0 + #endif + +#endif // EIGEN_HAS_CONSTEXPR // Does the compiler support C++11 math? // Let's be conservative and enable the default C++11 implementation only if we are sure it exists #ifndef EIGEN_HAS_CXX11_MATH - #if (__cplusplus > 201103L) || (__cplusplus >= 201103L) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC) \ - && (EIGEN_ARCH_i386_OR_x86_64) && (EIGEN_OS_GNULINUX || EIGEN_OS_WIN_STRICT || EIGEN_OS_MAC) + #if EIGEN_MAX_CPP_VER>=11 && ((__cplusplus > 201103L) || (__cplusplus >= 201103L) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC) \ + && (EIGEN_ARCH_i386_OR_x86_64) && (EIGEN_OS_GNULINUX || EIGEN_OS_WIN_STRICT || EIGEN_OS_MAC)) #define EIGEN_HAS_CXX11_MATH 1 #else #define EIGEN_HAS_CXX11_MATH 0 @@ -392,9 +655,10 @@ // Does the compiler support proper C++11 containers? #ifndef EIGEN_HAS_CXX11_CONTAINERS - #if (__cplusplus > 201103L) \ + #if EIGEN_MAX_CPP_VER>=11 && \ + ((__cplusplus > 201103L) \ || ((__cplusplus >= 201103L) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_ICC>=1400)) \ - || EIGEN_COMP_MSVC >= 1900 + || EIGEN_COMP_MSVC >= 1900) #define EIGEN_HAS_CXX11_CONTAINERS 1 #else #define EIGEN_HAS_CXX11_CONTAINERS 0 @@ -403,30 +667,84 @@ // Does the compiler support C++11 noexcept? #ifndef EIGEN_HAS_CXX11_NOEXCEPT - #if (__cplusplus > 201103L) \ + #if EIGEN_MAX_CPP_VER>=11 && \ + (__has_feature(cxx_noexcept) \ + || (__cplusplus > 201103L) \ || ((__cplusplus >= 201103L) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_ICC>=1400)) \ - || EIGEN_COMP_MSVC >= 1900 + || EIGEN_COMP_MSVC >= 1900) #define EIGEN_HAS_CXX11_NOEXCEPT 1 #else #define EIGEN_HAS_CXX11_NOEXCEPT 0 #endif #endif -/** Allows to disable some optimizations which might affect the accuracy of the result. - * Such optimization are enabled by default, and set EIGEN_FAST_MATH to 0 to disable them. - * They currently include: - * - single precision ArrayBase::sin() and ArrayBase::cos() for SSE and AVX vectorization. - */ -#ifndef EIGEN_FAST_MATH -#define EIGEN_FAST_MATH 1 +#ifndef EIGEN_HAS_CXX11_ATOMIC + #if EIGEN_MAX_CPP_VER>=11 && \ + (__has_feature(cxx_atomic) \ + || (__cplusplus > 201103L) \ + || ((__cplusplus >= 201103L) && (EIGEN_COMP_MSVC==0 || EIGEN_COMP_MSVC >= 1700))) + #define EIGEN_HAS_CXX11_ATOMIC 1 + #else + #define EIGEN_HAS_CXX11_ATOMIC 0 + #endif #endif +#ifndef EIGEN_HAS_CXX11_OVERRIDE_FINAL + #if EIGEN_MAX_CPP_VER>=11 && \ + (__cplusplus >= 201103L || EIGEN_COMP_MSVC >= 1700) + #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 1 + #else + #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 0 + #endif +#endif + +// NOTE: the required Apple's clang version is very conservative +// and it could be that XCode 9 works just fine. +// NOTE: the MSVC version is based on https://en.cppreference.com/w/cpp/compiler_support +// and not tested. +#ifndef EIGEN_HAS_CXX17_OVERALIGN +#if EIGEN_MAX_CPP_VER>=17 && EIGEN_COMP_CXXVER>=17 && ( \ + (EIGEN_COMP_MSVC >= 1912) \ + || (EIGEN_GNUC_AT_LEAST(7,0)) \ + || ((!defined(__apple_build_version__)) && (EIGEN_COMP_CLANG>=500)) \ + || (( defined(__apple_build_version__)) && (__apple_build_version__>=10000000)) \ + ) +#define EIGEN_HAS_CXX17_OVERALIGN 1 +#else +#define EIGEN_HAS_CXX17_OVERALIGN 0 +#endif +#endif + +#if defined(EIGEN_CUDACC) && EIGEN_HAS_CONSTEXPR + // While available already with c++11, this is useful mostly starting with c++14 and relaxed constexpr rules + #if defined(__NVCC__) + // nvcc considers constexpr functions as __host__ __device__ with the option --expt-relaxed-constexpr + #ifdef __CUDACC_RELAXED_CONSTEXPR__ + #define EIGEN_CONSTEXPR_ARE_DEVICE_FUNC + #endif + #elif defined(__clang__) && defined(__CUDA__) && __has_feature(cxx_relaxed_constexpr) + // clang++ always considers constexpr functions as implicitly __host__ __device__ + #define EIGEN_CONSTEXPR_ARE_DEVICE_FUNC + #endif +#endif + + +//------------------------------------------------------------------------------------------ +// Preprocessor programming helpers +//------------------------------------------------------------------------------------------ + +// This macro can be used to prevent from macro expansion, e.g.: +// std::max EIGEN_NOT_A_MACRO(a,b) +#define EIGEN_NOT_A_MACRO + #define EIGEN_DEBUG_VAR(x) std::cerr << #x << " = " << x << std::endl; // concatenate two tokens #define EIGEN_CAT2(a,b) a ## b #define EIGEN_CAT(a,b) EIGEN_CAT2(a,b) +#define EIGEN_COMMA , + // convert a token to a string #define EIGEN_MAKESTRING2(a) #a #define EIGEN_MAKESTRING(a) EIGEN_MAKESTRING2(a) @@ -434,20 +752,23 @@ // EIGEN_STRONG_INLINE is a stronger version of the inline, using __forceinline on MSVC, // but it still doesn't use GCC's always_inline. This is useful in (common) situations where MSVC needs forceinline // but GCC is still doing fine with just inline. +#ifndef EIGEN_STRONG_INLINE #if EIGEN_COMP_MSVC || EIGEN_COMP_ICC #define EIGEN_STRONG_INLINE __forceinline #else #define EIGEN_STRONG_INLINE inline #endif +#endif // EIGEN_ALWAYS_INLINE is the stronget, it has the effect of making the function inline and adding every possible // attribute to maximize inlining. This should only be used when really necessary: in particular, // it uses __attribute__((always_inline)) on GCC, which most of the time is useless and can severely harm compile times. // FIXME with the always_inline attribute, -// gcc 3.4.x reports the following compilation error: +// gcc 3.4.x and 4.1 reports the following compilation error: // Eval.h:91: sorry, unimplemented: inlining failed in call to 'const Eigen::Eval<Derived> Eigen::MatrixBase<Scalar, Derived>::eval() const' // : function body not available -#if EIGEN_GNUC_AT_LEAST(4,0) +// See also bug 1367 +#if EIGEN_GNUC_AT_LEAST(4,2) #define EIGEN_ALWAYS_INLINE __attribute__((always_inline)) inline #else #define EIGEN_ALWAYS_INLINE EIGEN_STRONG_INLINE @@ -467,12 +788,38 @@ #define EIGEN_PERMISSIVE_EXPR #endif +// GPU stuff + +// Disable some features when compiling with GPU compilers (NVCC/clang-cuda/SYCL/HIPCC) +#if defined(EIGEN_CUDACC) || defined(__SYCL_DEVICE_ONLY__) || defined(EIGEN_HIPCC) + // Do not try asserts on device code + #ifndef EIGEN_NO_DEBUG + #define EIGEN_NO_DEBUG + #endif + + #ifdef EIGEN_INTERNAL_DEBUGGING + #undef EIGEN_INTERNAL_DEBUGGING + #endif + + #ifdef EIGEN_EXCEPTIONS + #undef EIGEN_EXCEPTIONS + #endif +#endif + +// All functions callable from CUDA/HIP code must be qualified with __device__ +#ifdef EIGEN_GPUCC + #define EIGEN_DEVICE_FUNC __host__ __device__ +#else + #define EIGEN_DEVICE_FUNC +#endif + + // this macro allows to get rid of linking errors about multiply defined functions. // - static is not very good because it prevents definitions from different object files to be merged. // So static causes the resulting linked executable to be bloated with multiple copies of the same function. // - inline is not perfect either as it unwantedly hints the compiler toward inlining the function. -#define EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS -#define EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS inline +#define EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_DEVICE_FUNC +#define EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_DEVICE_FUNC inline #ifdef NDEBUG # ifndef EIGEN_NO_DEBUG @@ -556,7 +903,7 @@ // Suppresses 'unused variable' warnings. namespace Eigen { namespace internal { - template<typename T> EIGEN_DEVICE_FUNC void ignore_unused_variable(const T&) {} + template<typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ignore_unused_variable(const T&) {} } } #define EIGEN_UNUSED_VARIABLE(var) Eigen::internal::ignore_unused_variable(var); @@ -570,161 +917,14 @@ #endif -//------------------------------------------------------------------------------------------ -// Static and dynamic alignment control -// -// The main purpose of this section is to define EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES -// as the maximal boundary in bytes on which dynamically and statically allocated data may be alignment respectively. -// The values of EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES can be specified by the user. If not, -// a default value is automatically computed based on architecture, compiler, and OS. -// -// This section also defines macros EIGEN_ALIGN_TO_BOUNDARY(N) and the shortcuts EIGEN_ALIGN{8,16,32,_MAX} -// to be used to declare statically aligned buffers. -//------------------------------------------------------------------------------------------ - - -/* EIGEN_ALIGN_TO_BOUNDARY(n) forces data to be n-byte aligned. This is used to satisfy SIMD requirements. - * However, we do that EVEN if vectorization (EIGEN_VECTORIZE) is disabled, - * so that vectorization doesn't affect binary compatibility. - * - * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link - * vectorized and non-vectorized code. - */ -#if (defined __CUDACC__) - #define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n) -#elif EIGEN_COMP_GNUC || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM - #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) -#elif EIGEN_COMP_MSVC - #define EIGEN_ALIGN_TO_BOUNDARY(n) __declspec(align(n)) -#elif EIGEN_COMP_SUNCC - // FIXME not sure about this one: - #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) +#if EIGEN_COMP_MSVC + // NOTE MSVC often gives C4127 warnings with compiletime if statements. See bug 1362. + // This workaround is ugly, but it does the job. +# define EIGEN_CONST_CONDITIONAL(cond) (void)0, cond #else - #error Please tell me what is the equivalent of __attribute__((aligned(n))) for your compiler +# define EIGEN_CONST_CONDITIONAL(cond) cond #endif -// If the user explicitly disable vectorization, then we also disable alignment -#if defined(EIGEN_DONT_VECTORIZE) - #define EIGEN_IDEAL_MAX_ALIGN_BYTES 0 -#elif defined(__AVX__) - // 32 bytes static alignmeent is preferred only if really required - #define EIGEN_IDEAL_MAX_ALIGN_BYTES 32 -#else - #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 -#endif - - -// EIGEN_MIN_ALIGN_BYTES defines the minimal value for which the notion of explicit alignment makes sense -#define EIGEN_MIN_ALIGN_BYTES 16 - -// Defined the boundary (in bytes) on which the data needs to be aligned. Note -// that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be -// aligned at all regardless of the value of this #define. - -#if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && EIGEN_MAX_STATIC_ALIGN_BYTES>0 -#error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY. -#endif - -// EIGEN_DONT_ALIGN_STATICALLY and EIGEN_DONT_ALIGN are deprectated -// They imply EIGEN_MAX_STATIC_ALIGN_BYTES=0 -#if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN) - #ifdef EIGEN_MAX_STATIC_ALIGN_BYTES - #undef EIGEN_MAX_STATIC_ALIGN_BYTES - #endif - #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 -#endif - -#ifndef EIGEN_MAX_STATIC_ALIGN_BYTES - - // Try to automatically guess what is the best default value for EIGEN_MAX_STATIC_ALIGN_BYTES - - // 16 byte alignment is only useful for vectorization. Since it affects the ABI, we need to enable - // 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always - // enable alignment, but it can be a cause of problems on some platforms, so we just disable it in - // certain common platform (compiler+architecture combinations) to avoid these problems. - // Only static alignment is really problematic (relies on nonstandard compiler extensions), - // try to keep heap alignment even when we have to disable static alignment. - #if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64) - #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 - #elif EIGEN_ARCH_ARM_OR_ARM64 && EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(4, 6) - // Old versions of GCC on ARM, at least 4.4, were once seen to have buggy static alignment support. - // Not sure which version fixed it, hopefully it doesn't affect 4.7, which is still somewhat in use. - // 4.8 and newer seem definitely unaffected. - #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 - #else - #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0 - #endif - - // static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX - #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT \ - && !EIGEN_GCC3_OR_OLDER \ - && !EIGEN_COMP_SUNCC \ - && !EIGEN_OS_QNX - #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1 - #else - #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0 - #endif - - #if EIGEN_ARCH_WANTS_STACK_ALIGNMENT - #define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES - #else - #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 - #endif - -#endif - -// If EIGEN_MAX_ALIGN_BYTES is defined, then it is considered as an upper bound for EIGEN_MAX_ALIGN_BYTES -#if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES<EIGEN_MAX_STATIC_ALIGN_BYTES -#undef EIGEN_MAX_STATIC_ALIGN_BYTES -#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES -#endif - -#if EIGEN_MAX_STATIC_ALIGN_BYTES==0 && !defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT) - #define EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT -#endif - -// At this stage, EIGEN_MAX_STATIC_ALIGN_BYTES>0 is the true test whether we want to align arrays on the stack or not. -// It takes into account both the user choice to explicitly enable/disable alignment (by settting EIGEN_MAX_STATIC_ALIGN_BYTES) -// and the architecture config (EIGEN_ARCH_WANTS_STACK_ALIGNMENT). -// Henceforth, only EIGEN_MAX_STATIC_ALIGN_BYTES should be used. - - -// Shortcuts to EIGEN_ALIGN_TO_BOUNDARY -#define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8) -#define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16) -#define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32) -#define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64) -#if EIGEN_MAX_STATIC_ALIGN_BYTES>0 -#define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES) -#else -#define EIGEN_ALIGN_MAX -#endif - - -// Dynamic alignment control - -#if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES>0 -#error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN. -#endif - -#ifdef EIGEN_DONT_ALIGN - #ifdef EIGEN_MAX_ALIGN_BYTES - #undef EIGEN_MAX_ALIGN_BYTES - #endif - #define EIGEN_MAX_ALIGN_BYTES 0 -#elif !defined(EIGEN_MAX_ALIGN_BYTES) - #define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES -#endif - -#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES -#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES -#else -#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES -#endif - -//---------------------------------------------------------------------- - - #ifdef EIGEN_DONT_USE_RESTRICT_KEYWORD #define EIGEN_RESTRICT #endif @@ -732,10 +932,6 @@ #define EIGEN_RESTRICT __restrict #endif -#ifndef EIGEN_STACK_ALLOCATION_LIMIT -// 131072 == 128 KB -#define EIGEN_STACK_ALLOCATION_LIMIT 131072 -#endif #ifndef EIGEN_DEFAULT_IO_FORMAT #ifdef EIGEN_MAKING_DOCS @@ -750,7 +946,20 @@ // just an empty macro ! #define EIGEN_EMPTY -#if EIGEN_COMP_MSVC_STRICT && EIGEN_COMP_MSVC < 1900 // for older MSVC versions using the base operator is sufficient (cf Bug 1000) + +// When compiling CUDA/HIP device code with NVCC or HIPCC +// pull in math functions from the global namespace. +// In host mode, and when device code is compiled with clang, +// use the std versions. +#if (defined(EIGEN_CUDA_ARCH) && defined(__NVCC__)) || defined(EIGEN_HIP_DEVICE_COMPILE) + #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC; +#else + #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC; +#endif + + +#if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC < 1900 || EIGEN_CUDACC_VER>0) + // for older MSVC versions, as well as 1900 && CUDA 8, using the base operator is sufficient (cf Bugs 1000, 1324) #define EIGEN_INHERIT_ASSIGNMENT_EQUAL_OPERATOR(Derived) \ using Base::operator =; #elif EIGEN_COMP_CLANG // workaround clang bug (see http://forum.kde.org/viewtopic.php?f=74&t=102653) @@ -791,7 +1000,8 @@ typedef typename Eigen::internal::ref_selector<Derived>::type Nested; \ typedef typename Eigen::internal::traits<Derived>::StorageKind StorageKind; \ typedef typename Eigen::internal::traits<Derived>::StorageIndex StorageIndex; \ - enum { RowsAtCompileTime = Eigen::internal::traits<Derived>::RowsAtCompileTime, \ + enum CompileTimeTraits \ + { RowsAtCompileTime = Eigen::internal::traits<Derived>::RowsAtCompileTime, \ ColsAtCompileTime = Eigen::internal::traits<Derived>::ColsAtCompileTime, \ Flags = Eigen::internal::traits<Derived>::Flags, \ SizeAtCompileTime = Base::SizeAtCompileTime, \ @@ -836,18 +1046,10 @@ #define EIGEN_IMPLIES(a,b) (!(a) || (b)) -#define EIGEN_MAKE_CWISE_BINARY_OP(METHOD,FUNCTOR) \ - template<typename OtherDerived> \ - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp<FUNCTOR<Scalar>, const Derived, const OtherDerived> \ - (METHOD)(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const \ - { \ - return CwiseBinaryOp<FUNCTOR<Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); \ - } - -// the expression type of a cwise product -#define EIGEN_CWISE_PRODUCT_RETURN_TYPE(LHS,RHS) \ +// the expression type of a standard coefficient wise binary operation +#define EIGEN_CWISE_BINARY_RETURN_TYPE(LHS,RHS,OPNAME) \ CwiseBinaryOp< \ - internal::scalar_product_op< \ + EIGEN_CAT(EIGEN_CAT(internal::scalar_,OPNAME),_op)< \ typename internal::traits<LHS>::Scalar, \ typename internal::traits<RHS>::Scalar \ >, \ @@ -855,15 +1057,72 @@ const RHS \ > +#define EIGEN_MAKE_CWISE_BINARY_OP(METHOD,OPNAME) \ + template<typename OtherDerived> \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,OPNAME) \ + (METHOD)(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const \ + { \ + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,OPNAME)(derived(), other.derived()); \ + } + +#define EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,TYPEA,TYPEB) \ + (Eigen::internal::has_ReturnType<Eigen::ScalarBinaryOpTraits<TYPEA,TYPEB,EIGEN_CAT(EIGEN_CAT(Eigen::internal::scalar_,OPNAME),_op)<TYPEA,TYPEB> > >::value) + +#define EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(EXPR,SCALAR,OPNAME) \ + CwiseBinaryOp<EIGEN_CAT(EIGEN_CAT(internal::scalar_,OPNAME),_op)<typename internal::traits<EXPR>::Scalar,SCALAR>, const EXPR, \ + const typename internal::plain_constant_type<EXPR,SCALAR>::type> + +#define EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(SCALAR,EXPR,OPNAME) \ + CwiseBinaryOp<EIGEN_CAT(EIGEN_CAT(internal::scalar_,OPNAME),_op)<SCALAR,typename internal::traits<EXPR>::Scalar>, \ + const typename internal::plain_constant_type<EXPR,SCALAR>::type, const EXPR> + +// Workaround for MSVC 2010 (see ML thread "patch with compile for for MSVC 2010") +#if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC_STRICT<=1600) +#define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) typename internal::enable_if<true,X>::type +#else +#define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) X +#endif + +#define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(METHOD,OPNAME) \ + template <typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ + EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg<Scalar EIGEN_COMMA T EIGEN_COMMA EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,Scalar,T)>::type,OPNAME))\ + (METHOD)(const T& scalar) const { \ + typedef typename internal::promote_scalar_arg<Scalar,T,EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,Scalar,T)>::type PromotedT; \ + return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedT,OPNAME)(derived(), \ + typename internal::plain_constant_type<Derived,PromotedT>::type(derived().rows(), derived().cols(), internal::scalar_constant_op<PromotedT>(scalar))); \ + } + +#define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(METHOD,OPNAME) \ + template <typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend \ + EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg<Scalar EIGEN_COMMA T EIGEN_COMMA EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,T,Scalar)>::type,Derived,OPNAME)) \ + (METHOD)(const T& scalar, const StorageBaseType& matrix) { \ + typedef typename internal::promote_scalar_arg<Scalar,T,EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,T,Scalar)>::type PromotedT; \ + return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedT,Derived,OPNAME)( \ + typename internal::plain_constant_type<Derived,PromotedT>::type(matrix.derived().rows(), matrix.derived().cols(), internal::scalar_constant_op<PromotedT>(scalar)), matrix.derived()); \ + } + +#define EIGEN_MAKE_SCALAR_BINARY_OP(METHOD,OPNAME) \ + EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(METHOD,OPNAME) \ + EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(METHOD,OPNAME) + + +#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(EIGEN_CUDA_ARCH) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL) && !defined(EIGEN_HIP_DEVICE_COMPILE) + #define EIGEN_EXCEPTIONS +#endif + + #ifdef EIGEN_EXCEPTIONS # define EIGEN_THROW_X(X) throw X # define EIGEN_THROW throw # define EIGEN_TRY try # define EIGEN_CATCH(X) catch (X) #else -# ifdef __CUDA_ARCH__ -# define EIGEN_THROW_X(X) asm("trap;") return {} -# define EIGEN_THROW asm("trap;"); return {} +# if defined(EIGEN_CUDA_ARCH) +# define EIGEN_THROW_X(X) asm("trap;") +# define EIGEN_THROW asm("trap;") +# elif defined(EIGEN_HIP_DEVICE_COMPILE) +# define EIGEN_THROW_X(X) asm("s_trap 0") +# define EIGEN_THROW asm("s_trap 0") # else # define EIGEN_THROW_X(X) std::abort() # define EIGEN_THROW std::abort() @@ -872,12 +1131,58 @@ # define EIGEN_CATCH(X) else #endif + #if EIGEN_HAS_CXX11_NOEXCEPT +# define EIGEN_INCLUDE_TYPE_TRAITS +# define EIGEN_NOEXCEPT noexcept +# define EIGEN_NOEXCEPT_IF(x) noexcept(x) # define EIGEN_NO_THROW noexcept(true) # define EIGEN_EXCEPTION_SPEC(X) noexcept(false) #else +# define EIGEN_NOEXCEPT +# define EIGEN_NOEXCEPT_IF(x) # define EIGEN_NO_THROW throw() -# define EIGEN_EXCEPTION_SPEC(X) throw(X) +# if EIGEN_COMP_MSVC || EIGEN_COMP_CXXVER>=17 + // MSVC does not support exception specifications (warning C4290), + // and they are deprecated in c++11 anyway. This is even an error in c++17. +# define EIGEN_EXCEPTION_SPEC(X) throw() +# else +# define EIGEN_EXCEPTION_SPEC(X) throw(X) +# endif +#endif + +#if EIGEN_HAS_VARIADIC_TEMPLATES +// The all function is used to enable a variadic version of eigen_assert which can take a parameter pack as its input. +namespace Eigen { +namespace internal { + +inline bool all(){ return true; } + +template<typename T, typename ...Ts> +bool all(T t, Ts ... ts){ return t && all(ts...); } + +} +} +#endif + +#if EIGEN_HAS_CXX11_OVERRIDE_FINAL +// provide override and final specifiers if they are available: +# define EIGEN_OVERRIDE override +# define EIGEN_FINAL final +#else +# define EIGEN_OVERRIDE +# define EIGEN_FINAL +#endif + +// Wrapping #pragma unroll in a macro since it is required for SYCL +#if defined(__SYCL_DEVICE_ONLY__) + #if defined(_MSC_VER) + #define EIGEN_UNROLL_LOOP __pragma(unroll) + #else + #define EIGEN_UNROLL_LOOP _Pragma("unroll") + #endif +#else + #define EIGEN_UNROLL_LOOP #endif #endif // EIGEN_MACROS_H
diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index 5f8bf15..efd7472 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h
@@ -63,14 +63,27 @@ namespace internal { -EIGEN_DEVICE_FUNC +EIGEN_DEVICE_FUNC inline void throw_std_bad_alloc() { #ifdef EIGEN_EXCEPTIONS throw std::bad_alloc(); #else std::size_t huge = static_cast<std::size_t>(-1); + #if defined(EIGEN_HIPCC) + // + // calls to "::operator new" are to be treated as opaque function calls (i.e no inlining), + // and as a consequence the code in the #else block triggers the hipcc warning : + // "no overloaded function has restriction specifiers that are compatible with the ambient context" + // + // "throw_std_bad_alloc" has the EIGEN_DEVICE_FUNC attribute, so it seems that hipcc expects + // the same on "operator new" + // Reverting code back to the old version in this #if block for the hipcc compiler + // new int[huge]; + #else + ::operator new(huge); + #endif #endif } @@ -83,19 +96,32 @@ /** \internal Like malloc, but the returned pointer is guaranteed to be 16-byte aligned. * Fast, but wastes 16 additional bytes of memory. Does not throw any exception. */ -inline void* handmade_aligned_malloc(std::size_t size) +EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) { - void *original = std::malloc(size+EIGEN_DEFAULT_ALIGN_BYTES); + eigen_assert(alignment >= sizeof(void*) && (alignment & (alignment-1)) == 0 && "Alignment must be at least sizeof(void*) and a power of 2"); + +#if defined(EIGEN_HIP_DEVICE_COMPILE) + void *original = ::malloc(size+alignment); +#else + void *original = std::malloc(size+alignment); +#endif + if (original == 0) return 0; - void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + EIGEN_DEFAULT_ALIGN_BYTES); + void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(alignment-1))) + alignment); *(reinterpret_cast<void**>(aligned) - 1) = original; return aligned; } /** \internal Frees memory allocated with handmade_aligned_malloc */ -inline void handmade_aligned_free(void *ptr) +EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void *ptr) { - if (ptr) std::free(*(reinterpret_cast<void**>(ptr) - 1)); + if (ptr) { +#if defined(EIGEN_HIP_DEVICE_COMPILE) + ::free(*(reinterpret_cast<void**>(ptr) - 1)); +#else + std::free(*(reinterpret_cast<void**>(ptr) - 1)); +#endif + } } /** \internal @@ -114,7 +140,7 @@ void *previous_aligned = static_cast<char *>(original)+previous_offset; if(aligned!=previous_aligned) std::memmove(aligned, previous_aligned, size); - + *(reinterpret_cast<void**>(aligned) - 1) = original; return aligned; } @@ -142,7 +168,7 @@ { eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)"); } -#else +#else EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {} #endif @@ -150,15 +176,21 @@ /** \internal Allocates \a size bytes. The returned pointer is guaranteed to have 16 or 32 bytes alignment depending on the requirements. * On allocation error, the returned pointer is null, and std::bad_alloc is thrown. */ -EIGEN_DEVICE_FUNC inline void* aligned_malloc(size_t size) +EIGEN_DEVICE_FUNC inline void* aligned_malloc(std::size_t size) { check_that_malloc_is_allowed(); void *result; #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED + + #if defined(EIGEN_HIP_DEVICE_COMPILE) + result = ::malloc(size); + #else result = std::malloc(size); + #endif + #if EIGEN_DEFAULT_ALIGN_BYTES==16 - eigen_assert((size<16 || (std::size_t(result)%16)==0) && "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback to handmade alignd memory allocator."); + eigen_assert((size<16 || (std::size_t(result)%16)==0) && "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback to handmade aligned memory allocator."); #endif #else result = handmade_aligned_malloc(size); @@ -174,7 +206,13 @@ EIGEN_DEVICE_FUNC inline void aligned_free(void *ptr) { #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED + + #if defined(EIGEN_HIP_DEVICE_COMPILE) + ::free(ptr); + #else std::free(ptr); + #endif + #else handmade_aligned_free(ptr); #endif @@ -185,7 +223,7 @@ * \brief Reallocates an aligned block of memory. * \throws std::bad_alloc on allocation failure */ -inline void* aligned_realloc(void *ptr, size_t new_size, size_t old_size) +inline void* aligned_realloc(void *ptr, std::size_t new_size, std::size_t old_size) { EIGEN_UNUSED_VARIABLE(old_size); @@ -209,16 +247,21 @@ /** \internal Allocates \a size bytes. If Align is true, then the returned ptr is 16-byte-aligned. * On allocation error, the returned pointer is null, and a std::bad_alloc is thrown. */ -template<bool Align> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(size_t size) +template<bool Align> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) { return aligned_malloc(size); } -template<> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc<false>(size_t size) +template<> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc<false>(std::size_t size) { check_that_malloc_is_allowed(); + #if defined(EIGEN_HIP_DEVICE_COMPILE) + void *result = ::malloc(size); + #else void *result = std::malloc(size); + #endif + if(!result && size) throw_std_bad_alloc(); return result; @@ -232,15 +275,19 @@ template<> EIGEN_DEVICE_FUNC inline void conditional_aligned_free<false>(void *ptr) { + #if defined(EIGEN_HIP_DEVICE_COMPILE) + ::free(ptr); + #else std::free(ptr); + #endif } -template<bool Align> inline void* conditional_aligned_realloc(void* ptr, size_t new_size, size_t old_size) +template<bool Align> inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) { return aligned_realloc(ptr, new_size, old_size); } -template<> inline void* conditional_aligned_realloc<false>(void* ptr, size_t new_size, size_t) +template<> inline void* conditional_aligned_realloc<false>(void* ptr, std::size_t new_size, std::size_t) { return std::realloc(ptr, new_size); } @@ -252,7 +299,7 @@ /** \internal Destructs the elements of an array. * The \a size parameters tells on how many objects to call the destructor of T. */ -template<typename T> EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T *ptr, size_t size) +template<typename T> EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T *ptr, std::size_t size) { // always destruct an array starting from the end. if(ptr) @@ -262,9 +309,9 @@ /** \internal Constructs the elements of an array. * The \a size parameter tells on how many objects to call the constructor of T. */ -template<typename T> EIGEN_DEVICE_FUNC inline T* construct_elements_of_array(T *ptr, size_t size) +template<typename T> EIGEN_DEVICE_FUNC inline T* construct_elements_of_array(T *ptr, std::size_t size) { - size_t i; + std::size_t i; EIGEN_TRY { for (i = 0; i < size; ++i) ::new (ptr + i) T; @@ -275,6 +322,7 @@ destruct_elements_of_array(ptr, i); EIGEN_THROW; } + return NULL; } /***************************************************************************** @@ -282,9 +330,9 @@ *****************************************************************************/ template<typename T> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(size_t size) +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(std::size_t size) { - if(size > size_t(-1) / sizeof(T)) + if(size > std::size_t(-1) / sizeof(T)) throw_std_bad_alloc(); } @@ -292,7 +340,7 @@ * On allocation error, the returned pointer is undefined, but a std::bad_alloc is thrown. * The default constructor of T is called. */ -template<typename T> EIGEN_DEVICE_FUNC inline T* aligned_new(size_t size) +template<typename T> EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) { check_size_for_overflow<T>(size); T *result = reinterpret_cast<T*>(aligned_malloc(sizeof(T)*size)); @@ -305,9 +353,10 @@ aligned_free(result); EIGEN_THROW; } + return result; } -template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(size_t size) +template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(std::size_t size) { check_size_for_overflow<T>(size); T *result = reinterpret_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T)*size)); @@ -320,12 +369,13 @@ conditional_aligned_free<Align>(result); EIGEN_THROW; } + return result; } /** \internal Deletes objects constructed with aligned_new * The \a size parameters tells on how many objects to call the destructor of T. */ -template<typename T> EIGEN_DEVICE_FUNC inline void aligned_delete(T *ptr, size_t size) +template<typename T> EIGEN_DEVICE_FUNC inline void aligned_delete(T *ptr, std::size_t size) { destruct_elements_of_array<T>(ptr, size); aligned_free(ptr); @@ -334,13 +384,13 @@ /** \internal Deletes objects constructed with conditional_aligned_new * The \a size parameters tells on how many objects to call the destructor of T. */ -template<typename T, bool Align> EIGEN_DEVICE_FUNC inline void conditional_aligned_delete(T *ptr, size_t size) +template<typename T, bool Align> EIGEN_DEVICE_FUNC inline void conditional_aligned_delete(T *ptr, std::size_t size) { destruct_elements_of_array<T>(ptr, size); conditional_aligned_free<Align>(ptr); } -template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new(T* pts, size_t new_size, size_t old_size) +template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size) { check_size_for_overflow<T>(new_size); check_size_for_overflow<T>(old_size); @@ -363,7 +413,7 @@ } -template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(size_t size) +template<typename T, bool Align> EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { if(size==0) return 0; // short-cut. Also fixes Bug 884 @@ -384,7 +434,7 @@ return result; } -template<typename T, bool Align> inline T* conditional_aligned_realloc_new_auto(T* pts, size_t new_size, size_t old_size) +template<typename T, bool Align> inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) { check_size_for_overflow<T>(new_size); check_size_for_overflow<T>(old_size); @@ -406,7 +456,7 @@ return result; } -template<typename T, bool Align> EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T *ptr, size_t size) +template<typename T, bool Align> EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T *ptr, std::size_t size) { if(NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size); @@ -445,7 +495,7 @@ // so that all elements of the array have the same alignment. return 0; } - else if( (std::size_t(array) & (sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0) + else if( (UIntPtr(array) & (sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0) { // The array is not aligned to the size of a single scalar, or the requested alignment is not a multiple of the scalar size. // Consequently, no element of the array is well aligned. @@ -453,7 +503,7 @@ } else { - Index first = (AlignmentSize - (Index((std::size_t(array)/sizeof(Scalar))) & AlignmentMask)) & AlignmentMask; + Index first = (AlignmentSize - (Index((UIntPtr(array)/sizeof(Scalar))) & AlignmentMask)) & AlignmentMask; return (first < size) ? first : size; } } @@ -468,8 +518,8 @@ } /** \internal Returns the smallest integer multiple of \a base and greater or equal to \a size - */ -template<typename Index> + */ +template<typename Index> inline Index first_multiple(Index size, Index base) { return ((size+base-1)/base)*base; @@ -487,10 +537,14 @@ template<typename T> struct smart_copy_helper<T,true> { EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) { - std::ptrdiff_t size = std::ptrdiff_t(end)-std::ptrdiff_t(start); + IntPtr size = IntPtr(end)-IntPtr(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); - memcpy(target, start, size); + #if defined(EIGEN_HIP_DEVICE_COMPILE) + ::memcpy(target, start, size); + #else + std::memcpy(target, start, size); + #endif } }; @@ -499,7 +553,7 @@ { std::copy(start, end, target); } }; -// intelligent memmove. falls back to std::memmove for POD types, uses std::copy otherwise. +// intelligent memmove. falls back to std::memmove for POD types, uses std::copy otherwise. template<typename T, bool UseMemmove> struct smart_memmove_helper; template<typename T> void smart_memmove(const T* start, const T* end, T* target) @@ -510,7 +564,7 @@ template<typename T> struct smart_memmove_helper<T,true> { static inline void run(const T* start, const T* end, T* target) { - std::ptrdiff_t size = std::ptrdiff_t(end)-std::ptrdiff_t(start); + IntPtr size = IntPtr(end)-IntPtr(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); std::memmove(target, start, size); @@ -519,15 +573,15 @@ template<typename T> struct smart_memmove_helper<T,false> { static inline void run(const T* start, const T* end, T* target) - { - if (uintptr_t(target) < uintptr_t(start)) + { + if (UIntPtr(target) < UIntPtr(start)) { std::copy(start, end, target); } - else + else { std::ptrdiff_t count = (std::ptrdiff_t(end)-std::ptrdiff_t(start)) / sizeof(T); - std::copy_backward(start, end, target + count); + std::copy_backward(start, end, target + count); } } }; @@ -539,7 +593,7 @@ // you can overwrite Eigen's default behavior regarding alloca by defining EIGEN_ALLOCA // to the appropriate stack allocation function -#ifndef EIGEN_ALLOCA +#if ! defined EIGEN_ALLOCA && ! defined EIGEN_GPU_COMPILE_PHASE #if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca) #define EIGEN_ALLOCA alloca #elif EIGEN_COMP_MSVC @@ -547,6 +601,15 @@ #endif #endif +// With clang -Oz -mthumb, alloca changes the stack pointer in a way that is +// not allowed in Thumb2. -DEIGEN_STACK_ALLOCATION_LIMIT=0 doesn't work because +// the compiler still emits bad code because stack allocation checks use "<=". +// TODO: Eliminate after https://bugs.llvm.org/show_bug.cgi?id=23772 +// is fixed. +#if defined(__clang__) && defined(__thumb__) + #undef EIGEN_ALLOCA +#endif + // This helper class construct the allocated memory, and takes care of destructing and freeing the handled data // at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions. template<typename T> class aligned_stack_memory_handler : noncopyable @@ -558,12 +621,14 @@ * In this case, the buffer elements will also be destructed when this handler will be destructed. * Finally, if \a dealloc is true, then the pointer \a ptr is freed. **/ - aligned_stack_memory_handler(T* ptr, size_t size, bool dealloc) + EIGEN_DEVICE_FUNC + aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc) : m_ptr(ptr), m_size(size), m_deallocate(dealloc) { if(NumTraits<T>::RequireInitialization && m_ptr) Eigen::internal::construct_elements_of_array(m_ptr, size); } + EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() { if(NumTraits<T>::RequireInitialization && m_ptr) @@ -573,10 +638,64 @@ } protected: T* m_ptr; - size_t m_size; + std::size_t m_size; bool m_deallocate; }; +#ifdef EIGEN_ALLOCA + +template<typename Xpr, int NbEvaluations, + bool MapExternalBuffer = nested_eval<Xpr,NbEvaluations>::Evaluate && Xpr::MaxSizeAtCompileTime==Dynamic + > +struct local_nested_eval_wrapper +{ + static const bool NeedExternalBuffer = false; + typedef typename Xpr::Scalar Scalar; + typedef typename nested_eval<Xpr,NbEvaluations>::type ObjectType; + ObjectType object; + + EIGEN_DEVICE_FUNC + local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) : object(xpr) + { + EIGEN_UNUSED_VARIABLE(ptr); + eigen_internal_assert(ptr==0); + } +}; + +template<typename Xpr, int NbEvaluations> +struct local_nested_eval_wrapper<Xpr,NbEvaluations,true> +{ + static const bool NeedExternalBuffer = true; + typedef typename Xpr::Scalar Scalar; + typedef typename plain_object_eval<Xpr>::type PlainObject; + typedef Map<PlainObject,EIGEN_DEFAULT_ALIGN_BYTES> ObjectType; + ObjectType object; + + EIGEN_DEVICE_FUNC + local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) + : object(ptr==0 ? reinterpret_cast<Scalar*>(Eigen::internal::aligned_malloc(sizeof(Scalar)*xpr.size())) : ptr, xpr.rows(), xpr.cols()), + m_deallocate(ptr==0) + { + if(NumTraits<Scalar>::RequireInitialization && object.data()) + Eigen::internal::construct_elements_of_array(object.data(), object.size()); + object = xpr; + } + + EIGEN_DEVICE_FUNC + ~local_nested_eval_wrapper() + { + if(NumTraits<Scalar>::RequireInitialization && object.data()) + Eigen::internal::destruct_elements_of_array(object.data(), object.size()); + if(m_deallocate) + Eigen::internal::aligned_free(object.data()); + } + +private: + bool m_deallocate; +}; + +#endif // EIGEN_ALLOCA + template<typename T> class scoped_array : noncopyable { T* m_ptr; @@ -600,13 +719,15 @@ { std::swap(a.ptr(),b.ptr()); } - + } // end namespace internal /** \internal - * Declares, allocates and construct an aligned buffer named NAME of SIZE elements of type TYPE on the stack - * if SIZE is smaller than EIGEN_STACK_ALLOCATION_LIMIT, and if stack allocation is supported by the platform - * (currently, this is Linux and Visual Studio only). Otherwise the memory is allocated on the heap. + * + * The macro ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) declares, allocates, + * and construct an aligned buffer named NAME of SIZE elements of type TYPE on the stack + * if the size in bytes is smaller than EIGEN_STACK_ALLOCATION_LIMIT, and if stack allocation is supported by the platform + * (currently, this is Linux, OSX and Visual Studio only). Otherwise the memory is allocated on the heap. * The allocated buffer is automatically deleted when exiting the scope of this declaration. * If BUFFER is non null, then the declared variable is simply an alias for BUFFER, and no allocation/deletion occurs. * Here is an example: @@ -617,13 +738,21 @@ * } * \endcode * The underlying stack allocation function can controlled with the EIGEN_ALLOCA preprocessor token. + * + * The macro ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) is analogue to + * \code + * typename internal::nested_eval<XPRT_T,N>::type NAME(XPR); + * \endcode + * with the advantage of using aligned stack allocation even if the maximal size of XPR at compile time is unknown. + * This is accomplished through alloca if this later is supported and if the required number of bytes + * is below EIGEN_STACK_ALLOCATION_LIMIT. */ #ifdef EIGEN_ALLOCA - + #if EIGEN_DEFAULT_ALIGN_BYTES>0 // We always manually re-align the result of EIGEN_ALLOCA. // If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment. - #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((reinterpret_cast<std::size_t>(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((internal::UIntPtr(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) #else #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE) #endif @@ -636,13 +765,23 @@ : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \ Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT) + + #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) \ + Eigen::internal::local_nested_eval_wrapper<XPR_T,N> EIGEN_CAT(NAME,_wrapper)(XPR, reinterpret_cast<typename XPR_T::Scalar*>( \ + ( (Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::NeedExternalBuffer) && ((sizeof(typename XPR_T::Scalar)*XPR.size())<=EIGEN_STACK_ALLOCATION_LIMIT) ) \ + ? EIGEN_ALIGNED_ALLOCA( sizeof(typename XPR_T::Scalar)*XPR.size() ) : 0 ) ) ; \ + typename Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::ObjectType NAME(EIGEN_CAT(NAME,_wrapper).object) + #else #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \ Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \ Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true) - + + +#define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) typename Eigen::internal::nested_eval<XPR_T,N>::type NAME(XPR) + #endif @@ -650,17 +789,28 @@ *** Implementation of EIGEN_MAKE_ALIGNED_OPERATOR_NEW [_IF] *** *****************************************************************************/ +#if EIGEN_HAS_CXX17_OVERALIGN + +// C++17 -> no need to bother about alignment anymore :) + +#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) +#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) +#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW +#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) + +#else + #if EIGEN_MAX_ALIGN_BYTES!=0 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \ - void* operator new(size_t size, const std::nothrow_t&) EIGEN_NO_THROW { \ + void* operator new(std::size_t size, const std::nothrow_t&) EIGEN_NO_THROW { \ EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \ EIGEN_CATCH (...) { return 0; } \ } #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \ - void *operator new(size_t size) { \ + void *operator new(std::size_t size) { \ return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \ } \ - void *operator new[](size_t size) { \ + void *operator new[](std::size_t size) { \ return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \ } \ void operator delete(void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \ @@ -670,8 +820,8 @@ /* in-place new and delete. since (at least afaik) there is no actual */ \ /* memory allocated we can safely let the default implementation handle */ \ /* this particular case. */ \ - static void *operator new(size_t size, void *ptr) { return ::operator new(size,ptr); } \ - static void *operator new[](size_t size, void* ptr) { return ::operator new[](size,ptr); } \ + static void *operator new(std::size_t size, void *ptr) { return ::operator new(size,ptr); } \ + static void *operator new[](std::size_t size, void* ptr) { return ::operator new[](size,ptr); } \ void operator delete(void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete(memory,ptr); } \ void operator delete[](void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete[](memory,ptr); } \ /* nothrow-new (returns zero instead of std::bad_alloc) */ \ @@ -685,20 +835,34 @@ #endif #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true) -#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \ - EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(((Size)!=Eigen::Dynamic) && ((sizeof(Scalar)*(Size))%EIGEN_MAX_ALIGN_BYTES==0))) +#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \ + EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool( \ + ((Size)!=Eigen::Dynamic) && \ + (((EIGEN_MAX_ALIGN_BYTES>=16) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES )==0)) || \ + ((EIGEN_MAX_ALIGN_BYTES>=32) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/2)==0)) || \ + ((EIGEN_MAX_ALIGN_BYTES>=64) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/4)==0)) ))) + +#endif /****************************************************************************/ /** \class aligned_allocator * \ingroup Core_Module * -* \brief STL compatible allocator to use with with 16 byte aligned types +* \brief STL compatible allocator to use with types requiring a non standrad alignment. +* +* The memory is aligned as for dynamically aligned matrix/array types such as MatrixXd. +* By default, it will thus provide at least 16 bytes alignment and more in following cases: +* - 32 bytes alignment if AVX is enabled. +* - 64 bytes alignment if AVX512 is enabled. +* +* This can be controlled using the \c EIGEN_MAX_ALIGN_BYTES macro as documented +* \link TopicPreprocessorDirectivesPerformance there \endlink. * * Example: * \code * // Matrix4f requires 16 bytes alignment: -* std::map< int, Matrix4f, std::less<int>, +* std::map< int, Matrix4f, std::less<int>, * aligned_allocator<std::pair<const int, Matrix4f> > > my_map_mat4; * // Vector3f does not require 16 bytes alignment, no need to use Eigen's allocator: * std::map< int, Vector3f > my_map_vec3; @@ -710,7 +874,7 @@ class aligned_allocator : public std::allocator<T> { public: - typedef size_t size_type; + typedef std::size_t size_type; typedef std::ptrdiff_t difference_type; typedef T* pointer; typedef const T* const_pointer; @@ -733,6 +897,15 @@ ~aligned_allocator() {} + #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_LEAST(7,0) + // In gcc std::allocator::max_size() is bugged making gcc triggers a warning: + // eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object size 9223372036854775807 + // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544 + size_type max_size() const { + return (std::numeric_limits<std::ptrdiff_t>::max)()/sizeof(T); + } + #endif + pointer allocate(size_type num, const void* /*hint*/ = 0) { internal::check_size_for_overflow<T>(num);
diff --git a/Eigen/src/Core/util/Meta.h b/Eigen/src/Core/util/Meta.h old mode 100644 new mode 100755 index 24e8a6d..8fcb18a --- a/Eigen/src/Core/util/Meta.h +++ b/Eigen/src/Core/util/Meta.h
@@ -11,13 +11,36 @@ #ifndef EIGEN_META_H #define EIGEN_META_H -#if defined(__CUDA_ARCH__) -#include <cfloat> -#include <math_constants.h> +#if defined(EIGEN_GPU_COMPILE_PHASE) + + #include <cfloat> + + #if defined(EIGEN_CUDA_ARCH) + #include <math_constants.h> + #endif + + #if defined(EIGEN_HIP_DEVICE_COMPILE) + #include "Eigen/src/Core/arch/HIP/hcc/math_constants.h" + #endif + +#endif + +#if EIGEN_COMP_ICC>=1600 && __cplusplus >= 201103L +#include <cstdint> #endif namespace Eigen { +typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE DenseIndex; + +/** + * \brief The Index type as used for the API. + * \details To change this, \c \#define the preprocessor symbol \c EIGEN_DEFAULT_DENSE_INDEX_TYPE. + * \sa \blank \ref TopicPreprocessorDirectives, StorageIndex. + */ + +typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE Index; + namespace internal { /** \internal @@ -27,6 +50,16 @@ * we however don't want to add a dependency to Boost. */ +// Only recent versions of ICC complain about using ptrdiff_t to hold pointers, +// and older versions do not provide *intptr_t types. +#if EIGEN_COMP_ICC>=1600 && __cplusplus >= 201103L +typedef std::intptr_t IntPtr; +typedef std::uintptr_t UIntPtr; +#else +typedef std::ptrdiff_t IntPtr; +typedef std::size_t UIntPtr; +#endif + struct true_type { enum { value = 1 }; }; struct false_type { enum { value = 0 }; }; @@ -73,17 +106,49 @@ template<> struct is_arithmetic<signed long> { enum { value = true }; }; template<> struct is_arithmetic<unsigned long> { enum { value = true }; }; -template<typename T> struct is_integral { enum { value = false }; }; -template<> struct is_integral<bool> { enum { value = true }; }; -template<> struct is_integral<char> { enum { value = true }; }; -template<> struct is_integral<signed char> { enum { value = true }; }; -template<> struct is_integral<unsigned char> { enum { value = true }; }; -template<> struct is_integral<signed short> { enum { value = true }; }; -template<> struct is_integral<unsigned short> { enum { value = true }; }; -template<> struct is_integral<signed int> { enum { value = true }; }; -template<> struct is_integral<unsigned int> { enum { value = true }; }; -template<> struct is_integral<signed long> { enum { value = true }; }; -template<> struct is_integral<unsigned long> { enum { value = true }; }; +#if EIGEN_HAS_CXX11 +template<> struct is_arithmetic<signed long long> { enum { value = true }; }; +template<> struct is_arithmetic<unsigned long long> { enum { value = true }; }; +using std::is_integral; +#else +template<typename T> struct is_integral { enum { value = false }; }; +template<> struct is_integral<bool> { enum { value = true }; }; +template<> struct is_integral<char> { enum { value = true }; }; +template<> struct is_integral<signed char> { enum { value = true }; }; +template<> struct is_integral<unsigned char> { enum { value = true }; }; +template<> struct is_integral<signed short> { enum { value = true }; }; +template<> struct is_integral<unsigned short> { enum { value = true }; }; +template<> struct is_integral<signed int> { enum { value = true }; }; +template<> struct is_integral<unsigned int> { enum { value = true }; }; +template<> struct is_integral<signed long> { enum { value = true }; }; +template<> struct is_integral<unsigned long> { enum { value = true }; }; +#if EIGEN_COMP_MSVC +template<> struct is_integral<signed __int64> { enum { value = true }; }; +template<> struct is_integral<unsigned __int64> { enum { value = true }; }; +#endif +#endif + +#if EIGEN_HAS_CXX11 +using std::make_unsigned; +#else +// TODO: Possibly improve this implementation of make_unsigned. +// It is currently used only by +// template<typename Scalar> struct random_default_impl<Scalar, false, true>. +template<typename> struct make_unsigned; +template<> struct make_unsigned<char> { typedef unsigned char type; }; +template<> struct make_unsigned<signed char> { typedef unsigned char type; }; +template<> struct make_unsigned<unsigned char> { typedef unsigned char type; }; +template<> struct make_unsigned<signed short> { typedef unsigned short type; }; +template<> struct make_unsigned<unsigned short> { typedef unsigned short type; }; +template<> struct make_unsigned<signed int> { typedef unsigned int type; }; +template<> struct make_unsigned<unsigned int> { typedef unsigned int type; }; +template<> struct make_unsigned<signed long> { typedef unsigned long type; }; +template<> struct make_unsigned<unsigned long> { typedef unsigned long type; }; +#if EIGEN_COMP_MSVC +template<> struct make_unsigned<signed __int64> { typedef unsigned __int64 type; }; +template<> struct make_unsigned<unsigned __int64> { typedef unsigned __int64 type; }; +#endif +#endif template <typename T> struct add_const { typedef const T type; }; template <typename T> struct add_const<T&> { typedef T& type; }; @@ -110,30 +175,39 @@ struct yes {int a[1];}; struct no {int a[2];}; - static yes test(const To&, int); + template<typename T> + static yes test(T, int); + + template<typename T> static no test(any_conversion, ...); public: - static From ms_from; - enum { value = sizeof(test(ms_from, 0))==sizeof(yes) }; + static typename internal::remove_reference<From>::type* ms_from; +#ifdef __INTEL_COMPILER + #pragma warning push + #pragma warning ( disable : 2259 ) +#endif + enum { value = sizeof(test<To>(*ms_from, 0))==sizeof(yes) }; +#ifdef __INTEL_COMPILER + #pragma warning pop +#endif }; template<typename From, typename To> struct is_convertible { - enum { value = is_convertible_impl<typename remove_all<From>::type, - typename remove_all<To >::type>::value }; + enum { value = is_convertible_impl<From,To>::value }; }; /** \internal Allows to enable/disable an overload * according to a compile time condition. */ -template<bool Condition, typename T> struct enable_if; +template<bool Condition, typename T=void> struct enable_if; template<typename T> struct enable_if<true,T> { typedef T type; }; -#if defined(__CUDA_ARCH__) +#if defined(EIGEN_GPU_COMPILE_PHASE) #if !defined(__FLT_EPSILON__) #define __FLT_EPSILON__ FLT_EPSILON #define __DBL_EPSILON__ DBL_EPSILON @@ -155,13 +229,31 @@ EIGEN_DEVICE_FUNC static float epsilon() { return __FLT_EPSILON__; } EIGEN_DEVICE_FUNC - static float (max)() { return CUDART_MAX_NORMAL_F; } + static float (max)() { + #if defined(EIGEN_CUDA_ARCH) + return CUDART_MAX_NORMAL_F; + #else + return HIPRT_MAX_NORMAL_F; + #endif + } EIGEN_DEVICE_FUNC static float (min)() { return FLT_MIN; } EIGEN_DEVICE_FUNC - static float infinity() { return CUDART_INF_F; } + static float infinity() { + #if defined(EIGEN_CUDA_ARCH) + return CUDART_INF_F; + #else + return HIPRT_INF_F; + #endif + } EIGEN_DEVICE_FUNC - static float quiet_NaN() { return CUDART_NAN_F; } + static float quiet_NaN() { + #if defined(EIGEN_CUDA_ARCH) + return CUDART_NAN_F; + #else + return HIPRT_NAN_F; + #endif + } }; template<> struct numeric_limits<double> { @@ -172,9 +264,21 @@ EIGEN_DEVICE_FUNC static double (min)() { return DBL_MIN; } EIGEN_DEVICE_FUNC - static double infinity() { return CUDART_INF; } + static double infinity() { + #if defined(EIGEN_CUDA_ARCH) + return CUDART_INF; + #else + return HIPRT_INF; + #endif + } EIGEN_DEVICE_FUNC - static double quiet_NaN() { return CUDART_NAN; } + static double quiet_NaN() { + #if defined(EIGEN_CUDA_ARCH) + return CUDART_NAN; + #else + return HIPRT_NAN; + #endif + } }; template<> struct numeric_limits<int> { @@ -236,7 +340,7 @@ #endif /** \internal - * A base class do disable default copy ctor and copy assignement operator. + * A base class do disable default copy ctor and copy assignment operator. */ class noncopyable { @@ -248,13 +352,66 @@ }; /** \internal + * Provides access to the number of elements in the object of as a compile-time constant expression. + * It "returns" Eigen::Dynamic if the size cannot be resolved at compile-time (default). + * + * Similar to std::tuple_size, but more general. + * + * It currently supports: + * - any types T defining T::SizeAtCompileTime + * - plain C arrays as T[N] + * - std::array (c++11) + * - some internal types such as SingleRange and AllRange + * + * The second template parameter eases SFINAE-based specializations. + */ +template<typename T, typename EnableIf = void> struct array_size { + enum { value = Dynamic }; +}; + +template<typename T> struct array_size<T,typename internal::enable_if<((T::SizeAtCompileTime&0)==0)>::type> { + enum { value = T::SizeAtCompileTime }; +}; + +template<typename T, int N> struct array_size<const T (&)[N]> { + enum { value = N }; +}; +template<typename T, int N> struct array_size<T (&)[N]> { + enum { value = N }; +}; + +#if EIGEN_HAS_CXX11 +template<typename T, std::size_t N> struct array_size<const std::array<T,N> > { + enum { value = N }; +}; +template<typename T, std::size_t N> struct array_size<std::array<T,N> > { + enum { value = N }; +}; +#endif + +/** \internal + * Analogue of the std::size free function. + * It returns the size of the container or view \a x of type \c T + * + * It currently supports: + * - any types T defining a member T::size() const + * - plain C arrays as T[N] + * + */ +template<typename T> +Index size(const T& x) { return x.size(); } + +template<typename T,std::size_t N> +Index size(const T (&) [N]) { return N; } + +/** \internal * Convenient struct to get the result type of a unary or binary functor. * * It supports both the current STL mechanism (using the result_type member) as well as * upcoming next STL generation (using a templated result member). * If none of these members is provided, then the type of the first argument is returned. FIXME, that behavior is a pretty bad hack. */ -#ifdef EIGEN_HAS_STD_RESULT_OF +#if EIGEN_HAS_STD_RESULT_OF template<typename T> struct result_of { typedef typename std::result_of<T>::type type1; typedef typename remove_all<type1>::type type; @@ -311,8 +468,74 @@ enum {FunctorType = sizeof(testFunctor(static_cast<Func*>(0)))}; typedef typename binary_result_of_select<Func, ArgType0, ArgType1, FunctorType>::type type; }; + +template<typename Func, typename ArgType0, typename ArgType1, typename ArgType2, int SizeOf=sizeof(has_none)> +struct ternary_result_of_select {typedef typename internal::remove_all<ArgType0>::type type;}; + +template<typename Func, typename ArgType0, typename ArgType1, typename ArgType2> +struct ternary_result_of_select<Func, ArgType0, ArgType1, ArgType2, sizeof(has_std_result_type)> +{typedef typename Func::result_type type;}; + +template<typename Func, typename ArgType0, typename ArgType1, typename ArgType2> +struct ternary_result_of_select<Func, ArgType0, ArgType1, ArgType2, sizeof(has_tr1_result)> +{typedef typename Func::template result<Func(ArgType0,ArgType1,ArgType2)>::type type;}; + +template<typename Func, typename ArgType0, typename ArgType1, typename ArgType2> +struct result_of<Func(ArgType0,ArgType1,ArgType2)> { + template<typename T> + static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); + template<typename T> + static has_tr1_result testFunctor(T const *, typename T::template result<T(ArgType0,ArgType1,ArgType2)>::type const * = 0); + static has_none testFunctor(...); + + // note that the following indirection is needed for gcc-3.3 + enum {FunctorType = sizeof(testFunctor(static_cast<Func*>(0)))}; + typedef typename ternary_result_of_select<Func, ArgType0, ArgType1, ArgType2, FunctorType>::type type; +}; #endif +struct meta_yes { char a[1]; }; +struct meta_no { char a[2]; }; + +// Check whether T::ReturnType does exist +template <typename T> +struct has_ReturnType +{ + template <typename C> static meta_yes testFunctor(C const *, typename C::ReturnType const * = 0); + template <typename C> static meta_no testFunctor(...); + + enum { value = sizeof(testFunctor<T>(static_cast<T*>(0))) == sizeof(meta_yes) }; +}; + +template<typename T> const T* return_ptr(); + +template <typename T, typename IndexType=Index> +struct has_nullary_operator +{ + template <typename C> static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr<C>()->operator()())>0)>::type * = 0); + static meta_no testFunctor(...); + + enum { value = sizeof(testFunctor(static_cast<T*>(0))) == sizeof(meta_yes) }; +}; + +template <typename T, typename IndexType=Index> +struct has_unary_operator +{ + template <typename C> static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr<C>()->operator()(IndexType(0)))>0)>::type * = 0); + static meta_no testFunctor(...); + + enum { value = sizeof(testFunctor(static_cast<T*>(0))) == sizeof(meta_yes) }; +}; + +template <typename T, typename IndexType=Index> +struct has_binary_operator +{ + template <typename C> static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr<C>()->operator()(IndexType(0),IndexType(0)))>0)>::type * = 0); + static meta_no testFunctor(...); + + enum { value = sizeof(testFunctor(static_cast<T*>(0))) == sizeof(meta_yes) }; +}; + /** \internal In short, it computes int(sqrt(\a Y)) with \a Y an integer. * Usage example: \code meta_sqrt<1023>::ret \endcode */ @@ -358,33 +581,6 @@ enum { Defined = 0 }; }; -template<typename T> struct scalar_product_traits<T,T> -{ - enum { - // Cost = NumTraits<T>::MulCost, - Defined = 1 - }; - typedef T ReturnType; -}; - -template<typename T> struct scalar_product_traits<T,std::complex<T> > -{ - enum { - // Cost = 2*NumTraits<T>::MulCost, - Defined = 1 - }; - typedef std::complex<T> ReturnType; -}; - -template<typename T> struct scalar_product_traits<std::complex<T>, T> -{ - enum { - // Cost = 2*NumTraits<T>::MulCost, - Defined = 1 - }; - typedef std::complex<T> ReturnType; -}; - // FIXME quick workaround around current limitation of result_of // template<typename Scalar, typename ArgType0, typename ArgType1> // struct result_of<scalar_product_op<Scalar>(ArgType0,ArgType1)> { @@ -394,14 +590,14 @@ } // end namespace internal namespace numext { - -#if defined(__CUDA_ARCH__) + +#if defined(EIGEN_GPU_COMPILE_PHASE) template<typename T> EIGEN_DEVICE_FUNC void swap(T &a, T &b) { T tmp = b; b = a; a = tmp; } #else template<typename T> EIGEN_STRONG_INLINE void swap(T &a, T &b) { std::swap(a,b); } #endif -#if defined(__CUDA_ARCH__) +#if defined(EIGEN_GPU_COMPILE_PHASE) using internal::device::numeric_limits; #else using std::numeric_limits; @@ -410,13 +606,71 @@ // Integer division with rounding up. // T is assumed to be an integer type with a>=0, and b>0 template<typename T> +EIGEN_DEVICE_FUNC T div_ceil(const T &a, const T &b) { return (a+b-1) / b; } +// The aim of the following functions is to bypass -Wfloat-equal warnings +// when we really want a strict equality comparison on floating points. +template<typename X, typename Y> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool equal_strict(const X& x,const Y& y) { return x == y; } + +#if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool equal_strict(const float& x,const float& y) { return std::equal_to<float>()(x,y); } + +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool equal_strict(const double& x,const double& y) { return std::equal_to<double>()(x,y); } +#endif + +template<typename X, typename Y> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool not_equal_strict(const X& x,const Y& y) { return x != y; } + +#if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool not_equal_strict(const float& x,const float& y) { return std::not_equal_to<float>()(x,y); } + +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC +bool not_equal_strict(const double& x,const double& y) { return std::not_equal_to<double>()(x,y); } +#endif + +/** \internal extract the bits of the float \a x */ +inline unsigned int as_uint(float x) +{ + unsigned int ret; + std::memcpy(&ret, &x, sizeof(float)); + return ret; +} + } // end namespace numext } // end namespace Eigen +// Define portable (u)int{32,64} types +#if EIGEN_HAS_CXX11 +#include <cstdint> +namespace Eigen { +namespace numext { +typedef std::uint32_t uint32_t; +typedef std::int32_t int32_t; +typedef std::uint64_t uint64_t; +typedef std::int64_t int64_t; +} +} +#else +// Without c++11, all compilers able to compile Eigen also +// provides the C99 stdint.h header file. +#include <stdint.h> +namespace Eigen { +namespace numext { +typedef ::uint32_t uint32_t; +typedef ::int32_t int32_t; +typedef ::uint64_t uint64_t; +typedef ::int64_t int64_t; +} +} +#endif + #endif // EIGEN_META_H
diff --git a/Eigen/src/Core/util/ReenableStupidWarnings.h b/Eigen/src/Core/util/ReenableStupidWarnings.h index a23fab1..1ce6fd1 100644 --- a/Eigen/src/Core/util/ReenableStupidWarnings.h +++ b/Eigen/src/Core/util/ReenableStupidWarnings.h
@@ -1,4 +1,8 @@ -#ifdef EIGEN_WARNINGS_DISABLED +#ifdef EIGEN_WARNINGS_DISABLED_2 +// "DisableStupidWarnings.h" was included twice recursively: Do not reenable warnings yet! +# undef EIGEN_WARNINGS_DISABLED_2 + +#elif defined(EIGEN_WARNINGS_DISABLED) #undef EIGEN_WARNINGS_DISABLED #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS @@ -8,17 +12,20 @@ #pragma warning pop #elif defined __clang__ #pragma clang diagnostic pop + #elif defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) + #pragma GCC diagnostic pop #endif #if defined __NVCC__ // Don't reenable the diagnostic messages, as it turns out these messages need // to be disabled at the point of the template instantiation (i.e the user code) -// otherwise they'll be triggeredby nvcc. +// otherwise they'll be triggered by nvcc. // #pragma diag_default code_is_unreachable // #pragma diag_default initialization_not_reachable // #pragma diag_default 2651 // #pragma diag_default 2653 #endif + #endif #endif // EIGEN_WARNINGS_DISABLED
diff --git a/Eigen/src/Core/util/ReshapedHelper.h b/Eigen/src/Core/util/ReshapedHelper.h new file mode 100644 index 0000000..4124321 --- /dev/null +++ b/Eigen/src/Core/util/ReshapedHelper.h
@@ -0,0 +1,51 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#ifndef EIGEN_RESHAPED_HELPER_H +#define EIGEN_RESHAPED_HELPER_H + +namespace Eigen { + +enum AutoSize_t { AutoSize }; +const int AutoOrder = 2; + +namespace internal { + +template<typename SizeType,typename OtherSize, int TotalSize> +struct get_compiletime_reshape_size { + enum { value = get_fixed_value<SizeType>::value }; +}; + +template<typename SizeType> +Index get_runtime_reshape_size(SizeType size, Index /*other*/, Index /*total*/) { + return internal::get_runtime_value(size); +} + +template<typename OtherSize, int TotalSize> +struct get_compiletime_reshape_size<AutoSize_t,OtherSize,TotalSize> { + enum { + other_size = get_fixed_value<OtherSize>::value, + value = (TotalSize==Dynamic || other_size==Dynamic) ? Dynamic : TotalSize / other_size }; +}; + +inline Index get_runtime_reshape_size(AutoSize_t /*size*/, Index other, Index total) { + return total/other; +} + +template<int Flags, int Order> +struct get_compiletime_reshape_order { + enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order }; +}; + +} + +} // end namespace Eigen + +#endif // EIGEN_RESHAPED_HELPER_H
diff --git a/Eigen/src/Core/util/StaticAssert.h b/Eigen/src/Core/util/StaticAssert.h index afae2e5..67714e4 100644 --- a/Eigen/src/Core/util/StaticAssert.h +++ b/Eigen/src/Core/util/StaticAssert.h
@@ -24,9 +24,10 @@ * */ +#ifndef EIGEN_STATIC_ASSERT #ifndef EIGEN_NO_STATIC_ASSERT - #if __has_feature(cxx_static_assert) || (defined(__cplusplus) && __cplusplus >= 201103L) || (EIGEN_COMP_MSVC >= 1600) + #if EIGEN_MAX_CPP_VER>=11 && (__has_feature(cxx_static_assert) || (defined(__cplusplus) && __cplusplus >= 201103L) || (EIGEN_COMP_MSVC >= 1600)) // if native static_assert is enabled, let's use it #define EIGEN_STATIC_ASSERT(X,MSG) static_assert(X,#MSG); @@ -44,61 +45,67 @@ struct static_assertion<true> { enum { - YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX, - YOU_MIXED_VECTORS_OF_DIFFERENT_SIZES, - YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES, - THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE, - THIS_METHOD_IS_ONLY_FOR_MATRICES_OF_A_SPECIFIC_SIZE, - THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE, - OUT_OF_RANGE_ACCESS, - YOU_MADE_A_PROGRAMMING_MISTAKE, - EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT, - EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE, - YOU_CALLED_A_FIXED_SIZE_METHOD_ON_A_DYNAMIC_SIZE_MATRIX_OR_VECTOR, - YOU_CALLED_A_DYNAMIC_SIZE_METHOD_ON_A_FIXED_SIZE_MATRIX_OR_VECTOR, - UNALIGNED_LOAD_AND_STORE_OPERATIONS_UNIMPLEMENTED_ON_ALTIVEC, - THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES, - FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED, - NUMERIC_TYPE_MUST_BE_REAL, - COEFFICIENT_WRITE_ACCESS_TO_SELFADJOINT_NOT_SUPPORTED, - WRITING_TO_TRIANGULAR_PART_WITH_UNIT_DIAGONAL_IS_NOT_SUPPORTED, - THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE, - INVALID_MATRIX_PRODUCT, - INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS, - INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION, - YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY, - THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES, - THIS_METHOD_IS_ONLY_FOR_ROW_MAJOR_MATRICES, - INVALID_MATRIX_TEMPLATE_PARAMETERS, - INVALID_MATRIXBASE_TEMPLATE_PARAMETERS, - BOTH_MATRICES_MUST_HAVE_THE_SAME_STORAGE_ORDER, - THIS_METHOD_IS_ONLY_FOR_DIAGONAL_MATRIX, - THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE, - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_WITH_DIRECT_MEMORY_ACCESS_SUCH_AS_MAP_OR_PLAIN_MATRICES, - YOU_ALREADY_SPECIFIED_THIS_STRIDE, - INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION, - THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD, - PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1, - THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS, - YOU_CANNOT_MIX_ARRAYS_AND_MATRICES, - YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION, - THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY, - YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT, - THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS, - THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS, - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL, - THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES, - YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED, - YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED, - THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE, - THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH, - OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG, - IMPLICIT_CONVERSION_TO_SCALAR_IS_FOR_INNER_PRODUCT_ONLY, - STORAGE_LAYOUT_DOES_NOT_MATCH, - EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT__INVALID_COST_VALUE, - THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS, - MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY, - THIS_TYPE_IS_NOT_SUPPORTED + YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX=1, + YOU_MIXED_VECTORS_OF_DIFFERENT_SIZES=1, + YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES=1, + THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE=1, + THIS_METHOD_IS_ONLY_FOR_MATRICES_OF_A_SPECIFIC_SIZE=1, + THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE=1, + OUT_OF_RANGE_ACCESS=1, + YOU_MADE_A_PROGRAMMING_MISTAKE=1, + EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT=1, + EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE=1, + YOU_CALLED_A_FIXED_SIZE_METHOD_ON_A_DYNAMIC_SIZE_MATRIX_OR_VECTOR=1, + YOU_CALLED_A_DYNAMIC_SIZE_METHOD_ON_A_FIXED_SIZE_MATRIX_OR_VECTOR=1, + UNALIGNED_LOAD_AND_STORE_OPERATIONS_UNIMPLEMENTED_ON_ALTIVEC=1, + THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES=1, + FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED=1, + NUMERIC_TYPE_MUST_BE_REAL=1, + COEFFICIENT_WRITE_ACCESS_TO_SELFADJOINT_NOT_SUPPORTED=1, + WRITING_TO_TRIANGULAR_PART_WITH_UNIT_DIAGONAL_IS_NOT_SUPPORTED=1, + THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE=1, + INVALID_MATRIX_PRODUCT=1, + INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS=1, + INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION=1, + YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY=1, + THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES=1, + THIS_METHOD_IS_ONLY_FOR_ROW_MAJOR_MATRICES=1, + INVALID_MATRIX_TEMPLATE_PARAMETERS=1, + INVALID_MATRIXBASE_TEMPLATE_PARAMETERS=1, + BOTH_MATRICES_MUST_HAVE_THE_SAME_STORAGE_ORDER=1, + THIS_METHOD_IS_ONLY_FOR_DIAGONAL_MATRIX=1, + THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE=1, + THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_WITH_DIRECT_MEMORY_ACCESS_SUCH_AS_MAP_OR_PLAIN_MATRICES=1, + YOU_ALREADY_SPECIFIED_THIS_STRIDE=1, + INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION=1, + THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD=1, + PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1=1, + THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS=1, + YOU_CANNOT_MIX_ARRAYS_AND_MATRICES=1, + YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION=1, + THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY=1, + YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT=1, + THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS=1, + THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS=1, + THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL=1, + THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES=1, + YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED=1, + YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED=1, + THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE=1, + THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH=1, + OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG=1, + IMPLICIT_CONVERSION_TO_SCALAR_IS_FOR_INNER_PRODUCT_ONLY=1, + STORAGE_LAYOUT_DOES_NOT_MATCH=1, + EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT__INVALID_COST_VALUE=1, + THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS=1, + MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY=1, + THIS_TYPE_IS_NOT_SUPPORTED=1, + STORAGE_KIND_MUST_MATCH=1, + STORAGE_INDEX_MUST_MATCH=1, + CHOLMOD_SUPPORTS_DOUBLE_PRECISION_ONLY=1, + SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY=1, + INVALID_TEMPLATE_PARAMETER=1, + GPU_TENSOR_CONTRACTION_DOES_NOT_SUPPORT_OUTPUT_KERNELS=1 }; }; @@ -128,7 +135,7 @@ #define EIGEN_STATIC_ASSERT(CONDITION,MSG) eigen_assert((CONDITION) && #MSG); #endif // EIGEN_NO_STATIC_ASSERT - +#endif // EIGEN_STATIC_ASSERT // static assertion failing if the type \a TYPE is not a vector type #define EIGEN_STATIC_ASSERT_VECTOR_ONLY(TYPE) \ @@ -165,7 +172,7 @@ #define EIGEN_PREDICATE_SAME_MATRIX_SIZE(TYPE0,TYPE1) \ ( \ - (int(internal::size_of_xpr_at_compile_time<TYPE0>::ret)==0 && int(internal::size_of_xpr_at_compile_time<TYPE1>::ret)==0) \ + (int(Eigen::internal::size_of_xpr_at_compile_time<TYPE0>::ret)==0 && int(Eigen::internal::size_of_xpr_at_compile_time<TYPE1>::ret)==0) \ || (\ (int(TYPE0::RowsAtCompileTime)==Eigen::Dynamic \ || int(TYPE1::RowsAtCompileTime)==Eigen::Dynamic \ @@ -192,16 +199,16 @@ THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS) #define EIGEN_STATIC_ASSERT_LVALUE(Derived) \ - EIGEN_STATIC_ASSERT(internal::is_lvalue<Derived>::value, \ + EIGEN_STATIC_ASSERT(Eigen::internal::is_lvalue<Derived>::value, \ THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY) #define EIGEN_STATIC_ASSERT_ARRAYXPR(Derived) \ - EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Derived>::XprKind, ArrayXpr>::value), \ + EIGEN_STATIC_ASSERT((Eigen::internal::is_same<typename Eigen::internal::traits<Derived>::XprKind, ArrayXpr>::value), \ THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES) #define EIGEN_STATIC_ASSERT_SAME_XPR_KIND(Derived1, Derived2) \ - EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Derived1>::XprKind, \ - typename internal::traits<Derived2>::XprKind \ + EIGEN_STATIC_ASSERT((Eigen::internal::is_same<typename Eigen::internal::traits<Derived1>::XprKind, \ + typename Eigen::internal::traits<Derived2>::XprKind \ >::value), \ YOU_CANNOT_MIX_ARRAYS_AND_MATRICES)
diff --git a/Eigen/src/Core/util/SymbolicIndex.h b/Eigen/src/Core/util/SymbolicIndex.h new file mode 100644 index 0000000..17cf46f --- /dev/null +++ b/Eigen/src/Core/util/SymbolicIndex.h
@@ -0,0 +1,293 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SYMBOLIC_INDEX_H +#define EIGEN_SYMBOLIC_INDEX_H + +namespace Eigen { + +/** \namespace Eigen::symbolic + * \ingroup Core_Module + * + * This namespace defines a set of classes and functions to build and evaluate symbolic expressions of scalar type Index. + * Here is a simple example: + * + * \code + * // First step, defines symbols: + * struct x_tag {}; static const symbolic::SymbolExpr<x_tag> x; + * struct y_tag {}; static const symbolic::SymbolExpr<y_tag> y; + * struct z_tag {}; static const symbolic::SymbolExpr<z_tag> z; + * + * // Defines an expression: + * auto expr = (x+3)/y+z; + * + * // And evaluate it: (c++14) + * std::cout << expr.eval(x=6,y=3,z=-13) << "\n"; + * + * // In c++98/11, only one symbol per expression is supported for now: + * auto expr98 = (3-x)/2; + * std::cout << expr98.eval(x=6) << "\n"; + * \endcode + * + * It is currently only used internally to define and manipulate the Eigen::last and Eigen::lastp1 symbols in Eigen::seq and Eigen::seqN. + * + */ +namespace symbolic { + +template<typename Tag> class Symbol; +template<typename Arg0> class NegateExpr; +template<typename Arg1,typename Arg2> class AddExpr; +template<typename Arg1,typename Arg2> class ProductExpr; +template<typename Arg1,typename Arg2> class QuotientExpr; + +// A simple wrapper around an integral value to provide the eval method. +// We could also use a free-function symbolic_eval... +template<typename IndexType=Index> +class ValueExpr { +public: + ValueExpr(IndexType val) : m_value(val) {} + template<typename T> + IndexType eval_impl(const T&) const { return m_value; } +protected: + IndexType m_value; +}; + +// Specialization for compile-time value, +// It is similar to ValueExpr(N) but this version helps the compiler to generate better code. +template<int N> +class ValueExpr<internal::FixedInt<N> > { +public: + ValueExpr() {} + template<typename T> + Index eval_impl(const T&) const { return N; } +}; + + +/** \class BaseExpr + * \ingroup Core_Module + * Common base class of any symbolic expressions + */ +template<typename Derived> +class BaseExpr +{ +public: + const Derived& derived() const { return *static_cast<const Derived*>(this); } + + /** Evaluate the expression given the \a values of the symbols. + * + * \param values defines the values of the symbols, it can either be a SymbolValue or a std::tuple of SymbolValue + * as constructed by SymbolExpr::operator= operator. + * + */ + template<typename T> + Index eval(const T& values) const { return derived().eval_impl(values); } + +#if EIGEN_HAS_CXX14 + template<typename... Types> + Index eval(Types&&... values) const { return derived().eval_impl(std::make_tuple(values...)); } +#endif + + NegateExpr<Derived> operator-() const { return NegateExpr<Derived>(derived()); } + + AddExpr<Derived,ValueExpr<> > operator+(Index b) const + { return AddExpr<Derived,ValueExpr<> >(derived(), b); } + AddExpr<Derived,ValueExpr<> > operator-(Index a) const + { return AddExpr<Derived,ValueExpr<> >(derived(), -a); } + ProductExpr<Derived,ValueExpr<> > operator*(Index a) const + { return ProductExpr<Derived,ValueExpr<> >(derived(),a); } + QuotientExpr<Derived,ValueExpr<> > operator/(Index a) const + { return QuotientExpr<Derived,ValueExpr<> >(derived(),a); } + + friend AddExpr<Derived,ValueExpr<> > operator+(Index a, const BaseExpr& b) + { return AddExpr<Derived,ValueExpr<> >(b.derived(), a); } + friend AddExpr<NegateExpr<Derived>,ValueExpr<> > operator-(Index a, const BaseExpr& b) + { return AddExpr<NegateExpr<Derived>,ValueExpr<> >(-b.derived(), a); } + friend ProductExpr<ValueExpr<>,Derived> operator*(Index a, const BaseExpr& b) + { return ProductExpr<ValueExpr<>,Derived>(a,b.derived()); } + friend QuotientExpr<ValueExpr<>,Derived> operator/(Index a, const BaseExpr& b) + { return QuotientExpr<ValueExpr<>,Derived>(a,b.derived()); } + + template<int N> + AddExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator+(internal::FixedInt<N>) const + { return AddExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + AddExpr<Derived,ValueExpr<internal::FixedInt<-N> > > operator-(internal::FixedInt<N>) const + { return AddExpr<Derived,ValueExpr<internal::FixedInt<-N> > >(derived(), ValueExpr<internal::FixedInt<-N> >()); } + template<int N> + ProductExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator*(internal::FixedInt<N>) const + { return ProductExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(),ValueExpr<internal::FixedInt<N> >()); } + template<int N> + QuotientExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator/(internal::FixedInt<N>) const + { return QuotientExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(),ValueExpr<internal::FixedInt<N> >()); } + + template<int N> + friend AddExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator+(internal::FixedInt<N>, const BaseExpr& b) + { return AddExpr<Derived,ValueExpr<internal::FixedInt<N> > >(b.derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + friend AddExpr<NegateExpr<Derived>,ValueExpr<internal::FixedInt<N> > > operator-(internal::FixedInt<N>, const BaseExpr& b) + { return AddExpr<NegateExpr<Derived>,ValueExpr<internal::FixedInt<N> > >(-b.derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + friend ProductExpr<ValueExpr<internal::FixedInt<N> >,Derived> operator*(internal::FixedInt<N>, const BaseExpr& b) + { return ProductExpr<ValueExpr<internal::FixedInt<N> >,Derived>(ValueExpr<internal::FixedInt<N> >(),b.derived()); } + template<int N> + friend QuotientExpr<ValueExpr<internal::FixedInt<N> >,Derived> operator/(internal::FixedInt<N>, const BaseExpr& b) + { return QuotientExpr<ValueExpr<internal::FixedInt<N> > ,Derived>(ValueExpr<internal::FixedInt<N> >(),b.derived()); } + +#if (!EIGEN_HAS_CXX14) + template<int N> + AddExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator+(internal::FixedInt<N> (*)()) const + { return AddExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + AddExpr<Derived,ValueExpr<internal::FixedInt<-N> > > operator-(internal::FixedInt<N> (*)()) const + { return AddExpr<Derived,ValueExpr<internal::FixedInt<-N> > >(derived(), ValueExpr<internal::FixedInt<-N> >()); } + template<int N> + ProductExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator*(internal::FixedInt<N> (*)()) const + { return ProductExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(),ValueExpr<internal::FixedInt<N> >()); } + template<int N> + QuotientExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator/(internal::FixedInt<N> (*)()) const + { return QuotientExpr<Derived,ValueExpr<internal::FixedInt<N> > >(derived(),ValueExpr<internal::FixedInt<N> >()); } + + template<int N> + friend AddExpr<Derived,ValueExpr<internal::FixedInt<N> > > operator+(internal::FixedInt<N> (*)(), const BaseExpr& b) + { return AddExpr<Derived,ValueExpr<internal::FixedInt<N> > >(b.derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + friend AddExpr<NegateExpr<Derived>,ValueExpr<internal::FixedInt<N> > > operator-(internal::FixedInt<N> (*)(), const BaseExpr& b) + { return AddExpr<NegateExpr<Derived>,ValueExpr<internal::FixedInt<N> > >(-b.derived(), ValueExpr<internal::FixedInt<N> >()); } + template<int N> + friend ProductExpr<ValueExpr<internal::FixedInt<N> >,Derived> operator*(internal::FixedInt<N> (*)(), const BaseExpr& b) + { return ProductExpr<ValueExpr<internal::FixedInt<N> >,Derived>(ValueExpr<internal::FixedInt<N> >(),b.derived()); } + template<int N> + friend QuotientExpr<ValueExpr<internal::FixedInt<N> >,Derived> operator/(internal::FixedInt<N> (*)(), const BaseExpr& b) + { return QuotientExpr<ValueExpr<internal::FixedInt<N> > ,Derived>(ValueExpr<internal::FixedInt<N> >(),b.derived()); } +#endif + + + template<typename OtherDerived> + AddExpr<Derived,OtherDerived> operator+(const BaseExpr<OtherDerived> &b) const + { return AddExpr<Derived,OtherDerived>(derived(), b.derived()); } + + template<typename OtherDerived> + AddExpr<Derived,NegateExpr<OtherDerived> > operator-(const BaseExpr<OtherDerived> &b) const + { return AddExpr<Derived,NegateExpr<OtherDerived> >(derived(), -b.derived()); } + + template<typename OtherDerived> + ProductExpr<Derived,OtherDerived> operator*(const BaseExpr<OtherDerived> &b) const + { return ProductExpr<Derived,OtherDerived>(derived(), b.derived()); } + + template<typename OtherDerived> + QuotientExpr<Derived,OtherDerived> operator/(const BaseExpr<OtherDerived> &b) const + { return QuotientExpr<Derived,OtherDerived>(derived(), b.derived()); } +}; + +template<typename T> +struct is_symbolic { + // BaseExpr has no conversion ctor, so we only have to check whether T can be statically cast to its base class BaseExpr<T>. + enum { value = internal::is_convertible<T,BaseExpr<T> >::value }; +}; + +/** Represents the actual value of a symbol identified by its tag + * + * It is the return type of SymbolValue::operator=, and most of the time this is only way it is used. + */ +template<typename Tag> +class SymbolValue +{ +public: + /** Default constructor from the value \a val */ + SymbolValue(Index val) : m_value(val) {} + + /** \returns the stored value of the symbol */ + Index value() const { return m_value; } +protected: + Index m_value; +}; + +/** Expression of a symbol uniquely identified by the template parameter type \c tag */ +template<typename tag> +class SymbolExpr : public BaseExpr<SymbolExpr<tag> > +{ +public: + /** Alias to the template parameter \c tag */ + typedef tag Tag; + + SymbolExpr() {} + + /** Associate the value \a val to the given symbol \c *this, uniquely identified by its \c Tag. + * + * The returned object should be passed to ExprBase::eval() to evaluate a given expression with this specified runtime-time value. + */ + SymbolValue<Tag> operator=(Index val) const { + return SymbolValue<Tag>(val); + } + + Index eval_impl(const SymbolValue<Tag> &values) const { return values.value(); } + +#if EIGEN_HAS_CXX14 + // C++14 versions suitable for multiple symbols + template<typename... Types> + Index eval_impl(const std::tuple<Types...>& values) const { return std::get<SymbolValue<Tag> >(values).value(); } +#endif +}; + +template<typename Arg0> +class NegateExpr : public BaseExpr<NegateExpr<Arg0> > +{ +public: + NegateExpr(const Arg0& arg0) : m_arg0(arg0) {} + + template<typename T> + Index eval_impl(const T& values) const { return -m_arg0.eval_impl(values); } +protected: + Arg0 m_arg0; +}; + +template<typename Arg0, typename Arg1> +class AddExpr : public BaseExpr<AddExpr<Arg0,Arg1> > +{ +public: + AddExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} + + template<typename T> + Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) + m_arg1.eval_impl(values); } +protected: + Arg0 m_arg0; + Arg1 m_arg1; +}; + +template<typename Arg0, typename Arg1> +class ProductExpr : public BaseExpr<ProductExpr<Arg0,Arg1> > +{ +public: + ProductExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} + + template<typename T> + Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) * m_arg1.eval_impl(values); } +protected: + Arg0 m_arg0; + Arg1 m_arg1; +}; + +template<typename Arg0, typename Arg1> +class QuotientExpr : public BaseExpr<QuotientExpr<Arg0,Arg1> > +{ +public: + QuotientExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} + + template<typename T> + Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) / m_arg1.eval_impl(values); } +protected: + Arg0 m_arg0; + Arg1 m_arg1; +}; + +} // end namespace symbolic + +} // end namespace Eigen + +#endif // EIGEN_SYMBOLIC_INDEX_H
diff --git a/Eigen/src/Core/util/XprHelper.h b/Eigen/src/Core/util/XprHelper.h index a001c47..91c2e42 100644 --- a/Eigen/src/Core/util/XprHelper.h +++ b/Eigen/src/Core/util/XprHelper.h
@@ -24,16 +24,6 @@ namespace Eigen { -typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE DenseIndex; - -/** - * \brief The Index type as used for the API. - * \details To change this, \c \#define the preprocessor symbol \c EIGEN_DEFAULT_DENSE_INDEX_TYPE. - * \sa \blank \ref TopicPreprocessorDirectives, StorageIndex. - */ - -typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE Index; - namespace internal { template<typename IndexDest, typename IndexSrc> @@ -44,6 +34,76 @@ return IndexDest(idx); } +// true if T can be considered as an integral index (i.e., and integral type or enum) +template<typename T> struct is_valid_index_type +{ + enum { value = +#if EIGEN_HAS_TYPE_TRAITS + internal::is_integral<T>::value || std::is_enum<T>::value +#elif EIGEN_COMP_MSVC + internal::is_integral<T>::value || __is_enum(T) +#else + // without C++11, we use is_convertible to Index instead of is_integral in order to treat enums as Index. + internal::is_convertible<T,Index>::value +#endif + }; +}; + +// true if both types are not valid index types +template<typename RowIndices, typename ColIndices> +struct valid_indexed_view_overload { + enum { value = !(internal::is_valid_index_type<RowIndices>::value && internal::is_valid_index_type<ColIndices>::value) }; +}; + +// promote_scalar_arg is an helper used in operation between an expression and a scalar, like: +// expression * scalar +// Its role is to determine how the type T of the scalar operand should be promoted given the scalar type ExprScalar of the given expression. +// The IsSupported template parameter must be provided by the caller as: internal::has_ReturnType<ScalarBinaryOpTraits<ExprScalar,T,op> >::value using the proper order for ExprScalar and T. +// Then the logic is as follows: +// - if the operation is natively supported as defined by IsSupported, then the scalar type is not promoted, and T is returned. +// - otherwise, NumTraits<ExprScalar>::Literal is returned if T is implicitly convertible to NumTraits<ExprScalar>::Literal AND that this does not imply a float to integer conversion. +// - otherwise, ExprScalar is returned if T is implicitly convertible to ExprScalar AND that this does not imply a float to integer conversion. +// - In all other cases, the promoted type is not defined, and the respective operation is thus invalid and not available (SFINAE). +template<typename ExprScalar,typename T, bool IsSupported> +struct promote_scalar_arg; + +template<typename S,typename T> +struct promote_scalar_arg<S,T,true> +{ + typedef T type; +}; + +// Recursively check safe conversion to PromotedType, and then ExprScalar if they are different. +template<typename ExprScalar,typename T,typename PromotedType, + bool ConvertibleToLiteral = internal::is_convertible<T,PromotedType>::value, + bool IsSafe = NumTraits<T>::IsInteger || !NumTraits<PromotedType>::IsInteger> +struct promote_scalar_arg_unsupported; + +// Start recursion with NumTraits<ExprScalar>::Literal +template<typename S,typename T> +struct promote_scalar_arg<S,T,false> : promote_scalar_arg_unsupported<S,T,typename NumTraits<S>::Literal> {}; + +// We found a match! +template<typename S,typename T, typename PromotedType> +struct promote_scalar_arg_unsupported<S,T,PromotedType,true,true> +{ + typedef PromotedType type; +}; + +// No match, but no real-to-integer issues, and ExprScalar and current PromotedType are different, +// so let's try to promote to ExprScalar +template<typename ExprScalar,typename T, typename PromotedType> +struct promote_scalar_arg_unsupported<ExprScalar,T,PromotedType,false,true> + : promote_scalar_arg_unsupported<ExprScalar,T,ExprScalar> +{}; + +// Unsafe real-to-integer, let's stop. +template<typename S,typename T, typename PromotedType, bool ConvertibleToLiteral> +struct promote_scalar_arg_unsupported<S,T,PromotedType,ConvertibleToLiteral,false> {}; + +// T is not even convertible to ExprScalar, let's stop. +template<typename S,typename T> +struct promote_scalar_arg_unsupported<S,T,S,false,true> {}; //classes inheriting no_assignment_operator don't generate a default operator=. class no_assignment_operator @@ -67,9 +127,10 @@ { public: EIGEN_EMPTY_STRUCT_CTOR(variable_if_dynamic) - EIGEN_DEVICE_FUNC explicit variable_if_dynamic(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } - EIGEN_DEVICE_FUNC static T value() { return T(Value); } - EIGEN_DEVICE_FUNC void setValue(T) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamic(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T value() { return T(Value); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator T() const { return T(Value); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T) {} }; template<typename T> class variable_if_dynamic<T, Dynamic> @@ -77,9 +138,10 @@ T m_value; EIGEN_DEVICE_FUNC variable_if_dynamic() { eigen_assert(false); } public: - EIGEN_DEVICE_FUNC explicit variable_if_dynamic(T value) : m_value(value) {} - EIGEN_DEVICE_FUNC T value() const { return m_value; } - EIGEN_DEVICE_FUNC void setValue(T value) { m_value = value; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamic(T value) : m_value(value) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T value() const { return m_value; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator T() const { return m_value; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T value) { m_value = value; } }; /** \internal like variable_if_dynamic but for DynamicIndex @@ -88,9 +150,9 @@ { public: EIGEN_EMPTY_STRUCT_CTOR(variable_if_dynamicindex) - EIGEN_DEVICE_FUNC explicit variable_if_dynamicindex(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } - EIGEN_DEVICE_FUNC static T value() { return T(Value); } - EIGEN_DEVICE_FUNC void setValue(T) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamicindex(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T value() { return T(Value); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T) {} }; template<typename T> class variable_if_dynamicindex<T, DynamicIndex> @@ -98,9 +160,9 @@ T m_value; EIGEN_DEVICE_FUNC variable_if_dynamicindex() { eigen_assert(false); } public: - EIGEN_DEVICE_FUNC explicit variable_if_dynamicindex(T value) : m_value(value) {} - EIGEN_DEVICE_FUNC T value() const { return m_value; } - EIGEN_DEVICE_FUNC void setValue(T value) { m_value = value; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamicindex(T value) : m_value(value) {} + EIGEN_DEVICE_FUNC T EIGEN_STRONG_INLINE value() const { return m_value; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T value) { m_value = value; } }; template<typename T> struct functor_traits @@ -122,7 +184,8 @@ enum { size = 1, - alignment = 1 + alignment = 1, + vectorizable = false }; }; @@ -151,7 +214,7 @@ #if EIGEN_MAX_STATIC_ALIGN_BYTES>0 template<int ArrayBytes, int AlignmentBytes, bool Match = bool((ArrayBytes%AlignmentBytes)==0), - bool TryHalf = bool(AlignmentBytes>EIGEN_MIN_ALIGN_BYTES) > + bool TryHalf = bool(EIGEN_MIN_ALIGN_BYTES<AlignmentBytes) > struct compute_default_alignment_helper { enum { value = 0 }; @@ -343,7 +406,7 @@ typedef Matrix<typename traits<T>::Scalar, Rows, Cols, - (MaxCols==1&&MaxRows!=1) ? RowMajor : ColMajor, + (MaxCols==1&&MaxRows!=1) ? ColMajor : RowMajor, MaxRows, MaxCols > type; @@ -398,22 +461,18 @@ { enum { ScalarReadCost = NumTraits<typename traits<T>::Scalar>::ReadCost, - CoeffReadCost = evaluator<T>::CoeffReadCost, // NOTE What if an evaluator evaluate itself into a tempory? + CoeffReadCost = evaluator<T>::CoeffReadCost, // NOTE What if an evaluator evaluate itself into a temporary? // Then CoeffReadCost will be small (e.g., 1) but we still have to evaluate, especially if n>1. // This situation is already taken care by the EvalBeforeNestingBit flag, which is turned ON // for all evaluator creating a temporary. This flag is then propagated by the parent evaluators. // Another solution could be to count the number of temps? NAsInteger = n == Dynamic ? HugeCost : n, CostEval = (NAsInteger+1) * ScalarReadCost + CoeffReadCost, - CostNoEval = NAsInteger * CoeffReadCost + CostNoEval = NAsInteger * CoeffReadCost, + Evaluate = (int(evaluator<T>::Flags) & EvalBeforeNestingBit) || (int(CostEval) < int(CostNoEval)) }; - typedef typename conditional< - ( (int(evaluator<T>::Flags) & EvalBeforeNestingBit) || - (int(CostEval) < int(CostNoEval)) ), - PlainObject, - typename ref_selector<T>::type - >::type type; + typedef typename conditional<Evaluate, PlainObject, typename ref_selector<T>::type>::type type; }; template<typename T> @@ -450,52 +509,6 @@ typedef typename dense_xpr_base<Derived,XprKind>::type type; }; -/** \internal Helper base class to add a scalar multiple operator - * overloads for complex types */ -template<typename Derived, typename Scalar, typename OtherScalar, typename BaseType, - bool EnableIt = !is_same<Scalar,OtherScalar>::value > -struct special_scalar_op_base : public BaseType -{ - // dummy operator* so that the - // "using special_scalar_op_base::operator*" compiles - struct dummy {}; - void operator*(dummy) const; - void operator/(dummy) const; -}; - -template<typename Derived,typename Scalar,typename OtherScalar, typename BaseType> -struct special_scalar_op_base<Derived,Scalar,OtherScalar,BaseType,true> : public BaseType -{ - const CwiseUnaryOp<scalar_multiple2_op<Scalar,OtherScalar>, const Derived> - operator*(const OtherScalar& scalar) const - { -#ifdef EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN - EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN -#endif - return CwiseUnaryOp<scalar_multiple2_op<Scalar,OtherScalar>, const Derived> - (*static_cast<const Derived*>(this), scalar_multiple2_op<Scalar,OtherScalar>(scalar)); - } - - inline friend const CwiseUnaryOp<scalar_multiple2_op<Scalar,OtherScalar>, const Derived> - operator*(const OtherScalar& scalar, const Derived& matrix) - { -#ifdef EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN - EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN -#endif - return static_cast<const special_scalar_op_base&>(matrix).operator*(scalar); - } - - const CwiseUnaryOp<scalar_quotient2_op<Scalar,OtherScalar>, const Derived> - operator/(const OtherScalar& scalar) const - { -#ifdef EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN - EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN -#endif - return CwiseUnaryOp<scalar_quotient2_op<Scalar,OtherScalar>, const Derived> - (*static_cast<const Derived*>(this), scalar_quotient2_op<Scalar,OtherScalar>(scalar)); - } -}; - template<typename XprType, typename CastType> struct cast_return_type { typedef typename XprType::Scalar CurrentScalarType; @@ -542,6 +555,15 @@ template <typename Functor> struct cwise_promote_storage_type<Sparse,Dense,Functor> { typedef Sparse ret; }; template <typename Functor> struct cwise_promote_storage_type<Dense,Sparse,Functor> { typedef Sparse ret; }; +template <typename LhsKind, typename RhsKind, int LhsOrder, int RhsOrder> struct cwise_promote_storage_order { + enum { value = LhsOrder }; +}; + +template <typename LhsKind, int LhsOrder, int RhsOrder> struct cwise_promote_storage_order<LhsKind,Sparse,LhsOrder,RhsOrder> { enum { value = RhsOrder }; }; +template <typename RhsKind, int LhsOrder, int RhsOrder> struct cwise_promote_storage_order<Sparse,RhsKind,LhsOrder,RhsOrder> { enum { value = LhsOrder }; }; +template <int Order> struct cwise_promote_storage_order<Sparse,Sparse,Order,Order> { enum { value = Order }; }; + + /** \internal Specify the "storage kind" of multiplying an expression of kind A with kind B. * The template parameter ProductTag permits to specialize the resulting storage kind wrt to * some compile-time properties of the product: GemmProduct, GemvProduct, OuterProduct, InnerProduct. @@ -622,10 +644,24 @@ >::type type; }; +template<typename Expr,typename Scalar = typename Expr::Scalar> +struct plain_constant_type +{ + enum { Options = (traits<Expr>::Flags&RowMajorBit)?RowMajor:0 }; + + typedef Array<Scalar, traits<Expr>::RowsAtCompileTime, traits<Expr>::ColsAtCompileTime, + Options, traits<Expr>::MaxRowsAtCompileTime,traits<Expr>::MaxColsAtCompileTime> array_type; + + typedef Matrix<Scalar, traits<Expr>::RowsAtCompileTime, traits<Expr>::ColsAtCompileTime, + Options, traits<Expr>::MaxRowsAtCompileTime,traits<Expr>::MaxColsAtCompileTime> matrix_type; + + typedef CwiseNullaryOp<scalar_constant_op<Scalar>, const typename conditional<is_same< typename traits<Expr>::XprKind, MatrixXpr >::value, matrix_type, array_type>::type > type; +}; + template<typename ExpressionType> struct is_lvalue { - enum { value = !bool(is_const<ExpressionType>::value) && + enum { value = (!bool(is_const<ExpressionType>::value)) && bool(traits<ExpressionType>::Flags & LvalueBit) }; }; @@ -641,25 +677,57 @@ template<typename T, int S> struct is_diagonal<DiagonalMatrix<T,S> > { enum { ret = true }; }; + +template<typename T> struct is_identity +{ enum { value = false }; }; + +template<typename T> struct is_identity<CwiseNullaryOp<internal::scalar_identity_op<typename T::Scalar>, T> > +{ enum { value = true }; }; + + template<typename S1, typename S2> struct glue_shapes; template<> struct glue_shapes<DenseShape,TriangularShape> { typedef TriangularShape type; }; template<typename T1, typename T2> -bool is_same_dense(const T1 &mat1, const T2 &mat2, typename enable_if<has_direct_access<T1>::ret&&has_direct_access<T2>::ret, T1>::type * = 0) +struct possibly_same_dense { + enum { value = has_direct_access<T1>::ret && has_direct_access<T2>::ret && is_same<typename T1::Scalar,typename T2::Scalar>::value }; +}; + +template<typename T1, typename T2> +EIGEN_DEVICE_FUNC +bool is_same_dense(const T1 &mat1, const T2 &mat2, typename enable_if<possibly_same_dense<T1,T2>::value>::type * = 0) { return (mat1.data()==mat2.data()) && (mat1.innerStride()==mat2.innerStride()) && (mat1.outerStride()==mat2.outerStride()); } template<typename T1, typename T2> -bool is_same_dense(const T1 &, const T2 &, typename enable_if<!(has_direct_access<T1>::ret&&has_direct_access<T2>::ret), T1>::type * = 0) +EIGEN_DEVICE_FUNC +bool is_same_dense(const T1 &, const T2 &, typename enable_if<!possibly_same_dense<T1,T2>::value>::type * = 0) { return false; } -template<typename T, typename U> struct is_same_or_void { enum { value = is_same<T,U>::value }; }; -template<typename T> struct is_same_or_void<void,T> { enum { value = 1 }; }; -template<typename T> struct is_same_or_void<T,void> { enum { value = 1 }; }; -template<> struct is_same_or_void<void,void> { enum { value = 1 }; }; +// Internal helper defining the cost of a scalar division for the type T. +// The default heuristic can be specialized for each scalar type and architecture. +template<typename T,bool Vectorized=false,typename EnableIf = void> +struct scalar_div_cost { + enum { value = 8*NumTraits<T>::MulCost }; +}; + +template<typename T,bool Vectorized> +struct scalar_div_cost<std::complex<T>, Vectorized> { + enum { value = 2*scalar_div_cost<T>::value + + 6*NumTraits<T>::MulCost + + 3*NumTraits<T>::AddCost + }; +}; + + +template<bool Vectorized> +struct scalar_div_cost<signed long,Vectorized,typename conditional<sizeof(long)==8,void,false_type>::type> { enum { value = 24 }; }; +template<bool Vectorized> +struct scalar_div_cost<unsigned long,Vectorized,typename conditional<sizeof(long)==8,void,false_type>::type> { enum { value = 21 }; }; + #ifdef EIGEN_DEBUG_ASSIGN std::string demangle_traversal(int t) @@ -695,17 +763,95 @@ } // end namespace internal -// we require Lhs and Rhs to have the same scalar type. Currently there is no example of a binary functor -// that would take two operands of different types. If there were such an example, then this check should be -// moved to the BinaryOp functors, on a per-case basis. This would however require a change in the BinaryOp functors, as -// currently they take only one typename Scalar template parameter. + +/** \class ScalarBinaryOpTraits + * \ingroup Core_Module + * + * \brief Determines whether the given binary operation of two numeric types is allowed and what the scalar return type is. + * + * This class permits to control the scalar return type of any binary operation performed on two different scalar types through (partial) template specializations. + * + * For instance, let \c U1, \c U2 and \c U3 be three user defined scalar types for which most operations between instances of \c U1 and \c U2 returns an \c U3. + * You can let %Eigen knows that by defining: + \code + template<typename BinaryOp> + struct ScalarBinaryOpTraits<U1,U2,BinaryOp> { typedef U3 ReturnType; }; + template<typename BinaryOp> + struct ScalarBinaryOpTraits<U2,U1,BinaryOp> { typedef U3 ReturnType; }; + \endcode + * You can then explicitly disable some particular operations to get more explicit error messages: + \code + template<> + struct ScalarBinaryOpTraits<U1,U2,internal::scalar_max_op<U1,U2> > {}; + \endcode + * Or customize the return type for individual operation: + \code + template<> + struct ScalarBinaryOpTraits<U1,U2,internal::scalar_sum_op<U1,U2> > { typedef U1 ReturnType; }; + \endcode + * + * By default, the following generic combinations are supported: + <table class="manual"> + <tr><th>ScalarA</th><th>ScalarB</th><th>BinaryOp</th><th>ReturnType</th><th>Note</th></tr> + <tr ><td>\c T </td><td>\c T </td><td>\c * </td><td>\c T </td><td></td></tr> + <tr class="alt"><td>\c NumTraits<T>::Real </td><td>\c T </td><td>\c * </td><td>\c T </td><td>Only if \c NumTraits<T>::IsComplex </td></tr> + <tr ><td>\c T </td><td>\c NumTraits<T>::Real </td><td>\c * </td><td>\c T </td><td>Only if \c NumTraits<T>::IsComplex </td></tr> + </table> + * + * \sa CwiseBinaryOp + */ +template<typename ScalarA, typename ScalarB, typename BinaryOp=internal::scalar_product_op<ScalarA,ScalarB> > +struct ScalarBinaryOpTraits +#ifndef EIGEN_PARSED_BY_DOXYGEN + // for backward compatibility, use the hints given by the (deprecated) internal::scalar_product_traits class. + : internal::scalar_product_traits<ScalarA,ScalarB> +#endif // EIGEN_PARSED_BY_DOXYGEN +{}; + +template<typename T, typename BinaryOp> +struct ScalarBinaryOpTraits<T,T,BinaryOp> +{ + typedef T ReturnType; +}; + +template <typename T, typename BinaryOp> +struct ScalarBinaryOpTraits<T, typename NumTraits<typename internal::enable_if<NumTraits<T>::IsComplex,T>::type>::Real, BinaryOp> +{ + typedef T ReturnType; +}; +template <typename T, typename BinaryOp> +struct ScalarBinaryOpTraits<typename NumTraits<typename internal::enable_if<NumTraits<T>::IsComplex,T>::type>::Real, T, BinaryOp> +{ + typedef T ReturnType; +}; + +// For Matrix * Permutation +template<typename T, typename BinaryOp> +struct ScalarBinaryOpTraits<T,void,BinaryOp> +{ + typedef T ReturnType; +}; + +// For Permutation * Matrix +template<typename T, typename BinaryOp> +struct ScalarBinaryOpTraits<void,T,BinaryOp> +{ + typedef T ReturnType; +}; + +// for Permutation*Permutation +template<typename BinaryOp> +struct ScalarBinaryOpTraits<void,void,BinaryOp> +{ + typedef void ReturnType; +}; + +// We require Lhs and Rhs to have "compatible" scalar types. // It is tempting to always allow mixing different types but remember that this is often impossible in the vectorized paths. // So allowing mixing different types gives very unexpected errors when enabling vectorization, when the user tries to // add together a float matrix and a double matrix. #define EIGEN_CHECK_BINARY_COMPATIBILIY(BINOP,LHS,RHS) \ - EIGEN_STATIC_ASSERT((internal::functor_is_product_like<BINOP>::ret \ - ? int(internal::scalar_product_traits<LHS, RHS>::Defined) \ - : int(internal::is_same_or_void<LHS, RHS>::value)), \ + EIGEN_STATIC_ASSERT((Eigen::internal::has_ReturnType<ScalarBinaryOpTraits<LHS, RHS,BINOP> >::value), \ YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) } // end namespace Eigen
diff --git a/Eigen/src/Eigenvalues/CMakeLists.txt b/Eigen/src/Eigenvalues/CMakeLists.txt deleted file mode 100644 index 193e026..0000000 --- a/Eigen/src/Eigenvalues/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_EIGENVALUES_SRCS "*.h") - -INSTALL(FILES - ${Eigen_EIGENVALUES_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Eigenvalues COMPONENT Devel - )
diff --git a/Eigen/src/Eigenvalues/ComplexEigenSolver.h b/Eigen/src/Eigenvalues/ComplexEigenSolver.h index ec3b163..081e918 100644 --- a/Eigen/src/Eigenvalues/ComplexEigenSolver.h +++ b/Eigen/src/Eigenvalues/ComplexEigenSolver.h
@@ -214,7 +214,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ ComputationInfo info() const { @@ -250,7 +250,7 @@ EigenvectorType m_matX; private: - void doComputeEigenvectors(const RealScalar& matrixnorm); + void doComputeEigenvectors(RealScalar matrixnorm); void sortEigenvalues(bool computeEigenvectors); }; @@ -284,10 +284,12 @@ template<typename MatrixType> -void ComplexEigenSolver<MatrixType>::doComputeEigenvectors(const RealScalar& matrixnorm) +void ComplexEigenSolver<MatrixType>::doComputeEigenvectors(RealScalar matrixnorm) { const Index n = m_eivalues.size(); + matrixnorm = numext::maxi(matrixnorm,(std::numeric_limits<RealScalar>::min)()); + // Compute X such that T = X D X^(-1), where D is the diagonal of T. // The matrix X is unit triangular. m_matX = EigenvectorType::Zero(n, n);
diff --git a/Eigen/src/Eigenvalues/ComplexSchur.h b/Eigen/src/Eigenvalues/ComplexSchur.h index 7f38919..fc71468 100644 --- a/Eigen/src/Eigenvalues/ComplexSchur.h +++ b/Eigen/src/Eigenvalues/ComplexSchur.h
@@ -212,7 +212,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ ComputationInfo info() const { @@ -300,10 +300,13 @@ ComplexScalar trace = t.coeff(0,0) + t.coeff(1,1); ComplexScalar eival1 = (trace + disc) / RealScalar(2); ComplexScalar eival2 = (trace - disc) / RealScalar(2); - - if(numext::norm1(eival1) > numext::norm1(eival2)) + RealScalar eival1_norm = numext::norm1(eival1); + RealScalar eival2_norm = numext::norm1(eival2); + // A division by zero can only occur if eival1==eival2==0. + // In this case, det==0, and all we have to do is checking that eival2_norm!=0 + if(eival1_norm > eival2_norm) eival2 = det / eival1; - else + else if(eival2_norm!=RealScalar(0)) eival1 = det / eival2; // choose the eigenvalue closest to the bottom entry of the diagonal
diff --git a/Eigen/src/Eigenvalues/ComplexSchur_MKL.h b/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h similarity index 69% rename from Eigen/src/Eigenvalues/ComplexSchur_MKL.h rename to Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h index e20c372..4980a3e 100644 --- a/Eigen/src/Eigenvalues/ComplexSchur_MKL.h +++ b/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h
@@ -25,21 +25,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Complex Schur needed to complex unsymmetrical eigenvalues/eigenvectors. ******************************************************************************** */ -#ifndef EIGEN_COMPLEX_SCHUR_MKL_H -#define EIGEN_COMPLEX_SCHUR_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" +#ifndef EIGEN_COMPLEX_SCHUR_LAPACKE_H +#define EIGEN_COMPLEX_SCHUR_LAPACKE_H namespace Eigen { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_SCHUR_COMPLEX(EIGTYPE, MKLTYPE, MKLPREFIX, MKLPREFIX_U, EIGCOLROW, MKLCOLROW) \ +#define EIGEN_LAPACKE_SCHUR_COMPLEX(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX, LAPACKE_PREFIX_U, EIGCOLROW, LAPACKE_COLROW) \ template<> template<typename InputType> inline \ ComplexSchur<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >& \ ComplexSchur<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >::compute(const EigenBase<InputType>& matrix, bool computeU) \ @@ -60,18 +58,18 @@ m_matUisUptodate = computeU; \ return *this; \ } \ - lapack_int n = matrix.cols(), sdim, info; \ - lapack_int matrix_order = MKLCOLROW; \ + lapack_int n = internal::convert_index<lapack_int>(matrix.cols()), sdim, info; \ + lapack_int matrix_order = LAPACKE_COLROW; \ char jobvs, sort='N'; \ - LAPACK_##MKLPREFIX_U##_SELECT1 select = 0; \ + LAPACK_##LAPACKE_PREFIX_U##_SELECT1 select = 0; \ jobvs = (computeU) ? 'V' : 'N'; \ m_matU.resize(n, n); \ - lapack_int ldvs = m_matU.outerStride(); \ + lapack_int ldvs = internal::convert_index<lapack_int>(m_matU.outerStride()); \ m_matT = matrix; \ - lapack_int lda = m_matT.outerStride(); \ + lapack_int lda = internal::convert_index<lapack_int>(m_matT.outerStride()); \ Matrix<EIGTYPE, Dynamic, Dynamic> w; \ w.resize(n, 1);\ - info = LAPACKE_##MKLPREFIX##gees( matrix_order, jobvs, sort, select, n, (MKLTYPE*)m_matT.data(), lda, &sdim, (MKLTYPE*)w.data(), (MKLTYPE*)m_matU.data(), ldvs ); \ + info = LAPACKE_##LAPACKE_PREFIX##gees( matrix_order, jobvs, sort, select, n, (LAPACKE_TYPE*)m_matT.data(), lda, &sdim, (LAPACKE_TYPE*)w.data(), (LAPACKE_TYPE*)m_matU.data(), ldvs ); \ if(info == 0) \ m_info = Success; \ else \ @@ -83,11 +81,11 @@ \ } -EIGEN_MKL_SCHUR_COMPLEX(dcomplex, MKL_Complex16, z, Z, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SCHUR_COMPLEX(scomplex, MKL_Complex8, c, C, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SCHUR_COMPLEX(dcomplex, MKL_Complex16, z, Z, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_SCHUR_COMPLEX(scomplex, MKL_Complex8, c, C, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SCHUR_COMPLEX(dcomplex, lapack_complex_double, z, Z, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SCHUR_COMPLEX(scomplex, lapack_complex_float, c, C, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SCHUR_COMPLEX(dcomplex, lapack_complex_double, z, Z, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SCHUR_COMPLEX(scomplex, lapack_complex_float, c, C, RowMajor, LAPACK_ROW_MAJOR) } // end namespace Eigen -#endif // EIGEN_COMPLEX_SCHUR_MKL_H +#endif // EIGEN_COMPLEX_SCHUR_LAPACKE_H
diff --git a/Eigen/src/Eigenvalues/EigenSolver.h b/Eigen/src/Eigenvalues/EigenSolver.h index 532ca7d..572b29e 100644 --- a/Eigen/src/Eigenvalues/EigenSolver.h +++ b/Eigen/src/Eigenvalues/EigenSolver.h
@@ -110,7 +110,7 @@ * * \sa compute() for an example. */ - EigenSolver() : m_eivec(), m_eivalues(), m_isInitialized(false), m_realSchur(), m_matT(), m_tmp() {} + EigenSolver() : m_eivec(), m_eivalues(), m_isInitialized(false), m_eigenvectorsOk(false), m_realSchur(), m_matT(), m_tmp() {} /** \brief Default constructor with memory preallocation * @@ -277,7 +277,7 @@ template<typename InputType> EigenSolver& compute(const EigenBase<InputType>& matrix, bool computeEigenvectors = true); - /** \returns NumericalIssue if the input contains INF or NaN values or overflow occured. Returns Success otherwise. */ + /** \returns NumericalIssue if the input contains INF or NaN values or overflow occurred. Returns Success otherwise. */ ComputationInfo info() const { eigen_assert(m_isInitialized && "EigenSolver is not initialized."); @@ -324,11 +324,12 @@ MatrixType EigenSolver<MatrixType>::pseudoEigenvalueMatrix() const { eigen_assert(m_isInitialized && "EigenSolver is not initialized."); + const RealScalar precision = RealScalar(2)*NumTraits<RealScalar>::epsilon(); Index n = m_eivalues.rows(); MatrixType matD = MatrixType::Zero(n,n); for (Index i=0; i<n; ++i) { - if (internal::isMuchSmallerThan(numext::imag(m_eivalues.coeff(i)), numext::real(m_eivalues.coeff(i)))) + if (internal::isMuchSmallerThan(numext::imag(m_eivalues.coeff(i)), numext::real(m_eivalues.coeff(i)), precision)) matD.coeffRef(i,i) = numext::real(m_eivalues.coeff(i)); else { @@ -345,11 +346,12 @@ { eigen_assert(m_isInitialized && "EigenSolver is not initialized."); eigen_assert(m_eigenvectorsOk && "The eigenvectors have not been computed together with the eigenvalues."); + const RealScalar precision = RealScalar(2)*NumTraits<RealScalar>::epsilon(); Index n = m_eivec.cols(); EigenvectorsType matV(n,n); for (Index j=0; j<n; ++j) { - if (internal::isMuchSmallerThan(numext::imag(m_eivalues.coeff(j)), numext::real(m_eivalues.coeff(j))) || j+1==n) + if (internal::isMuchSmallerThan(numext::imag(m_eivalues.coeff(j)), numext::real(m_eivalues.coeff(j)), precision) || j+1==n) { // we have a real eigen value matV.col(j) = m_eivec.col(j).template cast<ComplexScalar>(); @@ -451,26 +453,6 @@ return *this; } -// Complex scalar division. -template<typename Scalar> -std::complex<Scalar> cdiv(const Scalar& xr, const Scalar& xi, const Scalar& yr, const Scalar& yi) -{ - using std::abs; - Scalar r,d; - if (abs(yr) > abs(yi)) - { - r = yi/yr; - d = yr + r*yi; - return std::complex<Scalar>((xr + r*xi)/d, (xi - r*xr)/d); - } - else - { - r = yr/yi; - d = yi + r*yr; - return std::complex<Scalar>((r*xr + xi)/d, (r*xi - xr)/d); - } -} - template<typename MatrixType> void EigenSolver<MatrixType>::doComputeEigenvectors() @@ -503,7 +485,7 @@ Scalar lastr(0), lastw(0); Index l = n; - m_matT.coeffRef(n,n) = 1.0; + m_matT.coeffRef(n,n) = Scalar(1); for (Index i = n-1; i >= 0; i--) { Scalar w = m_matT.coeff(i,i) - p; @@ -557,7 +539,7 @@ } else { - std::complex<Scalar> cc = cdiv<Scalar>(Scalar(0),-m_matT.coeff(n-1,n),m_matT.coeff(n-1,n-1)-p,q); + ComplexScalar cc = ComplexScalar(Scalar(0),-m_matT.coeff(n-1,n)) / ComplexScalar(m_matT.coeff(n-1,n-1)-p,q); m_matT.coeffRef(n-1,n-1) = numext::real(cc); m_matT.coeffRef(n-1,n) = numext::imag(cc); } @@ -580,7 +562,7 @@ l = i; if (m_eivalues.coeff(i).imag() == RealScalar(0)) { - std::complex<Scalar> cc = cdiv(-ra,-sa,w,q); + ComplexScalar cc = ComplexScalar(-ra,-sa) / ComplexScalar(w,q); m_matT.coeffRef(i,n-1) = numext::real(cc); m_matT.coeffRef(i,n) = numext::imag(cc); } @@ -594,7 +576,7 @@ if ((vr == Scalar(0)) && (vi == Scalar(0))) vr = eps * norm * (abs(w) + abs(q) + abs(x) + abs(y) + abs(lastw)); - std::complex<Scalar> cc = cdiv(x*lastra-lastw*ra+q*sa,x*lastsa-lastw*sa-q*ra,vr,vi); + ComplexScalar cc = ComplexScalar(x*lastra-lastw*ra+q*sa,x*lastsa-lastw*sa-q*ra) / ComplexScalar(vr,vi); m_matT.coeffRef(i,n-1) = numext::real(cc); m_matT.coeffRef(i,n) = numext::imag(cc); if (abs(x) > (abs(lastw) + abs(q))) @@ -604,7 +586,7 @@ } else { - cc = cdiv(-lastra-y*m_matT.coeff(i,n-1),-lastsa-y*m_matT.coeff(i,n),lastw,q); + cc = ComplexScalar(-lastra-y*m_matT.coeff(i,n-1),-lastsa-y*m_matT.coeff(i,n)) / ComplexScalar(lastw,q); m_matT.coeffRef(i+1,n-1) = numext::real(cc); m_matT.coeffRef(i+1,n) = numext::imag(cc); }
diff --git a/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h b/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h index a9d6790..87d789b 100644 --- a/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h +++ b/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h
@@ -1,8 +1,9 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2012 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2012-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2010,2012 Jitse Niesen <jitse@maths.leeds.ac.uk> +// Copyright (C) 2016 Tobias Wood <tobias@spinicist.org.uk> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -89,7 +90,7 @@ */ typedef Matrix<Scalar, ColsAtCompileTime, 1, Options & ~RowMajor, MaxColsAtCompileTime, 1> VectorType; - /** \brief Type for vector of complex scalar values eigenvalues as returned by betas(). + /** \brief Type for vector of complex scalar values eigenvalues as returned by alphas(). * * This is a column vector with entries of type #ComplexScalar. * The length of the vector is the size of #MatrixType. @@ -114,7 +115,14 @@ * * \sa compute() for an example. */ - GeneralizedEigenSolver() : m_eivec(), m_alphas(), m_betas(), m_isInitialized(false), m_realQZ(), m_matS(), m_tmp() {} + GeneralizedEigenSolver() + : m_eivec(), + m_alphas(), + m_betas(), + m_valuesOkay(false), + m_vectorsOkay(false), + m_realQZ() + {} /** \brief Default constructor with memory preallocation * @@ -126,10 +134,9 @@ : m_eivec(size, size), m_alphas(size), m_betas(size), - m_isInitialized(false), - m_eigenvectorsOk(false), + m_valuesOkay(false), + m_vectorsOkay(false), m_realQZ(size), - m_matS(size, size), m_tmp(size) {} @@ -149,10 +156,9 @@ : m_eivec(A.rows(), A.cols()), m_alphas(A.cols()), m_betas(A.cols()), - m_isInitialized(false), - m_eigenvectorsOk(false), + m_valuesOkay(false), + m_vectorsOkay(false), m_realQZ(A.cols()), - m_matS(A.rows(), A.cols()), m_tmp(A.cols()) { compute(A, B, computeEigenvectors); @@ -160,22 +166,20 @@ /* \brief Returns the computed generalized eigenvectors. * - * \returns %Matrix whose columns are the (possibly complex) eigenvectors. + * \returns %Matrix whose columns are the (possibly complex) right eigenvectors. + * i.e. the eigenvectors that solve (A - l*B)x = 0. The ordering matches the eigenvalues. * * \pre Either the constructor * GeneralizedEigenSolver(const MatrixType&,const MatrixType&, bool) or the member function * compute(const MatrixType&, const MatrixType& bool) has been called before, and * \p computeEigenvectors was set to true (the default). * - * Column \f$ k \f$ of the returned matrix is an eigenvector corresponding - * to eigenvalue number \f$ k \f$ as returned by eigenvalues(). The - * eigenvectors are normalized to have (Euclidean) norm equal to one. The - * matrix returned by this function is the matrix \f$ V \f$ in the - * generalized eigendecomposition \f$ A = B V D V^{-1} \f$, if it exists. - * * \sa eigenvalues() */ -// EigenvectorsType eigenvectors() const; + EigenvectorsType eigenvectors() const { + eigen_assert(m_vectorsOkay && "Eigenvectors for GeneralizedEigenSolver were not calculated."); + return m_eivec; + } /** \brief Returns an expression of the computed generalized eigenvalues. * @@ -197,7 +201,7 @@ */ EigenvalueType eigenvalues() const { - eigen_assert(m_isInitialized && "GeneralizedEigenSolver is not initialized."); + eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); return EigenvalueType(m_alphas,m_betas); } @@ -208,7 +212,7 @@ * \sa betas(), eigenvalues() */ ComplexVectorType alphas() const { - eigen_assert(m_isInitialized && "GeneralizedEigenSolver is not initialized."); + eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); return m_alphas; } @@ -219,7 +223,7 @@ * \sa alphas(), eigenvalues() */ VectorType betas() const { - eigen_assert(m_isInitialized && "GeneralizedEigenSolver is not initialized."); + eigen_assert(m_valuesOkay && "GeneralizedEigenSolver is not initialized."); return m_betas; } @@ -250,7 +254,7 @@ ComputationInfo info() const { - eigen_assert(m_isInitialized && "EigenSolver is not initialized."); + eigen_assert(m_valuesOkay && "EigenSolver is not initialized."); return m_realQZ.info(); } @@ -270,29 +274,14 @@ EIGEN_STATIC_ASSERT(!NumTraits<Scalar>::IsComplex, NUMERIC_TYPE_MUST_BE_REAL); } - MatrixType m_eivec; + EigenvectorsType m_eivec; ComplexVectorType m_alphas; VectorType m_betas; - bool m_isInitialized; - bool m_eigenvectorsOk; + bool m_valuesOkay, m_vectorsOkay; RealQZ<MatrixType> m_realQZ; - MatrixType m_matS; - - typedef Matrix<Scalar, ColsAtCompileTime, 1, Options & ~RowMajor, MaxColsAtCompileTime, 1> ColumnVectorType; - ColumnVectorType m_tmp; + ComplexVectorType m_tmp; }; -//template<typename MatrixType> -//typename GeneralizedEigenSolver<MatrixType>::EigenvectorsType GeneralizedEigenSolver<MatrixType>::eigenvectors() const -//{ -// eigen_assert(m_isInitialized && "EigenSolver is not initialized."); -// eigen_assert(m_eigenvectorsOk && "The eigenvectors have not been computed together with the eigenvalues."); -// Index n = m_eivec.cols(); -// EigenvectorsType matV(n,n); -// // TODO -// return matV; -//} - template<typename MatrixType> GeneralizedEigenSolver<MatrixType>& GeneralizedEigenSolver<MatrixType>::compute(const MatrixType& A, const MatrixType& B, bool computeEigenvectors) @@ -302,46 +291,125 @@ using std::sqrt; using std::abs; eigen_assert(A.cols() == A.rows() && B.cols() == A.rows() && B.cols() == B.rows()); - + Index size = A.cols(); + m_valuesOkay = false; + m_vectorsOkay = false; // Reduce to generalized real Schur form: // A = Q S Z and B = Q T Z m_realQZ.compute(A, B, computeEigenvectors); - if (m_realQZ.info() == Success) { - m_matS = m_realQZ.matrixS(); + // Resize storage + m_alphas.resize(size); + m_betas.resize(size); if (computeEigenvectors) - m_eivec = m_realQZ.matrixZ().transpose(); - - // Compute eigenvalues from matS - m_alphas.resize(A.cols()); - m_betas.resize(A.cols()); - Index i = 0; - while (i < A.cols()) { - if (i == A.cols() - 1 || m_matS.coeff(i+1, i) == Scalar(0)) + m_eivec.resize(size,size); + m_tmp.resize(size); + } + + // Aliases: + Map<VectorType> v(reinterpret_cast<Scalar*>(m_tmp.data()), size); + ComplexVectorType &cv = m_tmp; + const MatrixType &mS = m_realQZ.matrixS(); + const MatrixType &mT = m_realQZ.matrixT(); + + Index i = 0; + while (i < size) + { + if (i == size - 1 || mS.coeff(i+1, i) == Scalar(0)) { - m_alphas.coeffRef(i) = m_matS.coeff(i, i); - m_betas.coeffRef(i) = m_realQZ.matrixT().coeff(i,i); + // Real eigenvalue + m_alphas.coeffRef(i) = mS.diagonal().coeff(i); + m_betas.coeffRef(i) = mT.diagonal().coeff(i); + if (computeEigenvectors) + { + v.setConstant(Scalar(0.0)); + v.coeffRef(i) = Scalar(1.0); + // For singular eigenvalues do nothing more + if(abs(m_betas.coeffRef(i)) >= (std::numeric_limits<RealScalar>::min)()) + { + // Non-singular eigenvalue + const Scalar alpha = real(m_alphas.coeffRef(i)); + const Scalar beta = m_betas.coeffRef(i); + for (Index j = i-1; j >= 0; j--) + { + const Index st = j+1; + const Index sz = i-j; + if (j > 0 && mS.coeff(j, j-1) != Scalar(0)) + { + // 2x2 block + Matrix<Scalar, 2, 1> rhs = (alpha*mT.template block<2,Dynamic>(j-1,st,2,sz) - beta*mS.template block<2,Dynamic>(j-1,st,2,sz)) .lazyProduct( v.segment(st,sz) ); + Matrix<Scalar, 2, 2> lhs = beta * mS.template block<2,2>(j-1,j-1) - alpha * mT.template block<2,2>(j-1,j-1); + v.template segment<2>(j-1) = lhs.partialPivLu().solve(rhs); + j--; + } + else + { + v.coeffRef(j) = -v.segment(st,sz).transpose().cwiseProduct(beta*mS.block(j,st,1,sz) - alpha*mT.block(j,st,1,sz)).sum() / (beta*mS.coeffRef(j,j) - alpha*mT.coeffRef(j,j)); + } + } + } + m_eivec.col(i).real().noalias() = m_realQZ.matrixZ().transpose() * v; + m_eivec.col(i).real().normalize(); + m_eivec.col(i).imag().setConstant(0); + } ++i; } else { - Scalar p = Scalar(0.5) * (m_matS.coeff(i, i) - m_matS.coeff(i+1, i+1)); - Scalar z = sqrt(abs(p * p + m_matS.coeff(i+1, i) * m_matS.coeff(i, i+1))); - m_alphas.coeffRef(i) = ComplexScalar(m_matS.coeff(i+1, i+1) + p, z); - m_alphas.coeffRef(i+1) = ComplexScalar(m_matS.coeff(i+1, i+1) + p, -z); + // We need to extract the generalized eigenvalues of the pair of a general 2x2 block S and a positive diagonal 2x2 block T + // Then taking beta=T_00*T_11, we can avoid any division, and alpha is the eigenvalues of A = (U^-1 * S * U) * diag(T_11,T_00): - m_betas.coeffRef(i) = m_realQZ.matrixT().coeff(i,i); - m_betas.coeffRef(i+1) = m_realQZ.matrixT().coeff(i,i); + // T = [a 0] + // [0 b] + RealScalar a = mT.diagonal().coeff(i), + b = mT.diagonal().coeff(i+1); + const RealScalar beta = m_betas.coeffRef(i) = m_betas.coeffRef(i+1) = a*b; + + // ^^ NOTE: using diagonal()(i) instead of coeff(i,i) workarounds a MSVC bug. + Matrix<RealScalar,2,2> S2 = mS.template block<2,2>(i,i) * Matrix<Scalar,2,1>(b,a).asDiagonal(); + + Scalar p = Scalar(0.5) * (S2.coeff(0,0) - S2.coeff(1,1)); + Scalar z = sqrt(abs(p * p + S2.coeff(1,0) * S2.coeff(0,1))); + const ComplexScalar alpha = ComplexScalar(S2.coeff(1,1) + p, (beta > 0) ? z : -z); + m_alphas.coeffRef(i) = conj(alpha); + m_alphas.coeffRef(i+1) = alpha; + + if (computeEigenvectors) { + // Compute eigenvector in position (i+1) and then position (i) is just the conjugate + cv.setZero(); + cv.coeffRef(i+1) = Scalar(1.0); + // here, the "static_cast" workaound expression template issues. + cv.coeffRef(i) = -(static_cast<Scalar>(beta*mS.coeffRef(i,i+1)) - alpha*mT.coeffRef(i,i+1)) + / (static_cast<Scalar>(beta*mS.coeffRef(i,i)) - alpha*mT.coeffRef(i,i)); + for (Index j = i-1; j >= 0; j--) + { + const Index st = j+1; + const Index sz = i+1-j; + if (j > 0 && mS.coeff(j, j-1) != Scalar(0)) + { + // 2x2 block + Matrix<ComplexScalar, 2, 1> rhs = (alpha*mT.template block<2,Dynamic>(j-1,st,2,sz) - beta*mS.template block<2,Dynamic>(j-1,st,2,sz)) .lazyProduct( cv.segment(st,sz) ); + Matrix<ComplexScalar, 2, 2> lhs = beta * mS.template block<2,2>(j-1,j-1) - alpha * mT.template block<2,2>(j-1,j-1); + cv.template segment<2>(j-1) = lhs.partialPivLu().solve(rhs); + j--; + } else { + cv.coeffRef(j) = cv.segment(st,sz).transpose().cwiseProduct(beta*mS.block(j,st,1,sz) - alpha*mT.block(j,st,1,sz)).sum() + / (alpha*mT.coeffRef(j,j) - static_cast<Scalar>(beta*mS.coeffRef(j,j))); + } + } + m_eivec.col(i+1).noalias() = (m_realQZ.matrixZ().transpose() * cv); + m_eivec.col(i+1).normalize(); + m_eivec.col(i) = m_eivec.col(i+1).conjugate(); + } i += 2; } } + + m_valuesOkay = true; + m_vectorsOkay = computeEigenvectors; } - - m_isInitialized = true; - m_eigenvectorsOk = false;//computeEigenvectors; - return *this; }
diff --git a/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h b/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h index 5f6bb82..d0f9091 100644 --- a/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h +++ b/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h
@@ -121,7 +121,7 @@ * * \returns Reference to \c *this * - * Accoring to \p options, this function computes eigenvalues and (if requested) + * According to \p options, this function computes eigenvalues and (if requested) * the eigenvectors of one of the following three generalized eigenproblems: * - \c Ax_lBx: \f$ Ax = \lambda B x \f$ * - \c ABx_lx: \f$ ABx = \lambda x \f$
diff --git a/Eigen/src/Eigenvalues/HessenbergDecomposition.h b/Eigen/src/Eigenvalues/HessenbergDecomposition.h index f647f69..d947dac 100644 --- a/Eigen/src/Eigenvalues/HessenbergDecomposition.h +++ b/Eigen/src/Eigenvalues/HessenbergDecomposition.h
@@ -315,7 +315,7 @@ // A = A H' matA.rightCols(remainingSize) - .applyHouseholderOnTheRight(matA.col(i).tail(remainingSize-1).conjugate(), numext::conj(h), &temp.coeffRef(0)); + .applyHouseholderOnTheRight(matA.col(i).tail(remainingSize-1), numext::conj(h), &temp.coeffRef(0)); } }
diff --git a/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h b/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h index 4fec8af..66e5a3d 100644 --- a/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h +++ b/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h
@@ -66,7 +66,6 @@ inline typename MatrixBase<Derived>::EigenvaluesReturnType MatrixBase<Derived>::eigenvalues() const { - typedef typename internal::traits<Derived>::Scalar Scalar; return internal::eigenvalues_selector<Derived, NumTraits<Scalar>::IsComplex>::run(derived()); } @@ -85,10 +84,9 @@ * \sa SelfAdjointEigenSolver::eigenvalues(), MatrixBase::eigenvalues() */ template<typename MatrixType, unsigned int UpLo> -inline typename SelfAdjointView<MatrixType, UpLo>::EigenvaluesReturnType +EIGEN_DEVICE_FUNC inline typename SelfAdjointView<MatrixType, UpLo>::EigenvaluesReturnType SelfAdjointView<MatrixType, UpLo>::eigenvalues() const { - typedef typename SelfAdjointView<MatrixType, UpLo>::PlainObject PlainObject; PlainObject thisAsMatrix(*this); return SelfAdjointEigenSolver<PlainObject>(thisAsMatrix, false).eigenvalues(); } @@ -149,7 +147,7 @@ * \sa eigenvalues(), MatrixBase::operatorNorm() */ template<typename MatrixType, unsigned int UpLo> -inline typename SelfAdjointView<MatrixType, UpLo>::RealScalar +EIGEN_DEVICE_FUNC inline typename SelfAdjointView<MatrixType, UpLo>::RealScalar SelfAdjointView<MatrixType, UpLo>::operatorNorm() const { return eigenvalues().cwiseAbs().maxCoeff();
diff --git a/Eigen/src/Eigenvalues/RealQZ.h b/Eigen/src/Eigenvalues/RealQZ.h index a62071d..5091301 100644 --- a/Eigen/src/Eigenvalues/RealQZ.h +++ b/Eigen/src/Eigenvalues/RealQZ.h
@@ -90,8 +90,9 @@ m_Z(size, size), m_workspace(size*2), m_maxIters(400), - m_isInitialized(false) - { } + m_isInitialized(false), + m_computeQZ(true) + {} /** \brief Constructor; computes real QZ decomposition of given matrices * @@ -108,9 +109,11 @@ m_Z(A.rows(),A.cols()), m_workspace(A.rows()*2), m_maxIters(400), - m_isInitialized(false) { - compute(A, B, computeQZ); - } + m_isInitialized(false), + m_computeQZ(true) + { + compute(A, B, computeQZ); + } /** \brief Returns matrix Q in the QZ decomposition. * @@ -161,7 +164,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ ComputationInfo info() const { @@ -552,7 +555,6 @@ m_T.coeffRef(l,l-1) = Scalar(0.0); } - template<typename MatrixType> RealQZ<MatrixType>& RealQZ<MatrixType>::compute(const MatrixType& A_in, const MatrixType& B_in, bool computeQZ) { @@ -616,6 +618,37 @@ } // check if we converged before reaching iterations limit m_info = (local_iter<m_maxIters) ? Success : NoConvergence; + + // For each non triangular 2x2 diagonal block of S, + // reduce the respective 2x2 diagonal block of T to positive diagonal form using 2x2 SVD. + // This step is not mandatory for QZ, but it does help further extraction of eigenvalues/eigenvectors, + // and is in par with Lapack/Matlab QZ. + if(m_info==Success) + { + for(Index i=0; i<dim-1; ++i) + { + if(m_S.coeff(i+1, i) != Scalar(0)) + { + JacobiRotation<Scalar> j_left, j_right; + internal::real_2x2_jacobi_svd(m_T, i, i+1, &j_left, &j_right); + + // Apply resulting Jacobi rotations + m_S.applyOnTheLeft(i,i+1,j_left); + m_S.applyOnTheRight(i,i+1,j_right); + m_T.applyOnTheLeft(i,i+1,j_left); + m_T.applyOnTheRight(i,i+1,j_right); + m_T(i+1,i) = m_T(i,i+1) = Scalar(0); + + if(m_computeQZ) { + m_Q.applyOnTheRight(i,i+1,j_left.transpose()); + m_Z.applyOnTheLeft(i,i+1,j_right.transpose()); + } + + i++; + } + } + } + return *this; } // end compute
diff --git a/Eigen/src/Eigenvalues/RealSchur.h b/Eigen/src/Eigenvalues/RealSchur.h index f4ded69..7304ef3 100644 --- a/Eigen/src/Eigenvalues/RealSchur.h +++ b/Eigen/src/Eigenvalues/RealSchur.h
@@ -190,7 +190,7 @@ RealSchur& computeFromHessenberg(const HessMatrixType& matrixH, const OrthMatrixType& matrixQ, bool computeU); /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ ComputationInfo info() const { @@ -236,7 +236,7 @@ typedef Matrix<Scalar,3,1> Vector3s; Scalar computeNormOfT(); - Index findSmallSubdiagEntry(Index iu); + Index findSmallSubdiagEntry(Index iu, const Scalar& considerAsZero); void splitOffTwoRows(Index iu, bool computeU, const Scalar& exshift); void computeShift(Index iu, Index iter, Scalar& exshift, Vector3s& shiftInfo); void initFrancisQRStep(Index il, Index iu, const Vector3s& shiftInfo, Index& im, Vector3s& firstHouseholderVector); @@ -248,31 +248,54 @@ template<typename InputType> RealSchur<MatrixType>& RealSchur<MatrixType>::compute(const EigenBase<InputType>& matrix, bool computeU) { + const Scalar considerAsZero = (std::numeric_limits<Scalar>::min)(); + eigen_assert(matrix.cols() == matrix.rows()); Index maxIters = m_maxIters; if (maxIters == -1) maxIters = m_maxIterationsPerRow * matrix.rows(); - // Step 1. Reduce to Hessenberg form - m_hess.compute(matrix.derived()); + Scalar scale = matrix.derived().cwiseAbs().maxCoeff(); + if(scale<considerAsZero) + { + m_matT.setZero(matrix.rows(),matrix.cols()); + if(computeU) + m_matU.setIdentity(matrix.rows(),matrix.cols()); + m_info = Success; + m_isInitialized = true; + m_matUisUptodate = computeU; + return *this; + } - // Step 2. Reduce to real Schur form - computeFromHessenberg(m_hess.matrixH(), m_hess.matrixQ(), computeU); + // Step 1. Reduce to Hessenberg form + m_hess.compute(matrix.derived()/scale); + + // Step 2. Reduce to real Schur form + // Note: we copy m_hess.matrixQ() into m_matU here and not in computeFromHessenberg + // to be able to pass our working-space buffer for the Householder to Dense evaluation. + m_workspaceVector.resize(matrix.cols()); + if(computeU) + m_hess.matrixQ().evalTo(m_matU, m_workspaceVector); + computeFromHessenberg(m_hess.matrixH(), m_matU, computeU); + + m_matT *= scale; return *this; } template<typename MatrixType> template<typename HessMatrixType, typename OrthMatrixType> RealSchur<MatrixType>& RealSchur<MatrixType>::computeFromHessenberg(const HessMatrixType& matrixH, const OrthMatrixType& matrixQ, bool computeU) -{ - m_matT = matrixH; - if(computeU) +{ + using std::abs; + + m_matT = matrixH; + m_workspaceVector.resize(m_matT.cols()); + if(computeU && !internal::is_same_dense(m_matU,matrixQ)) m_matU = matrixQ; Index maxIters = m_maxIters; if (maxIters == -1) maxIters = m_maxIterationsPerRow * matrixH.rows(); - m_workspaceVector.resize(m_matT.cols()); Scalar* workspace = &m_workspaceVector.coeffRef(0); // The matrix m_matT is divided in three parts. @@ -284,12 +307,16 @@ Index totalIter = 0; // iteration count for whole matrix Scalar exshift(0); // sum of exceptional shifts Scalar norm = computeNormOfT(); + // sub-diagonal entries smaller than considerAsZero will be treated as zero. + // We use eps^2 to enable more precision in small eigenvalues. + Scalar considerAsZero = numext::maxi<Scalar>( norm * numext::abs2(NumTraits<Scalar>::epsilon()), + (std::numeric_limits<Scalar>::min)() ); - if(norm!=0) + if(norm!=Scalar(0)) { while (iu >= 0) { - Index il = findSmallSubdiagEntry(iu); + Index il = findSmallSubdiagEntry(iu,considerAsZero); // Check for convergence if (il == iu) // One root found @@ -309,7 +336,7 @@ else // No convergence yet { // The firstHouseholderVector vector has to be initialized to something to get rid of a silly GCC warning (-O1 -Wall -DNDEBUG ) - Vector3s firstHouseholderVector(0,0,0), shiftInfo; + Vector3s firstHouseholderVector = Vector3s::Zero(), shiftInfo; computeShift(iu, iter, exshift, shiftInfo); iter = iter + 1; totalIter = totalIter + 1; @@ -346,14 +373,17 @@ /** \internal Look for single small sub-diagonal element and returns its index */ template<typename MatrixType> -inline Index RealSchur<MatrixType>::findSmallSubdiagEntry(Index iu) +inline Index RealSchur<MatrixType>::findSmallSubdiagEntry(Index iu, const Scalar& considerAsZero) { using std::abs; Index res = iu; while (res > 0) { Scalar s = abs(m_matT.coeff(res-1,res-1)) + abs(m_matT.coeff(res,res)); - if (abs(m_matT.coeff(res,res-1)) <= NumTraits<Scalar>::epsilon() * s) + + s = numext::maxi<Scalar>(s * NumTraits<Scalar>::epsilon(), considerAsZero); + + if (abs(m_matT.coeff(res,res-1)) <= s) break; res--; }
diff --git a/Eigen/src/Eigenvalues/RealSchur_MKL.h b/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h similarity index 66% rename from Eigen/src/Eigenvalues/RealSchur_MKL.h rename to Eigen/src/Eigenvalues/RealSchur_LAPACKE.h index e809264..2c22517 100644 --- a/Eigen/src/Eigenvalues/RealSchur_MKL.h +++ b/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h
@@ -25,39 +25,37 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Real Schur needed to real unsymmetrical eigenvalues/eigenvectors. ******************************************************************************** */ -#ifndef EIGEN_REAL_SCHUR_MKL_H -#define EIGEN_REAL_SCHUR_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" +#ifndef EIGEN_REAL_SCHUR_LAPACKE_H +#define EIGEN_REAL_SCHUR_LAPACKE_H namespace Eigen { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_SCHUR_REAL(EIGTYPE, MKLTYPE, MKLPREFIX, MKLPREFIX_U, EIGCOLROW, MKLCOLROW) \ +#define EIGEN_LAPACKE_SCHUR_REAL(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX, LAPACKE_PREFIX_U, EIGCOLROW, LAPACKE_COLROW) \ template<> template<typename InputType> inline \ RealSchur<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >& \ RealSchur<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >::compute(const EigenBase<InputType>& matrix, bool computeU) \ { \ eigen_assert(matrix.cols() == matrix.rows()); \ \ - lapack_int n = matrix.cols(), sdim, info; \ - lapack_int matrix_order = MKLCOLROW; \ + lapack_int n = internal::convert_index<lapack_int>(matrix.cols()), sdim, info; \ + lapack_int matrix_order = LAPACKE_COLROW; \ char jobvs, sort='N'; \ - LAPACK_##MKLPREFIX_U##_SELECT2 select = 0; \ + LAPACK_##LAPACKE_PREFIX_U##_SELECT2 select = 0; \ jobvs = (computeU) ? 'V' : 'N'; \ m_matU.resize(n, n); \ - lapack_int ldvs = m_matU.outerStride(); \ + lapack_int ldvs = internal::convert_index<lapack_int>(m_matU.outerStride()); \ m_matT = matrix; \ - lapack_int lda = m_matT.outerStride(); \ + lapack_int lda = internal::convert_index<lapack_int>(m_matT.outerStride()); \ Matrix<EIGTYPE, Dynamic, Dynamic> wr, wi; \ wr.resize(n, 1); wi.resize(n, 1); \ - info = LAPACKE_##MKLPREFIX##gees( matrix_order, jobvs, sort, select, n, (MKLTYPE*)m_matT.data(), lda, &sdim, (MKLTYPE*)wr.data(), (MKLTYPE*)wi.data(), (MKLTYPE*)m_matU.data(), ldvs ); \ + info = LAPACKE_##LAPACKE_PREFIX##gees( matrix_order, jobvs, sort, select, n, (LAPACKE_TYPE*)m_matT.data(), lda, &sdim, (LAPACKE_TYPE*)wr.data(), (LAPACKE_TYPE*)wi.data(), (LAPACKE_TYPE*)m_matU.data(), ldvs ); \ if(info == 0) \ m_info = Success; \ else \ @@ -69,11 +67,11 @@ \ } -EIGEN_MKL_SCHUR_REAL(double, double, d, D, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SCHUR_REAL(float, float, s, S, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SCHUR_REAL(double, double, d, D, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_SCHUR_REAL(float, float, s, S, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SCHUR_REAL(double, double, d, D, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SCHUR_REAL(float, float, s, S, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SCHUR_REAL(double, double, d, D, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SCHUR_REAL(float, float, s, S, RowMajor, LAPACK_ROW_MAJOR) } // end namespace Eigen -#endif // EIGEN_REAL_SCHUR_MKL_H +#endif // EIGEN_REAL_SCHUR_LAPACKE_H
diff --git a/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h b/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h index 469ea5e..9bbce65 100644 --- a/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h +++ b/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h
@@ -20,7 +20,9 @@ namespace internal { template<typename SolverType,int Size,bool IsComplex> struct direct_selfadjoint_eigenvalues; + template<typename MatrixType, typename DiagType, typename SubDiagType> +EIGEN_DEVICE_FUNC ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag, const Index maxIterations, bool computeEigenvectors, MatrixType& eivec); } @@ -119,7 +121,9 @@ : m_eivec(), m_eivalues(), m_subdiag(), - m_isInitialized(false) + m_info(InvalidInput), + m_isInitialized(false), + m_eigenvectorsOk(false) { } /** \brief Constructor, pre-allocates memory for dynamic-size matrices. @@ -139,7 +143,8 @@ : m_eivec(size, size), m_eivalues(size), m_subdiag(size > 1 ? size - 1 : 1), - m_isInitialized(false) + m_isInitialized(false), + m_eigenvectorsOk(false) {} /** \brief Constructor; computes eigendecomposition of given matrix. @@ -163,7 +168,8 @@ : m_eivec(matrix.rows(), matrix.cols()), m_eivalues(matrix.cols()), m_subdiag(matrix.rows() > 1 ? matrix.rows() - 1 : 1), - m_isInitialized(false) + m_isInitialized(false), + m_eigenvectorsOk(false) { compute(matrix.derived(), options); } @@ -337,7 +343,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ EIGEN_DEVICE_FUNC ComputationInfo info() const @@ -354,7 +360,8 @@ static const int m_maxIterations = 30; protected: - static void check_template_parameters() + static EIGEN_DEVICE_FUNC + void check_template_parameters() { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } @@ -403,7 +410,7 @@ const InputType &matrix(a_matrix.derived()); - using std::abs; + EIGEN_USING_STD_MATH(abs); eigen_assert(matrix.cols() == matrix.rows()); eigen_assert((options&~(EigVecMask|GenEigMask))==0 && (options&EigVecMask)!=EigVecMask @@ -414,7 +421,8 @@ if(n==1) { - m_eivalues.coeffRef(0,0) = numext::real(matrix(0,0)); + m_eivec = matrix; + m_eivalues.coeffRef(0,0) = numext::real(m_eivec.coeff(0,0)); if(computeEigenvectors) m_eivec.setOnes(n,n); m_info = Success; @@ -458,7 +466,7 @@ { m_eivec.setIdentity(diag.size(), diag.size()); } - m_info = computeFromTridiagonal_impl(m_eivalues, m_subdiag, m_maxIterations, computeEigenvectors, m_eivec); + m_info = internal::computeFromTridiagonal_impl(m_eivalues, m_subdiag, m_maxIterations, computeEigenvectors, m_eivec); m_isInitialized = true; m_eigenvectorsOk = computeEigenvectors; @@ -478,9 +486,10 @@ * \returns \c Success or \c NoConvergence */ template<typename MatrixType, typename DiagType, typename SubDiagType> +EIGEN_DEVICE_FUNC ComputationInfo computeFromTridiagonal_impl(DiagType& diag, SubDiagType& subdiag, const Index maxIterations, bool computeEigenvectors, MatrixType& eivec) { - using std::abs; + EIGEN_USING_STD_MATH(abs); ComputationInfo info; typedef typename MatrixType::Scalar Scalar; @@ -492,15 +501,16 @@ typedef typename DiagType::RealScalar RealScalar; const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); + const RealScalar precision = RealScalar(2)*NumTraits<RealScalar>::epsilon(); while (end>0) { for (Index i = start; i<end; ++i) - if (internal::isMuchSmallerThan(abs(subdiag[i]),(abs(diag[i])+abs(diag[i+1]))) || abs(subdiag[i]) <= considerAsZero) + if (internal::isMuchSmallerThan(abs(subdiag[i]),(abs(diag[i])+abs(diag[i+1])),precision) || abs(subdiag[i]) <= considerAsZero) subdiag[i] = 0; // find the largest unreduced block - while (end>0 && subdiag[end-1]==0) + while (end>0 && subdiag[end-1]==RealScalar(0)) { end--; } @@ -533,7 +543,7 @@ diag.segment(i,n-i).minCoeff(&k); if (k > 0) { - std::swap(diag[i], diag[k+i]); + numext::swap(diag[i], diag[k+i]); if(computeEigenvectors) eivec.col(i).swap(eivec.col(k+i)); } @@ -568,8 +578,8 @@ EIGEN_USING_STD_MATH(atan2) EIGEN_USING_STD_MATH(cos) EIGEN_USING_STD_MATH(sin) - const Scalar s_inv3 = Scalar(1.0)/Scalar(3.0); - const Scalar s_sqrt3 = sqrt(Scalar(3.0)); + const Scalar s_inv3 = Scalar(1)/Scalar(3); + const Scalar s_sqrt3 = sqrt(Scalar(3)); // The characteristic equation is x^3 - c2*x^2 + c1*x - c0 = 0. The // eigenvalues are the roots to this equation, all guaranteed to be @@ -603,7 +613,8 @@ EIGEN_DEVICE_FUNC static inline bool extract_kernel(MatrixType& mat, Ref<VectorType> res, Ref<VectorType> representative) { - using std::abs; + EIGEN_USING_STD_MATH(abs); + EIGEN_USING_STD_MATH(sqrt); Index i0; // Find non-zero column i0 (by construction, there must exist a non zero coefficient on the diagonal): mat.diagonal().cwiseAbs().maxCoeff(&i0); @@ -614,8 +625,8 @@ VectorType c0, c1; n0 = (c0 = representative.cross(mat.col((i0+1)%3))).squaredNorm(); n1 = (c1 = representative.cross(mat.col((i0+2)%3))).squaredNorm(); - if(n0>n1) res = c0/std::sqrt(n0); - else res = c1/std::sqrt(n1); + if(n0>n1) res = c0/sqrt(n0); + else res = c1/sqrt(n1); return true; } @@ -717,7 +728,7 @@ EIGEN_DEVICE_FUNC static inline void computeRoots(const MatrixType& m, VectorType& roots) { - using std::sqrt; + EIGEN_USING_STD_MATH(sqrt); const Scalar t0 = Scalar(0.5) * sqrt( numext::abs2(m(0,0)-m(1,1)) + Scalar(4)*numext::abs2(m(1,0))); const Scalar t1 = Scalar(0.5) * (m(0,0) + m(1,1)); roots(0) = t1 - t0; @@ -739,14 +750,18 @@ EigenvectorsType& eivecs = solver.m_eivec; VectorType& eivals = solver.m_eivalues; - // map the matrix coefficients to [-1:1] to avoid over- and underflow. - Scalar scale = mat.cwiseAbs().maxCoeff(); - scale = numext::maxi(scale,Scalar(1)); - MatrixType scaledMat = mat / scale; - + // Shift the matrix to the mean eigenvalue and map the matrix coefficients to [-1:1] to avoid over- and underflow. + Scalar shift = mat.trace() / Scalar(2); + MatrixType scaledMat = mat; + scaledMat.coeffRef(0,1) = mat.coeff(1,0); + scaledMat.diagonal().array() -= shift; + Scalar scale = scaledMat.cwiseAbs().maxCoeff(); + if(scale > Scalar(0)) + scaledMat /= scale; + // Compute the eigenvalues computeRoots(scaledMat,eivals); - + // compute the eigen vectors if(computeEigenvectors) { @@ -774,10 +789,11 @@ eivecs.col(0) << eivecs.col(1).unitOrthogonal(); } } - + // Rescale back to the original size. eivals *= scale; - + eivals.array() += shift; + solver.m_info = Success; solver.m_isInitialized = true; solver.m_eigenvectorsOk = computeEigenvectors; @@ -800,7 +816,7 @@ EIGEN_DEVICE_FUNC static void tridiagonal_qr_step(RealScalar* diag, RealScalar* subdiag, Index start, Index end, Scalar* matrixQ, Index n) { - using std::abs; + EIGEN_USING_STD_MATH(abs); RealScalar td = (diag[end-1] - diag[end])*RealScalar(0.5); RealScalar e = subdiag[end-1]; // Note that thanks to scaling, e^2 or td^2 cannot overflow, however they can still @@ -809,14 +825,14 @@ // RealScalar mu = diag[end] - e2 / (td + (td>0 ? 1 : -1) * sqrt(td*td + e2)); // This explain the following, somewhat more complicated, version: RealScalar mu = diag[end]; - if(td==0) + if(td==RealScalar(0)) mu -= abs(e); else { RealScalar e2 = numext::abs2(subdiag[end-1]); RealScalar h = numext::hypot(td,e); - if(e2==0) mu -= (e / (td + (td>0 ? 1 : -1))) * (e / h); - else mu -= e2 / (td + (td>0 ? h : -h)); + if(e2==RealScalar(0)) mu -= (e / (td + (td>RealScalar(0) ? RealScalar(1) : RealScalar(-1)))) * (e / h); + else mu -= e2 / (td + (td>RealScalar(0) ? h : -h)); } RealScalar x = diag[start] - mu;
diff --git a/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_MKL.h b/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h similarity index 69% rename from Eigen/src/Eigenvalues/SelfAdjointEigenSolver_MKL.h rename to Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h index 3499dc7..b0c947d 100644 --- a/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_MKL.h +++ b/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h
@@ -25,21 +25,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Self-adjoint eigenvalues/eigenvectors. ******************************************************************************** */ -#ifndef EIGEN_SAEIGENSOLVER_MKL_H -#define EIGEN_SAEIGENSOLVER_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" +#ifndef EIGEN_SAEIGENSOLVER_LAPACKE_H +#define EIGEN_SAEIGENSOLVER_LAPACKE_H namespace Eigen { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_EIG_SELFADJ(EIGTYPE, MKLTYPE, MKLRTYPE, MKLNAME, EIGCOLROW, MKLCOLROW ) \ +#define EIGEN_LAPACKE_EIG_SELFADJ_2(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_NAME, EIGCOLROW ) \ template<> template<typename InputType> inline \ SelfAdjointEigenSolver<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >& \ SelfAdjointEigenSolver<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW> >::compute(const EigenBase<InputType>& matrix, int options) \ @@ -49,7 +47,7 @@ && (options&EigVecMask)!=EigVecMask \ && "invalid option parameter"); \ bool computeEigenvectors = (options&ComputeEigenvectors)==ComputeEigenvectors; \ - lapack_int n = matrix.cols(), lda, matrix_order, info; \ + lapack_int n = internal::convert_index<lapack_int>(matrix.cols()), lda, info; \ m_eivalues.resize(n,1); \ m_subdiag.resize(n-1); \ m_eivec = matrix; \ @@ -64,28 +62,25 @@ return *this; \ } \ \ - lda = m_eivec.outerStride(); \ - matrix_order=MKLCOLROW; \ + lda = internal::convert_index<lapack_int>(m_eivec.outerStride()); \ char jobz, uplo='L'/*, range='A'*/; \ jobz = computeEigenvectors ? 'V' : 'N'; \ \ - info = LAPACKE_##MKLNAME( matrix_order, jobz, uplo, n, (MKLTYPE*)m_eivec.data(), lda, (MKLRTYPE*)m_eivalues.data() ); \ + info = LAPACKE_##LAPACKE_NAME( LAPACK_COL_MAJOR, jobz, uplo, n, (LAPACKE_TYPE*)m_eivec.data(), lda, (LAPACKE_RTYPE*)m_eivalues.data() ); \ m_info = (info==0) ? Success : NoConvergence; \ m_isInitialized = true; \ m_eigenvectorsOk = computeEigenvectors; \ return *this; \ } +#define EIGEN_LAPACKE_EIG_SELFADJ(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_NAME ) \ + EIGEN_LAPACKE_EIG_SELFADJ_2(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_NAME, ColMajor ) \ + EIGEN_LAPACKE_EIG_SELFADJ_2(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_NAME, RowMajor ) -EIGEN_MKL_EIG_SELFADJ(double, double, double, dsyev, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_EIG_SELFADJ(float, float, float, ssyev, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_EIG_SELFADJ(dcomplex, MKL_Complex16, double, zheev, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_EIG_SELFADJ(scomplex, MKL_Complex8, float, cheev, ColMajor, LAPACK_COL_MAJOR) - -EIGEN_MKL_EIG_SELFADJ(double, double, double, dsyev, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_EIG_SELFADJ(float, float, float, ssyev, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_EIG_SELFADJ(dcomplex, MKL_Complex16, double, zheev, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_EIG_SELFADJ(scomplex, MKL_Complex8, float, cheev, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_EIG_SELFADJ(double, double, double, dsyev) +EIGEN_LAPACKE_EIG_SELFADJ(float, float, float, ssyev) +EIGEN_LAPACKE_EIG_SELFADJ(dcomplex, lapack_complex_double, double, zheev) +EIGEN_LAPACKE_EIG_SELFADJ(scomplex, lapack_complex_float, float, cheev) } // end namespace Eigen
diff --git a/Eigen/src/Eigenvalues/Tridiagonalization.h b/Eigen/src/Eigenvalues/Tridiagonalization.h index 2030b5b..c5c1acf 100644 --- a/Eigen/src/Eigenvalues/Tridiagonalization.h +++ b/Eigen/src/Eigenvalues/Tridiagonalization.h
@@ -25,6 +25,7 @@ }; template<typename MatrixType, typename CoeffVectorType> +EIGEN_DEVICE_FUNC void tridiagonalization_inplace(MatrixType& matA, CoeffVectorType& hCoeffs); } @@ -344,6 +345,7 @@ * \sa Tridiagonalization::packedMatrix() */ template<typename MatrixType, typename CoeffVectorType> +EIGEN_DEVICE_FUNC void tridiagonalization_inplace(MatrixType& matA, CoeffVectorType& hCoeffs) { using numext::conj; @@ -367,10 +369,10 @@ hCoeffs.tail(n-i-1).noalias() = (matA.bottomRightCorner(remainingSize,remainingSize).template selfadjointView<Lower>() * (conj(h) * matA.col(i).tail(remainingSize))); - hCoeffs.tail(n-i-1) += (conj(h)*Scalar(-0.5)*(hCoeffs.tail(remainingSize).dot(matA.col(i).tail(remainingSize)))) * matA.col(i).tail(n-i-1); + hCoeffs.tail(n-i-1) += (conj(h)*RealScalar(-0.5)*(hCoeffs.tail(remainingSize).dot(matA.col(i).tail(remainingSize)))) * matA.col(i).tail(n-i-1); matA.bottomRightCorner(remainingSize, remainingSize).template selfadjointView<Lower>() - .rankUpdate(matA.col(i).tail(remainingSize), hCoeffs.tail(remainingSize), -1); + .rankUpdate(matA.col(i).tail(remainingSize), hCoeffs.tail(remainingSize), Scalar(-1)); matA.col(i).coeffRef(i+1) = beta; hCoeffs.coeffRef(i) = h; @@ -424,6 +426,7 @@ * \sa class Tridiagonalization */ template<typename MatrixType, typename DiagonalType, typename SubDiagonalType> +EIGEN_DEVICE_FUNC void tridiagonalization_inplace(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, bool extractQ) { eigen_assert(mat.cols()==mat.rows() && diag.size()==mat.rows() && subdiag.size()==mat.rows()-1); @@ -439,7 +442,8 @@ typedef typename Tridiagonalization<MatrixType>::CoeffVectorType CoeffVectorType; typedef typename Tridiagonalization<MatrixType>::HouseholderSequenceType HouseholderSequenceType; template<typename DiagonalType, typename SubDiagonalType> - static void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, bool extractQ) + static EIGEN_DEVICE_FUNC + void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType& subdiag, bool extractQ) { CoeffVectorType hCoeffs(mat.cols()-1); tridiagonalization_inplace(mat,hCoeffs); @@ -508,7 +512,8 @@ typedef typename MatrixType::Scalar Scalar; template<typename DiagonalType, typename SubDiagonalType> - static void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType&, bool extractQ) + static EIGEN_DEVICE_FUNC + void run(MatrixType& mat, DiagonalType& diag, SubDiagonalType&, bool extractQ) { diag(0,0) = numext::real(mat(0,0)); if(extractQ)
diff --git a/Eigen/src/Geometry/AlignedBox.h b/Eigen/src/Geometry/AlignedBox.h index 03f1a11..c902d8f 100644 --- a/Eigen/src/Geometry/AlignedBox.h +++ b/Eigen/src/Geometry/AlignedBox.h
@@ -36,8 +36,9 @@ typedef NumTraits<Scalar> ScalarTraits; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef typename ScalarTraits::Real RealScalar; - typedef typename ScalarTraits::NonInteger NonInteger; + typedef typename ScalarTraits::NonInteger NonInteger; typedef Matrix<Scalar,AmbientDimAtCompileTime,1> VectorType; + typedef CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const VectorType, const VectorType> VectorTypeSum; /** Define constants to name the corners of a 1D, 2D or 3D axis aligned bounding box */ enum CornerType @@ -61,77 +62,76 @@ /** Default constructor initializing a null box. */ - inline AlignedBox() - { if (AmbientDimAtCompileTime!=Dynamic) setEmpty(); } + EIGEN_DEVICE_FUNC inline AlignedBox() + { if (EIGEN_CONST_CONDITIONAL(AmbientDimAtCompileTime!=Dynamic)) setEmpty(); } /** Constructs a null box with \a _dim the dimension of the ambient space. */ - inline explicit AlignedBox(Index _dim) : m_min(_dim), m_max(_dim) + EIGEN_DEVICE_FUNC inline explicit AlignedBox(Index _dim) : m_min(_dim), m_max(_dim) { setEmpty(); } /** Constructs a box with extremities \a _min and \a _max. * \warning If either component of \a _min is larger than the same component of \a _max, the constructed box is empty. */ template<typename OtherVectorType1, typename OtherVectorType2> - inline AlignedBox(const OtherVectorType1& _min, const OtherVectorType2& _max) : m_min(_min), m_max(_max) {} + EIGEN_DEVICE_FUNC inline AlignedBox(const OtherVectorType1& _min, const OtherVectorType2& _max) : m_min(_min), m_max(_max) {} /** Constructs a box containing a single point \a p. */ template<typename Derived> - inline explicit AlignedBox(const MatrixBase<Derived>& p) : m_min(p), m_max(m_min) + EIGEN_DEVICE_FUNC inline explicit AlignedBox(const MatrixBase<Derived>& p) : m_min(p), m_max(m_min) { } - ~AlignedBox() {} + EIGEN_DEVICE_FUNC ~AlignedBox() {} /** \returns the dimension in which the box holds */ - inline Index dim() const { return AmbientDimAtCompileTime==Dynamic ? m_min.size() : Index(AmbientDimAtCompileTime); } + EIGEN_DEVICE_FUNC inline Index dim() const { return AmbientDimAtCompileTime==Dynamic ? m_min.size() : Index(AmbientDimAtCompileTime); } /** \deprecated use isEmpty() */ - inline bool isNull() const { return isEmpty(); } + EIGEN_DEVICE_FUNC inline bool isNull() const { return isEmpty(); } /** \deprecated use setEmpty() */ - inline void setNull() { setEmpty(); } + EIGEN_DEVICE_FUNC inline void setNull() { setEmpty(); } /** \returns true if the box is empty. * \sa setEmpty */ - inline bool isEmpty() const { return (m_min.array() > m_max.array()).any(); } + EIGEN_DEVICE_FUNC inline bool isEmpty() const { return (m_min.array() > m_max.array()).any(); } /** Makes \c *this an empty box. * \sa isEmpty */ - inline void setEmpty() + EIGEN_DEVICE_FUNC inline void setEmpty() { m_min.setConstant( ScalarTraits::highest() ); m_max.setConstant( ScalarTraits::lowest() ); } /** \returns the minimal corner */ - inline const VectorType& (min)() const { return m_min; } + EIGEN_DEVICE_FUNC inline const VectorType& (min)() const { return m_min; } /** \returns a non const reference to the minimal corner */ - inline VectorType& (min)() { return m_min; } + EIGEN_DEVICE_FUNC inline VectorType& (min)() { return m_min; } /** \returns the maximal corner */ - inline const VectorType& (max)() const { return m_max; } + EIGEN_DEVICE_FUNC inline const VectorType& (max)() const { return m_max; } /** \returns a non const reference to the maximal corner */ - inline VectorType& (max)() { return m_max; } + EIGEN_DEVICE_FUNC inline VectorType& (max)() { return m_max; } /** \returns the center of the box */ - inline const CwiseUnaryOp<internal::scalar_quotient1_op<Scalar>, - const CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const VectorType, const VectorType> > + EIGEN_DEVICE_FUNC inline const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(VectorTypeSum, RealScalar, quotient) center() const - { return (m_min+m_max)/2; } + { return (m_min+m_max)/RealScalar(2); } /** \returns the lengths of the sides of the bounding box. * Note that this function does not get the same * result for integral or floating scalar types: see */ - inline const CwiseBinaryOp< internal::scalar_difference_op<Scalar>, const VectorType, const VectorType> sizes() const + EIGEN_DEVICE_FUNC inline const CwiseBinaryOp< internal::scalar_difference_op<Scalar,Scalar>, const VectorType, const VectorType> sizes() const { return m_max - m_min; } /** \returns the volume of the bounding box */ - inline Scalar volume() const + EIGEN_DEVICE_FUNC inline Scalar volume() const { return sizes().prod(); } /** \returns an expression for the bounding box diagonal vector * if the length of the diagonal is needed: diagonal().norm() * will provide it. */ - inline CwiseBinaryOp< internal::scalar_difference_op<Scalar>, const VectorType, const VectorType> diagonal() const + EIGEN_DEVICE_FUNC inline CwiseBinaryOp< internal::scalar_difference_op<Scalar,Scalar>, const VectorType, const VectorType> diagonal() const { return sizes(); } /** \returns the vertex of the bounding box at the corner defined by @@ -143,7 +143,7 @@ * For 3D bounding boxes, the following names are added: * BottomLeftCeil, BottomRightCeil, TopLeftCeil, TopRightCeil. */ - inline VectorType corner(CornerType corner) const + EIGEN_DEVICE_FUNC inline VectorType corner(CornerType corner) const { EIGEN_STATIC_ASSERT(_AmbientDim <= 3, THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE); @@ -161,7 +161,7 @@ /** \returns a random point inside the bounding box sampled with * a uniform distribution */ - inline VectorType sample() const + EIGEN_DEVICE_FUNC inline VectorType sample() const { VectorType r(dim()); for(Index d=0; d<dim(); ++d) @@ -179,25 +179,25 @@ /** \returns true if the point \a p is inside the box \c *this. */ template<typename Derived> - inline bool contains(const MatrixBase<Derived>& p) const + EIGEN_DEVICE_FUNC inline bool contains(const MatrixBase<Derived>& p) const { typename internal::nested_eval<Derived,2>::type p_n(p.derived()); return (m_min.array()<=p_n.array()).all() && (p_n.array()<=m_max.array()).all(); } /** \returns true if the box \a b is entirely inside the box \c *this. */ - inline bool contains(const AlignedBox& b) const + EIGEN_DEVICE_FUNC inline bool contains(const AlignedBox& b) const { return (m_min.array()<=(b.min)().array()).all() && ((b.max)().array()<=m_max.array()).all(); } /** \returns true if the box \a b is intersecting the box \c *this. * \sa intersection, clamp */ - inline bool intersects(const AlignedBox& b) const + EIGEN_DEVICE_FUNC inline bool intersects(const AlignedBox& b) const { return (m_min.array()<=(b.max)().array()).all() && ((b.min)().array()<=m_max.array()).all(); } /** Extends \c *this such that it contains the point \a p and returns a reference to \c *this. * \sa extend(const AlignedBox&) */ template<typename Derived> - inline AlignedBox& extend(const MatrixBase<Derived>& p) + EIGEN_DEVICE_FUNC inline AlignedBox& extend(const MatrixBase<Derived>& p) { typename internal::nested_eval<Derived,2>::type p_n(p.derived()); m_min = m_min.cwiseMin(p_n); @@ -207,7 +207,7 @@ /** Extends \c *this such that it contains the box \a b and returns a reference to \c *this. * \sa merged, extend(const MatrixBase&) */ - inline AlignedBox& extend(const AlignedBox& b) + EIGEN_DEVICE_FUNC inline AlignedBox& extend(const AlignedBox& b) { m_min = m_min.cwiseMin(b.m_min); m_max = m_max.cwiseMax(b.m_max); @@ -217,7 +217,7 @@ /** Clamps \c *this by the box \a b and returns a reference to \c *this. * \note If the boxes don't intersect, the resulting box is empty. * \sa intersection(), intersects() */ - inline AlignedBox& clamp(const AlignedBox& b) + EIGEN_DEVICE_FUNC inline AlignedBox& clamp(const AlignedBox& b) { m_min = m_min.cwiseMax(b.m_min); m_max = m_max.cwiseMin(b.m_max); @@ -227,18 +227,18 @@ /** Returns an AlignedBox that is the intersection of \a b and \c *this * \note If the boxes don't intersect, the resulting box is empty. * \sa intersects(), clamp, contains() */ - inline AlignedBox intersection(const AlignedBox& b) const + EIGEN_DEVICE_FUNC inline AlignedBox intersection(const AlignedBox& b) const {return AlignedBox(m_min.cwiseMax(b.m_min), m_max.cwiseMin(b.m_max)); } /** Returns an AlignedBox that is the union of \a b and \c *this. * \note Merging with an empty box may result in a box bigger than \c *this. * \sa extend(const AlignedBox&) */ - inline AlignedBox merged(const AlignedBox& b) const + EIGEN_DEVICE_FUNC inline AlignedBox merged(const AlignedBox& b) const { return AlignedBox(m_min.cwiseMin(b.m_min), m_max.cwiseMax(b.m_max)); } /** Translate \c *this by the vector \a t and returns a reference to \c *this. */ template<typename Derived> - inline AlignedBox& translate(const MatrixBase<Derived>& a_t) + EIGEN_DEVICE_FUNC inline AlignedBox& translate(const MatrixBase<Derived>& a_t) { const typename internal::nested_eval<Derived,2>::type t(a_t.derived()); m_min += t; @@ -251,28 +251,28 @@ * \sa exteriorDistance(const MatrixBase&), squaredExteriorDistance(const AlignedBox&) */ template<typename Derived> - inline Scalar squaredExteriorDistance(const MatrixBase<Derived>& p) const; + EIGEN_DEVICE_FUNC inline Scalar squaredExteriorDistance(const MatrixBase<Derived>& p) const; /** \returns the squared distance between the boxes \a b and \c *this, * and zero if the boxes intersect. * \sa exteriorDistance(const AlignedBox&), squaredExteriorDistance(const MatrixBase&) */ - inline Scalar squaredExteriorDistance(const AlignedBox& b) const; + EIGEN_DEVICE_FUNC inline Scalar squaredExteriorDistance(const AlignedBox& b) const; /** \returns the distance between the point \a p and the box \c *this, * and zero if \a p is inside the box. * \sa squaredExteriorDistance(const MatrixBase&), exteriorDistance(const AlignedBox&) */ template<typename Derived> - inline NonInteger exteriorDistance(const MatrixBase<Derived>& p) const - { using std::sqrt; return sqrt(NonInteger(squaredExteriorDistance(p))); } + EIGEN_DEVICE_FUNC inline NonInteger exteriorDistance(const MatrixBase<Derived>& p) const + { EIGEN_USING_STD_MATH(sqrt) return sqrt(NonInteger(squaredExteriorDistance(p))); } /** \returns the distance between the boxes \a b and \c *this, * and zero if the boxes intersect. * \sa squaredExteriorDistance(const AlignedBox&), exteriorDistance(const MatrixBase&) */ - inline NonInteger exteriorDistance(const AlignedBox& b) const - { using std::sqrt; return sqrt(NonInteger(squaredExteriorDistance(b))); } + EIGEN_DEVICE_FUNC inline NonInteger exteriorDistance(const AlignedBox& b) const + { EIGEN_USING_STD_MATH(sqrt) return sqrt(NonInteger(squaredExteriorDistance(b))); } /** \returns \c *this with scalar type casted to \a NewScalarType * @@ -280,7 +280,7 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<AlignedBox, + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<AlignedBox, AlignedBox<NewScalarType,AmbientDimAtCompileTime> >::type cast() const { return typename internal::cast_return_type<AlignedBox, @@ -289,7 +289,7 @@ /** Copy constructor with scalar type conversion */ template<typename OtherScalarType> - inline explicit AlignedBox(const AlignedBox<OtherScalarType,AmbientDimAtCompileTime>& other) + EIGEN_DEVICE_FUNC inline explicit AlignedBox(const AlignedBox<OtherScalarType,AmbientDimAtCompileTime>& other) { m_min = (other.min)().template cast<Scalar>(); m_max = (other.max)().template cast<Scalar>(); @@ -299,7 +299,7 @@ * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const AlignedBox& other, const RealScalar& prec = ScalarTraits::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const AlignedBox& other, const RealScalar& prec = ScalarTraits::dummy_precision()) const { return m_min.isApprox(other.m_min, prec) && m_max.isApprox(other.m_max, prec); } protected: @@ -311,7 +311,7 @@ template<typename Scalar,int AmbientDim> template<typename Derived> -inline Scalar AlignedBox<Scalar,AmbientDim>::squaredExteriorDistance(const MatrixBase<Derived>& a_p) const +EIGEN_DEVICE_FUNC inline Scalar AlignedBox<Scalar,AmbientDim>::squaredExteriorDistance(const MatrixBase<Derived>& a_p) const { typename internal::nested_eval<Derived,2*AmbientDim>::type p(a_p.derived()); Scalar dist2(0); @@ -333,7 +333,7 @@ } template<typename Scalar,int AmbientDim> -inline Scalar AlignedBox<Scalar,AmbientDim>::squaredExteriorDistance(const AlignedBox& b) const +EIGEN_DEVICE_FUNC inline Scalar AlignedBox<Scalar,AmbientDim>::squaredExteriorDistance(const AlignedBox& b) const { Scalar dist2(0); Scalar aux;
diff --git a/Eigen/src/Geometry/AngleAxis.h b/Eigen/src/Geometry/AngleAxis.h index 7fdb8ae..83ee1be 100644 --- a/Eigen/src/Geometry/AngleAxis.h +++ b/Eigen/src/Geometry/AngleAxis.h
@@ -69,59 +69,61 @@ public: /** Default constructor without initialization. */ - AngleAxis() {} + EIGEN_DEVICE_FUNC AngleAxis() {} /** Constructs and initialize the angle-axis rotation from an \a angle in radian * and an \a axis which \b must \b be \b normalized. * * \warning If the \a axis vector is not normalized, then the angle-axis object * represents an invalid rotation. */ template<typename Derived> + EIGEN_DEVICE_FUNC inline AngleAxis(const Scalar& angle, const MatrixBase<Derived>& axis) : m_axis(axis), m_angle(angle) {} /** Constructs and initialize the angle-axis rotation from a quaternion \a q. * This function implicitly normalizes the quaternion \a q. */ - template<typename QuatDerived> inline explicit AngleAxis(const QuaternionBase<QuatDerived>& q) { *this = q; } + template<typename QuatDerived> + EIGEN_DEVICE_FUNC inline explicit AngleAxis(const QuaternionBase<QuatDerived>& q) { *this = q; } /** Constructs and initialize the angle-axis rotation from a 3x3 rotation matrix. */ template<typename Derived> - inline explicit AngleAxis(const MatrixBase<Derived>& m) { *this = m; } + EIGEN_DEVICE_FUNC inline explicit AngleAxis(const MatrixBase<Derived>& m) { *this = m; } /** \returns the value of the rotation angle in radian */ - Scalar angle() const { return m_angle; } + EIGEN_DEVICE_FUNC Scalar angle() const { return m_angle; } /** \returns a read-write reference to the stored angle in radian */ - Scalar& angle() { return m_angle; } + EIGEN_DEVICE_FUNC Scalar& angle() { return m_angle; } /** \returns the rotation axis */ - const Vector3& axis() const { return m_axis; } + EIGEN_DEVICE_FUNC const Vector3& axis() const { return m_axis; } /** \returns a read-write reference to the stored rotation axis. * * \warning The rotation axis must remain a \b unit vector. */ - Vector3& axis() { return m_axis; } + EIGEN_DEVICE_FUNC Vector3& axis() { return m_axis; } /** Concatenates two rotations */ - inline QuaternionType operator* (const AngleAxis& other) const + EIGEN_DEVICE_FUNC inline QuaternionType operator* (const AngleAxis& other) const { return QuaternionType(*this) * QuaternionType(other); } /** Concatenates two rotations */ - inline QuaternionType operator* (const QuaternionType& other) const + EIGEN_DEVICE_FUNC inline QuaternionType operator* (const QuaternionType& other) const { return QuaternionType(*this) * other; } /** Concatenates two rotations */ - friend inline QuaternionType operator* (const QuaternionType& a, const AngleAxis& b) + friend EIGEN_DEVICE_FUNC inline QuaternionType operator* (const QuaternionType& a, const AngleAxis& b) { return a * QuaternionType(b); } /** \returns the inverse rotation, i.e., an angle-axis with opposite rotation angle */ - AngleAxis inverse() const + EIGEN_DEVICE_FUNC AngleAxis inverse() const { return AngleAxis(-m_angle, m_axis); } template<class QuatDerived> - AngleAxis& operator=(const QuaternionBase<QuatDerived>& q); + EIGEN_DEVICE_FUNC AngleAxis& operator=(const QuaternionBase<QuatDerived>& q); template<typename Derived> - AngleAxis& operator=(const MatrixBase<Derived>& m); + EIGEN_DEVICE_FUNC AngleAxis& operator=(const MatrixBase<Derived>& m); template<typename Derived> - AngleAxis& fromRotationMatrix(const MatrixBase<Derived>& m); - Matrix3 toRotationMatrix(void) const; + EIGEN_DEVICE_FUNC AngleAxis& fromRotationMatrix(const MatrixBase<Derived>& m); + EIGEN_DEVICE_FUNC Matrix3 toRotationMatrix(void) const; /** \returns \c *this with scalar type casted to \a NewScalarType * @@ -129,24 +131,24 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<AngleAxis,AngleAxis<NewScalarType> >::type cast() const + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<AngleAxis,AngleAxis<NewScalarType> >::type cast() const { return typename internal::cast_return_type<AngleAxis,AngleAxis<NewScalarType> >::type(*this); } /** Copy constructor with scalar type conversion */ template<typename OtherScalarType> - inline explicit AngleAxis(const AngleAxis<OtherScalarType>& other) + EIGEN_DEVICE_FUNC inline explicit AngleAxis(const AngleAxis<OtherScalarType>& other) { m_axis = other.axis().template cast<Scalar>(); m_angle = Scalar(other.angle()); } - static inline const AngleAxis Identity() { return AngleAxis(Scalar(0), Vector3::UnitX()); } + EIGEN_DEVICE_FUNC static inline const AngleAxis Identity() { return AngleAxis(Scalar(0), Vector3::UnitX()); } /** \returns \c true if \c *this is approximately equal to \a other, within the precision * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const AngleAxis& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const AngleAxis& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return m_axis.isApprox(other.m_axis, prec) && internal::isApprox(m_angle,other.m_angle, prec); } }; @@ -158,21 +160,26 @@ typedef AngleAxis<double> AngleAxisd; /** Set \c *this from a \b unit quaternion. - * The resulting axis is normalized. + * + * The resulting axis is normalized, and the computed angle is in the [0,pi] range. * * This function implicitly normalizes the quaternion \a q. */ template<typename Scalar> template<typename QuatDerived> -AngleAxis<Scalar>& AngleAxis<Scalar>::operator=(const QuaternionBase<QuatDerived>& q) +EIGEN_DEVICE_FUNC AngleAxis<Scalar>& AngleAxis<Scalar>::operator=(const QuaternionBase<QuatDerived>& q) { - using std::atan2; + EIGEN_USING_STD_MATH(atan2) + EIGEN_USING_STD_MATH(abs) Scalar n = q.vec().norm(); if(n<NumTraits<Scalar>::epsilon()) n = q.vec().stableNorm(); - if (n > Scalar(0)) + + if (n != Scalar(0)) { - m_angle = Scalar(2)*atan2(n, q.w()); + m_angle = Scalar(2)*atan2(n, abs(q.w())); + if(q.w() < Scalar(0)) + n = -n; m_axis = q.vec() / n; } else @@ -187,7 +194,7 @@ */ template<typename Scalar> template<typename Derived> -AngleAxis<Scalar>& AngleAxis<Scalar>::operator=(const MatrixBase<Derived>& mat) +EIGEN_DEVICE_FUNC AngleAxis<Scalar>& AngleAxis<Scalar>::operator=(const MatrixBase<Derived>& mat) { // Since a direct conversion would not be really faster, // let's use the robust Quaternion implementation: @@ -199,7 +206,7 @@ **/ template<typename Scalar> template<typename Derived> -AngleAxis<Scalar>& AngleAxis<Scalar>::fromRotationMatrix(const MatrixBase<Derived>& mat) +EIGEN_DEVICE_FUNC AngleAxis<Scalar>& AngleAxis<Scalar>::fromRotationMatrix(const MatrixBase<Derived>& mat) { return *this = QuaternionType(mat); } @@ -208,10 +215,10 @@ */ template<typename Scalar> typename AngleAxis<Scalar>::Matrix3 -AngleAxis<Scalar>::toRotationMatrix(void) const +EIGEN_DEVICE_FUNC AngleAxis<Scalar>::toRotationMatrix(void) const { - using std::sin; - using std::cos; + EIGEN_USING_STD_MATH(sin) + EIGEN_USING_STD_MATH(cos) Matrix3 res; Vector3 sin_axis = sin(m_angle) * m_axis; Scalar c = cos(m_angle);
diff --git a/Eigen/src/Geometry/CMakeLists.txt b/Eigen/src/Geometry/CMakeLists.txt deleted file mode 100644 index f8f728b..0000000 --- a/Eigen/src/Geometry/CMakeLists.txt +++ /dev/null
@@ -1,8 +0,0 @@ -FILE(GLOB Eigen_Geometry_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Geometry_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry COMPONENT Devel - ) - -ADD_SUBDIRECTORY(arch)
diff --git a/Eigen/src/Geometry/EulerAngles.h b/Eigen/src/Geometry/EulerAngles.h index b875b7a..c633268 100644 --- a/Eigen/src/Geometry/EulerAngles.h +++ b/Eigen/src/Geometry/EulerAngles.h
@@ -33,12 +33,12 @@ * \sa class AngleAxis */ template<typename Derived> -inline Matrix<typename MatrixBase<Derived>::Scalar,3,1> +EIGEN_DEVICE_FUNC inline Matrix<typename MatrixBase<Derived>::Scalar,3,1> MatrixBase<Derived>::eulerAngles(Index a0, Index a1, Index a2) const { - using std::atan2; - using std::sin; - using std::cos; + EIGEN_USING_STD_MATH(atan2) + EIGEN_USING_STD_MATH(sin) + EIGEN_USING_STD_MATH(cos) /* Implemented from Graphics Gems IV */ EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Derived,3,3) @@ -55,7 +55,12 @@ res[0] = atan2(coeff(j,i), coeff(k,i)); if((odd && res[0]<Scalar(0)) || ((!odd) && res[0]>Scalar(0))) { - res[0] = (res[0] > Scalar(0)) ? res[0] - Scalar(EIGEN_PI) : res[0] + Scalar(EIGEN_PI); + if(res[0] > Scalar(0)) { + res[0] -= Scalar(EIGEN_PI); + } + else { + res[0] += Scalar(EIGEN_PI); + } Scalar s2 = Vector2(coeff(j,i), coeff(k,i)).norm(); res[1] = -atan2(s2, coeff(i,i)); } @@ -84,7 +89,12 @@ res[0] = atan2(coeff(j,k), coeff(k,k)); Scalar c2 = Vector2(coeff(i,i), coeff(i,j)).norm(); if((odd && res[0]<Scalar(0)) || ((!odd) && res[0]>Scalar(0))) { - res[0] = (res[0] > Scalar(0)) ? res[0] - Scalar(EIGEN_PI) : res[0] + Scalar(EIGEN_PI); + if(res[0] > Scalar(0)) { + res[0] -= Scalar(EIGEN_PI); + } + else { + res[0] += Scalar(EIGEN_PI); + } res[1] = atan2(-coeff(i,k), -c2); } else
diff --git a/Eigen/src/Geometry/Homogeneous.h b/Eigen/src/Geometry/Homogeneous.h index cd52b54..5f0da1a 100644 --- a/Eigen/src/Geometry/Homogeneous.h +++ b/Eigen/src/Geometry/Homogeneous.h
@@ -68,17 +68,17 @@ typedef MatrixBase<Homogeneous> Base; EIGEN_DENSE_PUBLIC_INTERFACE(Homogeneous) - explicit inline Homogeneous(const MatrixType& matrix) + EIGEN_DEVICE_FUNC explicit inline Homogeneous(const MatrixType& matrix) : m_matrix(matrix) {} - inline Index rows() const { return m_matrix.rows() + (int(Direction)==Vertical ? 1 : 0); } - inline Index cols() const { return m_matrix.cols() + (int(Direction)==Horizontal ? 1 : 0); } + EIGEN_DEVICE_FUNC inline Index rows() const { return m_matrix.rows() + (int(Direction)==Vertical ? 1 : 0); } + EIGEN_DEVICE_FUNC inline Index cols() const { return m_matrix.cols() + (int(Direction)==Horizontal ? 1 : 0); } - const NestedExpression& nestedExpression() const { return m_matrix; } + EIGEN_DEVICE_FUNC const NestedExpression& nestedExpression() const { return m_matrix; } template<typename Rhs> - inline const Product<Homogeneous,Rhs> + EIGEN_DEVICE_FUNC inline const Product<Homogeneous,Rhs> operator* (const MatrixBase<Rhs>& rhs) const { eigen_assert(int(Direction)==Horizontal); @@ -86,7 +86,7 @@ } template<typename Lhs> friend - inline const Product<Lhs,Homogeneous> + EIGEN_DEVICE_FUNC inline const Product<Lhs,Homogeneous> operator* (const MatrixBase<Lhs>& lhs, const Homogeneous& rhs) { eigen_assert(int(Direction)==Vertical); @@ -94,7 +94,7 @@ } template<typename Scalar, int Dim, int Mode, int Options> friend - inline const Product<Transform<Scalar,Dim,Mode,Options>, Homogeneous > + EIGEN_DEVICE_FUNC inline const Product<Transform<Scalar,Dim,Mode,Options>, Homogeneous > operator* (const Transform<Scalar,Dim,Mode,Options>& lhs, const Homogeneous& rhs) { eigen_assert(int(Direction)==Vertical); @@ -102,7 +102,7 @@ } template<typename Func> - EIGEN_STRONG_INLINE typename internal::result_of<Func(Scalar,Scalar)>::type + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::result_of<Func(Scalar,Scalar)>::type redux(const Func& func) const { return func(m_matrix.redux(func), Scalar(1)); @@ -114,7 +114,9 @@ /** \geometry_module \ingroup Geometry_Module * - * \return an expression of the equivalent homogeneous vector + * \returns a vector expression that is one longer than the vector argument, with the value 1 symbolically appended as the last coefficient. + * + * This can be used to convert affine coordinates to homogeneous coordinates. * * \only_for_vectors * @@ -124,7 +126,7 @@ * \sa VectorwiseOp::homogeneous(), class Homogeneous */ template<typename Derived> -inline typename MatrixBase<Derived>::HomogeneousReturnType +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::HomogeneousReturnType MatrixBase<Derived>::homogeneous() const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); @@ -133,14 +135,16 @@ /** \geometry_module \ingroup Geometry_Module * - * \returns a matrix expression of homogeneous column (or row) vectors + * \returns an expression where the value 1 is symbolically appended as the final coefficient to each column (or row) of the matrix. + * + * This can be used to convert affine coordinates to homogeneous coordinates. * * Example: \include VectorwiseOp_homogeneous.cpp * Output: \verbinclude VectorwiseOp_homogeneous.out * * \sa MatrixBase::homogeneous(), class Homogeneous */ template<typename ExpressionType, int Direction> -inline Homogeneous<ExpressionType,Direction> +EIGEN_DEVICE_FUNC inline Homogeneous<ExpressionType,Direction> VectorwiseOp<ExpressionType,Direction>::homogeneous() const { return HomogeneousReturnType(_expression()); @@ -148,14 +152,23 @@ /** \geometry_module \ingroup Geometry_Module * - * \returns an expression of the homogeneous normalized vector of \c *this + * \brief homogeneous normalization + * + * \returns a vector expression of the N-1 first coefficients of \c *this divided by that last coefficient. + * + * This can be used to convert homogeneous coordinates to affine coordinates. + * + * It is essentially a shortcut for: + * \code + this->head(this->size()-1)/this->coeff(this->size()-1); + \endcode * * Example: \include MatrixBase_hnormalized.cpp * Output: \verbinclude MatrixBase_hnormalized.out * * \sa VectorwiseOp::hnormalized() */ template<typename Derived> -inline const typename MatrixBase<Derived>::HNormalizedReturnType +EIGEN_DEVICE_FUNC inline const typename MatrixBase<Derived>::HNormalizedReturnType MatrixBase<Derived>::hnormalized() const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); @@ -166,14 +179,20 @@ /** \geometry_module \ingroup Geometry_Module * - * \returns an expression of the homogeneous normalized vector of \c *this + * \brief column or row-wise homogeneous normalization + * + * \returns an expression of the first N-1 coefficients of each column (or row) of \c *this divided by the last coefficient of each column (or row). + * + * This can be used to convert homogeneous coordinates to affine coordinates. + * + * It is conceptually equivalent to calling MatrixBase::hnormalized() to each column (or row) of \c *this. * * Example: \include DirectionWise_hnormalized.cpp * Output: \verbinclude DirectionWise_hnormalized.out * * \sa MatrixBase::hnormalized() */ template<typename ExpressionType, int Direction> -inline const typename VectorwiseOp<ExpressionType,Direction>::HNormalizedReturnType +EIGEN_DEVICE_FUNC inline const typename VectorwiseOp<ExpressionType,Direction>::HNormalizedReturnType VectorwiseOp<ExpressionType,Direction>::hnormalized() const { return HNormalized_Block(_expression(),0,0, @@ -197,7 +216,7 @@ struct take_matrix_for_product { typedef MatrixOrTransformType type; - static const type& run(const type &x) { return x; } + EIGEN_DEVICE_FUNC static const type& run(const type &x) { return x; } }; template<typename Scalar, int Dim, int Mode,int Options> @@ -205,7 +224,7 @@ { typedef Transform<Scalar, Dim, Mode, Options> TransformType; typedef typename internal::add_const<typename TransformType::ConstAffinePart>::type type; - static type run (const TransformType& x) { return x.affine(); } + EIGEN_DEVICE_FUNC static type run (const TransformType& x) { return x.affine(); } }; template<typename Scalar, int Dim, int Options> @@ -213,7 +232,7 @@ { typedef Transform<Scalar, Dim, Projective, Options> TransformType; typedef typename TransformType::MatrixType type; - static const type& run (const TransformType& x) { return x.matrix(); } + EIGEN_DEVICE_FUNC static const type& run (const TransformType& x) { return x.matrix(); } }; template<typename MatrixType,typename Lhs> @@ -238,15 +257,15 @@ typedef typename traits<homogeneous_left_product_impl>::LhsMatrixType LhsMatrixType; typedef typename remove_all<LhsMatrixType>::type LhsMatrixTypeCleaned; typedef typename remove_all<typename LhsMatrixTypeCleaned::Nested>::type LhsMatrixTypeNested; - homogeneous_left_product_impl(const Lhs& lhs, const MatrixType& rhs) + EIGEN_DEVICE_FUNC homogeneous_left_product_impl(const Lhs& lhs, const MatrixType& rhs) : m_lhs(take_matrix_for_product<Lhs>::run(lhs)), m_rhs(rhs) {} - inline Index rows() const { return m_lhs.rows(); } - inline Index cols() const { return m_rhs.cols(); } + EIGEN_DEVICE_FUNC inline Index rows() const { return m_lhs.rows(); } + EIGEN_DEVICE_FUNC inline Index cols() const { return m_rhs.cols(); } - template<typename Dest> void evalTo(Dest& dst) const + template<typename Dest> EIGEN_DEVICE_FUNC void evalTo(Dest& dst) const { // FIXME investigate how to allow lazy evaluation of this product when possible dst = Block<const LhsMatrixTypeNested, @@ -277,14 +296,14 @@ : public ReturnByValue<homogeneous_right_product_impl<Homogeneous<MatrixType,Horizontal>,Rhs> > { typedef typename remove_all<typename Rhs::Nested>::type RhsNested; - homogeneous_right_product_impl(const MatrixType& lhs, const Rhs& rhs) + EIGEN_DEVICE_FUNC homogeneous_right_product_impl(const MatrixType& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) {} - inline Index rows() const { return m_lhs.rows(); } - inline Index cols() const { return m_rhs.cols(); } + EIGEN_DEVICE_FUNC inline Index rows() const { return m_lhs.rows(); } + EIGEN_DEVICE_FUNC inline Index cols() const { return m_rhs.cols(); } - template<typename Dest> void evalTo(Dest& dst) const + template<typename Dest> EIGEN_DEVICE_FUNC void evalTo(Dest& dst) const { // FIXME investigate how to allow lazy evaluation of this product when possible dst = m_lhs * Block<const RhsNested, @@ -317,7 +336,7 @@ typedef typename XprType::PlainObject PlainObject; typedef evaluator<PlainObject> Base; - explicit unary_evaluator(const XprType& op) + EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) : Base(), m_temp(op) { ::new (static_cast<Base*>(this)) Base(m_temp); @@ -329,11 +348,16 @@ // dense = homogeneous template< typename DstXprType, typename ArgType, typename Scalar> -struct Assignment<DstXprType, Homogeneous<ArgType,Vertical>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, Homogeneous<ArgType,Vertical>, internal::assign_op<Scalar,typename ArgType::Scalar>, Dense2Dense> { typedef Homogeneous<ArgType,Vertical> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,typename ArgType::Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + dst.template topRows<ArgType::RowsAtCompileTime>(src.nestedExpression().rows()) = src.nestedExpression(); dst.row(dst.rows()-1).setOnes(); } @@ -341,11 +365,16 @@ // dense = homogeneous template< typename DstXprType, typename ArgType, typename Scalar> -struct Assignment<DstXprType, Homogeneous<ArgType,Horizontal>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, Homogeneous<ArgType,Horizontal>, internal::assign_op<Scalar,typename ArgType::Scalar>, Dense2Dense> { typedef Homogeneous<ArgType,Horizontal> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,typename ArgType::Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + dst.template leftCols<ArgType::ColsAtCompileTime>(src.nestedExpression().cols()) = src.nestedExpression(); dst.col(dst.cols()-1).setOnes(); } @@ -355,7 +384,7 @@ struct generic_product_impl<Homogeneous<LhsArg,Horizontal>, Rhs, HomogeneousShape, DenseShape, ProductTag> { template<typename Dest> - static void evalTo(Dest& dst, const Homogeneous<LhsArg,Horizontal>& lhs, const Rhs& rhs) + EIGEN_DEVICE_FUNC static void evalTo(Dest& dst, const Homogeneous<LhsArg,Horizontal>& lhs, const Rhs& rhs) { homogeneous_right_product_impl<Homogeneous<LhsArg,Horizontal>, Rhs>(lhs.nestedExpression(), rhs).evalTo(dst); } @@ -373,7 +402,7 @@ typedef typename Rhs::ConstRowXpr ConstantColumn; typedef Replicate<const ConstantColumn,Rows,1> ConstantBlock; typedef Product<Lhs,LinearBlock,LazyProduct> LinearProduct; - typedef CwiseBinaryOp<internal::scalar_sum_op<typename Lhs::Scalar>, const LinearProduct, const ConstantBlock> Xpr; + typedef CwiseBinaryOp<internal::scalar_sum_op<typename Lhs::Scalar,typename Rhs::Scalar>, const LinearProduct, const ConstantBlock> Xpr; }; template<typename Lhs, typename Rhs, int ProductTag> @@ -396,12 +425,24 @@ struct generic_product_impl<Lhs, Homogeneous<RhsArg,Vertical>, DenseShape, HomogeneousShape, ProductTag> { template<typename Dest> - static void evalTo(Dest& dst, const Lhs& lhs, const Homogeneous<RhsArg,Vertical>& rhs) + EIGEN_DEVICE_FUNC static void evalTo(Dest& dst, const Lhs& lhs, const Homogeneous<RhsArg,Vertical>& rhs) { homogeneous_left_product_impl<Homogeneous<RhsArg,Vertical>, Lhs>(lhs, rhs.nestedExpression()).evalTo(dst); } }; +// TODO: the following specialization is to address a regression from 3.2 to 3.3 +// In the future, this path should be optimized. +template<typename Lhs, typename RhsArg, int ProductTag> +struct generic_product_impl<Lhs, Homogeneous<RhsArg,Vertical>, TriangularShape, HomogeneousShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Homogeneous<RhsArg,Vertical>& rhs) + { + dst.noalias() = lhs * rhs.eval(); + } +}; + template<typename Lhs,typename Rhs> struct homogeneous_left_product_refactoring_helper { @@ -414,7 +455,7 @@ typedef typename Lhs::ConstColXpr ConstantColumn; typedef Replicate<const ConstantColumn,1,Cols> ConstantBlock; typedef Product<LinearBlock,Rhs,LazyProduct> LinearProduct; - typedef CwiseBinaryOp<internal::scalar_sum_op<typename Lhs::Scalar>, const LinearProduct, const ConstantBlock> Xpr; + typedef CwiseBinaryOp<internal::scalar_sum_op<typename Lhs::Scalar,typename Rhs::Scalar>, const LinearProduct, const ConstantBlock> Xpr; }; template<typename Lhs, typename Rhs, int ProductTag> @@ -438,7 +479,7 @@ { typedef Transform<Scalar,Dim,Mode,Options> TransformType; template<typename Dest> - static void evalTo(Dest& dst, const TransformType& lhs, const Homogeneous<RhsArg,Vertical>& rhs) + EIGEN_DEVICE_FUNC static void evalTo(Dest& dst, const TransformType& lhs, const Homogeneous<RhsArg,Vertical>& rhs) { homogeneous_left_product_impl<Homogeneous<RhsArg,Vertical>, TransformType>(lhs, rhs.nestedExpression()).evalTo(dst); }
diff --git a/Eigen/src/Geometry/Hyperplane.h b/Eigen/src/Geometry/Hyperplane.h index cc89639..cebe035 100644 --- a/Eigen/src/Geometry/Hyperplane.h +++ b/Eigen/src/Geometry/Hyperplane.h
@@ -50,21 +50,21 @@ typedef const Block<const Coefficients,AmbientDimAtCompileTime,1> ConstNormalReturnType; /** Default constructor without initialization */ - inline Hyperplane() {} + EIGEN_DEVICE_FUNC inline Hyperplane() {} template<int OtherOptions> - Hyperplane(const Hyperplane<Scalar,AmbientDimAtCompileTime,OtherOptions>& other) + EIGEN_DEVICE_FUNC Hyperplane(const Hyperplane<Scalar,AmbientDimAtCompileTime,OtherOptions>& other) : m_coeffs(other.coeffs()) {} /** Constructs a dynamic-size hyperplane with \a _dim the dimension * of the ambient space */ - inline explicit Hyperplane(Index _dim) : m_coeffs(_dim+1) {} + EIGEN_DEVICE_FUNC inline explicit Hyperplane(Index _dim) : m_coeffs(_dim+1) {} /** Construct a plane from its normal \a n and a point \a e onto the plane. * \warning the vector normal is assumed to be normalized. */ - inline Hyperplane(const VectorType& n, const VectorType& e) + EIGEN_DEVICE_FUNC inline Hyperplane(const VectorType& n, const VectorType& e) : m_coeffs(n.size()+1) { normal() = n; @@ -75,7 +75,7 @@ * such that the algebraic equation of the plane is \f$ n \cdot x + d = 0 \f$. * \warning the vector normal is assumed to be normalized. */ - inline Hyperplane(const VectorType& n, const Scalar& d) + EIGEN_DEVICE_FUNC inline Hyperplane(const VectorType& n, const Scalar& d) : m_coeffs(n.size()+1) { normal() = n; @@ -85,7 +85,7 @@ /** Constructs a hyperplane passing through the two points. If the dimension of the ambient space * is greater than 2, then there isn't uniqueness, so an arbitrary choice is made. */ - static inline Hyperplane Through(const VectorType& p0, const VectorType& p1) + EIGEN_DEVICE_FUNC static inline Hyperplane Through(const VectorType& p0, const VectorType& p1) { Hyperplane result(p0.size()); result.normal() = (p1 - p0).unitOrthogonal(); @@ -96,7 +96,7 @@ /** Constructs a hyperplane passing through the three points. The dimension of the ambient space * is required to be exactly 3. */ - static inline Hyperplane Through(const VectorType& p0, const VectorType& p1, const VectorType& p2) + EIGEN_DEVICE_FUNC static inline Hyperplane Through(const VectorType& p0, const VectorType& p1, const VectorType& p2) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(VectorType, 3) Hyperplane result(p0.size()); @@ -119,20 +119,20 @@ * If the dimension of the ambient space is greater than 2, then there isn't uniqueness, * so an arbitrary choice is made. */ - // FIXME to be consitent with the rest this could be implemented as a static Through function ?? - explicit Hyperplane(const ParametrizedLine<Scalar, AmbientDimAtCompileTime>& parametrized) + // FIXME to be consistent with the rest this could be implemented as a static Through function ?? + EIGEN_DEVICE_FUNC explicit Hyperplane(const ParametrizedLine<Scalar, AmbientDimAtCompileTime>& parametrized) { normal() = parametrized.direction().unitOrthogonal(); offset() = -parametrized.origin().dot(normal()); } - ~Hyperplane() {} + EIGEN_DEVICE_FUNC ~Hyperplane() {} /** \returns the dimension in which the plane holds */ - inline Index dim() const { return AmbientDimAtCompileTime==Dynamic ? m_coeffs.size()-1 : Index(AmbientDimAtCompileTime); } + EIGEN_DEVICE_FUNC inline Index dim() const { return AmbientDimAtCompileTime==Dynamic ? m_coeffs.size()-1 : Index(AmbientDimAtCompileTime); } /** normalizes \c *this */ - void normalize(void) + EIGEN_DEVICE_FUNC void normalize(void) { m_coeffs /= normal().norm(); } @@ -140,45 +140,45 @@ /** \returns the signed distance between the plane \c *this and a point \a p. * \sa absDistance() */ - inline Scalar signedDistance(const VectorType& p) const { return normal().dot(p) + offset(); } + EIGEN_DEVICE_FUNC inline Scalar signedDistance(const VectorType& p) const { return normal().dot(p) + offset(); } /** \returns the absolute distance between the plane \c *this and a point \a p. * \sa signedDistance() */ - inline Scalar absDistance(const VectorType& p) const { using std::abs; return abs(signedDistance(p)); } + EIGEN_DEVICE_FUNC inline Scalar absDistance(const VectorType& p) const { return numext::abs(signedDistance(p)); } /** \returns the projection of a point \a p onto the plane \c *this. */ - inline VectorType projection(const VectorType& p) const { return p - signedDistance(p) * normal(); } + EIGEN_DEVICE_FUNC inline VectorType projection(const VectorType& p) const { return p - signedDistance(p) * normal(); } /** \returns a constant reference to the unit normal vector of the plane, which corresponds * to the linear part of the implicit equation. */ - inline ConstNormalReturnType normal() const { return ConstNormalReturnType(m_coeffs,0,0,dim(),1); } + EIGEN_DEVICE_FUNC inline ConstNormalReturnType normal() const { return ConstNormalReturnType(m_coeffs,0,0,dim(),1); } /** \returns a non-constant reference to the unit normal vector of the plane, which corresponds * to the linear part of the implicit equation. */ - inline NormalReturnType normal() { return NormalReturnType(m_coeffs,0,0,dim(),1); } + EIGEN_DEVICE_FUNC inline NormalReturnType normal() { return NormalReturnType(m_coeffs,0,0,dim(),1); } /** \returns the distance to the origin, which is also the "constant term" of the implicit equation * \warning the vector normal is assumed to be normalized. */ - inline const Scalar& offset() const { return m_coeffs.coeff(dim()); } + EIGEN_DEVICE_FUNC inline const Scalar& offset() const { return m_coeffs.coeff(dim()); } /** \returns a non-constant reference to the distance to the origin, which is also the constant part * of the implicit equation */ - inline Scalar& offset() { return m_coeffs(dim()); } + EIGEN_DEVICE_FUNC inline Scalar& offset() { return m_coeffs(dim()); } /** \returns a constant reference to the coefficients c_i of the plane equation: * \f$ c_0*x_0 + ... + c_{d-1}*x_{d-1} + c_d = 0 \f$ */ - inline const Coefficients& coeffs() const { return m_coeffs; } + EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs; } /** \returns a non-constant reference to the coefficients c_i of the plane equation: * \f$ c_0*x_0 + ... + c_{d-1}*x_{d-1} + c_d = 0 \f$ */ - inline Coefficients& coeffs() { return m_coeffs; } + EIGEN_DEVICE_FUNC inline Coefficients& coeffs() { return m_coeffs; } /** \returns the intersection of *this with \a other. * @@ -186,16 +186,15 @@ * * \note If \a other is approximately parallel to *this, this method will return any point on *this. */ - VectorType intersection(const Hyperplane& other) const + EIGEN_DEVICE_FUNC VectorType intersection(const Hyperplane& other) const { - using std::abs; EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(VectorType, 2) Scalar det = coeffs().coeff(0) * other.coeffs().coeff(1) - coeffs().coeff(1) * other.coeffs().coeff(0); // since the line equations ax+by=c are normalized with a^2+b^2=1, the following tests // whether the two lines are approximately parallel. if(internal::isMuchSmallerThan(det, Scalar(1))) { // special case where the two lines are approximately parallel. Pick any point on the first line. - if(abs(coeffs().coeff(1))>abs(coeffs().coeff(0))) + if(numext::abs(coeffs().coeff(1))>numext::abs(coeffs().coeff(0))) return VectorType(coeffs().coeff(1), -coeffs().coeff(2)/coeffs().coeff(1)-coeffs().coeff(0)); else return VectorType(-coeffs().coeff(2)/coeffs().coeff(0)-coeffs().coeff(1), coeffs().coeff(0)); @@ -215,10 +214,13 @@ * or a more generic #Affine transformation. The default is #Affine. */ template<typename XprType> - inline Hyperplane& transform(const MatrixBase<XprType>& mat, TransformTraits traits = Affine) + EIGEN_DEVICE_FUNC inline Hyperplane& transform(const MatrixBase<XprType>& mat, TransformTraits traits = Affine) { if (traits==Affine) + { normal() = mat.inverse().transpose() * normal(); + m_coeffs /= normal().norm(); + } else if (traits==Isometry) normal() = mat * normal(); else @@ -236,7 +238,7 @@ * Other kind of transformations are not supported. */ template<int TrOptions> - inline Hyperplane& transform(const Transform<Scalar,AmbientDimAtCompileTime,Affine,TrOptions>& t, + EIGEN_DEVICE_FUNC inline Hyperplane& transform(const Transform<Scalar,AmbientDimAtCompileTime,Affine,TrOptions>& t, TransformTraits traits = Affine) { transform(t.linear(), traits); @@ -250,7 +252,7 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<Hyperplane, + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<Hyperplane, Hyperplane<NewScalarType,AmbientDimAtCompileTime,Options> >::type cast() const { return typename internal::cast_return_type<Hyperplane, @@ -259,7 +261,7 @@ /** Copy constructor with scalar type conversion */ template<typename OtherScalarType,int OtherOptions> - inline explicit Hyperplane(const Hyperplane<OtherScalarType,AmbientDimAtCompileTime,OtherOptions>& other) + EIGEN_DEVICE_FUNC inline explicit Hyperplane(const Hyperplane<OtherScalarType,AmbientDimAtCompileTime,OtherOptions>& other) { m_coeffs = other.coeffs().template cast<Scalar>(); } /** \returns \c true if \c *this is approximately equal to \a other, within the precision @@ -267,7 +269,7 @@ * * \sa MatrixBase::isApprox() */ template<int OtherOptions> - bool isApprox(const Hyperplane<Scalar,AmbientDimAtCompileTime,OtherOptions>& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const Hyperplane<Scalar,AmbientDimAtCompileTime,OtherOptions>& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return m_coeffs.isApprox(other.m_coeffs, prec); } protected:
diff --git a/Eigen/src/Geometry/OrthoMethods.h b/Eigen/src/Geometry/OrthoMethods.h index c3648f5..524aebe 100644 --- a/Eigen/src/Geometry/OrthoMethods.h +++ b/Eigen/src/Geometry/OrthoMethods.h
@@ -27,9 +27,10 @@ template<typename Derived> template<typename OtherDerived> #ifndef EIGEN_PARSED_BY_DOXYGEN -inline typename MatrixBase<Derived>::template cross_product_return_type<OtherDerived>::type +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename MatrixBase<Derived>::template cross_product_return_type<OtherDerived>::type #else -inline typename MatrixBase<Derived>::PlainObject +typename MatrixBase<Derived>::PlainObject #endif MatrixBase<Derived>::cross(const MatrixBase<OtherDerived>& other) const { @@ -53,7 +54,7 @@ typename Scalar = typename VectorLhs::Scalar, bool Vectorizable = bool((VectorLhs::Flags&VectorRhs::Flags)&PacketAccessBit)> struct cross3_impl { - static inline typename internal::plain_matrix_type<VectorLhs>::type + EIGEN_DEVICE_FUNC static inline typename internal::plain_matrix_type<VectorLhs>::type run(const VectorLhs& lhs, const VectorRhs& rhs) { return typename internal::plain_matrix_type<VectorLhs>::type( @@ -78,7 +79,7 @@ */ template<typename Derived> template<typename OtherDerived> -inline typename MatrixBase<Derived>::PlainObject +EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::cross3(const MatrixBase<OtherDerived>& other) const { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived,4) @@ -105,6 +106,7 @@ * \sa MatrixBase::cross() */ template<typename ExpressionType, int Direction> template<typename OtherDerived> +EIGEN_DEVICE_FUNC const typename VectorwiseOp<ExpressionType,Direction>::CrossReturnType VectorwiseOp<ExpressionType,Direction>::cross(const MatrixBase<OtherDerived>& other) const { @@ -221,7 +223,7 @@ * \sa cross() */ template<typename Derived> -typename MatrixBase<Derived>::PlainObject +EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::unitOrthogonal() const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
diff --git a/Eigen/src/Geometry/ParametrizedLine.h b/Eigen/src/Geometry/ParametrizedLine.h index c43dce7..3929ca8 100644 --- a/Eigen/src/Geometry/ParametrizedLine.h +++ b/Eigen/src/Geometry/ParametrizedLine.h
@@ -41,45 +41,45 @@ typedef Matrix<Scalar,AmbientDimAtCompileTime,1,Options> VectorType; /** Default constructor without initialization */ - inline ParametrizedLine() {} + EIGEN_DEVICE_FUNC inline ParametrizedLine() {} template<int OtherOptions> - ParametrizedLine(const ParametrizedLine<Scalar,AmbientDimAtCompileTime,OtherOptions>& other) + EIGEN_DEVICE_FUNC ParametrizedLine(const ParametrizedLine<Scalar,AmbientDimAtCompileTime,OtherOptions>& other) : m_origin(other.origin()), m_direction(other.direction()) {} /** Constructs a dynamic-size line with \a _dim the dimension * of the ambient space */ - inline explicit ParametrizedLine(Index _dim) : m_origin(_dim), m_direction(_dim) {} + EIGEN_DEVICE_FUNC inline explicit ParametrizedLine(Index _dim) : m_origin(_dim), m_direction(_dim) {} /** Initializes a parametrized line of direction \a direction and origin \a origin. * \warning the vector direction is assumed to be normalized. */ - ParametrizedLine(const VectorType& origin, const VectorType& direction) + EIGEN_DEVICE_FUNC ParametrizedLine(const VectorType& origin, const VectorType& direction) : m_origin(origin), m_direction(direction) {} template <int OtherOptions> - explicit ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane); + EIGEN_DEVICE_FUNC explicit ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane); /** Constructs a parametrized line going from \a p0 to \a p1. */ - static inline ParametrizedLine Through(const VectorType& p0, const VectorType& p1) + EIGEN_DEVICE_FUNC static inline ParametrizedLine Through(const VectorType& p0, const VectorType& p1) { return ParametrizedLine(p0, (p1-p0).normalized()); } - ~ParametrizedLine() {} + EIGEN_DEVICE_FUNC ~ParametrizedLine() {} /** \returns the dimension in which the line holds */ - inline Index dim() const { return m_direction.size(); } + EIGEN_DEVICE_FUNC inline Index dim() const { return m_direction.size(); } - const VectorType& origin() const { return m_origin; } - VectorType& origin() { return m_origin; } + EIGEN_DEVICE_FUNC const VectorType& origin() const { return m_origin; } + EIGEN_DEVICE_FUNC VectorType& origin() { return m_origin; } - const VectorType& direction() const { return m_direction; } - VectorType& direction() { return m_direction; } + EIGEN_DEVICE_FUNC const VectorType& direction() const { return m_direction; } + EIGEN_DEVICE_FUNC VectorType& direction() { return m_direction; } /** \returns the squared distance of a point \a p to its projection onto the line \c *this. * \sa distance() */ - RealScalar squaredDistance(const VectorType& p) const + EIGEN_DEVICE_FUNC RealScalar squaredDistance(const VectorType& p) const { VectorType diff = p - origin(); return (diff - direction().dot(diff) * direction()).squaredNorm(); @@ -87,30 +87,67 @@ /** \returns the distance of a point \a p to its projection onto the line \c *this. * \sa squaredDistance() */ - RealScalar distance(const VectorType& p) const { using std::sqrt; return sqrt(squaredDistance(p)); } + EIGEN_DEVICE_FUNC RealScalar distance(const VectorType& p) const { EIGEN_USING_STD_MATH(sqrt) return sqrt(squaredDistance(p)); } /** \returns the projection of a point \a p onto the line \c *this. */ - VectorType projection(const VectorType& p) const + EIGEN_DEVICE_FUNC VectorType projection(const VectorType& p) const { return origin() + direction().dot(p-origin()) * direction(); } - VectorType pointAt(const Scalar& t) const; + EIGEN_DEVICE_FUNC VectorType pointAt(const Scalar& t) const; template <int OtherOptions> - Scalar intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; + EIGEN_DEVICE_FUNC Scalar intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; template <int OtherOptions> - Scalar intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; + EIGEN_DEVICE_FUNC Scalar intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; template <int OtherOptions> - VectorType intersectionPoint(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; + EIGEN_DEVICE_FUNC VectorType intersectionPoint(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const; - /** \returns \c *this with scalar type casted to \a NewScalarType + /** Applies the transformation matrix \a mat to \c *this and returns a reference to \c *this. + * + * \param mat the Dim x Dim transformation matrix + * \param traits specifies whether the matrix \a mat represents an #Isometry + * or a more generic #Affine transformation. The default is #Affine. + */ + template<typename XprType> + EIGEN_DEVICE_FUNC inline ParametrizedLine& transform(const MatrixBase<XprType>& mat, TransformTraits traits = Affine) + { + if (traits==Affine) + direction() = (mat * direction()).normalized(); + else if (traits==Isometry) + direction() = mat * direction(); + else + { + eigen_assert(0 && "invalid traits value in ParametrizedLine::transform()"); + } + origin() = mat * origin(); + return *this; + } + + /** Applies the transformation \a t to \c *this and returns a reference to \c *this. + * + * \param t the transformation of dimension Dim + * \param traits specifies whether the transformation \a t represents an #Isometry + * or a more generic #Affine transformation. The default is #Affine. + * Other kind of transformations are not supported. + */ + template<int TrOptions> + EIGEN_DEVICE_FUNC inline ParametrizedLine& transform(const Transform<Scalar,AmbientDimAtCompileTime,Affine,TrOptions>& t, + TransformTraits traits = Affine) + { + transform(t.linear(), traits); + origin() += t.translation(); + return *this; + } + +/** \returns \c *this with scalar type casted to \a NewScalarType * * Note that if \a NewScalarType is equal to the current scalar type of \c *this * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<ParametrizedLine, + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<ParametrizedLine, ParametrizedLine<NewScalarType,AmbientDimAtCompileTime,Options> >::type cast() const { return typename internal::cast_return_type<ParametrizedLine, @@ -119,7 +156,7 @@ /** Copy constructor with scalar type conversion */ template<typename OtherScalarType,int OtherOptions> - inline explicit ParametrizedLine(const ParametrizedLine<OtherScalarType,AmbientDimAtCompileTime,OtherOptions>& other) + EIGEN_DEVICE_FUNC inline explicit ParametrizedLine(const ParametrizedLine<OtherScalarType,AmbientDimAtCompileTime,OtherOptions>& other) { m_origin = other.origin().template cast<Scalar>(); m_direction = other.direction().template cast<Scalar>(); @@ -129,7 +166,7 @@ * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const ParametrizedLine& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const ParametrizedLine& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return m_origin.isApprox(other.m_origin, prec) && m_direction.isApprox(other.m_direction, prec); } protected: @@ -143,7 +180,7 @@ */ template <typename _Scalar, int _AmbientDim, int _Options> template <int OtherOptions> -inline ParametrizedLine<_Scalar, _AmbientDim,_Options>::ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim,OtherOptions>& hyperplane) +EIGEN_DEVICE_FUNC inline ParametrizedLine<_Scalar, _AmbientDim,_Options>::ParametrizedLine(const Hyperplane<_Scalar, _AmbientDim,OtherOptions>& hyperplane) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(VectorType, 2) direction() = hyperplane.normal().unitOrthogonal(); @@ -153,7 +190,7 @@ /** \returns the point at \a t along this line */ template <typename _Scalar, int _AmbientDim, int _Options> -inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType +EIGEN_DEVICE_FUNC inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType ParametrizedLine<_Scalar, _AmbientDim,_Options>::pointAt(const _Scalar& t) const { return origin() + (direction()*t); @@ -163,7 +200,7 @@ */ template <typename _Scalar, int _AmbientDim, int _Options> template <int OtherOptions> -inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const +EIGEN_DEVICE_FUNC inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersectionParameter(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const { return -(hyperplane.offset()+hyperplane.normal().dot(origin())) / hyperplane.normal().dot(direction()); @@ -175,7 +212,7 @@ */ template <typename _Scalar, int _AmbientDim, int _Options> template <int OtherOptions> -inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const +EIGEN_DEVICE_FUNC inline _Scalar ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersection(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const { return intersectionParameter(hyperplane); } @@ -184,7 +221,7 @@ */ template <typename _Scalar, int _AmbientDim, int _Options> template <int OtherOptions> -inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType +EIGEN_DEVICE_FUNC inline typename ParametrizedLine<_Scalar, _AmbientDim,_Options>::VectorType ParametrizedLine<_Scalar, _AmbientDim,_Options>::intersectionPoint(const Hyperplane<_Scalar, _AmbientDim, OtherOptions>& hyperplane) const { return pointAt(intersectionParameter(hyperplane));
diff --git a/Eigen/src/Geometry/Quaternion.h b/Eigen/src/Geometry/Quaternion.h index 32d1499..faea62f 100644 --- a/Eigen/src/Geometry/Quaternion.h +++ b/Eigen/src/Geometry/Quaternion.h
@@ -43,6 +43,11 @@ typedef typename internal::traits<Derived>::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename internal::traits<Derived>::Coefficients Coefficients; + typedef typename Coefficients::CoeffReturnType CoeffReturnType; + typedef typename internal::conditional<bool(internal::traits<Derived>::Flags&LvalueBit), + Scalar&, CoeffReturnType>::type NonConstCoeffReturnType; + + enum { Flags = Eigen::internal::traits<Derived>::Flags }; @@ -58,37 +63,37 @@ /** \returns the \c x coefficient */ - inline Scalar x() const { return this->derived().coeffs().coeff(0); } + EIGEN_DEVICE_FUNC inline CoeffReturnType x() const { return this->derived().coeffs().coeff(0); } /** \returns the \c y coefficient */ - inline Scalar y() const { return this->derived().coeffs().coeff(1); } + EIGEN_DEVICE_FUNC inline CoeffReturnType y() const { return this->derived().coeffs().coeff(1); } /** \returns the \c z coefficient */ - inline Scalar z() const { return this->derived().coeffs().coeff(2); } + EIGEN_DEVICE_FUNC inline CoeffReturnType z() const { return this->derived().coeffs().coeff(2); } /** \returns the \c w coefficient */ - inline Scalar w() const { return this->derived().coeffs().coeff(3); } + EIGEN_DEVICE_FUNC inline CoeffReturnType w() const { return this->derived().coeffs().coeff(3); } - /** \returns a reference to the \c x coefficient */ - inline Scalar& x() { return this->derived().coeffs().coeffRef(0); } - /** \returns a reference to the \c y coefficient */ - inline Scalar& y() { return this->derived().coeffs().coeffRef(1); } - /** \returns a reference to the \c z coefficient */ - inline Scalar& z() { return this->derived().coeffs().coeffRef(2); } - /** \returns a reference to the \c w coefficient */ - inline Scalar& w() { return this->derived().coeffs().coeffRef(3); } + /** \returns a reference to the \c x coefficient (if Derived is a non-const lvalue) */ + EIGEN_DEVICE_FUNC inline NonConstCoeffReturnType x() { return this->derived().coeffs().x(); } + /** \returns a reference to the \c y coefficient (if Derived is a non-const lvalue) */ + EIGEN_DEVICE_FUNC inline NonConstCoeffReturnType y() { return this->derived().coeffs().y(); } + /** \returns a reference to the \c z coefficient (if Derived is a non-const lvalue) */ + EIGEN_DEVICE_FUNC inline NonConstCoeffReturnType z() { return this->derived().coeffs().z(); } + /** \returns a reference to the \c w coefficient (if Derived is a non-const lvalue) */ + EIGEN_DEVICE_FUNC inline NonConstCoeffReturnType w() { return this->derived().coeffs().w(); } /** \returns a read-only vector expression of the imaginary part (x,y,z) */ - inline const VectorBlock<const Coefficients,3> vec() const { return coeffs().template head<3>(); } + EIGEN_DEVICE_FUNC inline const VectorBlock<const Coefficients,3> vec() const { return coeffs().template head<3>(); } /** \returns a vector expression of the imaginary part (x,y,z) */ - inline VectorBlock<Coefficients,3> vec() { return coeffs().template head<3>(); } + EIGEN_DEVICE_FUNC inline VectorBlock<Coefficients,3> vec() { return coeffs().template head<3>(); } /** \returns a read-only vector expression of the coefficients (x,y,z,w) */ - inline const typename internal::traits<Derived>::Coefficients& coeffs() const { return derived().coeffs(); } + EIGEN_DEVICE_FUNC inline const typename internal::traits<Derived>::Coefficients& coeffs() const { return derived().coeffs(); } /** \returns a vector expression of the coefficients (x,y,z,w) */ - inline typename internal::traits<Derived>::Coefficients& coeffs() { return derived().coeffs(); } + EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Coefficients& coeffs() { return derived().coeffs(); } - EIGEN_STRONG_INLINE QuaternionBase<Derived>& operator=(const QuaternionBase<Derived>& other); - template<class OtherDerived> EIGEN_STRONG_INLINE Derived& operator=(const QuaternionBase<OtherDerived>& other); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE QuaternionBase<Derived>& operator=(const QuaternionBase<Derived>& other); + template<class OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const QuaternionBase<OtherDerived>& other); // disabled this copy operator as it is giving very strange compilation errors when compiling // test_stdvector with GCC 4.4.2. This looks like a GCC bug though, so feel free to re-enable it if it's @@ -97,72 +102,72 @@ // Derived& operator=(const QuaternionBase& other) // { return operator=<Derived>(other); } - Derived& operator=(const AngleAxisType& aa); - template<class OtherDerived> Derived& operator=(const MatrixBase<OtherDerived>& m); + EIGEN_DEVICE_FUNC Derived& operator=(const AngleAxisType& aa); + template<class OtherDerived> EIGEN_DEVICE_FUNC Derived& operator=(const MatrixBase<OtherDerived>& m); /** \returns a quaternion representing an identity rotation * \sa MatrixBase::Identity() */ - static inline Quaternion<Scalar> Identity() { return Quaternion<Scalar>(Scalar(1), Scalar(0), Scalar(0), Scalar(0)); } + EIGEN_DEVICE_FUNC static inline Quaternion<Scalar> Identity() { return Quaternion<Scalar>(Scalar(1), Scalar(0), Scalar(0), Scalar(0)); } /** \sa QuaternionBase::Identity(), MatrixBase::setIdentity() */ - inline QuaternionBase& setIdentity() { coeffs() << Scalar(0), Scalar(0), Scalar(0), Scalar(1); return *this; } + EIGEN_DEVICE_FUNC inline QuaternionBase& setIdentity() { coeffs() << Scalar(0), Scalar(0), Scalar(0), Scalar(1); return *this; } /** \returns the squared norm of the quaternion's coefficients * \sa QuaternionBase::norm(), MatrixBase::squaredNorm() */ - inline Scalar squaredNorm() const { return coeffs().squaredNorm(); } + EIGEN_DEVICE_FUNC inline Scalar squaredNorm() const { return coeffs().squaredNorm(); } /** \returns the norm of the quaternion's coefficients * \sa QuaternionBase::squaredNorm(), MatrixBase::norm() */ - inline Scalar norm() const { return coeffs().norm(); } + EIGEN_DEVICE_FUNC inline Scalar norm() const { return coeffs().norm(); } /** Normalizes the quaternion \c *this * \sa normalized(), MatrixBase::normalize() */ - inline void normalize() { coeffs().normalize(); } + EIGEN_DEVICE_FUNC inline void normalize() { coeffs().normalize(); } /** \returns a normalized copy of \c *this * \sa normalize(), MatrixBase::normalized() */ - inline Quaternion<Scalar> normalized() const { return Quaternion<Scalar>(coeffs().normalized()); } + EIGEN_DEVICE_FUNC inline Quaternion<Scalar> normalized() const { return Quaternion<Scalar>(coeffs().normalized()); } /** \returns the dot product of \c *this and \a other * Geometrically speaking, the dot product of two unit quaternions * corresponds to the cosine of half the angle between the two rotations. * \sa angularDistance() */ - template<class OtherDerived> inline Scalar dot(const QuaternionBase<OtherDerived>& other) const { return coeffs().dot(other.coeffs()); } + template<class OtherDerived> EIGEN_DEVICE_FUNC inline Scalar dot(const QuaternionBase<OtherDerived>& other) const { return coeffs().dot(other.coeffs()); } - template<class OtherDerived> Scalar angularDistance(const QuaternionBase<OtherDerived>& other) const; + template<class OtherDerived> EIGEN_DEVICE_FUNC Scalar angularDistance(const QuaternionBase<OtherDerived>& other) const; /** \returns an equivalent 3x3 rotation matrix */ - Matrix3 toRotationMatrix() const; + EIGEN_DEVICE_FUNC Matrix3 toRotationMatrix() const; /** \returns the quaternion which transform \a a into \a b through a rotation */ template<typename Derived1, typename Derived2> - Derived& setFromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b); + EIGEN_DEVICE_FUNC Derived& setFromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b); - template<class OtherDerived> EIGEN_STRONG_INLINE Quaternion<Scalar> operator* (const QuaternionBase<OtherDerived>& q) const; - template<class OtherDerived> EIGEN_STRONG_INLINE Derived& operator*= (const QuaternionBase<OtherDerived>& q); + template<class OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Quaternion<Scalar> operator* (const QuaternionBase<OtherDerived>& q) const; + template<class OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*= (const QuaternionBase<OtherDerived>& q); /** \returns the quaternion describing the inverse rotation */ - Quaternion<Scalar> inverse() const; + EIGEN_DEVICE_FUNC Quaternion<Scalar> inverse() const; /** \returns the conjugated quaternion */ - Quaternion<Scalar> conjugate() const; + EIGEN_DEVICE_FUNC Quaternion<Scalar> conjugate() const; - template<class OtherDerived> Quaternion<Scalar> slerp(const Scalar& t, const QuaternionBase<OtherDerived>& other) const; + template<class OtherDerived> EIGEN_DEVICE_FUNC Quaternion<Scalar> slerp(const Scalar& t, const QuaternionBase<OtherDerived>& other) const; /** \returns \c true if \c *this is approximately equal to \a other, within the precision * determined by \a prec. * * \sa MatrixBase::isApprox() */ template<class OtherDerived> - bool isApprox(const QuaternionBase<OtherDerived>& other, const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const QuaternionBase<OtherDerived>& other, const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const { return coeffs().isApprox(other.coeffs(), prec); } /** return the result vector of \a v through the rotation*/ - EIGEN_STRONG_INLINE Vector3 _transformVector(const Vector3& v) const; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Vector3 _transformVector(const Vector3& v) const; /** \returns \c *this with scalar type casted to \a NewScalarType * @@ -170,7 +175,7 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<Derived,Quaternion<NewScalarType> >::type cast() const + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<Derived,Quaternion<NewScalarType> >::type cast() const { return typename internal::cast_return_type<Derived,Quaternion<NewScalarType> >::type(derived()); } @@ -239,7 +244,7 @@ typedef typename Base::AngleAxisType AngleAxisType; /** Default constructor leaving the quaternion uninitialized. */ - inline Quaternion() {} + EIGEN_DEVICE_FUNC inline Quaternion() {} /** Constructs and initializes the quaternion \f$ w+xi+yj+zk \f$ from * its four coefficients \a w, \a x, \a y and \a z. @@ -248,34 +253,57 @@ * while internally the coefficients are stored in the following order: * [\c x, \c y, \c z, \c w] */ - inline Quaternion(const Scalar& w, const Scalar& x, const Scalar& y, const Scalar& z) : m_coeffs(x, y, z, w){} + EIGEN_DEVICE_FUNC inline Quaternion(const Scalar& w, const Scalar& x, const Scalar& y, const Scalar& z) : m_coeffs(x, y, z, w){} /** Constructs and initialize a quaternion from the array data */ - explicit inline Quaternion(const Scalar* data) : m_coeffs(data) {} + EIGEN_DEVICE_FUNC explicit inline Quaternion(const Scalar* data) : m_coeffs(data) {} /** Copy constructor */ - template<class Derived> EIGEN_STRONG_INLINE Quaternion(const QuaternionBase<Derived>& other) { this->Base::operator=(other); } + template<class Derived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Quaternion(const QuaternionBase<Derived>& other) { this->Base::operator=(other); } /** Constructs and initializes a quaternion from the angle-axis \a aa */ - explicit inline Quaternion(const AngleAxisType& aa) { *this = aa; } + EIGEN_DEVICE_FUNC explicit inline Quaternion(const AngleAxisType& aa) { *this = aa; } /** Constructs and initializes a quaternion from either: * - a rotation matrix expression, * - a 4D vector expression representing quaternion coefficients. */ template<typename Derived> - explicit inline Quaternion(const MatrixBase<Derived>& other) { *this = other; } + EIGEN_DEVICE_FUNC explicit inline Quaternion(const MatrixBase<Derived>& other) { *this = other; } /** Explicit copy constructor with scalar conversion */ template<typename OtherScalar, int OtherOptions> - explicit inline Quaternion(const Quaternion<OtherScalar, OtherOptions>& other) + EIGEN_DEVICE_FUNC explicit inline Quaternion(const Quaternion<OtherScalar, OtherOptions>& other) { m_coeffs = other.coeffs().template cast<Scalar>(); } - template<typename Derived1, typename Derived2> - static Quaternion FromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b); +#if EIGEN_HAS_RVALUE_REFERENCES + // We define a copy constructor, which means we don't get an implicit move constructor or assignment operator. + /** Default move constructor */ + EIGEN_DEVICE_FUNC inline Quaternion(Quaternion&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible<Scalar>::value) + : m_coeffs(std::move(other.coeffs())) + {} - inline Coefficients& coeffs() { return m_coeffs;} - inline const Coefficients& coeffs() const { return m_coeffs;} + /** Default move assignment operator */ + EIGEN_DEVICE_FUNC Quaternion& operator=(Quaternion&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable<Scalar>::value) + { + m_coeffs = std::move(other.coeffs()); + return *this; + } + + // And now because we declared a constructor, we don't get an implicit copy constructor. Say we want one. + /** Default copy constructor */ + EIGEN_DEVICE_FUNC Quaternion(const Quaternion& other) + : m_coeffs(other.coeffs()) + {} +#endif + + EIGEN_DEVICE_FUNC static Quaternion UnitRandom(); + + template<typename Derived1, typename Derived2> + EIGEN_DEVICE_FUNC static Quaternion FromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b); + + EIGEN_DEVICE_FUNC inline Coefficients& coeffs() { return m_coeffs;} + EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs;} EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(NeedsAlignment)) @@ -355,9 +383,9 @@ * \code *coeffs == {x, y, z, w} \endcode * * If the template parameter _Options is set to #Aligned, then the pointer coeffs must be aligned. */ - explicit EIGEN_STRONG_INLINE Map(const Scalar* coeffs) : m_coeffs(coeffs) {} + EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Map(const Scalar* coeffs) : m_coeffs(coeffs) {} - inline const Coefficients& coeffs() const { return m_coeffs;} + EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs;} protected: const Coefficients m_coeffs; @@ -392,10 +420,10 @@ * \code *coeffs == {x, y, z, w} \endcode * * If the template parameter _Options is set to #Aligned, then the pointer coeffs must be aligned. */ - explicit EIGEN_STRONG_INLINE Map(Scalar* coeffs) : m_coeffs(coeffs) {} + EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Map(Scalar* coeffs) : m_coeffs(coeffs) {} - inline Coefficients& coeffs() { return m_coeffs; } - inline const Coefficients& coeffs() const { return m_coeffs; } + EIGEN_DEVICE_FUNC inline Coefficients& coeffs() { return m_coeffs; } + EIGEN_DEVICE_FUNC inline const Coefficients& coeffs() const { return m_coeffs; } protected: Coefficients m_coeffs; @@ -421,9 +449,9 @@ // Generic Quaternion * Quaternion product // This product can be specialized for a given architecture via the Arch template argument. namespace internal { -template<int Arch, class Derived1, class Derived2, typename Scalar, int _Options> struct quat_product +template<int Arch, class Derived1, class Derived2, typename Scalar> struct quat_product { - static EIGEN_STRONG_INLINE Quaternion<Scalar> run(const QuaternionBase<Derived1>& a, const QuaternionBase<Derived2>& b){ + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Quaternion<Scalar> run(const QuaternionBase<Derived1>& a, const QuaternionBase<Derived2>& b){ return Quaternion<Scalar> ( a.w() * b.w() - a.x() * b.x() - a.y() * b.y() - a.z() * b.z(), @@ -438,20 +466,19 @@ /** \returns the concatenation of two rotations as a quaternion-quaternion product */ template <class Derived> template <class OtherDerived> -EIGEN_STRONG_INLINE Quaternion<typename internal::traits<Derived>::Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Quaternion<typename internal::traits<Derived>::Scalar> QuaternionBase<Derived>::operator* (const QuaternionBase<OtherDerived>& other) const { EIGEN_STATIC_ASSERT((internal::is_same<typename Derived::Scalar, typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) return internal::quat_product<Architecture::Target, Derived, OtherDerived, - typename internal::traits<Derived>::Scalar, - EIGEN_PLAIN_ENUM_MIN(internal::traits<Derived>::Alignment, internal::traits<OtherDerived>::Alignment)>::run(*this, other); + typename internal::traits<Derived>::Scalar>::run(*this, other); } /** \sa operator*(Quaternion) */ template <class Derived> template <class OtherDerived> -EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator*= (const QuaternionBase<OtherDerived>& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator*= (const QuaternionBase<OtherDerived>& other) { derived() = derived() * other.derived(); return derived(); @@ -465,7 +492,7 @@ * - Via a Matrix3: 24 + 15n */ template <class Derived> -EIGEN_STRONG_INLINE typename QuaternionBase<Derived>::Vector3 +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename QuaternionBase<Derived>::Vector3 QuaternionBase<Derived>::_transformVector(const Vector3& v) const { // Note that this algorithm comes from the optimization by hand @@ -479,7 +506,7 @@ } template<class Derived> -EIGEN_STRONG_INLINE QuaternionBase<Derived>& QuaternionBase<Derived>::operator=(const QuaternionBase<Derived>& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE QuaternionBase<Derived>& QuaternionBase<Derived>::operator=(const QuaternionBase<Derived>& other) { coeffs() = other.coeffs(); return derived(); @@ -487,7 +514,7 @@ template<class Derived> template<class OtherDerived> -EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator=(const QuaternionBase<OtherDerived>& other) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator=(const QuaternionBase<OtherDerived>& other) { coeffs() = other.coeffs(); return derived(); @@ -496,10 +523,10 @@ /** Set \c *this from an angle-axis \a aa and returns a reference to \c *this */ template<class Derived> -EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator=(const AngleAxisType& aa) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& QuaternionBase<Derived>::operator=(const AngleAxisType& aa) { - using std::cos; - using std::sin; + EIGEN_USING_STD_MATH(cos) + EIGEN_USING_STD_MATH(sin) Scalar ha = Scalar(0.5)*aa.angle(); // Scalar(0.5) to suppress precision loss warnings this->w() = cos(ha); this->vec() = sin(ha) * aa.axis(); @@ -514,7 +541,7 @@ template<class Derived> template<class MatrixDerived> -inline Derived& QuaternionBase<Derived>::operator=(const MatrixBase<MatrixDerived>& xpr) +EIGEN_DEVICE_FUNC inline Derived& QuaternionBase<Derived>::operator=(const MatrixBase<MatrixDerived>& xpr) { EIGEN_STATIC_ASSERT((internal::is_same<typename Derived::Scalar, typename MatrixDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) @@ -526,7 +553,7 @@ * be normalized, otherwise the result is undefined. */ template<class Derived> -inline typename QuaternionBase<Derived>::Matrix3 +EIGEN_DEVICE_FUNC inline typename QuaternionBase<Derived>::Matrix3 QuaternionBase<Derived>::toRotationMatrix(void) const { // NOTE if inlined, then gcc 4.2 and 4.4 get rid of the temporary (not gcc 4.3 !!) @@ -573,9 +600,9 @@ */ template<class Derived> template<typename Derived1, typename Derived2> -inline Derived& QuaternionBase<Derived>::setFromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b) +EIGEN_DEVICE_FUNC inline Derived& QuaternionBase<Derived>::setFromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b) { - using std::sqrt; + EIGEN_USING_STD_MATH(sqrt) Vector3 v0 = a.normalized(); Vector3 v1 = b.normalized(); Scalar c = v1.dot(v0); @@ -609,6 +636,24 @@ return derived(); } +/** \returns a random unit quaternion following a uniform distribution law on SO(3) + * + * \note The implementation is based on http://planning.cs.uiuc.edu/node198.html + */ +template<typename Scalar, int Options> +EIGEN_DEVICE_FUNC Quaternion<Scalar,Options> Quaternion<Scalar,Options>::UnitRandom() +{ + EIGEN_USING_STD_MATH(sqrt) + EIGEN_USING_STD_MATH(sin) + EIGEN_USING_STD_MATH(cos) + const Scalar u1 = internal::random<Scalar>(0, 1), + u2 = internal::random<Scalar>(0, 2*EIGEN_PI), + u3 = internal::random<Scalar>(0, 2*EIGEN_PI); + const Scalar a = sqrt(Scalar(1) - u1), + b = sqrt(u1); + return Quaternion (a * sin(u2), a * cos(u2), b * sin(u3), b * cos(u3)); +} + /** Returns a quaternion representing a rotation between * the two arbitrary vectors \a a and \a b. In other words, the built @@ -622,7 +667,7 @@ */ template<typename Scalar, int Options> template<typename Derived1, typename Derived2> -Quaternion<Scalar,Options> Quaternion<Scalar,Options>::FromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b) +EIGEN_DEVICE_FUNC Quaternion<Scalar,Options> Quaternion<Scalar,Options>::FromTwoVectors(const MatrixBase<Derived1>& a, const MatrixBase<Derived2>& b) { Quaternion quat; quat.setFromTwoVectors(a, b); @@ -637,7 +682,7 @@ * \sa QuaternionBase::conjugate() */ template <class Derived> -inline Quaternion<typename internal::traits<Derived>::Scalar> QuaternionBase<Derived>::inverse() const +EIGEN_DEVICE_FUNC inline Quaternion<typename internal::traits<Derived>::Scalar> QuaternionBase<Derived>::inverse() const { // FIXME should this function be called multiplicativeInverse and conjugate() be called inverse() or opposite() ?? Scalar n2 = this->squaredNorm(); @@ -652,9 +697,9 @@ // Generic conjugate of a Quaternion namespace internal { -template<int Arch, class Derived, typename Scalar, int _Options> struct quat_conj +template<int Arch, class Derived, typename Scalar> struct quat_conj { - static EIGEN_STRONG_INLINE Quaternion<Scalar> run(const QuaternionBase<Derived>& q){ + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Quaternion<Scalar> run(const QuaternionBase<Derived>& q){ return Quaternion<Scalar>(q.w(),-q.x(),-q.y(),-q.z()); } }; @@ -667,12 +712,11 @@ * \sa Quaternion2::inverse() */ template <class Derived> -inline Quaternion<typename internal::traits<Derived>::Scalar> +EIGEN_DEVICE_FUNC inline Quaternion<typename internal::traits<Derived>::Scalar> QuaternionBase<Derived>::conjugate() const { return internal::quat_conj<Architecture::Target, Derived, - typename internal::traits<Derived>::Scalar, - internal::traits<Derived>::Alignment>::run(*this); + typename internal::traits<Derived>::Scalar>::run(*this); } @@ -681,13 +725,12 @@ */ template <class Derived> template <class OtherDerived> -inline typename internal::traits<Derived>::Scalar +EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar QuaternionBase<Derived>::angularDistance(const QuaternionBase<OtherDerived>& other) const { - using std::atan2; - using std::abs; + EIGEN_USING_STD_MATH(atan2) Quaternion<Scalar> d = (*this) * other.conjugate(); - return Scalar(2) * atan2( d.vec().norm(), abs(d.w()) ); + return Scalar(2) * atan2( d.vec().norm(), numext::abs(d.w()) ); } @@ -700,15 +743,14 @@ */ template <class Derived> template <class OtherDerived> -Quaternion<typename internal::traits<Derived>::Scalar> +EIGEN_DEVICE_FUNC Quaternion<typename internal::traits<Derived>::Scalar> QuaternionBase<Derived>::slerp(const Scalar& t, const QuaternionBase<OtherDerived>& other) const { - using std::acos; - using std::sin; - using std::abs; - static const Scalar one = Scalar(1) - NumTraits<Scalar>::epsilon(); + EIGEN_USING_STD_MATH(acos) + EIGEN_USING_STD_MATH(sin) + const Scalar one = Scalar(1) - NumTraits<Scalar>::epsilon(); Scalar d = this->dot(other); - Scalar absD = abs(d); + Scalar absD = numext::abs(d); Scalar scale0; Scalar scale1; @@ -739,10 +781,10 @@ struct quaternionbase_assign_impl<Other,3,3> { typedef typename Other::Scalar Scalar; - template<class Derived> static inline void run(QuaternionBase<Derived>& q, const Other& a_mat) + template<class Derived> EIGEN_DEVICE_FUNC static inline void run(QuaternionBase<Derived>& q, const Other& a_mat) { const typename internal::nested_eval<Other,2>::type mat(a_mat); - using std::sqrt; + EIGEN_USING_STD_MATH(sqrt) // This algorithm comes from "Quaternion Calculus and Fast Animation", // Ken Shoemake, 1987 SIGGRAPH course notes Scalar t = mat.trace(); @@ -780,7 +822,7 @@ struct quaternionbase_assign_impl<Other,4,1> { typedef typename Other::Scalar Scalar; - template<class Derived> static inline void run(QuaternionBase<Derived>& q, const Other& vec) + template<class Derived> EIGEN_DEVICE_FUNC static inline void run(QuaternionBase<Derived>& q, const Other& vec) { q.coeffs() = vec; }
diff --git a/Eigen/src/Geometry/Rotation2D.h b/Eigen/src/Geometry/Rotation2D.h index 5ab0d59..884b7d0 100644 --- a/Eigen/src/Geometry/Rotation2D.h +++ b/Eigen/src/Geometry/Rotation2D.h
@@ -59,59 +59,59 @@ public: /** Construct a 2D counter clock wise rotation from the angle \a a in radian. */ - explicit inline Rotation2D(const Scalar& a) : m_angle(a) {} + EIGEN_DEVICE_FUNC explicit inline Rotation2D(const Scalar& a) : m_angle(a) {} /** Default constructor wihtout initialization. The represented rotation is undefined. */ - Rotation2D() {} + EIGEN_DEVICE_FUNC Rotation2D() {} /** Construct a 2D rotation from a 2x2 rotation matrix \a mat. * * \sa fromRotationMatrix() */ template<typename Derived> - explicit Rotation2D(const MatrixBase<Derived>& m) + EIGEN_DEVICE_FUNC explicit Rotation2D(const MatrixBase<Derived>& m) { fromRotationMatrix(m.derived()); } /** \returns the rotation angle */ - inline Scalar angle() const { return m_angle; } + EIGEN_DEVICE_FUNC inline Scalar angle() const { return m_angle; } /** \returns a read-write reference to the rotation angle */ - inline Scalar& angle() { return m_angle; } + EIGEN_DEVICE_FUNC inline Scalar& angle() { return m_angle; } /** \returns the rotation angle in [0,2pi] */ - inline Scalar smallestPositiveAngle() const { - Scalar tmp = fmod(m_angle,Scalar(2)*EIGEN_PI); - return tmp<Scalar(0) ? tmp + Scalar(2)*EIGEN_PI : tmp; + EIGEN_DEVICE_FUNC inline Scalar smallestPositiveAngle() const { + Scalar tmp = numext::fmod(m_angle,Scalar(2*EIGEN_PI)); + return tmp<Scalar(0) ? tmp + Scalar(2*EIGEN_PI) : tmp; } /** \returns the rotation angle in [-pi,pi] */ - inline Scalar smallestAngle() const { - Scalar tmp = fmod(m_angle,Scalar(2)*EIGEN_PI); - if(tmp>Scalar(EIGEN_PI)) tmp -= Scalar(2)*Scalar(EIGEN_PI); - else if(tmp<-Scalar(EIGEN_PI)) tmp += Scalar(2)*Scalar(EIGEN_PI); + EIGEN_DEVICE_FUNC inline Scalar smallestAngle() const { + Scalar tmp = numext::fmod(m_angle,Scalar(2*EIGEN_PI)); + if(tmp>Scalar(EIGEN_PI)) tmp -= Scalar(2*EIGEN_PI); + else if(tmp<-Scalar(EIGEN_PI)) tmp += Scalar(2*EIGEN_PI); return tmp; } /** \returns the inverse rotation */ - inline Rotation2D inverse() const { return Rotation2D(-m_angle); } + EIGEN_DEVICE_FUNC inline Rotation2D inverse() const { return Rotation2D(-m_angle); } /** Concatenates two rotations */ - inline Rotation2D operator*(const Rotation2D& other) const + EIGEN_DEVICE_FUNC inline Rotation2D operator*(const Rotation2D& other) const { return Rotation2D(m_angle + other.m_angle); } /** Concatenates two rotations */ - inline Rotation2D& operator*=(const Rotation2D& other) + EIGEN_DEVICE_FUNC inline Rotation2D& operator*=(const Rotation2D& other) { m_angle += other.m_angle; return *this; } /** Applies the rotation to a 2D vector */ - Vector2 operator* (const Vector2& vec) const + EIGEN_DEVICE_FUNC Vector2 operator* (const Vector2& vec) const { return toRotationMatrix() * vec; } template<typename Derived> - Rotation2D& fromRotationMatrix(const MatrixBase<Derived>& m); - Matrix2 toRotationMatrix() const; + EIGEN_DEVICE_FUNC Rotation2D& fromRotationMatrix(const MatrixBase<Derived>& m); + EIGEN_DEVICE_FUNC Matrix2 toRotationMatrix() const; /** Set \c *this from a 2x2 rotation matrix \a mat. * In other words, this function extract the rotation angle from the rotation matrix. @@ -121,13 +121,13 @@ * \sa fromRotationMatrix() */ template<typename Derived> - Rotation2D& operator=(const MatrixBase<Derived>& m) + EIGEN_DEVICE_FUNC Rotation2D& operator=(const MatrixBase<Derived>& m) { return fromRotationMatrix(m.derived()); } /** \returns the spherical interpolation between \c *this and \a other using * parameter \a t. It is in fact equivalent to a linear interpolation. */ - inline Rotation2D slerp(const Scalar& t, const Rotation2D& other) const + EIGEN_DEVICE_FUNC inline Rotation2D slerp(const Scalar& t, const Rotation2D& other) const { Scalar dist = Rotation2D(other.m_angle-m_angle).smallestAngle(); return Rotation2D(m_angle + dist*t); @@ -139,23 +139,23 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<Rotation2D,Rotation2D<NewScalarType> >::type cast() const + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<Rotation2D,Rotation2D<NewScalarType> >::type cast() const { return typename internal::cast_return_type<Rotation2D,Rotation2D<NewScalarType> >::type(*this); } /** Copy constructor with scalar type conversion */ template<typename OtherScalarType> - inline explicit Rotation2D(const Rotation2D<OtherScalarType>& other) + EIGEN_DEVICE_FUNC inline explicit Rotation2D(const Rotation2D<OtherScalarType>& other) { m_angle = Scalar(other.angle()); } - static inline Rotation2D Identity() { return Rotation2D(0); } + EIGEN_DEVICE_FUNC static inline Rotation2D Identity() { return Rotation2D(0); } /** \returns \c true if \c *this is approximately equal to \a other, within the precision * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const Rotation2D& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const Rotation2D& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return internal::isApprox(m_angle,other.m_angle, prec); } }; @@ -173,9 +173,9 @@ */ template<typename Scalar> template<typename Derived> -Rotation2D<Scalar>& Rotation2D<Scalar>::fromRotationMatrix(const MatrixBase<Derived>& mat) +EIGEN_DEVICE_FUNC Rotation2D<Scalar>& Rotation2D<Scalar>::fromRotationMatrix(const MatrixBase<Derived>& mat) { - using std::atan2; + EIGEN_USING_STD_MATH(atan2) EIGEN_STATIC_ASSERT(Derived::RowsAtCompileTime==2 && Derived::ColsAtCompileTime==2,YOU_MADE_A_PROGRAMMING_MISTAKE) m_angle = atan2(mat.coeff(1,0), mat.coeff(0,0)); return *this; @@ -185,10 +185,10 @@ */ template<typename Scalar> typename Rotation2D<Scalar>::Matrix2 -Rotation2D<Scalar>::toRotationMatrix(void) const +EIGEN_DEVICE_FUNC Rotation2D<Scalar>::toRotationMatrix(void) const { - using std::sin; - using std::cos; + EIGEN_USING_STD_MATH(sin) + EIGEN_USING_STD_MATH(cos) Scalar sinA = sin(m_angle); Scalar cosA = cos(m_angle); return (Matrix2() << cosA, -sinA, sinA, cosA).finished();
diff --git a/Eigen/src/Geometry/RotationBase.h b/Eigen/src/Geometry/RotationBase.h index fadfd91..f0ee0bd 100644 --- a/Eigen/src/Geometry/RotationBase.h +++ b/Eigen/src/Geometry/RotationBase.h
@@ -38,26 +38,26 @@ typedef Matrix<Scalar,Dim,1> VectorType; public: - inline const Derived& derived() const { return *static_cast<const Derived*>(this); } - inline Derived& derived() { return *static_cast<Derived*>(this); } + EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast<const Derived*>(this); } + EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast<Derived*>(this); } /** \returns an equivalent rotation matrix */ - inline RotationMatrixType toRotationMatrix() const { return derived().toRotationMatrix(); } + EIGEN_DEVICE_FUNC inline RotationMatrixType toRotationMatrix() const { return derived().toRotationMatrix(); } /** \returns an equivalent rotation matrix * This function is added to be conform with the Transform class' naming scheme. */ - inline RotationMatrixType matrix() const { return derived().toRotationMatrix(); } + EIGEN_DEVICE_FUNC inline RotationMatrixType matrix() const { return derived().toRotationMatrix(); } /** \returns the inverse rotation */ - inline Derived inverse() const { return derived().inverse(); } + EIGEN_DEVICE_FUNC inline Derived inverse() const { return derived().inverse(); } /** \returns the concatenation of the rotation \c *this with a translation \a t */ - inline Transform<Scalar,Dim,Isometry> operator*(const Translation<Scalar,Dim>& t) const + EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Isometry> operator*(const Translation<Scalar,Dim>& t) const { return Transform<Scalar,Dim,Isometry>(*this) * t; } /** \returns the concatenation of the rotation \c *this with a uniform scaling \a s */ - inline RotationMatrixType operator*(const UniformScaling<Scalar>& s) const + EIGEN_DEVICE_FUNC inline RotationMatrixType operator*(const UniformScaling<Scalar>& s) const { return toRotationMatrix() * s.factor(); } /** \returns the concatenation of the rotation \c *this with a generic expression \a e @@ -67,17 +67,17 @@ * - a vector of size Dim */ template<typename OtherDerived> - EIGEN_STRONG_INLINE typename internal::rotation_base_generic_product_selector<Derived,OtherDerived,OtherDerived::IsVectorAtCompileTime>::ReturnType + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::rotation_base_generic_product_selector<Derived,OtherDerived,OtherDerived::IsVectorAtCompileTime>::ReturnType operator*(const EigenBase<OtherDerived>& e) const { return internal::rotation_base_generic_product_selector<Derived,OtherDerived>::run(derived(), e.derived()); } /** \returns the concatenation of a linear transformation \a l with the rotation \a r */ template<typename OtherDerived> friend - inline RotationMatrixType operator*(const EigenBase<OtherDerived>& l, const Derived& r) + EIGEN_DEVICE_FUNC inline RotationMatrixType operator*(const EigenBase<OtherDerived>& l, const Derived& r) { return l.derived() * r.toRotationMatrix(); } /** \returns the concatenation of a scaling \a l with the rotation \a r */ - friend inline Transform<Scalar,Dim,Affine> operator*(const DiagonalMatrix<Scalar,Dim>& l, const Derived& r) + EIGEN_DEVICE_FUNC friend inline Transform<Scalar,Dim,Affine> operator*(const DiagonalMatrix<Scalar,Dim>& l, const Derived& r) { Transform<Scalar,Dim,Affine> res(r); res.linear().applyOnTheLeft(l); @@ -86,11 +86,11 @@ /** \returns the concatenation of the rotation \c *this with a transformation \a t */ template<int Mode, int Options> - inline Transform<Scalar,Dim,Mode> operator*(const Transform<Scalar,Dim,Mode,Options>& t) const + EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode> operator*(const Transform<Scalar,Dim,Mode,Options>& t) const { return toRotationMatrix() * t; } template<typename OtherVectorType> - inline VectorType _transformVector(const OtherVectorType& v) const + EIGEN_DEVICE_FUNC inline VectorType _transformVector(const OtherVectorType& v) const { return toRotationMatrix() * v; } }; @@ -102,7 +102,7 @@ { enum { Dim = RotationDerived::Dim }; typedef Matrix<typename RotationDerived::Scalar,Dim,Dim> ReturnType; - static inline ReturnType run(const RotationDerived& r, const MatrixType& m) + EIGEN_DEVICE_FUNC static inline ReturnType run(const RotationDerived& r, const MatrixType& m) { return r.toRotationMatrix() * m; } }; @@ -110,7 +110,7 @@ struct rotation_base_generic_product_selector< RotationDerived, DiagonalMatrix<Scalar,Dim,MaxDim>, false > { typedef Transform<Scalar,Dim,Affine> ReturnType; - static inline ReturnType run(const RotationDerived& r, const DiagonalMatrix<Scalar,Dim,MaxDim>& m) + EIGEN_DEVICE_FUNC static inline ReturnType run(const RotationDerived& r, const DiagonalMatrix<Scalar,Dim,MaxDim>& m) { ReturnType res(r); res.linear() *= m; @@ -123,7 +123,7 @@ { enum { Dim = RotationDerived::Dim }; typedef Matrix<typename RotationDerived::Scalar,Dim,1> ReturnType; - static EIGEN_STRONG_INLINE ReturnType run(const RotationDerived& r, const OtherVectorType& v) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE ReturnType run(const RotationDerived& r, const OtherVectorType& v) { return r._transformVector(v); } @@ -137,7 +137,7 @@ */ template<typename _Scalar, int _Rows, int _Cols, int _Storage, int _MaxRows, int _MaxCols> template<typename OtherDerived> -Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols> +EIGEN_DEVICE_FUNC Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols> ::Matrix(const RotationBase<OtherDerived,ColsAtCompileTime>& r) { EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Matrix,int(OtherDerived::Dim),int(OtherDerived::Dim)) @@ -150,7 +150,7 @@ */ template<typename _Scalar, int _Rows, int _Cols, int _Storage, int _MaxRows, int _MaxCols> template<typename OtherDerived> -Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols>& +EIGEN_DEVICE_FUNC Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols>& Matrix<_Scalar, _Rows, _Cols, _Storage, _MaxRows, _MaxCols> ::operator=(const RotationBase<OtherDerived,ColsAtCompileTime>& r) { @@ -179,20 +179,20 @@ * \sa class Transform, class Rotation2D, class Quaternion, class AngleAxis */ template<typename Scalar, int Dim> -static inline Matrix<Scalar,2,2> toRotationMatrix(const Scalar& s) +EIGEN_DEVICE_FUNC static inline Matrix<Scalar,2,2> toRotationMatrix(const Scalar& s) { EIGEN_STATIC_ASSERT(Dim==2,YOU_MADE_A_PROGRAMMING_MISTAKE) return Rotation2D<Scalar>(s).toRotationMatrix(); } template<typename Scalar, int Dim, typename OtherDerived> -static inline Matrix<Scalar,Dim,Dim> toRotationMatrix(const RotationBase<OtherDerived,Dim>& r) +EIGEN_DEVICE_FUNC static inline Matrix<Scalar,Dim,Dim> toRotationMatrix(const RotationBase<OtherDerived,Dim>& r) { return r.toRotationMatrix(); } template<typename Scalar, int Dim, typename OtherDerived> -static inline const MatrixBase<OtherDerived>& toRotationMatrix(const MatrixBase<OtherDerived>& mat) +EIGEN_DEVICE_FUNC static inline const MatrixBase<OtherDerived>& toRotationMatrix(const MatrixBase<OtherDerived>& mat) { EIGEN_STATIC_ASSERT(OtherDerived::RowsAtCompileTime==Dim && OtherDerived::ColsAtCompileTime==Dim, YOU_MADE_A_PROGRAMMING_MISTAKE)
diff --git a/Eigen/src/Geometry/Scaling.h b/Eigen/src/Geometry/Scaling.h old mode 100644 new mode 100755 index 6431381..df650fd --- a/Eigen/src/Geometry/Scaling.h +++ b/Eigen/src/Geometry/Scaling.h
@@ -29,6 +29,22 @@ * * \sa Scaling(), class DiagonalMatrix, MatrixBase::asDiagonal(), class Translation, class Transform */ + +namespace internal +{ + // This helper helps nvcc+MSVC to properly parse this file. + // See bug 1412. + template <typename Scalar, int Dim, int Mode> + struct uniformscaling_times_affine_returntype + { + enum + { + NewMode = int(Mode) == int(Isometry) ? Affine : Mode + }; + typedef Transform <Scalar, Dim, NewMode> type; + }; +} + template<typename _Scalar> class UniformScaling { @@ -60,9 +76,11 @@ /** Concatenates a uniform scaling and an affine transformation */ template<int Dim, int Mode, int Options> - inline Transform<Scalar,Dim,(int(Mode)==int(Isometry)?Affine:Mode)> operator* (const Transform<Scalar,Dim, Mode, Options>& t) const + inline typename + internal::uniformscaling_times_affine_returntype<Scalar,Dim,Mode>::type + operator* (const Transform<Scalar, Dim, Mode, Options>& t) const { - Transform<Scalar,Dim,(int(Mode)==int(Isometry)?Affine:Mode)> res = t; + typename internal::uniformscaling_times_affine_returntype<Scalar,Dim,Mode>::type res = t; res.prescale(factor()); return res; } @@ -70,7 +88,7 @@ /** Concatenates a uniform scaling and a linear transformation matrix */ // TODO returns an expression template<typename Derived> - inline typename internal::plain_matrix_type<Derived>::type operator* (const MatrixBase<Derived>& other) const + inline typename Eigen::internal::plain_matrix_type<Derived>::type operator* (const MatrixBase<Derived>& other) const { return other * m_factor; } template<typename Derived,int Dim> @@ -107,36 +125,39 @@ /** \addtogroup Geometry_Module */ //@{ -/** Concatenates a linear transformation matrix and a uniform scaling */ -// NOTE this operator is defiend in MatrixBase and not as a friend function +/** Concatenates a linear transformation matrix and a uniform scaling + * \relates UniformScaling + */ +// NOTE this operator is defined in MatrixBase and not as a friend function // of UniformScaling to fix an internal crash of Intel's ICC -template<typename Derived> typename MatrixBase<Derived>::ScalarMultipleReturnType -MatrixBase<Derived>::operator*(const UniformScaling<Scalar>& s) const -{ return derived() * s.factor(); } +template<typename Derived,typename Scalar> +EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,Scalar,product) +operator*(const MatrixBase<Derived>& matrix, const UniformScaling<Scalar>& s) +{ return matrix.derived() * s.factor(); } /** Constructs a uniform scaling from scale factor \a s */ -static inline UniformScaling<float> Scaling(float s) { return UniformScaling<float>(s); } +inline UniformScaling<float> Scaling(float s) { return UniformScaling<float>(s); } /** Constructs a uniform scaling from scale factor \a s */ -static inline UniformScaling<double> Scaling(double s) { return UniformScaling<double>(s); } +inline UniformScaling<double> Scaling(double s) { return UniformScaling<double>(s); } /** Constructs a uniform scaling from scale factor \a s */ template<typename RealScalar> -static inline UniformScaling<std::complex<RealScalar> > Scaling(const std::complex<RealScalar>& s) +inline UniformScaling<std::complex<RealScalar> > Scaling(const std::complex<RealScalar>& s) { return UniformScaling<std::complex<RealScalar> >(s); } /** Constructs a 2D axis aligned scaling */ template<typename Scalar> -static inline DiagonalMatrix<Scalar,2> Scaling(const Scalar& sx, const Scalar& sy) +inline DiagonalMatrix<Scalar,2> Scaling(const Scalar& sx, const Scalar& sy) { return DiagonalMatrix<Scalar,2>(sx, sy); } /** Constructs a 3D axis aligned scaling */ template<typename Scalar> -static inline DiagonalMatrix<Scalar,3> Scaling(const Scalar& sx, const Scalar& sy, const Scalar& sz) +inline DiagonalMatrix<Scalar,3> Scaling(const Scalar& sx, const Scalar& sy, const Scalar& sz) { return DiagonalMatrix<Scalar,3>(sx, sy, sz); } /** Constructs an axis aligned scaling expression from vector expression \a coeffs * This is an alias for coeffs.asDiagonal() */ template<typename Derived> -static inline const DiagonalWrapper<const Derived> Scaling(const MatrixBase<Derived>& coeffs) +inline const DiagonalWrapper<const Derived> Scaling(const MatrixBase<Derived>& coeffs) { return coeffs.asDiagonal(); } /** \deprecated */
diff --git a/Eigen/src/Geometry/Transform.h b/Eigen/src/Geometry/Transform.h index 75f20bd..3090351 100644 --- a/Eigen/src/Geometry/Transform.h +++ b/Eigen/src/Geometry/Transform.h
@@ -32,7 +32,8 @@ typename MatrixType, int Case = transform_traits<TransformType>::IsProjective ? 0 : int(MatrixType::RowsAtCompileTime) == int(transform_traits<TransformType>::HDim) ? 1 - : 2> + : 2, + int RhsCols = MatrixType::ColsAtCompileTime> struct transform_right_product_impl; template< typename Other, @@ -96,6 +97,9 @@ * - #AffineCompact: the transformation is stored as a (Dim)x(Dim+1) matrix. * - #Projective: the transformation is stored as a (Dim+1)^2 matrix * without any assumption. + * - #Isometry: same as #Affine with the additional assumption that + * the linear part represents a rotation. This assumption is exploited + * to speed up some functions such as inverse() and rotation(). * \tparam _Options has the same meaning as in class Matrix. It allows to specify DontAlign and/or RowMajor. * These Options are passed directly to the underlying matrix type. * @@ -114,7 +118,7 @@ * \end{array} \right) \f$ * * Note that for a projective transformation the last row can be anything, - * and then the interpretation of different parts might be sightly different. + * and then the interpretation of different parts might be slightly different. * * However, unlike a plain matrix, the Transform class provides many features * simplifying both its assembly and usage. In particular, it can be composed @@ -192,7 +196,7 @@ * preprocessor token EIGEN_QT_SUPPORT is defined. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_TRANSFORM_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_TRANSFORM_PLUGIN. * * \sa class Matrix, class Quaternion */ @@ -251,44 +255,44 @@ public: /** Default constructor without initialization of the meaningful coefficients. - * If Mode==Affine, then the last row is set to [0 ... 0 1] */ - inline Transform() + * If Mode==Affine or Mode==Isometry, then the last row is set to [0 ... 0 1] */ + EIGEN_DEVICE_FUNC inline Transform() { check_template_params(); - internal::transform_make_affine<(int(Mode)==Affine) ? Affine : AffineCompact>::run(m_matrix); + internal::transform_make_affine<(int(Mode)==Affine || int(Mode)==Isometry) ? Affine : AffineCompact>::run(m_matrix); } - inline Transform(const Transform& other) + EIGEN_DEVICE_FUNC inline Transform(const Transform& other) { check_template_params(); m_matrix = other.m_matrix; } - inline explicit Transform(const TranslationType& t) + EIGEN_DEVICE_FUNC inline explicit Transform(const TranslationType& t) { check_template_params(); *this = t; } - inline explicit Transform(const UniformScaling<Scalar>& s) + EIGEN_DEVICE_FUNC inline explicit Transform(const UniformScaling<Scalar>& s) { check_template_params(); *this = s; } template<typename Derived> - inline explicit Transform(const RotationBase<Derived, Dim>& r) + EIGEN_DEVICE_FUNC inline explicit Transform(const RotationBase<Derived, Dim>& r) { check_template_params(); *this = r; } - inline Transform& operator=(const Transform& other) + EIGEN_DEVICE_FUNC inline Transform& operator=(const Transform& other) { m_matrix = other.m_matrix; return *this; } typedef internal::transform_take_affine_part<Transform> take_affine_part; /** Constructs and initializes a transformation from a Dim^2 or a (Dim+1)^2 matrix. */ template<typename OtherDerived> - inline explicit Transform(const EigenBase<OtherDerived>& other) + EIGEN_DEVICE_FUNC inline explicit Transform(const EigenBase<OtherDerived>& other) { EIGEN_STATIC_ASSERT((internal::is_same<Scalar,typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY); @@ -299,7 +303,7 @@ /** Set \c *this from a Dim^2 or (Dim+1)^2 matrix. */ template<typename OtherDerived> - inline Transform& operator=(const EigenBase<OtherDerived>& other) + EIGEN_DEVICE_FUNC inline Transform& operator=(const EigenBase<OtherDerived>& other) { EIGEN_STATIC_ASSERT((internal::is_same<Scalar,typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY); @@ -309,7 +313,7 @@ } template<int OtherOptions> - inline Transform(const Transform<Scalar,Dim,Mode,OtherOptions>& other) + EIGEN_DEVICE_FUNC inline Transform(const Transform<Scalar,Dim,Mode,OtherOptions>& other) { check_template_params(); // only the options change, we can directly copy the matrices @@ -317,7 +321,7 @@ } template<int OtherMode,int OtherOptions> - inline Transform(const Transform<Scalar,Dim,OtherMode,OtherOptions>& other) + EIGEN_DEVICE_FUNC inline Transform(const Transform<Scalar,Dim,OtherMode,OtherOptions>& other) { check_template_params(); // prevent conversions as: @@ -334,7 +338,7 @@ OtherModeIsAffineCompact = OtherMode == int(AffineCompact) }; - if(ModeIsAffineCompact == OtherModeIsAffineCompact) + if(EIGEN_CONST_CONDITIONAL(ModeIsAffineCompact == OtherModeIsAffineCompact)) { // We need the block expression because the code is compiled for all // combinations of transformations and will trigger a compile time error @@ -342,7 +346,7 @@ m_matrix.template block<Dim,Dim+1>(0,0) = other.matrix().template block<Dim,Dim+1>(0,0); makeAffine(); } - else if(OtherModeIsAffineCompact) + else if(EIGEN_CONST_CONDITIONAL(OtherModeIsAffineCompact)) { typedef typename Transform<Scalar,Dim,OtherMode,OtherOptions>::MatrixType OtherMatrixType; internal::transform_construct_from_matrix<OtherMatrixType,Mode,Options,Dim,HDim>::run(this, other.matrix()); @@ -358,14 +362,14 @@ } template<typename OtherDerived> - Transform(const ReturnByValue<OtherDerived>& other) + EIGEN_DEVICE_FUNC Transform(const ReturnByValue<OtherDerived>& other) { check_template_params(); other.evalTo(*this); } template<typename OtherDerived> - Transform& operator=(const ReturnByValue<OtherDerived>& other) + EIGEN_DEVICE_FUNC Transform& operator=(const ReturnByValue<OtherDerived>& other) { other.evalTo(*this); return *this; @@ -380,35 +384,35 @@ inline QTransform toQTransform(void) const; #endif - Index rows() const { return int(Mode)==int(Projective) ? m_matrix.cols() : (m_matrix.cols()-1); } - Index cols() const { return m_matrix.cols(); } + EIGEN_DEVICE_FUNC Index rows() const { return int(Mode)==int(Projective) ? m_matrix.cols() : (m_matrix.cols()-1); } + EIGEN_DEVICE_FUNC Index cols() const { return m_matrix.cols(); } /** shortcut for m_matrix(row,col); * \sa MatrixBase::operator(Index,Index) const */ - inline Scalar operator() (Index row, Index col) const { return m_matrix(row,col); } + EIGEN_DEVICE_FUNC inline Scalar operator() (Index row, Index col) const { return m_matrix(row,col); } /** shortcut for m_matrix(row,col); * \sa MatrixBase::operator(Index,Index) */ - inline Scalar& operator() (Index row, Index col) { return m_matrix(row,col); } + EIGEN_DEVICE_FUNC inline Scalar& operator() (Index row, Index col) { return m_matrix(row,col); } /** \returns a read-only expression of the transformation matrix */ - inline const MatrixType& matrix() const { return m_matrix; } + EIGEN_DEVICE_FUNC inline const MatrixType& matrix() const { return m_matrix; } /** \returns a writable expression of the transformation matrix */ - inline MatrixType& matrix() { return m_matrix; } + EIGEN_DEVICE_FUNC inline MatrixType& matrix() { return m_matrix; } /** \returns a read-only expression of the linear part of the transformation */ - inline ConstLinearPart linear() const { return ConstLinearPart(m_matrix,0,0); } + EIGEN_DEVICE_FUNC inline ConstLinearPart linear() const { return ConstLinearPart(m_matrix,0,0); } /** \returns a writable expression of the linear part of the transformation */ - inline LinearPart linear() { return LinearPart(m_matrix,0,0); } + EIGEN_DEVICE_FUNC inline LinearPart linear() { return LinearPart(m_matrix,0,0); } /** \returns a read-only expression of the Dim x HDim affine part of the transformation */ - inline ConstAffinePart affine() const { return take_affine_part::run(m_matrix); } + EIGEN_DEVICE_FUNC inline ConstAffinePart affine() const { return take_affine_part::run(m_matrix); } /** \returns a writable expression of the Dim x HDim affine part of the transformation */ - inline AffinePart affine() { return take_affine_part::run(m_matrix); } + EIGEN_DEVICE_FUNC inline AffinePart affine() { return take_affine_part::run(m_matrix); } /** \returns a read-only expression of the translation vector of the transformation */ - inline ConstTranslationPart translation() const { return ConstTranslationPart(m_matrix,0,Dim); } + EIGEN_DEVICE_FUNC inline ConstTranslationPart translation() const { return ConstTranslationPart(m_matrix,0,Dim); } /** \returns a writable expression of the translation vector of the transformation */ - inline TranslationPart translation() { return TranslationPart(m_matrix,0,Dim); } + EIGEN_DEVICE_FUNC inline TranslationPart translation() { return TranslationPart(m_matrix,0,Dim); } /** \returns an expression of the product between the transform \c *this and a matrix expression \a other. * @@ -436,7 +440,7 @@ */ // note: this function is defined here because some compilers cannot find the respective declaration template<typename OtherDerived> - EIGEN_STRONG_INLINE const typename OtherDerived::PlainObject + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename internal::transform_right_product_impl<Transform, OtherDerived>::ResultType operator * (const EigenBase<OtherDerived> &other) const { return internal::transform_right_product_impl<Transform, OtherDerived>::run(*this,other.derived()); } @@ -448,7 +452,7 @@ * \li a general transformation matrix of size Dim+1 x Dim+1. */ template<typename OtherDerived> friend - inline const typename internal::transform_left_product_impl<OtherDerived,Mode,Options,_Dim,_Dim+1>::ResultType + EIGEN_DEVICE_FUNC inline const typename internal::transform_left_product_impl<OtherDerived,Mode,Options,_Dim,_Dim+1>::ResultType operator * (const EigenBase<OtherDerived> &a, const Transform &b) { return internal::transform_left_product_impl<OtherDerived,Mode,Options,Dim,HDim>::run(a.derived(),b); } @@ -459,11 +463,11 @@ * mode is no isometry. In that case, the returned transform is an affinity. */ template<typename DiagonalDerived> - inline const TransformTimeDiagonalReturnType + EIGEN_DEVICE_FUNC inline const TransformTimeDiagonalReturnType operator * (const DiagonalBase<DiagonalDerived> &b) const { TransformTimeDiagonalReturnType res(*this); - res.linear() *= b; + res.linearExt() *= b; return res; } @@ -474,22 +478,22 @@ * mode is no isometry. In that case, the returned transform is an affinity. */ template<typename DiagonalDerived> - friend inline TransformTimeDiagonalReturnType + EIGEN_DEVICE_FUNC friend inline TransformTimeDiagonalReturnType operator * (const DiagonalBase<DiagonalDerived> &a, const Transform &b) { TransformTimeDiagonalReturnType res; res.linear().noalias() = a*b.linear(); res.translation().noalias() = a*b.translation(); - if (Mode!=int(AffineCompact)) + if (EIGEN_CONST_CONDITIONAL(Mode!=int(AffineCompact))) res.matrix().row(Dim) = b.matrix().row(Dim); return res; } template<typename OtherDerived> - inline Transform& operator*=(const EigenBase<OtherDerived>& other) { return *this = *this * other; } + EIGEN_DEVICE_FUNC inline Transform& operator*=(const EigenBase<OtherDerived>& other) { return *this = *this * other; } /** Concatenates two transformations */ - inline const Transform operator * (const Transform& other) const + EIGEN_DEVICE_FUNC inline const Transform operator * (const Transform& other) const { return internal::transform_transform_product_impl<Transform,Transform>::run(*this,other); } @@ -521,7 +525,7 @@ #else /** Concatenates two different transformations */ template<int OtherMode,int OtherOptions> - inline typename internal::transform_transform_product_impl<Transform,Transform<Scalar,Dim,OtherMode,OtherOptions> >::ResultType + EIGEN_DEVICE_FUNC inline typename internal::transform_transform_product_impl<Transform,Transform<Scalar,Dim,OtherMode,OtherOptions> >::ResultType operator * (const Transform<Scalar,Dim,OtherMode,OtherOptions>& other) const { return internal::transform_transform_product_impl<Transform,Transform<Scalar,Dim,OtherMode,OtherOptions> >::run(*this,other); @@ -529,47 +533,61 @@ #endif /** \sa MatrixBase::setIdentity() */ - void setIdentity() { m_matrix.setIdentity(); } + EIGEN_DEVICE_FUNC void setIdentity() { m_matrix.setIdentity(); } /** * \brief Returns an identity transformation. * \todo In the future this function should be returning a Transform expression. */ - static const Transform Identity() + EIGEN_DEVICE_FUNC static const Transform Identity() { return Transform(MatrixType::Identity()); } template<typename OtherDerived> + EIGEN_DEVICE_FUNC inline Transform& scale(const MatrixBase<OtherDerived> &other); template<typename OtherDerived> + EIGEN_DEVICE_FUNC inline Transform& prescale(const MatrixBase<OtherDerived> &other); - inline Transform& scale(const Scalar& s); - inline Transform& prescale(const Scalar& s); + EIGEN_DEVICE_FUNC inline Transform& scale(const Scalar& s); + EIGEN_DEVICE_FUNC inline Transform& prescale(const Scalar& s); template<typename OtherDerived> + EIGEN_DEVICE_FUNC inline Transform& translate(const MatrixBase<OtherDerived> &other); template<typename OtherDerived> + EIGEN_DEVICE_FUNC inline Transform& pretranslate(const MatrixBase<OtherDerived> &other); template<typename RotationType> + EIGEN_DEVICE_FUNC inline Transform& rotate(const RotationType& rotation); template<typename RotationType> + EIGEN_DEVICE_FUNC inline Transform& prerotate(const RotationType& rotation); - Transform& shear(const Scalar& sx, const Scalar& sy); - Transform& preshear(const Scalar& sx, const Scalar& sy); + EIGEN_DEVICE_FUNC Transform& shear(const Scalar& sx, const Scalar& sy); + EIGEN_DEVICE_FUNC Transform& preshear(const Scalar& sx, const Scalar& sy); - inline Transform& operator=(const TranslationType& t); + EIGEN_DEVICE_FUNC inline Transform& operator=(const TranslationType& t); + + EIGEN_DEVICE_FUNC inline Transform& operator*=(const TranslationType& t) { return translate(t.vector()); } - inline Transform operator*(const TranslationType& t) const; + + EIGEN_DEVICE_FUNC inline Transform operator*(const TranslationType& t) const; + EIGEN_DEVICE_FUNC inline Transform& operator=(const UniformScaling<Scalar>& t); + + EIGEN_DEVICE_FUNC inline Transform& operator*=(const UniformScaling<Scalar>& s) { return scale(s.factor()); } + + EIGEN_DEVICE_FUNC inline TransformTimeDiagonalReturnType operator*(const UniformScaling<Scalar>& s) const { TransformTimeDiagonalReturnType res = *this; @@ -577,31 +595,38 @@ return res; } - inline Transform& operator*=(const DiagonalMatrix<Scalar,Dim>& s) { linear() *= s; return *this; } + EIGEN_DEVICE_FUNC + inline Transform& operator*=(const DiagonalMatrix<Scalar,Dim>& s) { linearExt() *= s; return *this; } template<typename Derived> - inline Transform& operator=(const RotationBase<Derived,Dim>& r); + EIGEN_DEVICE_FUNC inline Transform& operator=(const RotationBase<Derived,Dim>& r); template<typename Derived> - inline Transform& operator*=(const RotationBase<Derived,Dim>& r) { return rotate(r.toRotationMatrix()); } + EIGEN_DEVICE_FUNC inline Transform& operator*=(const RotationBase<Derived,Dim>& r) { return rotate(r.toRotationMatrix()); } template<typename Derived> - inline Transform operator*(const RotationBase<Derived,Dim>& r) const; + EIGEN_DEVICE_FUNC inline Transform operator*(const RotationBase<Derived,Dim>& r) const; - const LinearMatrixType rotation() const; + typedef typename internal::conditional<int(Mode)==Isometry,ConstLinearPart,const LinearMatrixType>::type RotationReturnType; + EIGEN_DEVICE_FUNC RotationReturnType rotation() const; + template<typename RotationMatrixType, typename ScalingMatrixType> + EIGEN_DEVICE_FUNC void computeRotationScaling(RotationMatrixType *rotation, ScalingMatrixType *scaling) const; template<typename ScalingMatrixType, typename RotationMatrixType> + EIGEN_DEVICE_FUNC void computeScalingRotation(ScalingMatrixType *scaling, RotationMatrixType *rotation) const; template<typename PositionDerived, typename OrientationType, typename ScaleDerived> + EIGEN_DEVICE_FUNC Transform& fromPositionOrientationScale(const MatrixBase<PositionDerived> &position, const OrientationType& orientation, const MatrixBase<ScaleDerived> &scale); + EIGEN_DEVICE_FUNC inline Transform inverse(TransformTraits traits = (TransformTraits)Mode) const; /** \returns a const pointer to the column major internal matrix */ - const Scalar* data() const { return m_matrix.data(); } + EIGEN_DEVICE_FUNC const Scalar* data() const { return m_matrix.data(); } /** \returns a non-const pointer to the column major internal matrix */ - Scalar* data() { return m_matrix.data(); } + EIGEN_DEVICE_FUNC Scalar* data() { return m_matrix.data(); } /** \returns \c *this with scalar type casted to \a NewScalarType * @@ -609,12 +634,12 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<Transform,Transform<NewScalarType,Dim,Mode,Options> >::type cast() const + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<Transform,Transform<NewScalarType,Dim,Mode,Options> >::type cast() const { return typename internal::cast_return_type<Transform,Transform<NewScalarType,Dim,Mode,Options> >::type(*this); } /** Copy constructor with scalar type conversion */ template<typename OtherScalarType> - inline explicit Transform(const Transform<OtherScalarType,Dim,Mode,Options>& other) + EIGEN_DEVICE_FUNC inline explicit Transform(const Transform<OtherScalarType,Dim,Mode,Options>& other) { check_template_params(); m_matrix = other.matrix().template cast<Scalar>(); @@ -624,12 +649,12 @@ * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const Transform& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const Transform& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return m_matrix.isApprox(other.m_matrix, prec); } /** Sets the last row to [0 ... 0 1] */ - void makeAffine() + EIGEN_DEVICE_FUNC void makeAffine() { internal::transform_make_affine<int(Mode)>::run(m_matrix); } @@ -638,26 +663,26 @@ * \returns the Dim x Dim linear part if the transformation is affine, * and the HDim x Dim part for projective transformations. */ - inline Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,Dim> linearExt() + EIGEN_DEVICE_FUNC inline Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,Dim> linearExt() { return m_matrix.template block<int(Mode)==int(Projective)?HDim:Dim,Dim>(0,0); } /** \internal * \returns the Dim x Dim linear part if the transformation is affine, * and the HDim x Dim part for projective transformations. */ - inline const Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,Dim> linearExt() const + EIGEN_DEVICE_FUNC inline const Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,Dim> linearExt() const { return m_matrix.template block<int(Mode)==int(Projective)?HDim:Dim,Dim>(0,0); } /** \internal * \returns the translation part if the transformation is affine, * and the last column for projective transformations. */ - inline Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,1> translationExt() + EIGEN_DEVICE_FUNC inline Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,1> translationExt() { return m_matrix.template block<int(Mode)==int(Projective)?HDim:Dim,1>(0,Dim); } /** \internal * \returns the translation part if the transformation is affine, * and the last column for projective transformations. */ - inline const Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,1> translationExt() const + EIGEN_DEVICE_FUNC inline const Block<MatrixType,int(Mode)==int(Projective)?HDim:Dim,1> translationExt() const { return m_matrix.template block<int(Mode)==int(Projective)?HDim:Dim,1>(0,Dim); } @@ -667,7 +692,7 @@ protected: #ifndef EIGEN_PARSED_BY_DOXYGEN - static EIGEN_STRONG_INLINE void check_template_params() + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void check_template_params() { EIGEN_STATIC_ASSERT((Options & (DontAlign|RowMajor)) == Options, INVALID_MATRIX_TEMPLATE_PARAMETERS) } @@ -735,7 +760,7 @@ Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const QMatrix& other) { EIGEN_STATIC_ASSERT(Dim==2, YOU_MADE_A_PROGRAMMING_MISTAKE) - if (Mode == int(AffineCompact)) + if (EIGEN_CONST_CONDITIONAL(Mode == int(AffineCompact))) m_matrix << other.m11(), other.m21(), other.dx(), other.m12(), other.m22(), other.dy(); else @@ -781,7 +806,7 @@ { check_template_params(); EIGEN_STATIC_ASSERT(Dim==2, YOU_MADE_A_PROGRAMMING_MISTAKE) - if (Mode == int(AffineCompact)) + if (EIGEN_CONST_CONDITIONAL(Mode == int(AffineCompact))) m_matrix << other.m11(), other.m21(), other.dx(), other.m12(), other.m22(), other.dy(); else @@ -799,7 +824,7 @@ QTransform Transform<Scalar,Dim,Mode,Options>::toQTransform(void) const { EIGEN_STATIC_ASSERT(Dim==2, YOU_MADE_A_PROGRAMMING_MISTAKE) - if (Mode == int(AffineCompact)) + if (EIGEN_CONST_CONDITIONAL(Mode == int(AffineCompact))) return QTransform(m_matrix.coeff(0,0), m_matrix.coeff(1,0), m_matrix.coeff(0,1), m_matrix.coeff(1,1), m_matrix.coeff(0,2), m_matrix.coeff(1,2)); @@ -820,7 +845,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename OtherDerived> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::scale(const MatrixBase<OtherDerived> &other) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,int(Dim)) @@ -834,7 +859,7 @@ * \sa prescale(Scalar) */ template<typename Scalar, int Dim, int Mode, int Options> -inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::scale(const Scalar& s) +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::scale(const Scalar& s) { EIGEN_STATIC_ASSERT(Mode!=int(Isometry), THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS) linearExt() *= s; @@ -847,12 +872,12 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename OtherDerived> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::prescale(const MatrixBase<OtherDerived> &other) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,int(Dim)) EIGEN_STATIC_ASSERT(Mode!=int(Isometry), THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS) - m_matrix.template block<Dim,HDim>(0,0).noalias() = (other.asDiagonal() * m_matrix.template block<Dim,HDim>(0,0)); + affine().noalias() = (other.asDiagonal() * affine()); return *this; } @@ -861,7 +886,7 @@ * \sa scale(Scalar) */ template<typename Scalar, int Dim, int Mode, int Options> -inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::prescale(const Scalar& s) +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::prescale(const Scalar& s) { EIGEN_STATIC_ASSERT(Mode!=int(Isometry), THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS) m_matrix.template topRows<Dim>() *= s; @@ -874,7 +899,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename OtherDerived> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::translate(const MatrixBase<OtherDerived> &other) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,int(Dim)) @@ -888,11 +913,11 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename OtherDerived> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::pretranslate(const MatrixBase<OtherDerived> &other) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,int(Dim)) - if(int(Mode)==int(Projective)) + if(EIGEN_CONST_CONDITIONAL(int(Mode)==int(Projective))) affine() += other * m_matrix.row(Dim); else translation() += other; @@ -918,7 +943,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename RotationType> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::rotate(const RotationType& rotation) { linearExt() *= internal::toRotationMatrix<Scalar,Dim>(rotation); @@ -934,7 +959,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename RotationType> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::prerotate(const RotationType& rotation) { m_matrix.template block<Dim,HDim>(0,0) = internal::toRotationMatrix<Scalar,Dim>(rotation) @@ -948,7 +973,7 @@ * \sa preshear() */ template<typename Scalar, int Dim, int Mode, int Options> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::shear(const Scalar& sx, const Scalar& sy) { EIGEN_STATIC_ASSERT(int(Dim)==2, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -964,7 +989,7 @@ * \sa shear() */ template<typename Scalar, int Dim, int Mode, int Options> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::preshear(const Scalar& sx, const Scalar& sy) { EIGEN_STATIC_ASSERT(int(Dim)==2, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -978,7 +1003,7 @@ ******************************************************/ template<typename Scalar, int Dim, int Mode, int Options> -inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const TranslationType& t) +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const TranslationType& t) { linear().setIdentity(); translation() = t.vector(); @@ -987,7 +1012,7 @@ } template<typename Scalar, int Dim, int Mode, int Options> -inline Transform<Scalar,Dim,Mode,Options> Transform<Scalar,Dim,Mode,Options>::operator*(const TranslationType& t) const +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options> Transform<Scalar,Dim,Mode,Options>::operator*(const TranslationType& t) const { Transform res = *this; res.translate(t.vector()); @@ -995,7 +1020,7 @@ } template<typename Scalar, int Dim, int Mode, int Options> -inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const UniformScaling<Scalar>& s) +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const UniformScaling<Scalar>& s) { m_matrix.setZero(); linear().diagonal().fill(s.factor()); @@ -1005,7 +1030,7 @@ template<typename Scalar, int Dim, int Mode, int Options> template<typename Derived> -inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const RotationBase<Derived,Dim>& r) +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::operator=(const RotationBase<Derived,Dim>& r) { linear() = internal::toRotationMatrix<Scalar,Dim>(r); translation().setZero(); @@ -1015,7 +1040,7 @@ template<typename Scalar, int Dim, int Mode, int Options> template<typename Derived> -inline Transform<Scalar,Dim,Mode,Options> Transform<Scalar,Dim,Mode,Options>::operator*(const RotationBase<Derived,Dim>& r) const +EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode,Options> Transform<Scalar,Dim,Mode,Options>::operator*(const RotationBase<Derived,Dim>& r) const { Transform res = *this; res.rotate(r.derived()); @@ -1026,20 +1051,43 @@ *** Special functions *** ************************/ +namespace internal { +template<int Mode> struct transform_rotation_impl { + template<typename TransformType> + EIGEN_DEVICE_FUNC static inline + const typename TransformType::LinearMatrixType run(const TransformType& t) + { + typedef typename TransformType::LinearMatrixType LinearMatrixType; + LinearMatrixType result; + t.computeRotationScaling(&result, (LinearMatrixType*)0); + return result; + } +}; +template<> struct transform_rotation_impl<Isometry> { + template<typename TransformType> + EIGEN_DEVICE_FUNC static inline + typename TransformType::ConstLinearPart run(const TransformType& t) + { + return t.linear(); + } +}; +} /** \returns the rotation part of the transformation * + * If Mode==Isometry, then this method is an alias for linear(), + * otherwise it calls computeRotationScaling() to extract the rotation + * through a SVD decomposition. * * \svd_module * * \sa computeRotationScaling(), computeScalingRotation(), class SVD */ template<typename Scalar, int Dim, int Mode, int Options> -const typename Transform<Scalar,Dim,Mode,Options>::LinearMatrixType +EIGEN_DEVICE_FUNC +typename Transform<Scalar,Dim,Mode,Options>::RotationReturnType Transform<Scalar,Dim,Mode,Options>::rotation() const { - LinearMatrixType result; - computeRotationScaling(&result, (LinearMatrixType*)0); - return result; + return internal::transform_rotation_impl<Mode>::run(*this); } @@ -1056,7 +1104,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename RotationMatrixType, typename ScalingMatrixType> -void Transform<Scalar,Dim,Mode,Options>::computeRotationScaling(RotationMatrixType *rotation, ScalingMatrixType *scaling) const +EIGEN_DEVICE_FUNC void Transform<Scalar,Dim,Mode,Options>::computeRotationScaling(RotationMatrixType *rotation, ScalingMatrixType *scaling) const { JacobiSVD<LinearMatrixType> svd(linear(), ComputeFullU | ComputeFullV); @@ -1072,7 +1120,7 @@ } } -/** decomposes the linear part of the transformation as a product rotation x scaling, the scaling being +/** decomposes the linear part of the transformation as a product scaling x rotation, the scaling being * not necessarily positive. * * If either pointer is zero, the corresponding computation is skipped. @@ -1085,7 +1133,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename ScalingMatrixType, typename RotationMatrixType> -void Transform<Scalar,Dim,Mode,Options>::computeScalingRotation(ScalingMatrixType *scaling, RotationMatrixType *rotation) const +EIGEN_DEVICE_FUNC void Transform<Scalar,Dim,Mode,Options>::computeScalingRotation(ScalingMatrixType *scaling, RotationMatrixType *rotation) const { JacobiSVD<LinearMatrixType> svd(linear(), ComputeFullU | ComputeFullV); @@ -1106,7 +1154,7 @@ */ template<typename Scalar, int Dim, int Mode, int Options> template<typename PositionDerived, typename OrientationType, typename ScaleDerived> -Transform<Scalar,Dim,Mode,Options>& +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options>& Transform<Scalar,Dim,Mode,Options>::fromPositionOrientationScale(const MatrixBase<PositionDerived> &position, const OrientationType& orientation, const MatrixBase<ScaleDerived> &scale) { @@ -1123,7 +1171,7 @@ struct transform_make_affine { template<typename MatrixType> - static void run(MatrixType &mat) + EIGEN_DEVICE_FUNC static void run(MatrixType &mat) { static const int Dim = MatrixType::ColsAtCompileTime-1; mat.template block<1,Dim>(Dim,0).setZero(); @@ -1134,21 +1182,21 @@ template<> struct transform_make_affine<AffineCompact> { - template<typename MatrixType> static void run(MatrixType &) { } + template<typename MatrixType> EIGEN_DEVICE_FUNC static void run(MatrixType &) { } }; // selector needed to avoid taking the inverse of a 3x4 matrix template<typename TransformType, int Mode=TransformType::Mode> struct projective_transform_inverse { - static inline void run(const TransformType&, TransformType&) + EIGEN_DEVICE_FUNC static inline void run(const TransformType&, TransformType&) {} }; template<typename TransformType> struct projective_transform_inverse<TransformType, Projective> { - static inline void run(const TransformType& m, TransformType& res) + EIGEN_DEVICE_FUNC static inline void run(const TransformType& m, TransformType& res) { res.matrix() = m.matrix().inverse(); } @@ -1178,7 +1226,7 @@ * \sa MatrixBase::inverse() */ template<typename Scalar, int Dim, int Mode, int Options> -Transform<Scalar,Dim,Mode,Options> +EIGEN_DEVICE_FUNC Transform<Scalar,Dim,Mode,Options> Transform<Scalar,Dim,Mode,Options>::inverse(TransformTraits hint) const { Transform res; @@ -1287,8 +1335,8 @@ }; }; -template< typename TransformType, typename MatrixType > -struct transform_right_product_impl< TransformType, MatrixType, 0 > +template< typename TransformType, typename MatrixType, int RhsCols> +struct transform_right_product_impl< TransformType, MatrixType, 0, RhsCols> { typedef typename MatrixType::PlainObject ResultType; @@ -1298,8 +1346,8 @@ } }; -template< typename TransformType, typename MatrixType > -struct transform_right_product_impl< TransformType, MatrixType, 1 > +template< typename TransformType, typename MatrixType, int RhsCols> +struct transform_right_product_impl< TransformType, MatrixType, 1, RhsCols> { enum { Dim = TransformType::Dim, @@ -1324,8 +1372,8 @@ } }; -template< typename TransformType, typename MatrixType > -struct transform_right_product_impl< TransformType, MatrixType, 2 > +template< typename TransformType, typename MatrixType, int RhsCols> +struct transform_right_product_impl< TransformType, MatrixType, 2, RhsCols> { enum { Dim = TransformType::Dim, @@ -1348,6 +1396,30 @@ } }; +template< typename TransformType, typename MatrixType > +struct transform_right_product_impl< TransformType, MatrixType, 2, 1> // rhs is a vector of size Dim +{ + typedef typename TransformType::MatrixType TransformMatrix; + enum { + Dim = TransformType::Dim, + HDim = TransformType::HDim, + OtherRows = MatrixType::RowsAtCompileTime, + WorkingRows = EIGEN_PLAIN_ENUM_MIN(TransformMatrix::RowsAtCompileTime,HDim) + }; + + typedef typename MatrixType::PlainObject ResultType; + + static EIGEN_STRONG_INLINE ResultType run(const TransformType& T, const MatrixType& other) + { + EIGEN_STATIC_ASSERT(OtherRows==Dim, YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES); + + Matrix<typename ResultType::Scalar, Dim+1, 1> rhs; + rhs.template head<Dim>() = other; rhs[Dim] = typename ResultType::Scalar(1); + Matrix<typename ResultType::Scalar, WorkingRows, 1> res(T.matrix() * rhs); + return res.template head<Dim>(); + } +}; + /********************************************************** *** Specializations of operator* with lhs EigenBase *** **********************************************************/
diff --git a/Eigen/src/Geometry/Translation.h b/Eigen/src/Geometry/Translation.h index 82d7777..23b19f7 100644 --- a/Eigen/src/Geometry/Translation.h +++ b/Eigen/src/Geometry/Translation.h
@@ -51,16 +51,16 @@ public: /** Default constructor without initialization. */ - Translation() {} + EIGEN_DEVICE_FUNC Translation() {} /** */ - inline Translation(const Scalar& sx, const Scalar& sy) + EIGEN_DEVICE_FUNC inline Translation(const Scalar& sx, const Scalar& sy) { eigen_assert(Dim==2); m_coeffs.x() = sx; m_coeffs.y() = sy; } /** */ - inline Translation(const Scalar& sx, const Scalar& sy, const Scalar& sz) + EIGEN_DEVICE_FUNC inline Translation(const Scalar& sx, const Scalar& sy, const Scalar& sz) { eigen_assert(Dim==3); m_coeffs.x() = sx; @@ -68,48 +68,48 @@ m_coeffs.z() = sz; } /** Constructs and initialize the translation transformation from a vector of translation coefficients */ - explicit inline Translation(const VectorType& vector) : m_coeffs(vector) {} + EIGEN_DEVICE_FUNC explicit inline Translation(const VectorType& vector) : m_coeffs(vector) {} - /** \brief Retruns the x-translation by value. **/ - inline Scalar x() const { return m_coeffs.x(); } - /** \brief Retruns the y-translation by value. **/ - inline Scalar y() const { return m_coeffs.y(); } - /** \brief Retruns the z-translation by value. **/ - inline Scalar z() const { return m_coeffs.z(); } + /** \brief Returns the x-translation by value. **/ + EIGEN_DEVICE_FUNC inline Scalar x() const { return m_coeffs.x(); } + /** \brief Returns the y-translation by value. **/ + EIGEN_DEVICE_FUNC inline Scalar y() const { return m_coeffs.y(); } + /** \brief Returns the z-translation by value. **/ + EIGEN_DEVICE_FUNC inline Scalar z() const { return m_coeffs.z(); } - /** \brief Retruns the x-translation as a reference. **/ - inline Scalar& x() { return m_coeffs.x(); } - /** \brief Retruns the y-translation as a reference. **/ - inline Scalar& y() { return m_coeffs.y(); } - /** \brief Retruns the z-translation as a reference. **/ - inline Scalar& z() { return m_coeffs.z(); } + /** \brief Returns the x-translation as a reference. **/ + EIGEN_DEVICE_FUNC inline Scalar& x() { return m_coeffs.x(); } + /** \brief Returns the y-translation as a reference. **/ + EIGEN_DEVICE_FUNC inline Scalar& y() { return m_coeffs.y(); } + /** \brief Returns the z-translation as a reference. **/ + EIGEN_DEVICE_FUNC inline Scalar& z() { return m_coeffs.z(); } - const VectorType& vector() const { return m_coeffs; } - VectorType& vector() { return m_coeffs; } + EIGEN_DEVICE_FUNC const VectorType& vector() const { return m_coeffs; } + EIGEN_DEVICE_FUNC VectorType& vector() { return m_coeffs; } - const VectorType& translation() const { return m_coeffs; } - VectorType& translation() { return m_coeffs; } + EIGEN_DEVICE_FUNC const VectorType& translation() const { return m_coeffs; } + EIGEN_DEVICE_FUNC VectorType& translation() { return m_coeffs; } /** Concatenates two translation */ - inline Translation operator* (const Translation& other) const + EIGEN_DEVICE_FUNC inline Translation operator* (const Translation& other) const { return Translation(m_coeffs + other.m_coeffs); } /** Concatenates a translation and a uniform scaling */ - inline AffineTransformType operator* (const UniformScaling<Scalar>& other) const; + EIGEN_DEVICE_FUNC inline AffineTransformType operator* (const UniformScaling<Scalar>& other) const; /** Concatenates a translation and a linear transformation */ template<typename OtherDerived> - inline AffineTransformType operator* (const EigenBase<OtherDerived>& linear) const; + EIGEN_DEVICE_FUNC inline AffineTransformType operator* (const EigenBase<OtherDerived>& linear) const; /** Concatenates a translation and a rotation */ template<typename Derived> - inline IsometryTransformType operator*(const RotationBase<Derived,Dim>& r) const + EIGEN_DEVICE_FUNC inline IsometryTransformType operator*(const RotationBase<Derived,Dim>& r) const { return *this * IsometryTransformType(r); } /** \returns the concatenation of a linear transformation \a l with the translation \a t */ // its a nightmare to define a templated friend function outside its declaration template<typename OtherDerived> friend - inline AffineTransformType operator*(const EigenBase<OtherDerived>& linear, const Translation& t) + EIGEN_DEVICE_FUNC inline AffineTransformType operator*(const EigenBase<OtherDerived>& linear, const Translation& t) { AffineTransformType res; res.matrix().setZero(); @@ -122,7 +122,7 @@ /** Concatenates a translation and a transformation */ template<int Mode, int Options> - inline Transform<Scalar,Dim,Mode> operator* (const Transform<Scalar,Dim,Mode,Options>& t) const + EIGEN_DEVICE_FUNC inline Transform<Scalar,Dim,Mode> operator* (const Transform<Scalar,Dim,Mode,Options>& t) const { Transform<Scalar,Dim,Mode> res = t; res.pretranslate(m_coeffs); @@ -130,8 +130,10 @@ } /** Applies translation to vector */ - inline VectorType operator* (const VectorType& other) const - { return m_coeffs + other; } + template<typename Derived> + inline typename internal::enable_if<Derived::IsVectorAtCompileTime,VectorType>::type + operator* (const MatrixBase<Derived>& vec) const + { return m_coeffs + vec.derived(); } /** \returns the inverse translation (opposite) */ Translation inverse() const { return Translation(-m_coeffs); } @@ -150,19 +152,19 @@ * then this function smartly returns a const reference to \c *this. */ template<typename NewScalarType> - inline typename internal::cast_return_type<Translation,Translation<NewScalarType,Dim> >::type cast() const + EIGEN_DEVICE_FUNC inline typename internal::cast_return_type<Translation,Translation<NewScalarType,Dim> >::type cast() const { return typename internal::cast_return_type<Translation,Translation<NewScalarType,Dim> >::type(*this); } /** Copy constructor with scalar type conversion */ template<typename OtherScalarType> - inline explicit Translation(const Translation<OtherScalarType,Dim>& other) + EIGEN_DEVICE_FUNC inline explicit Translation(const Translation<OtherScalarType,Dim>& other) { m_coeffs = other.vector().template cast<Scalar>(); } /** \returns \c true if \c *this is approximately equal to \a other, within the precision * determined by \a prec. * * \sa MatrixBase::isApprox() */ - bool isApprox(const Translation& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const + EIGEN_DEVICE_FUNC bool isApprox(const Translation& other, const typename NumTraits<Scalar>::Real& prec = NumTraits<Scalar>::dummy_precision()) const { return m_coeffs.isApprox(other.m_coeffs, prec); } }; @@ -176,7 +178,7 @@ //@} template<typename Scalar, int Dim> -inline typename Translation<Scalar,Dim>::AffineTransformType +EIGEN_DEVICE_FUNC inline typename Translation<Scalar,Dim>::AffineTransformType Translation<Scalar,Dim>::operator* (const UniformScaling<Scalar>& other) const { AffineTransformType res; @@ -189,7 +191,7 @@ template<typename Scalar, int Dim> template<typename OtherDerived> -inline typename Translation<Scalar,Dim>::AffineTransformType +EIGEN_DEVICE_FUNC inline typename Translation<Scalar,Dim>::AffineTransformType Translation<Scalar,Dim>::operator* (const EigenBase<OtherDerived>& linear) const { AffineTransformType res;
diff --git a/Eigen/src/Geometry/arch/CMakeLists.txt b/Eigen/src/Geometry/arch/CMakeLists.txt deleted file mode 100644 index 1267a79..0000000 --- a/Eigen/src/Geometry/arch/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Geometry_arch_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Geometry_arch_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Geometry/arch COMPONENT Devel - )
diff --git a/Eigen/src/Geometry/arch/Geometry_SSE.h b/Eigen/src/Geometry/arch/Geometry_SSE.h index 1a86ff8..d4346aa 100644 --- a/Eigen/src/Geometry/arch/Geometry_SSE.h +++ b/Eigen/src/Geometry/arch/Geometry_SSE.h
@@ -16,34 +16,43 @@ namespace internal { template<class Derived, class OtherDerived> -struct quat_product<Architecture::SSE, Derived, OtherDerived, float, Aligned16> +struct quat_product<Architecture::SSE, Derived, OtherDerived, float> { + enum { + AAlignment = traits<Derived>::Alignment, + BAlignment = traits<OtherDerived>::Alignment, + ResAlignment = traits<Quaternion<float> >::Alignment + }; static inline Quaternion<float> run(const QuaternionBase<Derived>& _a, const QuaternionBase<OtherDerived>& _b) { Quaternion<float> res; - const __m128 mask = _mm_setr_ps(0.f,0.f,0.f,-0.f); - __m128 a = _a.coeffs().template packet<Aligned16>(0); - __m128 b = _b.coeffs().template packet<Aligned16>(0); - __m128 s1 = _mm_mul_ps(vec4f_swizzle1(a,1,2,0,2),vec4f_swizzle1(b,2,0,1,2)); - __m128 s2 = _mm_mul_ps(vec4f_swizzle1(a,3,3,3,1),vec4f_swizzle1(b,0,1,2,1)); - pstore(&res.x(), - _mm_add_ps(_mm_sub_ps(_mm_mul_ps(a,vec4f_swizzle1(b,3,3,3,3)), - _mm_mul_ps(vec4f_swizzle1(a,2,0,1,0), + const Packet4f mask = _mm_setr_ps(0.f,0.f,0.f,-0.f); + Packet4f a = _a.coeffs().template packet<AAlignment>(0); + Packet4f b = _b.coeffs().template packet<BAlignment>(0); + Packet4f s1 = pmul(vec4f_swizzle1(a,1,2,0,2),vec4f_swizzle1(b,2,0,1,2)); + Packet4f s2 = pmul(vec4f_swizzle1(a,3,3,3,1),vec4f_swizzle1(b,0,1,2,1)); + pstoret<float,Packet4f,ResAlignment>( + &res.x(), + padd(psub(pmul(a,vec4f_swizzle1(b,3,3,3,3)), + pmul(vec4f_swizzle1(a,2,0,1,0), vec4f_swizzle1(b,1,2,0,0))), - _mm_xor_ps(mask,_mm_add_ps(s1,s2)))); + pxor(mask,padd(s1,s2)))); return res; } }; -template<class Derived, int Alignment> -struct quat_conj<Architecture::SSE, Derived, float, Alignment> +template<class Derived> +struct quat_conj<Architecture::SSE, Derived, float> { + enum { + ResAlignment = traits<Quaternion<float> >::Alignment + }; static inline Quaternion<float> run(const QuaternionBase<Derived>& q) { Quaternion<float> res; const __m128 mask = _mm_setr_ps(-0.f,-0.f,-0.f,0.f); - pstore(&res.x(), _mm_xor_ps(mask, q.coeffs().template packet<Alignment>(0))); + pstoret<float,Packet4f,ResAlignment>(&res.x(), _mm_xor_ps(mask, q.coeffs().template packet<traits<Derived>::Alignment>(0))); return res; } }; @@ -52,6 +61,9 @@ template<typename VectorLhs,typename VectorRhs> struct cross3_impl<Architecture::SSE,VectorLhs,VectorRhs,float,true> { + enum { + ResAlignment = traits<typename plain_matrix_type<VectorLhs>::type>::Alignment + }; static inline typename plain_matrix_type<VectorLhs>::type run(const VectorLhs& lhs, const VectorRhs& rhs) { @@ -60,7 +72,7 @@ __m128 mul1=_mm_mul_ps(vec4f_swizzle1(a,1,2,0,3),vec4f_swizzle1(b,2,0,1,3)); __m128 mul2=_mm_mul_ps(vec4f_swizzle1(a,2,0,1,3),vec4f_swizzle1(b,1,2,0,3)); typename plain_matrix_type<VectorLhs>::type res; - pstore(&res.x(),_mm_sub_ps(mul1,mul2)); + pstoret<float,Packet4f,ResAlignment>(&res.x(),_mm_sub_ps(mul1,mul2)); return res; } }; @@ -68,9 +80,14 @@ -template<class Derived, class OtherDerived, int Alignment> -struct quat_product<Architecture::SSE, Derived, OtherDerived, double, Alignment> +template<class Derived, class OtherDerived> +struct quat_product<Architecture::SSE, Derived, OtherDerived, double> { + enum { + BAlignment = traits<OtherDerived>::Alignment, + ResAlignment = traits<Quaternion<double> >::Alignment + }; + static inline Quaternion<double> run(const QuaternionBase<Derived>& _a, const QuaternionBase<OtherDerived>& _b) { const Packet2d mask = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0)); @@ -78,8 +95,8 @@ Quaternion<double> res; const double* a = _a.coeffs().data(); - Packet2d b_xy = _b.coeffs().template packet<Alignment>(0); - Packet2d b_zw = _b.coeffs().template packet<Alignment>(2); + Packet2d b_xy = _b.coeffs().template packet<BAlignment>(0); + Packet2d b_zw = _b.coeffs().template packet<BAlignment>(2); Packet2d a_xx = pset1<Packet2d>(a[0]); Packet2d a_yy = pset1<Packet2d>(a[1]); Packet2d a_zz = pset1<Packet2d>(a[2]); @@ -97,9 +114,9 @@ t2 = psub(pmul(a_zz, b_xy), pmul(a_xx, b_zw)); #ifdef EIGEN_VECTORIZE_SSE3 EIGEN_UNUSED_VARIABLE(mask) - pstore(&res.x(), _mm_addsub_pd(t1, preverse(t2))); + pstoret<double,Packet2d,ResAlignment>(&res.x(), _mm_addsub_pd(t1, preverse(t2))); #else - pstore(&res.x(), padd(t1, pxor(mask,preverse(t2)))); + pstoret<double,Packet2d,ResAlignment>(&res.x(), padd(t1, pxor(mask,preverse(t2)))); #endif /* @@ -111,25 +128,28 @@ t2 = padd(pmul(a_zz, b_zw), pmul(a_xx, b_xy)); #ifdef EIGEN_VECTORIZE_SSE3 EIGEN_UNUSED_VARIABLE(mask) - pstore(&res.z(), preverse(_mm_addsub_pd(preverse(t1), t2))); + pstoret<double,Packet2d,ResAlignment>(&res.z(), preverse(_mm_addsub_pd(preverse(t1), t2))); #else - pstore(&res.z(), psub(t1, pxor(mask,preverse(t2)))); + pstoret<double,Packet2d,ResAlignment>(&res.z(), psub(t1, pxor(mask,preverse(t2)))); #endif return res; } }; -template<class Derived, int Alignment> -struct quat_conj<Architecture::SSE, Derived, double, Alignment> +template<class Derived> +struct quat_conj<Architecture::SSE, Derived, double> { + enum { + ResAlignment = traits<Quaternion<double> >::Alignment + }; static inline Quaternion<double> run(const QuaternionBase<Derived>& q) { Quaternion<double> res; const __m128d mask0 = _mm_setr_pd(-0.,-0.); const __m128d mask2 = _mm_setr_pd(-0.,0.); - pstore(&res.x(), _mm_xor_pd(mask0, q.coeffs().template packet<Alignment>(0))); - pstore(&res.z(), _mm_xor_pd(mask2, q.coeffs().template packet<Alignment>(2))); + pstoret<double,Packet2d,ResAlignment>(&res.x(), _mm_xor_pd(mask0, q.coeffs().template packet<traits<Derived>::Alignment>(0))); + pstoret<double,Packet2d,ResAlignment>(&res.z(), _mm_xor_pd(mask2, q.coeffs().template packet<traits<Derived>::Alignment>(2))); return res; } };
diff --git a/Eigen/src/Householder/BlockHouseholder.h b/Eigen/src/Householder/BlockHouseholder.h index 39bf8c8..39ce1c2 100644 --- a/Eigen/src/Householder/BlockHouseholder.h +++ b/Eigen/src/Householder/BlockHouseholder.h
@@ -63,8 +63,15 @@ triFactor.row(i).tail(rt).noalias() = -hCoeffs(i) * vectors.col(i).tail(rs).adjoint() * vectors.bottomRightCorner(rs, rt).template triangularView<UnitLower>(); - // FIXME add .noalias() once the triangular product can work inplace - triFactor.row(i).tail(rt) = triFactor.row(i).tail(rt) * triFactor.bottomRightCorner(rt,rt).template triangularView<Upper>(); + // FIXME use the following line with .noalias() once the triangular product can work inplace + // triFactor.row(i).tail(rt) = triFactor.row(i).tail(rt) * triFactor.bottomRightCorner(rt,rt).template triangularView<Upper>(); + for(Index j=nbVecs-1; j>i; --j) + { + typename TriangularFactorType::Scalar z = triFactor(i,j); + triFactor(i,j) = z * triFactor(j,j); + if(nbVecs-j-1>0) + triFactor.row(i).tail(nbVecs-j-1) += z * triFactor.row(j).tail(nbVecs-j-1); + } } triFactor(i,i) = hCoeffs(i); @@ -87,7 +94,8 @@ const TriangularView<const VectorsType, UnitLower> V(vectors); // A -= V T V^* A - Matrix<typename MatrixType::Scalar,VectorsType::ColsAtCompileTime,MatrixType::ColsAtCompileTime,0, + Matrix<typename MatrixType::Scalar,VectorsType::ColsAtCompileTime,MatrixType::ColsAtCompileTime, + (VectorsType::MaxColsAtCompileTime==1 && MatrixType::MaxColsAtCompileTime!=1)?RowMajor:ColMajor, VectorsType::MaxColsAtCompileTime,MatrixType::MaxColsAtCompileTime> tmp = V.adjoint() * mat; // FIXME add .noalias() once the triangular product can work inplace if(forward) tmp = T.template triangularView<Upper>() * tmp;
diff --git a/Eigen/src/Householder/CMakeLists.txt b/Eigen/src/Householder/CMakeLists.txt deleted file mode 100644 index ce4937d..0000000 --- a/Eigen/src/Householder/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Householder_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Householder_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Householder COMPONENT Devel - )
diff --git a/Eigen/src/Householder/Householder.h b/Eigen/src/Householder/Householder.h index 4c1f499..5bc037f 100644 --- a/Eigen/src/Householder/Householder.h +++ b/Eigen/src/Householder/Householder.h
@@ -39,6 +39,7 @@ * MatrixBase::applyHouseholderOnTheRight() */ template<typename Derived> +EIGEN_DEVICE_FUNC void MatrixBase<Derived>::makeHouseholderInPlace(Scalar& tau, RealScalar& beta) { VectorBlock<Derived, internal::decrement_size<Base::SizeAtCompileTime>::ret> essentialPart(derived(), 1, size()-1); @@ -62,6 +63,7 @@ */ template<typename Derived> template<typename EssentialPart> +EIGEN_DEVICE_FUNC void MatrixBase<Derived>::makeHouseholder( EssentialPart& essential, Scalar& tau, @@ -103,13 +105,14 @@ * \param essential the essential part of the vector \c v * \param tau the scaling factor of the Householder transformation * \param workspace a pointer to working space with at least - * this->cols() * essential.size() entries + * this->cols() entries * * \sa MatrixBase::makeHouseholder(), MatrixBase::makeHouseholderInPlace(), * MatrixBase::applyHouseholderOnTheRight() */ template<typename Derived> template<typename EssentialPart> +EIGEN_DEVICE_FUNC void MatrixBase<Derived>::applyHouseholderOnTheLeft( const EssentialPart& essential, const Scalar& tau, @@ -119,7 +122,7 @@ { *this *= Scalar(1)-tau; } - else + else if(tau!=Scalar(0)) { Map<typename internal::plain_row_type<PlainObject>::type> tmp(workspace,cols()); Block<Derived, EssentialPart::SizeAtCompileTime, Derived::ColsAtCompileTime> bottom(derived(), 1, 0, rows()-1, cols()); @@ -140,13 +143,14 @@ * \param essential the essential part of the vector \c v * \param tau the scaling factor of the Householder transformation * \param workspace a pointer to working space with at least - * this->cols() * essential.size() entries + * this->rows() entries * * \sa MatrixBase::makeHouseholder(), MatrixBase::makeHouseholderInPlace(), * MatrixBase::applyHouseholderOnTheLeft() */ template<typename Derived> template<typename EssentialPart> +EIGEN_DEVICE_FUNC void MatrixBase<Derived>::applyHouseholderOnTheRight( const EssentialPart& essential, const Scalar& tau, @@ -156,14 +160,14 @@ { *this *= Scalar(1)-tau; } - else + else if(tau!=Scalar(0)) { Map<typename internal::plain_col_type<PlainObject>::type> tmp(workspace,rows()); Block<Derived, Derived::RowsAtCompileTime, EssentialPart::SizeAtCompileTime> right(derived(), 0, 1, rows(), cols()-1); - tmp.noalias() = right * essential.conjugate(); + tmp.noalias() = right * essential; tmp += this->col(0); this->col(0) -= tau * tmp; - right.noalias() -= tau * tmp * essential.transpose(); + right.noalias() -= tau * tmp * essential.adjoint(); } }
diff --git a/Eigen/src/Householder/HouseholderSequence.h b/Eigen/src/Householder/HouseholderSequence.h index e9f3ebf..9318c28 100644 --- a/Eigen/src/Householder/HouseholderSequence.h +++ b/Eigen/src/Householder/HouseholderSequence.h
@@ -87,7 +87,7 @@ { typedef Block<const VectorsType, Dynamic, 1> EssentialVectorType; typedef HouseholderSequence<VectorsType, CoeffsType, OnTheLeft> HouseholderSequenceType; - static inline const EssentialVectorType essentialVector(const HouseholderSequenceType& h, Index k) + static EIGEN_DEVICE_FUNC inline const EssentialVectorType essentialVector(const HouseholderSequenceType& h, Index k) { Index start = k+1+h.m_shift; return Block<const VectorsType,Dynamic,1>(h.m_vectors, start, k, h.rows()-start, 1); @@ -108,7 +108,7 @@ template<typename OtherScalarType, typename MatrixType> struct matrix_type_times_scalar_type { - typedef typename scalar_product_traits<OtherScalarType, typename MatrixType::Scalar>::ReturnType + typedef typename ScalarBinaryOpTraits<OtherScalarType, typename MatrixType::Scalar>::ReturnType ResultScalar; typedef Matrix<ResultScalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime, 0, MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime> Type; @@ -140,6 +140,28 @@ Side > ConjugateReturnType; + typedef HouseholderSequence< + VectorsType, + typename internal::conditional<NumTraits<Scalar>::IsComplex, + typename internal::remove_all<typename CoeffsType::ConjugateReturnType>::type, + CoeffsType>::type, + Side + > AdjointReturnType; + + typedef HouseholderSequence< + typename internal::conditional<NumTraits<Scalar>::IsComplex, + typename internal::remove_all<typename VectorsType::ConjugateReturnType>::type, + VectorsType>::type, + CoeffsType, + Side + > TransposeReturnType; + + typedef HouseholderSequence< + typename internal::add_const<VectorsType>::type, + typename internal::add_const<CoeffsType>::type, + Side + > ConstHouseholderSequence; + /** \brief Constructor. * \param[in] v %Matrix containing the essential parts of the Householder vectors * \param[in] h Vector containing the Householder coefficients @@ -157,17 +179,19 @@ * * \sa setLength(), setShift() */ + EIGEN_DEVICE_FUNC HouseholderSequence(const VectorsType& v, const CoeffsType& h) - : m_vectors(v), m_coeffs(h), m_trans(false), m_length(v.diagonalSize()), + : m_vectors(v), m_coeffs(h), m_reverse(false), m_length(v.diagonalSize()), m_shift(0) { } /** \brief Copy constructor. */ + EIGEN_DEVICE_FUNC HouseholderSequence(const HouseholderSequence& other) : m_vectors(other.m_vectors), m_coeffs(other.m_coeffs), - m_trans(other.m_trans), + m_reverse(other.m_reverse), m_length(other.m_length), m_shift(other.m_shift) { @@ -177,12 +201,14 @@ * \returns Number of rows * \details This equals the dimension of the space that the transformation acts on. */ + EIGEN_DEVICE_FUNC Index rows() const { return Side==OnTheLeft ? m_vectors.rows() : m_vectors.cols(); } /** \brief Number of columns of transformation viewed as a matrix. * \returns Number of columns * \details This equals the dimension of the space that the transformation acts on. */ + EIGEN_DEVICE_FUNC Index cols() const { return rows(); } /** \brief Essential part of a Householder vector. @@ -199,6 +225,7 @@ * * \sa setShift(), shift() */ + EIGEN_DEVICE_FUNC const EssentialVectorType essentialVector(Index k) const { eigen_assert(k >= 0 && k < m_length); @@ -206,31 +233,51 @@ } /** \brief %Transpose of the Householder sequence. */ - HouseholderSequence transpose() const + TransposeReturnType transpose() const { - return HouseholderSequence(*this).setTrans(!m_trans); + return TransposeReturnType(m_vectors.conjugate(), m_coeffs) + .setReverseFlag(!m_reverse) + .setLength(m_length) + .setShift(m_shift); } /** \brief Complex conjugate of the Householder sequence. */ ConjugateReturnType conjugate() const { return ConjugateReturnType(m_vectors.conjugate(), m_coeffs.conjugate()) - .setTrans(m_trans) + .setReverseFlag(m_reverse) .setLength(m_length) .setShift(m_shift); } - /** \brief Adjoint (conjugate transpose) of the Householder sequence. */ - ConjugateReturnType adjoint() const + /** \returns an expression of the complex conjugate of \c *this if Cond==true, + * returns \c *this otherwise. + */ + template<bool Cond> + EIGEN_DEVICE_FUNC + inline typename internal::conditional<Cond,ConjugateReturnType,ConstHouseholderSequence>::type + conjugateIf() const { - return conjugate().setTrans(!m_trans); + typedef typename internal::conditional<Cond,ConjugateReturnType,ConstHouseholderSequence>::type ReturnType; + return ReturnType(m_vectors.template conjugateIf<Cond>(), m_coeffs.template conjugateIf<Cond>()); + } + + /** \brief Adjoint (conjugate transpose) of the Householder sequence. */ + AdjointReturnType adjoint() const + { + return AdjointReturnType(m_vectors, m_coeffs.conjugate()) + .setReverseFlag(!m_reverse) + .setLength(m_length) + .setShift(m_shift); } /** \brief Inverse of the Householder sequence (equals the adjoint). */ - ConjugateReturnType inverse() const { return adjoint(); } + AdjointReturnType inverse() const { return adjoint(); } /** \internal */ - template<typename DestType> inline void evalTo(DestType& dst) const + template<typename DestType> + inline EIGEN_DEVICE_FUNC + void evalTo(DestType& dst) const { Matrix<Scalar, DestType::RowsAtCompileTime, 1, AutoAlign|ColMajor, DestType::MaxRowsAtCompileTime, 1> workspace(rows()); @@ -239,11 +286,12 @@ /** \internal */ template<typename Dest, typename Workspace> + EIGEN_DEVICE_FUNC void evalTo(Dest& dst, Workspace& workspace) const { workspace.resize(rows()); Index vecs = m_length; - if(is_same_dense(dst,m_vectors)) + if(internal::is_same_dense(dst,m_vectors)) { // in-place dst.diagonal().setOnes(); @@ -251,7 +299,7 @@ for(Index k = vecs-1; k >= 0; --k) { Index cornerSize = rows() - k - m_shift; - if(m_trans) + if(m_reverse) dst.bottomRightCorner(cornerSize, cornerSize) .applyHouseholderOnTheRight(essentialVector(k), m_coeffs.coeff(k), workspace.data()); else @@ -265,18 +313,26 @@ for(Index k = 0; k<cols()-vecs ; ++k) dst.col(k).tail(rows()-k-1).setZero(); } + else if(m_length>BlockSize) + { + dst.setIdentity(rows(), rows()); + if(m_reverse) + applyThisOnTheLeft(dst,workspace,true); + else + applyThisOnTheLeft(dst,workspace,true); + } else { dst.setIdentity(rows(), rows()); for(Index k = vecs-1; k >= 0; --k) { Index cornerSize = rows() - k - m_shift; - if(m_trans) + if(m_reverse) dst.bottomRightCorner(cornerSize, cornerSize) - .applyHouseholderOnTheRight(essentialVector(k), m_coeffs.coeff(k), &workspace.coeffRef(0)); + .applyHouseholderOnTheRight(essentialVector(k), m_coeffs.coeff(k), workspace.data()); else dst.bottomRightCorner(cornerSize, cornerSize) - .applyHouseholderOnTheLeft(essentialVector(k), m_coeffs.coeff(k), &workspace.coeffRef(0)); + .applyHouseholderOnTheLeft(essentialVector(k), m_coeffs.coeff(k), workspace.data()); } } } @@ -295,31 +351,34 @@ workspace.resize(dst.rows()); for(Index k = 0; k < m_length; ++k) { - Index actual_k = m_trans ? m_length-k-1 : k; + Index actual_k = m_reverse ? m_length-k-1 : k; dst.rightCols(rows()-m_shift-actual_k) .applyHouseholderOnTheRight(essentialVector(actual_k), m_coeffs.coeff(actual_k), workspace.data()); } } /** \internal */ - template<typename Dest> inline void applyThisOnTheLeft(Dest& dst) const + template<typename Dest> inline void applyThisOnTheLeft(Dest& dst, bool inputIsIdentity = false) const { - Matrix<Scalar,1,Dest::ColsAtCompileTime,RowMajor,1,Dest::MaxColsAtCompileTime> workspace(dst.cols()); - applyThisOnTheLeft(dst, workspace); + Matrix<Scalar,1,Dest::ColsAtCompileTime,RowMajor,1,Dest::MaxColsAtCompileTime> workspace; + applyThisOnTheLeft(dst, workspace, inputIsIdentity); } /** \internal */ template<typename Dest, typename Workspace> - inline void applyThisOnTheLeft(Dest& dst, Workspace& workspace) const + inline void applyThisOnTheLeft(Dest& dst, Workspace& workspace, bool inputIsIdentity = false) const { - const Index BlockSize = 48; + if(inputIsIdentity && m_reverse) + inputIsIdentity = false; // if the entries are large enough, then apply the reflectors by block if(m_length>=BlockSize && dst.cols()>1) { - for(Index i = 0; i < m_length; i+=BlockSize) + // Make sure we have at least 2 useful blocks, otherwise it is point-less: + Index blockSize = m_length<Index(2*BlockSize) ? (m_length+1)/2 : Index(BlockSize); + for(Index i = 0; i < m_length; i+=blockSize) { - Index end = m_trans ? (std::min)(m_length,i+BlockSize) : m_length-i; - Index k = m_trans ? i : (std::max)(Index(0),end-BlockSize); + Index end = m_reverse ? (std::min)(m_length,i+blockSize) : m_length-i; + Index k = m_reverse ? i : (std::max)(Index(0),end-blockSize); Index bs = end-k; Index start = k + m_shift; @@ -329,8 +388,15 @@ Side==OnTheRight ? bs : m_vectors.rows()-start, Side==OnTheRight ? m_vectors.cols()-start : bs); typename internal::conditional<Side==OnTheRight, Transpose<SubVectorsType>, SubVectorsType&>::type sub_vecs(sub_vecs1); - Block<Dest,Dynamic,Dynamic> sub_dst(dst,dst.rows()-rows()+m_shift+k,0, rows()-m_shift-k,dst.cols()); - apply_block_householder_on_the_left(sub_dst, sub_vecs, m_coeffs.segment(k, bs), !m_trans); + + Index dstStart = dst.rows()-rows()+m_shift+k; + Index dstRows = rows()-m_shift-k; + Block<Dest,Dynamic,Dynamic> sub_dst(dst, + dstStart, + inputIsIdentity ? dstStart : 0, + dstRows, + inputIsIdentity ? dstRows : dst.cols()); + apply_block_householder_on_the_left(sub_dst, sub_vecs, m_coeffs.segment(k, bs), !m_reverse); } } else @@ -338,8 +404,9 @@ workspace.resize(dst.cols()); for(Index k = 0; k < m_length; ++k) { - Index actual_k = m_trans ? k : m_length-k-1; - dst.bottomRows(rows()-m_shift-actual_k) + Index actual_k = m_reverse ? k : m_length-k-1; + Index dstStart = rows()-m_shift-actual_k; + dst.bottomRightCorner(dstStart, inputIsIdentity ? dstStart : dst.cols()) .applyHouseholderOnTheLeft(essentialVector(actual_k), m_coeffs.coeff(actual_k), workspace.data()); } } @@ -357,7 +424,7 @@ { typename internal::matrix_type_times_scalar_type<Scalar, OtherDerived>::Type res(other.template cast<typename internal::matrix_type_times_scalar_type<Scalar,OtherDerived>::ResultScalar>()); - applyThisOnTheLeft(res); + applyThisOnTheLeft(res, internal::is_identity<OtherDerived>::value && res.rows()==res.cols()); return res; } @@ -372,6 +439,7 @@ * * \sa length() */ + EIGEN_DEVICE_FUNC HouseholderSequence& setLength(Index length) { m_length = length; @@ -389,13 +457,17 @@ * * \sa shift() */ + EIGEN_DEVICE_FUNC HouseholderSequence& setShift(Index shift) { m_shift = shift; return *this; } + EIGEN_DEVICE_FUNC Index length() const { return m_length; } /**< \brief Returns the length of the Householder sequence. */ + + EIGEN_DEVICE_FUNC Index shift() const { return m_shift; } /**< \brief Returns the shift of the Householder sequence. */ /* Necessary for .adjoint() and .conjugate() */ @@ -403,27 +475,30 @@ protected: - /** \brief Sets the transpose flag. - * \param [in] trans New value of the transpose flag. + /** \internal + * \brief Sets the reverse flag. + * \param [in] reverse New value of the reverse flag. * - * By default, the transpose flag is not set. If the transpose flag is set, then this object represents - * \f$ H^T = H_{n-1}^T \ldots H_1^T H_0^T \f$ instead of \f$ H = H_0 H_1 \ldots H_{n-1} \f$. + * By default, the reverse flag is not set. If the reverse flag is set, then this object represents + * \f$ H^r = H_{n-1} \ldots H_1 H_0 \f$ instead of \f$ H = H_0 H_1 \ldots H_{n-1} \f$. + * \note For real valued HouseholderSequence this is equivalent to transposing \f$ H \f$. * - * \sa trans() + * \sa reverseFlag(), transpose(), adjoint() */ - HouseholderSequence& setTrans(bool trans) + HouseholderSequence& setReverseFlag(bool reverse) { - m_trans = trans; + m_reverse = reverse; return *this; } - bool trans() const { return m_trans; } /**< \brief Returns the transpose flag. */ + bool reverseFlag() const { return m_reverse; } /**< \internal \brief Returns the reverse flag. */ typename VectorsType::Nested m_vectors; typename CoeffsType::Nested m_coeffs; - bool m_trans; + bool m_reverse; Index m_length; Index m_shift; + enum { BlockSize = 48 }; }; /** \brief Computes the product of a matrix with a Householder sequence.
diff --git a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h index 358444a..f66c846 100644 --- a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h +++ b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h
@@ -152,13 +152,28 @@ { // Compute the inverse squared-norm of each column of mat m_invdiag.resize(mat.cols()); - for(Index j=0; j<mat.outerSize(); ++j) + if(MatType::IsRowMajor) { - RealScalar sum = mat.innerVector(j).squaredNorm(); - if(sum>0) - m_invdiag(j) = RealScalar(1)/sum; - else - m_invdiag(j) = RealScalar(1); + m_invdiag.setZero(); + for(Index j=0; j<mat.outerSize(); ++j) + { + for(typename MatType::InnerIterator it(mat,j); it; ++it) + m_invdiag(it.index()) += numext::abs2(it.value()); + } + for(Index j=0; j<mat.cols(); ++j) + if(numext::real(m_invdiag(j))>RealScalar(0)) + m_invdiag(j) = RealScalar(1)/numext::real(m_invdiag(j)); + } + else + { + for(Index j=0; j<mat.outerSize(); ++j) + { + RealScalar sum = mat.col(j).squaredNorm(); + if(sum>RealScalar(0)) + m_invdiag(j) = RealScalar(1)/sum; + else + m_invdiag(j) = RealScalar(1); + } } Base::m_isInitialized = true; return *this;
diff --git a/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h b/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h index 454f468..153acef 100644 --- a/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h +++ b/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h
@@ -191,32 +191,16 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const { - bool failed = false; - for(Index j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - if(!internal::bicgstab(matrix(), b.col(j), xj, Base::m_preconditioner, m_iterations, m_error)) - failed = true; - } - m_info = failed ? NumericalIssue + m_iterations = Base::maxIterations(); + m_error = Base::m_tolerance; + + bool ret = internal::bicgstab(matrix(), b, x, Base::m_preconditioner, m_iterations, m_error); + + m_info = (!ret) ? NumericalIssue : m_error <= Base::m_tolerance ? Success : NoConvergence; - m_isInitialized = true; - } - - /** \internal */ - using Base::_solve_impl; - template<typename Rhs,typename Dest> - void _solve_impl(const MatrixBase<Rhs>& b, Dest& x) const - { - x.resize(this->rows(),b.cols()); - x.setZero(); - _solve_with_guess_impl(b,x); } protected:
diff --git a/Eigen/src/IterativeLinearSolvers/CMakeLists.txt b/Eigen/src/IterativeLinearSolvers/CMakeLists.txt deleted file mode 100644 index 59ccc00..0000000 --- a/Eigen/src/IterativeLinearSolvers/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_IterativeLinearSolvers_SRCS "*.h") - -INSTALL(FILES - ${Eigen_IterativeLinearSolvers_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/IterativeLinearSolvers COMPONENT Devel - )
diff --git a/Eigen/src/IterativeLinearSolvers/ConjugateGradient.h b/Eigen/src/IterativeLinearSolvers/ConjugateGradient.h index 395daa8..96e8b9f 100644 --- a/Eigen/src/IterativeLinearSolvers/ConjugateGradient.h +++ b/Eigen/src/IterativeLinearSolvers/ConjugateGradient.h
@@ -50,7 +50,8 @@ tol_error = 0; return; } - RealScalar threshold = tol*tol*rhsNorm2; + const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); + RealScalar threshold = numext::maxi(tol*tol*rhsNorm2,considerAsZero); RealScalar residualNorm2 = residual.squaredNorm(); if (residualNorm2 < threshold) { @@ -58,7 +59,7 @@ tol_error = sqrt(residualNorm2 / rhsNorm2); return; } - + VectorType p(n); p = precond.solve(residual); // initial search direction @@ -194,7 +195,7 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const { typedef typename Base::MatrixWrapper MatrixWrapper; typedef typename Base::ActualMatrixType ActualMatrixType; @@ -210,31 +211,14 @@ RowMajorWrapper, typename MatrixWrapper::template ConstSelfAdjointViewReturnType<UpLo>::Type >::type SelfAdjointWrapper; + m_iterations = Base::maxIterations(); m_error = Base::m_tolerance; - for(Index j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - RowMajorWrapper row_mat(matrix()); - internal::conjugate_gradient(SelfAdjointWrapper(row_mat), b.col(j), xj, Base::m_preconditioner, m_iterations, m_error); - } - - m_isInitialized = true; + RowMajorWrapper row_mat(matrix()); + internal::conjugate_gradient(SelfAdjointWrapper(row_mat), b, x, Base::m_preconditioner, m_iterations, m_error); m_info = m_error <= Base::m_tolerance ? Success : NoConvergence; } - - /** \internal */ - using Base::_solve_impl; - template<typename Rhs,typename Dest> - void _solve_impl(const MatrixBase<Rhs>& b, Dest& x) const - { - x.setZero(); - _solve_with_guess_impl(b.derived(),x); - } protected:
diff --git a/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h b/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h index e45c272..5a827c5 100644 --- a/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h +++ b/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h
@@ -76,12 +76,12 @@ * * \sa IncompleteCholesky(const MatrixType&) */ - IncompleteCholesky() : m_initialShift(1e-3),m_factorizationIsOk(false) {} + IncompleteCholesky() : m_initialShift(1e-3),m_analysisIsOk(false),m_factorizationIsOk(false) {} /** Constructor computing the incomplete factorization for the given matrix \a matrix. */ template<typename MatrixType> - IncompleteCholesky(const MatrixType& matrix) : m_initialShift(1e-3),m_factorizationIsOk(false) + IncompleteCholesky(const MatrixType& matrix) : m_initialShift(1e-3),m_analysisIsOk(false),m_factorizationIsOk(false) { compute(matrix); }
diff --git a/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h b/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h index 338e6f1..43bd8e8 100644 --- a/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h +++ b/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h
@@ -136,7 +136,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix.appears to be negative. */ ComputationInfo info() const @@ -230,7 +230,7 @@ SparseMatrix<Scalar,ColMajor, StorageIndex> mat1 = amat; SparseMatrix<Scalar,ColMajor, StorageIndex> mat2 = amat.transpose(); // FIXME for a matrix with nearly symmetric pattern, mat2+mat1 is the appropriate choice. - // on the other hand for a really non-symmetric pattern, mat2*mat1 should be prefered... + // on the other hand for a really non-symmetric pattern, mat2*mat1 should be preferred... SparseMatrix<Scalar,ColMajor, StorageIndex> AtA = mat2 + mat1; AMDOrdering<StorageIndex> ordering; ordering(AtA,m_P);
diff --git a/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h b/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h index 3d62fef..13ba9a5 100644 --- a/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h +++ b/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h
@@ -275,7 +275,7 @@ const Preconditioner& preconditioner() const { return m_preconditioner; } /** \returns the max number of iterations. - * It is either the value setted by setMaxIterations or, by default, + * It is either the value set by setMaxIterations or, by default, * twice the number of columns of the matrix. */ Index maxIterations() const @@ -330,25 +330,77 @@ } /** \internal */ - template<typename Rhs, typename DestScalar, int DestOptions, typename DestIndex> - void _solve_impl(const Rhs& b, SparseMatrix<DestScalar,DestOptions,DestIndex> &dest) const + template<typename Rhs, typename DestDerived> + void _solve_with_guess_impl(const Rhs& b, SparseMatrixBase<DestDerived> &aDest) const { eigen_assert(rows()==b.rows()); Index rhsCols = b.cols(); Index size = b.rows(); + DestDerived& dest(aDest.derived()); + typedef typename DestDerived::Scalar DestScalar; Eigen::Matrix<DestScalar,Dynamic,1> tb(size); Eigen::Matrix<DestScalar,Dynamic,1> tx(cols()); // We do not directly fill dest because sparse expressions have to be free of aliasing issue. // For non square least-square problems, b and dest might not have the same size whereas they might alias each-other. - SparseMatrix<DestScalar,DestOptions,DestIndex> tmp(cols(),rhsCols); + typename DestDerived::PlainObject tmp(cols(),rhsCols); + ComputationInfo global_info = Success; for(Index k=0; k<rhsCols; ++k) { tb = b.col(k); - tx = derived().solve(tb); + tx = dest.col(k); + derived()._solve_vector_with_guess_impl(tb,tx); tmp.col(k) = tx.sparseView(0); + + // The call to _solve_vector_with_guess_impl updates m_info, so if it failed for a previous column + // we need to restore it to the worst value. + if(m_info==NumericalIssue) + global_info = NumericalIssue; + else if(m_info==NoConvergence) + global_info = NoConvergence; } - tmp.swap(dest); + m_info = global_info; + dest.swap(tmp); + } + + template<typename Rhs, typename DestDerived> + typename internal::enable_if<Rhs::ColsAtCompileTime!=1 && DestDerived::ColsAtCompileTime!=1>::type + _solve_with_guess_impl(const Rhs& b, MatrixBase<DestDerived> &aDest) const + { + eigen_assert(rows()==b.rows()); + + Index rhsCols = b.cols(); + DestDerived& dest(aDest.derived()); + ComputationInfo global_info = Success; + for(Index k=0; k<rhsCols; ++k) + { + typename DestDerived::ColXpr xk(dest,k); + typename Rhs::ConstColXpr bk(b,k); + derived()._solve_vector_with_guess_impl(bk,xk); + + // The call to _solve_vector_with_guess updates m_info, so if it failed for a previous column + // we need to restore it to the worst value. + if(m_info==NumericalIssue) + global_info = NumericalIssue; + else if(m_info==NoConvergence) + global_info = NoConvergence; + } + m_info = global_info; + } + + template<typename Rhs, typename DestDerived> + typename internal::enable_if<Rhs::ColsAtCompileTime==1 || DestDerived::ColsAtCompileTime==1>::type + _solve_with_guess_impl(const Rhs& b, MatrixBase<DestDerived> &dest) const + { + derived()._solve_vector_with_guess_impl(b,dest.derived()); + } + + /** \internal default initial guess = 0 */ + template<typename Rhs,typename Dest> + void _solve_impl(const Rhs& b, Dest& x) const + { + x.setZero(); + derived()._solve_with_guess_impl(b,x); } protected:
diff --git a/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h b/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h index 0aea0e0..203fd0e 100644 --- a/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h +++ b/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h
@@ -182,32 +182,14 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const { m_iterations = Base::maxIterations(); m_error = Base::m_tolerance; - for(Index j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - internal::least_square_conjugate_gradient(matrix(), b.col(j), xj, Base::m_preconditioner, m_iterations, m_error); - } - - m_isInitialized = true; + internal::least_square_conjugate_gradient(matrix(), b, x, Base::m_preconditioner, m_iterations, m_error); m_info = m_error <= Base::m_tolerance ? Success : NoConvergence; } - - /** \internal */ - using Base::_solve_impl; - template<typename Rhs,typename Dest> - void _solve_impl(const MatrixBase<Rhs>& b, Dest& x) const - { - x.setZero(); - _solve_with_guess_impl(b.derived(),x); - } };
diff --git a/Eigen/src/IterativeLinearSolvers/SolveWithGuess.h b/Eigen/src/IterativeLinearSolvers/SolveWithGuess.h index 35923be..79e1e48 100644 --- a/Eigen/src/IterativeLinearSolvers/SolveWithGuess.h +++ b/Eigen/src/IterativeLinearSolvers/SolveWithGuess.h
@@ -44,6 +44,7 @@ typedef typename internal::traits<SolveWithGuess>::Scalar Scalar; typedef typename internal::traits<SolveWithGuess>::PlainObject PlainObject; typedef typename internal::generic_xpr_base<SolveWithGuess<Decomposition,RhsType,GuessType>, MatrixXpr, typename internal::traits<RhsType>::StorageKind>::type Base; + typedef typename internal::ref_selector<SolveWithGuess>::type Nested; SolveWithGuess(const Decomposition &dec, const RhsType &rhs, const GuessType &guess) : m_dec(dec), m_rhs(rhs), m_guess(guess) @@ -81,7 +82,8 @@ : m_result(solve.rows(), solve.cols()) { ::new (static_cast<Base*>(this)) Base(m_result); - solve.dec()._solve_with_guess_impl(solve.rhs(), m_result, solve().guess()); + m_result = solve.guess(); + solve.dec()._solve_with_guess_impl(solve.rhs(), m_result); } protected: @@ -91,18 +93,22 @@ // Specialization for "dst = dec.solveWithGuess(rhs)" // NOTE we need to specialize it for Dense2Dense to avoid ambiguous specialization error and a Sparse2Sparse specialization must exist somewhere template<typename DstXprType, typename DecType, typename RhsType, typename GuessType, typename Scalar> -struct Assignment<DstXprType, SolveWithGuess<DecType,RhsType,GuessType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +struct Assignment<DstXprType, SolveWithGuess<DecType,RhsType,GuessType>, internal::assign_op<Scalar,Scalar>, Dense2Dense> { typedef SolveWithGuess<DecType,RhsType,GuessType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { - // FIXME shall we resize dst here? + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + dst = src.guess(); src.dec()._solve_with_guess_impl(src.rhs(), dst/*, src.guess()*/); } }; -} // end namepsace internal +} // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Jacobi/CMakeLists.txt b/Eigen/src/Jacobi/CMakeLists.txt deleted file mode 100644 index 490dac6..0000000 --- a/Eigen/src/Jacobi/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Jacobi_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Jacobi_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/Jacobi COMPONENT Devel - )
diff --git a/Eigen/src/Jacobi/Jacobi.h b/Eigen/src/Jacobi/Jacobi.h index 55de15e..bfb9dcb 100644 --- a/Eigen/src/Jacobi/Jacobi.h +++ b/Eigen/src/Jacobi/Jacobi.h
@@ -11,7 +11,7 @@ #ifndef EIGEN_JACOBI_H #define EIGEN_JACOBI_H -namespace Eigen { +namespace Eigen { /** \ingroup Jacobi_Module * \jacobi_module @@ -37,17 +37,20 @@ typedef typename NumTraits<Scalar>::Real RealScalar; /** Default constructor without any initialization. */ + EIGEN_DEVICE_FUNC JacobiRotation() {} /** Construct a planar rotation from a cosine-sine pair (\a c, \c s). */ + EIGEN_DEVICE_FUNC JacobiRotation(const Scalar& c, const Scalar& s) : m_c(c), m_s(s) {} - Scalar& c() { return m_c; } - Scalar c() const { return m_c; } - Scalar& s() { return m_s; } - Scalar s() const { return m_s; } + EIGEN_DEVICE_FUNC Scalar& c() { return m_c; } + EIGEN_DEVICE_FUNC Scalar c() const { return m_c; } + EIGEN_DEVICE_FUNC Scalar& s() { return m_s; } + EIGEN_DEVICE_FUNC Scalar s() const { return m_s; } /** Concatenates two planar rotation */ + EIGEN_DEVICE_FUNC JacobiRotation operator*(const JacobiRotation& other) { using numext::conj; @@ -56,20 +59,27 @@ } /** Returns the transposed transformation */ + EIGEN_DEVICE_FUNC JacobiRotation transpose() const { using numext::conj; return JacobiRotation(m_c, -conj(m_s)); } /** Returns the adjoint transformation */ + EIGEN_DEVICE_FUNC JacobiRotation adjoint() const { using numext::conj; return JacobiRotation(conj(m_c), -m_s); } template<typename Derived> + EIGEN_DEVICE_FUNC bool makeJacobi(const MatrixBase<Derived>&, Index p, Index q); + EIGEN_DEVICE_FUNC bool makeJacobi(const RealScalar& x, const Scalar& y, const RealScalar& z); - void makeGivens(const Scalar& p, const Scalar& q, Scalar* z=0); + EIGEN_DEVICE_FUNC + void makeGivens(const Scalar& p, const Scalar& q, Scalar* r=0); protected: - void makeGivens(const Scalar& p, const Scalar& q, Scalar* z, internal::true_type); - void makeGivens(const Scalar& p, const Scalar& q, Scalar* z, internal::false_type); + EIGEN_DEVICE_FUNC + void makeGivens(const Scalar& p, const Scalar& q, Scalar* r, internal::true_type); + EIGEN_DEVICE_FUNC + void makeGivens(const Scalar& p, const Scalar& q, Scalar* r, internal::false_type); Scalar m_c, m_s; }; @@ -80,12 +90,14 @@ * \sa MatrixBase::makeJacobi(const MatrixBase<Derived>&, Index, Index), MatrixBase::applyOnTheLeft(), MatrixBase::applyOnTheRight() */ template<typename Scalar> +EIGEN_DEVICE_FUNC bool JacobiRotation<Scalar>::makeJacobi(const RealScalar& x, const Scalar& y, const RealScalar& z) { using std::sqrt; using std::abs; - typedef typename NumTraits<Scalar>::Real RealScalar; - if(y == Scalar(0)) + + RealScalar deno = RealScalar(2)*abs(y); + if(deno < (std::numeric_limits<RealScalar>::min)()) { m_c = Scalar(1); m_s = Scalar(0); @@ -93,7 +105,7 @@ } else { - RealScalar tau = (x-z)/(RealScalar(2)*abs(y)); + RealScalar tau = (x-z)/deno; RealScalar w = sqrt(numext::abs2(tau) + RealScalar(1)); RealScalar t; if(tau>RealScalar(0)) @@ -123,6 +135,7 @@ */ template<typename Scalar> template<typename Derived> +EIGEN_DEVICE_FUNC inline bool JacobiRotation<Scalar>::makeJacobi(const MatrixBase<Derived>& m, Index p, Index q) { return makeJacobi(numext::real(m.coeff(p,p)), m.coeff(p,q), numext::real(m.coeff(q,q))); @@ -132,7 +145,7 @@ * \f$ V = \left ( \begin{array}{c} p \\ q \end{array} \right )\f$ yields: * \f$ G^* V = \left ( \begin{array}{c} r \\ 0 \end{array} \right )\f$. * - * The value of \a z is returned if \a z is not null (the default is null). + * The value of \a r is returned if \a r is not null (the default is null). * Also note that G is built such that the cosine is always real. * * Example: \include Jacobi_makeGivens.cpp @@ -145,20 +158,22 @@ * \sa MatrixBase::applyOnTheLeft(), MatrixBase::applyOnTheRight() */ template<typename Scalar> -void JacobiRotation<Scalar>::makeGivens(const Scalar& p, const Scalar& q, Scalar* z) +EIGEN_DEVICE_FUNC +void JacobiRotation<Scalar>::makeGivens(const Scalar& p, const Scalar& q, Scalar* r) { - makeGivens(p, q, z, typename internal::conditional<NumTraits<Scalar>::IsComplex, internal::true_type, internal::false_type>::type()); + makeGivens(p, q, r, typename internal::conditional<NumTraits<Scalar>::IsComplex, internal::true_type, internal::false_type>::type()); } // specialization for complexes template<typename Scalar> +EIGEN_DEVICE_FUNC void JacobiRotation<Scalar>::makeGivens(const Scalar& p, const Scalar& q, Scalar* r, internal::true_type) { using std::sqrt; using std::abs; using numext::conj; - + if(q==Scalar(0)) { m_c = numext::real(p)<0 ? Scalar(-1) : Scalar(1); @@ -212,6 +227,7 @@ // specialization for reals template<typename Scalar> +EIGEN_DEVICE_FUNC void JacobiRotation<Scalar>::makeGivens(const Scalar& p, const Scalar& q, Scalar* r, internal::false_type) { using std::sqrt; @@ -257,12 +273,13 @@ namespace internal { /** \jacobi_module - * Applies the clock wise 2D rotation \a j to the set of 2D vectors of cordinates \a x and \a y: + * Applies the clock wise 2D rotation \a j to the set of 2D vectors of coordinates \a x and \a y: * \f$ \left ( \begin{array}{cc} x \\ y \end{array} \right ) = J \left ( \begin{array}{cc} x \\ y \end{array} \right ) \f$ * * \sa MatrixBase::applyOnTheLeft(), MatrixBase::applyOnTheRight() */ template<typename VectorX, typename VectorY, typename OtherScalar> +EIGEN_DEVICE_FUNC void apply_rotation_in_the_plane(DenseBase<VectorX>& xpr_x, DenseBase<VectorY>& xpr_y, const JacobiRotation<OtherScalar>& j); } @@ -274,6 +291,7 @@ */ template<typename Derived> template<typename OtherScalar> +EIGEN_DEVICE_FUNC inline void MatrixBase<Derived>::applyOnTheLeft(Index p, Index q, const JacobiRotation<OtherScalar>& j) { RowXpr x(this->row(p)); @@ -289,6 +307,7 @@ */ template<typename Derived> template<typename OtherScalar> +EIGEN_DEVICE_FUNC inline void MatrixBase<Derived>::applyOnTheRight(Index p, Index q, const JacobiRotation<OtherScalar>& j) { ColXpr x(this->col(p)); @@ -297,121 +316,13 @@ } namespace internal { -template<typename VectorX, typename VectorY, typename OtherScalar> -void /*EIGEN_DONT_INLINE*/ apply_rotation_in_the_plane(DenseBase<VectorX>& xpr_x, DenseBase<VectorY>& xpr_y, const JacobiRotation<OtherScalar>& j) + +template<typename Scalar, typename OtherScalar, + int SizeAtCompileTime, int MinAlignment, bool Vectorizable> +struct apply_rotation_in_the_plane_selector { - typedef typename VectorX::Scalar Scalar; - enum { PacketSize = packet_traits<Scalar>::size }; - typedef typename packet_traits<Scalar>::type Packet; - eigen_assert(xpr_x.size() == xpr_y.size()); - Index size = xpr_x.size(); - Index incrx = xpr_x.derived().innerStride(); - Index incry = xpr_y.derived().innerStride(); - - Scalar* EIGEN_RESTRICT x = &xpr_x.derived().coeffRef(0); - Scalar* EIGEN_RESTRICT y = &xpr_y.derived().coeffRef(0); - - OtherScalar c = j.c(); - OtherScalar s = j.s(); - if (c==OtherScalar(1) && s==OtherScalar(0)) - return; - - /*** dynamic-size vectorized paths ***/ - - if(VectorX::SizeAtCompileTime == Dynamic && - (VectorX::Flags & VectorY::Flags & PacketAccessBit) && - ((incrx==1 && incry==1) || PacketSize == 1)) - { - // both vectors are sequentially stored in memory => vectorization - enum { Peeling = 2 }; - - Index alignedStart = internal::first_default_aligned(y, size); - Index alignedEnd = alignedStart + ((size-alignedStart)/PacketSize)*PacketSize; - - const Packet pc = pset1<Packet>(c); - const Packet ps = pset1<Packet>(s); - conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex,false> pcj; - - for(Index i=0; i<alignedStart; ++i) - { - Scalar xi = x[i]; - Scalar yi = y[i]; - x[i] = c * xi + numext::conj(s) * yi; - y[i] = -s * xi + numext::conj(c) * yi; - } - - Scalar* EIGEN_RESTRICT px = x + alignedStart; - Scalar* EIGEN_RESTRICT py = y + alignedStart; - - if(internal::first_default_aligned(x, size)==alignedStart) - { - for(Index i=alignedStart; i<alignedEnd; i+=PacketSize) - { - Packet xi = pload<Packet>(px); - Packet yi = pload<Packet>(py); - pstore(px, padd(pmul(pc,xi),pcj.pmul(ps,yi))); - pstore(py, psub(pcj.pmul(pc,yi),pmul(ps,xi))); - px += PacketSize; - py += PacketSize; - } - } - else - { - Index peelingEnd = alignedStart + ((size-alignedStart)/(Peeling*PacketSize))*(Peeling*PacketSize); - for(Index i=alignedStart; i<peelingEnd; i+=Peeling*PacketSize) - { - Packet xi = ploadu<Packet>(px); - Packet xi1 = ploadu<Packet>(px+PacketSize); - Packet yi = pload <Packet>(py); - Packet yi1 = pload <Packet>(py+PacketSize); - pstoreu(px, padd(pmul(pc,xi),pcj.pmul(ps,yi))); - pstoreu(px+PacketSize, padd(pmul(pc,xi1),pcj.pmul(ps,yi1))); - pstore (py, psub(pcj.pmul(pc,yi),pmul(ps,xi))); - pstore (py+PacketSize, psub(pcj.pmul(pc,yi1),pmul(ps,xi1))); - px += Peeling*PacketSize; - py += Peeling*PacketSize; - } - if(alignedEnd!=peelingEnd) - { - Packet xi = ploadu<Packet>(x+peelingEnd); - Packet yi = pload <Packet>(y+peelingEnd); - pstoreu(x+peelingEnd, padd(pmul(pc,xi),pcj.pmul(ps,yi))); - pstore (y+peelingEnd, psub(pcj.pmul(pc,yi),pmul(ps,xi))); - } - } - - for(Index i=alignedEnd; i<size; ++i) - { - Scalar xi = x[i]; - Scalar yi = y[i]; - x[i] = c * xi + numext::conj(s) * yi; - y[i] = -s * xi + numext::conj(c) * yi; - } - } - - /*** fixed-size vectorized path ***/ - else if(VectorX::SizeAtCompileTime != Dynamic && - (VectorX::Flags & VectorY::Flags & PacketAccessBit) && - (EIGEN_PLAIN_ENUM_MIN(evaluator<VectorX>::Alignment, evaluator<VectorY>::Alignment)>0)) // FIXME should be compared to the required alignment - { - const Packet pc = pset1<Packet>(c); - const Packet ps = pset1<Packet>(s); - conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex,false> pcj; - Scalar* EIGEN_RESTRICT px = x; - Scalar* EIGEN_RESTRICT py = y; - for(Index i=0; i<size; i+=PacketSize) - { - Packet xi = pload<Packet>(px); - Packet yi = pload<Packet>(py); - pstore(px, padd(pmul(pc,xi),pcj.pmul(ps,yi))); - pstore(py, psub(pcj.pmul(pc,yi),pmul(ps,xi))); - px += PacketSize; - py += PacketSize; - } - } - - /*** non-vectorized path ***/ - else + static EIGEN_DEVICE_FUNC + inline void run(Scalar *x, Index incrx, Scalar *y, Index incry, Index size, OtherScalar c, OtherScalar s) { for(Index i=0; i<size; ++i) { @@ -423,6 +334,146 @@ y += incry; } } +}; + +template<typename Scalar, typename OtherScalar, + int SizeAtCompileTime, int MinAlignment> +struct apply_rotation_in_the_plane_selector<Scalar,OtherScalar,SizeAtCompileTime,MinAlignment,true /* vectorizable */> +{ + static inline void run(Scalar *x, Index incrx, Scalar *y, Index incry, Index size, OtherScalar c, OtherScalar s) + { + enum { + PacketSize = packet_traits<Scalar>::size, + OtherPacketSize = packet_traits<OtherScalar>::size + }; + typedef typename packet_traits<Scalar>::type Packet; + typedef typename packet_traits<OtherScalar>::type OtherPacket; + + /*** dynamic-size vectorized paths ***/ + if(SizeAtCompileTime == Dynamic && ((incrx==1 && incry==1) || PacketSize == 1)) + { + // both vectors are sequentially stored in memory => vectorization + enum { Peeling = 2 }; + + Index alignedStart = internal::first_default_aligned(y, size); + Index alignedEnd = alignedStart + ((size-alignedStart)/PacketSize)*PacketSize; + + const OtherPacket pc = pset1<OtherPacket>(c); + const OtherPacket ps = pset1<OtherPacket>(s); + conj_helper<OtherPacket,Packet,NumTraits<OtherScalar>::IsComplex,false> pcj; + conj_helper<OtherPacket,Packet,false,false> pm; + + for(Index i=0; i<alignedStart; ++i) + { + Scalar xi = x[i]; + Scalar yi = y[i]; + x[i] = c * xi + numext::conj(s) * yi; + y[i] = -s * xi + numext::conj(c) * yi; + } + + Scalar* EIGEN_RESTRICT px = x + alignedStart; + Scalar* EIGEN_RESTRICT py = y + alignedStart; + + if(internal::first_default_aligned(x, size)==alignedStart) + { + for(Index i=alignedStart; i<alignedEnd; i+=PacketSize) + { + Packet xi = pload<Packet>(px); + Packet yi = pload<Packet>(py); + pstore(px, padd(pm.pmul(pc,xi),pcj.pmul(ps,yi))); + pstore(py, psub(pcj.pmul(pc,yi),pm.pmul(ps,xi))); + px += PacketSize; + py += PacketSize; + } + } + else + { + Index peelingEnd = alignedStart + ((size-alignedStart)/(Peeling*PacketSize))*(Peeling*PacketSize); + for(Index i=alignedStart; i<peelingEnd; i+=Peeling*PacketSize) + { + Packet xi = ploadu<Packet>(px); + Packet xi1 = ploadu<Packet>(px+PacketSize); + Packet yi = pload <Packet>(py); + Packet yi1 = pload <Packet>(py+PacketSize); + pstoreu(px, padd(pm.pmul(pc,xi),pcj.pmul(ps,yi))); + pstoreu(px+PacketSize, padd(pm.pmul(pc,xi1),pcj.pmul(ps,yi1))); + pstore (py, psub(pcj.pmul(pc,yi),pm.pmul(ps,xi))); + pstore (py+PacketSize, psub(pcj.pmul(pc,yi1),pm.pmul(ps,xi1))); + px += Peeling*PacketSize; + py += Peeling*PacketSize; + } + if(alignedEnd!=peelingEnd) + { + Packet xi = ploadu<Packet>(x+peelingEnd); + Packet yi = pload <Packet>(y+peelingEnd); + pstoreu(x+peelingEnd, padd(pm.pmul(pc,xi),pcj.pmul(ps,yi))); + pstore (y+peelingEnd, psub(pcj.pmul(pc,yi),pm.pmul(ps,xi))); + } + } + + for(Index i=alignedEnd; i<size; ++i) + { + Scalar xi = x[i]; + Scalar yi = y[i]; + x[i] = c * xi + numext::conj(s) * yi; + y[i] = -s * xi + numext::conj(c) * yi; + } + } + + /*** fixed-size vectorized path ***/ + else if(SizeAtCompileTime != Dynamic && MinAlignment>0) // FIXME should be compared to the required alignment + { + const OtherPacket pc = pset1<OtherPacket>(c); + const OtherPacket ps = pset1<OtherPacket>(s); + conj_helper<OtherPacket,Packet,NumTraits<OtherPacket>::IsComplex,false> pcj; + conj_helper<OtherPacket,Packet,false,false> pm; + Scalar* EIGEN_RESTRICT px = x; + Scalar* EIGEN_RESTRICT py = y; + for(Index i=0; i<size; i+=PacketSize) + { + Packet xi = pload<Packet>(px); + Packet yi = pload<Packet>(py); + pstore(px, padd(pm.pmul(pc,xi),pcj.pmul(ps,yi))); + pstore(py, psub(pcj.pmul(pc,yi),pm.pmul(ps,xi))); + px += PacketSize; + py += PacketSize; + } + } + + /*** non-vectorized path ***/ + else + { + apply_rotation_in_the_plane_selector<Scalar,OtherScalar,SizeAtCompileTime,MinAlignment,false>::run(x,incrx,y,incry,size,c,s); + } + } +}; + +template<typename VectorX, typename VectorY, typename OtherScalar> +EIGEN_DEVICE_FUNC +void /*EIGEN_DONT_INLINE*/ apply_rotation_in_the_plane(DenseBase<VectorX>& xpr_x, DenseBase<VectorY>& xpr_y, const JacobiRotation<OtherScalar>& j) +{ + typedef typename VectorX::Scalar Scalar; + const bool Vectorizable = (VectorX::Flags & VectorY::Flags & PacketAccessBit) + && (int(packet_traits<Scalar>::size) == int(packet_traits<OtherScalar>::size)); + + eigen_assert(xpr_x.size() == xpr_y.size()); + Index size = xpr_x.size(); + Index incrx = xpr_x.derived().innerStride(); + Index incry = xpr_y.derived().innerStride(); + + Scalar* EIGEN_RESTRICT x = &xpr_x.derived().coeffRef(0); + Scalar* EIGEN_RESTRICT y = &xpr_y.derived().coeffRef(0); + + OtherScalar c = j.c(); + OtherScalar s = j.s(); + if (c==OtherScalar(1) && s==OtherScalar(0)) + return; + + apply_rotation_in_the_plane_selector< + Scalar,OtherScalar, + VectorX::SizeAtCompileTime, + EIGEN_PLAIN_ENUM_MIN(evaluator<VectorX>::Alignment, evaluator<VectorY>::Alignment), + Vectorizable>::run(x,incrx,y,incry,size,c,s); } } // end namespace internal
diff --git a/Eigen/src/KLUSupport/KLUSupport.h b/Eigen/src/KLUSupport/KLUSupport.h new file mode 100644 index 0000000..d2633a9 --- /dev/null +++ b/Eigen/src/KLUSupport/KLUSupport.h
@@ -0,0 +1,358 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Kyle Macfarlan <kyle.macfarlan@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_KLUSUPPORT_H +#define EIGEN_KLUSUPPORT_H + +namespace Eigen { + +/* TODO extract L, extract U, compute det, etc... */ + +/** \ingroup KLUSupport_Module + * \brief A sparse LU factorization and solver based on KLU + * + * This class allows to solve for A.X = B sparse linear problems via a LU factorization + * using the KLU library. The sparse matrix A must be squared and full rank. + * The vectors or matrices X and B can be either dense or sparse. + * + * \warning The input matrix A should be in a \b compressed and \b column-major form. + * Otherwise an expensive copy will be made. You can call the inexpensive makeCompressed() to get a compressed matrix. + * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> + * + * \implsparsesolverconcept + * + * \sa \ref TutorialSparseSolverConcept, class UmfPackLU, class SparseLU + */ + + +inline int klu_solve(klu_symbolic *Symbolic, klu_numeric *Numeric, Index ldim, Index nrhs, double B [ ], klu_common *Common, double) { + return klu_solve(Symbolic, Numeric, internal::convert_index<int>(ldim), internal::convert_index<int>(nrhs), B, Common); +} + +inline int klu_solve(klu_symbolic *Symbolic, klu_numeric *Numeric, Index ldim, Index nrhs, std::complex<double>B[], klu_common *Common, std::complex<double>) { + return klu_z_solve(Symbolic, Numeric, internal::convert_index<int>(ldim), internal::convert_index<int>(nrhs), &numext::real_ref(B[0]), Common); +} + +inline int klu_tsolve(klu_symbolic *Symbolic, klu_numeric *Numeric, Index ldim, Index nrhs, double B[], klu_common *Common, double) { + return klu_tsolve(Symbolic, Numeric, internal::convert_index<int>(ldim), internal::convert_index<int>(nrhs), B, Common); +} + +inline int klu_tsolve(klu_symbolic *Symbolic, klu_numeric *Numeric, Index ldim, Index nrhs, std::complex<double>B[], klu_common *Common, std::complex<double>) { + return klu_z_tsolve(Symbolic, Numeric, internal::convert_index<int>(ldim), internal::convert_index<int>(nrhs), &numext::real_ref(B[0]), 0, Common); +} + +inline klu_numeric* klu_factor(int Ap [ ], int Ai [ ], double Ax [ ], klu_symbolic *Symbolic, klu_common *Common, double) { + return klu_factor(Ap, Ai, Ax, Symbolic, Common); +} + +inline klu_numeric* klu_factor(int Ap[], int Ai[], std::complex<double> Ax[], klu_symbolic *Symbolic, klu_common *Common, std::complex<double>) { + return klu_z_factor(Ap, Ai, &numext::real_ref(Ax[0]), Symbolic, Common); +} + + +template<typename _MatrixType> +class KLU : public SparseSolverBase<KLU<_MatrixType> > +{ + protected: + typedef SparseSolverBase<KLU<_MatrixType> > Base; + using Base::m_isInitialized; + public: + using Base::_solve_impl; + typedef _MatrixType MatrixType; + typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; + typedef typename MatrixType::StorageIndex StorageIndex; + typedef Matrix<Scalar,Dynamic,1> Vector; + typedef Matrix<int, 1, MatrixType::ColsAtCompileTime> IntRowVectorType; + typedef Matrix<int, MatrixType::RowsAtCompileTime, 1> IntColVectorType; + typedef SparseMatrix<Scalar> LUMatrixType; + typedef SparseMatrix<Scalar,ColMajor,int> KLUMatrixType; + typedef Ref<const KLUMatrixType, StandardCompressedFormat> KLUMatrixRef; + enum { + ColsAtCompileTime = MatrixType::ColsAtCompileTime, + MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime + }; + + public: + + KLU() + : m_dummy(0,0), mp_matrix(m_dummy) + { + init(); + } + + template<typename InputMatrixType> + explicit KLU(const InputMatrixType& matrix) + : mp_matrix(matrix) + { + init(); + compute(matrix); + } + + ~KLU() + { + if(m_symbolic) klu_free_symbolic(&m_symbolic,&m_common); + if(m_numeric) klu_free_numeric(&m_numeric,&m_common); + } + + inline Index rows() const { return mp_matrix.rows(); } + inline Index cols() const { return mp_matrix.cols(); } + + /** \brief Reports whether previous computation was successful. + * + * \returns \c Success if computation was successful, + * \c NumericalIssue if the matrix.appears to be negative. + */ + ComputationInfo info() const + { + eigen_assert(m_isInitialized && "Decomposition is not initialized."); + return m_info; + } +#if 0 // not implemented yet + inline const LUMatrixType& matrixL() const + { + if (m_extractedDataAreDirty) extractData(); + return m_l; + } + + inline const LUMatrixType& matrixU() const + { + if (m_extractedDataAreDirty) extractData(); + return m_u; + } + + inline const IntColVectorType& permutationP() const + { + if (m_extractedDataAreDirty) extractData(); + return m_p; + } + + inline const IntRowVectorType& permutationQ() const + { + if (m_extractedDataAreDirty) extractData(); + return m_q; + } +#endif + /** Computes the sparse Cholesky decomposition of \a matrix + * Note that the matrix should be column-major, and in compressed format for best performance. + * \sa SparseMatrix::makeCompressed(). + */ + template<typename InputMatrixType> + void compute(const InputMatrixType& matrix) + { + if(m_symbolic) klu_free_symbolic(&m_symbolic, &m_common); + if(m_numeric) klu_free_numeric(&m_numeric, &m_common); + grab(matrix.derived()); + analyzePattern_impl(); + factorize_impl(); + } + + /** Performs a symbolic decomposition on the sparcity of \a matrix. + * + * This function is particularly useful when solving for several problems having the same structure. + * + * \sa factorize(), compute() + */ + template<typename InputMatrixType> + void analyzePattern(const InputMatrixType& matrix) + { + if(m_symbolic) klu_free_symbolic(&m_symbolic, &m_common); + if(m_numeric) klu_free_numeric(&m_numeric, &m_common); + + grab(matrix.derived()); + + analyzePattern_impl(); + } + + + /** Provides access to the control settings array used by KLU. + * + * See KLU documentation for details. + */ + inline const klu_common& kluCommon() const + { + return m_common; + } + + /** Provides access to the control settings array used by UmfPack. + * + * If this array contains NaN's, the default values are used. + * + * See KLU documentation for details. + */ + inline klu_common& kluCommon() + { + return m_common; + } + + /** Performs a numeric decomposition of \a matrix + * + * The given matrix must has the same sparcity than the matrix on which the pattern anylysis has been performed. + * + * \sa analyzePattern(), compute() + */ + template<typename InputMatrixType> + void factorize(const InputMatrixType& matrix) + { + eigen_assert(m_analysisIsOk && "KLU: you must first call analyzePattern()"); + if(m_numeric) + klu_free_numeric(&m_numeric,&m_common); + + grab(matrix.derived()); + + factorize_impl(); + } + + /** \internal */ + template<typename BDerived,typename XDerived> + bool _solve_impl(const MatrixBase<BDerived> &b, MatrixBase<XDerived> &x) const; + +#if 0 // not implemented yet + Scalar determinant() const; + + void extractData() const; +#endif + + protected: + + void init() + { + m_info = InvalidInput; + m_isInitialized = false; + m_numeric = 0; + m_symbolic = 0; + m_extractedDataAreDirty = true; + + klu_defaults(&m_common); + } + + void analyzePattern_impl() + { + m_info = InvalidInput; + m_analysisIsOk = false; + m_factorizationIsOk = false; + m_symbolic = klu_analyze(internal::convert_index<int>(mp_matrix.rows()), + const_cast<StorageIndex*>(mp_matrix.outerIndexPtr()), const_cast<StorageIndex*>(mp_matrix.innerIndexPtr()), + &m_common); + if (m_symbolic) { + m_isInitialized = true; + m_info = Success; + m_analysisIsOk = true; + m_extractedDataAreDirty = true; + } + } + + void factorize_impl() + { + + m_numeric = klu_factor(const_cast<StorageIndex*>(mp_matrix.outerIndexPtr()), const_cast<StorageIndex*>(mp_matrix.innerIndexPtr()), const_cast<Scalar*>(mp_matrix.valuePtr()), + m_symbolic, &m_common, Scalar()); + + + m_info = m_numeric ? Success : NumericalIssue; + m_factorizationIsOk = m_numeric ? 1 : 0; + m_extractedDataAreDirty = true; + } + + template<typename MatrixDerived> + void grab(const EigenBase<MatrixDerived> &A) + { + mp_matrix.~KLUMatrixRef(); + ::new (&mp_matrix) KLUMatrixRef(A.derived()); + } + + void grab(const KLUMatrixRef &A) + { + if(&(A.derived()) != &mp_matrix) + { + mp_matrix.~KLUMatrixRef(); + ::new (&mp_matrix) KLUMatrixRef(A); + } + } + + // cached data to reduce reallocation, etc. +#if 0 // not implemented yet + mutable LUMatrixType m_l; + mutable LUMatrixType m_u; + mutable IntColVectorType m_p; + mutable IntRowVectorType m_q; +#endif + + KLUMatrixType m_dummy; + KLUMatrixRef mp_matrix; + + klu_numeric* m_numeric; + klu_symbolic* m_symbolic; + klu_common m_common; + mutable ComputationInfo m_info; + int m_factorizationIsOk; + int m_analysisIsOk; + mutable bool m_extractedDataAreDirty; + + private: + KLU(const KLU& ) { } +}; + +#if 0 // not implemented yet +template<typename MatrixType> +void KLU<MatrixType>::extractData() const +{ + if (m_extractedDataAreDirty) + { + eigen_assert(false && "KLU: extractData Not Yet Implemented"); + + // get size of the data + int lnz, unz, rows, cols, nz_udiag; + umfpack_get_lunz(&lnz, &unz, &rows, &cols, &nz_udiag, m_numeric, Scalar()); + + // allocate data + m_l.resize(rows,(std::min)(rows,cols)); + m_l.resizeNonZeros(lnz); + + m_u.resize((std::min)(rows,cols),cols); + m_u.resizeNonZeros(unz); + + m_p.resize(rows); + m_q.resize(cols); + + // extract + umfpack_get_numeric(m_l.outerIndexPtr(), m_l.innerIndexPtr(), m_l.valuePtr(), + m_u.outerIndexPtr(), m_u.innerIndexPtr(), m_u.valuePtr(), + m_p.data(), m_q.data(), 0, 0, 0, m_numeric); + + m_extractedDataAreDirty = false; + } +} + +template<typename MatrixType> +typename KLU<MatrixType>::Scalar KLU<MatrixType>::determinant() const +{ + eigen_assert(false && "KLU: extractData Not Yet Implemented"); + return Scalar(); +} +#endif + +template<typename MatrixType> +template<typename BDerived,typename XDerived> +bool KLU<MatrixType>::_solve_impl(const MatrixBase<BDerived> &b, MatrixBase<XDerived> &x) const +{ + Index rhsCols = b.cols(); + EIGEN_STATIC_ASSERT((XDerived::Flags&RowMajorBit)==0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); + eigen_assert(m_factorizationIsOk && "The decomposition is not in a valid state for solving, you must first call either compute() or analyzePattern()/factorize()"); + + x = b; + int info = klu_solve(m_symbolic, m_numeric, b.rows(), rhsCols, x.const_cast_derived().data(), const_cast<klu_common*>(&m_common), Scalar()); + + m_info = info!=0 ? Success : NumericalIssue; + return true; +} + +} // end namespace Eigen + +#endif // EIGEN_KLUSUPPORT_H
diff --git a/Eigen/src/LU/CMakeLists.txt b/Eigen/src/LU/CMakeLists.txt deleted file mode 100644 index e0d8d78..0000000 --- a/Eigen/src/LU/CMakeLists.txt +++ /dev/null
@@ -1,8 +0,0 @@ -FILE(GLOB Eigen_LU_SRCS "*.h") - -INSTALL(FILES - ${Eigen_LU_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU COMPONENT Devel - ) - -ADD_SUBDIRECTORY(arch)
diff --git a/Eigen/src/LU/Determinant.h b/Eigen/src/LU/Determinant.h index d6a3c1e..3a41e6f 100644 --- a/Eigen/src/LU/Determinant.h +++ b/Eigen/src/LU/Determinant.h
@@ -15,6 +15,7 @@ namespace internal { template<typename Derived> +EIGEN_DEVICE_FUNC inline const typename Derived::Scalar bruteforce_det3_helper (const MatrixBase<Derived>& matrix, int a, int b, int c) { @@ -22,14 +23,6 @@ * (matrix.coeff(1,b) * matrix.coeff(2,c) - matrix.coeff(1,c) * matrix.coeff(2,b)); } -template<typename Derived> -const typename Derived::Scalar bruteforce_det4_helper -(const MatrixBase<Derived>& matrix, int j, int k, int m, int n) -{ - return (matrix.coeff(j,0) * matrix.coeff(k,1) - matrix.coeff(k,0) * matrix.coeff(j,1)) - * (matrix.coeff(m,2) * matrix.coeff(n,3) - matrix.coeff(n,2) * matrix.coeff(m,3)); -} - template<typename Derived, int DeterminantType = Derived::RowsAtCompileTime > struct determinant_impl @@ -44,7 +37,8 @@ template<typename Derived> struct determinant_impl<Derived, 1> { - static inline typename traits<Derived>::Scalar run(const Derived& m) + static inline EIGEN_DEVICE_FUNC + typename traits<Derived>::Scalar run(const Derived& m) { return m.coeff(0,0); } @@ -52,7 +46,8 @@ template<typename Derived> struct determinant_impl<Derived, 2> { - static inline typename traits<Derived>::Scalar run(const Derived& m) + static inline EIGEN_DEVICE_FUNC + typename traits<Derived>::Scalar run(const Derived& m) { return m.coeff(0,0) * m.coeff(1,1) - m.coeff(1,0) * m.coeff(0,1); } @@ -60,7 +55,8 @@ template<typename Derived> struct determinant_impl<Derived, 3> { - static inline typename traits<Derived>::Scalar run(const Derived& m) + static inline EIGEN_DEVICE_FUNC + typename traits<Derived>::Scalar run(const Derived& m) { return bruteforce_det3_helper(m,0,1,2) - bruteforce_det3_helper(m,1,0,2) @@ -70,15 +66,34 @@ template<typename Derived> struct determinant_impl<Derived, 4> { - static typename traits<Derived>::Scalar run(const Derived& m) + typedef typename traits<Derived>::Scalar Scalar; + static EIGEN_DEVICE_FUNC + Scalar run(const Derived& m) { - // trick by Martin Costabel to compute 4x4 det with only 30 muls - return bruteforce_det4_helper(m,0,1,2,3) - - bruteforce_det4_helper(m,0,2,1,3) - + bruteforce_det4_helper(m,0,3,1,2) - + bruteforce_det4_helper(m,1,2,0,3) - - bruteforce_det4_helper(m,1,3,0,2) - + bruteforce_det4_helper(m,2,3,0,1); + Scalar d2_01 = det2(m, 0, 1); + Scalar d2_02 = det2(m, 0, 2); + Scalar d2_03 = det2(m, 0, 3); + Scalar d2_12 = det2(m, 1, 2); + Scalar d2_13 = det2(m, 1, 3); + Scalar d2_23 = det2(m, 2, 3); + Scalar d3_0 = det3(m, 1,d2_23, 2,d2_13, 3,d2_12); + Scalar d3_1 = det3(m, 0,d2_23, 2,d2_03, 3,d2_02); + Scalar d3_2 = det3(m, 0,d2_13, 1,d2_03, 3,d2_01); + Scalar d3_3 = det3(m, 0,d2_12, 1,d2_02, 2,d2_01); + return internal::pmadd(-m(0,3),d3_0, m(1,3)*d3_1) + + internal::pmadd(-m(2,3),d3_2, m(3,3)*d3_3); + } +protected: + static EIGEN_DEVICE_FUNC + Scalar det2(const Derived& m, Index i0, Index i1) + { + return m(i0,0) * m(i1,1) - m(i1,0) * m(i0,1); + } + + static EIGEN_DEVICE_FUNC + Scalar det3(const Derived& m, Index i0, const Scalar& d0, Index i1, const Scalar& d1, Index i2, const Scalar& d2) + { + return internal::pmadd(m(i0,2), d0, internal::pmadd(-m(i1,2), d1, m(i2,2)*d2)); } }; @@ -89,6 +104,7 @@ * \returns the determinant of this matrix */ template<typename Derived> +EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar MatrixBase<Derived>::determinant() const { eigen_assert(rows() == cols());
diff --git a/Eigen/src/LU/FullPivLU.h b/Eigen/src/LU/FullPivLU.h index 64b9eb7..ef93ec5 100644 --- a/Eigen/src/LU/FullPivLU.h +++ b/Eigen/src/LU/FullPivLU.h
@@ -18,6 +18,7 @@ { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; + typedef int StorageIndex; enum { Flags = 0 }; }; @@ -48,10 +49,12 @@ * The data of the LU decomposition can be directly accessed through the methods matrixLU(), * permutationP(), permutationQ(). * - * As an exemple, here is how the original matrix can be retrieved: + * As an example, here is how the original matrix can be retrieved: * \include class_FullPivLU.cpp * Output: \verbinclude class_FullPivLU.out * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::fullPivLu(), MatrixBase::determinant(), MatrixBase::inverse() */ template<typename _MatrixType> class FullPivLU @@ -60,9 +63,9 @@ public: typedef _MatrixType MatrixType; typedef SolverBase<FullPivLU> Base; + friend class SolverBase<FullPivLU>; EIGEN_GENERIC_PUBLIC_INTERFACE(FullPivLU) - // FIXME StorageIndex defined in EIGEN_GENERIC_PUBLIC_INTERFACE should be int enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime @@ -97,6 +100,15 @@ template<typename InputType> explicit FullPivLU(const EigenBase<InputType>& matrix); + /** \brief Constructs a LU factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when \c MatrixType is a Eigen::Ref. + * + * \sa FullPivLU(const EigenBase&) + */ + template<typename InputType> + explicit FullPivLU(EigenBase<InputType>& matrix); + /** Computes the LU decomposition of the given matrix. * * \param matrix the matrix of which to compute the LU decomposition. @@ -105,7 +117,11 @@ * \returns a reference to *this */ template<typename InputType> - FullPivLU& compute(const EigenBase<InputType>& matrix); + FullPivLU& compute(const EigenBase<InputType>& matrix) { + m_lu = matrix.derived(); + computeInPlace(); + return *this; + } /** \returns the LU decomposition matrix: the upper-triangular part is U, the * unit-lower-triangular part is L (at least for square matrices; in the non-square @@ -141,7 +157,7 @@ * * \sa permutationQ() */ - inline const PermutationPType& permutationP() const + EIGEN_DEVICE_FUNC inline const PermutationPType& permutationP() const { eigen_assert(m_isInitialized && "LU is not initialized."); return m_p; @@ -203,6 +219,7 @@ return internal::image_retval<FullPivLU>(*this, originalMatrix); } + #ifdef EIGEN_PARSED_BY_DOXYGEN /** \return a solution x to the equation Ax=b, where A is the matrix of which * *this is the LU decomposition. * @@ -222,14 +239,10 @@ * * \sa TriangularView::solve(), kernel(), inverse() */ - // FIXME this is a copy-paste of the base-class member to add the isInitialized assertion. template<typename Rhs> inline const Solve<FullPivLU, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "LU is not initialized."); - return Solve<FullPivLU, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif /** \returns an estimate of the reciprocal condition number of the matrix of which \c *this is the LU decomposition. @@ -305,7 +318,7 @@ return m_usePrescribedThreshold ? m_prescribedThreshold // this formula comes from experimenting (see "LU precision tuning" thread on the list) // and turns out to be identical to Higham's formula used already in LDLt. - : NumTraits<Scalar>::epsilon() * m_lu.diagonalSize(); + : NumTraits<Scalar>::epsilon() * RealScalar(m_lu.diagonalSize()); } /** \returns the rank of the matrix of which *this is the LU decomposition. @@ -391,16 +404,14 @@ MatrixType reconstructedMatrix() const; - inline Index rows() const { return m_lu.rows(); } - inline Index cols() const { return m_lu.cols(); } + EIGEN_DEVICE_FUNC inline Index rows() const { return m_lu.rows(); } + EIGEN_DEVICE_FUNC inline Index cols() const { return m_lu.cols(); } #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; template<bool Conjugate, typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif @@ -418,9 +429,10 @@ PermutationQType m_q; IntColVectorType m_rowsTranspositions; IntRowVectorType m_colsTranspositions; - Index m_det_pq, m_nonzero_pivots; + Index m_nonzero_pivots; RealScalar m_l1_norm; RealScalar m_maxpivot, m_prescribedThreshold; + signed char m_det_pq; bool m_isInitialized, m_usePrescribedThreshold; }; @@ -458,25 +470,28 @@ template<typename MatrixType> template<typename InputType> -FullPivLU<MatrixType>& FullPivLU<MatrixType>::compute(const EigenBase<InputType>& matrix) +FullPivLU<MatrixType>::FullPivLU(EigenBase<InputType>& matrix) + : m_lu(matrix.derived()), + m_p(matrix.rows()), + m_q(matrix.cols()), + m_rowsTranspositions(matrix.rows()), + m_colsTranspositions(matrix.cols()), + m_isInitialized(false), + m_usePrescribedThreshold(false) { - check_template_parameters(); - - // the permutations are stored as int indices, so just to be sure: - eigen_assert(matrix.rows()<=NumTraits<int>::highest() && matrix.cols()<=NumTraits<int>::highest()); - - m_lu = matrix.derived(); - m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); - computeInPlace(); - - m_isInitialized = true; - return *this; } template<typename MatrixType> void FullPivLU<MatrixType>::computeInPlace() { + check_template_parameters(); + + // the permutations are stored as int indices, so just to be sure: + eigen_assert(m_lu.rows()<=NumTraits<int>::highest() && m_lu.cols()<=NumTraits<int>::highest()); + + m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); + const Index size = m_lu.diagonalSize(); const Index rows = m_lu.rows(); const Index cols = m_lu.cols(); @@ -512,8 +527,8 @@ m_nonzero_pivots = k; for(Index i = k; i < size; ++i) { - m_rowsTranspositions.coeffRef(i) = i; - m_colsTranspositions.coeffRef(i) = i; + m_rowsTranspositions.coeffRef(i) = internal::convert_index<StorageIndex>(i); + m_colsTranspositions.coeffRef(i) = internal::convert_index<StorageIndex>(i); } break; } @@ -524,8 +539,8 @@ // Now that we've found the pivot, we need to apply the row/col swaps to // bring it to the location (k,k). - m_rowsTranspositions.coeffRef(k) = row_of_biggest_in_corner; - m_colsTranspositions.coeffRef(k) = col_of_biggest_in_corner; + m_rowsTranspositions.coeffRef(k) = internal::convert_index<StorageIndex>(row_of_biggest_in_corner); + m_colsTranspositions.coeffRef(k) = internal::convert_index<StorageIndex>(col_of_biggest_in_corner); if(k != row_of_biggest_in_corner) { m_lu.row(k).swap(m_lu.row(row_of_biggest_in_corner)); ++number_of_transpositions; @@ -556,6 +571,8 @@ m_q.applyTranspositionOnTheRight(k, m_colsTranspositions.coeff(k)); m_det_pq = (number_of_transpositions%2) ? -1 : 1; + + m_isInitialized = true; } template<typename MatrixType> @@ -736,7 +753,6 @@ const Index rows = this->rows(), cols = this->cols(), nonzero_pivots = this->rank(); - eigen_assert(rhs.rows() == rows); const Index smalldim = (std::min)(rows, cols); if(nonzero_pivots == 0) @@ -786,7 +802,6 @@ const Index rows = this->rows(), cols = this->cols(), nonzero_pivots = this->rank(); - eigen_assert(rhs.rows() == cols); const Index smalldim = (std::min)(rows, cols); if(nonzero_pivots == 0) @@ -800,29 +815,19 @@ // Step 1 c = permutationQ().inverse() * rhs; - if (Conjugate) { - // Step 2 - m_lu.topLeftCorner(nonzero_pivots, nonzero_pivots) - .template triangularView<Upper>() - .adjoint() - .solveInPlace(c.topRows(nonzero_pivots)); - // Step 3 - m_lu.topLeftCorner(smalldim, smalldim) - .template triangularView<UnitLower>() - .adjoint() - .solveInPlace(c.topRows(smalldim)); - } else { - // Step 2 - m_lu.topLeftCorner(nonzero_pivots, nonzero_pivots) - .template triangularView<Upper>() - .transpose() - .solveInPlace(c.topRows(nonzero_pivots)); - // Step 3 - m_lu.topLeftCorner(smalldim, smalldim) - .template triangularView<UnitLower>() - .transpose() - .solveInPlace(c.topRows(smalldim)); - } + // Step 2 + m_lu.topLeftCorner(nonzero_pivots, nonzero_pivots) + .template triangularView<Upper>() + .transpose() + .template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(nonzero_pivots)); + + // Step 3 + m_lu.topLeftCorner(smalldim, smalldim) + .template triangularView<UnitLower>() + .transpose() + .template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(smalldim)); // Step 4 PermutationPType invp = permutationP().inverse().eval(); @@ -838,12 +843,12 @@ /***** Implementation of inverse() *****************************************************/ -template<typename DstXprType, typename MatrixType, typename Scalar> -struct Assignment<DstXprType, Inverse<FullPivLU<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename MatrixType> +struct Assignment<DstXprType, Inverse<FullPivLU<MatrixType> >, internal::assign_op<typename DstXprType::Scalar,typename FullPivLU<MatrixType>::Scalar>, Dense2Dense> { typedef FullPivLU<MatrixType> LuType; typedef Inverse<LuType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename MatrixType::Scalar> &) { dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } @@ -858,14 +863,12 @@ * * \sa class FullPivLU */ -#ifndef __CUDACC__ template<typename Derived> inline const FullPivLU<typename MatrixBase<Derived>::PlainObject> MatrixBase<Derived>::fullPivLu() const { return FullPivLU<PlainObject>(eval()); } -#endif } // end namespace Eigen
diff --git a/Eigen/src/LU/InverseImpl.h b/Eigen/src/LU/InverseImpl.h index e202a55..1bab00c 100644 --- a/Eigen/src/LU/InverseImpl.h +++ b/Eigen/src/LU/InverseImpl.h
@@ -286,13 +286,18 @@ namespace internal { // Specialization for "dense = dense_xpr.inverse()" -template<typename DstXprType, typename XprType, typename Scalar> -struct Assignment<DstXprType, Inverse<XprType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename XprType> +struct Assignment<DstXprType, Inverse<XprType>, internal::assign_op<typename DstXprType::Scalar,typename XprType::Scalar>, Dense2Dense> { typedef Inverse<XprType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + EIGEN_DEVICE_FUNC + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename XprType::Scalar> &) { - // FIXME shall we resize dst here? + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + const int Size = EIGEN_PLAIN_ENUM_MIN(XprType::ColsAtCompileTime,DstXprType::ColsAtCompileTime); EIGEN_ONLY_USED_FOR_DEBUG(Size); eigen_assert(( (Size<=1) || (Size>4) || (extract_data(src.nestedExpression())!=extract_data(dst))) @@ -328,6 +333,7 @@ * \sa computeInverseAndDetWithCheck() */ template<typename Derived> +EIGEN_DEVICE_FUNC inline const Inverse<Derived> MatrixBase<Derived>::inverse() const { EIGEN_STATIC_ASSERT(!NumTraits<Scalar>::IsInteger,THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES) @@ -400,7 +406,7 @@ const RealScalar& absDeterminantThreshold ) const { - RealScalar determinant; + Scalar determinant; // i'd love to put some static assertions there, but SFINAE means that they have no effect... eigen_assert(rows() == cols()); computeInverseAndDetWithCheck(inverse,determinant,invertible,absDeterminantThreshold);
diff --git a/Eigen/src/LU/PartialPivLU.h b/Eigen/src/LU/PartialPivLU.h index 2e6d919..b893801 100644 --- a/Eigen/src/LU/PartialPivLU.h +++ b/Eigen/src/LU/PartialPivLU.h
@@ -19,6 +19,7 @@ { typedef MatrixXpr XprKind; typedef SolverStorage StorageKind; + typedef int StorageIndex; typedef traits<_MatrixType> BaseTraits; enum { Flags = BaseTraits::Flags & RowMajorBit, @@ -26,6 +27,17 @@ }; }; +template<typename T,typename Derived> +struct enable_if_ref; +// { +// typedef Derived type; +// }; + +template<typename T,typename Derived> +struct enable_if_ref<Ref<T>,Derived> { + typedef Derived type; +}; + } // end namespace internal /** \ingroup LU_Module @@ -57,6 +69,8 @@ * * The data of the LU decomposition can be directly accessed through the methods matrixLU(), permutationP(). * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::partialPivLu(), MatrixBase::determinant(), MatrixBase::inverse(), MatrixBase::computeInverse(), class FullPivLU */ template<typename _MatrixType> class PartialPivLU @@ -66,8 +80,9 @@ typedef _MatrixType MatrixType; typedef SolverBase<PartialPivLU> Base; + friend class SolverBase<PartialPivLU>; + EIGEN_GENERIC_PUBLIC_INTERFACE(PartialPivLU) - // FIXME StorageIndex defined in EIGEN_GENERIC_PUBLIC_INTERFACE should be int enum { MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime @@ -102,8 +117,22 @@ template<typename InputType> explicit PartialPivLU(const EigenBase<InputType>& matrix); + /** Constructor for \link InplaceDecomposition inplace decomposition \endlink + * + * \param matrix the matrix of which to compute the LU decomposition. + * + * \warning The matrix should have full rank (e.g. if it's square, it should be invertible). + * If you need to deal with non-full rank, use class FullPivLU instead. + */ template<typename InputType> - PartialPivLU& compute(const EigenBase<InputType>& matrix); + explicit PartialPivLU(EigenBase<InputType>& matrix); + + template<typename InputType> + PartialPivLU& compute(const EigenBase<InputType>& matrix) { + m_lu = matrix.derived(); + compute(); + return *this; + } /** \returns the LU decomposition matrix: the upper-triangular part is U, the * unit-lower-triangular part is L (at least for square matrices; in the non-square @@ -125,6 +154,7 @@ return m_p; } + #ifdef EIGEN_PARSED_BY_DOXYGEN /** This method returns the solution x to the equation Ax=b, where A is the matrix of which * *this is the LU decomposition. * @@ -142,14 +172,10 @@ * * \sa TriangularView::solve(), inverse(), computeInverse() */ - // FIXME this is a copy-paste of the base-class member to add the isInitialized assertion. template<typename Rhs> inline const Solve<PartialPivLU, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); - return Solve<PartialPivLU, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif /** \returns an estimate of the reciprocal condition number of the matrix of which \c *this is the LU decomposition. @@ -204,8 +230,6 @@ * Step 3: replace c by the solution x to Ux = c. */ - eigen_assert(rhs.rows() == m_lu.rows()); - // Step 1 dst = permutationP() * rhs; @@ -219,26 +243,21 @@ template<bool Conjugate, typename RhsType, typename DstType> EIGEN_DEVICE_FUNC void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const { - /* The decomposition PA = LU can be rewritten as A = P^{-1} L U. + /* The decomposition PA = LU can be rewritten as A^T = U^T L^T P. * So we proceed as follows: - * Step 1: compute c = Pb. - * Step 2: replace c by the solution x to Lx = c. - * Step 3: replace c by the solution x to Ux = c. + * Step 1: compute c as the solution to L^T c = b + * Step 2: replace c by the solution x to U^T x = c. + * Step 3: update c = P^-1 c. */ eigen_assert(rhs.rows() == m_lu.cols()); - if (Conjugate) { - // Step 1 - dst = m_lu.template triangularView<Upper>().adjoint().solve(rhs); - // Step 2 - m_lu.template triangularView<UnitLower>().adjoint().solveInPlace(dst); - } else { - // Step 1 - dst = m_lu.template triangularView<Upper>().transpose().solve(rhs); - // Step 2 - m_lu.template triangularView<UnitLower>().transpose().solveInPlace(dst); - } + // Step 1 + dst = m_lu.template triangularView<Upper>().transpose() + .template conjugateIf<Conjugate>().solve(rhs); + // Step 2 + m_lu.template triangularView<UnitLower>().transpose() + .template conjugateIf<Conjugate>().solveInPlace(dst); // Step 3 dst = permutationP().transpose() * dst; } @@ -251,11 +270,13 @@ EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } + void compute(); + MatrixType m_lu; PermutationType m_p; TranspositionType m_rowsTranspositions; - Index m_det_p; RealScalar m_l1_norm; + signed char m_det_p; bool m_isInitialized; }; @@ -264,8 +285,8 @@ : m_lu(), m_p(), m_rowsTranspositions(), - m_det_p(0), m_l1_norm(0), + m_det_p(0), m_isInitialized(false) { } @@ -275,8 +296,8 @@ : m_lu(size, size), m_p(size), m_rowsTranspositions(size), - m_det_p(0), m_l1_norm(0), + m_det_p(0), m_isInitialized(false) { } @@ -284,30 +305,44 @@ template<typename MatrixType> template<typename InputType> PartialPivLU<MatrixType>::PartialPivLU(const EigenBase<InputType>& matrix) - : m_lu(matrix.rows(), matrix.rows()), + : m_lu(matrix.rows(),matrix.cols()), m_p(matrix.rows()), m_rowsTranspositions(matrix.rows()), - m_det_p(0), m_l1_norm(0), + m_det_p(0), m_isInitialized(false) { compute(matrix.derived()); } +template<typename MatrixType> +template<typename InputType> +PartialPivLU<MatrixType>::PartialPivLU(EigenBase<InputType>& matrix) + : m_lu(matrix.derived()), + m_p(matrix.rows()), + m_rowsTranspositions(matrix.rows()), + m_l1_norm(0), + m_det_p(0), + m_isInitialized(false) +{ + compute(); +} + namespace internal { /** \internal This is the blocked version of fullpivlu_unblocked() */ -template<typename Scalar, int StorageOrder, typename PivIndex> +template<typename Scalar, int StorageOrder, typename PivIndex, int SizeAtCompileTime=Dynamic> struct partial_lu_impl { - // FIXME add a stride to Map, so that the following mapping becomes easier, - // another option would be to create an expression being able to automatically - // warp any Map, Matrix, and Block expressions as a unique type, but since that's exactly - // a Map + stride, why not adding a stride to Map, and convenient ctors from a Matrix, - // and Block. - typedef Map<Matrix<Scalar, Dynamic, Dynamic, StorageOrder> > MapLU; - typedef Block<MapLU, Dynamic, Dynamic> MatrixType; - typedef Block<MatrixType,Dynamic,Dynamic> BlockType; + static const int UnBlockedBound = 16; + static const bool UnBlockedAtCompileTime = SizeAtCompileTime!=Dynamic && SizeAtCompileTime<=UnBlockedBound; + static const int ActualSizeAtCompileTime = UnBlockedAtCompileTime ? SizeAtCompileTime : Dynamic; + // Remaining rows and columns at compile-time: + static const int RRows = SizeAtCompileTime==2 ? 1 : Dynamic; + static const int RCols = SizeAtCompileTime==2 ? 1 : Dynamic; + typedef Matrix<Scalar, ActualSizeAtCompileTime, ActualSizeAtCompileTime, StorageOrder> MatrixType; + typedef Ref<MatrixType> MatrixTypeRef; + typedef Ref<Matrix<Scalar, Dynamic, Dynamic, StorageOrder> > BlockType; typedef typename MatrixType::RealScalar RealScalar; /** \internal performs the LU decomposition in-place of the matrix \a lu @@ -320,19 +355,22 @@ * * \returns The index of the first pivot which is exactly zero if any, or a negative number otherwise. */ - static Index unblocked_lu(MatrixType& lu, PivIndex* row_transpositions, PivIndex& nb_transpositions) + static Index unblocked_lu(MatrixTypeRef& lu, PivIndex* row_transpositions, PivIndex& nb_transpositions) { typedef scalar_score_coeff_op<Scalar> Scoring; typedef typename Scoring::result_type Score; const Index rows = lu.rows(); const Index cols = lu.cols(); const Index size = (std::min)(rows,cols); + // For small compile-time matrices it is worth processing the last row separately: + // speedup: +100% for 2x2, +10% for others. + const Index endk = UnBlockedAtCompileTime ? size-1 : size; nb_transpositions = 0; Index first_zero_pivot = -1; - for(Index k = 0; k < size; ++k) + for(Index k = 0; k < endk; ++k) { - Index rrows = rows-k-1; - Index rcols = cols-k-1; + int rrows = internal::convert_index<int>(rows-k-1); + int rcols = internal::convert_index<int>(cols-k-1); Index row_of_biggest_in_col; Score biggest_in_corner @@ -349,9 +387,7 @@ ++nb_transpositions; } - // FIXME shall we introduce a safe quotient expression in cas 1/lu.coeff(k,k) - // overflow but not the actual quotient? - lu.col(k).tail(rrows) /= lu.coeff(k,k); + lu.col(k).tail(fix<RRows>(rrows)) /= lu.coeff(k,k); } else if(first_zero_pivot==-1) { @@ -361,8 +397,18 @@ } if(k<rows-1) - lu.bottomRightCorner(rrows,rcols).noalias() -= lu.col(k).tail(rrows) * lu.row(k).tail(rcols); + lu.bottomRightCorner(fix<RRows>(rrows),fix<RCols>(rcols)).noalias() -= lu.col(k).tail(fix<RRows>(rrows)) * lu.row(k).tail(fix<RCols>(rcols)); } + + // special handling of the last entry + if(UnBlockedAtCompileTime) + { + Index k = endk; + row_transpositions[k] = PivIndex(k); + if (Scoring()(lu(k, k)) == Score(0) && first_zero_pivot == -1) + first_zero_pivot = k; + } + return first_zero_pivot; } @@ -378,18 +424,17 @@ * \returns The index of the first pivot which is exactly zero if any, or a negative number otherwise. * * \note This very low level interface using pointers, etc. is to: - * 1 - reduce the number of instanciations to the strict minimum - * 2 - avoid infinite recursion of the instanciations with Block<Block<Block<...> > > + * 1 - reduce the number of instantiations to the strict minimum + * 2 - avoid infinite recursion of the instantiations with Block<Block<Block<...> > > */ static Index blocked_lu(Index rows, Index cols, Scalar* lu_data, Index luStride, PivIndex* row_transpositions, PivIndex& nb_transpositions, Index maxBlockSize=256) { - MapLU lu1(lu_data,StorageOrder==RowMajor?rows:luStride,StorageOrder==RowMajor?luStride:cols); - MatrixType lu(lu1,0,0,rows,cols); + MatrixTypeRef lu = MatrixType::Map(lu_data,rows, cols, OuterStride<>(luStride)); const Index size = (std::min)(rows,cols); // if the matrix is too small, no blocking: - if(size<=16) + if(UnBlockedAtCompileTime || size<=UnBlockedBound) { return unblocked_lu(lu, row_transpositions, nb_transpositions); } @@ -415,12 +460,12 @@ // A00 | A01 | A02 // lu = A_0 | A_1 | A_2 = A10 | A11 | A12 // A20 | A21 | A22 - BlockType A_0(lu,0,0,rows,k); - BlockType A_2(lu,0,k+bs,rows,tsize); - BlockType A11(lu,k,k,bs,bs); - BlockType A12(lu,k,k+bs,bs,tsize); - BlockType A21(lu,k+bs,k,trows,bs); - BlockType A22(lu,k+bs,k+bs,trows,tsize); + BlockType A_0 = lu.block(0,0,rows,k); + BlockType A_2 = lu.block(0,k+bs,rows,tsize); + BlockType A11 = lu.block(k,k,bs,bs); + BlockType A12 = lu.block(k,k+bs,bs,tsize); + BlockType A21 = lu.block(k+bs,k,trows,bs); + BlockType A22 = lu.block(k+bs,k+bs,trows,tsize); PivIndex nb_transpositions_in_panel; // recursively call the blocked LU algorithm on [A11^T A21^T]^T @@ -434,7 +479,7 @@ // update permutations and apply them to A_0 for(Index i=k; i<k+bs; ++i) { - Index piv = (row_transpositions[i] += k); + Index piv = (row_transpositions[i] += internal::convert_index<PivIndex>(k)); A_0.row(i).swap(A_0.row(piv)); } @@ -463,26 +508,29 @@ eigen_assert((&row_transpositions.coeffRef(1)-&row_transpositions.coeffRef(0)) == 1); partial_lu_impl - <typename MatrixType::Scalar, MatrixType::Flags&RowMajorBit?RowMajor:ColMajor, typename TranspositionType::StorageIndex> + < typename MatrixType::Scalar, MatrixType::Flags&RowMajorBit?RowMajor:ColMajor, + typename TranspositionType::StorageIndex, + EIGEN_SIZE_MIN_PREFER_FIXED(MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime)> ::blocked_lu(lu.rows(), lu.cols(), &lu.coeffRef(0,0), lu.outerStride(), &row_transpositions.coeffRef(0), nb_transpositions); } } // end namespace internal template<typename MatrixType> -template<typename InputType> -PartialPivLU<MatrixType>& PartialPivLU<MatrixType>::compute(const EigenBase<InputType>& matrix) +void PartialPivLU<MatrixType>::compute() { check_template_parameters(); // the row permutation is stored as int indices, so just to be sure: - eigen_assert(matrix.rows()<NumTraits<int>::highest()); + eigen_assert(m_lu.rows()<NumTraits<int>::highest()); - m_lu = matrix.derived(); - m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); + if(m_lu.cols()>0) + m_l1_norm = m_lu.cwiseAbs().colwise().sum().maxCoeff(); + else + m_l1_norm = RealScalar(0); - eigen_assert(matrix.rows() == matrix.cols() && "PartialPivLU is only for square (and moreover invertible) matrices"); - const Index size = matrix.rows(); + eigen_assert(m_lu.rows() == m_lu.cols() && "PartialPivLU is only for square (and moreover invertible) matrices"); + const Index size = m_lu.rows(); m_rowsTranspositions.resize(size); @@ -493,7 +541,6 @@ m_p = m_rowsTranspositions; m_isInitialized = true; - return *this; } template<typename MatrixType> @@ -525,12 +572,12 @@ namespace internal { /***** Implementation of inverse() *****************************************************/ -template<typename DstXprType, typename MatrixType, typename Scalar> -struct Assignment<DstXprType, Inverse<PartialPivLU<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename MatrixType> +struct Assignment<DstXprType, Inverse<PartialPivLU<MatrixType> >, internal::assign_op<typename DstXprType::Scalar,typename PartialPivLU<MatrixType>::Scalar>, Dense2Dense> { typedef PartialPivLU<MatrixType> LuType; typedef Inverse<LuType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename LuType::Scalar> &) { dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } @@ -545,14 +592,12 @@ * * \sa class PartialPivLU */ -#ifndef __CUDACC__ template<typename Derived> inline const PartialPivLU<typename MatrixBase<Derived>::PlainObject> MatrixBase<Derived>::partialPivLu() const { return PartialPivLU<PlainObject>(eval()); } -#endif /** \lu_module * @@ -562,14 +607,12 @@ * * \sa class PartialPivLU */ -#ifndef __CUDACC__ template<typename Derived> inline const PartialPivLU<typename MatrixBase<Derived>::PlainObject> MatrixBase<Derived>::lu() const { return PartialPivLU<PlainObject>(eval()); } -#endif } // end namespace Eigen
diff --git a/Eigen/src/LU/PartialPivLU_MKL.h b/Eigen/src/LU/PartialPivLU_LAPACKE.h similarity index 76% rename from Eigen/src/LU/PartialPivLU_MKL.h rename to Eigen/src/LU/PartialPivLU_LAPACKE.h index 9035953..755168a 100644 --- a/Eigen/src/LU/PartialPivLU_MKL.h +++ b/Eigen/src/LU/PartialPivLU_LAPACKE.h
@@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * LU decomposition with partial pivoting based on LAPACKE_?getrf function. ******************************************************************************** */ @@ -33,20 +33,18 @@ #ifndef EIGEN_PARTIALLU_LAPACK_H #define EIGEN_PARTIALLU_LAPACK_H -#include "Eigen/src/Core/util/MKL_support.h" - namespace Eigen { namespace internal { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_LU_PARTPIV(EIGTYPE, MKLTYPE, MKLPREFIX) \ +#define EIGEN_LAPACKE_LU_PARTPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ template<int StorageOrder> \ struct partial_lu_impl<EIGTYPE, StorageOrder, lapack_int> \ { \ /* \internal performs the LU decomposition in-place of the matrix represented */ \ - static lapack_int blocked_lu(lapack_int rows, lapack_int cols, EIGTYPE* lu_data, lapack_int luStride, lapack_int* row_transpositions, lapack_int& nb_transpositions, lapack_int maxBlockSize=256) \ + static lapack_int blocked_lu(Index rows, Index cols, EIGTYPE* lu_data, Index luStride, lapack_int* row_transpositions, lapack_int& nb_transpositions, lapack_int maxBlockSize=256) \ { \ EIGEN_UNUSED_VARIABLE(maxBlockSize);\ lapack_int matrix_order, first_zero_pivot; \ @@ -54,14 +52,14 @@ EIGTYPE* a; \ /* Set up parameters for ?getrf */ \ matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ - lda = luStride; \ + lda = convert_index<lapack_int>(luStride); \ a = lu_data; \ ipiv = row_transpositions; \ - m = rows; \ - n = cols; \ + m = convert_index<lapack_int>(rows); \ + n = convert_index<lapack_int>(cols); \ nb_transpositions = 0; \ \ - info = LAPACKE_##MKLPREFIX##getrf( matrix_order, m, n, (MKLTYPE*)a, lda, ipiv ); \ + info = LAPACKE_##LAPACKE_PREFIX##getrf( matrix_order, m, n, (LAPACKE_TYPE*)a, lda, ipiv ); \ \ for(int i=0;i<m;i++) { ipiv[i]--; if (ipiv[i]!=i) nb_transpositions++; } \ \ @@ -73,10 +71,10 @@ } \ }; -EIGEN_MKL_LU_PARTPIV(double, double, d) -EIGEN_MKL_LU_PARTPIV(float, float, s) -EIGEN_MKL_LU_PARTPIV(dcomplex, MKL_Complex16, z) -EIGEN_MKL_LU_PARTPIV(scomplex, MKL_Complex8, c) +EIGEN_LAPACKE_LU_PARTPIV(double, double, d) +EIGEN_LAPACKE_LU_PARTPIV(float, float, s) +EIGEN_LAPACKE_LU_PARTPIV(dcomplex, lapack_complex_double, z) +EIGEN_LAPACKE_LU_PARTPIV(scomplex, lapack_complex_float, c) } // end namespace internal
diff --git a/Eigen/src/LU/arch/CMakeLists.txt b/Eigen/src/LU/arch/CMakeLists.txt deleted file mode 100644 index f6b7ed9..0000000 --- a/Eigen/src/LU/arch/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_LU_arch_SRCS "*.h") - -INSTALL(FILES - ${Eigen_LU_arch_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/LU/arch COMPONENT Devel - )
diff --git a/Eigen/src/LU/arch/Inverse_SSE.h b/Eigen/src/LU/arch/Inverse_SSE.h index e1470c6..4dce2ef 100644 --- a/Eigen/src/LU/arch/Inverse_SSE.h +++ b/Eigen/src/LU/arch/Inverse_SSE.h
@@ -44,7 +44,7 @@ static void run(const MatrixType& mat, ResultType& result) { ActualMatrixType matrix(mat); - EIGEN_ALIGN16 const unsigned int _Sign_PNNP[4] = { 0x00000000, 0x80000000, 0x80000000, 0x00000000 }; + const Packet4f p4f_sign_PNNP = _mm_castsi128_ps(_mm_set_epi32(0x00000000, 0x80000000, 0x80000000, 0x00000000)); // Load the full matrix into registers __m128 _L1 = matrix.template packet<MatrixAlignment>( 0); @@ -139,7 +139,7 @@ iC = _mm_sub_ps(iC, _mm_mul_ps(_mm_shuffle_ps(A,A,0xB1), _mm_shuffle_ps(DC,DC,0x66))); rd = _mm_shuffle_ps(rd,rd,0); - rd = _mm_xor_ps(rd, _mm_load_ps((float*)_Sign_PNNP)); + rd = _mm_xor_ps(rd, p4f_sign_PNNP); // iB = C*|B| - D*B#*A iB = _mm_sub_ps(_mm_mul_ps(C,_mm_shuffle_ps(dB,dB,0)), iB); @@ -153,10 +153,12 @@ iC = _mm_mul_ps(rd,iC); iD = _mm_mul_ps(rd,iD); - result.template writePacket<ResultAlignment>( 0, _mm_shuffle_ps(iA,iB,0x77)); - result.template writePacket<ResultAlignment>( 4, _mm_shuffle_ps(iA,iB,0x22)); - result.template writePacket<ResultAlignment>( 8, _mm_shuffle_ps(iC,iD,0x77)); - result.template writePacket<ResultAlignment>(12, _mm_shuffle_ps(iC,iD,0x22)); + Index res_stride = result.outerStride(); + float* res = result.data(); + pstoret<float, Packet4f, ResultAlignment>(res+0, _mm_shuffle_ps(iA,iB,0x77)); + pstoret<float, Packet4f, ResultAlignment>(res+res_stride, _mm_shuffle_ps(iA,iB,0x22)); + pstoret<float, Packet4f, ResultAlignment>(res+2*res_stride, _mm_shuffle_ps(iC,iD,0x77)); + pstoret<float, Packet4f, ResultAlignment>(res+3*res_stride, _mm_shuffle_ps(iC,iD,0x22)); } }; @@ -316,14 +318,16 @@ iC1 = _mm_sub_pd(_mm_mul_pd(B1, dC), iC1); iC2 = _mm_sub_pd(_mm_mul_pd(B2, dC), iC2); - result.template writePacket<ResultAlignment>( 0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1)); // iA# / det - result.template writePacket<ResultAlignment>( 4, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2)); - result.template writePacket<ResultAlignment>( 2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1)); // iB# / det - result.template writePacket<ResultAlignment>( 6, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2)); - result.template writePacket<ResultAlignment>( 8, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1)); // iC# / det - result.template writePacket<ResultAlignment>(12, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2)); - result.template writePacket<ResultAlignment>(10, _mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1)); // iD# / det - result.template writePacket<ResultAlignment>(14, _mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2)); + Index res_stride = result.outerStride(); + double* res = result.data(); + pstoret<double, Packet2d, ResultAlignment>(res+0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1)); + pstoret<double, Packet2d, ResultAlignment>(res+res_stride, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2)); + pstoret<double, Packet2d, ResultAlignment>(res+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1)); + pstoret<double, Packet2d, ResultAlignment>(res+res_stride+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2)); + pstoret<double, Packet2d, ResultAlignment>(res+2*res_stride, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1)); + pstoret<double, Packet2d, ResultAlignment>(res+3*res_stride, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2)); + pstoret<double, Packet2d, ResultAlignment>(res+2*res_stride+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1)); + pstoret<double, Packet2d, ResultAlignment>(res+3*res_stride+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2)); } };
diff --git a/Eigen/src/MetisSupport/CMakeLists.txt b/Eigen/src/MetisSupport/CMakeLists.txt deleted file mode 100644 index 2bad314..0000000 --- a/Eigen/src/MetisSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_MetisSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_MetisSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/MetisSupport COMPONENT Devel - )
diff --git a/Eigen/src/OrderingMethods/CMakeLists.txt b/Eigen/src/OrderingMethods/CMakeLists.txt deleted file mode 100644 index 9f4bb27..0000000 --- a/Eigen/src/OrderingMethods/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_OrderingMethods_SRCS "*.h") - -INSTALL(FILES - ${Eigen_OrderingMethods_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/OrderingMethods COMPONENT Devel - )
diff --git a/Eigen/src/OrderingMethods/Eigen_Colamd.h b/Eigen/src/OrderingMethods/Eigen_Colamd.h index 933cd56..67fcad3 100644 --- a/Eigen/src/OrderingMethods/Eigen_Colamd.h +++ b/Eigen/src/OrderingMethods/Eigen_Colamd.h
@@ -1004,7 +1004,7 @@ COLAMD_ASSERT (head [min_score] >= COLAMD_EMPTY) ; /* get pivot column from head of minimum degree list */ - while (head [min_score] == COLAMD_EMPTY && min_score < n_col) + while (min_score < n_col && head [min_score] == COLAMD_EMPTY) { min_score++ ; } @@ -1493,7 +1493,7 @@ c = Col [c].shared1.parent ; /* continue until we hit an ordered column. There are */ - /* guarranteed not to be anymore unordered columns */ + /* guaranteed not to be anymore unordered columns */ /* above an ordered column */ } while (Col [c].shared2.order == COLAMD_EMPTY) ; @@ -1638,7 +1638,7 @@ COLAMD_ASSERT (ROW_IS_ALIVE (*cp1)) ; COLAMD_ASSERT (ROW_IS_ALIVE (*cp2)) ; /* row indices will same order for both supercols, */ - /* no gather scatter nessasary */ + /* no gather scatter necessary */ if (*cp1++ != *cp2++) { break ; @@ -1688,7 +1688,7 @@ /* Defragments and compacts columns and rows in the workspace A. Used when - all avaliable memory has been used while performing row merging. Returns + all available memory has been used while performing row merging. Returns the index of the first free position in A, after garbage collection. The time taken by this routine is linear is the size of the array A, which is itself linear in the number of nonzeros in the input matrix.
diff --git a/Eigen/src/OrderingMethods/Ordering.h b/Eigen/src/OrderingMethods/Ordering.h index 7ea9b14..34dbef4 100644 --- a/Eigen/src/OrderingMethods/Ordering.h +++ b/Eigen/src/OrderingMethods/Ordering.h
@@ -31,7 +31,7 @@ for (int i = 0; i < C.rows(); i++) { for (typename MatrixType::InnerIterator it(C, i); it; ++it) - it.valueRef() = 0.0; + it.valueRef() = typename MatrixType::Scalar(0); } symmat = C + A; }
diff --git a/Eigen/src/PaStiXSupport/CMakeLists.txt b/Eigen/src/PaStiXSupport/CMakeLists.txt deleted file mode 100644 index 28c657e..0000000 --- a/Eigen/src/PaStiXSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_PastixSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_PastixSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PaStiXSupport COMPONENT Devel - )
diff --git a/Eigen/src/PaStiXSupport/PaStiXSupport.h b/Eigen/src/PaStiXSupport/PaStiXSupport.h index d2ebfd7..3742687 100644 --- a/Eigen/src/PaStiXSupport/PaStiXSupport.h +++ b/Eigen/src/PaStiXSupport/PaStiXSupport.h
@@ -64,28 +64,28 @@ typedef typename _MatrixType::StorageIndex StorageIndex; }; - void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, float *vals, int *perm, int * invp, float *x, int nbrhs, int *iparm, double *dparm) + inline void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, float *vals, int *perm, int * invp, float *x, int nbrhs, int *iparm, double *dparm) { if (n == 0) { ptr = NULL; idx = NULL; vals = NULL; } if (nbrhs == 0) {x = NULL; nbrhs=1;} s_pastix(pastix_data, pastix_comm, n, ptr, idx, vals, perm, invp, x, nbrhs, iparm, dparm); } - void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, double *vals, int *perm, int * invp, double *x, int nbrhs, int *iparm, double *dparm) + inline void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, double *vals, int *perm, int * invp, double *x, int nbrhs, int *iparm, double *dparm) { if (n == 0) { ptr = NULL; idx = NULL; vals = NULL; } if (nbrhs == 0) {x = NULL; nbrhs=1;} d_pastix(pastix_data, pastix_comm, n, ptr, idx, vals, perm, invp, x, nbrhs, iparm, dparm); } - void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex<float> *vals, int *perm, int * invp, std::complex<float> *x, int nbrhs, int *iparm, double *dparm) + inline void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex<float> *vals, int *perm, int * invp, std::complex<float> *x, int nbrhs, int *iparm, double *dparm) { if (n == 0) { ptr = NULL; idx = NULL; vals = NULL; } if (nbrhs == 0) {x = NULL; nbrhs=1;} c_pastix(pastix_data, pastix_comm, n, ptr, idx, reinterpret_cast<PASTIX_COMPLEX*>(vals), perm, invp, reinterpret_cast<PASTIX_COMPLEX*>(x), nbrhs, iparm, dparm); } - void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex<double> *vals, int *perm, int * invp, std::complex<double> *x, int nbrhs, int *iparm, double *dparm) + inline void eigen_pastix(pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex<double> *vals, int *perm, int * invp, std::complex<double> *x, int nbrhs, int *iparm, double *dparm) { if (n == 0) { ptr = NULL; idx = NULL; vals = NULL; } if (nbrhs == 0) {x = NULL; nbrhs=1;} @@ -203,7 +203,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the PaStiX reports a problem * \c InvalidInput if the input matrix is invalid *
diff --git a/Eigen/src/PardisoSupport/CMakeLists.txt b/Eigen/src/PardisoSupport/CMakeLists.txt deleted file mode 100644 index a097ab4..0000000 --- a/Eigen/src/PardisoSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_PardisoSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_PardisoSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/PardisoSupport COMPONENT Devel - )
diff --git a/Eigen/src/PardisoSupport/PardisoSupport.h b/Eigen/src/PardisoSupport/PardisoSupport.h index 80d914f..70afcb3 100644 --- a/Eigen/src/PardisoSupport/PardisoSupport.h +++ b/Eigen/src/PardisoSupport/PardisoSupport.h
@@ -123,6 +123,7 @@ }; PardisoImpl() + : m_analysisIsOk(false), m_factorizationIsOk(false), m_pt(0) { eigen_assert((sizeof(StorageIndex) >= sizeof(_INTEGER_t) && sizeof(StorageIndex) <= 8) && "Non-supported index type"); m_iparm.setZero(); @@ -140,7 +141,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix appears to be negative. */ ComputationInfo info() const @@ -183,7 +184,7 @@ { if(m_isInitialized) // Factorization ran at least once { - internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, -1, m_size,0, 0, 0, m_perm.data(), 0, + internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, -1, internal::convert_index<StorageIndex>(m_size),0, 0, 0, m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); m_isInitialized = false; } @@ -194,11 +195,11 @@ m_type = type; bool symmetric = std::abs(m_type) < 10; m_iparm[0] = 1; // No solver default - m_iparm[1] = 3; // use Metis for the ordering - m_iparm[2] = 1; // Numbers of processors, value of OMP_NUM_THREADS + m_iparm[1] = 2; // use Metis for the ordering + m_iparm[2] = 0; // Reserved. Set to zero. (??Numbers of processors, value of OMP_NUM_THREADS??) m_iparm[3] = 0; // No iterative-direct algorithm m_iparm[4] = 0; // No user fill-in reducing permutation - m_iparm[5] = 0; // Write solution into x + m_iparm[5] = 0; // Write solution into x, b is left unchanged m_iparm[6] = 0; // Not in use m_iparm[7] = 2; // Max numbers of iterative refinement steps m_iparm[8] = 0; // Not in use @@ -219,7 +220,8 @@ m_iparm[26] = 0; // No matrix checker m_iparm[27] = (sizeof(RealScalar) == 4) ? 1 : 0; m_iparm[34] = 1; // C indexing - m_iparm[59] = 1; // Automatic switch between In-Core and Out-of-Core modes + m_iparm[36] = 0; // CSR + m_iparm[59] = 0; // 0 - In-Core ; 1 - Automatic switch between In-Core and Out-of-Core modes ; 2 - Out-of-Core memset(m_pt, 0, sizeof(m_pt)); } @@ -246,7 +248,7 @@ mutable SparseMatrixType m_matrix; mutable ComputationInfo m_info; bool m_analysisIsOk, m_factorizationIsOk; - Index m_type, m_msglvl; + StorageIndex m_type, m_msglvl; mutable void *m_pt[64]; mutable ParameterType m_iparm; mutable IntColVectorType m_perm; @@ -265,10 +267,9 @@ derived().getMatrix(a); Index error; - error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 12, m_size, + error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 12, internal::convert_index<StorageIndex>(m_size), m_matrix.valuePtr(), m_matrix.outerIndexPtr(), m_matrix.innerIndexPtr(), m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); - manageErrorCode(error); m_analysisIsOk = true; m_factorizationIsOk = true; @@ -287,7 +288,7 @@ derived().getMatrix(a); Index error; - error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 11, m_size, + error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 11, internal::convert_index<StorageIndex>(m_size), m_matrix.valuePtr(), m_matrix.outerIndexPtr(), m_matrix.innerIndexPtr(), m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); @@ -306,8 +307,8 @@ derived().getMatrix(a); - Index error; - error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 22, m_size, + Index error; + error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 22, internal::convert_index<StorageIndex>(m_size), m_matrix.valuePtr(), m_matrix.outerIndexPtr(), m_matrix.innerIndexPtr(), m_perm.data(), 0, m_iparm.data(), m_msglvl, NULL, NULL); @@ -354,9 +355,9 @@ } Index error; - error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 33, m_size, + error = internal::pardiso_run_selector<StorageIndex>::run(m_pt, 1, 1, m_type, 33, internal::convert_index<StorageIndex>(m_size), m_matrix.valuePtr(), m_matrix.outerIndexPtr(), m_matrix.innerIndexPtr(), - m_perm.data(), nrhs, m_iparm.data(), m_msglvl, + m_perm.data(), internal::convert_index<StorageIndex>(nrhs), m_iparm.data(), m_msglvl, rhs_ptr, x.derived().data()); manageErrorCode(error); @@ -371,6 +372,9 @@ * using the Intel MKL PARDISO library. The sparse matrix A must be squared and invertible. * The vectors or matrices X and B can be either dense or sparse. * + * By default, it runs in in-core mode. To enable PARDISO's out-of-core feature, set: + * \code solver.pardisoParameterArray()[59] = 1; \endcode + * * \tparam _MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> * * \implsparsesolverconcept @@ -421,6 +425,9 @@ * using the Intel MKL PARDISO library. The sparse matrix A must be selfajoint and positive definite. * The vectors or matrices X and B can be either dense or sparse. * + * By default, it runs in in-core mode. To enable PARDISO's out-of-core feature, set: + * \code solver.pardisoParameterArray()[59] = 1; \endcode + * * \tparam MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> * \tparam UpLo can be any bitwise combination of Upper, Lower. The default is Upper, meaning only the upper triangular part has to be used. * Upper|Lower can be used to tell both triangular parts can be used as input. @@ -480,6 +487,9 @@ * For complex matrices, A can also be symmetric only, see the \a Options template parameter. * The vectors or matrices X and B can be either dense or sparse. * + * By default, it runs in in-core mode. To enable PARDISO's out-of-core feature, set: + * \code solver.pardisoParameterArray()[59] = 1; \endcode + * * \tparam MatrixType the type of the sparse matrix A, it must be a SparseMatrix<> * \tparam Options can be any bitwise combination of Upper, Lower, and Symmetric. The default is Upper, meaning only the upper triangular part has to be used. * Symmetric can be used for symmetric, non-selfadjoint complex matrices, the default being to assume a selfadjoint matrix.
diff --git a/Eigen/src/QR/CMakeLists.txt b/Eigen/src/QR/CMakeLists.txt deleted file mode 100644 index 96f43d7..0000000 --- a/Eigen/src/QR/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_QR_SRCS "*.h") - -INSTALL(FILES - ${Eigen_QR_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/QR COMPONENT Devel - )
diff --git a/Eigen/src/QR/ColPivHouseholderQR.h b/Eigen/src/QR/ColPivHouseholderQR.h index 7c559f9..9b677e9 100644 --- a/Eigen/src/QR/ColPivHouseholderQR.h +++ b/Eigen/src/QR/ColPivHouseholderQR.h
@@ -17,6 +17,9 @@ template<typename _MatrixType> struct traits<ColPivHouseholderQR<_MatrixType> > : traits<_MatrixType> { + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; enum { Flags = 0 }; }; @@ -41,25 +44,24 @@ * This decomposition performs column pivoting in order to be rank-revealing and improve * numerical stability. It is slower than HouseholderQR, and faster than FullPivHouseholderQR. * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::colPivHouseholderQr() */ template<typename _MatrixType> class ColPivHouseholderQR + : public SolverBase<ColPivHouseholderQR<_MatrixType> > { public: typedef _MatrixType MatrixType; + typedef SolverBase<ColPivHouseholderQR> Base; + friend class SolverBase<ColPivHouseholderQR>; + + EIGEN_GENERIC_PUBLIC_INTERFACE(ColPivHouseholderQR) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::RealScalar RealScalar; - // FIXME should be int - typedef typename MatrixType::StorageIndex StorageIndex; - typedef Matrix<Scalar, RowsAtCompileTime, RowsAtCompileTime, Options, MaxRowsAtCompileTime, MaxRowsAtCompileTime> MatrixQType; typedef typename internal::plain_diag_type<MatrixType>::type HCoeffsType; typedef PermutationMatrix<ColsAtCompileTime, MaxColsAtCompileTime> PermutationType; typedef typename internal::plain_row_type<MatrixType, Index>::type IntRowVectorType; @@ -135,6 +137,28 @@ compute(matrix.derived()); } + /** \brief Constructs a QR factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when \c MatrixType is a Eigen::Ref. + * + * \sa ColPivHouseholderQR(const EigenBase&) + */ + template<typename InputType> + explicit ColPivHouseholderQR(EigenBase<InputType>& matrix) + : m_qr(matrix.derived()), + m_hCoeffs((std::min)(matrix.rows(),matrix.cols())), + m_colsPermutation(PermIndexType(matrix.cols())), + m_colsTranspositions(matrix.cols()), + m_temp(matrix.cols()), + m_colNormsUpdated(matrix.cols()), + m_colNormsDirect(matrix.cols()), + m_isInitialized(false), + m_usePrescribedThreshold(false) + { + computeInPlace(); + } + + #ifdef EIGEN_PARSED_BY_DOXYGEN /** This method finds a solution x to the equation Ax=b, where A is the matrix of which * *this is the QR decomposition, if any exists. * @@ -142,9 +166,6 @@ * * \returns a solution. * - * \note The case where b is a matrix is not yet implemented. Also, this - * code is space inefficient. - * * \note_about_checking_solutions * * \note_about_arbitrary_choice_of_solution @@ -154,11 +175,8 @@ */ template<typename Rhs> inline const Solve<ColPivHouseholderQR, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); - return Solve<ColPivHouseholderQR, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif HouseholderSequenceType householderQ() const; HouseholderSequenceType matrixQ() const @@ -384,7 +402,7 @@ */ RealScalar maxPivot() const { return m_maxpivot; } - /** \brief Reports whether the QR factorization was succesful. + /** \brief Reports whether the QR factorization was successful. * * \note This function always returns \c Success. It is provided for compatibility * with other factorization routines. @@ -398,8 +416,10 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: @@ -453,21 +473,19 @@ template<typename InputType> ColPivHouseholderQR<MatrixType>& ColPivHouseholderQR<MatrixType>::compute(const EigenBase<InputType>& matrix) { - check_template_parameters(); - - // the column permutation is stored as int indices, so just to be sure: - eigen_assert(matrix.cols()<=NumTraits<int>::highest()); - - m_qr = matrix; - + m_qr = matrix.derived(); computeInPlace(); - return *this; } template<typename MatrixType> void ColPivHouseholderQR<MatrixType>::computeInPlace() { + check_template_parameters(); + + // the column permutation is stored as int indices, so just to be sure: + eigen_assert(m_qr.cols()<=NumTraits<int>::highest()); + using std::abs; Index rows = m_qr.rows(); @@ -490,8 +508,8 @@ m_colNormsUpdated.coeffRef(k) = m_colNormsDirect.coeffRef(k); } - RealScalar threshold_helper = numext::abs2(m_colNormsUpdated.maxCoeff() * NumTraits<Scalar>::epsilon()) / RealScalar(rows); - RealScalar norm_downdate_threshold = numext::sqrt(NumTraits<Scalar>::epsilon()); + RealScalar threshold_helper = numext::abs2<RealScalar>(m_colNormsUpdated.maxCoeff() * NumTraits<RealScalar>::epsilon()) / RealScalar(rows); + RealScalar norm_downdate_threshold = numext::sqrt(NumTraits<RealScalar>::epsilon()); m_nonzero_pivots = size; // the generic case is that in which all pivots are nonzero (invertible case) m_maxpivot = RealScalar(0); @@ -537,12 +555,12 @@ // http://www.netlib.org/lapack/lawnspdf/lawn176.pdf // and used in LAPACK routines xGEQPF and xGEQP3. // See lines 278-297 in http://www.netlib.org/lapack/explore-html/dc/df4/sgeqpf_8f_source.html - if (m_colNormsUpdated.coeffRef(j) != 0) { + if (m_colNormsUpdated.coeffRef(j) != RealScalar(0)) { RealScalar temp = abs(m_qr.coeffRef(k, j)) / m_colNormsUpdated.coeffRef(j); temp = (RealScalar(1) + temp) * (RealScalar(1) - temp); - temp = temp < 0 ? 0 : temp; - RealScalar temp2 = temp * numext::abs2(m_colNormsUpdated.coeffRef(j) / - m_colNormsDirect.coeffRef(j)); + temp = temp < RealScalar(0) ? RealScalar(0) : temp; + RealScalar temp2 = temp * numext::abs2<RealScalar>(m_colNormsUpdated.coeffRef(j) / + m_colNormsDirect.coeffRef(j)); if (temp2 <= norm_downdate_threshold) { // The updated norm has become too inaccurate so re-compute the column // norm directly. @@ -568,8 +586,6 @@ template<typename RhsType, typename DstType> void ColPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const { - eigen_assert(rhs.rows() == rows()); - const Index nonzero_pivots = nonzeroPivots(); if(nonzero_pivots == 0) @@ -580,11 +596,7 @@ typename RhsType::PlainObject c(rhs); - // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T - c.applyOnTheLeft(householderSequence(m_qr, m_hCoeffs) - .setLength(nonzero_pivots) - .transpose() - ); + c.applyOnTheLeft(householderQ().setLength(nonzero_pivots).adjoint() ); m_qr.topLeftCorner(nonzero_pivots, nonzero_pivots) .template triangularView<Upper>() @@ -593,16 +605,41 @@ for(Index i = 0; i < nonzero_pivots; ++i) dst.row(m_colsPermutation.indices().coeff(i)) = c.row(i); for(Index i = nonzero_pivots; i < cols(); ++i) dst.row(m_colsPermutation.indices().coeff(i)).setZero(); } + +template<typename _MatrixType> +template<bool Conjugate, typename RhsType, typename DstType> +void ColPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + const Index nonzero_pivots = nonzeroPivots(); + + if(nonzero_pivots == 0) + { + dst.setZero(); + return; + } + + typename RhsType::PlainObject c(m_colsPermutation.transpose()*rhs); + + m_qr.topLeftCorner(nonzero_pivots, nonzero_pivots) + .template triangularView<Upper>() + .transpose().template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(nonzero_pivots)); + + dst.topRows(nonzero_pivots) = c.topRows(nonzero_pivots); + dst.bottomRows(rows()-nonzero_pivots).setZero(); + + dst.applyOnTheLeft(householderQ().setLength(nonzero_pivots).template conjugateIf<!Conjugate>() ); +} #endif namespace internal { -template<typename DstXprType, typename MatrixType, typename Scalar> -struct Assignment<DstXprType, Inverse<ColPivHouseholderQR<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename MatrixType> +struct Assignment<DstXprType, Inverse<ColPivHouseholderQR<MatrixType> >, internal::assign_op<typename DstXprType::Scalar,typename ColPivHouseholderQR<MatrixType>::Scalar>, Dense2Dense> { typedef ColPivHouseholderQR<MatrixType> QrType; typedef Inverse<QrType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename QrType::Scalar> &) { dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } @@ -621,7 +658,6 @@ return HouseholderSequenceType(m_qr, m_hCoeffs.conjugate()); } -#ifndef __CUDACC__ /** \return the column-pivoting Householder QR decomposition of \c *this. * * \sa class ColPivHouseholderQR @@ -632,7 +668,6 @@ { return ColPivHouseholderQR<PlainObject>(eval()); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/QR/ColPivHouseholderQR_MKL.h b/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h similarity index 66% rename from Eigen/src/QR/ColPivHouseholderQR_MKL.h rename to Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h index 1203d0d..4e9651f 100644 --- a/Eigen/src/QR/ColPivHouseholderQR_MKL.h +++ b/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h
@@ -25,22 +25,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Householder QR decomposition of a matrix with column pivoting based on * LAPACKE_?geqp3 function. ******************************************************************************** */ -#ifndef EIGEN_COLPIVOTINGHOUSEHOLDERQR_MKL_H -#define EIGEN_COLPIVOTINGHOUSEHOLDERQR_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" +#ifndef EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H +#define EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H namespace Eigen { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_QR_COLPIV(EIGTYPE, MKLTYPE, MKLPREFIX, EIGCOLROW, MKLCOLROW) \ +#define EIGEN_LAPACKE_QR_COLPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX, EIGCOLROW, LAPACKE_COLROW) \ template<> template<typename InputType> inline \ ColPivHouseholderQR<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynamic> >& \ ColPivHouseholderQR<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynamic> >::compute( \ @@ -65,34 +63,35 @@ m_colsPermutation.resize(cols); \ m_colsPermutation.indices().setZero(); \ \ - lapack_int lda = m_qr.outerStride(), i; \ - lapack_int matrix_order = MKLCOLROW; \ - LAPACKE_##MKLPREFIX##geqp3( matrix_order, rows, cols, (MKLTYPE*)m_qr.data(), lda, (lapack_int*)m_colsPermutation.indices().data(), (MKLTYPE*)m_hCoeffs.data()); \ + lapack_int lda = internal::convert_index<lapack_int,Index>(m_qr.outerStride()); \ + lapack_int matrix_order = LAPACKE_COLROW; \ + LAPACKE_##LAPACKE_PREFIX##geqp3( matrix_order, internal::convert_index<lapack_int,Index>(rows), internal::convert_index<lapack_int,Index>(cols), \ + (LAPACKE_TYPE*)m_qr.data(), lda, (lapack_int*)m_colsPermutation.indices().data(), (LAPACKE_TYPE*)m_hCoeffs.data()); \ m_isInitialized = true; \ m_maxpivot=m_qr.diagonal().cwiseAbs().maxCoeff(); \ m_hCoeffs.adjointInPlace(); \ RealScalar premultiplied_threshold = abs(m_maxpivot) * threshold(); \ lapack_int *perm = m_colsPermutation.indices().data(); \ - for(i=0;i<size;i++) { \ + for(Index i=0;i<size;i++) { \ m_nonzero_pivots += (abs(m_qr.coeff(i,i)) > premultiplied_threshold);\ } \ - for(i=0;i<cols;i++) perm[i]--;\ + for(Index i=0;i<cols;i++) perm[i]--;\ \ /*m_det_pq = (number_of_transpositions%2) ? -1 : 1; // TODO: It's not needed now; fix upon availability in Eigen */ \ \ return *this; \ } -EIGEN_MKL_QR_COLPIV(double, double, d, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_QR_COLPIV(float, float, s, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_QR_COLPIV(dcomplex, MKL_Complex16, z, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_QR_COLPIV(scomplex, MKL_Complex8, c, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(double, double, d, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(float, float, s, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(dcomplex, lapack_complex_double, z, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(scomplex, lapack_complex_float, c, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_QR_COLPIV(double, double, d, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_QR_COLPIV(float, float, s, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_QR_COLPIV(dcomplex, MKL_Complex16, z, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_QR_COLPIV(scomplex, MKL_Complex8, c, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(double, double, d, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(float, float, s, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(dcomplex, lapack_complex_double, z, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_QR_COLPIV(scomplex, lapack_complex_float, c, RowMajor, LAPACK_ROW_MAJOR) } // end namespace Eigen -#endif // EIGEN_COLPIVOTINGHOUSEHOLDERQR_MKL_H +#endif // EIGEN_COLPIVOTINGHOUSEHOLDERQR_LAPACKE_H
diff --git a/Eigen/src/QR/CompleteOrthogonalDecomposition.h b/Eigen/src/QR/CompleteOrthogonalDecomposition.h index 230d0d2..2fc3c87 100644 --- a/Eigen/src/QR/CompleteOrthogonalDecomposition.h +++ b/Eigen/src/QR/CompleteOrthogonalDecomposition.h
@@ -16,6 +16,9 @@ template <typename _MatrixType> struct traits<CompleteOrthogonalDecomposition<_MatrixType> > : traits<_MatrixType> { + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; enum { Flags = 0 }; }; @@ -29,35 +32,36 @@ * * \param MatrixType the type of the matrix of which we are computing the COD. * - * This class performs a rank-revealing complete ortogonal decomposition of a + * This class performs a rank-revealing complete orthogonal decomposition of a * matrix \b A into matrices \b P, \b Q, \b T, and \b Z such that * \f[ - * \mathbf{A} \, \mathbf{P} = \mathbf{Q} \, \begin{matrix} \mathbf{T} & - * \mathbf{0} \\ \mathbf{0} & \mathbf{0} \end{matrix} \, \mathbf{Z} + * \mathbf{A} \, \mathbf{P} = \mathbf{Q} \, + * \begin{bmatrix} \mathbf{T} & \mathbf{0} \\ + * \mathbf{0} & \mathbf{0} \end{bmatrix} \, \mathbf{Z} * \f] * by using Householder transformations. Here, \b P is a permutation matrix, * \b Q and \b Z are unitary matrices and \b T an upper triangular matrix of * size rank-by-rank. \b A may be rank deficient. * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::completeOrthogonalDecomposition() */ -template <typename _MatrixType> -class CompleteOrthogonalDecomposition { +template <typename _MatrixType> class CompleteOrthogonalDecomposition + : public SolverBase<CompleteOrthogonalDecomposition<_MatrixType> > +{ public: typedef _MatrixType MatrixType; + typedef SolverBase<CompleteOrthogonalDecomposition> Base; + + template<typename Derived> + friend struct internal::solve_assertion; + + EIGEN_GENERIC_PUBLIC_INTERFACE(CompleteOrthogonalDecomposition) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::RealScalar RealScalar; - typedef typename MatrixType::StorageIndex StorageIndex; - typedef Matrix<Scalar, RowsAtCompileTime, RowsAtCompileTime, Options, - MaxRowsAtCompileTime, MaxRowsAtCompileTime> - MatrixQType; typedef typename internal::plain_diag_type<MatrixType>::type HCoeffsType; typedef PermutationMatrix<ColsAtCompileTime, MaxColsAtCompileTime> PermutationType; @@ -114,26 +118,40 @@ explicit CompleteOrthogonalDecomposition(const EigenBase<InputType>& matrix) : m_cpqr(matrix.rows(), matrix.cols()), m_zCoeffs((std::min)(matrix.rows(), matrix.cols())), - m_temp(matrix.cols()) { + m_temp(matrix.cols()) + { compute(matrix.derived()); } + /** \brief Constructs a complete orthogonal decomposition from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when \c MatrixType is a Eigen::Ref. + * + * \sa CompleteOrthogonalDecomposition(const EigenBase&) + */ + template<typename InputType> + explicit CompleteOrthogonalDecomposition(EigenBase<InputType>& matrix) + : m_cpqr(matrix.derived()), + m_zCoeffs((std::min)(matrix.rows(), matrix.cols())), + m_temp(matrix.cols()) + { + computeInPlace(); + } + + #ifdef EIGEN_PARSED_BY_DOXYGEN /** This method computes the minimum-norm solution X to a least squares - * problem \f[\mathrm{minimize} ||A X - B|| \f], where \b A is the matrix of + * problem \f[\mathrm{minimize} \|A X - B\|, \f] where \b A is the matrix of * which \c *this is the complete orthogonal decomposition. * - * \param B the right-hand sides of the problem to solve. + * \param b the right-hand sides of the problem to solve. * * \returns a solution. * */ template <typename Rhs> inline const Solve<CompleteOrthogonalDecomposition, Rhs> solve( - const MatrixBase<Rhs>& b) const { - eigen_assert(m_cpqr.m_isInitialized && - "CompleteOrthogonalDecomposition is not initialized."); - return Solve<CompleteOrthogonalDecomposition, Rhs>(*this, b.derived()); - } + const MatrixBase<Rhs>& b) const; + #endif HouseholderSequenceType householderQ(void) const; HouseholderSequenceType matrixQ(void) const { return m_cpqr.householderQ(); } @@ -142,8 +160,8 @@ */ MatrixType matrixZ() const { MatrixType Z = MatrixType::Identity(m_cpqr.cols(), m_cpqr.cols()); - applyZAdjointOnTheLeftInPlace(Z); - return Z.adjoint(); + applyZOnTheLeftInPlace<false>(Z); + return Z; } /** \returns a reference to the matrix where the complete orthogonal @@ -165,7 +183,12 @@ const MatrixType& matrixT() const { return m_cpqr.matrixQR(); } template <typename InputType> - CompleteOrthogonalDecomposition& compute(const EigenBase<InputType>& matrix); + CompleteOrthogonalDecomposition& compute(const EigenBase<InputType>& matrix) { + // Compute the column pivoted QR factorization A P = Q R. + m_cpqr.compute(matrix); + computeInPlace(); + return *this; + } /** \returns a const reference to the column permutation matrix */ const PermutationType& colsPermutation() const { @@ -254,6 +277,7 @@ */ inline const Inverse<CompleteOrthogonalDecomposition> pseudoInverse() const { + eigen_assert(m_cpqr.m_isInitialized && "CompleteOrthogonalDecomposition is not initialized."); return Inverse<CompleteOrthogonalDecomposition>(*this); } @@ -332,7 +356,7 @@ inline RealScalar maxPivot() const { return m_cpqr.maxPivot(); } /** \brief Reports whether the complete orthogonal decomposition was - * succesful. + * successful. * * \note This function always returns \c Success. It is provided for * compatibility @@ -346,7 +370,10 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN template <typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType& rhs, DstType& dst) const; + void _solve_impl(const RhsType& rhs, DstType& dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: @@ -354,6 +381,22 @@ EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } + template<bool Transpose_, typename Rhs> + void _check_solve_assertion(const Rhs& b) const { + EIGEN_ONLY_USED_FOR_DEBUG(b); + eigen_assert(m_cpqr.m_isInitialized && "CompleteOrthogonalDecomposition is not initialized."); + eigen_assert((Transpose_?derived().cols():derived().rows())==b.rows() && "CompleteOrthogonalDecomposition::solve(): invalid number of rows of the right hand side matrix b"); + } + + void computeInPlace(); + + /** Overwrites \b rhs with \f$ \mathbf{Z} * \mathbf{rhs} \f$ or + * \f$ \mathbf{\overline Z} * \mathbf{rhs} \f$ if \c Conjugate + * is set to \c true. + */ + template <bool Conjugate, typename Rhs> + void applyZOnTheLeftInPlace(Rhs& rhs) const; + /** Overwrites \b rhs with \f$ \mathbf{Z}^* * \mathbf{rhs} \f$. */ template <typename Rhs> @@ -384,20 +427,16 @@ * CompleteOrthogonalDecomposition(const MatrixType&) */ template <typename MatrixType> -template <typename InputType> -CompleteOrthogonalDecomposition<MatrixType>& CompleteOrthogonalDecomposition< - MatrixType>::compute(const EigenBase<InputType>& matrix) { +void CompleteOrthogonalDecomposition<MatrixType>::computeInPlace() +{ check_template_parameters(); // the column permutation is stored as int indices, so just to be sure: - eigen_assert(matrix.cols() <= NumTraits<int>::highest()); - - // Compute the column pivoted QR factorization A P = Q R. - m_cpqr.compute(matrix); + eigen_assert(m_cpqr.cols() <= NumTraits<int>::highest()); const Index rank = m_cpqr.rank(); - const Index cols = matrix.cols(); - const Index rows = matrix.rows(); + const Index cols = m_cpqr.cols(); + const Index rows = m_cpqr.rows(); m_zCoeffs.resize((std::min)(rows, cols)); m_temp.resize(cols); @@ -433,7 +472,7 @@ // Apply Z(k) to the first k rows of X_k m_cpqr.m_qr.topRightCorner(k, cols - rank + 1) .applyHouseholderOnTheRight( - m_cpqr.m_qr.row(k).tail(cols - rank).transpose(), m_zCoeffs(k), + m_cpqr.m_qr.row(k).tail(cols - rank).adjoint(), m_zCoeffs(k), &m_temp(0)); } if (k != rank - 1) { @@ -443,7 +482,28 @@ } } } - return *this; +} + +template <typename MatrixType> +template <bool Conjugate, typename Rhs> +void CompleteOrthogonalDecomposition<MatrixType>::applyZOnTheLeftInPlace( + Rhs& rhs) const { + const Index cols = this->cols(); + const Index nrhs = rhs.cols(); + const Index rank = this->rank(); + Matrix<typename Rhs::Scalar, Dynamic, 1> temp((std::max)(cols, nrhs)); + for (Index k = rank-1; k >= 0; --k) { + if (k != rank - 1) { + rhs.row(k).swap(rhs.row(rank - 1)); + } + rhs.middleRows(rank - 1, cols - rank + 1) + .applyHouseholderOnTheLeft( + matrixQTZ().row(k).tail(cols - rank).transpose().template conjugateIf<!Conjugate>(), zCoeffs().template conjugateIf<Conjugate>()(k), + &temp(0)); + if (k != rank - 1) { + rhs.row(k).swap(rhs.row(rank - 1)); + } + } } template <typename MatrixType> @@ -453,7 +513,7 @@ const Index cols = this->cols(); const Index nrhs = rhs.cols(); const Index rank = this->rank(); - Matrix<typename MatrixType::Scalar, Dynamic, 1> temp((std::max)(cols, nrhs)); + Matrix<typename Rhs::Scalar, Dynamic, 1> temp((std::max)(cols, nrhs)); for (Index k = 0; k < rank; ++k) { if (k != rank - 1) { rhs.row(k).swap(rhs.row(rank - 1)); @@ -473,8 +533,6 @@ template <typename RhsType, typename DstType> void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl( const RhsType& rhs, DstType& dst) const { - eigen_assert(rhs.rows() == this->rows()); - const Index rank = this->rank(); if (rank == 0) { dst.setZero(); @@ -482,11 +540,8 @@ } // Compute c = Q^* * rhs - // Note that the matrix Q = H_0^* H_1^*... so its inverse is - // Q^* = (H_0 H_1 ...)^T typename RhsType::PlainObject c(rhs); - c.applyOnTheLeft( - householderSequence(matrixQTZ(), hCoeffs()).setLength(rank).transpose()); + c.applyOnTheLeft(matrixQ().setLength(rank).adjoint()); // Solve T z = c(1:rank, :) dst.topRows(rank) = matrixT() @@ -505,16 +560,44 @@ // Undo permutation to get x = P^{-1} * y. dst = colsPermutation() * dst; } + +template<typename _MatrixType> +template<bool Conjugate, typename RhsType, typename DstType> +void CompleteOrthogonalDecomposition<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + const Index rank = this->rank(); + + if (rank == 0) { + dst.setZero(); + return; + } + + typename RhsType::PlainObject c(colsPermutation().transpose()*rhs); + + if (rank < cols()) { + applyZOnTheLeftInPlace<!Conjugate>(c); + } + + matrixT().topLeftCorner(rank, rank) + .template triangularView<Upper>() + .transpose().template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(rank)); + + dst.topRows(rank) = c.topRows(rank); + dst.bottomRows(rows()-rank).setZero(); + + dst.applyOnTheLeft(householderQ().setLength(rank).template conjugateIf<!Conjugate>() ); +} #endif namespace internal { -template<typename DstXprType, typename MatrixType, typename Scalar> -struct Assignment<DstXprType, Inverse<CompleteOrthogonalDecomposition<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename MatrixType> +struct Assignment<DstXprType, Inverse<CompleteOrthogonalDecomposition<MatrixType> >, internal::assign_op<typename DstXprType::Scalar,typename CompleteOrthogonalDecomposition<MatrixType>::Scalar>, Dense2Dense> { typedef CompleteOrthogonalDecomposition<MatrixType> CodType; typedef Inverse<CodType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename CodType::Scalar> &) { dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.rows())); } @@ -529,7 +612,6 @@ return m_cpqr.householderQ(); } -#ifndef __CUDACC__ /** \return the complete orthogonal decomposition of \c *this. * * \sa class CompleteOrthogonalDecomposition @@ -539,7 +621,6 @@ MatrixBase<Derived>::completeOrthogonalDecomposition() const { return CompleteOrthogonalDecomposition<PlainObject>(eval()); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/QR/FullPivHouseholderQR.h b/Eigen/src/QR/FullPivHouseholderQR.h index 32a10f3..d0664a1 100644 --- a/Eigen/src/QR/FullPivHouseholderQR.h +++ b/Eigen/src/QR/FullPivHouseholderQR.h
@@ -18,6 +18,9 @@ template<typename _MatrixType> struct traits<FullPivHouseholderQR<_MatrixType> > : traits<_MatrixType> { + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; enum { Flags = 0 }; }; @@ -50,24 +53,24 @@ * This decomposition performs a very prudent full pivoting in order to be rank-revealing and achieve optimal * numerical stability. The trade-off is that it is slower than HouseholderQR and ColPivHouseholderQR. * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::fullPivHouseholderQr() */ template<typename _MatrixType> class FullPivHouseholderQR + : public SolverBase<FullPivHouseholderQR<_MatrixType> > { public: typedef _MatrixType MatrixType; + typedef SolverBase<FullPivHouseholderQR> Base; + friend class SolverBase<FullPivHouseholderQR>; + + EIGEN_GENERIC_PUBLIC_INTERFACE(FullPivHouseholderQR) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::RealScalar RealScalar; - // FIXME should be int - typedef typename MatrixType::StorageIndex StorageIndex; typedef internal::FullPivHouseholderQRMatrixQReturnType<MatrixType> MatrixQReturnType; typedef typename internal::plain_diag_type<MatrixType>::type HCoeffsType; typedef Matrix<StorageIndex, 1, @@ -135,6 +138,27 @@ compute(matrix.derived()); } + /** \brief Constructs a QR factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when \c MatrixType is a Eigen::Ref. + * + * \sa FullPivHouseholderQR(const EigenBase&) + */ + template<typename InputType> + explicit FullPivHouseholderQR(EigenBase<InputType>& matrix) + : m_qr(matrix.derived()), + m_hCoeffs((std::min)(matrix.rows(), matrix.cols())), + m_rows_transpositions((std::min)(matrix.rows(), matrix.cols())), + m_cols_transpositions((std::min)(matrix.rows(), matrix.cols())), + m_cols_permutation(matrix.cols()), + m_temp(matrix.cols()), + m_isInitialized(false), + m_usePrescribedThreshold(false) + { + computeInPlace(); + } + + #ifdef EIGEN_PARSED_BY_DOXYGEN /** This method finds a solution x to the equation Ax=b, where A is the matrix of which * \c *this is the QR decomposition. * @@ -143,9 +167,6 @@ * \returns the exact or least-square solution if the rank is greater or equal to the number of columns of A, * and an arbitrary solution otherwise. * - * \note The case where b is a matrix is not yet implemented. Also, this - * code is space inefficient. - * * \note_about_checking_solutions * * \note_about_arbitrary_choice_of_solution @@ -155,11 +176,8 @@ */ template<typename Rhs> inline const Solve<FullPivHouseholderQR, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); - return Solve<FullPivHouseholderQR, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif /** \returns Expression object representing the matrix Q */ @@ -374,22 +392,24 @@ * diagonal coefficient of U. */ RealScalar maxPivot() const { return m_maxpivot; } - + #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: - + static void check_template_parameters() { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } - + void computeInPlace(); - + MatrixType m_qr; HCoeffsType m_hCoeffs; IntDiagSizeVectorType m_rows_transpositions; @@ -430,18 +450,16 @@ template<typename InputType> FullPivHouseholderQR<MatrixType>& FullPivHouseholderQR<MatrixType>::compute(const EigenBase<InputType>& matrix) { - check_template_parameters(); - m_qr = matrix.derived(); - computeInPlace(); - return *this; } template<typename MatrixType> void FullPivHouseholderQR<MatrixType>::computeInPlace() { + check_template_parameters(); + using std::abs; Index rows = m_qr.rows(); Index cols = m_qr.cols(); @@ -483,15 +501,15 @@ m_nonzero_pivots = k; for(Index i = k; i < size; i++) { - m_rows_transpositions.coeffRef(i) = i; - m_cols_transpositions.coeffRef(i) = i; + m_rows_transpositions.coeffRef(i) = internal::convert_index<StorageIndex>(i); + m_cols_transpositions.coeffRef(i) = internal::convert_index<StorageIndex>(i); m_hCoeffs.coeffRef(i) = Scalar(0); } break; } - m_rows_transpositions.coeffRef(k) = row_of_biggest_in_corner; - m_cols_transpositions.coeffRef(k) = col_of_biggest_in_corner; + m_rows_transpositions.coeffRef(k) = internal::convert_index<StorageIndex>(row_of_biggest_in_corner); + m_cols_transpositions.coeffRef(k) = internal::convert_index<StorageIndex>(col_of_biggest_in_corner); if(k != row_of_biggest_in_corner) { m_qr.row(k).tail(cols-k).swap(m_qr.row(row_of_biggest_in_corner).tail(cols-k)); ++number_of_transpositions; @@ -525,7 +543,6 @@ template<typename RhsType, typename DstType> void FullPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const { - eigen_assert(rhs.rows() == rows()); const Index l_rank = rank(); // FIXME introduce nonzeroPivots() and use it here. and more generally, @@ -538,7 +555,7 @@ typename RhsType::PlainObject c(rhs); - Matrix<Scalar,1,RhsType::ColsAtCompileTime> temp(rhs.cols()); + Matrix<typename RhsType::Scalar,1,RhsType::ColsAtCompileTime> temp(rhs.cols()); for (Index k = 0; k < l_rank; ++k) { Index remainingSize = rows()-k; @@ -555,16 +572,52 @@ for(Index i = 0; i < l_rank; ++i) dst.row(m_cols_permutation.indices().coeff(i)) = c.row(i); for(Index i = l_rank; i < cols(); ++i) dst.row(m_cols_permutation.indices().coeff(i)).setZero(); } + +template<typename _MatrixType> +template<bool Conjugate, typename RhsType, typename DstType> +void FullPivHouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + const Index l_rank = rank(); + + if(l_rank == 0) + { + dst.setZero(); + return; + } + + typename RhsType::PlainObject c(m_cols_permutation.transpose()*rhs); + + m_qr.topLeftCorner(l_rank, l_rank) + .template triangularView<Upper>() + .transpose().template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(l_rank)); + + dst.topRows(l_rank) = c.topRows(l_rank); + dst.bottomRows(rows()-l_rank).setZero(); + + Matrix<Scalar, 1, DstType::ColsAtCompileTime> temp(dst.cols()); + const Index size = (std::min)(rows(), cols()); + for (Index k = size-1; k >= 0; --k) + { + Index remainingSize = rows()-k; + + dst.bottomRightCorner(remainingSize, dst.cols()) + .applyHouseholderOnTheLeft(m_qr.col(k).tail(remainingSize-1).template conjugateIf<!Conjugate>(), + m_hCoeffs.template conjugateIf<Conjugate>().coeff(k), &temp.coeffRef(0)); + + dst.row(k).swap(dst.row(m_rows_transpositions.coeff(k))); + } +} #endif namespace internal { -template<typename DstXprType, typename MatrixType, typename Scalar> -struct Assignment<DstXprType, Inverse<FullPivHouseholderQR<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +template<typename DstXprType, typename MatrixType> +struct Assignment<DstXprType, Inverse<FullPivHouseholderQR<MatrixType> >, internal::assign_op<typename DstXprType::Scalar,typename FullPivHouseholderQR<MatrixType>::Scalar>, Dense2Dense> { typedef FullPivHouseholderQR<MatrixType> QrType; typedef Inverse<QrType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename QrType::Scalar> &) { dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } @@ -644,7 +697,6 @@ return MatrixQReturnType(m_qr, m_hCoeffs, m_rows_transpositions); } -#ifndef __CUDACC__ /** \return the full-pivoting Householder QR decomposition of \c *this. * * \sa class FullPivHouseholderQR @@ -655,7 +707,6 @@ { return FullPivHouseholderQR<PlainObject>(eval()); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/QR/HouseholderQR.h b/Eigen/src/QR/HouseholderQR.h index 03bc8e6..801739f 100644 --- a/Eigen/src/QR/HouseholderQR.h +++ b/Eigen/src/QR/HouseholderQR.h
@@ -14,6 +14,18 @@ namespace Eigen { +namespace internal { +template<typename _MatrixType> struct traits<HouseholderQR<_MatrixType> > + : traits<_MatrixType> +{ + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; + enum { Flags = 0 }; +}; + +} // end namespace internal + /** \ingroup QR_Module * * @@ -37,24 +49,24 @@ * This Householder QR decomposition is faster, but less numerically stable and less feature-full than * FullPivHouseholderQR or ColPivHouseholderQR. * + * This class supports the \link InplaceDecomposition inplace decomposition \endlink mechanism. + * * \sa MatrixBase::householderQr() */ template<typename _MatrixType> class HouseholderQR + : public SolverBase<HouseholderQR<_MatrixType> > { public: typedef _MatrixType MatrixType; + typedef SolverBase<HouseholderQR> Base; + friend class SolverBase<HouseholderQR>; + + EIGEN_GENERIC_PUBLIC_INTERFACE(HouseholderQR) enum { - RowsAtCompileTime = MatrixType::RowsAtCompileTime, - ColsAtCompileTime = MatrixType::ColsAtCompileTime, - Options = MatrixType::Options, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime }; - typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::RealScalar RealScalar; - // FIXME should be int - typedef typename MatrixType::StorageIndex StorageIndex; typedef Matrix<Scalar, RowsAtCompileTime, RowsAtCompileTime, (MatrixType::Flags&RowMajorBit) ? RowMajor : ColMajor, MaxRowsAtCompileTime, MaxRowsAtCompileTime> MatrixQType; typedef typename internal::plain_diag_type<MatrixType>::type HCoeffsType; typedef typename internal::plain_row_type<MatrixType>::type RowVectorType; @@ -102,6 +114,25 @@ compute(matrix.derived()); } + + /** \brief Constructs a QR factorization from a given matrix + * + * This overloaded constructor is provided for \link InplaceDecomposition inplace decomposition \endlink when + * \c MatrixType is a Eigen::Ref. + * + * \sa HouseholderQR(const EigenBase&) + */ + template<typename InputType> + explicit HouseholderQR(EigenBase<InputType>& matrix) + : m_qr(matrix.derived()), + m_hCoeffs((std::min)(matrix.rows(),matrix.cols())), + m_temp(matrix.cols()), + m_isInitialized(false) + { + computeInPlace(); + } + + #ifdef EIGEN_PARSED_BY_DOXYGEN /** This method finds a solution x to the equation Ax=b, where A is the matrix of which * *this is the QR decomposition, if any exists. * @@ -109,9 +140,6 @@ * * \returns a solution. * - * \note The case where b is a matrix is not yet implemented. Also, this - * code is space inefficient. - * * \note_about_checking_solutions * * \note_about_arbitrary_choice_of_solution @@ -121,11 +149,8 @@ */ template<typename Rhs> inline const Solve<HouseholderQR, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); - return Solve<HouseholderQR, Rhs>(*this, b.derived()); - } + solve(const MatrixBase<Rhs>& b) const; + #endif /** This method returns an expression of the unitary matrix Q as a sequence of Householder transformations. * @@ -151,7 +176,11 @@ } template<typename InputType> - HouseholderQR& compute(const EigenBase<InputType>& matrix); + HouseholderQR& compute(const EigenBase<InputType>& matrix) { + m_qr = matrix.derived(); + computeInPlace(); + return *this; + } /** \returns the absolute value of the determinant of the matrix of which * *this is the QR decomposition. It has only linear complexity @@ -184,26 +213,30 @@ inline Index rows() const { return m_qr.rows(); } inline Index cols() const { return m_qr.cols(); } - + /** \returns a const reference to the vector of Householder coefficients used to represent the factor \c Q. * * For advanced uses only. */ const HCoeffsType& hCoeffs() const { return m_hCoeffs; } - + #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: - + static void check_template_parameters() { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } - + + void computeInPlace(); + MatrixType m_qr; HCoeffsType m_hCoeffs; RowVectorType m_temp; @@ -270,7 +303,7 @@ bool InnerStrideIsOne = (MatrixQR::InnerStrideAtCompileTime == 1 && HCoeffs::InnerStrideAtCompileTime == 1)> struct householder_qr_inplace_blocked { - // This is specialized for MKL-supported Scalar types in HouseholderQR_MKL.h + // This is specialized for LAPACK-supported Scalar types in HouseholderQR_LAPACKE.h static void run(MatrixQR& mat, HCoeffs& hCoeffs, Index maxBlockSize=32, typename MatrixQR::Scalar* tempData = 0) { @@ -328,15 +361,10 @@ void HouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const { const Index rank = (std::min)(rows(), cols()); - eigen_assert(rhs.rows() == rows()); typename RhsType::PlainObject c(rhs); - // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T - c.applyOnTheLeft(householderSequence( - m_qr.leftCols(rank), - m_hCoeffs.head(rank)).transpose() - ); + c.applyOnTheLeft(householderQ().setLength(rank).adjoint() ); m_qr.topLeftCorner(rank, rank) .template triangularView<Upper>() @@ -345,6 +373,25 @@ dst.topRows(rank) = c.topRows(rank); dst.bottomRows(cols()-rank).setZero(); } + +template<typename _MatrixType> +template<bool Conjugate, typename RhsType, typename DstType> +void HouseholderQR<_MatrixType>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + const Index rank = (std::min)(rows(), cols()); + + typename RhsType::PlainObject c(rhs); + + m_qr.topLeftCorner(rank, rank) + .template triangularView<Upper>() + .transpose().template conjugateIf<Conjugate>() + .solveInPlace(c.topRows(rank)); + + dst.topRows(rank) = c.topRows(rank); + dst.bottomRows(rows()-rank).setZero(); + + dst.applyOnTheLeft(householderQ().setLength(rank).template conjugateIf<!Conjugate>() ); +} #endif /** Performs the QR factorization of the given matrix \a matrix. The result of @@ -354,16 +401,14 @@ * \sa class HouseholderQR, HouseholderQR(const MatrixType&) */ template<typename MatrixType> -template<typename InputType> -HouseholderQR<MatrixType>& HouseholderQR<MatrixType>::compute(const EigenBase<InputType>& matrix) +void HouseholderQR<MatrixType>::computeInPlace() { check_template_parameters(); - Index rows = matrix.rows(); - Index cols = matrix.cols(); + Index rows = m_qr.rows(); + Index cols = m_qr.cols(); Index size = (std::min)(rows,cols); - m_qr = matrix.derived(); m_hCoeffs.resize(size); m_temp.resize(cols); @@ -371,10 +416,8 @@ internal::householder_qr_inplace_blocked<MatrixType, HCoeffsType>::run(m_qr, m_hCoeffs, 48, m_temp.data()); m_isInitialized = true; - return *this; } -#ifndef __CUDACC__ /** \return the Householder QR decomposition of \c *this. * * \sa class HouseholderQR @@ -385,7 +428,6 @@ { return HouseholderQR<PlainObject>(eval()); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/QR/HouseholderQR_MKL.h b/Eigen/src/QR/HouseholderQR_LAPACKE.h similarity index 80% rename from Eigen/src/QR/HouseholderQR_MKL.h rename to Eigen/src/QR/HouseholderQR_LAPACKE.h index 84ab640..1dc7d53 100644 --- a/Eigen/src/QR/HouseholderQR_MKL.h +++ b/Eigen/src/QR/HouseholderQR_LAPACKE.h
@@ -25,24 +25,22 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Householder QR decomposition of a matrix w/o pivoting based on * LAPACKE_?geqrf function. ******************************************************************************** */ -#ifndef EIGEN_QR_MKL_H -#define EIGEN_QR_MKL_H - -#include "../Core/util/MKL_support.h" +#ifndef EIGEN_QR_LAPACKE_H +#define EIGEN_QR_LAPACKE_H namespace Eigen { namespace internal { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_QR_NOPIV(EIGTYPE, MKLTYPE, MKLPREFIX) \ +#define EIGEN_LAPACKE_QR_NOPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ template<typename MatrixQR, typename HCoeffs> \ struct householder_qr_inplace_blocked<MatrixQR, HCoeffs, EIGTYPE, true> \ { \ @@ -53,18 +51,18 @@ lapack_int n = (lapack_int) mat.cols(); \ lapack_int lda = (lapack_int) mat.outerStride(); \ lapack_int matrix_order = (MatrixQR::IsRowMajor) ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ - LAPACKE_##MKLPREFIX##geqrf( matrix_order, m, n, (MKLTYPE*)mat.data(), lda, (MKLTYPE*)hCoeffs.data()); \ + LAPACKE_##LAPACKE_PREFIX##geqrf( matrix_order, m, n, (LAPACKE_TYPE*)mat.data(), lda, (LAPACKE_TYPE*)hCoeffs.data()); \ hCoeffs.adjointInPlace(); \ } \ }; -EIGEN_MKL_QR_NOPIV(double, double, d) -EIGEN_MKL_QR_NOPIV(float, float, s) -EIGEN_MKL_QR_NOPIV(dcomplex, MKL_Complex16, z) -EIGEN_MKL_QR_NOPIV(scomplex, MKL_Complex8, c) +EIGEN_LAPACKE_QR_NOPIV(double, double, d) +EIGEN_LAPACKE_QR_NOPIV(float, float, s) +EIGEN_LAPACKE_QR_NOPIV(dcomplex, lapack_complex_double, z) +EIGEN_LAPACKE_QR_NOPIV(scomplex, lapack_complex_float, c) } // end namespace internal } // end namespace Eigen -#endif // EIGEN_QR_MKL_H +#endif // EIGEN_QR_LAPACKE_H
diff --git a/Eigen/src/SPQRSupport/CMakeLists.txt b/Eigen/src/SPQRSupport/CMakeLists.txt deleted file mode 100644 index 4968bea..0000000 --- a/Eigen/src/SPQRSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SPQRSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SPQRSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SPQRSupport/ COMPONENT Devel - )
diff --git a/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h b/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h index d9c3113..013c7ae 100644 --- a/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h +++ b/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h
@@ -74,13 +74,35 @@ }; public: SPQR() - : m_ordering(SPQR_ORDERING_DEFAULT), m_allow_tol(SPQR_DEFAULT_TOL), m_tolerance (NumTraits<Scalar>::epsilon()), m_useDefaultThreshold(true) + : m_analysisIsOk(false), + m_factorizationIsOk(false), + m_isRUpToDate(false), + m_ordering(SPQR_ORDERING_DEFAULT), + m_allow_tol(SPQR_DEFAULT_TOL), + m_tolerance (NumTraits<Scalar>::epsilon()), + m_cR(0), + m_E(0), + m_H(0), + m_HPinv(0), + m_HTau(0), + m_useDefaultThreshold(true) { cholmod_l_start(&m_cc); } explicit SPQR(const _MatrixType& matrix) - : m_ordering(SPQR_ORDERING_DEFAULT), m_allow_tol(SPQR_DEFAULT_TOL), m_tolerance (NumTraits<Scalar>::epsilon()), m_useDefaultThreshold(true) + : m_analysisIsOk(false), + m_factorizationIsOk(false), + m_isRUpToDate(false), + m_ordering(SPQR_ORDERING_DEFAULT), + m_allow_tol(SPQR_DEFAULT_TOL), + m_tolerance (NumTraits<Scalar>::epsilon()), + m_cR(0), + m_E(0), + m_H(0), + m_HPinv(0), + m_HTau(0), + m_useDefaultThreshold(true) { cholmod_l_start(&m_cc); compute(matrix); @@ -119,16 +141,16 @@ max2Norm = RealScalar(1); pivotThreshold = 20 * (mat.rows() + mat.cols()) * max2Norm * NumTraits<RealScalar>::epsilon(); } - cholmod_sparse A; A = viewAsCholmod(mat); + m_rows = matrix.rows(); Index col = matrix.cols(); m_rank = SuiteSparseQR<Scalar>(m_ordering, pivotThreshold, col, &A, &m_cR, &m_E, &m_H, &m_HPinv, &m_HTau, &m_cc); if (!m_cR) { - m_info = NumericalIssue; + m_info = NumericalIssue; m_isInitialized = false; return; } @@ -139,7 +161,7 @@ /** * Get the number of rows of the input matrix and the Q matrix */ - inline Index rows() const {return m_cR->nrow; } + inline Index rows() const {return m_rows; } /** * Get the number of columns of the input matrix. @@ -220,7 +242,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the sparse QR can not be computed */ ComputationInfo info() const @@ -245,6 +267,7 @@ mutable Index m_rank; // The rank of the matrix mutable cholmod_common m_cc; // Workspace and parameters bool m_useDefaultThreshold; // Use default threshold + Index m_rows; template<typename ,typename > friend struct SPQR_QProduct; };
diff --git a/Eigen/src/SVD/BDCSVD.h b/Eigen/src/SVD/BDCSVD.h index 3552c87..e3fddac 100644 --- a/Eigen/src/SVD/BDCSVD.h +++ b/Eigen/src/SVD/BDCSVD.h
@@ -11,7 +11,7 @@ // Copyright (C) 2013 Jean Ceccato <jean.ceccato@ensimag.fr> // Copyright (C) 2013 Pierre Zoppitelli <pierre.zoppitelli@ensimag.fr> // Copyright (C) 2013 Jitse Niesen <jitse@maths.leeds.ac.uk> -// Copyright (C) 2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2014-2017 Gael Guennebaud <gael.guennebaud@inria.fr> // // Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -21,6 +21,12 @@ #define EIGEN_BDCSVD_H // #define EIGEN_BDCSVD_DEBUG_VERBOSE // #define EIGEN_BDCSVD_SANITY_CHECKS + +#ifdef EIGEN_BDCSVD_SANITY_CHECKS +#undef eigen_internal_assert +#define eigen_internal_assert(X) assert(X); +#endif + namespace Eigen { #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE @@ -33,6 +39,7 @@ template<typename _MatrixType> struct traits<BDCSVD<_MatrixType> > + : traits<_MatrixType> { typedef _MatrixType MatrixType; }; @@ -49,6 +56,18 @@ * * \tparam _MatrixType the type of the matrix of which we are computing the SVD decomposition * + * This class first reduces the input matrix to bi-diagonal form using class UpperBidiagonalization, + * and then performs a divide-and-conquer diagonalization. Small blocks are diagonalized using class JacobiSVD. + * You can control the switching size with the setSwitchSize() method, default is 16. + * For small matrice (<16), it is thus preferable to directly use JacobiSVD. For larger ones, BDCSVD is highly + * recommended and can several order of magnitude faster. + * + * \warning this algorithm is unlikely to provide accurate result when compiled with unsafe math optimizations. + * For instance, this concerns Intel's compiler (ICC), which performs such optimization by default unless + * you compile with the \c -fp-model \c precise option. Likewise, the \c -ffast-math option of GCC or clang will + * significantly degrade the accuracy. + * + * \sa class JacobiSVD */ template<typename _MatrixType> class BDCSVD : public SVDBase<BDCSVD<_MatrixType> > @@ -64,6 +83,7 @@ typedef _MatrixType MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; + typedef typename NumTraits<RealScalar>::Literal Literal; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -91,7 +111,7 @@ * The default constructor is useful in cases in which the user intends to * perform decompositions via BDCSVD::compute(const MatrixType&). */ - BDCSVD() : m_algoswap(16), m_numIters(0) + BDCSVD() : m_algoswap(16), m_isTranspose(false), m_compU(false), m_compV(false), m_numIters(0) {} @@ -198,7 +218,7 @@ // Method to allocate and initialize matrix and attributes template<typename MatrixType> -void BDCSVD<MatrixType>::allocate(Index rows, Index cols, unsigned int computationOptions) +void BDCSVD<MatrixType>::allocate(Eigen::Index rows, Eigen::Index cols, unsigned int computationOptions) { m_isTranspose = (cols > rows); @@ -228,6 +248,8 @@ #endif allocate(matrix.rows(), matrix.cols(), computationOptions); using std::abs; + + const RealScalar considerZero = (std::numeric_limits<RealScalar>::min)(); //**** step -1 - If the problem is too small, directly falls back to JacobiSVD and return if(matrix.cols() < m_algoswap) @@ -244,7 +266,7 @@ //**** step 0 - Copy the input matrix and apply scaling to reduce over/under-flows RealScalar scale = matrix.cwiseAbs().maxCoeff(); - if(scale==RealScalar(0)) scale = RealScalar(1); + if(scale==Literal(0)) scale = Literal(1); MatrixX copy; if (m_isTranspose) copy = matrix.adjoint()/scale; else copy = matrix/scale; @@ -266,7 +288,7 @@ { RealScalar a = abs(m_computed.coeff(i, i)); m_singularValues.coeffRef(i) = a * scale; - if (a == 0) + if (a<considerZero) { m_nonzeroSingularValues = i; m_singularValues.tail(m_diagSize - i - 1).setZero(); @@ -336,13 +358,13 @@ Index k1=0, k2=0; for(Index j=0; j<n; ++j) { - if( (A.col(j).head(n1).array()!=0).any() ) + if( (A.col(j).head(n1).array()!=Literal(0)).any() ) { A1.col(k1) = A.col(j).head(n1); B1.row(k1) = B.row(j); ++k1; } - if( (A.col(j).tail(n2).array()!=0).any() ) + if( (A.col(j).tail(n2).array()!=Literal(0)).any() ) { A2.col(k2) = A.col(j).tail(n2); B2.row(k2) = B.row(j); @@ -372,7 +394,7 @@ //@param shift : Each time one takes the left submatrix, one must add 1 to the shift. Why? Because! We actually want the last column of the U submatrix // to become the first column (*coeff) and to shift all the other columns to the right. There are more details on the reference paper. template<typename MatrixType> -void BDCSVD<MatrixType>::divide (Index firstCol, Index lastCol, Index firstRowW, Index firstColW, Index shift) +void BDCSVD<MatrixType>::divide (Eigen::Index firstCol, Eigen::Index lastCol, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index shift) { // requires rows = cols + 1; using std::pow; @@ -380,6 +402,7 @@ using std::abs; const Index n = lastCol - firstCol + 1; const Index k = n/2; + const RealScalar considerZero = (std::numeric_limits<RealScalar>::min)(); RealScalar alphaK; RealScalar betaK; RealScalar r0; @@ -433,11 +456,11 @@ l = m_naiveU.row(1).segment(firstCol, k); f = m_naiveU.row(0).segment(firstCol + k + 1, n - k - 1); } - if (m_compV) m_naiveV(firstRowW+k, firstColW) = 1; - if (r0 == 0) + if (m_compV) m_naiveV(firstRowW+k, firstColW) = Literal(1); + if (r0<considerZero) { - c0 = 1; - s0 = 0; + c0 = Literal(1); + s0 = Literal(0); } else { @@ -551,12 +574,14 @@ // handling of round-off errors, be consistent in ordering // For instance, to solve the secular equation using FMM, see http://www.stat.uchicago.edu/~lekheng/courses/302/classics/greengard-rokhlin.pdf template <typename MatrixType> -void BDCSVD<MatrixType>::computeSVDofM(Index firstCol, Index n, MatrixXr& U, VectorType& singVals, MatrixXr& V) +void BDCSVD<MatrixType>::computeSVDofM(Eigen::Index firstCol, Eigen::Index n, MatrixXr& U, VectorType& singVals, MatrixXr& V) { + const RealScalar considerZero = (std::numeric_limits<RealScalar>::min)(); + using std::abs; ArrayRef col0 = m_computed.col(firstCol).segment(firstCol, n); m_workspace.head(n) = m_computed.block(firstCol, firstCol, n, n).diagonal(); ArrayRef diag = m_workspace.head(n); - diag(0) = 0; + diag(0) = Literal(0); // Allocate space for singular values and vectors singVals.resize(n); @@ -572,10 +597,10 @@ // but others are interleaved and we must ignore them at this stage. // To this end, let's compute a permutation skipping them: Index actual_n = n; - while(actual_n>1 && diag(actual_n-1)==0) --actual_n; + while(actual_n>1 && diag(actual_n-1)==Literal(0)) {--actual_n; eigen_internal_assert(col0(actual_n)==Literal(0)); } Index m = 0; // size of the deflated problem for(Index k=0;k<actual_n;++k) - if(col0(k)!=0) + if(abs(col0(k))>considerZero) m_workspaceI(m++) = k; Map<ArrayXi> perm(m_workspaceI.data(),m); @@ -599,13 +624,11 @@ std::cout << " shift: " << shifts.transpose() << "\n"; { - Index actual_n = n; - while(actual_n>1 && col0(actual_n-1)==0) --actual_n; std::cout << "\n\n mus: " << mus.head(actual_n).transpose() << "\n\n"; std::cout << " check1 (expect0) : " << ((singVals.array()-(shifts+mus)) / singVals.array()).head(actual_n).transpose() << "\n\n"; + assert((((singVals.array()-(shifts+mus)) / singVals.array()).head(actual_n) >= 0).all()); std::cout << " check2 (>0) : " << ((singVals.array()-diag) / singVals.array()).head(actual_n).transpose() << "\n\n"; - std::cout << " check3 (>0) : " << ((diag.segment(1,actual_n-1)-singVals.head(actual_n-1).array()) / singVals.head(actual_n-1).array()).transpose() << "\n\n\n"; - std::cout << " check4 (>0) : " << ((singVals.segment(1,actual_n-1)-singVals.head(actual_n-1))).transpose() << "\n\n\n"; + assert((((singVals.array()-diag) / singVals.array()).head(actual_n) >= 0).all()); } #endif @@ -633,13 +656,13 @@ #endif #ifdef EIGEN_BDCSVD_SANITY_CHECKS - assert(U.allFinite()); - assert(V.allFinite()); - assert((U.transpose() * U - MatrixXr(MatrixXr::Identity(U.cols(),U.cols()))).norm() < 1e-14 * n); - assert((V.transpose() * V - MatrixXr(MatrixXr::Identity(V.cols(),V.cols()))).norm() < 1e-14 * n); assert(m_naiveU.allFinite()); assert(m_naiveV.allFinite()); assert(m_computed.allFinite()); + assert(U.allFinite()); + assert(V.allFinite()); +// assert((U.transpose() * U - MatrixXr(MatrixXr::Identity(U.cols(),U.cols()))).norm() < 100*NumTraits<RealScalar>::epsilon() * n); +// assert((V.transpose() * V - MatrixXr(MatrixXr::Identity(V.cols(),V.cols()))).norm() < 100*NumTraits<RealScalar>::epsilon() * n); #endif // Because of deflation, the singular values might not be completely sorted. @@ -654,6 +677,15 @@ if(m_compV) V.col(i).swap(V.col(i+1)); } } + +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + { + bool singular_values_sorted = (((singVals.segment(1,actual_n-1)-singVals.head(actual_n-1))).array() >= 0).all(); + if(!singular_values_sorted) + std::cout << "Singular values are not sorted: " << singVals.segment(1,actual_n).transpose() << "\n"; + assert(singular_values_sorted); + } +#endif // Reverse order so that singular values in increased order // Because of deflation, the zeros singular-values are already at the end @@ -673,13 +705,16 @@ typename BDCSVD<MatrixType>::RealScalar BDCSVD<MatrixType>::secularEq(RealScalar mu, const ArrayRef& col0, const ArrayRef& diag, const IndicesRef &perm, const ArrayRef& diagShifted, RealScalar shift) { Index m = perm.size(); - RealScalar res = 1; + RealScalar res = Literal(1); for(Index i=0; i<m; ++i) { Index j = perm(i); - res += numext::abs2(col0(j)) / ((diagShifted(j) - mu) * (diag(j) + shift + mu)); + // The following expression could be rewritten to involve only a single division, + // but this would make the expression more sensitive to overflow. + res += (col0(j) / (diagShifted(j) - mu)) * (col0(j) / (diag(j) + shift + mu)); } return res; + } template <typename MatrixType> @@ -688,19 +723,22 @@ { using std::abs; using std::swap; + using std::sqrt; Index n = col0.size(); Index actual_n = n; - while(actual_n>1 && col0(actual_n-1)==0) --actual_n; + // Note that here actual_n is computed based on col0(i)==0 instead of diag(i)==0 as above + // because 1) we have diag(i)==0 => col0(i)==0 and 2) if col0(i)==0, then diag(i) is already a singular value. + while(actual_n>1 && col0(actual_n-1)==Literal(0)) --actual_n; for (Index k = 0; k < n; ++k) { - if (col0(k) == 0 || actual_n==1) + if (col0(k) == Literal(0) || actual_n==1) { // if col0(k) == 0, then entry is deflated, so singular value is on diagonal // if actual_n==1, then the deflated problem is already diagonalized singVals(k) = k==0 ? col0(0) : diag(k); - mus(k) = 0; + mus(k) = Literal(0); shifts(k) = k==0 ? col0(0) : diag(k); continue; } @@ -712,31 +750,36 @@ right = (diag(actual_n-1) + col0.matrix().norm()); else { - // Skip deflated singular values + // Skip deflated singular values, + // recall that at this stage we assume that z[j]!=0 and all entries for which z[j]==0 have been put aside. + // This should be equivalent to using perm[] Index l = k+1; - while(col0(l)==0) { ++l; eigen_internal_assert(l<actual_n); } + while(col0(l)==Literal(0)) { ++l; eigen_internal_assert(l<actual_n); } right = diag(l); } // first decide whether it's closer to the left end or the right end - RealScalar mid = left + (right-left) / 2; - RealScalar fMid = secularEq(mid, col0, diag, perm, diag, 0); + RealScalar mid = left + (right-left) / Literal(2); + RealScalar fMid = secularEq(mid, col0, diag, perm, diag, Literal(0)); #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE - std::cout << right-left << "\n"; - std::cout << "fMid = " << fMid << " " << secularEq(mid-left, col0, diag, perm, diag-left, left) << " " << secularEq(mid-right, col0, diag, perm, diag-right, right) << "\n"; - std::cout << " = " << secularEq(0.1*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.2*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.3*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.4*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.49*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.5*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.51*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.6*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.7*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.8*(left+right), col0, diag, perm, diag, 0) - << " " << secularEq(0.9*(left+right), col0, diag, perm, diag, 0) << "\n"; + std::cout << "right-left = " << right-left << "\n"; +// std::cout << "fMid = " << fMid << " " << secularEq(mid-left, col0, diag, perm, ArrayXr(diag-left), left) +// << " " << secularEq(mid-right, col0, diag, perm, ArrayXr(diag-right), right) << "\n"; + std::cout << " = " << secularEq(left+RealScalar(0.000001)*(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.1) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.2) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.3) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.4) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.49) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.5) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.51) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.6) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.7) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.8) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.9) *(right-left), col0, diag, perm, diag, 0) + << " " << secularEq(left+RealScalar(0.999999)*(right-left), col0, diag, perm, diag, 0) << "\n"; #endif - RealScalar shift = (k == actual_n-1 || fMid > 0) ? left : right; + RealScalar shift = (k == actual_n-1 || fMid > Literal(0)) ? left : right; // measure everything relative to shift Map<ArrayXr> diagShifted(m_workspace.data()+4*n, n); @@ -746,14 +789,14 @@ RealScalar muPrev, muCur; if (shift == left) { - muPrev = (right - left) * 0.1; + muPrev = (right - left) * RealScalar(0.1); if (k == actual_n-1) muCur = right - left; - else muCur = (right - left) * 0.5; + else muCur = (right - left) * RealScalar(0.5); } else { - muPrev = -(right - left) * 0.1; - muCur = -(right - left) * 0.5; + muPrev = -(right - left) * RealScalar(0.1); + muCur = -(right - left) * RealScalar(0.5); } RealScalar fPrev = secularEq(muPrev, col0, diag, perm, diagShifted, shift); @@ -766,26 +809,29 @@ // rational interpolation: fit a function of the form a / mu + b through the two previous // iterates and use its zero to compute the next iterate - bool useBisection = fPrev*fCur>0; - while (fCur!=0 && abs(muCur - muPrev) > 8 * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(abs(muCur), abs(muPrev)) && abs(fCur - fPrev)>NumTraits<RealScalar>::epsilon() && !useBisection) + bool useBisection = fPrev*fCur>Literal(0); + while (fCur!=Literal(0) && abs(muCur - muPrev) > Literal(8) * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(abs(muCur), abs(muPrev)) && abs(fCur - fPrev)>NumTraits<RealScalar>::epsilon() && !useBisection) { ++m_numIters; // Find a and b such that the function f(mu) = a / mu + b matches the current and previous samples. - RealScalar a = (fCur - fPrev) / (1/muCur - 1/muPrev); + RealScalar a = (fCur - fPrev) / (Literal(1)/muCur - Literal(1)/muPrev); RealScalar b = fCur - a / muCur; // And find mu such that f(mu)==0: RealScalar muZero = -a/b; RealScalar fZero = secularEq(muZero, col0, diag, perm, diagShifted, shift); + +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + assert((std::isfinite)(fZero)); +#endif muPrev = muCur; fPrev = fCur; muCur = muZero; fCur = fZero; - - if (shift == left && (muCur < 0 || muCur > right - left)) useBisection = true; - if (shift == right && (muCur < -(right - left) || muCur > 0)) useBisection = true; + if (shift == left && (muCur < Literal(0) || muCur > right - left)) useBisection = true; + if (shift == right && (muCur < -(right - left) || muCur > Literal(0))) useBisection = true; if (abs(fCur)>abs(fPrev)) useBisection = true; } @@ -798,34 +844,59 @@ RealScalar leftShifted, rightShifted; if (shift == left) { - leftShifted = RealScalar(1)/NumTraits<RealScalar>::highest(); + // to avoid overflow, we must have mu > max(real_min, |z(k)|/sqrt(real_max)), + // the factor 2 is to be more conservative + leftShifted = numext::maxi<RealScalar>( (std::numeric_limits<RealScalar>::min)(), Literal(2) * abs(col0(k)) / sqrt((std::numeric_limits<RealScalar>::max)()) ); + + // check that we did it right: + eigen_internal_assert( (numext::isfinite)( (col0(k)/leftShifted)*(col0(k)/(diag(k)+shift+leftShifted)) ) ); // I don't understand why the case k==0 would be special there: - // if (k == 0) rightShifted = right - left; else - rightShifted = (k==actual_n-1) ? right : ((right - left) * 0.6); // theoretically we can take 0.5, but let's be safe + // if (k == 0) rightShifted = right - left; else + rightShifted = (k==actual_n-1) ? right : ((right - left) * RealScalar(0.51)); // theoretically we can take 0.5, but let's be safe } else { - leftShifted = -(right - left) * 0.6; - rightShifted = -RealScalar(1)/NumTraits<RealScalar>::highest(); + leftShifted = -(right - left) * RealScalar(0.51); + if(k+1<n) + rightShifted = -numext::maxi<RealScalar>( (std::numeric_limits<RealScalar>::min)(), abs(col0(k+1)) / sqrt((std::numeric_limits<RealScalar>::max)()) ); + else + rightShifted = -(std::numeric_limits<RealScalar>::min)(); } - + RealScalar fLeft = secularEq(leftShifted, col0, diag, perm, diagShifted, shift); -#if defined EIGEN_INTERNAL_DEBUGGING || defined EIGEN_BDCSVD_DEBUG_VERBOSE +#if defined EIGEN_INTERNAL_DEBUGGING || defined EIGEN_BDCSVD_SANITY_CHECKS RealScalar fRight = secularEq(rightShifted, col0, diag, perm, diagShifted, shift); #endif +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + if(!(std::isfinite)(fLeft)) + std::cout << "f(" << leftShifted << ") =" << fLeft << " ; " << left << " " << shift << " " << right << "\n"; + assert((std::isfinite)(fLeft)); + + if(!(std::isfinite)(fRight)) + std::cout << "f(" << rightShifted << ") =" << fRight << " ; " << left << " " << shift << " " << right << "\n"; +// assert((std::isfinite)(fRight)); +#endif + #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE if(!(fLeft * fRight<0)) - std::cout << k << " : " << fLeft << " * " << fRight << " == " << fLeft * fRight << " ; " << left << " - " << right << " -> " << leftShifted << " " << rightShifted << " shift=" << shift << "\n"; -#endif - eigen_internal_assert(fLeft * fRight < 0); - - while (rightShifted - leftShifted > 2 * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(abs(leftShifted), abs(rightShifted))) { - RealScalar midShifted = (leftShifted + rightShifted) / 2; + std::cout << "f(leftShifted) using leftShifted=" << leftShifted << " ; diagShifted(1:10):" << diagShifted.head(10).transpose() << "\n ; " + << "left==shift=" << bool(left==shift) << " ; left-shift = " << (left-shift) << "\n"; + std::cout << "k=" << k << ", " << fLeft << " * " << fRight << " == " << fLeft * fRight << " ; " + << "[" << left << " .. " << right << "] -> [" << leftShifted << " " << rightShifted << "], shift=" << shift << " , f(right)=" << secularEq(0, col0, diag, perm, diagShifted, shift) << " == " << secularEq(right, col0, diag, perm, diag, 0) << "\n"; + } +#endif + eigen_internal_assert(fLeft * fRight < Literal(0)); + + while (rightShifted - leftShifted > Literal(2) * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(abs(leftShifted), abs(rightShifted))) + { + RealScalar midShifted = (leftShifted + rightShifted) / Literal(2); fMid = secularEq(midShifted, col0, diag, perm, diagShifted, shift); - if (fLeft * fMid < 0) + eigen_internal_assert((numext::isfinite)(fMid)); + + if (fLeft * fMid < Literal(0)) { rightShifted = midShifted; } @@ -836,13 +907,22 @@ } } - muCur = (leftShifted + rightShifted) / 2; + muCur = (leftShifted + rightShifted) / Literal(2); } singVals[k] = shift + muCur; shifts[k] = shift; mus[k] = muCur; +#ifdef EIGEN_BDCSVD_DEBUG_VERBOSE + if(k+1<n) + std::cout << "found " << singVals[k] << " == " << shift << " + " << muCur << " from " << diag(k) << " .. " << diag(k+1) << "\n"; +#endif +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + assert(k==0 || singVals[k]>=singVals[k-1]); + assert(singVals[k]>=diag(k)); +#endif + // perturb singular value slightly if it equals diagonal entry to avoid division by zero later // (deflation is supposed to avoid this from happening) // - this does no seem to be necessary anymore - @@ -866,25 +946,53 @@ zhat.setZero(); return; } - Index last = perm(m-1); + Index lastIdx = perm(m-1); // The offset permits to skip deflated entries while computing zhat for (Index k = 0; k < n; ++k) { - if (col0(k) == 0) // deflated - zhat(k) = 0; + if (col0(k) == Literal(0)) // deflated + zhat(k) = Literal(0); else { // see equation (3.6) RealScalar dk = diag(k); - RealScalar prod = (singVals(last) + dk) * (mus(last) + (shifts(last) - dk)); + RealScalar prod = (singVals(lastIdx) + dk) * (mus(lastIdx) + (shifts(lastIdx) - dk)); +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + if(prod<0) { + std::cout << "k = " << k << " ; z(k)=" << col0(k) << ", diag(k)=" << dk << "\n"; + std::cout << "prod = " << "(" << singVals(lastIdx) << " + " << dk << ") * (" << mus(lastIdx) << " + (" << shifts(lastIdx) << " - " << dk << "))" << "\n"; + std::cout << " = " << singVals(lastIdx) + dk << " * " << mus(lastIdx) + (shifts(lastIdx) - dk) << "\n"; + } + assert(prod>=0); +#endif for(Index l = 0; l<m; ++l) { Index i = perm(l); if(i!=k) { +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + if(i>=k && (l==0 || l-1>=m)) + { + std::cout << "Error in perturbCol0\n"; + std::cout << " " << k << "/" << n << " " << l << "/" << m << " " << i << "/" << n << " ; " << col0(k) << " " << diag(k) << " " << "\n"; + std::cout << " " <<diag(i) << "\n"; + Index j = (i<k /*|| l==0*/) ? i : perm(l-1); + std::cout << " " << "j=" << j << "\n"; + } +#endif Index j = i<k ? i : perm(l-1); +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + if(!(dk!=Literal(0) || diag(i)!=Literal(0))) + { + std::cout << "k=" << k << ", i=" << i << ", l=" << l << ", perm.size()=" << perm.size() << "\n"; + } + assert(dk!=Literal(0) || diag(i)!=Literal(0)); +#endif prod *= ((singVals(j)+dk) / ((diag(i)+dk))) * ((mus(j)+(shifts(j)-dk)) / ((diag(i)-dk))); +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + assert(prod>=0); +#endif #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE if(i!=k && std::abs(((singVals(j)+dk)*(mus(j)+(shifts(j)-dk)))/((diag(i)+dk)*(diag(i)-dk)) - 1) > 0.9 ) std::cout << " " << ((singVals(j)+dk)*(mus(j)+(shifts(j)-dk)))/((diag(i)+dk)*(diag(i)-dk)) << " == (" << (singVals(j)+dk) << " * " << (mus(j)+(shifts(j)-dk)) @@ -893,10 +1001,13 @@ } } #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE - std::cout << "zhat(" << k << ") = sqrt( " << prod << ") ; " << (singVals(last) + dk) << " * " << mus(last) + shifts(last) << " - " << dk << "\n"; + std::cout << "zhat(" << k << ") = sqrt( " << prod << ") ; " << (singVals(lastIdx) + dk) << " * " << mus(lastIdx) + shifts(lastIdx) << " - " << dk << "\n"; #endif RealScalar tmp = sqrt(prod); - zhat(k) = col0(k) > 0 ? tmp : -tmp; +#ifdef EIGEN_BDCSVD_SANITY_CHECKS + assert((std::isfinite)(tmp)); +#endif + zhat(k) = col0(k) > Literal(0) ? RealScalar(tmp) : RealScalar(-tmp); } } } @@ -912,7 +1023,7 @@ for (Index k = 0; k < n; ++k) { - if (zhat(k) == 0) + if (zhat(k) == Literal(0)) { U.col(k) = VectorType::Unit(n+1, k); if (m_compV) V.col(k) = VectorType::Unit(n, k); @@ -925,7 +1036,7 @@ Index i = perm(l); U(i,k) = zhat(i)/(((diag(i) - shifts(k)) - mus(k)) )/( (diag(i) + singVals[k])); } - U(n,k) = 0; + U(n,k) = Literal(0); U.col(k).normalize(); if (m_compV) @@ -936,7 +1047,7 @@ Index i = perm(l); V(i,k) = diag(i) * zhat(i) / (((diag(i) - shifts(k)) - mus(k)) )/( (diag(i) + singVals[k])); } - V(0,k) = -1; + V(0,k) = Literal(-1); V.col(k).normalize(); } } @@ -949,7 +1060,7 @@ // i >= 1, di almost null and zi non null. // We use a rotation to zero out zi applied to the left of M template <typename MatrixType> -void BDCSVD<MatrixType>::deflation43(Index firstCol, Index shift, Index i, Index size) +void BDCSVD<MatrixType>::deflation43(Eigen::Index firstCol, Eigen::Index shift, Eigen::Index i, Eigen::Index size) { using std::abs; using std::sqrt; @@ -957,15 +1068,15 @@ Index start = firstCol + shift; RealScalar c = m_computed(start, start); RealScalar s = m_computed(start+i, start); - RealScalar r = sqrt(numext::abs2(c) + numext::abs2(s)); - if (r == 0) + RealScalar r = numext::hypot(c,s); + if (r == Literal(0)) { - m_computed(start+i, start+i) = 0; + m_computed(start+i, start+i) = Literal(0); return; } m_computed(start,start) = r; - m_computed(start+i, start) = 0; - m_computed(start+i, start+i) = 0; + m_computed(start+i, start) = Literal(0); + m_computed(start+i, start+i) = Literal(0); JacobiRotation<RealScalar> J(c/r,-s/r); if (m_compU) m_naiveU.middleRows(firstCol, size+1).applyOnTheRight(firstCol, firstCol+i, J); @@ -978,7 +1089,7 @@ // We apply two rotations to have zj = 0; // TODO deflation44 is still broken and not properly tested template <typename MatrixType> -void BDCSVD<MatrixType>::deflation44(Index firstColu , Index firstColm, Index firstRowW, Index firstColW, Index i, Index j, Index size) +void BDCSVD<MatrixType>::deflation44(Eigen::Index firstColu , Eigen::Index firstColm, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index i, Eigen::Index j, Eigen::Index size) { using std::abs; using std::sqrt; @@ -998,16 +1109,16 @@ << m_computed(firstColm + i+1, firstColm+i+1) << " " << m_computed(firstColm + i+2, firstColm+i+2) << "\n"; #endif - if (r==0) + if (r==Literal(0)) { m_computed(firstColm + i, firstColm + i) = m_computed(firstColm + j, firstColm + j); return; } c/=r; s/=r; - m_computed(firstColm + i, firstColm) = r; + m_computed(firstColm + i, firstColm) = r; m_computed(firstColm + j, firstColm + j) = m_computed(firstColm + i, firstColm + i); - m_computed(firstColm + j, firstColm) = 0; + m_computed(firstColm + j, firstColm) = Literal(0); JacobiRotation<RealScalar> J(c,-s); if (m_compU) m_naiveU.middleRows(firstColu, size+1).applyOnTheRight(firstColu + i, firstColu + j, J); @@ -1018,7 +1129,7 @@ // acts on block from (firstCol+shift, firstCol+shift) to (lastCol+shift, lastCol+shift) [inclusive] template <typename MatrixType> -void BDCSVD<MatrixType>::deflation(Index firstCol, Index lastCol, Index k, Index firstRowW, Index firstColW, Index shift) +void BDCSVD<MatrixType>::deflation(Eigen::Index firstCol, Eigen::Index lastCol, Eigen::Index k, Eigen::Index firstRowW, Eigen::Index firstColW, Eigen::Index shift) { using std::sqrt; using std::abs; @@ -1028,9 +1139,10 @@ Diagonal<MatrixXr> fulldiag(m_computed); VectorBlock<Diagonal<MatrixXr>,Dynamic> diag(fulldiag, firstCol+shift, length); + const RealScalar considerZero = (std::numeric_limits<RealScalar>::min)(); RealScalar maxDiag = diag.tail((std::max)(Index(1),length-1)).cwiseAbs().maxCoeff(); - RealScalar epsilon_strict = NumTraits<RealScalar>::epsilon() * maxDiag; - RealScalar epsilon_coarse = 8 * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(col0.cwiseAbs().maxCoeff(), maxDiag); + RealScalar epsilon_strict = numext::maxi<RealScalar>(considerZero,NumTraits<RealScalar>::epsilon() * maxDiag); + RealScalar epsilon_coarse = Literal(8) * NumTraits<RealScalar>::epsilon() * numext::maxi<RealScalar>(col0.cwiseAbs().maxCoeff(), maxDiag); #ifdef EIGEN_BDCSVD_SANITY_CHECKS assert(m_naiveU.allFinite()); @@ -1058,7 +1170,7 @@ #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "deflation 4.2, set z(" << i << ") to zero because " << abs(col0(i)) << " < " << epsilon_strict << " (diag(" << i << ")=" << diag(i) << ")\n"; #endif - col0(i) = 0; + col0(i) = Literal(0); } //condition 4.3 @@ -1078,11 +1190,12 @@ #endif #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE std::cout << "to be sorted: " << diag.transpose() << "\n\n"; + std::cout << " : " << col0.transpose() << "\n\n"; #endif { // Check for total deflation // If we have a total deflation, then we have to consider col0(0)==diag(0) as a singular value during sorting - bool total_deflation = (col0.tail(length-1).array()==RealScalar(0)).all(); + bool total_deflation = (col0.tail(length-1).array()<considerZero).all(); // Sort the diagonal entries, since diag(1:k-1) and diag(k:length) are already sorted, let's do a sorted merge. // First, compute the respective permutation. @@ -1093,7 +1206,7 @@ // Move deflated diagonal entries at the end. for(Index i=1; i<length; ++i) - if(diag(i)==0) + if(abs(diag(i))<considerZero) permutation[p++] = i; Index i=1, j=k+1; @@ -1112,7 +1225,7 @@ for(Index i=1; i<length; ++i) { Index pi = permutation[i]; - if(diag(pi)==0 || diag(0)<diag(pi)) + if(abs(diag(pi))<considerZero || diag(0)<diag(pi)) permutation[i-1] = permutation[i]; else { @@ -1163,12 +1276,12 @@ //condition 4.4 { Index i = length-1; - while(i>0 && (diag(i)==0 || col0(i)==0)) --i; + while(i>0 && (abs(diag(i))<considerZero || abs(col0(i))<considerZero)) --i; for(; i>1;--i) if( (diag(i) - diag(i-1)) < NumTraits<RealScalar>::epsilon()*maxDiag ) { #ifdef EIGEN_BDCSVD_DEBUG_VERBOSE - std::cout << "deflation 4.4 with i = " << i << " because " << (diag(i) - diag(i-1)) << " < " << NumTraits<RealScalar>::epsilon()*diag(i) << "\n"; + std::cout << "deflation 4.4 with i = " << i << " because " << diag(i) << " - " << diag(i-1) << " == " << (diag(i) - diag(i-1)) << " < " << NumTraits<RealScalar>::epsilon()*/*diag(i)*/maxDiag << "\n"; #endif eigen_internal_assert(abs(diag(i) - diag(i-1))<epsilon_coarse && " diagonal entries are not properly sorted"); deflation44(firstCol, firstCol + shift, firstRowW, firstColW, i-1, i, length); @@ -1177,7 +1290,7 @@ #ifdef EIGEN_BDCSVD_SANITY_CHECKS for(Index j=2;j<length;++j) - assert(diag(j-1)<=diag(j) || diag(j)==0); + assert(diag(j-1)<=diag(j) || abs(diag(j))<considerZero); #endif #ifdef EIGEN_BDCSVD_SANITY_CHECKS @@ -1187,7 +1300,7 @@ #endif }//end deflation -#ifndef __CUDACC__ +#if !defined(EIGEN_GPUCC) /** \svd_module * * \return the singular value decomposition of \c *this computed by Divide & Conquer algorithm
diff --git a/Eigen/src/SVD/CMakeLists.txt b/Eigen/src/SVD/CMakeLists.txt deleted file mode 100644 index 55efc44..0000000 --- a/Eigen/src/SVD/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SVD_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SVD_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SVD COMPONENT Devel - )
diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index 1940c82..2b68911 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h
@@ -112,9 +112,11 @@ ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Options = MatrixType::Options + TrOptions = RowsAtCompileTime==1 ? (MatrixType::Options & ~(RowMajor)) + : ColsAtCompileTime==1 ? (MatrixType::Options | RowMajor) + : MatrixType::Options }; - typedef Matrix<Scalar, ColsAtCompileTime, RowsAtCompileTime, Options, MaxColsAtCompileTime, MaxRowsAtCompileTime> + typedef Matrix<Scalar, ColsAtCompileTime, RowsAtCompileTime, TrOptions, MaxColsAtCompileTime, MaxRowsAtCompileTime> TransposeTypeWithSameStorageOrder; void allocate(const JacobiSVD<MatrixType, FullPivHouseholderQRPreconditioner>& svd) @@ -200,10 +202,12 @@ ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, - Options = MatrixType::Options + TrOptions = RowsAtCompileTime==1 ? (MatrixType::Options & ~(RowMajor)) + : ColsAtCompileTime==1 ? (MatrixType::Options | RowMajor) + : MatrixType::Options }; - typedef Matrix<Scalar, ColsAtCompileTime, RowsAtCompileTime, Options, MaxColsAtCompileTime, MaxRowsAtCompileTime> + typedef Matrix<Scalar, ColsAtCompileTime, RowsAtCompileTime, TrOptions, MaxColsAtCompileTime, MaxRowsAtCompileTime> TransposeTypeWithSameStorageOrder; void allocate(const JacobiSVD<MatrixType, ColPivHouseholderQRPreconditioner>& svd) @@ -412,47 +416,16 @@ } // update largest diagonal entry - maxDiagEntry = numext::maxi(maxDiagEntry,numext::maxi(abs(work_matrix.coeff(p,p)), abs(work_matrix.coeff(q,q)))); + maxDiagEntry = numext::maxi<RealScalar>(maxDiagEntry,numext::maxi<RealScalar>(abs(work_matrix.coeff(p,p)), abs(work_matrix.coeff(q,q)))); // and check whether the 2x2 block is already diagonal RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry); return abs(work_matrix.coeff(p,q))>threshold || abs(work_matrix.coeff(q,p)) > threshold; } }; -template<typename MatrixType, typename RealScalar, typename Index> -void real_2x2_jacobi_svd(const MatrixType& matrix, Index p, Index q, - JacobiRotation<RealScalar> *j_left, - JacobiRotation<RealScalar> *j_right) -{ - using std::sqrt; - using std::abs; - Matrix<RealScalar,2,2> m; - m << numext::real(matrix.coeff(p,p)), numext::real(matrix.coeff(p,q)), - numext::real(matrix.coeff(q,p)), numext::real(matrix.coeff(q,q)); - JacobiRotation<RealScalar> rot1; - RealScalar t = m.coeff(0,0) + m.coeff(1,1); - RealScalar d = m.coeff(1,0) - m.coeff(0,1); - if(d == RealScalar(0)) - { - rot1.s() = RealScalar(0); - rot1.c() = RealScalar(1); - } - else - { - // If d!=0, then t/d cannot overflow because the magnitude of the - // entries forming d are not too small compared to the ones forming t. - RealScalar u = t / d; - RealScalar tmp = sqrt(RealScalar(1) + numext::abs2(u)); - rot1.s() = RealScalar(1) / tmp; - rot1.c() = u / tmp; - } - m.applyOnTheLeft(0,1,rot1); - j_right->makeJacobi(m,0,1); - *j_left = rot1 * j_right->transpose(); -} - template<typename _MatrixType, int QRPreconditioner> struct traits<JacobiSVD<_MatrixType,QRPreconditioner> > + : traits<_MatrixType> { typedef _MatrixType MatrixType; }; @@ -638,7 +611,7 @@ }; template<typename MatrixType, int QRPreconditioner> -void JacobiSVD<MatrixType, QRPreconditioner>::allocate(Index rows, Index cols, unsigned int computationOptions) +void JacobiSVD<MatrixType, QRPreconditioner>::allocate(Eigen::Index rows, Eigen::Index cols, unsigned int computationOptions) { eigen_assert(rows >= 0 && cols >= 0); @@ -697,10 +670,8 @@ // only worsening the precision of U and V as we accumulate more rotations const RealScalar precision = RealScalar(2) * NumTraits<Scalar>::epsilon(); - // limit for very small denormal numbers to be considered zero in order to avoid infinite loops (see bug 286) - // FIXME What about considerering any denormal numbers as zero, using: - // const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); - const RealScalar considerAsZero = RealScalar(2) * std::numeric_limits<RealScalar>::denorm_min(); + // limit for denormal numbers to be considered zero in order to avoid infinite loops (see bug 286) + const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); // Scaling factor to reduce over/under-flows RealScalar scale = matrix.cwiseAbs().maxCoeff(); @@ -745,7 +716,7 @@ { finished = false; // perform SVD decomposition of 2x2 sub-matrix corresponding to indices p,q to make it diagonal - // the complex to real operation returns true is the updated 2x2 block is not already diagonal + // the complex to real operation returns true if the updated 2x2 block is not already diagonal if(internal::svd_precondition_2x2_block_to_be_real<MatrixType, QRPreconditioner>::run(m_workMatrix, *this, p, q, maxDiagEntry)) { JacobiRotation<RealScalar> j_left, j_right; @@ -759,7 +730,7 @@ if(computeV()) m_matrixV.applyOnTheRight(p,q,j_right); // keep track of the largest diagonal coefficient - maxDiagEntry = numext::maxi(maxDiagEntry,numext::maxi(abs(m_workMatrix.coeff(p,p)), abs(m_workMatrix.coeff(q,q)))); + maxDiagEntry = numext::maxi<RealScalar>(maxDiagEntry,numext::maxi<RealScalar>(abs(m_workMatrix.coeff(p,p)), abs(m_workMatrix.coeff(q,q)))); } } } @@ -770,9 +741,22 @@ for(Index i = 0; i < m_diagSize; ++i) { - RealScalar a = abs(m_workMatrix.coeff(i,i)); - m_singularValues.coeffRef(i) = a; - if(computeU() && (a!=RealScalar(0))) m_matrixU.col(i) *= m_workMatrix.coeff(i,i)/a; + // For a complex matrix, some diagonal coefficients might note have been + // treated by svd_precondition_2x2_block_to_be_real, and the imaginary part + // of some diagonal entry might not be null. + if(NumTraits<Scalar>::IsComplex && abs(numext::imag(m_workMatrix.coeff(i,i)))>considerAsZero) + { + RealScalar a = abs(m_workMatrix.coeff(i,i)); + m_singularValues.coeffRef(i) = abs(a); + if(computeU()) m_matrixU.col(i) *= m_workMatrix.coeff(i,i)/a; + } + else + { + // m_workMatrix.coeff(i,i) is already real, no difficulty: + RealScalar a = numext::real(m_workMatrix.coeff(i,i)); + m_singularValues.coeffRef(i) = abs(a); + if(computeU() && (a<RealScalar(0))) m_matrixU.col(i) = -m_matrixU.col(i); + } } m_singularValues *= scale; @@ -802,7 +786,6 @@ return *this; } -#ifndef __CUDACC__ /** \svd_module * * \return the singular value decomposition of \c *this computed by two-sided @@ -816,7 +799,6 @@ { return JacobiSVD<PlainObject>(*this, computationOptions); } -#endif // __CUDACC__ } // end namespace Eigen
diff --git a/Eigen/src/SVD/JacobiSVD_MKL.h b/Eigen/src/SVD/JacobiSVD_LAPACKE.h similarity index 60% rename from Eigen/src/SVD/JacobiSVD_MKL.h rename to Eigen/src/SVD/JacobiSVD_LAPACKE.h index 14e461c..ff0516f 100644 --- a/Eigen/src/SVD/JacobiSVD_MKL.h +++ b/Eigen/src/SVD/JacobiSVD_LAPACKE.h
@@ -25,21 +25,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** - * Content : Eigen bindings to Intel(R) MKL + * Content : Eigen bindings to LAPACKe * Singular Value Decomposition - SVD. ******************************************************************************** */ -#ifndef EIGEN_JACOBISVD_MKL_H -#define EIGEN_JACOBISVD_MKL_H - -#include "Eigen/src/Core/util/MKL_support.h" +#ifndef EIGEN_JACOBISVD_LAPACKE_H +#define EIGEN_JACOBISVD_LAPACKE_H namespace Eigen { -/** \internal Specialization for the data types supported by MKL */ +/** \internal Specialization for the data types supported by LAPACKe */ -#define EIGEN_MKL_SVD(EIGTYPE, MKLTYPE, MKLRTYPE, MKLPREFIX, EIGCOLROW, MKLCOLROW) \ +#define EIGEN_LAPACKE_SVD(EIGTYPE, LAPACKE_TYPE, LAPACKE_RTYPE, LAPACKE_PREFIX, EIGCOLROW, LAPACKE_COLROW) \ template<> inline \ JacobiSVD<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynamic>, ColPivHouseholderQRPreconditioner>& \ JacobiSVD<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynamic>, ColPivHouseholderQRPreconditioner>::compute(const Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynamic>& matrix, unsigned int computationOptions) \ @@ -52,41 +50,42 @@ /*const RealScalar precision = RealScalar(2) * NumTraits<Scalar>::epsilon();*/ \ m_nonzeroSingularValues = m_diagSize; \ \ - lapack_int lda = matrix.outerStride(), ldu, ldvt; \ - lapack_int matrix_order = MKLCOLROW; \ + lapack_int lda = internal::convert_index<lapack_int>(matrix.outerStride()), ldu, ldvt; \ + lapack_int matrix_order = LAPACKE_COLROW; \ char jobu, jobvt; \ - MKLTYPE *u, *vt, dummy; \ + LAPACKE_TYPE *u, *vt, dummy; \ jobu = (m_computeFullU) ? 'A' : (m_computeThinU) ? 'S' : 'N'; \ jobvt = (m_computeFullV) ? 'A' : (m_computeThinV) ? 'S' : 'N'; \ if (computeU()) { \ - ldu = m_matrixU.outerStride(); \ - u = (MKLTYPE*)m_matrixU.data(); \ + ldu = internal::convert_index<lapack_int>(m_matrixU.outerStride()); \ + u = (LAPACKE_TYPE*)m_matrixU.data(); \ } else { ldu=1; u=&dummy; }\ MatrixType localV; \ - ldvt = (m_computeFullV) ? m_cols : (m_computeThinV) ? m_diagSize : 1; \ + lapack_int vt_rows = (m_computeFullV) ? internal::convert_index<lapack_int>(m_cols) : (m_computeThinV) ? internal::convert_index<lapack_int>(m_diagSize) : 1; \ if (computeV()) { \ - localV.resize(ldvt, m_cols); \ - vt = (MKLTYPE*)localV.data(); \ + localV.resize(vt_rows, m_cols); \ + ldvt = internal::convert_index<lapack_int>(localV.outerStride()); \ + vt = (LAPACKE_TYPE*)localV.data(); \ } else { ldvt=1; vt=&dummy; }\ - Matrix<MKLRTYPE, Dynamic, Dynamic> superb; superb.resize(m_diagSize, 1); \ + Matrix<LAPACKE_RTYPE, Dynamic, Dynamic> superb; superb.resize(m_diagSize, 1); \ MatrixType m_temp; m_temp = matrix; \ - LAPACKE_##MKLPREFIX##gesvd( matrix_order, jobu, jobvt, m_rows, m_cols, (MKLTYPE*)m_temp.data(), lda, (MKLRTYPE*)m_singularValues.data(), u, ldu, vt, ldvt, superb.data()); \ + LAPACKE_##LAPACKE_PREFIX##gesvd( matrix_order, jobu, jobvt, internal::convert_index<lapack_int>(m_rows), internal::convert_index<lapack_int>(m_cols), (LAPACKE_TYPE*)m_temp.data(), lda, (LAPACKE_RTYPE*)m_singularValues.data(), u, ldu, vt, ldvt, superb.data()); \ if (computeV()) m_matrixV = localV.adjoint(); \ /* for(int i=0;i<m_diagSize;i++) if (m_singularValues.coeffRef(i) < precision) { m_nonzeroSingularValues--; m_singularValues.coeffRef(i)=RealScalar(0);}*/ \ m_isInitialized = true; \ return *this; \ } -EIGEN_MKL_SVD(double, double, double, d, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SVD(float, float, float , s, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SVD(dcomplex, MKL_Complex16, double, z, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SVD(scomplex, MKL_Complex8, float , c, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SVD(double, double, double, d, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SVD(float, float, float , s, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SVD(dcomplex, lapack_complex_double, double, z, ColMajor, LAPACK_COL_MAJOR) +EIGEN_LAPACKE_SVD(scomplex, lapack_complex_float, float , c, ColMajor, LAPACK_COL_MAJOR) -EIGEN_MKL_SVD(double, double, double, d, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_SVD(float, float, float , s, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_SVD(dcomplex, MKL_Complex16, double, z, RowMajor, LAPACK_ROW_MAJOR) -EIGEN_MKL_SVD(scomplex, MKL_Complex8, float , c, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SVD(double, double, double, d, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SVD(float, float, float , s, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SVD(dcomplex, lapack_complex_double, double, z, RowMajor, LAPACK_ROW_MAJOR) +EIGEN_LAPACKE_SVD(scomplex, lapack_complex_float, float , c, RowMajor, LAPACK_ROW_MAJOR) } // end namespace Eigen -#endif // EIGEN_JACOBISVD_MKL_H +#endif // EIGEN_JACOBISVD_LAPACKE_H
diff --git a/Eigen/src/SVD/SVDBase.h b/Eigen/src/SVD/SVDBase.h index e2d77a7..68df489 100644 --- a/Eigen/src/SVD/SVDBase.h +++ b/Eigen/src/SVD/SVDBase.h
@@ -17,6 +17,18 @@ #define EIGEN_SVDBASE_H namespace Eigen { + +namespace internal { +template<typename Derived> struct traits<SVDBase<Derived> > + : traits<Derived> +{ + typedef MatrixXpr XprKind; + typedef SolverStorage StorageKind; + typedef int StorageIndex; + enum { Flags = 0 }; +}; +} + /** \ingroup SVD_Module * * @@ -44,15 +56,18 @@ * terminate in finite (and reasonable) time. * \sa class BDCSVD, class JacobiSVD */ -template<typename Derived> -class SVDBase +template<typename Derived> class SVDBase + : public SolverBase<SVDBase<Derived> > { +public: + + template<typename Derived_> + friend struct internal::solve_assertion; -public: typedef typename internal::traits<Derived>::MatrixType MatrixType; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; - typedef typename MatrixType::StorageIndex StorageIndex; + typedef typename Eigen::internal::traits<SVDBase>::StorageIndex StorageIndex; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, @@ -130,10 +145,9 @@ inline Index rank() const { using std::abs; - using std::max; eigen_assert(m_isInitialized && "JacobiSVD is not initialized."); if(m_singularValues.size()==0) return 0; - RealScalar premultiplied_threshold = (max)(m_singularValues.coeff(0) * threshold(), (std::numeric_limits<RealScalar>::min)()); + RealScalar premultiplied_threshold = numext::maxi<RealScalar>(m_singularValues.coeff(0) * threshold(), (std::numeric_limits<RealScalar>::min)()); Index i = m_nonzeroSingularValues-1; while(i>=0 && m_singularValues.coeff(i) < premultiplied_threshold) --i; return i+1; @@ -181,8 +195,10 @@ RealScalar threshold() const { eigen_assert(m_isInitialized || m_usePrescribedThreshold); + // this temporary is needed to workaround a MSVC issue + Index diagSize = (std::max<Index>)(1,m_diagSize); return m_usePrescribedThreshold ? m_prescribedThreshold - : (std::max<Index>)(1,m_diagSize)*NumTraits<Scalar>::epsilon(); + : diagSize*NumTraits<Scalar>::epsilon(); } /** \returns true if \a U (full or thin) is asked for in this SVD decomposition */ @@ -193,6 +209,7 @@ inline Index rows() const { return m_rows; } inline Index cols() const { return m_cols; } + #ifdef EIGEN_PARSED_BY_DOXYGEN /** \returns a (least squares) solution of \f$ A x = b \f$ using the current SVD decomposition of A. * * \param b the right-hand-side of the equation to solve. @@ -204,17 +221,15 @@ */ template<typename Rhs> inline const Solve<Derived, Rhs> - solve(const MatrixBase<Rhs>& b) const - { - eigen_assert(m_isInitialized && "SVD is not initialized."); - eigen_assert(computeU() && computeV() && "SVD::solve() requires both unitaries U and V to be computed (thin unitaries suffice)."); - return Solve<Derived, Rhs>(derived(), b.derived()); - } - + solve(const MatrixBase<Rhs>& b) const; + #endif + #ifndef EIGEN_PARSED_BY_DOXYGEN template<typename RhsType, typename DstType> - EIGEN_DEVICE_FUNC void _solve_impl(const RhsType &rhs, DstType &dst) const; + + template<bool Conjugate, typename RhsType, typename DstType> + void _solve_impl_transposed(const RhsType &rhs, DstType &dst) const; #endif protected: @@ -223,6 +238,14 @@ { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar); } + + template<bool Transpose_, typename Rhs> + void _check_solve_assertion(const Rhs& b) const { + EIGEN_ONLY_USED_FOR_DEBUG(b); + eigen_assert(m_isInitialized && "SVD is not initialized."); + eigen_assert(computeU() && computeV() && "SVDBase::solve(): Both unitaries U and V are required to be computed (thin unitaries suffice)."); + eigen_assert((Transpose_?cols():rows())==b.rows() && "SVDBase::solve(): invalid number of rows of the right hand side matrix b"); + } // return true if already allocated bool allocate(Index rows, Index cols, unsigned int computationOptions) ; @@ -245,6 +268,10 @@ : m_isInitialized(false), m_isAllocated(false), m_usePrescribedThreshold(false), + m_computeFullU(false), + m_computeThinU(false), + m_computeFullV(false), + m_computeThinV(false), m_computationOptions(0), m_rows(-1), m_cols(-1), m_diagSize(0) { @@ -259,17 +286,30 @@ template<typename RhsType, typename DstType> void SVDBase<Derived>::_solve_impl(const RhsType &rhs, DstType &dst) const { - eigen_assert(rhs.rows() == rows()); - // A = U S V^* // So A^{-1} = V S^{-1} U^* - Matrix<Scalar, Dynamic, RhsType::ColsAtCompileTime, 0, MatrixType::MaxRowsAtCompileTime, RhsType::MaxColsAtCompileTime> tmp; + Matrix<typename RhsType::Scalar, Dynamic, RhsType::ColsAtCompileTime, 0, MatrixType::MaxRowsAtCompileTime, RhsType::MaxColsAtCompileTime> tmp; Index l_rank = rank(); tmp.noalias() = m_matrixU.leftCols(l_rank).adjoint() * rhs; tmp = m_singularValues.head(l_rank).asDiagonal().inverse() * tmp; dst = m_matrixV.leftCols(l_rank) * tmp; } + +template<typename Derived> +template<bool Conjugate, typename RhsType, typename DstType> +void SVDBase<Derived>::_solve_impl_transposed(const RhsType &rhs, DstType &dst) const +{ + // A = U S V^* + // So A^{-*} = U S^{-1} V^* + // And A^{-T} = U_conj S^{-1} V^T + Matrix<typename RhsType::Scalar, Dynamic, RhsType::ColsAtCompileTime, 0, MatrixType::MaxRowsAtCompileTime, RhsType::MaxColsAtCompileTime> tmp; + Index l_rank = rank(); + + tmp.noalias() = m_matrixV.leftCols(l_rank).transpose().template conjugateIf<Conjugate>() * rhs; + tmp = m_singularValues.head(l_rank).asDiagonal().inverse() * tmp; + dst = m_matrixU.template conjugateIf<!Conjugate>().leftCols(l_rank) * tmp; +} #endif template<typename MatrixType>
diff --git a/Eigen/src/SVD/UpperBidiagonalization.h b/Eigen/src/SVD/UpperBidiagonalization.h index 0b14608..997defc 100644 --- a/Eigen/src/SVD/UpperBidiagonalization.h +++ b/Eigen/src/SVD/UpperBidiagonalization.h
@@ -127,7 +127,7 @@ .makeHouseholderInPlace(mat.coeffRef(k,k+1), upper_diagonal[k]); // apply householder transform to remaining part of mat on the left mat.bottomRightCorner(remainingRows-1, remainingCols) - .applyHouseholderOnTheRight(mat.row(k).tail(remainingCols-1).transpose(), mat.coeff(k,k+1), tempData); + .applyHouseholderOnTheRight(mat.row(k).tail(remainingCols-1).adjoint(), mat.coeff(k,k+1), tempData); } } @@ -159,6 +159,8 @@ traits<MatrixType>::Flags & RowMajorBit> > Y) { typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; + typedef typename NumTraits<RealScalar>::Literal Literal; enum { StorageOrder = traits<MatrixType>::Flags & RowMajorBit }; typedef InnerStride<int(StorageOrder) == int(ColMajor) ? 1 : Dynamic> ColInnerStride; typedef InnerStride<int(StorageOrder) == int(ColMajor) ? Dynamic : 1> RowInnerStride; @@ -200,7 +202,7 @@ { SubColumnType y_k( Y.col(k).tail(remainingCols) ); - // let's use the begining of column k of Y as a temporary vector + // let's use the beginning of column k of Y as a temporary vector SubColumnType tmp( Y.col(k).head(k) ); y_k.noalias() = A.block(k,k+1, remainingRows,remainingCols).adjoint() * v_k; // bottleneck tmp.noalias() = V_k1.adjoint() * v_k; @@ -229,7 +231,7 @@ { SubColumnType x_k ( X.col(k).tail(remainingRows-1) ); - // let's use the begining of column k of X as a temporary vectors + // let's use the beginning of column k of X as a temporary vectors // note that tmp0 and tmp1 overlaps SubColumnType tmp0 ( X.col(k).head(k) ), tmp1 ( X.col(k).head(k+1) ); @@ -263,7 +265,7 @@ SubMatType A10( A.block(bs,0, brows-bs,bs) ); SubMatType A01( A.block(0,bs, bs,bcols-bs) ); Scalar tmp = A01(bs-1,0); - A01(bs-1,0) = 1; + A01(bs-1,0) = Literal(1); A11.noalias() -= A10 * Y.topLeftCorner(bcols,bs).bottomRows(bcols-bs).adjoint(); A11.noalias() -= X.topLeftCorner(brows,bs).bottomRows(brows-bs) * A01; A01(bs-1,0) = tmp;
diff --git a/Eigen/src/SparseCholesky/CMakeLists.txt b/Eigen/src/SparseCholesky/CMakeLists.txt deleted file mode 100644 index 375a59d..0000000 --- a/Eigen/src/SparseCholesky/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SparseCholesky_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SparseCholesky_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCholesky COMPONENT Devel - )
diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky.h b/Eigen/src/SparseCholesky/SimplicialCholesky.h index 2907f65..1ee4fad 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky.h
@@ -80,11 +80,19 @@ /** Default constructor */ SimplicialCholeskyBase() - : m_info(Success), m_shiftOffset(0), m_shiftScale(1) + : m_info(Success), + m_factorizationIsOk(false), + m_analysisIsOk(false), + m_shiftOffset(0), + m_shiftScale(1) {} explicit SimplicialCholeskyBase(const MatrixType& matrix) - : m_info(Success), m_shiftOffset(0), m_shiftScale(1) + : m_info(Success), + m_factorizationIsOk(false), + m_analysisIsOk(false), + m_shiftOffset(0), + m_shiftScale(1) { derived().compute(matrix); } @@ -101,7 +109,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix.appears to be negative. */ ComputationInfo info() const
diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h index 31e0699..0aa92f8 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h
@@ -5,7 +5,7 @@ /* -NOTE: thes functions vave been adapted from the LDL library: +NOTE: these functions have been adapted from the LDL library: LDL Copyright (c) 2005 by Timothy A. Davis. All Rights Reserved. @@ -122,7 +122,7 @@ for(StorageIndex k = 0; k < size; ++k) { // compute nonzero pattern of kth row of L, in topological order - y[k] = 0.0; // Y(0:k) is now all zero + y[k] = Scalar(0); // Y(0:k) is now all zero StorageIndex top = size; // stack for pattern is empty tags[k] = k; // mark node k as visited m_nonZerosPerCol[k] = 0; // count of nonzeros in column k of L @@ -146,12 +146,12 @@ /* compute numerical values kth row of L (a sparse triangular solve) */ RealScalar d = numext::real(y[k]) * m_shiftScale + m_shiftOffset; // get D(k,k), apply the shift function, and clear Y(k) - y[k] = 0.0; + y[k] = Scalar(0); for(; top < size; ++top) { Index i = pattern[top]; /* pattern[top:n-1] is pattern of L(:,k) */ Scalar yi = y[i]; /* get and clear Y(i) */ - y[i] = 0.0; + y[i] = Scalar(0); /* the nonzero entry L(k,i) */ Scalar l_ki;
diff --git a/Eigen/src/SparseCore/AmbiVector.h b/Eigen/src/SparseCore/AmbiVector.h index 1233e16..e0295f2 100644 --- a/Eigen/src/SparseCore/AmbiVector.h +++ b/Eigen/src/SparseCore/AmbiVector.h
@@ -94,7 +94,7 @@ Index allocSize = m_allocatedElements * sizeof(ListEl); allocSize = (allocSize + sizeof(Scalar) - 1)/sizeof(Scalar); Scalar* newBuffer = new Scalar[allocSize]; - memcpy(newBuffer, m_buffer, copyElements * sizeof(ListEl)); + std::memcpy(newBuffer, m_buffer, copyElements * sizeof(ListEl)); delete[] m_buffer; m_buffer = newBuffer; } @@ -336,7 +336,7 @@ { do { ++m_cachedIndex; - } while (m_cachedIndex<m_vector.m_end && abs(m_vector.m_buffer[m_cachedIndex])<m_epsilon); + } while (m_cachedIndex<m_vector.m_end && abs(m_vector.m_buffer[m_cachedIndex])<=m_epsilon); if (m_cachedIndex<m_vector.m_end) m_cachedValue = m_vector.m_buffer[m_cachedIndex]; else @@ -347,7 +347,7 @@ ListEl* EIGEN_RESTRICT llElements = reinterpret_cast<ListEl*>(m_vector.m_buffer); do { m_currentEl = llElements[m_currentEl].next; - } while (m_currentEl>=0 && abs(llElements[m_currentEl].value)<m_epsilon); + } while (m_currentEl>=0 && abs(llElements[m_currentEl].value)<=m_epsilon); if (m_currentEl<0) { m_cachedIndex = -1; @@ -363,9 +363,9 @@ protected: const AmbiVector& m_vector; // the target vector - StorageIndex m_currentEl; // the current element in sparse/linked-list mode + StorageIndex m_currentEl; // the current element in sparse/linked-list mode RealScalar m_epsilon; // epsilon used to prune zero coefficients - StorageIndex m_cachedIndex; // current coordinate + StorageIndex m_cachedIndex; // current coordinate Scalar m_cachedValue; // current value bool m_isDense; // mode of the vector };
diff --git a/Eigen/src/SparseCore/CMakeLists.txt b/Eigen/src/SparseCore/CMakeLists.txt deleted file mode 100644 index d860452..0000000 --- a/Eigen/src/SparseCore/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SparseCore_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SparseCore_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseCore COMPONENT Devel - )
diff --git a/Eigen/src/SparseCore/CompressedStorage.h b/Eigen/src/SparseCore/CompressedStorage.h index d89fa0d..acd986f 100644 --- a/Eigen/src/SparseCore/CompressedStorage.h +++ b/Eigen/src/SparseCore/CompressedStorage.h
@@ -207,6 +207,22 @@ return m_values[id]; } + void moveChunk(Index from, Index to, Index chunkSize) + { + eigen_internal_assert(to+chunkSize <= m_size); + if(to>from && from+chunkSize>to) + { + // move backward + internal::smart_memmove(m_values+from, m_values+from+chunkSize, m_values+to); + internal::smart_memmove(m_indices+from, m_indices+from+chunkSize, m_indices+to); + } + else + { + internal::smart_copy(m_values+from, m_values+from+chunkSize, m_values+to); + internal::smart_copy(m_indices+from, m_indices+from+chunkSize, m_indices+to); + } + } + void prune(const Scalar& reference, const RealScalar& epsilon = NumTraits<RealScalar>::dummy_precision()) { Index k = 0;
diff --git a/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h b/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h index 0f68358..9db119b 100644 --- a/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h +++ b/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h
@@ -17,7 +17,9 @@ template<typename Lhs, typename Rhs, typename ResultType> static void conservative_sparse_sparse_product_impl(const Lhs& lhs, const Rhs& rhs, ResultType& res, bool sortedInsertion = false) { - typedef typename remove_all<Lhs>::type::Scalar Scalar; + typedef typename remove_all<Lhs>::type::Scalar LhsScalar; + typedef typename remove_all<Rhs>::type::Scalar RhsScalar; + typedef typename remove_all<ResultType>::type::Scalar ResScalar; // make sure to call innerSize/outerSize since we fake the storage order. Index rows = lhs.innerSize(); @@ -25,7 +27,7 @@ eigen_assert(lhs.outerSize() == rhs.innerSize()); ei_declare_aligned_stack_constructed_variable(bool, mask, rows, 0); - ei_declare_aligned_stack_constructed_variable(Scalar, values, rows, 0); + ei_declare_aligned_stack_constructed_variable(ResScalar, values, rows, 0); ei_declare_aligned_stack_constructed_variable(Index, indices, rows, 0); std::memset(mask,0,sizeof(bool)*rows); @@ -51,12 +53,12 @@ Index nnz = 0; for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) { - Scalar y = rhsIt.value(); + RhsScalar y = rhsIt.value(); Index k = rhsIt.index(); for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, k); lhsIt; ++lhsIt) { Index i = lhsIt.index(); - Scalar x = lhsIt.value(); + LhsScalar x = lhsIt.value(); if(!mask[i]) { mask[i] = true; @@ -143,7 +145,7 @@ // If the result is tall and thin (in the extreme case a column vector) // then it is faster to sort the coefficients inplace instead of transposing twice. // FIXME, the following heuristic is probably not very good. - if(lhs.rows()>=rhs.cols()) + if(lhs.rows()>rhs.cols()) { ColMajorMatrix resCol(lhs.rows(),rhs.cols()); // perform sorted insertion @@ -166,11 +168,12 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorMatrix; - RowMajorMatrix rhsRow = rhs; - RowMajorMatrix resRow(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl<RowMajorMatrix,Lhs,RowMajorMatrix>(rhsRow, lhs, resRow); - res = resRow; + typedef SparseMatrix<typename Rhs::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorRhs; + typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorRes; + RowMajorRhs rhsRow = rhs; + RowMajorRes resRow(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl<RowMajorRhs,Lhs,RowMajorRes>(rhsRow, lhs, resRow); + res = resRow; } }; @@ -179,10 +182,11 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorMatrix; - RowMajorMatrix lhsRow = lhs; - RowMajorMatrix resRow(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl<Rhs,RowMajorMatrix,RowMajorMatrix>(rhs, lhsRow, resRow); + typedef SparseMatrix<typename Lhs::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorLhs; + typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename ResultType::StorageIndex> RowMajorRes; + RowMajorLhs lhsRow = lhs; + RowMajorRes resRow(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl<Rhs,RowMajorLhs,RowMajorRes>(rhs, lhsRow, resRow); res = resRow; } }; @@ -219,10 +223,11 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorMatrix; - ColMajorMatrix lhsCol = lhs; - ColMajorMatrix resCol(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl<ColMajorMatrix,Rhs,ColMajorMatrix>(lhsCol, rhs, resCol); + typedef SparseMatrix<typename Lhs::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorLhs; + typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorRes; + ColMajorLhs lhsCol = lhs; + ColMajorRes resCol(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl<ColMajorLhs,Rhs,ColMajorRes>(lhsCol, rhs, resCol); res = resCol; } }; @@ -232,10 +237,11 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorMatrix; - ColMajorMatrix rhsCol = rhs; - ColMajorMatrix resCol(lhs.rows(), rhs.cols()); - internal::conservative_sparse_sparse_product_impl<Lhs,ColMajorMatrix,ColMajorMatrix>(lhs, rhsCol, resCol); + typedef SparseMatrix<typename Rhs::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorRhs; + typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorRes; + ColMajorRhs rhsCol = rhs; + ColMajorRes resCol(lhs.rows(), rhs.cols()); + internal::conservative_sparse_sparse_product_impl<Lhs,ColMajorRhs,ColMajorRes>(lhs, rhsCol, resCol); res = resCol; } }; @@ -263,7 +269,8 @@ template<typename Lhs, typename Rhs, typename ResultType> static void sparse_sparse_to_dense_product_impl(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef typename remove_all<Lhs>::type::Scalar Scalar; + typedef typename remove_all<Lhs>::type::Scalar LhsScalar; + typedef typename remove_all<Rhs>::type::Scalar RhsScalar; Index cols = rhs.outerSize(); eigen_assert(lhs.outerSize() == rhs.innerSize()); @@ -274,12 +281,12 @@ { for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) { - Scalar y = rhsIt.value(); + RhsScalar y = rhsIt.value(); Index k = rhsIt.index(); for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, k); lhsIt; ++lhsIt) { Index i = lhsIt.index(); - Scalar x = lhsIt.value(); + LhsScalar x = lhsIt.value(); res.coeffRef(i,j) += x * y; } } @@ -310,9 +317,9 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorMatrix; - ColMajorMatrix lhsCol(lhs); - internal::sparse_sparse_to_dense_product_impl<ColMajorMatrix,Rhs,ResultType>(lhsCol, rhs, res); + typedef SparseMatrix<typename Lhs::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorLhs; + ColMajorLhs lhsCol(lhs); + internal::sparse_sparse_to_dense_product_impl<ColMajorLhs,Rhs,ResultType>(lhsCol, rhs, res); } }; @@ -321,9 +328,9 @@ { static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorMatrix; - ColMajorMatrix rhsCol(rhs); - internal::sparse_sparse_to_dense_product_impl<Lhs,ColMajorMatrix,ResultType>(lhs, rhsCol, res); + typedef SparseMatrix<typename Rhs::Scalar,ColMajor,typename ResultType::StorageIndex> ColMajorRhs; + ColMajorRhs rhsCol(rhs); + internal::sparse_sparse_to_dense_product_impl<Lhs,ColMajorRhs,ResultType>(lhs, rhsCol, res); } };
diff --git a/Eigen/src/SparseCore/SparseAssign.h b/Eigen/src/SparseCore/SparseAssign.h index 4a8dd12..905485c 100644 --- a/Eigen/src/SparseCore/SparseAssign.h +++ b/Eigen/src/SparseCore/SparseAssign.h
@@ -34,8 +34,8 @@ inline Derived& SparseMatrixBase<Derived>::operator=(const SparseMatrixBase<OtherDerived>& other) { // by default sparse evaluation do not alias, so we can safely bypass the generic call_assignment routine - internal::Assignment<Derived,OtherDerived,internal::assign_op<Scalar> > - ::run(derived(), other.derived(), internal::assign_op<Scalar>()); + internal::Assignment<Derived,OtherDerived,internal::assign_op<Scalar,typename OtherDerived::Scalar> > + ::run(derived(), other.derived(), internal::assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -83,7 +83,7 @@ // eval without temporary dst.resize(src.rows(), src.cols()); dst.setZero(); - dst.reserve((std::max)(src.rows(),src.cols())*2); + dst.reserve((std::min)(src.rows()*src.cols(), (std::max)(src.rows(),src.cols())*2)); for (Index j=0; j<outerEvaluationSize; ++j) { dst.startVec(j); @@ -107,7 +107,7 @@ DstXprType temp(src.rows(), src.cols()); - temp.reserve((std::max)(src.rows(),src.cols())*2); + temp.reserve((std::min)(src.rows()*src.cols(), (std::max)(src.rows(),src.cols())*2)); for (Index j=0; j<outerEvaluationSize; ++j) { temp.startVec(j); @@ -124,28 +124,28 @@ } // Generic Sparse to Sparse assignment -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Sparse, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Sparse> { - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) { assign_sparse_to_sparse(dst.derived(), src.derived()); } }; // Generic Sparse to Dense assignment -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Dense, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor, typename Weak> +struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Dense, Weak> { static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { - eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); - - if(internal::is_same<Functor,internal::assign_op<Scalar> >::value) + if(internal::is_same<Functor,internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> >::value) dst.setZero(); internal::evaluator<SrcXprType> srcEval(src); + resize_if_allowed(dst, src, func); internal::evaluator<DstXprType> dstEval(dst); + const Index outerEvaluationSize = (internal::evaluator<SrcXprType>::Flags&RowMajorBit) ? src.rows() : src.cols(); for (Index j=0; j<outerEvaluationSize; ++j) for (typename internal::evaluator<SrcXprType>::InnerIterator i(srcEval,j); i; ++i) @@ -153,14 +153,86 @@ } }; +// Specialization for dense ?= dense +/- sparse and dense ?= sparse +/- dense +template<typename DstXprType, typename Func1, typename Func2> +struct assignment_from_dense_op_sparse +{ + template<typename SrcXprType, typename InitialFunc> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void run(DstXprType &dst, const SrcXprType &src, const InitialFunc& /*func*/) + { + #ifdef EIGEN_SPARSE_ASSIGNMENT_FROM_DENSE_OP_SPARSE_PLUGIN + EIGEN_SPARSE_ASSIGNMENT_FROM_DENSE_OP_SPARSE_PLUGIN + #endif + + call_assignment_no_alias(dst, src.lhs(), Func1()); + call_assignment_no_alias(dst, src.rhs(), Func2()); + } + + // Specialization for dense1 = sparse + dense2; -> dense1 = dense2; dense1 += sparse; + template<typename Lhs, typename Rhs, typename Scalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::enable_if<internal::is_same<typename internal::evaluator_traits<Rhs>::Shape,DenseShape>::value>::type + run(DstXprType &dst, const CwiseBinaryOp<internal::scalar_sum_op<Scalar,Scalar>, const Lhs, const Rhs> &src, + const internal::assign_op<typename DstXprType::Scalar,Scalar>& /*func*/) + { + #ifdef EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_ADD_DENSE_PLUGIN + EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_ADD_DENSE_PLUGIN + #endif + + // Apply the dense matrix first, then the sparse one. + call_assignment_no_alias(dst, src.rhs(), Func1()); + call_assignment_no_alias(dst, src.lhs(), Func2()); + } + + // Specialization for dense1 = sparse - dense2; -> dense1 = -dense2; dense1 += sparse; + template<typename Lhs, typename Rhs, typename Scalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::enable_if<internal::is_same<typename internal::evaluator_traits<Rhs>::Shape,DenseShape>::value>::type + run(DstXprType &dst, const CwiseBinaryOp<internal::scalar_difference_op<Scalar,Scalar>, const Lhs, const Rhs> &src, + const internal::assign_op<typename DstXprType::Scalar,Scalar>& /*func*/) + { + #ifdef EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_SUB_DENSE_PLUGIN + EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_SUB_DENSE_PLUGIN + #endif + + // Apply the dense matrix first, then the sparse one. + call_assignment_no_alias(dst, -src.rhs(), Func1()); + call_assignment_no_alias(dst, src.lhs(), add_assign_op<typename DstXprType::Scalar,typename Lhs::Scalar>()); + } +}; + +#define EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(ASSIGN_OP,BINOP,ASSIGN_OP2) \ + template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> \ + struct Assignment<DstXprType, CwiseBinaryOp<internal::BINOP<Scalar,Scalar>, const Lhs, const Rhs>, internal::ASSIGN_OP<typename DstXprType::Scalar,Scalar>, \ + Sparse2Dense, \ + typename internal::enable_if< internal::is_same<typename internal::evaluator_traits<Lhs>::Shape,DenseShape>::value \ + || internal::is_same<typename internal::evaluator_traits<Rhs>::Shape,DenseShape>::value>::type> \ + : assignment_from_dense_op_sparse<DstXprType, internal::ASSIGN_OP<typename DstXprType::Scalar,typename Lhs::Scalar>, internal::ASSIGN_OP2<typename DstXprType::Scalar,typename Rhs::Scalar> > \ + {} + +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(assign_op, scalar_sum_op,add_assign_op); +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(add_assign_op,scalar_sum_op,add_assign_op); +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(sub_assign_op,scalar_sum_op,sub_assign_op); + +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(assign_op, scalar_difference_op,sub_assign_op); +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(add_assign_op,scalar_difference_op,sub_assign_op); +EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE(sub_assign_op,scalar_difference_op,add_assign_op); + + // Specialization for "dst = dec.solve(rhs)" // NOTE we need to specialize it for Sparse2Sparse to avoid ambiguous specialization error template<typename DstXprType, typename DecType, typename RhsType, typename Scalar> -struct Assignment<DstXprType, Solve<DecType,RhsType>, internal::assign_op<Scalar>, Sparse2Sparse, Scalar> +struct Assignment<DstXprType, Solve<DecType,RhsType>, internal::assign_op<Scalar,Scalar>, Sparse2Sparse> { typedef Solve<DecType,RhsType> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + src.dec()._solve_impl(src.rhs(), dst); } }; @@ -169,34 +241,27 @@ template<> struct AssignmentKind<SparseShape,DiagonalShape> { typedef Diagonal2Sparse Kind; }; -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Sparse, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Sparse> { typedef typename DstXprType::StorageIndex StorageIndex; - typedef Array<StorageIndex,Dynamic,1> ArrayXI; - typedef Array<Scalar,Dynamic,1> ArrayXS; - template<int Options> - static void run(SparseMatrix<Scalar,Options,StorageIndex> &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) - { - Index size = src.diagonal().size(); - dst.makeCompressed(); - dst.resizeNonZeros(size); - Map<ArrayXI>(dst.innerIndexPtr(), size).setLinSpaced(0,StorageIndex(size)-1); - Map<ArrayXI>(dst.outerIndexPtr(), size+1).setLinSpaced(0,StorageIndex(size)); - Map<ArrayXS>(dst.valuePtr(), size) = src.diagonal(); - } + typedef typename DstXprType::Scalar Scalar; + + template<int Options, typename AssignFunc> + static void run(SparseMatrix<Scalar,Options,StorageIndex> &dst, const SrcXprType &src, const AssignFunc &func) + { dst.assignDiagonal(src.diagonal(), func); } template<typename DstDerived> - static void run(SparseMatrixBase<DstDerived> &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) - { - dst.diagonal() = src.diagonal(); - } + static void run(SparseMatrixBase<DstDerived> &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) + { dst.derived().diagonal() = src.diagonal(); } - static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar> &/*func*/) - { dst.diagonal() += src.diagonal(); } + template<typename DstDerived> + static void run(SparseMatrixBase<DstDerived> &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) + { dst.derived().diagonal() += src.diagonal(); } - static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar> &/*func*/) - { dst.diagonal() -= src.diagonal(); } + template<typename DstDerived> + static void run(SparseMatrixBase<DstDerived> &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/) + { dst.derived().diagonal() -= src.diagonal(); } }; } // end namespace internal
diff --git a/Eigen/src/SparseCore/SparseBlock.h b/Eigen/src/SparseCore/SparseBlock.h index 82fae8c..5ed7c43 100644 --- a/Eigen/src/SparseCore/SparseBlock.h +++ b/Eigen/src/SparseCore/SparseBlock.h
@@ -1,601 +1,563 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_SPARSE_BLOCK_H -#define EIGEN_SPARSE_BLOCK_H - -namespace Eigen { - -// Subset of columns or rows -template<typename XprType, int BlockRows, int BlockCols> -class BlockImpl<XprType,BlockRows,BlockCols,true,Sparse> - : public SparseMatrixBase<Block<XprType,BlockRows,BlockCols,true> > -{ - typedef typename internal::remove_all<typename XprType::Nested>::type _MatrixTypeNested; - typedef Block<XprType, BlockRows, BlockCols, true> BlockType; -public: - enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; -protected: - enum { OuterSize = IsRowMajor ? BlockRows : BlockCols }; - typedef SparseMatrixBase<BlockType> Base; - using Base::convert_index; -public: - EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) - - inline BlockImpl(XprType& xpr, Index i) - : m_matrix(xpr), m_outerStart(convert_index(i)), m_outerSize(OuterSize) - {} - - inline BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : m_matrix(xpr), m_outerStart(convert_index(IsRowMajor ? startRow : startCol)), m_outerSize(convert_index(IsRowMajor ? blockRows : blockCols)) - {} - - EIGEN_STRONG_INLINE Index rows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } - EIGEN_STRONG_INLINE Index cols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } - - Index nonZeros() const - { - typedef internal::evaluator<XprType> EvaluatorType; - EvaluatorType matEval(m_matrix); - Index nnz = 0; - Index end = m_outerStart + m_outerSize.value(); - for(Index j=m_outerStart; j<end; ++j) - for(typename EvaluatorType::InnerIterator it(matEval, j); it; ++it) - ++nnz; - return nnz; - } - - inline const Scalar coeff(Index row, Index col) const - { - return m_matrix.coeff(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); - } - - inline const Scalar coeff(Index index) const - { - return m_matrix.coeff(IsRowMajor ? m_outerStart : index, IsRowMajor ? index : m_outerStart); - } - - inline const XprType& nestedExpression() const { return m_matrix; } - inline XprType& nestedExpression() { return m_matrix; } - Index startRow() const { return IsRowMajor ? m_outerStart : 0; } - Index startCol() const { return IsRowMajor ? 0 : m_outerStart; } - Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } - Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } - - protected: - - typename internal::ref_selector<XprType>::non_const_type m_matrix; - Index m_outerStart; - const internal::variable_if_dynamic<Index, OuterSize> m_outerSize; - - protected: - // Disable assignment with clear error message. - // Note that simply removing operator= yields compilation errors with ICC+MSVC - template<typename T> - BlockImpl& operator=(const T&) - { - EIGEN_STATIC_ASSERT(sizeof(T)==0, THIS_SPARSE_BLOCK_SUBEXPRESSION_IS_READ_ONLY); - return *this; - } -}; - - -/*************************************************************************** -* specialization for SparseMatrix -***************************************************************************/ - -namespace internal { - -template<typename SparseMatrixType, int BlockRows, int BlockCols> -class sparse_matrix_block_impl - : public SparseCompressedBase<Block<SparseMatrixType,BlockRows,BlockCols,true> > -{ - typedef typename internal::remove_all<typename SparseMatrixType::Nested>::type _MatrixTypeNested; - typedef Block<SparseMatrixType, BlockRows, BlockCols, true> BlockType; - typedef SparseCompressedBase<Block<SparseMatrixType,BlockRows,BlockCols,true> > Base; - using Base::convert_index; -public: - enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; - EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) -protected: - typedef typename Base::IndexVector IndexVector; - enum { OuterSize = IsRowMajor ? BlockRows : BlockCols }; -public: - - inline sparse_matrix_block_impl(SparseMatrixType& xpr, Index i) - : m_matrix(xpr), m_outerStart(convert_index(i)), m_outerSize(OuterSize) - {} - - inline sparse_matrix_block_impl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : m_matrix(xpr), m_outerStart(convert_index(IsRowMajor ? startRow : startCol)), m_outerSize(convert_index(IsRowMajor ? blockRows : blockCols)) - {} - - template<typename OtherDerived> - inline BlockType& operator=(const SparseMatrixBase<OtherDerived>& other) - { - typedef typename internal::remove_all<typename SparseMatrixType::Nested>::type _NestedMatrixType; - _NestedMatrixType& matrix = m_matrix; - // This assignment is slow if this vector set is not empty - // and/or it is not at the end of the nonzeros of the underlying matrix. - - // 1 - eval to a temporary to avoid transposition and/or aliasing issues - Ref<const SparseMatrix<Scalar, IsRowMajor ? RowMajor : ColMajor, StorageIndex> > tmp(other.derived()); - eigen_internal_assert(tmp.outerSize()==m_outerSize.value()); - - // 2 - let's check whether there is enough allocated memory - Index nnz = tmp.nonZeros(); - Index start = m_outerStart==0 ? 0 : matrix.outerIndexPtr()[m_outerStart]; // starting position of the current block - Index end = m_matrix.outerIndexPtr()[m_outerStart+m_outerSize.value()]; // ending position of the current block - Index block_size = end - start; // available room in the current block - Index tail_size = m_matrix.outerIndexPtr()[m_matrix.outerSize()] - end; - - Index free_size = m_matrix.isCompressed() - ? Index(matrix.data().allocatedSize()) + block_size - : block_size; - - bool update_trailing_pointers = false; - if(nnz>free_size) - { - // realloc manually to reduce copies - typename SparseMatrixType::Storage newdata(m_matrix.data().allocatedSize() - block_size + nnz); - - internal::smart_copy(m_matrix.valuePtr(), m_matrix.valuePtr() + start, newdata.valuePtr()); - internal::smart_copy(m_matrix.innerIndexPtr(), m_matrix.innerIndexPtr() + start, newdata.indexPtr()); - - internal::smart_copy(tmp.valuePtr(), tmp.valuePtr() + nnz, newdata.valuePtr() + start); - internal::smart_copy(tmp.innerIndexPtr(), tmp.innerIndexPtr() + nnz, newdata.indexPtr() + start); - - internal::smart_copy(matrix.valuePtr()+end, matrix.valuePtr()+end + tail_size, newdata.valuePtr()+start+nnz); - internal::smart_copy(matrix.innerIndexPtr()+end, matrix.innerIndexPtr()+end + tail_size, newdata.indexPtr()+start+nnz); - - newdata.resize(m_matrix.outerIndexPtr()[m_matrix.outerSize()] - block_size + nnz); - - matrix.data().swap(newdata); - - update_trailing_pointers = true; - } - else - { - if(m_matrix.isCompressed()) - { - // no need to realloc, simply copy the tail at its respective position and insert tmp - matrix.data().resize(start + nnz + tail_size); - - internal::smart_memmove(matrix.valuePtr()+end, matrix.valuePtr() + end+tail_size, matrix.valuePtr() + start+nnz); - internal::smart_memmove(matrix.innerIndexPtr()+end, matrix.innerIndexPtr() + end+tail_size, matrix.innerIndexPtr() + start+nnz); - - update_trailing_pointers = true; - } - - internal::smart_copy(tmp.valuePtr(), tmp.valuePtr() + nnz, matrix.valuePtr() + start); - internal::smart_copy(tmp.innerIndexPtr(), tmp.innerIndexPtr() + nnz, matrix.innerIndexPtr() + start); - } - - // update outer index pointers and innerNonZeros - if(IsVectorAtCompileTime) - { - if(!m_matrix.isCompressed()) - matrix.innerNonZeroPtr()[m_outerStart] = StorageIndex(nnz); - matrix.outerIndexPtr()[m_outerStart] = StorageIndex(start); - } - else - { - StorageIndex p = StorageIndex(start); - for(Index k=0; k<m_outerSize.value(); ++k) - { - Index nnz_k = tmp.innerVector(k).nonZeros(); - if(!m_matrix.isCompressed()) - matrix.innerNonZeroPtr()[m_outerStart+k] = StorageIndex(nnz_k); - matrix.outerIndexPtr()[m_outerStart+k] = p; - p += nnz_k; - } - } - - if(update_trailing_pointers) - { - StorageIndex offset = internal::convert_index<StorageIndex>(nnz - block_size); - for(Index k = m_outerStart + m_outerSize.value(); k<=matrix.outerSize(); ++k) - { - matrix.outerIndexPtr()[k] += offset; - } - } - - return derived(); - } - - inline BlockType& operator=(const BlockType& other) - { - return operator=<BlockType>(other); - } - - inline const Scalar* valuePtr() const - { return m_matrix.valuePtr(); } - inline Scalar* valuePtr() - { return m_matrix.valuePtr(); } - - inline const StorageIndex* innerIndexPtr() const - { return m_matrix.innerIndexPtr(); } - inline StorageIndex* innerIndexPtr() - { return m_matrix.innerIndexPtr(); } - - inline const StorageIndex* outerIndexPtr() const - { return m_matrix.outerIndexPtr() + m_outerStart; } - inline StorageIndex* outerIndexPtr() - { return m_matrix.outerIndexPtr() + m_outerStart; } - - inline const StorageIndex* innerNonZeroPtr() const - { return isCompressed() ? 0 : (m_matrix.innerNonZeroPtr()+m_outerStart); } - inline StorageIndex* innerNonZeroPtr() - { return isCompressed() ? 0 : (m_matrix.innerNonZeroPtr()+m_outerStart); } - - bool isCompressed() const { return m_matrix.innerNonZeroPtr()==0; } - - inline Scalar& coeffRef(Index row, Index col) - { - return m_matrix.coeffRef(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); - } - - inline const Scalar coeff(Index row, Index col) const - { - return m_matrix.coeff(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); - } - - inline const Scalar coeff(Index index) const - { - return m_matrix.coeff(IsRowMajor ? m_outerStart : index, IsRowMajor ? index : m_outerStart); - } - - const Scalar& lastCoeff() const - { - EIGEN_STATIC_ASSERT_VECTOR_ONLY(sparse_matrix_block_impl); - eigen_assert(Base::nonZeros()>0); - if(m_matrix.isCompressed()) - return m_matrix.valuePtr()[m_matrix.outerIndexPtr()[m_outerStart+1]-1]; - else - return m_matrix.valuePtr()[m_matrix.outerIndexPtr()[m_outerStart]+m_matrix.innerNonZeroPtr()[m_outerStart]-1]; - } - - EIGEN_STRONG_INLINE Index rows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } - EIGEN_STRONG_INLINE Index cols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } - - inline const SparseMatrixType& nestedExpression() const { return m_matrix; } - inline SparseMatrixType& nestedExpression() { return m_matrix; } - Index startRow() const { return IsRowMajor ? m_outerStart : 0; } - Index startCol() const { return IsRowMajor ? 0 : m_outerStart; } - Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } - Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } - - protected: - - typename internal::ref_selector<SparseMatrixType>::non_const_type m_matrix; - Index m_outerStart; - const internal::variable_if_dynamic<Index, OuterSize> m_outerSize; - -}; - -} // namespace internal - -template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> -class BlockImpl<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true,Sparse> - : public internal::sparse_matrix_block_impl<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols> -{ -public: - typedef _StorageIndex StorageIndex; - typedef SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; - typedef internal::sparse_matrix_block_impl<SparseMatrixType,BlockRows,BlockCols> Base; - inline BlockImpl(SparseMatrixType& xpr, Index i) - : Base(xpr, i) - {} - - inline BlockImpl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : Base(xpr, startRow, startCol, blockRows, blockCols) - {} - - using Base::operator=; -}; - -template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> -class BlockImpl<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true,Sparse> - : public internal::sparse_matrix_block_impl<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols> -{ -public: - typedef _StorageIndex StorageIndex; - typedef const SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; - typedef internal::sparse_matrix_block_impl<SparseMatrixType,BlockRows,BlockCols> Base; - inline BlockImpl(SparseMatrixType& xpr, Index i) - : Base(xpr, i) - {} - - inline BlockImpl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : Base(xpr, startRow, startCol, blockRows, blockCols) - {} - - using Base::operator=; -private: - template<typename Derived> BlockImpl(const SparseMatrixBase<Derived>& xpr, Index i); - template<typename Derived> BlockImpl(const SparseMatrixBase<Derived>& xpr); -}; - -//---------- - -/** \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this - * is col-major (resp. row-major). - */ -template<typename Derived> -typename SparseMatrixBase<Derived>::InnerVectorReturnType SparseMatrixBase<Derived>::innerVector(Index outer) -{ return InnerVectorReturnType(derived(), outer); } - -/** \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this - * is col-major (resp. row-major). Read-only. - */ -template<typename Derived> -const typename SparseMatrixBase<Derived>::ConstInnerVectorReturnType SparseMatrixBase<Derived>::innerVector(Index outer) const -{ return ConstInnerVectorReturnType(derived(), outer); } - -/** \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this - * is col-major (resp. row-major). - */ -template<typename Derived> -typename SparseMatrixBase<Derived>::InnerVectorsReturnType -SparseMatrixBase<Derived>::innerVectors(Index outerStart, Index outerSize) -{ - return Block<Derived,Dynamic,Dynamic,true>(derived(), - IsRowMajor ? outerStart : 0, IsRowMajor ? 0 : outerStart, - IsRowMajor ? outerSize : rows(), IsRowMajor ? cols() : outerSize); - -} - -/** \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this - * is col-major (resp. row-major). Read-only. - */ -template<typename Derived> -const typename SparseMatrixBase<Derived>::ConstInnerVectorsReturnType -SparseMatrixBase<Derived>::innerVectors(Index outerStart, Index outerSize) const -{ - return Block<const Derived,Dynamic,Dynamic,true>(derived(), - IsRowMajor ? outerStart : 0, IsRowMajor ? 0 : outerStart, - IsRowMajor ? outerSize : rows(), IsRowMajor ? cols() : outerSize); - -} - -/** Generic implementation of sparse Block expression. - * Real-only. - */ -template<typename XprType, int BlockRows, int BlockCols, bool InnerPanel> -class BlockImpl<XprType,BlockRows,BlockCols,InnerPanel,Sparse> - : public SparseMatrixBase<Block<XprType,BlockRows,BlockCols,InnerPanel> >, internal::no_assignment_operator -{ - typedef Block<XprType, BlockRows, BlockCols, InnerPanel> BlockType; - typedef SparseMatrixBase<BlockType> Base; - using Base::convert_index; -public: - enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; - EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) - - typedef typename internal::remove_all<typename XprType::Nested>::type _MatrixTypeNested; - - /** Column or Row constructor - */ - inline BlockImpl(XprType& xpr, Index i) - : m_matrix(xpr), - m_startRow( (BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) ? convert_index(i) : 0), - m_startCol( (BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) ? convert_index(i) : 0), - m_blockRows(BlockRows==1 ? 1 : xpr.rows()), - m_blockCols(BlockCols==1 ? 1 : xpr.cols()) - {} - - /** Dynamic-size constructor - */ - inline BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : m_matrix(xpr), m_startRow(convert_index(startRow)), m_startCol(convert_index(startCol)), m_blockRows(convert_index(blockRows)), m_blockCols(convert_index(blockCols)) - {} - - inline Index rows() const { return m_blockRows.value(); } - inline Index cols() const { return m_blockCols.value(); } - - inline Scalar& coeffRef(Index row, Index col) - { - return m_matrix.coeffRef(row + m_startRow.value(), col + m_startCol.value()); - } - - inline const Scalar coeff(Index row, Index col) const - { - return m_matrix.coeff(row + m_startRow.value(), col + m_startCol.value()); - } - - inline Scalar& coeffRef(Index index) - { - return m_matrix.coeffRef(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index), - m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0)); - } - - inline const Scalar coeff(Index index) const - { - return m_matrix.coeff(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index), - m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0)); - } - - inline const XprType& nestedExpression() const { return m_matrix; } - inline XprType& nestedExpression() { return m_matrix; } - Index startRow() const { return m_startRow.value(); } - Index startCol() const { return m_startCol.value(); } - Index blockRows() const { return m_blockRows.value(); } - Index blockCols() const { return m_blockCols.value(); } - - protected: -// friend class internal::GenericSparseBlockInnerIteratorImpl<XprType,BlockRows,BlockCols,InnerPanel>; - friend class ReverseInnerIterator; - friend struct internal::unary_evaluator<Block<XprType,BlockRows,BlockCols,InnerPanel>, internal::IteratorBased, Scalar >; - - Index nonZeros() const { return Dynamic; } - - typename internal::ref_selector<XprType>::non_const_type m_matrix; - const internal::variable_if_dynamic<Index, XprType::RowsAtCompileTime == 1 ? 0 : Dynamic> m_startRow; - const internal::variable_if_dynamic<Index, XprType::ColsAtCompileTime == 1 ? 0 : Dynamic> m_startCol; - const internal::variable_if_dynamic<Index, RowsAtCompileTime> m_blockRows; - const internal::variable_if_dynamic<Index, ColsAtCompileTime> m_blockCols; - - protected: - // Disable assignment with clear error message. - // Note that simply removing operator= yields compilation errors with ICC+MSVC - template<typename T> - BlockImpl& operator=(const T&) - { - EIGEN_STATIC_ASSERT(sizeof(T)==0, THIS_SPARSE_BLOCK_SUBEXPRESSION_IS_READ_ONLY); - return *this; - } - -}; - -namespace internal { - -template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> -struct unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased > - : public evaluator_base<Block<ArgType,BlockRows,BlockCols,InnerPanel> > -{ - class InnerVectorInnerIterator; - class OuterVectorInnerIterator; - public: - typedef Block<ArgType,BlockRows,BlockCols,InnerPanel> XprType; - typedef typename XprType::StorageIndex StorageIndex; - typedef typename XprType::Scalar Scalar; - - class ReverseInnerIterator; - - enum { - IsRowMajor = XprType::IsRowMajor, - - OuterVector = (BlockCols==1 && ArgType::IsRowMajor) - | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&". - // revert to || as soon as not needed anymore. - (BlockRows==1 && !ArgType::IsRowMajor), - - CoeffReadCost = evaluator<ArgType>::CoeffReadCost, - Flags = XprType::Flags - }; - - typedef typename internal::conditional<OuterVector,OuterVectorInnerIterator,InnerVectorInnerIterator>::type InnerIterator; - - explicit unary_evaluator(const XprType& op) - : m_argImpl(op.nestedExpression()), m_block(op) - {} - - inline Index nonZerosEstimate() const { - Index nnz = m_block.nonZeros(); - if(nnz<0) - return m_argImpl.nonZerosEstimate() * m_block.size() / m_block.nestedExpression().size(); - return nnz; - } - - protected: - typedef typename evaluator<ArgType>::InnerIterator EvalIterator; - - evaluator<ArgType> m_argImpl; - const XprType &m_block; -}; - -template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> -class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::InnerVectorInnerIterator - : public EvalIterator -{ - const XprType& m_block; - Index m_end; -public: - - EIGEN_STRONG_INLINE InnerVectorInnerIterator(const unary_evaluator& aEval, Index outer) - : EvalIterator(aEval.m_argImpl, outer + (IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol())), - m_block(aEval.m_block), - m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows()) - { - while( (EvalIterator::operator bool()) && (EvalIterator::index() < (IsRowMajor ? m_block.startCol() : m_block.startRow())) ) - EvalIterator::operator++(); - } - - inline StorageIndex index() const { return EvalIterator::index() - convert_index<StorageIndex>(IsRowMajor ? m_block.startCol() : m_block.startRow()); } - inline Index outer() const { return EvalIterator::outer() - (IsRowMajor ? m_block.startRow() : m_block.startCol()); } - inline Index row() const { return EvalIterator::row() - m_block.startRow(); } - inline Index col() const { return EvalIterator::col() - m_block.startCol(); } - - inline operator bool() const { return EvalIterator::operator bool() && EvalIterator::index() < m_end; } -}; - -template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> -class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::OuterVectorInnerIterator -{ - const unary_evaluator& m_eval; - Index m_outerPos; - Index m_innerIndex; - Scalar m_value; - Index m_end; -public: - - EIGEN_STRONG_INLINE OuterVectorInnerIterator(const unary_evaluator& aEval, Index outer) - : m_eval(aEval), - m_outerPos( (IsRowMajor ? aEval.m_block.startCol() : aEval.m_block.startRow()) - 1), // -1 so that operator++ finds the first non-zero entry - m_innerIndex(IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol()), - m_value(0), - m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows()) - { - EIGEN_UNUSED_VARIABLE(outer); - eigen_assert(outer==0); - - ++(*this); - } - - inline StorageIndex index() const { return convert_index<StorageIndex>(m_outerPos - (IsRowMajor ? m_eval.m_block.startCol() : m_eval.m_block.startRow())); } - inline Index outer() const { return 0; } - inline Index row() const { return IsRowMajor ? 0 : index(); } - inline Index col() const { return IsRowMajor ? index() : 0; } - - inline Scalar value() const { return m_value; } - - inline OuterVectorInnerIterator& operator++() - { - // search next non-zero entry - while(++m_outerPos<m_end) - { - EvalIterator it(m_eval.m_argImpl, m_outerPos); - // search for the key m_innerIndex in the current outer-vector - while(it && it.index() < m_innerIndex) ++it; - if(it && it.index()==m_innerIndex) - { - m_value = it.value(); - break; - } - } - return *this; - } - - inline operator bool() const { return m_outerPos < m_end; } -}; - -template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> -struct unary_evaluator<Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true>, IteratorBased> - : evaluator<SparseCompressedBase<Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> > > -{ - typedef Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> XprType; - typedef evaluator<SparseCompressedBase<XprType> > Base; - explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} -}; - -template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> -struct unary_evaluator<Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true>, IteratorBased> - : evaluator<SparseCompressedBase<Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> > > -{ - typedef Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> XprType; - typedef evaluator<SparseCompressedBase<XprType> > Base; - explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} -}; - -} // end namespace internal - - -} // end namespace Eigen - -#endif // EIGEN_SPARSE_BLOCK_H +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPARSE_BLOCK_H +#define EIGEN_SPARSE_BLOCK_H + +namespace Eigen { + +// Subset of columns or rows +template<typename XprType, int BlockRows, int BlockCols> +class BlockImpl<XprType,BlockRows,BlockCols,true,Sparse> + : public SparseMatrixBase<Block<XprType,BlockRows,BlockCols,true> > +{ + typedef typename internal::remove_all<typename XprType::Nested>::type _MatrixTypeNested; + typedef Block<XprType, BlockRows, BlockCols, true> BlockType; +public: + enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; +protected: + enum { OuterSize = IsRowMajor ? BlockRows : BlockCols }; + typedef SparseMatrixBase<BlockType> Base; + using Base::convert_index; +public: + EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) + + inline BlockImpl(XprType& xpr, Index i) + : m_matrix(xpr), m_outerStart(convert_index(i)), m_outerSize(OuterSize) + {} + + inline BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + : m_matrix(xpr), m_outerStart(convert_index(IsRowMajor ? startRow : startCol)), m_outerSize(convert_index(IsRowMajor ? blockRows : blockCols)) + {} + + EIGEN_STRONG_INLINE Index rows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } + EIGEN_STRONG_INLINE Index cols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } + + Index nonZeros() const + { + typedef internal::evaluator<XprType> EvaluatorType; + EvaluatorType matEval(m_matrix); + Index nnz = 0; + Index end = m_outerStart + m_outerSize.value(); + for(Index j=m_outerStart; j<end; ++j) + for(typename EvaluatorType::InnerIterator it(matEval, j); it; ++it) + ++nnz; + return nnz; + } + + inline const Scalar coeff(Index row, Index col) const + { + return m_matrix.coeff(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); + } + + inline const Scalar coeff(Index index) const + { + return m_matrix.coeff(IsRowMajor ? m_outerStart : index, IsRowMajor ? index : m_outerStart); + } + + inline const XprType& nestedExpression() const { return m_matrix; } + inline XprType& nestedExpression() { return m_matrix; } + Index startRow() const { return IsRowMajor ? m_outerStart : 0; } + Index startCol() const { return IsRowMajor ? 0 : m_outerStart; } + Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } + Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } + + protected: + + typename internal::ref_selector<XprType>::non_const_type m_matrix; + Index m_outerStart; + const internal::variable_if_dynamic<Index, OuterSize> m_outerSize; + + protected: + // Disable assignment with clear error message. + // Note that simply removing operator= yields compilation errors with ICC+MSVC + template<typename T> + BlockImpl& operator=(const T&) + { + EIGEN_STATIC_ASSERT(sizeof(T)==0, THIS_SPARSE_BLOCK_SUBEXPRESSION_IS_READ_ONLY); + return *this; + } +}; + + +/*************************************************************************** +* specialization for SparseMatrix +***************************************************************************/ + +namespace internal { + +template<typename SparseMatrixType, int BlockRows, int BlockCols> +class sparse_matrix_block_impl + : public SparseCompressedBase<Block<SparseMatrixType,BlockRows,BlockCols,true> > +{ + typedef typename internal::remove_all<typename SparseMatrixType::Nested>::type _MatrixTypeNested; + typedef Block<SparseMatrixType, BlockRows, BlockCols, true> BlockType; + typedef SparseCompressedBase<Block<SparseMatrixType,BlockRows,BlockCols,true> > Base; + using Base::convert_index; +public: + enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; + EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) +protected: + typedef typename Base::IndexVector IndexVector; + enum { OuterSize = IsRowMajor ? BlockRows : BlockCols }; +public: + + inline sparse_matrix_block_impl(SparseMatrixType& xpr, Index i) + : m_matrix(xpr), m_outerStart(convert_index(i)), m_outerSize(OuterSize) + {} + + inline sparse_matrix_block_impl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + : m_matrix(xpr), m_outerStart(convert_index(IsRowMajor ? startRow : startCol)), m_outerSize(convert_index(IsRowMajor ? blockRows : blockCols)) + {} + + template<typename OtherDerived> + inline BlockType& operator=(const SparseMatrixBase<OtherDerived>& other) + { + typedef typename internal::remove_all<typename SparseMatrixType::Nested>::type _NestedMatrixType; + _NestedMatrixType& matrix = m_matrix; + // This assignment is slow if this vector set is not empty + // and/or it is not at the end of the nonzeros of the underlying matrix. + + // 1 - eval to a temporary to avoid transposition and/or aliasing issues + Ref<const SparseMatrix<Scalar, IsRowMajor ? RowMajor : ColMajor, StorageIndex> > tmp(other.derived()); + eigen_internal_assert(tmp.outerSize()==m_outerSize.value()); + + // 2 - let's check whether there is enough allocated memory + Index nnz = tmp.nonZeros(); + Index start = m_outerStart==0 ? 0 : m_matrix.outerIndexPtr()[m_outerStart]; // starting position of the current block + Index end = m_matrix.outerIndexPtr()[m_outerStart+m_outerSize.value()]; // ending position of the current block + Index block_size = end - start; // available room in the current block + Index tail_size = m_matrix.outerIndexPtr()[m_matrix.outerSize()] - end; + + Index free_size = m_matrix.isCompressed() + ? Index(matrix.data().allocatedSize()) + block_size + : block_size; + + Index tmp_start = tmp.outerIndexPtr()[0]; + + bool update_trailing_pointers = false; + if(nnz>free_size) + { + // realloc manually to reduce copies + typename SparseMatrixType::Storage newdata(m_matrix.data().allocatedSize() - block_size + nnz); + + internal::smart_copy(m_matrix.valuePtr(), m_matrix.valuePtr() + start, newdata.valuePtr()); + internal::smart_copy(m_matrix.innerIndexPtr(), m_matrix.innerIndexPtr() + start, newdata.indexPtr()); + + internal::smart_copy(tmp.valuePtr() + tmp_start, tmp.valuePtr() + tmp_start + nnz, newdata.valuePtr() + start); + internal::smart_copy(tmp.innerIndexPtr() + tmp_start, tmp.innerIndexPtr() + tmp_start + nnz, newdata.indexPtr() + start); + + internal::smart_copy(matrix.valuePtr()+end, matrix.valuePtr()+end + tail_size, newdata.valuePtr()+start+nnz); + internal::smart_copy(matrix.innerIndexPtr()+end, matrix.innerIndexPtr()+end + tail_size, newdata.indexPtr()+start+nnz); + + newdata.resize(m_matrix.outerIndexPtr()[m_matrix.outerSize()] - block_size + nnz); + + matrix.data().swap(newdata); + + update_trailing_pointers = true; + } + else + { + if(m_matrix.isCompressed()) + { + // no need to realloc, simply copy the tail at its respective position and insert tmp + matrix.data().resize(start + nnz + tail_size); + + internal::smart_memmove(matrix.valuePtr()+end, matrix.valuePtr() + end+tail_size, matrix.valuePtr() + start+nnz); + internal::smart_memmove(matrix.innerIndexPtr()+end, matrix.innerIndexPtr() + end+tail_size, matrix.innerIndexPtr() + start+nnz); + + update_trailing_pointers = true; + } + + internal::smart_copy(tmp.valuePtr() + tmp_start, tmp.valuePtr() + tmp_start + nnz, matrix.valuePtr() + start); + internal::smart_copy(tmp.innerIndexPtr() + tmp_start, tmp.innerIndexPtr() + tmp_start + nnz, matrix.innerIndexPtr() + start); + } + + // update outer index pointers and innerNonZeros + if(IsVectorAtCompileTime) + { + if(!m_matrix.isCompressed()) + matrix.innerNonZeroPtr()[m_outerStart] = StorageIndex(nnz); + matrix.outerIndexPtr()[m_outerStart] = StorageIndex(start); + } + else + { + StorageIndex p = StorageIndex(start); + for(Index k=0; k<m_outerSize.value(); ++k) + { + StorageIndex nnz_k = internal::convert_index<StorageIndex>(tmp.innerVector(k).nonZeros()); + if(!m_matrix.isCompressed()) + matrix.innerNonZeroPtr()[m_outerStart+k] = nnz_k; + matrix.outerIndexPtr()[m_outerStart+k] = p; + p += nnz_k; + } + } + + if(update_trailing_pointers) + { + StorageIndex offset = internal::convert_index<StorageIndex>(nnz - block_size); + for(Index k = m_outerStart + m_outerSize.value(); k<=matrix.outerSize(); ++k) + { + matrix.outerIndexPtr()[k] += offset; + } + } + + return derived(); + } + + inline BlockType& operator=(const BlockType& other) + { + return operator=<BlockType>(other); + } + + inline const Scalar* valuePtr() const + { return m_matrix.valuePtr(); } + inline Scalar* valuePtr() + { return m_matrix.valuePtr(); } + + inline const StorageIndex* innerIndexPtr() const + { return m_matrix.innerIndexPtr(); } + inline StorageIndex* innerIndexPtr() + { return m_matrix.innerIndexPtr(); } + + inline const StorageIndex* outerIndexPtr() const + { return m_matrix.outerIndexPtr() + m_outerStart; } + inline StorageIndex* outerIndexPtr() + { return m_matrix.outerIndexPtr() + m_outerStart; } + + inline const StorageIndex* innerNonZeroPtr() const + { return isCompressed() ? 0 : (m_matrix.innerNonZeroPtr()+m_outerStart); } + inline StorageIndex* innerNonZeroPtr() + { return isCompressed() ? 0 : (m_matrix.innerNonZeroPtr()+m_outerStart); } + + bool isCompressed() const { return m_matrix.innerNonZeroPtr()==0; } + + inline Scalar& coeffRef(Index row, Index col) + { + return m_matrix.coeffRef(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); + } + + inline const Scalar coeff(Index row, Index col) const + { + return m_matrix.coeff(row + (IsRowMajor ? m_outerStart : 0), col + (IsRowMajor ? 0 : m_outerStart)); + } + + inline const Scalar coeff(Index index) const + { + return m_matrix.coeff(IsRowMajor ? m_outerStart : index, IsRowMajor ? index : m_outerStart); + } + + const Scalar& lastCoeff() const + { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(sparse_matrix_block_impl); + eigen_assert(Base::nonZeros()>0); + if(m_matrix.isCompressed()) + return m_matrix.valuePtr()[m_matrix.outerIndexPtr()[m_outerStart+1]-1]; + else + return m_matrix.valuePtr()[m_matrix.outerIndexPtr()[m_outerStart]+m_matrix.innerNonZeroPtr()[m_outerStart]-1]; + } + + EIGEN_STRONG_INLINE Index rows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } + EIGEN_STRONG_INLINE Index cols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } + + inline const SparseMatrixType& nestedExpression() const { return m_matrix; } + inline SparseMatrixType& nestedExpression() { return m_matrix; } + Index startRow() const { return IsRowMajor ? m_outerStart : 0; } + Index startCol() const { return IsRowMajor ? 0 : m_outerStart; } + Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); } + Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); } + + protected: + + typename internal::ref_selector<SparseMatrixType>::non_const_type m_matrix; + Index m_outerStart; + const internal::variable_if_dynamic<Index, OuterSize> m_outerSize; + +}; + +} // namespace internal + +template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> +class BlockImpl<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true,Sparse> + : public internal::sparse_matrix_block_impl<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols> +{ +public: + typedef _StorageIndex StorageIndex; + typedef SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; + typedef internal::sparse_matrix_block_impl<SparseMatrixType,BlockRows,BlockCols> Base; + inline BlockImpl(SparseMatrixType& xpr, Index i) + : Base(xpr, i) + {} + + inline BlockImpl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + : Base(xpr, startRow, startCol, blockRows, blockCols) + {} + + using Base::operator=; +}; + +template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> +class BlockImpl<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true,Sparse> + : public internal::sparse_matrix_block_impl<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols> +{ +public: + typedef _StorageIndex StorageIndex; + typedef const SparseMatrix<_Scalar, _Options, _StorageIndex> SparseMatrixType; + typedef internal::sparse_matrix_block_impl<SparseMatrixType,BlockRows,BlockCols> Base; + inline BlockImpl(SparseMatrixType& xpr, Index i) + : Base(xpr, i) + {} + + inline BlockImpl(SparseMatrixType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + : Base(xpr, startRow, startCol, blockRows, blockCols) + {} + + using Base::operator=; +private: + template<typename Derived> BlockImpl(const SparseMatrixBase<Derived>& xpr, Index i); + template<typename Derived> BlockImpl(const SparseMatrixBase<Derived>& xpr); +}; + +//---------- + +/** Generic implementation of sparse Block expression. + * Real-only. + */ +template<typename XprType, int BlockRows, int BlockCols, bool InnerPanel> +class BlockImpl<XprType,BlockRows,BlockCols,InnerPanel,Sparse> + : public SparseMatrixBase<Block<XprType,BlockRows,BlockCols,InnerPanel> >, internal::no_assignment_operator +{ + typedef Block<XprType, BlockRows, BlockCols, InnerPanel> BlockType; + typedef SparseMatrixBase<BlockType> Base; + using Base::convert_index; +public: + enum { IsRowMajor = internal::traits<BlockType>::IsRowMajor }; + EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType) + + typedef typename internal::remove_all<typename XprType::Nested>::type _MatrixTypeNested; + + /** Column or Row constructor + */ + inline BlockImpl(XprType& xpr, Index i) + : m_matrix(xpr), + m_startRow( (BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) ? convert_index(i) : 0), + m_startCol( (BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) ? convert_index(i) : 0), + m_blockRows(BlockRows==1 ? 1 : xpr.rows()), + m_blockCols(BlockCols==1 ? 1 : xpr.cols()) + {} + + /** Dynamic-size constructor + */ + inline BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) + : m_matrix(xpr), m_startRow(convert_index(startRow)), m_startCol(convert_index(startCol)), m_blockRows(convert_index(blockRows)), m_blockCols(convert_index(blockCols)) + {} + + inline Index rows() const { return m_blockRows.value(); } + inline Index cols() const { return m_blockCols.value(); } + + inline Scalar& coeffRef(Index row, Index col) + { + return m_matrix.coeffRef(row + m_startRow.value(), col + m_startCol.value()); + } + + inline const Scalar coeff(Index row, Index col) const + { + return m_matrix.coeff(row + m_startRow.value(), col + m_startCol.value()); + } + + inline Scalar& coeffRef(Index index) + { + return m_matrix.coeffRef(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index), + m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0)); + } + + inline const Scalar coeff(Index index) const + { + return m_matrix.coeff(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index), + m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0)); + } + + inline const XprType& nestedExpression() const { return m_matrix; } + inline XprType& nestedExpression() { return m_matrix; } + Index startRow() const { return m_startRow.value(); } + Index startCol() const { return m_startCol.value(); } + Index blockRows() const { return m_blockRows.value(); } + Index blockCols() const { return m_blockCols.value(); } + + protected: +// friend class internal::GenericSparseBlockInnerIteratorImpl<XprType,BlockRows,BlockCols,InnerPanel>; + friend struct internal::unary_evaluator<Block<XprType,BlockRows,BlockCols,InnerPanel>, internal::IteratorBased, Scalar >; + + Index nonZeros() const { return Dynamic; } + + typename internal::ref_selector<XprType>::non_const_type m_matrix; + const internal::variable_if_dynamic<Index, XprType::RowsAtCompileTime == 1 ? 0 : Dynamic> m_startRow; + const internal::variable_if_dynamic<Index, XprType::ColsAtCompileTime == 1 ? 0 : Dynamic> m_startCol; + const internal::variable_if_dynamic<Index, RowsAtCompileTime> m_blockRows; + const internal::variable_if_dynamic<Index, ColsAtCompileTime> m_blockCols; + + protected: + // Disable assignment with clear error message. + // Note that simply removing operator= yields compilation errors with ICC+MSVC + template<typename T> + BlockImpl& operator=(const T&) + { + EIGEN_STATIC_ASSERT(sizeof(T)==0, THIS_SPARSE_BLOCK_SUBEXPRESSION_IS_READ_ONLY); + return *this; + } + +}; + +namespace internal { + +template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> +struct unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased > + : public evaluator_base<Block<ArgType,BlockRows,BlockCols,InnerPanel> > +{ + class InnerVectorInnerIterator; + class OuterVectorInnerIterator; + public: + typedef Block<ArgType,BlockRows,BlockCols,InnerPanel> XprType; + typedef typename XprType::StorageIndex StorageIndex; + typedef typename XprType::Scalar Scalar; + + enum { + IsRowMajor = XprType::IsRowMajor, + + OuterVector = (BlockCols==1 && ArgType::IsRowMajor) + | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&". + // revert to || as soon as not needed anymore. + (BlockRows==1 && !ArgType::IsRowMajor), + + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = XprType::Flags + }; + + typedef typename internal::conditional<OuterVector,OuterVectorInnerIterator,InnerVectorInnerIterator>::type InnerIterator; + + explicit unary_evaluator(const XprType& op) + : m_argImpl(op.nestedExpression()), m_block(op) + {} + + inline Index nonZerosEstimate() const { + Index nnz = m_block.nonZeros(); + if(nnz<0) + return m_argImpl.nonZerosEstimate() * m_block.size() / m_block.nestedExpression().size(); + return nnz; + } + + protected: + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; + + evaluator<ArgType> m_argImpl; + const XprType &m_block; +}; + +template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> +class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::InnerVectorInnerIterator + : public EvalIterator +{ + enum { IsRowMajor = unary_evaluator::IsRowMajor }; + const XprType& m_block; + Index m_end; +public: + + EIGEN_STRONG_INLINE InnerVectorInnerIterator(const unary_evaluator& aEval, Index outer) + : EvalIterator(aEval.m_argImpl, outer + (IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol())), + m_block(aEval.m_block), + m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows()) + { + while( (EvalIterator::operator bool()) && (EvalIterator::index() < (IsRowMajor ? m_block.startCol() : m_block.startRow())) ) + EvalIterator::operator++(); + } + + inline StorageIndex index() const { return EvalIterator::index() - convert_index<StorageIndex>(IsRowMajor ? m_block.startCol() : m_block.startRow()); } + inline Index outer() const { return EvalIterator::outer() - (IsRowMajor ? m_block.startRow() : m_block.startCol()); } + inline Index row() const { return EvalIterator::row() - m_block.startRow(); } + inline Index col() const { return EvalIterator::col() - m_block.startCol(); } + + inline operator bool() const { return EvalIterator::operator bool() && EvalIterator::index() < m_end; } +}; + +template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> +class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::OuterVectorInnerIterator +{ + enum { IsRowMajor = unary_evaluator::IsRowMajor }; + const unary_evaluator& m_eval; + Index m_outerPos; + const Index m_innerIndex; + Index m_end; + EvalIterator m_it; +public: + + EIGEN_STRONG_INLINE OuterVectorInnerIterator(const unary_evaluator& aEval, Index outer) + : m_eval(aEval), + m_outerPos( (IsRowMajor ? aEval.m_block.startCol() : aEval.m_block.startRow()) ), + m_innerIndex(IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol()), + m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows()), + m_it(m_eval.m_argImpl, m_outerPos) + { + EIGEN_UNUSED_VARIABLE(outer); + eigen_assert(outer==0); + + while(m_it && m_it.index() < m_innerIndex) ++m_it; + if((!m_it) || (m_it.index()!=m_innerIndex)) + ++(*this); + } + + inline StorageIndex index() const { return convert_index<StorageIndex>(m_outerPos - (IsRowMajor ? m_eval.m_block.startCol() : m_eval.m_block.startRow())); } + inline Index outer() const { return 0; } + inline Index row() const { return IsRowMajor ? 0 : index(); } + inline Index col() const { return IsRowMajor ? index() : 0; } + + inline Scalar value() const { return m_it.value(); } + inline Scalar& valueRef() { return m_it.valueRef(); } + + inline OuterVectorInnerIterator& operator++() + { + // search next non-zero entry + while(++m_outerPos<m_end) + { + // Restart iterator at the next inner-vector: + m_it.~EvalIterator(); + ::new (&m_it) EvalIterator(m_eval.m_argImpl, m_outerPos); + // search for the key m_innerIndex in the current outer-vector + while(m_it && m_it.index() < m_innerIndex) ++m_it; + if(m_it && m_it.index()==m_innerIndex) break; + } + return *this; + } + + inline operator bool() const { return m_outerPos < m_end; } +}; + +template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> +struct unary_evaluator<Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true>, IteratorBased> + : evaluator<SparseCompressedBase<Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> > > +{ + typedef Block<SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> XprType; + typedef evaluator<SparseCompressedBase<XprType> > Base; + explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} +}; + +template<typename _Scalar, int _Options, typename _StorageIndex, int BlockRows, int BlockCols> +struct unary_evaluator<Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true>, IteratorBased> + : evaluator<SparseCompressedBase<Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> > > +{ + typedef Block<const SparseMatrix<_Scalar, _Options, _StorageIndex>,BlockRows,BlockCols,true> XprType; + typedef evaluator<SparseCompressedBase<XprType> > Base; + explicit unary_evaluator(const XprType &xpr) : Base(xpr) {} +}; + +} // end namespace internal + + +} // end namespace Eigen + +#endif // EIGEN_SPARSE_BLOCK_H
diff --git a/Eigen/src/SparseCore/SparseCompressedBase.h b/Eigen/src/SparseCore/SparseCompressedBase.h index 15854a7..6a2c7a8 100644 --- a/Eigen/src/SparseCore/SparseCompressedBase.h +++ b/Eigen/src/SparseCore/SparseCompressedBase.h
@@ -106,9 +106,50 @@ /** \returns whether \c *this is in compressed form. */ inline bool isCompressed() const { return innerNonZeroPtr()==0; } + /** \returns a read-only view of the stored coefficients as a 1D array expression. + * + * \warning this method is for \b compressed \b storage \b only, and it will trigger an assertion otherwise. + * + * \sa valuePtr(), isCompressed() */ + const Map<const Array<Scalar,Dynamic,1> > coeffs() const { eigen_assert(isCompressed()); return Array<Scalar,Dynamic,1>::Map(valuePtr(),nonZeros()); } + + /** \returns a read-write view of the stored coefficients as a 1D array expression + * + * \warning this method is for \b compressed \b storage \b only, and it will trigger an assertion otherwise. + * + * Here is an example: + * \include SparseMatrix_coeffs.cpp + * and the output is: + * \include SparseMatrix_coeffs.out + * + * \sa valuePtr(), isCompressed() */ + Map<Array<Scalar,Dynamic,1> > coeffs() { eigen_assert(isCompressed()); return Array<Scalar,Dynamic,1>::Map(valuePtr(),nonZeros()); } + protected: /** Default constructor. Do nothing. */ SparseCompressedBase() {} + + /** \internal return the index of the coeff at (row,col) or just before if it does not exist. + * This is an analogue of std::lower_bound. + */ + internal::LowerBoundIndex lower_bound(Index row, Index col) const + { + eigen_internal_assert(row>=0 && row<this->rows() && col>=0 && col<this->cols()); + + const Index outer = Derived::IsRowMajor ? row : col; + const Index inner = Derived::IsRowMajor ? col : row; + + Index start = this->outerIndexPtr()[outer]; + Index end = this->isCompressed() ? this->outerIndexPtr()[outer+1] : this->outerIndexPtr()[outer] + this->innerNonZeroPtr()[outer]; + eigen_assert(end>=start && "you are using a non finalized sparse matrix or written coefficient does not exist"); + internal::LowerBoundIndex p; + p.value = std::lower_bound(this->innerIndexPtr()+start, this->innerIndexPtr()+end,inner) - this->innerIndexPtr(); + p.found = (p.value<end) && (this->innerIndexPtr()[p.value]==inner); + return p; + } + + friend struct internal::evaluator<SparseCompressedBase<Derived> >; + private: template<typename OtherDerived> explicit SparseCompressedBase(const SparseCompressedBase<OtherDerived>&); }; @@ -166,6 +207,14 @@ } inline InnerIterator& operator++() { m_id++; return *this; } + inline InnerIterator& operator+=(Index i) { m_id += i ; return *this; } + + inline InnerIterator operator+(Index i) + { + InnerIterator result = *this; + result += i; + return result; + } inline const Scalar& value() const { return m_values[m_id]; } inline Scalar& valueRef() { return const_cast<Scalar&>(m_values[m_id]); } @@ -205,11 +254,11 @@ } else { - m_start.value() = mat.outerIndexPtr()[outer]; + m_start = mat.outerIndexPtr()[outer]; if(mat.isCompressed()) m_id = mat.outerIndexPtr()[outer+1]; else - m_id = m_start.value() + mat.innerNonZeroPtr()[outer]; + m_id = m_start + mat.innerNonZeroPtr()[outer]; } } @@ -226,6 +275,14 @@ } inline ReverseInnerIterator& operator--() { --m_id; return *this; } + inline ReverseInnerIterator& operator-=(Index i) { m_id -= i; return *this; } + + inline ReverseInnerIterator operator-(Index i) + { + ReverseInnerIterator result = *this; + result -= i; + return result; + } inline const Scalar& value() const { return m_values[m_id-1]; } inline Scalar& valueRef() { return const_cast<Scalar&>(m_values[m_id-1]); } @@ -235,14 +292,15 @@ inline Index row() const { return IsRowMajor ? m_outer.value() : index(); } inline Index col() const { return IsRowMajor ? index() : m_outer.value(); } - inline operator bool() const { return (m_id > m_start.value()); } + inline operator bool() const { return (m_id > m_start); } protected: const Scalar* m_values; const StorageIndex* m_indices; - const internal::variable_if_dynamic<Index,Derived::IsVectorAtCompileTime?0:Dynamic> m_outer; + typedef internal::variable_if_dynamic<Index,Derived::IsVectorAtCompileTime?0:Dynamic> OuterType; + const OuterType m_outer; + Index m_start; Index m_id; - const internal::variable_if_dynamic<Index,Derived::IsVectorAtCompileTime?0:Dynamic> m_start; }; namespace internal { @@ -253,18 +311,17 @@ { typedef typename Derived::Scalar Scalar; typedef typename Derived::InnerIterator InnerIterator; - typedef typename Derived::ReverseInnerIterator ReverseInnerIterator; enum { CoeffReadCost = NumTraits<Scalar>::ReadCost, Flags = Derived::Flags }; - evaluator() : m_matrix(0) + evaluator() : m_matrix(0), m_zero(0) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } - explicit evaluator(const Derived &mat) : m_matrix(&mat) + explicit evaluator(const Derived &mat) : m_matrix(&mat), m_zero(0) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } @@ -277,26 +334,33 @@ operator const Derived&() const { return *m_matrix; } typedef typename DenseCoeffsBase<Derived,ReadOnlyAccessors>::CoeffReturnType CoeffReturnType; - Scalar coeff(Index row, Index col) const - { return m_matrix->coeff(row,col); } - + const Scalar& coeff(Index row, Index col) const + { + Index p = find(row,col); + + if(p==Dynamic) + return m_zero; + else + return m_matrix->const_cast_derived().valuePtr()[p]; + } + Scalar& coeffRef(Index row, Index col) { - eigen_internal_assert(row>=0 && row<m_matrix->rows() && col>=0 && col<m_matrix->cols()); - - const Index outer = Derived::IsRowMajor ? row : col; - const Index inner = Derived::IsRowMajor ? col : row; - - Index start = m_matrix->outerIndexPtr()[outer]; - Index end = m_matrix->isCompressed() ? m_matrix->outerIndexPtr()[outer+1] : m_matrix->outerIndexPtr()[outer] + m_matrix->innerNonZeroPtr()[outer]; - eigen_assert(end>start && "you are using a non finalized sparse matrix or written coefficient does not exist"); - const Index p = std::lower_bound(m_matrix->innerIndexPtr()+start, m_matrix->innerIndexPtr()+end,inner) - - m_matrix->innerIndexPtr(); - eigen_assert((p<end) && (m_matrix->innerIndexPtr()[p]==inner) && "written coefficient does not exist"); + Index p = find(row,col); + eigen_assert(p!=Dynamic && "written coefficient does not exist"); return m_matrix->const_cast_derived().valuePtr()[p]; } +protected: + + Index find(Index row, Index col) const + { + internal::LowerBoundIndex p = m_matrix->lower_bound(row,col); + return p.found ? p.value : Dynamic; + } + const Derived *m_matrix; + const Scalar m_zero; }; }
diff --git a/Eigen/src/SparseCore/SparseCwiseBinaryOp.h b/Eigen/src/SparseCore/SparseCwiseBinaryOp.h index c57d9ac..c41c07a 100644 --- a/Eigen/src/SparseCore/SparseCwiseBinaryOp.h +++ b/Eigen/src/SparseCore/SparseCwiseBinaryOp.h
@@ -28,6 +28,9 @@ // generic sparse // 4 - dense op dense product dense // generic dense +// +// TODO to ease compiler job, we could specialize product/quotient with a scalar +// and fallback to cwise-unary evaluator using bind1st_op and bind2nd_op. template<typename BinaryOp, typename Lhs, typename Rhs> class CwiseBinaryOpImpl<BinaryOp, Lhs, Rhs, Sparse> @@ -42,7 +45,7 @@ EIGEN_STATIC_ASSERT(( (!internal::is_same<typename internal::traits<Lhs>::StorageKind, typename internal::traits<Rhs>::StorageKind>::value) - || ((Lhs::Flags&RowMajorBit) == (Rhs::Flags&RowMajorBit))), + || ((internal::evaluator<Lhs>::Flags&RowMajorBit) == (internal::evaluator<Rhs>::Flags&RowMajorBit))), THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH); } }; @@ -65,7 +68,6 @@ typedef typename XprType::StorageIndex StorageIndex; public: - class ReverseInnerIterator; class InnerIterator { public: @@ -108,6 +110,7 @@ EIGEN_STRONG_INLINE Scalar value() const { return m_value; } EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return Lhs::IsRowMajor ? m_lhsIter.row() : index(); } EIGEN_STRONG_INLINE Index col() const { return Lhs::IsRowMajor ? index() : m_lhsIter.col(); } @@ -158,14 +161,13 @@ typedef typename XprType::StorageIndex StorageIndex; public: - class ReverseInnerIterator; class InnerIterator { enum { IsRowMajor = (int(Rhs::Flags)&RowMajorBit)==RowMajorBit }; public: EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) - : m_lhsEval(aEval.m_lhsImpl), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_id(-1), m_innerSize(aEval.m_expr.rhs().innerSize()) + : m_lhsEval(aEval.m_lhsImpl), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_value(0), m_id(-1), m_innerSize(aEval.m_expr.rhs().innerSize()) { this->operator++(); } @@ -189,9 +191,10 @@ return *this; } - EIGEN_STRONG_INLINE Scalar value() const { return m_value; } + EIGEN_STRONG_INLINE Scalar value() const { eigen_internal_assert(m_id<m_innerSize); return m_value; } EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_rhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_rhsIter.outer() : m_id; } EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_rhsIter.outer(); } @@ -209,8 +212,7 @@ enum { CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, - // Expose storage order of the sparse expression - Flags = (XprType::Flags & ~RowMajorBit) | (int(Rhs::Flags)&RowMajorBit) + Flags = XprType::Flags }; explicit binary_evaluator(const XprType& xpr) @@ -246,14 +248,13 @@ typedef typename XprType::StorageIndex StorageIndex; public: - class ReverseInnerIterator; class InnerIterator { enum { IsRowMajor = (int(Lhs::Flags)&RowMajorBit)==RowMajorBit }; public: EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) - : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsEval(aEval.m_rhsImpl), m_functor(aEval.m_functor), m_id(-1), m_innerSize(aEval.m_expr.lhs().innerSize()) + : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsEval(aEval.m_rhsImpl), m_functor(aEval.m_functor), m_value(0), m_id(-1), m_innerSize(aEval.m_expr.lhs().innerSize()) { this->operator++(); } @@ -277,9 +278,10 @@ return *this; } - EIGEN_STRONG_INLINE Scalar value() const { return m_value; } + EIGEN_STRONG_INLINE Scalar value() const { eigen_internal_assert(m_id<m_innerSize); return m_value; } EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_lhsIter.outer() : m_id; } EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_lhsIter.outer(); } @@ -297,8 +299,7 @@ enum { CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, - // Expose storage order of the sparse expression - Flags = (XprType::Flags & ~RowMajorBit) | (int(Lhs::Flags)&RowMajorBit) + Flags = XprType::Flags }; explicit binary_evaluator(const XprType& xpr) @@ -322,26 +323,98 @@ const XprType &m_expr; }; +template<typename T, + typename LhsKind = typename evaluator_traits<typename T::Lhs>::Kind, + typename RhsKind = typename evaluator_traits<typename T::Rhs>::Kind, + typename LhsScalar = typename traits<typename T::Lhs>::Scalar, + typename RhsScalar = typename traits<typename T::Rhs>::Scalar> struct sparse_conjunction_evaluator; + // "sparse .* sparse" -template<typename T, typename Lhs, typename Rhs> -struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IteratorBased, IteratorBased> - : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +template<typename T1, typename T2, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs>, IteratorBased, IteratorBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; +// "dense .* sparse" +template<typename T1, typename T2, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs>, IndexBased, IteratorBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; +// "sparse .* dense" +template<typename T1, typename T2, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs>, IteratorBased, IndexBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_product_op<T1,T2>, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; + +// "sparse ./ dense" +template<typename T1, typename T2, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_quotient_op<T1,T2>, Lhs, Rhs>, IteratorBased, IndexBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_quotient_op<T1,T2>, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_quotient_op<T1,T2>, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; + +// "sparse && sparse" +template<typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs>, IteratorBased, IteratorBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; +// "dense && sparse" +template<typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs>, IndexBased, IteratorBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; +// "sparse && dense" +template<typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs>, IteratorBased, IndexBased> + : sparse_conjunction_evaluator<CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<scalar_boolean_and_op, Lhs, Rhs> XprType; + typedef sparse_conjunction_evaluator<XprType> Base; + explicit binary_evaluator(const XprType& xpr) : Base(xpr) {} +}; + +// "sparse ^ sparse" +template<typename XprType> +struct sparse_conjunction_evaluator<XprType, IteratorBased, IteratorBased> + : evaluator_base<XprType> { protected: - typedef scalar_product_op<T> BinaryOp; - typedef typename evaluator<Lhs>::InnerIterator LhsIterator; - typedef typename evaluator<Rhs>::InnerIterator RhsIterator; - typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef typename evaluator<LhsArg>::InnerIterator LhsIterator; + typedef typename evaluator<RhsArg>::InnerIterator RhsIterator; typedef typename XprType::StorageIndex StorageIndex; typedef typename traits<XprType>::Scalar Scalar; public: - class ReverseInnerIterator; class InnerIterator { public: - EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + EIGEN_STRONG_INLINE InnerIterator(const sparse_conjunction_evaluator& aEval, Index outer) : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor) { while (m_lhsIter && m_rhsIter && (m_lhsIter.index() != m_rhsIter.index())) @@ -370,6 +443,7 @@ EIGEN_STRONG_INLINE Scalar value() const { return m_functor(m_lhsIter.value(), m_rhsIter.value()); } EIGEN_STRONG_INLINE StorageIndex index() const { return m_lhsIter.index(); } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return m_lhsIter.row(); } EIGEN_STRONG_INLINE Index col() const { return m_lhsIter.col(); } @@ -383,11 +457,11 @@ enum { - CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + CoeffReadCost = evaluator<LhsArg>::CoeffReadCost + evaluator<RhsArg>::CoeffReadCost + functor_traits<BinaryOp>::Cost, Flags = XprType::Flags }; - explicit binary_evaluator(const XprType& xpr) + explicit sparse_conjunction_evaluator(const XprType& xpr) : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()) @@ -402,32 +476,32 @@ protected: const BinaryOp m_functor; - evaluator<Lhs> m_lhsImpl; - evaluator<Rhs> m_rhsImpl; + evaluator<LhsArg> m_lhsImpl; + evaluator<RhsArg> m_rhsImpl; }; -// "dense .* sparse" -template<typename T, typename Lhs, typename Rhs> -struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IndexBased, IteratorBased> - : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +// "dense ^ sparse" +template<typename XprType> +struct sparse_conjunction_evaluator<XprType, IndexBased, IteratorBased> + : evaluator_base<XprType> { protected: - typedef scalar_product_op<T> BinaryOp; - typedef evaluator<Lhs> LhsEvaluator; - typedef typename evaluator<Rhs>::InnerIterator RhsIterator; - typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef evaluator<LhsArg> LhsEvaluator; + typedef typename evaluator<RhsArg>::InnerIterator RhsIterator; typedef typename XprType::StorageIndex StorageIndex; typedef typename traits<XprType>::Scalar Scalar; public: - class ReverseInnerIterator; class InnerIterator { - enum { IsRowMajor = (int(Rhs::Flags)&RowMajorBit)==RowMajorBit }; + enum { IsRowMajor = (int(RhsArg::Flags)&RowMajorBit)==RowMajorBit }; public: - EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + EIGEN_STRONG_INLINE InnerIterator(const sparse_conjunction_evaluator& aEval, Index outer) : m_lhsEval(aEval.m_lhsImpl), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_outer(outer) {} @@ -441,6 +515,7 @@ { return m_functor(m_lhsEval.coeff(IsRowMajor?m_outer:m_rhsIter.index(),IsRowMajor?m_rhsIter.index():m_outer), m_rhsIter.value()); } EIGEN_STRONG_INLINE StorageIndex index() const { return m_rhsIter.index(); } + EIGEN_STRONG_INLINE Index outer() const { return m_rhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return m_rhsIter.row(); } EIGEN_STRONG_INLINE Index col() const { return m_rhsIter.col(); } @@ -455,12 +530,11 @@ enum { - CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, - // Expose storage order of the sparse expression - Flags = (XprType::Flags & ~RowMajorBit) | (int(Rhs::Flags)&RowMajorBit) + CoeffReadCost = evaluator<LhsArg>::CoeffReadCost + evaluator<RhsArg>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags }; - explicit binary_evaluator(const XprType& xpr) + explicit sparse_conjunction_evaluator(const XprType& xpr) : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()) @@ -475,32 +549,32 @@ protected: const BinaryOp m_functor; - evaluator<Lhs> m_lhsImpl; - evaluator<Rhs> m_rhsImpl; + evaluator<LhsArg> m_lhsImpl; + evaluator<RhsArg> m_rhsImpl; }; -// "sparse .* dense" -template<typename T, typename Lhs, typename Rhs> -struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IteratorBased, IndexBased> - : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +// "sparse ^ dense" +template<typename XprType> +struct sparse_conjunction_evaluator<XprType, IteratorBased, IndexBased> + : evaluator_base<XprType> { protected: - typedef scalar_product_op<T> BinaryOp; - typedef typename evaluator<Lhs>::InnerIterator LhsIterator; - typedef evaluator<Rhs> RhsEvaluator; - typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + typedef typename XprType::Functor BinaryOp; + typedef typename XprType::Lhs LhsArg; + typedef typename XprType::Rhs RhsArg; + typedef typename evaluator<LhsArg>::InnerIterator LhsIterator; + typedef evaluator<RhsArg> RhsEvaluator; typedef typename XprType::StorageIndex StorageIndex; typedef typename traits<XprType>::Scalar Scalar; public: - class ReverseInnerIterator; class InnerIterator { - enum { IsRowMajor = (int(Lhs::Flags)&RowMajorBit)==RowMajorBit }; + enum { IsRowMajor = (int(LhsArg::Flags)&RowMajorBit)==RowMajorBit }; public: - EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + EIGEN_STRONG_INLINE InnerIterator(const sparse_conjunction_evaluator& aEval, Index outer) : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsEval(aEval.m_rhsImpl), m_functor(aEval.m_functor), m_outer(outer) {} @@ -515,6 +589,7 @@ m_rhsEval.coeff(IsRowMajor?m_outer:m_lhsIter.index(),IsRowMajor?m_lhsIter.index():m_outer)); } EIGEN_STRONG_INLINE StorageIndex index() const { return m_lhsIter.index(); } + EIGEN_STRONG_INLINE Index outer() const { return m_lhsIter.outer(); } EIGEN_STRONG_INLINE Index row() const { return m_lhsIter.row(); } EIGEN_STRONG_INLINE Index col() const { return m_lhsIter.col(); } @@ -522,19 +597,18 @@ protected: LhsIterator m_lhsIter; - const evaluator<Rhs> &m_rhsEval; + const evaluator<RhsArg> &m_rhsEval; const BinaryOp& m_functor; const Index m_outer; }; enum { - CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, - // Expose storage order of the sparse expression - Flags = (XprType::Flags & ~RowMajorBit) | (int(Lhs::Flags)&RowMajorBit) + CoeffReadCost = evaluator<LhsArg>::CoeffReadCost + evaluator<RhsArg>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags }; - explicit binary_evaluator(const XprType& xpr) + explicit sparse_conjunction_evaluator(const XprType& xpr) : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()) @@ -549,8 +623,8 @@ protected: const BinaryOp m_functor; - evaluator<Lhs> m_lhsImpl; - evaluator<Rhs> m_rhsImpl; + evaluator<LhsArg> m_lhsImpl; + evaluator<RhsArg> m_rhsImpl; }; } @@ -561,6 +635,22 @@ template<typename Derived> template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator+=(const EigenBase<OtherDerived> &other) +{ + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator-=(const EigenBase<OtherDerived> &other) +{ + call_assignment(derived(), other.derived(), internal::assign_op<Scalar,typename OtherDerived::Scalar>()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> EIGEN_STRONG_INLINE Derived & SparseMatrixBase<Derived>::operator-=(const SparseMatrixBase<OtherDerived> &other) { @@ -579,7 +669,7 @@ template<typename OtherDerived> Derived& SparseMatrixBase<Derived>::operator+=(const DiagonalBase<OtherDerived>& other) { - call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op<Scalar>()); + call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -587,7 +677,7 @@ template<typename OtherDerived> Derived& SparseMatrixBase<Derived>::operator-=(const DiagonalBase<OtherDerived>& other) { - call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op<Scalar,typename OtherDerived::Scalar>()); return derived(); } @@ -600,31 +690,31 @@ } template<typename DenseDerived, typename SparseDerived> -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar,typename SparseDerived::Scalar>, const DenseDerived, const SparseDerived> operator+(const MatrixBase<DenseDerived> &a, const SparseMatrixBase<SparseDerived> &b) { - return CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived()); + return CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar,typename SparseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived()); } template<typename SparseDerived, typename DenseDerived> -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename SparseDerived::Scalar,typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived> operator+(const SparseMatrixBase<SparseDerived> &a, const MatrixBase<DenseDerived> &b) { - return CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived()); + return CwiseBinaryOp<internal::scalar_sum_op<typename SparseDerived::Scalar,typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived()); } template<typename DenseDerived, typename SparseDerived> -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar,typename SparseDerived::Scalar>, const DenseDerived, const SparseDerived> operator-(const MatrixBase<DenseDerived> &a, const SparseMatrixBase<SparseDerived> &b) { - return CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived()); + return CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar,typename SparseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived()); } template<typename SparseDerived, typename DenseDerived> -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename SparseDerived::Scalar,typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived> operator-(const SparseMatrixBase<SparseDerived> &a, const MatrixBase<DenseDerived> &b) { - return CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived()); + return CwiseBinaryOp<internal::scalar_difference_op<typename SparseDerived::Scalar,typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived()); } } // end namespace Eigen
diff --git a/Eigen/src/SparseCore/SparseCwiseUnaryOp.h b/Eigen/src/SparseCore/SparseCwiseUnaryOp.h index fe4a971..ea79737 100644 --- a/Eigen/src/SparseCore/SparseCwiseUnaryOp.h +++ b/Eigen/src/SparseCore/SparseCwiseUnaryOp.h
@@ -22,7 +22,6 @@ typedef CwiseUnaryOp<UnaryOp, ArgType> XprType; class InnerIterator; -// class ReverseInnerIterator; enum { CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<UnaryOp>::Cost, @@ -41,7 +40,6 @@ protected: typedef typename evaluator<ArgType>::InnerIterator EvalIterator; -// typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; const UnaryOp m_functor; evaluator<ArgType> m_argImpl; @@ -70,33 +68,6 @@ Scalar& valueRef(); }; -// template<typename UnaryOp, typename ArgType> -// class unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::ReverseInnerIterator -// : public unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalReverseIterator -// { -// typedef typename XprType::Scalar Scalar; -// typedef typename unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalReverseIterator Base; -// public: -// -// EIGEN_STRONG_INLINE ReverseInnerIterator(const XprType& unaryOp, typename XprType::Index outer) -// : Base(unaryOp.derived().nestedExpression(),outer), m_functor(unaryOp.derived().functor()) -// {} -// -// EIGEN_STRONG_INLINE ReverseInnerIterator& operator--() -// { Base::operator--(); return *this; } -// -// EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } -// -// protected: -// const UnaryOp m_functor; -// private: -// Scalar& valueRef(); -// }; - - - - - template<typename ViewOp, typename ArgType> struct unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased> : public evaluator_base<CwiseUnaryView<ViewOp,ArgType> > @@ -105,7 +76,6 @@ typedef CwiseUnaryView<ViewOp, ArgType> XprType; class InnerIterator; - class ReverseInnerIterator; enum { CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<ViewOp>::Cost, @@ -120,7 +90,6 @@ protected: typedef typename evaluator<ArgType>::InnerIterator EvalIterator; -// typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; const ViewOp m_functor; evaluator<ArgType> m_argImpl; @@ -148,37 +117,16 @@ const ViewOp m_functor; }; -// template<typename ViewOp, typename ArgType> -// class unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::ReverseInnerIterator -// : public unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalReverseIterator -// { -// typedef typename XprType::Scalar Scalar; -// typedef typename unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalReverseIterator Base; -// public: -// -// EIGEN_STRONG_INLINE ReverseInnerIterator(const XprType& unaryOp, typename XprType::Index outer) -// : Base(unaryOp.derived().nestedExpression(),outer), m_functor(unaryOp.derived().functor()) -// {} -// -// EIGEN_STRONG_INLINE ReverseInnerIterator& operator--() -// { Base::operator--(); return *this; } -// -// EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } -// EIGEN_STRONG_INLINE Scalar& valueRef() { return m_functor(Base::valueRef()); } -// -// protected: -// const ViewOp m_functor; -// }; - - } // end namespace internal template<typename Derived> EIGEN_STRONG_INLINE Derived& SparseMatrixBase<Derived>::operator*=(const Scalar& other) { + typedef typename internal::evaluator<Derived>::InnerIterator EvalIterator; + internal::evaluator<Derived> thisEval(derived()); for (Index j=0; j<outerSize(); ++j) - for (typename Derived::InnerIterator i(derived(),j); i; ++i) + for (EvalIterator i(thisEval,j); i; ++i) i.valueRef() *= other; return derived(); } @@ -187,8 +135,10 @@ EIGEN_STRONG_INLINE Derived& SparseMatrixBase<Derived>::operator/=(const Scalar& other) { + typedef typename internal::evaluator<Derived>::InnerIterator EvalIterator; + internal::evaluator<Derived> thisEval(derived()); for (Index j=0; j<outerSize(); ++j) - for (typename Derived::InnerIterator i(derived(),j); i; ++i) + for (EvalIterator i(thisEval,j); i; ++i) i.valueRef() /= other; return derived(); }
diff --git a/Eigen/src/SparseCore/SparseDenseProduct.h b/Eigen/src/SparseCore/SparseDenseProduct.h index c9da8a2..f005a18 100644 --- a/Eigen/src/SparseCore/SparseDenseProduct.h +++ b/Eigen/src/SparseCore/SparseDenseProduct.h
@@ -72,30 +72,33 @@ }; // FIXME: what is the purpose of the following specialization? Is it for the BlockedSparse format? -template<typename T1, typename T2/*, int _Options, typename _StrideType*/> -struct scalar_product_traits<T1, Ref<T2/*, _Options, _StrideType*/> > -{ - enum { - Defined = 1 - }; - typedef typename CwiseUnaryOp<scalar_multiple2_op<T1, typename T2::Scalar>, T2>::PlainObject ReturnType; -}; +// -> let's disable it for now as it is conflicting with generic scalar*matrix and matrix*scalar operators +// template<typename T1, typename T2/*, int _Options, typename _StrideType*/> +// struct ScalarBinaryOpTraits<T1, Ref<T2/*, _Options, _StrideType*/> > +// { +// enum { +// Defined = 1 +// }; +// typedef typename CwiseUnaryOp<scalar_multiple2_op<T1, typename T2::Scalar>, T2>::PlainObject ReturnType; +// }; + template<typename SparseLhsType, typename DenseRhsType, typename DenseResType, typename AlphaType> struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, AlphaType, ColMajor, true> { typedef typename internal::remove_all<SparseLhsType>::type Lhs; typedef typename internal::remove_all<DenseRhsType>::type Rhs; typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; + typedef evaluator<Lhs> LhsEval; + typedef typename LhsEval::InnerIterator LhsInnerIterator; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) { - evaluator<Lhs> lhsEval(lhs); + LhsEval lhsEval(lhs); for(Index c=0; c<rhs.cols(); ++c) { for(Index j=0; j<lhs.outerSize(); ++j) { // typename Res::Scalar rhs_j = alpha * rhs.coeff(j,c); - typename internal::scalar_product_traits<AlphaType, typename Rhs::Scalar>::ReturnType rhs_j(alpha * rhs.coeff(j,c)); + typename ScalarBinaryOpTraits<AlphaType, typename Rhs::Scalar>::ReturnType rhs_j(alpha * rhs.coeff(j,c)); for(LhsInnerIterator it(lhsEval,j); it ;++it) res.coeffRef(it.index(),c) += it.value() * rhs_j; } @@ -109,16 +112,37 @@ typedef typename internal::remove_all<SparseLhsType>::type Lhs; typedef typename internal::remove_all<DenseRhsType>::type Rhs; typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; + typedef evaluator<Lhs> LhsEval; + typedef typename LhsEval::InnerIterator LhsInnerIterator; static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) { - evaluator<Lhs> lhsEval(lhs); - for(Index j=0; j<lhs.outerSize(); ++j) + Index n = lhs.rows(); + LhsEval lhsEval(lhs); + +#ifdef EIGEN_HAS_OPENMP + Eigen::initParallel(); + Index threads = Eigen::nbThreads(); + // This 20000 threshold has been found experimentally on 2D and 3D Poisson problems. + // It basically represents the minimal amount of work to be done to be worth it. + if(threads>1 && lhsEval.nonZerosEstimate()*rhs.cols() > 20000) { - typename Res::RowXpr res_j(res.row(j)); - for(LhsInnerIterator it(lhsEval,j); it ;++it) - res_j += (alpha*it.value()) * rhs.row(it.index()); + #pragma omp parallel for schedule(dynamic,(n+threads*4-1)/(threads*4)) num_threads(threads) + for(Index i=0; i<n; ++i) + processRow(lhsEval,rhs,res,alpha,i); } + else +#endif + { + for(Index i=0; i<n; ++i) + processRow(lhsEval, rhs, res, alpha, i); + } + } + + static void processRow(const LhsEval& lhsEval, const DenseRhsType& rhs, Res& res, const typename Res::Scalar& alpha, Index i) + { + typename Res::RowXpr res_i(res.row(i)); + for(LhsInnerIterator it(lhsEval,i); it ;++it) + res_i += (alpha*it.value()) * rhs.row(it.index()); } };
diff --git a/Eigen/src/SparseCore/SparseDiagonalProduct.h b/Eigen/src/SparseCore/SparseDiagonalProduct.h index e4af49e..941c03b 100644 --- a/Eigen/src/SparseCore/SparseDiagonalProduct.h +++ b/Eigen/src/SparseCore/SparseDiagonalProduct.h
@@ -80,6 +80,8 @@ sparse_diagonal_product_evaluator(const SparseXprType &sparseXpr, const DiagonalCoeffType &diagCoeff) : m_sparseXprImpl(sparseXpr), m_diagCoeffImpl(diagCoeff) {} + + Index nonZerosEstimate() const { return m_sparseXprImpl.nonZerosEstimate(); } protected: evaluator<SparseXprType> m_sparseXprImpl; @@ -121,6 +123,8 @@ sparse_diagonal_product_evaluator(const SparseXprType &sparseXpr, const DiagCoeffType &diagCoeff) : m_sparseXprEval(sparseXpr), m_diagCoeffNested(diagCoeff) {} + + Index nonZerosEstimate() const { return m_sparseXprEval.nonZerosEstimate(); } protected: evaluator<SparseXprType> m_sparseXprEval;
diff --git a/Eigen/src/SparseCore/SparseMap.h b/Eigen/src/SparseCore/SparseMap.h index eb241c3..f99be33 100644 --- a/Eigen/src/SparseCore/SparseMap.h +++ b/Eigen/src/SparseCore/SparseMap.h
@@ -166,12 +166,17 @@ using Base::innerIndexPtr; using Base::outerIndexPtr; using Base::innerNonZeroPtr; - inline Scalar* valuePtr() { return Base::m_values; } + /** \copydoc SparseMatrix::valuePtr */ + inline Scalar* valuePtr() { return Base::m_values; } + /** \copydoc SparseMatrix::innerIndexPtr */ inline StorageIndex* innerIndexPtr() { return Base::m_innerIndices; } + /** \copydoc SparseMatrix::outerIndexPtr */ inline StorageIndex* outerIndexPtr() { return Base::m_outerIndex; } + /** \copydoc SparseMatrix::innerNonZeroPtr */ inline StorageIndex* innerNonZeroPtr() { return Base::m_innerNonZeros; } //---------------------------------------- + /** \copydoc SparseMatrix::coeffRef */ inline Scalar& coeffRef(Index row, Index col) { const Index outer = IsRowMajor ? row : col; @@ -181,14 +186,14 @@ Index end = Base::isCompressed() ? Base::m_outerIndex[outer+1] : start + Base::m_innerNonZeros[outer]; eigen_assert(end>=start && "you probably called coeffRef on a non finalized matrix"); eigen_assert(end>start && "coeffRef cannot be called on a zero coefficient"); - Index* r = std::lower_bound(&Base::m_innerIndices[start],&Base::m_innerIndices[end],inner); + StorageIndex* r = std::lower_bound(&Base::m_innerIndices[start],&Base::m_innerIndices[end],inner); const Index id = r - &Base::m_innerIndices[0]; eigen_assert((*r==inner) && (id<end) && "coeffRef cannot be called on a zero coefficient"); return const_cast<Scalar*>(Base::m_values)[id]; } inline SparseMapBase(Index rows, Index cols, Index nnz, StorageIndex* outerIndexPtr, StorageIndex* innerIndexPtr, - Scalar* valuePtr, StorageIndex* innerNonZerosPtr = 0) + Scalar* valuePtr, StorageIndex* innerNonZerosPtr = 0) : Base(rows, cols, nnz, outerIndexPtr, innerIndexPtr, valuePtr, innerNonZerosPtr) {} @@ -233,13 +238,15 @@ * stored as a sparse format as defined by the pointers \a outerIndexPtr, \a innerIndexPtr, and \a valuePtr. * If the optional parameter \a innerNonZerosPtr is the null pointer, then a standard compressed format is assumed. * + * This constructor is available only if \c SparseMatrixType is non-const. + * * More details on the expected storage schemes are given in the \ref TutorialSparse "manual pages". */ inline Map(Index rows, Index cols, Index nnz, StorageIndex* outerIndexPtr, StorageIndex* innerIndexPtr, Scalar* valuePtr, StorageIndex* innerNonZerosPtr = 0) : Base(rows, cols, nnz, outerIndexPtr, innerIndexPtr, valuePtr, innerNonZerosPtr) {} - +#ifndef EIGEN_PARSED_BY_DOXYGEN /** Empty destructor */ inline ~Map() {} }; @@ -254,7 +261,12 @@ enum { IsRowMajor = Base::IsRowMajor }; public: - +#endif + /** This is the const version of the above constructor. + * + * This constructor is available only if \c SparseMatrixType is const, e.g.: + * \code Map<const SparseMatrix<double> > \endcode + */ inline Map(Index rows, Index cols, Index nnz, const StorageIndex* outerIndexPtr, const StorageIndex* innerIndexPtr, const Scalar* valuePtr, const StorageIndex* innerNonZerosPtr = 0) : Base(rows, cols, nnz, outerIndexPtr, innerIndexPtr, valuePtr, innerNonZerosPtr)
diff --git a/Eigen/src/SparseCore/SparseMatrix.h b/Eigen/src/SparseCore/SparseMatrix.h index 760e151..63dd1cc 100644 --- a/Eigen/src/SparseCore/SparseMatrix.h +++ b/Eigen/src/SparseCore/SparseMatrix.h
@@ -21,7 +21,7 @@ * This class implements a more versatile variants of the common \em compressed row/column storage format. * Each colmun's (resp. row) non zeros are stored as a pair of value with associated row (resp. colmiun) index. * All the non zeros are stored in a single large buffer. Unlike the \em compressed format, there might be extra - * space inbetween the nonzeros of two successive colmuns (resp. rows) such that insertion of new non-zero + * space in between the nonzeros of two successive colmuns (resp. rows) such that insertion of new non-zero * can be done with limited memory reallocation and copies. * * A call to the function makeCompressed() turns the matrix into the standard \em compressed format @@ -32,18 +32,22 @@ * \tparam _Scalar the scalar type, i.e. the type of the coefficients * \tparam _Options Union of bit flags controlling the storage scheme. Currently the only possibility * is ColMajor or RowMajor. The default is 0 which means column-major. - * \tparam _Index the type of the indices. It has to be a \b signed type (e.g., short, int, std::ptrdiff_t). Default is \c int. + * \tparam _StorageIndex the type of the indices. It has to be a \b signed type (e.g., short, int, std::ptrdiff_t). Default is \c int. + * + * \warning In %Eigen 3.2, the undocumented type \c SparseMatrix::Index was improperly defined as the storage index type (e.g., int), + * whereas it is now (starting from %Eigen 3.3) deprecated and always defined as Eigen::Index. + * Codes making use of \c SparseMatrix::Index, might thus likely have to be changed to use \c SparseMatrix::StorageIndex instead. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_SPARSEMATRIX_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_SPARSEMATRIX_PLUGIN. */ namespace internal { -template<typename _Scalar, int _Options, typename _Index> -struct traits<SparseMatrix<_Scalar, _Options, _Index> > +template<typename _Scalar, int _Options, typename _StorageIndex> +struct traits<SparseMatrix<_Scalar, _Options, _StorageIndex> > { typedef _Scalar Scalar; - typedef _Index StorageIndex; + typedef _StorageIndex StorageIndex; typedef Sparse StorageKind; typedef MatrixXpr XprKind; enum { @@ -56,16 +60,16 @@ }; }; -template<typename _Scalar, int _Options, typename _Index, int DiagIndex> -struct traits<Diagonal<SparseMatrix<_Scalar, _Options, _Index>, DiagIndex> > +template<typename _Scalar, int _Options, typename _StorageIndex, int DiagIndex> +struct traits<Diagonal<SparseMatrix<_Scalar, _Options, _StorageIndex>, DiagIndex> > { - typedef SparseMatrix<_Scalar, _Options, _Index> MatrixType; + typedef SparseMatrix<_Scalar, _Options, _StorageIndex> MatrixType; typedef typename ref_selector<MatrixType>::type MatrixTypeNested; typedef typename remove_reference<MatrixTypeNested>::type _MatrixTypeNested; typedef _Scalar Scalar; typedef Dense StorageKind; - typedef _Index StorageIndex; + typedef _StorageIndex StorageIndex; typedef MatrixXpr XprKind; enum { @@ -77,9 +81,9 @@ }; }; -template<typename _Scalar, int _Options, typename _Index, int DiagIndex> -struct traits<Diagonal<const SparseMatrix<_Scalar, _Options, _Index>, DiagIndex> > - : public traits<Diagonal<SparseMatrix<_Scalar, _Options, _Index>, DiagIndex> > +template<typename _Scalar, int _Options, typename _StorageIndex, int DiagIndex> +struct traits<Diagonal<const SparseMatrix<_Scalar, _Options, _StorageIndex>, DiagIndex> > + : public traits<Diagonal<SparseMatrix<_Scalar, _Options, _StorageIndex>, DiagIndex> > { enum { Flags = 0 @@ -88,12 +92,15 @@ } // end namespace internal -template<typename _Scalar, int _Options, typename _Index> +template<typename _Scalar, int _Options, typename _StorageIndex> class SparseMatrix - : public SparseCompressedBase<SparseMatrix<_Scalar, _Options, _Index> > + : public SparseCompressedBase<SparseMatrix<_Scalar, _Options, _StorageIndex> > { typedef SparseCompressedBase<SparseMatrix> Base; using Base::convert_index; + friend class SparseVector<_Scalar,0,_StorageIndex>; + template<typename, typename, typename, typename, typename> + friend struct internal::Assignment; public: using Base::isCompressed; using Base::nonZeros; @@ -440,7 +447,7 @@ template<typename InputIterators,typename DupFunctor> void setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func); - void sumupDuplicates() { collapseDuplicates(internal::scalar_sum_op<Scalar>()); } + void sumupDuplicates() { collapseDuplicates(internal::scalar_sum_op<Scalar,Scalar>()); } template<typename DupFunctor> void collapseDuplicates(DupFunctor dup_func = DupFunctor()); @@ -497,8 +504,8 @@ m_innerNonZeros[i] = m_outerIndex[i+1] - m_outerIndex[i]; } } - - /** Suppresses all nonzeros which are \b much \b smaller \b than \a reference under the tolerence \a epsilon */ + + /** Suppresses all nonzeros which are \b much \b smaller \b than \a reference under the tolerance \a epsilon */ void prune(const Scalar& reference, const RealScalar& epsilon = NumTraits<RealScalar>::dummy_precision()) { prune(default_prunning_func(reference,epsilon)); @@ -599,9 +606,9 @@ m_outerIndex = newOuterIndex; if (outerChange > 0) { - StorageIndex last = m_outerSize == 0 ? 0 : m_outerIndex[m_outerSize]; + StorageIndex lastIdx = m_outerSize == 0 ? 0 : m_outerIndex[m_outerSize]; for(Index i=m_outerSize; i<m_outerSize+outerChange+1; i++) - m_outerIndex[i] = last; + m_outerIndex[i] = lastIdx; } m_outerSize += outerChange; } @@ -785,30 +792,38 @@ EIGEN_DBG_SPARSE( s << "Nonzero entries:\n"; if(m.isCompressed()) + { for (Index i=0; i<m.nonZeros(); ++i) s << "(" << m.m_data.value(i) << "," << m.m_data.index(i) << ") "; + } else + { for (Index i=0; i<m.outerSize(); ++i) { Index p = m.m_outerIndex[i]; Index pe = m.m_outerIndex[i]+m.m_innerNonZeros[i]; Index k=p; - for (; k<pe; ++k) + for (; k<pe; ++k) { s << "(" << m.m_data.value(k) << "," << m.m_data.index(k) << ") "; - for (; k<m.m_outerIndex[i+1]; ++k) + } + for (; k<m.m_outerIndex[i+1]; ++k) { s << "(_,_) "; + } } + } s << std::endl; s << std::endl; s << "Outer pointers:\n"; - for (Index i=0; i<m.outerSize(); ++i) + for (Index i=0; i<m.outerSize(); ++i) { s << m.m_outerIndex[i] << " "; + } s << " $" << std::endl; if(!m.isCompressed()) { s << "Inner non zeros:\n"; - for (Index i=0; i<m.outerSize(); ++i) + for (Index i=0; i<m.outerSize(); ++i) { s << m.m_innerNonZeros[i] << " "; + } s << " $" << std::endl; } s << std::endl; @@ -880,7 +895,114 @@ Index p = m_outerIndex[outer] + m_innerNonZeros[outer]++; m_data.index(p) = convert_index(inner); - return (m_data.value(p) = 0); + return (m_data.value(p) = Scalar(0)); + } +protected: + struct IndexPosPair { + IndexPosPair(Index a_i, Index a_p) : i(a_i), p(a_p) {} + Index i; + Index p; + }; + + /** \internal assign \a diagXpr to the diagonal of \c *this + * There are different strategies: + * 1 - if *this is overwritten (Func==assign_op) or *this is empty, then we can work treat *this as a dense vector expression. + * 2 - otherwise, for each diagonal coeff, + * 2.a - if it already exists, then we update it, + * 2.b - otherwise, if *this is uncompressed and that the current inner-vector has empty room for at least 1 element, then we perform an in-place insertion. + * 2.c - otherwise, we'll have to reallocate and copy everything, so instead of doing so for each new element, it is recorded in a std::vector. + * 3 - at the end, if some entries failed to be inserted in-place, then we alloc a new buffer, copy each chunk at the right position, and insert the new elements. + * + * TODO: some piece of code could be isolated and reused for a general in-place update strategy. + * TODO: if we start to defer the insertion of some elements (i.e., case 2.c executed once), + * then it *might* be better to disable case 2.b since they will have to be copied anyway. + */ + template<typename DiagXpr, typename Func> + void assignDiagonal(const DiagXpr diagXpr, const Func& assignFunc) + { + Index n = diagXpr.size(); + + const bool overwrite = internal::is_same<Func, internal::assign_op<Scalar,Scalar> >::value; + if(overwrite) + { + if((this->rows()!=n) || (this->cols()!=n)) + this->resize(n, n); + } + + if(m_data.size()==0 || overwrite) + { + typedef Array<StorageIndex,Dynamic,1> ArrayXI; + this->makeCompressed(); + this->resizeNonZeros(n); + Eigen::Map<ArrayXI>(this->innerIndexPtr(), n).setLinSpaced(0,StorageIndex(n)-1); + Eigen::Map<ArrayXI>(this->outerIndexPtr(), n+1).setLinSpaced(0,StorageIndex(n)); + Eigen::Map<Array<Scalar,Dynamic,1> > values = this->coeffs(); + values.setZero(); + internal::call_assignment_no_alias(values, diagXpr, assignFunc); + } + else + { + bool isComp = isCompressed(); + internal::evaluator<DiagXpr> diaEval(diagXpr); + std::vector<IndexPosPair> newEntries; + + // 1 - try in-place update and record insertion failures + for(Index i = 0; i<n; ++i) + { + internal::LowerBoundIndex lb = this->lower_bound(i,i); + Index p = lb.value; + if(lb.found) + { + // the coeff already exists + assignFunc.assignCoeff(m_data.value(p), diaEval.coeff(i)); + } + else if((!isComp) && m_innerNonZeros[i] < (m_outerIndex[i+1]-m_outerIndex[i])) + { + // non compressed mode with local room for inserting one element + m_data.moveChunk(p, p+1, m_outerIndex[i]+m_innerNonZeros[i]-p); + m_innerNonZeros[i]++; + m_data.value(p) = Scalar(0); + m_data.index(p) = StorageIndex(i); + assignFunc.assignCoeff(m_data.value(p), diaEval.coeff(i)); + } + else + { + // defer insertion + newEntries.push_back(IndexPosPair(i,p)); + } + } + // 2 - insert deferred entries + Index n_entries = Index(newEntries.size()); + if(n_entries>0) + { + Storage newData(m_data.size()+n_entries); + Index prev_p = 0; + Index prev_i = 0; + for(Index k=0; k<n_entries;++k) + { + Index i = newEntries[k].i; + Index p = newEntries[k].p; + internal::smart_copy(m_data.valuePtr()+prev_p, m_data.valuePtr()+p, newData.valuePtr()+prev_p+k); + internal::smart_copy(m_data.indexPtr()+prev_p, m_data.indexPtr()+p, newData.indexPtr()+prev_p+k); + for(Index j=prev_i;j<i;++j) + m_outerIndex[j+1] += k; + if(!isComp) + m_innerNonZeros[i]++; + prev_p = p; + prev_i = i; + newData.value(p+k) = Scalar(0); + newData.index(p+k) = StorageIndex(i); + assignFunc.assignCoeff(newData.value(p+k), diaEval.coeff(i)); + } + { + internal::smart_copy(m_data.valuePtr()+prev_p, m_data.valuePtr()+m_data.size(), newData.valuePtr()+prev_p+n_entries); + internal::smart_copy(m_data.indexPtr()+prev_p, m_data.indexPtr()+m_data.size(), newData.indexPtr()+prev_p+n_entries); + for(Index j=prev_i+1;j<=m_outerSize;++j) + m_outerIndex[j] += n_entries; + } + m_data.swap(newData); + } + } } private: @@ -973,13 +1095,13 @@ * * \warning The list of triplets is read multiple times (at least twice). Therefore, it is not recommended to define * an abstract iterator over a complex data-structure that would be expensive to evaluate. The triplets should rather - * be explicitely stored into a std::vector for instance. + * be explicitly stored into a std::vector for instance. */ -template<typename Scalar, int _Options, typename _Index> +template<typename Scalar, int _Options, typename _StorageIndex> template<typename InputIterators> -void SparseMatrix<Scalar,_Options,_Index>::setFromTriplets(const InputIterators& begin, const InputIterators& end) +void SparseMatrix<Scalar,_Options,_StorageIndex>::setFromTriplets(const InputIterators& begin, const InputIterators& end) { - internal::set_from_triplets<InputIterators, SparseMatrix<Scalar,_Options,_Index> >(begin, end, *this, internal::scalar_sum_op<Scalar>()); + internal::set_from_triplets<InputIterators, SparseMatrix<Scalar,_Options,_StorageIndex> >(begin, end, *this, internal::scalar_sum_op<Scalar,Scalar>()); } /** The same as setFromTriplets but when duplicates are met the functor \a dup_func is applied: @@ -991,17 +1113,17 @@ * mat.setFromTriplets(triplets.begin(), triplets.end(), [] (const Scalar&,const Scalar &b) { return b; }); * \endcode */ -template<typename Scalar, int _Options, typename _Index> +template<typename Scalar, int _Options, typename _StorageIndex> template<typename InputIterators,typename DupFunctor> -void SparseMatrix<Scalar,_Options,_Index>::setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) +void SparseMatrix<Scalar,_Options,_StorageIndex>::setFromTriplets(const InputIterators& begin, const InputIterators& end, DupFunctor dup_func) { - internal::set_from_triplets<InputIterators, SparseMatrix<Scalar,_Options,_Index>, DupFunctor>(begin, end, *this, dup_func); + internal::set_from_triplets<InputIterators, SparseMatrix<Scalar,_Options,_StorageIndex>, DupFunctor>(begin, end, *this, dup_func); } /** \internal */ -template<typename Scalar, int _Options, typename _Index> +template<typename Scalar, int _Options, typename _StorageIndex> template<typename DupFunctor> -void SparseMatrix<Scalar,_Options,_Index>::collapseDuplicates(DupFunctor dup_func) +void SparseMatrix<Scalar,_Options,_StorageIndex>::collapseDuplicates(DupFunctor dup_func) { eigen_assert(!isCompressed()); // TODO, in practice we should be able to use m_innerNonZeros for that task @@ -1039,9 +1161,9 @@ m_data.resize(m_outerIndex[m_outerSize]); } -template<typename Scalar, int _Options, typename _Index> +template<typename Scalar, int _Options, typename _StorageIndex> template<typename OtherDerived> -EIGEN_DONT_INLINE SparseMatrix<Scalar,_Options,_Index>& SparseMatrix<Scalar,_Options,_Index>::operator=(const SparseMatrixBase<OtherDerived>& other) +EIGEN_DONT_INLINE SparseMatrix<Scalar,_Options,_StorageIndex>& SparseMatrix<Scalar,_Options,_StorageIndex>::operator=(const SparseMatrixBase<OtherDerived>& other) { EIGEN_STATIC_ASSERT((internal::is_same<Scalar, typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) @@ -1080,7 +1202,7 @@ IndexVector positions(dest.outerSize()); for (Index j=0; j<dest.outerSize(); ++j) { - Index tmp = dest.m_outerIndex[j]; + StorageIndex tmp = dest.m_outerIndex[j]; dest.m_outerIndex[j] = count; positions[j] = count; count += tmp; @@ -1112,8 +1234,8 @@ } } -template<typename _Scalar, int _Options, typename _Index> -typename SparseMatrix<_Scalar,_Options,_Index>::Scalar& SparseMatrix<_Scalar,_Options,_Index>::insert(Index row, Index col) +template<typename _Scalar, int _Options, typename _StorageIndex> +typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insert(Index row, Index col) { eigen_assert(row>=0 && row<rows() && col>=0 && col<cols()); @@ -1232,8 +1354,8 @@ return insertUncompressed(row,col); } -template<typename _Scalar, int _Options, typename _Index> -EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_Index>::Scalar& SparseMatrix<_Scalar,_Options,_Index>::insertUncompressed(Index row, Index col) +template<typename _Scalar, int _Options, typename _StorageIndex> +EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insertUncompressed(Index row, Index col) { eigen_assert(!isCompressed()); @@ -1261,11 +1383,11 @@ m_innerNonZeros[outer]++; m_data.index(p) = inner; - return (m_data.value(p) = 0); + return (m_data.value(p) = Scalar(0)); } -template<typename _Scalar, int _Options, typename _Index> -EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_Index>::Scalar& SparseMatrix<_Scalar,_Options,_Index>::insertCompressed(Index row, Index col) +template<typename _Scalar, int _Options, typename _StorageIndex> +EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_StorageIndex>::Scalar& SparseMatrix<_Scalar,_Options,_StorageIndex>::insertCompressed(Index row, Index col) { eigen_assert(isCompressed()); @@ -1288,11 +1410,11 @@ // starts with: [ 0 0 0 0 0 1 ...] and we are inserted in, e.g., // the 2nd inner vector... bool isLastVec = (!(previousOuter==-1 && m_data.size()!=0)) - && (size_t(m_outerIndex[outer+1]) == m_data.size()); + && (std::size_t(m_outerIndex[outer+1]) == m_data.size()); - size_t startId = m_outerIndex[outer]; - // FIXME let's make sure sizeof(long int) == sizeof(size_t) - size_t p = m_outerIndex[outer+1]; + std::size_t startId = m_outerIndex[outer]; + // FIXME let's make sure sizeof(long int) == sizeof(std::size_t) + std::size_t p = m_outerIndex[outer+1]; ++m_outerIndex[outer+1]; double reallocRatio = 1; @@ -1368,17 +1490,17 @@ } m_data.index(p) = inner; - return (m_data.value(p) = 0); + return (m_data.value(p) = Scalar(0)); } namespace internal { -template<typename _Scalar, int _Options, typename _Index> -struct evaluator<SparseMatrix<_Scalar,_Options,_Index> > - : evaluator<SparseCompressedBase<SparseMatrix<_Scalar,_Options,_Index> > > +template<typename _Scalar, int _Options, typename _StorageIndex> +struct evaluator<SparseMatrix<_Scalar,_Options,_StorageIndex> > + : evaluator<SparseCompressedBase<SparseMatrix<_Scalar,_Options,_StorageIndex> > > { - typedef evaluator<SparseCompressedBase<SparseMatrix<_Scalar,_Options,_Index> > > Base; - typedef SparseMatrix<_Scalar,_Options,_Index> SparseMatrixType; + typedef evaluator<SparseCompressedBase<SparseMatrix<_Scalar,_Options,_StorageIndex> > > Base; + typedef SparseMatrix<_Scalar,_Options,_StorageIndex> SparseMatrixType; evaluator() : Base() {} explicit evaluator(const SparseMatrixType &mat) : Base(mat) {} };
diff --git a/Eigen/src/SparseCore/SparseMatrixBase.h b/Eigen/src/SparseCore/SparseMatrixBase.h index 2a90f40..229449f 100644 --- a/Eigen/src/SparseCore/SparseMatrixBase.h +++ b/Eigen/src/SparseCore/SparseMatrixBase.h
@@ -21,16 +21,10 @@ * \tparam Derived is the derived type, e.g. a sparse matrix type, or an expression, etc. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_SPARSEMATRIXBASE_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_SPARSEMATRIXBASE_PLUGIN. */ template<typename Derived> class SparseMatrixBase -#ifndef EIGEN_PARSED_BY_DOXYGEN - : public internal::special_scalar_op_base<Derived,typename internal::traits<Derived>::Scalar, - typename NumTraits<typename internal::traits<Derived>::Scalar>::Real, - EigenBase<Derived> > -#else : public EigenBase<Derived> -#endif // not EIGEN_PARSED_BY_DOXYGEN { public: @@ -43,7 +37,11 @@ typedef typename internal::packet_traits<Scalar>::type PacketScalar; typedef typename internal::traits<Derived>::StorageKind StorageKind; + + /** The integer type used to \b store indices within a SparseMatrix. + * For a \c SparseMatrix<Scalar,Options,IndexType> it an alias of the third template parameter \c IndexType. */ typedef typename internal::traits<Derived>::StorageIndex StorageIndex; + typedef typename internal::add_const_on_value_type_if_arithmetic< typename internal::packet_traits<Scalar>::type >::type PacketReturnType; @@ -89,6 +87,11 @@ * we are dealing with a column-vector (if there is only one column) or with * a row-vector (if there is only one row). */ + NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2, + /**< This value is equal to Tensor::NumDimensions, i.e. 0 for scalars, 1 for vectors, + * and 2 for matrices. + */ + Flags = internal::traits<Derived>::Flags, /**< This stores expression \ref flags flags which may or may not be inherited by new expressions * constructed from this one. See the \ref flags "list of flags". @@ -142,12 +145,20 @@ inline Derived& const_cast_derived() const { return *static_cast<Derived*>(const_cast<SparseMatrixBase*>(this)); } - typedef internal::special_scalar_op_base<Derived, Scalar, RealScalar, EigenBase<Derived> > Base; - using Base::operator*; - using Base::operator/; + typedef EigenBase<Derived> Base; + #endif // not EIGEN_PARSED_BY_DOXYGEN #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::SparseMatrixBase +#ifdef EIGEN_PARSED_BY_DOXYGEN +#define EIGEN_DOC_UNARY_ADDONS(METHOD,OP) /** <p>This method does not change the sparsity of \c *this: the OP is applied to explicitly stored coefficients only. \sa SparseCompressedBase::coeffs() </p> */ +#define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL /** <p> \warning This method returns a read-only expression for any sparse matrices. \sa \ref TutorialSparse_SubMatrices "Sparse block operations" </p> */ +#define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) /** <p> \warning This method returns a read-write expression for COND sparse matrices only. Otherwise, the returned expression is read-only. \sa \ref TutorialSparse_SubMatrices "Sparse block operations" </p> */ +#else +#define EIGEN_DOC_UNARY_ADDONS(X,Y) +#define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +#define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) +#endif # include "../plugins/CommonCwiseUnaryOps.h" # include "../plugins/CommonCwiseBinaryOps.h" # include "../plugins/MatrixCwiseUnaryOps.h" @@ -156,8 +167,10 @@ # ifdef EIGEN_SPARSEMATRIXBASE_PLUGIN # include EIGEN_SPARSEMATRIXBASE_PLUGIN # endif -# undef EIGEN_CURRENT_STORAGE_BASE_CLASS #undef EIGEN_CURRENT_STORAGE_BASE_CLASS +#undef EIGEN_DOC_UNARY_ADDONS +#undef EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +#undef EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF /** \returns the number of rows. \sa cols() */ inline Index rows() const { return derived().rows(); } @@ -209,11 +222,12 @@ if (Flags&RowMajorBit) { - const Nested nm(m.derived()); + Nested nm(m.derived()); + internal::evaluator<NestedCleaned> thisEval(nm); for (Index row=0; row<nm.outerSize(); ++row) { Index col = 0; - for (typename NestedCleaned::InnerIterator it(nm.derived(), row); it; ++it) + for (typename internal::evaluator<NestedCleaned>::InnerIterator it(thisEval, row); it; ++it) { for ( ; col<it.index(); ++col) s << "0 "; @@ -227,10 +241,11 @@ } else { - const Nested nm(m.derived()); + Nested nm(m.derived()); + internal::evaluator<NestedCleaned> thisEval(nm); if (m.cols() == 1) { Index row = 0; - for (typename NestedCleaned::InnerIterator it(nm.derived(), 0); it; ++it) + for (typename internal::evaluator<NestedCleaned>::InnerIterator it(thisEval, 0); it; ++it) { for ( ; row<it.index(); ++row) s << "0" << std::endl; @@ -259,11 +274,16 @@ template<typename OtherDerived> Derived& operator-=(const DiagonalBase<OtherDerived>& other); + template<typename OtherDerived> + Derived& operator+=(const EigenBase<OtherDerived> &other); + template<typename OtherDerived> + Derived& operator-=(const EigenBase<OtherDerived> &other); + Derived& operator*=(const Scalar& other); Derived& operator/=(const Scalar& other); template<typename OtherDerived> struct CwiseProductDenseReturnType { - typedef CwiseBinaryOp<internal::scalar_product_op<typename internal::scalar_product_traits< + typedef CwiseBinaryOp<internal::scalar_product_op<typename ScalarBinaryOpTraits< typename internal::traits<Derived>::Scalar, typename internal::traits<OtherDerived>::Scalar >::ReturnType>, @@ -335,18 +355,6 @@ const ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(derived()); } const AdjointReturnType adjoint() const { return AdjointReturnType(transpose()); } - // inner-vector - typedef Block<Derived,IsRowMajor?1:Dynamic,IsRowMajor?Dynamic:1,true> InnerVectorReturnType; - typedef Block<const Derived,IsRowMajor?1:Dynamic,IsRowMajor?Dynamic:1,true> ConstInnerVectorReturnType; - InnerVectorReturnType innerVector(Index outer); - const ConstInnerVectorReturnType innerVector(Index outer) const; - - // set of inner-vectors - typedef Block<Derived,Dynamic,Dynamic,true> InnerVectorsReturnType; - typedef Block<const Derived,Dynamic,Dynamic,true> ConstInnerVectorsReturnType; - InnerVectorsReturnType innerVectors(Index outerStart, Index outerSize); - const ConstInnerVectorsReturnType innerVectors(Index outerStart, Index outerSize) const; - DenseMatrixType toDense() const { return DenseMatrixType(derived());
diff --git a/Eigen/src/SparseCore/SparseProduct.h b/Eigen/src/SparseCore/SparseProduct.h index cbd0db7..c495a73 100644 --- a/Eigen/src/SparseCore/SparseProduct.h +++ b/Eigen/src/SparseCore/SparseProduct.h
@@ -17,7 +17,7 @@ * The automatic pruning of the small values can be achieved by calling the pruned() function * in which case a totally different product algorithm is employed: * \code - * C = (A*B).pruned(); // supress numerical zeros (exact) + * C = (A*B).pruned(); // suppress numerical zeros (exact) * C = (A*B).pruned(ref); * C = (A*B).pruned(ref,epsilon); * \endcode @@ -45,7 +45,7 @@ // dense += sparse * sparse template<typename Dest,typename ActualLhs> - static void addTo(Dest& dst, const ActualLhs& lhs, const Rhs& rhs, int* = typename enable_if<is_same<typename evaluator_traits<Dest>::Shape,DenseShape>::value,int*>::type(0) ) + static void addTo(Dest& dst, const ActualLhs& lhs, const Rhs& rhs, typename enable_if<is_same<typename evaluator_traits<Dest>::Shape,DenseShape>::value,int*>::type* = 0) { typedef typename nested_eval<ActualLhs,Dynamic>::type LhsNested; typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; @@ -57,7 +57,7 @@ // dense -= sparse * sparse template<typename Dest> - static void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, int* = typename enable_if<is_same<typename evaluator_traits<Dest>::Shape,DenseShape>::value,int*>::type(0) ) + static void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, typename enable_if<is_same<typename evaluator_traits<Dest>::Shape,DenseShape>::value,int*>::type* = 0) { addTo(dst, -lhs, rhs); } @@ -99,21 +99,26 @@ // dense = sparse-product (can be sparse*sparse, sparse*perm, etc.) template< typename DstXprType, typename Lhs, typename Rhs> -struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::assign_op<typename DstXprType::Scalar>, Sparse2Dense> +struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::assign_op<typename DstXprType::Scalar,typename Product<Lhs,Rhs,AliasFreeProduct>::Scalar>, Sparse2Dense> { typedef Product<Lhs,Rhs,AliasFreeProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &) { + Index dstRows = src.rows(); + Index dstCols = src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + generic_product_impl<Lhs, Rhs>::evalTo(dst,src.lhs(),src.rhs()); } }; // dense += sparse-product (can be sparse*sparse, sparse*perm, etc.) template< typename DstXprType, typename Lhs, typename Rhs> -struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::add_assign_op<typename DstXprType::Scalar>, Sparse2Dense> +struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::add_assign_op<typename DstXprType::Scalar,typename Product<Lhs,Rhs,AliasFreeProduct>::Scalar>, Sparse2Dense> { typedef Product<Lhs,Rhs,AliasFreeProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &) { generic_product_impl<Lhs, Rhs>::addTo(dst,src.lhs(),src.rhs()); } @@ -121,24 +126,24 @@ // dense -= sparse-product (can be sparse*sparse, sparse*perm, etc.) template< typename DstXprType, typename Lhs, typename Rhs> -struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::sub_assign_op<typename DstXprType::Scalar>, Sparse2Dense> +struct Assignment<DstXprType, Product<Lhs,Rhs,AliasFreeProduct>, internal::sub_assign_op<typename DstXprType::Scalar,typename Product<Lhs,Rhs,AliasFreeProduct>::Scalar>, Sparse2Dense> { typedef Product<Lhs,Rhs,AliasFreeProduct> SrcXprType; - static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar> &) + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &) { generic_product_impl<Lhs, Rhs>::subTo(dst,src.lhs(),src.rhs()); } }; template<typename Lhs, typename Rhs, int Options> -struct evaluator<SparseView<Product<Lhs, Rhs, Options> > > +struct unary_evaluator<SparseView<Product<Lhs, Rhs, Options> >, IteratorBased> : public evaluator<typename Product<Lhs, Rhs, DefaultProduct>::PlainObject> { typedef SparseView<Product<Lhs, Rhs, Options> > XprType; typedef typename XprType::PlainObject PlainObject; typedef evaluator<PlainObject> Base; - - explicit evaluator(const XprType& xpr) + + explicit unary_evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) { using std::abs; @@ -147,13 +152,13 @@ typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; LhsNested lhsNested(xpr.nestedExpression().lhs()); RhsNested rhsNested(xpr.nestedExpression().rhs()); - + internal::sparse_sparse_product_with_pruning_selector<typename remove_all<LhsNested>::type, typename remove_all<RhsNested>::type, PlainObject>::run(lhsNested,rhsNested,m_result, abs(xpr.reference())*xpr.epsilon()); } - -protected: + +protected: PlainObject m_result; };
diff --git a/Eigen/src/SparseCore/SparseRedux.h b/Eigen/src/SparseCore/SparseRedux.h index 2a9718c..4587749 100644 --- a/Eigen/src/SparseCore/SparseRedux.h +++ b/Eigen/src/SparseCore/SparseRedux.h
@@ -30,7 +30,10 @@ SparseMatrix<_Scalar,_Options,_Index>::sum() const { eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); - return Matrix<Scalar,1,Dynamic>::Map(m_data.valuePtr(), m_data.size()).sum(); + if(this->isCompressed()) + return Matrix<Scalar,1,Dynamic>::Map(m_data.valuePtr(), m_data.size()).sum(); + else + return Base::sum(); } template<typename _Scalar, int _Options, typename _Index>
diff --git a/Eigen/src/SparseCore/SparseRef.h b/Eigen/src/SparseCore/SparseRef.h index a558230..d91f38f 100644 --- a/Eigen/src/SparseCore/SparseRef.h +++ b/Eigen/src/SparseCore/SparseRef.h
@@ -185,20 +185,27 @@ EIGEN_SPARSE_PUBLIC_INTERFACE(Ref) template<typename Derived> - inline Ref(const SparseMatrixBase<Derived>& expr) + inline Ref(const SparseMatrixBase<Derived>& expr) : m_hasCopy(false) { construct(expr.derived(), typename Traits::template match<Derived>::type()); } - inline Ref(const Ref& other) : Base(other) { + inline Ref(const Ref& other) : Base(other), m_hasCopy(false) { // copy constructor shall not copy the m_object, to avoid unnecessary malloc and copy } template<typename OtherRef> - inline Ref(const RefBase<OtherRef>& other) { + inline Ref(const RefBase<OtherRef>& other) : m_hasCopy(false) { construct(other.derived(), typename Traits::template match<OtherRef>::type()); } + ~Ref() { + if(m_hasCopy) { + TPlainObjectType* obj = reinterpret_cast<TPlainObjectType*>(m_object_bytes); + obj->~TPlainObjectType(); + } + } + protected: template<typename Expression> @@ -208,6 +215,7 @@ { TPlainObjectType* obj = reinterpret_cast<TPlainObjectType*>(m_object_bytes); ::new (obj) TPlainObjectType(expr); + m_hasCopy = true; Base::construct(*obj); } else @@ -221,11 +229,13 @@ { TPlainObjectType* obj = reinterpret_cast<TPlainObjectType*>(m_object_bytes); ::new (obj) TPlainObjectType(expr); + m_hasCopy = true; Base::construct(*obj); } protected: char m_object_bytes[sizeof(TPlainObjectType)]; + bool m_hasCopy; }; @@ -293,20 +303,27 @@ EIGEN_SPARSE_PUBLIC_INTERFACE(Ref) template<typename Derived> - inline Ref(const SparseMatrixBase<Derived>& expr) + inline Ref(const SparseMatrixBase<Derived>& expr) : m_hasCopy(false) { construct(expr.derived(), typename Traits::template match<Derived>::type()); } - inline Ref(const Ref& other) : Base(other) { + inline Ref(const Ref& other) : Base(other), m_hasCopy(false) { // copy constructor shall not copy the m_object, to avoid unnecessary malloc and copy } template<typename OtherRef> - inline Ref(const RefBase<OtherRef>& other) { + inline Ref(const RefBase<OtherRef>& other) : m_hasCopy(false) { construct(other.derived(), typename Traits::template match<OtherRef>::type()); } + ~Ref() { + if(m_hasCopy) { + TPlainObjectType* obj = reinterpret_cast<TPlainObjectType*>(m_object_bytes); + obj->~TPlainObjectType(); + } + } + protected: template<typename Expression> @@ -320,11 +337,13 @@ { TPlainObjectType* obj = reinterpret_cast<TPlainObjectType*>(m_object_bytes); ::new (obj) TPlainObjectType(expr); + m_hasCopy = true; Base::construct(*obj); } protected: char m_object_bytes[sizeof(TPlainObjectType)]; + bool m_hasCopy; }; namespace internal {
diff --git a/Eigen/src/SparseCore/SparseSelfAdjointView.h b/Eigen/src/SparseCore/SparseSelfAdjointView.h index b92bb17..65611b3 100644 --- a/Eigen/src/SparseCore/SparseSelfAdjointView.h +++ b/Eigen/src/SparseCore/SparseSelfAdjointView.h
@@ -47,6 +47,7 @@ enum { Mode = _Mode, + TransposeMode = ((Mode & Upper) ? Lower : 0) | ((Mode & Lower) ? Upper : 0), RowsAtCompileTime = internal::traits<SparseSelfAdjointView>::RowsAtCompileTime, ColsAtCompileTime = internal::traits<SparseSelfAdjointView>::ColsAtCompileTime }; @@ -218,18 +219,47 @@ template<> struct AssignmentKind<SparseShape,SparseSelfAdjointShape> { typedef SparseSelfAdjoint2Sparse Kind; }; template<> struct AssignmentKind<SparseSelfAdjointShape,SparseShape> { typedef Sparse2Sparse Kind; }; -template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> -struct Assignment<DstXprType, SrcXprType, Functor, SparseSelfAdjoint2Sparse, Scalar> +template< typename DstXprType, typename SrcXprType, typename Functor> +struct Assignment<DstXprType, SrcXprType, Functor, SparseSelfAdjoint2Sparse> { typedef typename DstXprType::StorageIndex StorageIndex; + typedef internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> AssignOpType; + template<typename DestScalar,int StorageOrder> - static void run(SparseMatrix<DestScalar,StorageOrder,StorageIndex> &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(SparseMatrix<DestScalar,StorageOrder,StorageIndex> &dst, const SrcXprType &src, const AssignOpType&/*func*/) { internal::permute_symm_to_fullsymm<SrcXprType::Mode>(src.matrix(), dst); } + + // FIXME: the handling of += and -= in sparse matrices should be cleanup so that next two overloads could be reduced to: + template<typename DestScalar,int StorageOrder,typename AssignFunc> + static void run(SparseMatrix<DestScalar,StorageOrder,StorageIndex> &dst, const SrcXprType &src, const AssignFunc& func) + { + SparseMatrix<DestScalar,StorageOrder,StorageIndex> tmp(src.rows(),src.cols()); + run(tmp, src, AssignOpType()); + call_assignment_no_alias_no_transpose(dst, tmp, func); + } + + template<typename DestScalar,int StorageOrder> + static void run(SparseMatrix<DestScalar,StorageOrder,StorageIndex> &dst, const SrcXprType &src, + const internal::add_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>& /* func */) + { + SparseMatrix<DestScalar,StorageOrder,StorageIndex> tmp(src.rows(),src.cols()); + run(tmp, src, AssignOpType()); + dst += tmp; + } + + template<typename DestScalar,int StorageOrder> + static void run(SparseMatrix<DestScalar,StorageOrder,StorageIndex> &dst, const SrcXprType &src, + const internal::sub_assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>& /* func */) + { + SparseMatrix<DestScalar,StorageOrder,StorageIndex> tmp(src.rows(),src.cols()); + run(tmp, src, AssignOpType()); + dst -= tmp; + } template<typename DestScalar> - static void run(DynamicSparseMatrix<DestScalar,ColMajor,StorageIndex>& dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + static void run(DynamicSparseMatrix<DestScalar,ColMajor,StorageIndex>& dst, const SrcXprType &src, const AssignOpType&/*func*/) { // TODO directly evaluate into dst; SparseMatrix<DestScalar,ColMajor,StorageIndex> tmp(dst.rows(),dst.cols()); @@ -250,11 +280,11 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) { EIGEN_ONLY_USED_FOR_DEBUG(alpha); - // TODO use alpha - eigen_assert(alpha==AlphaType(1) && "alpha != 1 is not implemented yet, sorry"); - typedef evaluator<SparseLhsType> LhsEval; - typedef typename evaluator<SparseLhsType>::InnerIterator LhsIterator; + typedef typename internal::nested_eval<SparseLhsType,DenseRhsType::MaxColsAtCompileTime>::type SparseLhsTypeNested; + typedef typename internal::remove_all<SparseLhsTypeNested>::type SparseLhsTypeNestedCleaned; + typedef evaluator<SparseLhsTypeNestedCleaned> LhsEval; + typedef typename LhsEval::InnerIterator LhsIterator; typedef typename SparseLhsType::Scalar LhsScalar; enum { @@ -266,39 +296,53 @@ ProcessSecondHalf = !ProcessFirstHalf }; - LhsEval lhsEval(lhs); - - for (Index j=0; j<lhs.outerSize(); ++j) + SparseLhsTypeNested lhs_nested(lhs); + LhsEval lhsEval(lhs_nested); + + // work on one column at once + for (Index k=0; k<rhs.cols(); ++k) { - LhsIterator i(lhsEval,j); - if (ProcessSecondHalf) + for (Index j=0; j<lhs.outerSize(); ++j) { - while (i && i.index()<j) ++i; - if(i && i.index()==j) + LhsIterator i(lhsEval,j); + // handle diagonal coeff + if (ProcessSecondHalf) { - res.row(j) += i.value() * rhs.row(j); - ++i; + while (i && i.index()<j) ++i; + if(i && i.index()==j) + { + res.coeffRef(j,k) += alpha * i.value() * rhs.coeff(j,k); + ++i; + } } + + // premultiplied rhs for scatters + typename ScalarBinaryOpTraits<AlphaType, typename DenseRhsType::Scalar>::ReturnType rhs_j(alpha*rhs(j,k)); + // accumulator for partial scalar product + typename DenseResType::Scalar res_j(0); + for(; (ProcessFirstHalf ? i && i.index() < j : i) ; ++i) + { + LhsScalar lhs_ij = i.value(); + if(!LhsIsRowMajor) lhs_ij = numext::conj(lhs_ij); + res_j += lhs_ij * rhs.coeff(i.index(),k); + res(i.index(),k) += numext::conj(lhs_ij) * rhs_j; + } + res.coeffRef(j,k) += alpha * res_j; + + // handle diagonal coeff + if (ProcessFirstHalf && i && (i.index()==j)) + res.coeffRef(j,k) += alpha * i.value() * rhs.coeff(j,k); } - for(; (ProcessFirstHalf ? i && i.index() < j : i) ; ++i) - { - Index a = LhsIsRowMajor ? j : i.index(); - Index b = LhsIsRowMajor ? i.index() : j; - LhsScalar v = i.value(); - res.row(a) += (v) * rhs.row(b); - res.row(b) += numext::conj(v) * rhs.row(a); - } - if (ProcessFirstHalf && i && (i.index()==j)) - res.row(j) += i.value() * rhs.row(j); } } template<typename LhsView, typename Rhs, int ProductType> struct generic_product_impl<LhsView, Rhs, SparseSelfAdjointShape, DenseShape, ProductType> +: generic_product_impl_base<LhsView, Rhs, generic_product_impl<LhsView, Rhs, SparseSelfAdjointShape, DenseShape, ProductType> > { template<typename Dest> - static void evalTo(Dest& dst, const LhsView& lhsView, const Rhs& rhs) + static void scaleAndAddTo(Dest& dst, const LhsView& lhsView, const Rhs& rhs, const typename Dest::Scalar& alpha) { typedef typename LhsView::_MatrixTypeNested Lhs; typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; @@ -306,16 +350,16 @@ LhsNested lhsNested(lhsView.matrix()); RhsNested rhsNested(rhs); - dst.setZero(); - internal::sparse_selfadjoint_time_dense_product<LhsView::Mode>(lhsNested, rhsNested, dst, typename Dest::Scalar(1)); + internal::sparse_selfadjoint_time_dense_product<LhsView::Mode>(lhsNested, rhsNested, dst, alpha); } }; template<typename Lhs, typename RhsView, int ProductType> struct generic_product_impl<Lhs, RhsView, DenseShape, SparseSelfAdjointShape, ProductType> +: generic_product_impl_base<Lhs, RhsView, generic_product_impl<Lhs, RhsView, DenseShape, SparseSelfAdjointShape, ProductType> > { template<typename Dest> - static void evalTo(Dest& dst, const Lhs& lhs, const RhsView& rhsView) + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const RhsView& rhsView, const typename Dest::Scalar& alpha) { typedef typename RhsView::_MatrixTypeNested Rhs; typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; @@ -323,10 +367,9 @@ LhsNested lhsNested(lhs); RhsNested rhsNested(rhsView.matrix()); - dst.setZero(); - // transpoe everything + // transpose everything Transpose<Dest> dstT(dst); - internal::sparse_selfadjoint_time_dense_product<RhsView::Mode>(rhsNested.transpose(), lhsNested.transpose(), dstT, typename Dest::Scalar(1)); + internal::sparse_selfadjoint_time_dense_product<RhsView::TransposeMode>(rhsNested.transpose(), lhsNested.transpose(), dstT, alpha); } }; @@ -586,12 +629,12 @@ namespace internal { template<typename DstXprType, typename MatrixType, int Mode, typename Scalar> -struct Assignment<DstXprType, SparseSymmetricPermutationProduct<MatrixType,Mode>, internal::assign_op<Scalar>, Sparse2Sparse> +struct Assignment<DstXprType, SparseSymmetricPermutationProduct<MatrixType,Mode>, internal::assign_op<Scalar,typename MatrixType::Scalar>, Sparse2Sparse> { typedef SparseSymmetricPermutationProduct<MatrixType,Mode> SrcXprType; typedef typename DstXprType::StorageIndex DstIndex; template<int Options> - static void run(SparseMatrix<Scalar,Options,DstIndex> &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(SparseMatrix<Scalar,Options,DstIndex> &dst, const SrcXprType &src, const internal::assign_op<Scalar,typename MatrixType::Scalar> &) { // internal::permute_symm_to_fullsymm<Mode>(m_matrix,_dest,m_perm.indices().data()); SparseMatrix<Scalar,(Options&RowMajor)==RowMajor ? ColMajor : RowMajor, DstIndex> tmp; @@ -600,7 +643,7 @@ } template<typename DestType,unsigned int DestMode> - static void run(SparseSelfAdjointView<DestType,DestMode>& dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + static void run(SparseSelfAdjointView<DestType,DestMode>& dst, const SrcXprType &src, const internal::assign_op<Scalar,typename MatrixType::Scalar> &) { internal::permute_symm_to_symm<Mode,DestMode>(src.matrix(),dst.matrix(),src.perm().indices().data()); }
diff --git a/Eigen/src/SparseCore/SparseSolverBase.h b/Eigen/src/SparseCore/SparseSolverBase.h index 1cb7080..b4c9a42 100644 --- a/Eigen/src/SparseCore/SparseSolverBase.h +++ b/Eigen/src/SparseCore/SparseSolverBase.h
@@ -19,7 +19,8 @@ * The rhs is decomposed into small vertical panels which are solved through dense temporaries. */ template<typename Decomposition, typename Rhs, typename Dest> -void solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest &dest) +typename enable_if<Rhs::ColsAtCompileTime!=1 && Dest::ColsAtCompileTime!=1>::type +solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest &dest) { EIGEN_STATIC_ASSERT((Dest::Flags&RowMajorBit)==0,THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES); typedef typename Dest::Scalar DestScalar; @@ -40,6 +41,19 @@ } } +// Overload for vector as rhs +template<typename Decomposition, typename Rhs, typename Dest> +typename enable_if<Rhs::ColsAtCompileTime==1 || Dest::ColsAtCompileTime==1>::type +solve_sparse_through_dense_panels(const Decomposition &dec, const Rhs& rhs, Dest &dest) +{ + typedef typename Dest::Scalar DestScalar; + Index size = rhs.rows(); + Eigen::Matrix<DestScalar,Dynamic,1> rhs_dense(rhs); + Eigen::Matrix<DestScalar,Dynamic,1> dest_dense(size); + dest_dense = dec.solve(rhs_dense); + dest = dest_dense.sparseView(); +} + } // end namespace internal /** \class SparseSolverBase
diff --git a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h index 20078f7..88820a4 100644 --- a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h +++ b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h
@@ -21,7 +21,8 @@ { // return sparse_sparse_product_with_pruning_impl2(lhs,rhs,res); - typedef typename remove_all<Lhs>::type::Scalar Scalar; + typedef typename remove_all<Rhs>::type::Scalar RhsScalar; + typedef typename remove_all<ResultType>::type::Scalar ResScalar; typedef typename remove_all<Lhs>::type::StorageIndex StorageIndex; // make sure to call innerSize/outerSize since we fake the storage order. @@ -31,7 +32,7 @@ eigen_assert(lhs.outerSize() == rhs.innerSize()); // allocate a temporary buffer - AmbiVector<Scalar,StorageIndex> tempVector(rows); + AmbiVector<ResScalar,StorageIndex> tempVector(rows); // mimics a resizeByInnerOuter: if(ResultType::IsRowMajor) @@ -51,7 +52,7 @@ Index estimated_nnz_prod = lhsEval.nonZerosEstimate() + rhsEval.nonZerosEstimate(); res.reserve(estimated_nnz_prod); - double ratioColRes = double(estimated_nnz_prod)/double(lhs.rows()*rhs.cols()); + double ratioColRes = double(estimated_nnz_prod)/(double(lhs.rows())*double(rhs.cols())); for (Index j=0; j<cols; ++j) { // FIXME: @@ -63,14 +64,14 @@ { // FIXME should be written like this: tmp += rhsIt.value() * lhs.col(rhsIt.index()) tempVector.restart(); - Scalar x = rhsIt.value(); + RhsScalar x = rhsIt.value(); for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, rhsIt.index()); lhsIt; ++lhsIt) { tempVector.coeffRef(lhsIt.index()) += lhsIt.value() * x; } } res.startVec(j); - for (typename AmbiVector<Scalar,StorageIndex>::Iterator it(tempVector,tolerance); it; ++it) + for (typename AmbiVector<ResScalar,StorageIndex>::Iterator it(tempVector,tolerance); it; ++it) res.insertBackByOuterInner(j,it.index()) = it.value(); } res.finalize(); @@ -85,7 +86,6 @@ template<typename Lhs, typename Rhs, typename ResultType> struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,ColMajor,ColMajor,ColMajor> { - typedef typename traits<typename remove_all<Lhs>::type>::Scalar Scalar; typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) @@ -129,8 +129,8 @@ typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixLhs; - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixRhs; + typedef SparseMatrix<typename Lhs::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixLhs; + typedef SparseMatrix<typename Rhs::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixRhs; ColMajorMatrixLhs colLhs(lhs); ColMajorMatrixRhs colRhs(rhs); internal::sparse_sparse_product_with_pruning_impl<ColMajorMatrixLhs,ColMajorMatrixRhs,ResultType>(colLhs, colRhs, res, tolerance); @@ -149,7 +149,7 @@ typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) { - typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename Lhs::StorageIndex> RowMajorMatrixLhs; + typedef SparseMatrix<typename Lhs::Scalar,RowMajor,typename Lhs::StorageIndex> RowMajorMatrixLhs; RowMajorMatrixLhs rowLhs(lhs); sparse_sparse_product_with_pruning_selector<RowMajorMatrixLhs,Rhs,ResultType,RowMajor,RowMajor>(rowLhs,rhs,res,tolerance); } @@ -161,7 +161,7 @@ typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) { - typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename Lhs::StorageIndex> RowMajorMatrixRhs; + typedef SparseMatrix<typename Rhs::Scalar,RowMajor,typename Lhs::StorageIndex> RowMajorMatrixRhs; RowMajorMatrixRhs rowRhs(rhs); sparse_sparse_product_with_pruning_selector<Lhs,RowMajorMatrixRhs,ResultType,RowMajor,RowMajor,RowMajor>(lhs,rowRhs,res,tolerance); } @@ -173,7 +173,7 @@ typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixRhs; + typedef SparseMatrix<typename Rhs::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixRhs; ColMajorMatrixRhs colRhs(rhs); internal::sparse_sparse_product_with_pruning_impl<Lhs,ColMajorMatrixRhs,ResultType>(lhs, colRhs, res, tolerance); } @@ -185,7 +185,7 @@ typedef typename ResultType::RealScalar RealScalar; static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) { - typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixLhs; + typedef SparseMatrix<typename Lhs::Scalar,ColMajor,typename Lhs::StorageIndex> ColMajorMatrixLhs; ColMajorMatrixLhs colLhs(lhs); internal::sparse_sparse_product_with_pruning_impl<ColMajorMatrixLhs,Rhs,ResultType>(colLhs, rhs, res, tolerance); }
diff --git a/Eigen/src/SparseCore/SparseTranspose.h b/Eigen/src/SparseCore/SparseTranspose.h index b6f180a..3757d4c 100644 --- a/Eigen/src/SparseCore/SparseTranspose.h +++ b/Eigen/src/SparseCore/SparseTranspose.h
@@ -56,7 +56,6 @@ : public evaluator_base<Transpose<ArgType> > { typedef typename evaluator<ArgType>::InnerIterator EvalIterator; - typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; public: typedef Transpose<ArgType> XprType; @@ -75,17 +74,6 @@ Index col() const { return EvalIterator::row(); } }; - class ReverseInnerIterator : public EvalReverseIterator - { - public: - EIGEN_STRONG_INLINE ReverseInnerIterator(const unary_evaluator& unaryOp, Index outer) - : EvalReverseIterator(unaryOp.m_argImpl,outer) - {} - - Index row() const { return EvalReverseIterator::col(); } - Index col() const { return EvalReverseIterator::row(); } - }; - enum { CoeffReadCost = evaluator<ArgType>::CoeffReadCost, Flags = XprType::Flags
diff --git a/Eigen/src/SparseCore/SparseTriangularView.h b/Eigen/src/SparseCore/SparseTriangularView.h index 0c27855..9ac1202 100644 --- a/Eigen/src/SparseCore/SparseTriangularView.h +++ b/Eigen/src/SparseCore/SparseTriangularView.h
@@ -55,7 +55,10 @@ this->solveInPlace(dst); } + /** Applies the inverse of \c *this to the dense vector or matrix \a other, "in-place" */ template<typename OtherDerived> void solveInPlace(MatrixBase<OtherDerived>& other) const; + + /** Applies the inverse of \c *this to the sparse vector or matrix \a other, "in-place" */ template<typename OtherDerived> void solveInPlace(SparseMatrixBase<OtherDerived>& other) const; };
diff --git a/Eigen/src/SparseCore/SparseUtil.h b/Eigen/src/SparseCore/SparseUtil.h index 74df0d4..ceb9368 100644 --- a/Eigen/src/SparseCore/SparseUtil.h +++ b/Eigen/src/SparseCore/SparseUtil.h
@@ -140,6 +140,14 @@ template<> struct glue_shapes<SparseShape,SelfAdjointShape> { typedef SparseSelfAdjointShape type; }; template<> struct glue_shapes<SparseShape,TriangularShape > { typedef SparseTriangularShape type; }; +// return type of SparseCompressedBase::lower_bound; +struct LowerBoundIndex { + LowerBoundIndex() : value(-1), found(false) {} + LowerBoundIndex(Index val, bool ok) : value(val), found(ok) {} + Index value; + bool found; +}; + } // end namespace internal /** \ingroup SparseCore_Module
diff --git a/Eigen/src/SparseCore/SparseVector.h b/Eigen/src/SparseCore/SparseVector.h index 167a988..05779be 100644 --- a/Eigen/src/SparseCore/SparseVector.h +++ b/Eigen/src/SparseCore/SparseVector.h
@@ -22,7 +22,7 @@ * See http://www.netlib.org/linalg/html_templates/node91.html for details on the storage scheme. * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_SPARSEVECTOR_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_SPARSEVECTOR_PLUGIN. */ namespace internal { @@ -281,7 +281,7 @@ } /** Swaps the values of \c *this and \a other. - * Overloaded for performance: this version performs a \em shallow swap by swaping pointers and attributes only. + * Overloaded for performance: this version performs a \em shallow swap by swapping pointers and attributes only. * \sa SparseMatrixBase::swap() */ inline void swap(SparseVector& other) @@ -290,6 +290,14 @@ m_data.swap(other.m_data); } + template<int OtherOptions> + inline void swap(SparseMatrix<Scalar,OtherOptions,StorageIndex>& other) + { + eigen_assert(other.outerSize()==1); + std::swap(m_size, other.m_innerSize); + m_data.swap(other.m_data); + } + inline SparseVector& operator=(const SparseVector& other) { if (other.isRValue()) @@ -403,6 +411,7 @@ : evaluator_base<SparseVector<_Scalar,_Options,_Index> > { typedef SparseVector<_Scalar,_Options,_Index> SparseVectorType; + typedef evaluator_base<SparseVectorType> Base; typedef typename SparseVectorType::InnerIterator InnerIterator; typedef typename SparseVectorType::ReverseInnerIterator ReverseInnerIterator; @@ -410,20 +419,22 @@ CoeffReadCost = NumTraits<_Scalar>::ReadCost, Flags = SparseVectorType::Flags }; + + evaluator() : Base() {} - explicit evaluator(const SparseVectorType &mat) : m_matrix(mat) + explicit evaluator(const SparseVectorType &mat) : m_matrix(&mat) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } inline Index nonZerosEstimate() const { - return m_matrix.nonZeros(); + return m_matrix->nonZeros(); } - operator SparseVectorType&() { return m_matrix.const_cast_derived(); } - operator const SparseVectorType&() const { return m_matrix; } + operator SparseVectorType&() { return m_matrix->const_cast_derived(); } + operator const SparseVectorType&() const { return *m_matrix; } - const SparseVectorType &m_matrix; + const SparseVectorType *m_matrix; }; template< typename Dest, typename Src>
diff --git a/Eigen/src/SparseCore/SparseView.h b/Eigen/src/SparseCore/SparseView.h index b867877..7c4aea7 100644 --- a/Eigen/src/SparseCore/SparseView.h +++ b/Eigen/src/SparseCore/SparseView.h
@@ -27,6 +27,20 @@ } // end namespace internal +/** \ingroup SparseCore_Module + * \class SparseView + * + * \brief Expression of a dense or sparse matrix with zero or too small values removed + * + * \tparam MatrixType the type of the object of which we are removing the small entries + * + * This class represents an expression of a given dense or sparse matrix with + * entries smaller than \c reference * \c epsilon are removed. + * It is the return type of MatrixBase::sparseView() and SparseMatrixBase::pruned() + * and most of the time this is the only way it is used. + * + * \sa MatrixBase::sparseView(), SparseMatrixBase::pruned() + */ template<typename MatrixType> class SparseView : public SparseMatrixBase<SparseView<MatrixType> > { @@ -190,6 +204,23 @@ } // end namespace internal +/** \ingroup SparseCore_Module + * + * \returns a sparse expression of the dense expression \c *this with values smaller than + * \a reference * \a epsilon removed. + * + * This method is typically used when prototyping to convert a quickly assembled dense Matrix \c D to a SparseMatrix \c S: + * \code + * MatrixXd D(n,m); + * SparseMatrix<double> S; + * S = D.sparseView(); // suppress numerical zeros (exact) + * S = D.sparseView(reference); + * S = D.sparseView(reference,epsilon); + * \endcode + * where \a reference is a meaningful non zero reference value, + * and \a epsilon is a tolerance factor defaulting to NumTraits<Scalar>::dummy_precision(). + * + * \sa SparseMatrixBase::pruned(), class SparseView */ template<typename Derived> const SparseView<Derived> MatrixBase<Derived>::sparseView(const Scalar& reference, const typename NumTraits<Scalar>::Real& epsilon) const @@ -198,7 +229,7 @@ } /** \returns an expression of \c *this with values smaller than - * \a reference * \a epsilon are removed. + * \a reference * \a epsilon removed. * * This method is typically used in conjunction with the product of two sparse matrices * to automatically prune the smallest values as follows:
diff --git a/Eigen/src/SparseCore/TriangularSolver.h b/Eigen/src/SparseCore/TriangularSolver.h index 19f8f67..f9c56ba 100644 --- a/Eigen/src/SparseCore/TriangularSolver.h +++ b/Eigen/src/SparseCore/TriangularSolver.h
@@ -171,6 +171,8 @@ } // end namespace internal +#ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename ExpressionType,unsigned int Mode> template<typename OtherDerived> void TriangularViewImpl<ExpressionType,Mode,Sparse>::solveInPlace(MatrixBase<OtherDerived>& other) const @@ -189,6 +191,7 @@ if (copy) other = otherCopy; } +#endif // pure sparse path @@ -286,6 +289,7 @@ } // end namespace internal +#ifndef EIGEN_PARSED_BY_DOXYGEN template<typename ExpressionType,unsigned int Mode> template<typename OtherDerived> void TriangularViewImpl<ExpressionType,Mode,Sparse>::solveInPlace(SparseMatrixBase<OtherDerived>& other) const @@ -304,6 +308,7 @@ // if (copy) // other = otherCopy; } +#endif } // end namespace Eigen
diff --git a/Eigen/src/SparseLU/CMakeLists.txt b/Eigen/src/SparseLU/CMakeLists.txt deleted file mode 100644 index 69729ee..0000000 --- a/Eigen/src/SparseLU/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SparseLU_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SparseLU_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseLU COMPONENT Devel - )
diff --git a/Eigen/src/SparseLU/SparseLU.h b/Eigen/src/SparseLU/SparseLU.h index 8d03870..b23125e 100644 --- a/Eigen/src/SparseLU/SparseLU.h +++ b/Eigen/src/SparseLU/SparseLU.h
@@ -26,7 +26,7 @@ * This class implements the supernodal LU factorization for general matrices. * It uses the main techniques from the sequential SuperLU package * (http://crd-legacy.lbl.gov/~xiaoye/SuperLU/). It handles transparently real - * and complex arithmetics with single and double precision, depending on the + * and complex arithmetic with single and double precision, depending on the * scalar type of your input matrix. * The code has been optimized to provide BLAS-3 operations during supernode-panel updates. * It benefits directly from the built-in high-performant Eigen BLAS routines. @@ -193,7 +193,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the LU factorization reports a problem, zero diagonal for instance * \c InvalidInput if the input matrix is invalid * @@ -499,11 +499,8 @@ eigen_assert(m_analysisIsOk && "analyzePattern() should be called first"); eigen_assert((matrix.rows() == matrix.cols()) && "Only for squared matrices"); - typedef typename IndexVector::Scalar StorageIndex; - m_isInitialized = true; - // Apply the column permutation computed in analyzepattern() // m_mat = matrix * m_perm_c.inverse(); m_mat = matrix; @@ -706,8 +703,8 @@ typedef typename MappedSupernodalType::Scalar Scalar; explicit SparseLUMatrixLReturnType(const MappedSupernodalType& mapL) : m_mapL(mapL) { } - Index rows() { return m_mapL.rows(); } - Index cols() { return m_mapL.cols(); } + Index rows() const { return m_mapL.rows(); } + Index cols() const { return m_mapL.cols(); } template<typename Dest> void solveInPlace( MatrixBase<Dest> &X) const { @@ -723,8 +720,8 @@ SparseLUMatrixUReturnType(const MatrixLType& mapL, const MatrixUType& mapU) : m_mapL(mapL),m_mapU(mapU) { } - Index rows() { return m_mapL.rows(); } - Index cols() { return m_mapL.cols(); } + Index rows() const { return m_mapL.rows(); } + Index cols() const { return m_mapL.cols(); } template<typename Dest> void solveInPlace(MatrixBase<Dest> &X) const { @@ -747,8 +744,9 @@ } else { + // FIXME: the following lines should use Block expressions and not Map! Map<const Matrix<Scalar,Dynamic,Dynamic, ColMajor>, 0, OuterStride<> > A( &(m_mapL.valuePtr()[luptr]), nsupc, nsupc, OuterStride<>(lda) ); - Map< Matrix<Scalar,Dynamic,Dynamic, ColMajor>, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); + Map< Matrix<Scalar,Dynamic,Dest::ColsAtCompileTime, ColMajor>, 0, OuterStride<> > U (&(X.coeffRef(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); U = A.template triangularView<Upper>().solve(U); }
diff --git a/Eigen/src/SparseLU/SparseLU_Memory.h b/Eigen/src/SparseLU/SparseLU_Memory.h index 4dc42e8..349bfd5 100644 --- a/Eigen/src/SparseLU/SparseLU_Memory.h +++ b/Eigen/src/SparseLU/SparseLU_Memory.h
@@ -51,7 +51,7 @@ /** - * Expand the existing storage to accomodate more fill-ins + * Expand the existing storage to accommodate more fill-ins * \param vec Valid pointer to the vector to allocate or expand * \param[in,out] length At input, contain the current length of the vector that is to be increased. At output, length of the newly allocated vector * \param[in] nbElts Current number of elements in the factors
diff --git a/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h b/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h index f0856db..8583b1b 100644 --- a/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h +++ b/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h
@@ -75,12 +75,12 @@ /** * Number of rows */ - Index rows() { return m_row; } + Index rows() const { return m_row; } /** * Number of columns */ - Index cols() { return m_col; } + Index cols() const { return m_col; } /** * Return the array of nonzero values packed by column @@ -239,7 +239,7 @@ Index n = int(X.rows()); Index nrhs = Index(X.cols()); const Scalar * Lval = valuePtr(); // Nonzero values - Matrix<Scalar,Dynamic,Dynamic, ColMajor> work(n, nrhs); // working vector + Matrix<Scalar,Dynamic,Dest::ColsAtCompileTime, ColMajor> work(n, nrhs); // working vector work.setZero(); for (Index k = 0; k <= nsuper(); k ++) { @@ -271,12 +271,12 @@ // Triangular solve Map<const Matrix<Scalar,Dynamic,Dynamic, ColMajor>, 0, OuterStride<> > A( &(Lval[luptr]), nsupc, nsupc, OuterStride<>(lda) ); - Map< Matrix<Scalar,Dynamic,Dynamic, ColMajor>, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); + Map< Matrix<Scalar,Dynamic,Dest::ColsAtCompileTime, ColMajor>, 0, OuterStride<> > U (&(X(fsupc,0)), nsupc, nrhs, OuterStride<>(n) ); U = A.template triangularView<UnitLower>().solve(U); // Matrix-vector product new (&A) Map<const Matrix<Scalar,Dynamic,Dynamic, ColMajor>, 0, OuterStride<> > ( &(Lval[luptr+nsupc]), nrow, nsupc, OuterStride<>(lda) ); - work.block(0, 0, nrow, nrhs) = A * U; + work.topRows(nrow).noalias() = A * U; //Begin Scatter for (Index j = 0; j < nrhs; j++)
diff --git a/Eigen/src/SparseLU/SparseLU_column_dfs.h b/Eigen/src/SparseLU/SparseLU_column_dfs.h index c98b30e..5a2c941 100644 --- a/Eigen/src/SparseLU/SparseLU_column_dfs.h +++ b/Eigen/src/SparseLU/SparseLU_column_dfs.h
@@ -151,7 +151,7 @@ StorageIndex ito = glu.xlsub(fsupc+1); glu.xlsub(jcolm1) = ito; StorageIndex istop = ito + jptr - jm1ptr; - xprune(jcolm1) = istop; // intialize xprune(jcol-1) + xprune(jcolm1) = istop; // initialize xprune(jcol-1) glu.xlsub(jcol) = istop; for (StorageIndex ifrom = jm1ptr; ifrom < nextl; ++ifrom, ++ito) @@ -166,7 +166,7 @@ // Tidy up the pointers before exit glu.xsup(nsuper+1) = jcolp1; glu.supno(jcolp1) = nsuper; - xprune(jcol) = StorageIndex(nextl); // Intialize upper bound for pruning + xprune(jcol) = StorageIndex(nextl); // Initialize upper bound for pruning glu.xlsub(jcolp1) = StorageIndex(nextl); return 0;
diff --git a/Eigen/src/SparseLU/SparseLU_gemm_kernel.h b/Eigen/src/SparseLU/SparseLU_gemm_kernel.h index ae3685a..e37c2fe 100644 --- a/Eigen/src/SparseLU/SparseLU_gemm_kernel.h +++ b/Eigen/src/SparseLU/SparseLU_gemm_kernel.h
@@ -72,14 +72,14 @@ // load and expand a RN x RK block of B Packet b00, b10, b20, b30, b01, b11, b21, b31; - b00 = pset1<Packet>(Bc0[0]); - b10 = pset1<Packet>(Bc0[1]); - if(RK==4) b20 = pset1<Packet>(Bc0[2]); - if(RK==4) b30 = pset1<Packet>(Bc0[3]); - b01 = pset1<Packet>(Bc1[0]); - b11 = pset1<Packet>(Bc1[1]); - if(RK==4) b21 = pset1<Packet>(Bc1[2]); - if(RK==4) b31 = pset1<Packet>(Bc1[3]); + { b00 = pset1<Packet>(Bc0[0]); } + { b10 = pset1<Packet>(Bc0[1]); } + if(RK==4) { b20 = pset1<Packet>(Bc0[2]); } + if(RK==4) { b30 = pset1<Packet>(Bc0[3]); } + { b01 = pset1<Packet>(Bc1[0]); } + { b11 = pset1<Packet>(Bc1[1]); } + if(RK==4) { b21 = pset1<Packet>(Bc1[2]); } + if(RK==4) { b31 = pset1<Packet>(Bc1[3]); } Packet a0, a1, a2, a3, c0, c1, t0, t1; @@ -106,22 +106,22 @@ #define KMADD(c, a, b, tmp) {tmp = b; tmp = pmul(a,tmp); c = padd(c,tmp);} #define WORK(I) \ - c0 = pload<Packet>(C0+i+(I)*PacketSize); \ - c1 = pload<Packet>(C1+i+(I)*PacketSize); \ - KMADD(c0, a0, b00, t0) \ - KMADD(c1, a0, b01, t1) \ - a0 = pload<Packet>(A0+i+(I+1)*PacketSize); \ - KMADD(c0, a1, b10, t0) \ - KMADD(c1, a1, b11, t1) \ - a1 = pload<Packet>(A1+i+(I+1)*PacketSize); \ - if(RK==4) KMADD(c0, a2, b20, t0) \ - if(RK==4) KMADD(c1, a2, b21, t1) \ - if(RK==4) a2 = pload<Packet>(A2+i+(I+1)*PacketSize); \ - if(RK==4) KMADD(c0, a3, b30, t0) \ - if(RK==4) KMADD(c1, a3, b31, t1) \ - if(RK==4) a3 = pload<Packet>(A3+i+(I+1)*PacketSize); \ - pstore(C0+i+(I)*PacketSize, c0); \ - pstore(C1+i+(I)*PacketSize, c1) + c0 = pload<Packet>(C0+i+(I)*PacketSize); \ + c1 = pload<Packet>(C1+i+(I)*PacketSize); \ + KMADD(c0, a0, b00, t0) \ + KMADD(c1, a0, b01, t1) \ + a0 = pload<Packet>(A0+i+(I+1)*PacketSize); \ + KMADD(c0, a1, b10, t0) \ + KMADD(c1, a1, b11, t1) \ + a1 = pload<Packet>(A1+i+(I+1)*PacketSize); \ + if(RK==4){ KMADD(c0, a2, b20, t0) }\ + if(RK==4){ KMADD(c1, a2, b21, t1) }\ + if(RK==4){ a2 = pload<Packet>(A2+i+(I+1)*PacketSize); }\ + if(RK==4){ KMADD(c0, a3, b30, t0) }\ + if(RK==4){ KMADD(c1, a3, b31, t1) }\ + if(RK==4){ a3 = pload<Packet>(A3+i+(I+1)*PacketSize); }\ + pstore(C0+i+(I)*PacketSize, c0); \ + pstore(C1+i+(I)*PacketSize, c1) // process rows of A' - C' with aggressive vectorization and peeling for(Index i=0; i<actual_b_end1; i+=PacketSize*8) @@ -131,14 +131,15 @@ prefetch((A1+i+(5)*PacketSize)); if(RK==4) prefetch((A2+i+(5)*PacketSize)); if(RK==4) prefetch((A3+i+(5)*PacketSize)); - WORK(0); - WORK(1); - WORK(2); - WORK(3); - WORK(4); - WORK(5); - WORK(6); - WORK(7); + + WORK(0); + WORK(1); + WORK(2); + WORK(3); + WORK(4); + WORK(5); + WORK(6); + WORK(7); } // process the remaining rows with vectorization only for(Index i=actual_b_end1; i<actual_b_end2; i+=PacketSize) @@ -203,18 +204,18 @@ } #define WORK(I) \ - c0 = pload<Packet>(C0+i+(I)*PacketSize); \ - KMADD(c0, a0, b00, t0) \ - a0 = pload<Packet>(A0+i+(I+1)*PacketSize); \ - KMADD(c0, a1, b10, t0) \ - a1 = pload<Packet>(A1+i+(I+1)*PacketSize); \ - if(RK==4) KMADD(c0, a2, b20, t0) \ - if(RK==4) a2 = pload<Packet>(A2+i+(I+1)*PacketSize); \ - if(RK==4) KMADD(c0, a3, b30, t0) \ - if(RK==4) a3 = pload<Packet>(A3+i+(I+1)*PacketSize); \ - pstore(C0+i+(I)*PacketSize, c0); + c0 = pload<Packet>(C0+i+(I)*PacketSize); \ + KMADD(c0, a0, b00, t0) \ + a0 = pload<Packet>(A0+i+(I+1)*PacketSize); \ + KMADD(c0, a1, b10, t0) \ + a1 = pload<Packet>(A1+i+(I+1)*PacketSize); \ + if(RK==4){ KMADD(c0, a2, b20, t0) }\ + if(RK==4){ a2 = pload<Packet>(A2+i+(I+1)*PacketSize); }\ + if(RK==4){ KMADD(c0, a3, b30, t0) }\ + if(RK==4){ a3 = pload<Packet>(A3+i+(I+1)*PacketSize); }\ + pstore(C0+i+(I)*PacketSize, c0); - // agressive vectorization and peeling + // aggressive vectorization and peeling for(Index i=0; i<actual_b_end1; i+=PacketSize*8) { EIGEN_ASM_COMMENT("SPARSELU_GEMML_KERNEL2");
diff --git a/Eigen/src/SparseLU/SparseLU_panel_bmod.h b/Eigen/src/SparseLU/SparseLU_panel_bmod.h index 822cf32..f052001 100644 --- a/Eigen/src/SparseLU/SparseLU_panel_bmod.h +++ b/Eigen/src/SparseLU/SparseLU_panel_bmod.h
@@ -38,7 +38,7 @@ * \brief Performs numeric block updates (sup-panel) in topological order. * * Before entering this routine, the original nonzeros in the panel - * were already copied i nto the spa[m,w] + * were already copied into the spa[m,w] * * \param m number of rows in the matrix * \param w Panel size
diff --git a/Eigen/src/SparseQR/CMakeLists.txt b/Eigen/src/SparseQR/CMakeLists.txt deleted file mode 100644 index f9ddf2b..0000000 --- a/Eigen/src/SparseQR/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SparseQR_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SparseQR_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SparseQR/ COMPONENT Devel - )
diff --git a/Eigen/src/SparseQR/SparseQR.h b/Eigen/src/SparseQR/SparseQR.h index acd7f7e..d1fb96f 100644 --- a/Eigen/src/SparseQR/SparseQR.h +++ b/Eigen/src/SparseQR/SparseQR.h
@@ -41,18 +41,19 @@ /** * \ingroup SparseQR_Module * \class SparseQR - * \brief Sparse left-looking rank-revealing QR factorization + * \brief Sparse left-looking QR factorization with numerical column pivoting * - * This class implements a left-looking rank-revealing QR decomposition - * of sparse matrices. When a column has a norm less than a given tolerance + * This class implements a left-looking QR decomposition of sparse matrices + * with numerical column pivoting. + * When a column has a norm less than a given tolerance * it is implicitly permuted to the end. The QR factorization thus obtained is * given by A*P = Q*R where R is upper triangular or trapezoidal. * * P is the column permutation which is the product of the fill-reducing and the - * rank-revealing permutations. Use colsPermutation() to get it. + * numerical permutations. Use colsPermutation() to get it. * * Q is the orthogonal matrix represented as products of Householder reflectors. - * Use matrixQ() to get an expression and matrixQ().transpose() to get the transpose. + * Use matrixQ() to get an expression and matrixQ().adjoint() to get the adjoint. * You can then apply it to a vector. * * R is the sparse triangular or trapezoidal matrix. The later occurs when A is rank-deficient. @@ -64,7 +65,19 @@ * * \implsparsesolverconcept * + * The numerical pivoting strategy and default threshold are the same as in SuiteSparse QR, and + * detailed in the following paper: + * <i> + * Tim Davis, "Algorithm 915, SuiteSparseQR: Multifrontal Multithreaded Rank-Revealing + * Sparse QR Factorization, ACM Trans. on Math. Soft. 38(1), 2011. + * </i> + * Even though it is qualified as "rank-revealing", this strategy might fail for some + * rank deficient problems. When this class is used to solve linear or least-square problems + * it is thus strongly recommended to check the accuracy of the computed solution. If it + * failed, it usually helps to increase the threshold with setPivotThreshold. + * * \warning The input sparse matrix A must be in compressed mode (see SparseMatrix::makeCompressed()). + * \warning For complex matrices matrixQ().transpose() will actually return the adjoint matrix. * */ template<typename _MatrixType, typename _OrderingType> @@ -196,9 +209,9 @@ Index rank = this->rank(); - // Compute Q^T * b; + // Compute Q^* * b; typename Dest::PlainObject y, b; - y = this->matrixQ().transpose() * B; + y = this->matrixQ().adjoint() * B; b = y; // Solve with the triangular matrix R @@ -330,7 +343,7 @@ m_R.resize(m, n); m_Q.resize(m, diagSize); - // Allocate space for nonzero elements : rough estimation + // Allocate space for nonzero elements: rough estimation m_R.reserve(2*mat.nonZeros()); //FIXME Get a more accurate estimation through symbolic factorization with the etree m_Q.reserve(2*mat.nonZeros()); m_hcoeffs.resize(diagSize); @@ -604,7 +617,7 @@ // Get the references SparseQR_QProduct(const SparseQRType& qr, const Derived& other, bool transpose) : m_qr(qr),m_other(other),m_transpose(transpose) {} - inline Index rows() const { return m_transpose ? m_qr.rows() : m_qr.cols(); } + inline Index rows() const { return m_qr.matrixQ().rows(); } inline Index cols() const { return m_other.cols(); } // Assign to a vector @@ -632,16 +645,20 @@ } else { - eigen_assert(m_qr.m_Q.rows() == m_other.rows() && "Non conforming object sizes"); + eigen_assert(m_qr.matrixQ().cols() == m_other.rows() && "Non conforming object sizes"); + + res.conservativeResize(rows(), cols()); + // Compute res = Q * other column by column for(Index j = 0; j < res.cols(); j++) { - for (Index k = diagSize-1; k >=0; k--) + Index start_k = internal::is_identity<Derived>::value ? numext::mini(j,diagSize-1) : diagSize-1; + for (Index k = start_k; k >=0; k--) { Scalar tau = Scalar(0); tau = m_qr.m_Q.col(k).dot(res.col(j)); if(tau==Scalar(0)) continue; - tau = tau * m_qr.m_hcoeffs(k); + tau = tau * numext::conj(m_qr.m_hcoeffs(k)); res.col(j) -= tau * m_qr.m_Q.col(k); } } @@ -650,7 +667,7 @@ const SparseQRType& m_qr; const Derived& m_other; - bool m_transpose; + bool m_transpose; // TODO this actually means adjoint }; template<typename SparseQRType> @@ -668,13 +685,14 @@ { return SparseQR_QProduct<SparseQRType,Derived>(m_qr,other.derived(),false); } + // To use for operations with the adjoint of Q SparseQRMatrixQTransposeReturnType<SparseQRType> adjoint() const { return SparseQRMatrixQTransposeReturnType<SparseQRType>(m_qr); } inline Index rows() const { return m_qr.rows(); } - inline Index cols() const { return (std::min)(m_qr.rows(),m_qr.cols()); } - // To use for operations with the transpose of Q + inline Index cols() const { return m_qr.rows(); } + // To use for operations with the transpose of Q FIXME this is the same as adjoint at the moment SparseQRMatrixQTransposeReturnType<SparseQRType> transpose() const { return SparseQRMatrixQTransposeReturnType<SparseQRType>(m_qr); @@ -682,6 +700,7 @@ const SparseQRType& m_qr; }; +// TODO this actually represents the adjoint of Q template<typename SparseQRType> struct SparseQRMatrixQTransposeReturnType { @@ -705,14 +724,14 @@ }; template< typename DstXprType, typename SparseQRType> -struct Assignment<DstXprType, SparseQRMatrixQReturnType<SparseQRType>, internal::assign_op<typename DstXprType::Scalar>, Sparse2Sparse> +struct Assignment<DstXprType, SparseQRMatrixQReturnType<SparseQRType>, internal::assign_op<typename DstXprType::Scalar,typename DstXprType::Scalar>, Sparse2Sparse> { typedef SparseQRMatrixQReturnType<SparseQRType> SrcXprType; typedef typename DstXprType::Scalar Scalar; typedef typename DstXprType::StorageIndex StorageIndex; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &/*func*/) { - typename DstXprType::PlainObject idMat(src.m_qr.rows(), src.m_qr.rows()); + typename DstXprType::PlainObject idMat(src.rows(), src.cols()); idMat.setIdentity(); // Sort the sparse householder reflectors if needed const_cast<SparseQRType *>(&src.m_qr)->_sort_matrix_Q(); @@ -721,12 +740,12 @@ }; template< typename DstXprType, typename SparseQRType> -struct Assignment<DstXprType, SparseQRMatrixQReturnType<SparseQRType>, internal::assign_op<typename DstXprType::Scalar>, Sparse2Dense> +struct Assignment<DstXprType, SparseQRMatrixQReturnType<SparseQRType>, internal::assign_op<typename DstXprType::Scalar,typename DstXprType::Scalar>, Sparse2Dense> { typedef SparseQRMatrixQReturnType<SparseQRType> SrcXprType; typedef typename DstXprType::Scalar Scalar; typedef typename DstXprType::StorageIndex StorageIndex; - static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &/*func*/) + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar,Scalar> &/*func*/) { dst = src.m_qr.matrixQ() * DstXprType::Identity(src.m_qr.rows(), src.m_qr.rows()); }
diff --git a/Eigen/src/StlSupport/CMakeLists.txt b/Eigen/src/StlSupport/CMakeLists.txt deleted file mode 100644 index 0f094f6..0000000 --- a/Eigen/src/StlSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_StlSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_StlSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/StlSupport COMPONENT Devel - )
diff --git a/Eigen/src/StlSupport/StdDeque.h b/Eigen/src/StlSupport/StdDeque.h index cf1fedf..045da7b 100644 --- a/Eigen/src/StlSupport/StdDeque.h +++ b/Eigen/src/StlSupport/StdDeque.h
@@ -36,7 +36,7 @@ deque(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) : deque_base(first, last, a) {} \ deque(const deque& c) : deque_base(c) {} \ explicit deque(size_type num, const value_type& val = value_type()) : deque_base(num, val) {} \ - deque(iterator start, iterator end) : deque_base(start, end) {} \ + deque(iterator start_, iterator end_) : deque_base(start_, end_) {} \ deque& operator=(const deque& x) { \ deque_base::operator=(x); \ return *this; \ @@ -62,7 +62,7 @@ : deque_base(first, last, a) {} \ deque(const deque& c) : deque_base(c) {} \ explicit deque(size_type num, const value_type& val = value_type()) : deque_base(num, val) {} \ - deque(iterator start, iterator end) : deque_base(start, end) {} \ + deque(iterator start_, iterator end_) : deque_base(start_, end_) {} \ deque& operator=(const deque& x) { \ deque_base::operator=(x); \ return *this; \
diff --git a/Eigen/src/StlSupport/StdList.h b/Eigen/src/StlSupport/StdList.h index e1eba49..8ba3fad 100644 --- a/Eigen/src/StlSupport/StdList.h +++ b/Eigen/src/StlSupport/StdList.h
@@ -35,7 +35,7 @@ list(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) : list_base(first, last, a) {} \ list(const list& c) : list_base(c) {} \ explicit list(size_type num, const value_type& val = value_type()) : list_base(num, val) {} \ - list(iterator start, iterator end) : list_base(start, end) {} \ + list(iterator start_, iterator end_) : list_base(start_, end_) {} \ list& operator=(const list& x) { \ list_base::operator=(x); \ return *this; \ @@ -62,7 +62,7 @@ : list_base(first, last, a) {} \ list(const list& c) : list_base(c) {} \ explicit list(size_type num, const value_type& val = value_type()) : list_base(num, val) {} \ - list(iterator start, iterator end) : list_base(start, end) {} \ + list(iterator start_, iterator end_) : list_base(start_, end_) {} \ list& operator=(const list& x) { \ list_base::operator=(x); \ return *this; \
diff --git a/Eigen/src/StlSupport/StdVector.h b/Eigen/src/StlSupport/StdVector.h index ec22821..9fcf19b 100644 --- a/Eigen/src/StlSupport/StdVector.h +++ b/Eigen/src/StlSupport/StdVector.h
@@ -36,7 +36,7 @@ vector(InputIterator first, InputIterator last, const allocator_type& a = allocator_type()) : vector_base(first, last, a) {} \ vector(const vector& c) : vector_base(c) {} \ explicit vector(size_type num, const value_type& val = value_type()) : vector_base(num, val) {} \ - vector(iterator start, iterator end) : vector_base(start, end) {} \ + vector(iterator start_, iterator end_) : vector_base(start_, end_) {} \ vector& operator=(const vector& x) { \ vector_base::operator=(x); \ return *this; \ @@ -62,7 +62,7 @@ : vector_base(first, last, a) {} \ vector(const vector& c) : vector_base(c) {} \ explicit vector(size_type num, const value_type& val = value_type()) : vector_base(num, val) {} \ - vector(iterator start, iterator end) : vector_base(start, end) {} \ + vector(iterator start_, iterator end_) : vector_base(start_, end_) {} \ vector& operator=(const vector& x) { \ vector_base::operator=(x); \ return *this; \
diff --git a/Eigen/src/StlSupport/details.h b/Eigen/src/StlSupport/details.h index e42ec02..2cfd13e 100644 --- a/Eigen/src/StlSupport/details.h +++ b/Eigen/src/StlSupport/details.h
@@ -22,13 +22,13 @@ class aligned_allocator_indirection : public EIGEN_ALIGNED_ALLOCATOR<T> { public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + typedef T value_type; template<class U> struct rebind
diff --git a/Eigen/src/SuperLUSupport/CMakeLists.txt b/Eigen/src/SuperLUSupport/CMakeLists.txt deleted file mode 100644 index b28ebe5..0000000 --- a/Eigen/src/SuperLUSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SuperLUSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SuperLUSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/SuperLUSupport COMPONENT Devel - )
diff --git a/Eigen/src/SuperLUSupport/SuperLUSupport.h b/Eigen/src/SuperLUSupport/SuperLUSupport.h index 7e2efd4..354e33d 100644 --- a/Eigen/src/SuperLUSupport/SuperLUSupport.h +++ b/Eigen/src/SuperLUSupport/SuperLUSupport.h
@@ -10,15 +10,16 @@ #ifndef EIGEN_SUPERLUSUPPORT_H #define EIGEN_SUPERLUSUPPORT_H -namespace Eigen { +namespace Eigen { +#if defined(SUPERLU_MAJOR_VERSION) && (SUPERLU_MAJOR_VERSION >= 5) #define DECL_GSSVX(PREFIX,FLOATTYPE,KEYTYPE) \ extern "C" { \ extern void PREFIX##gssvx(superlu_options_t *, SuperMatrix *, int *, int *, int *, \ char *, FLOATTYPE *, FLOATTYPE *, SuperMatrix *, SuperMatrix *, \ void *, int, SuperMatrix *, SuperMatrix *, \ FLOATTYPE *, FLOATTYPE *, FLOATTYPE *, FLOATTYPE *, \ - mem_usage_t *, SuperLUStat_t *, int *); \ + GlobalLU_t *, mem_usage_t *, SuperLUStat_t *, int *); \ } \ inline float SuperLU_gssvx(superlu_options_t *options, SuperMatrix *A, \ int *perm_c, int *perm_r, int *etree, char *equed, \ @@ -28,12 +29,37 @@ FLOATTYPE *recip_pivot_growth, \ FLOATTYPE *rcond, FLOATTYPE *ferr, FLOATTYPE *berr, \ SuperLUStat_t *stats, int *info, KEYTYPE) { \ - mem_usage_t mem_usage; \ + mem_usage_t mem_usage; \ + GlobalLU_t gLU; \ + PREFIX##gssvx(options, A, perm_c, perm_r, etree, equed, R, C, L, \ + U, work, lwork, B, X, recip_pivot_growth, rcond, \ + ferr, berr, &gLU, &mem_usage, stats, info); \ + return mem_usage.for_lu; /* bytes used by the factor storage */ \ + } +#else // version < 5.0 +#define DECL_GSSVX(PREFIX,FLOATTYPE,KEYTYPE) \ + extern "C" { \ + extern void PREFIX##gssvx(superlu_options_t *, SuperMatrix *, int *, int *, int *, \ + char *, FLOATTYPE *, FLOATTYPE *, SuperMatrix *, SuperMatrix *, \ + void *, int, SuperMatrix *, SuperMatrix *, \ + FLOATTYPE *, FLOATTYPE *, FLOATTYPE *, FLOATTYPE *, \ + mem_usage_t *, SuperLUStat_t *, int *); \ + } \ + inline float SuperLU_gssvx(superlu_options_t *options, SuperMatrix *A, \ + int *perm_c, int *perm_r, int *etree, char *equed, \ + FLOATTYPE *R, FLOATTYPE *C, SuperMatrix *L, \ + SuperMatrix *U, void *work, int lwork, \ + SuperMatrix *B, SuperMatrix *X, \ + FLOATTYPE *recip_pivot_growth, \ + FLOATTYPE *rcond, FLOATTYPE *ferr, FLOATTYPE *berr, \ + SuperLUStat_t *stats, int *info, KEYTYPE) { \ + mem_usage_t mem_usage; \ PREFIX##gssvx(options, A, perm_c, perm_r, etree, equed, R, C, L, \ U, work, lwork, B, X, recip_pivot_growth, rcond, \ ferr, berr, &mem_usage, stats, info); \ return mem_usage.for_lu; /* bytes used by the factor storage */ \ } +#endif DECL_GSSVX(s,float,float) DECL_GSSVX(c,float,std::complex<float>) @@ -271,8 +297,8 @@ template<typename Scalar, int Flags, typename Index> MappedSparseMatrix<Scalar,Flags,Index> map_superlu(SluMatrix& sluMat) { - eigen_assert((Flags&RowMajor)==RowMajor && sluMat.Stype == SLU_NR - || (Flags&ColMajor)==ColMajor && sluMat.Stype == SLU_NC); + eigen_assert(((Flags&RowMajor)==RowMajor && sluMat.Stype == SLU_NR) + || ((Flags&ColMajor)==ColMajor && sluMat.Stype == SLU_NC)); Index outerSize = (Flags&RowMajor)==RowMajor ? sluMat.ncol : sluMat.nrow; @@ -326,7 +352,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix.appears to be negative. */ ComputationInfo info() const @@ -941,6 +967,7 @@ m_factorizationIsOk = true; } +#ifndef EIGEN_PARSED_BY_DOXYGEN template<typename MatrixType> template<typename Rhs,typename Dest> void SuperILU<MatrixType>::_solve_impl(const MatrixBase<Rhs> &b, MatrixBase<Dest>& x) const @@ -993,6 +1020,8 @@ } #endif +#endif + } // end namespace Eigen #endif // EIGEN_SUPERLUSUPPORT_H
diff --git a/Eigen/src/UmfPackSupport/CMakeLists.txt b/Eigen/src/UmfPackSupport/CMakeLists.txt deleted file mode 100644 index a57de00..0000000 --- a/Eigen/src/UmfPackSupport/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_UmfPackSupport_SRCS "*.h") - -INSTALL(FILES - ${Eigen_UmfPackSupport_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/UmfPackSupport COMPONENT Devel - )
diff --git a/Eigen/src/UmfPackSupport/UmfPackSupport.h b/Eigen/src/UmfPackSupport/UmfPackSupport.h index 929a01a..e3a333f 100644 --- a/Eigen/src/UmfPackSupport/UmfPackSupport.h +++ b/Eigen/src/UmfPackSupport/UmfPackSupport.h
@@ -10,31 +10,102 @@ #ifndef EIGEN_UMFPACKSUPPORT_H #define EIGEN_UMFPACKSUPPORT_H -namespace Eigen { +// for compatibility with super old version of umfpack, +// not sure this is really needed, but this is harmless. +#ifndef SuiteSparse_long +#ifdef UF_long +#define SuiteSparse_long UF_long +#else +#error neither SuiteSparse_long nor UF_long are defined +#endif +#endif + +namespace Eigen { /* TODO extract L, extract U, compute det, etc... */ // generic double/complex<double> wrapper functions: -inline void umfpack_defaults(double control[UMFPACK_CONTROL], double) + // Defaults +inline void umfpack_defaults(double control[UMFPACK_CONTROL], double, int) { umfpack_di_defaults(control); } -inline void umfpack_defaults(double control[UMFPACK_CONTROL], std::complex<double>) +inline void umfpack_defaults(double control[UMFPACK_CONTROL], std::complex<double>, int) { umfpack_zi_defaults(control); } -inline void umfpack_free_numeric(void **Numeric, double) +inline void umfpack_defaults(double control[UMFPACK_CONTROL], double, SuiteSparse_long) +{ umfpack_dl_defaults(control); } + +inline void umfpack_defaults(double control[UMFPACK_CONTROL], std::complex<double>, SuiteSparse_long) +{ umfpack_zl_defaults(control); } + +// Report info +inline void umfpack_report_info(double control[UMFPACK_CONTROL], double info[UMFPACK_INFO], double, int) +{ umfpack_di_report_info(control, info);} + +inline void umfpack_report_info(double control[UMFPACK_CONTROL], double info[UMFPACK_INFO], std::complex<double>, int) +{ umfpack_zi_report_info(control, info);} + +inline void umfpack_report_info(double control[UMFPACK_CONTROL], double info[UMFPACK_INFO], double, SuiteSparse_long) +{ umfpack_dl_report_info(control, info);} + +inline void umfpack_report_info(double control[UMFPACK_CONTROL], double info[UMFPACK_INFO], std::complex<double>, SuiteSparse_long) +{ umfpack_zl_report_info(control, info);} + +// Report status +inline void umfpack_report_status(double control[UMFPACK_CONTROL], int status, double, int) +{ umfpack_di_report_status(control, status);} + +inline void umfpack_report_status(double control[UMFPACK_CONTROL], int status, std::complex<double>, int) +{ umfpack_zi_report_status(control, status);} + +inline void umfpack_report_status(double control[UMFPACK_CONTROL], int status, double, SuiteSparse_long) +{ umfpack_dl_report_status(control, status);} + +inline void umfpack_report_status(double control[UMFPACK_CONTROL], int status, std::complex<double>, SuiteSparse_long) +{ umfpack_zl_report_status(control, status);} + +// report control +inline void umfpack_report_control(double control[UMFPACK_CONTROL], double, int) +{ umfpack_di_report_control(control);} + +inline void umfpack_report_control(double control[UMFPACK_CONTROL], std::complex<double>, int) +{ umfpack_zi_report_control(control);} + +inline void umfpack_report_control(double control[UMFPACK_CONTROL], double, SuiteSparse_long) +{ umfpack_dl_report_control(control);} + +inline void umfpack_report_control(double control[UMFPACK_CONTROL], std::complex<double>, SuiteSparse_long) +{ umfpack_zl_report_control(control);} + +// Free numeric +inline void umfpack_free_numeric(void **Numeric, double, int) { umfpack_di_free_numeric(Numeric); *Numeric = 0; } -inline void umfpack_free_numeric(void **Numeric, std::complex<double>) +inline void umfpack_free_numeric(void **Numeric, std::complex<double>, int) { umfpack_zi_free_numeric(Numeric); *Numeric = 0; } -inline void umfpack_free_symbolic(void **Symbolic, double) +inline void umfpack_free_numeric(void **Numeric, double, SuiteSparse_long) +{ umfpack_dl_free_numeric(Numeric); *Numeric = 0; } + +inline void umfpack_free_numeric(void **Numeric, std::complex<double>, SuiteSparse_long) +{ umfpack_zl_free_numeric(Numeric); *Numeric = 0; } + +// Free symbolic +inline void umfpack_free_symbolic(void **Symbolic, double, int) { umfpack_di_free_symbolic(Symbolic); *Symbolic = 0; } -inline void umfpack_free_symbolic(void **Symbolic, std::complex<double>) +inline void umfpack_free_symbolic(void **Symbolic, std::complex<double>, int) { umfpack_zi_free_symbolic(Symbolic); *Symbolic = 0; } +inline void umfpack_free_symbolic(void **Symbolic, double, SuiteSparse_long) +{ umfpack_dl_free_symbolic(Symbolic); *Symbolic = 0; } + +inline void umfpack_free_symbolic(void **Symbolic, std::complex<double>, SuiteSparse_long) +{ umfpack_zl_free_symbolic(Symbolic); *Symbolic = 0; } + +// Symbolic inline int umfpack_symbolic(int n_row,int n_col, const int Ap[], const int Ai[], const double Ax[], void **Symbolic, const double Control [UMFPACK_CONTROL], double Info [UMFPACK_INFO]) @@ -48,7 +119,21 @@ { return umfpack_zi_symbolic(n_row,n_col,Ap,Ai,&numext::real_ref(Ax[0]),0,Symbolic,Control,Info); } +inline SuiteSparse_long umfpack_symbolic( SuiteSparse_long n_row,SuiteSparse_long n_col, + const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const double Ax[], void **Symbolic, + const double Control [UMFPACK_CONTROL], double Info [UMFPACK_INFO]) +{ + return umfpack_dl_symbolic(n_row,n_col,Ap,Ai,Ax,Symbolic,Control,Info); +} +inline SuiteSparse_long umfpack_symbolic( SuiteSparse_long n_row,SuiteSparse_long n_col, + const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const std::complex<double> Ax[], void **Symbolic, + const double Control [UMFPACK_CONTROL], double Info [UMFPACK_INFO]) +{ + return umfpack_zl_symbolic(n_row,n_col,Ap,Ai,&numext::real_ref(Ax[0]),0,Symbolic,Control,Info); +} + +// Numeric inline int umfpack_numeric( const int Ap[], const int Ai[], const double Ax[], void *Symbolic, void **Numeric, const double Control[UMFPACK_CONTROL],double Info [UMFPACK_INFO]) @@ -62,7 +147,21 @@ { return umfpack_zi_numeric(Ap,Ai,&numext::real_ref(Ax[0]),0,Symbolic,Numeric,Control,Info); } +inline SuiteSparse_long umfpack_numeric(const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const double Ax[], + void *Symbolic, void **Numeric, + const double Control[UMFPACK_CONTROL],double Info [UMFPACK_INFO]) +{ + return umfpack_dl_numeric(Ap,Ai,Ax,Symbolic,Numeric,Control,Info); +} +inline SuiteSparse_long umfpack_numeric(const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const std::complex<double> Ax[], + void *Symbolic, void **Numeric, + const double Control[UMFPACK_CONTROL],double Info [UMFPACK_INFO]) +{ + return umfpack_zl_numeric(Ap,Ai,&numext::real_ref(Ax[0]),0,Symbolic,Numeric,Control,Info); +} + +// solve inline int umfpack_solve( int sys, const int Ap[], const int Ai[], const double Ax[], double X[], const double B[], void *Numeric, const double Control[UMFPACK_CONTROL], double Info[UMFPACK_INFO]) @@ -77,6 +176,21 @@ return umfpack_zi_solve(sys,Ap,Ai,&numext::real_ref(Ax[0]),0,&numext::real_ref(X[0]),0,&numext::real_ref(B[0]),0,Numeric,Control,Info); } +inline SuiteSparse_long umfpack_solve(int sys, const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const double Ax[], + double X[], const double B[], void *Numeric, + const double Control[UMFPACK_CONTROL], double Info[UMFPACK_INFO]) +{ + return umfpack_dl_solve(sys,Ap,Ai,Ax,X,B,Numeric,Control,Info); +} + +inline SuiteSparse_long umfpack_solve(int sys, const SuiteSparse_long Ap[], const SuiteSparse_long Ai[], const std::complex<double> Ax[], + std::complex<double> X[], const std::complex<double> B[], void *Numeric, + const double Control[UMFPACK_CONTROL], double Info[UMFPACK_INFO]) +{ + return umfpack_zl_solve(sys,Ap,Ai,&numext::real_ref(Ax[0]),0,&numext::real_ref(X[0]),0,&numext::real_ref(B[0]),0,Numeric,Control,Info); +} + +// Get Lunz inline int umfpack_get_lunz(int *lnz, int *unz, int *n_row, int *n_col, int *nz_udiag, void *Numeric, double) { return umfpack_di_get_lunz(lnz,unz,n_row,n_col,nz_udiag,Numeric); @@ -87,6 +201,19 @@ return umfpack_zi_get_lunz(lnz,unz,n_row,n_col,nz_udiag,Numeric); } +inline SuiteSparse_long umfpack_get_lunz( SuiteSparse_long *lnz, SuiteSparse_long *unz, SuiteSparse_long *n_row, SuiteSparse_long *n_col, + SuiteSparse_long *nz_udiag, void *Numeric, double) +{ + return umfpack_dl_get_lunz(lnz,unz,n_row,n_col,nz_udiag,Numeric); +} + +inline SuiteSparse_long umfpack_get_lunz( SuiteSparse_long *lnz, SuiteSparse_long *unz, SuiteSparse_long *n_row, SuiteSparse_long *n_col, + SuiteSparse_long *nz_udiag, void *Numeric, std::complex<double>) +{ + return umfpack_zl_get_lunz(lnz,unz,n_row,n_col,nz_udiag,Numeric); +} + +// Get Numeric inline int umfpack_get_numeric(int Lp[], int Lj[], double Lx[], int Up[], int Ui[], double Ux[], int P[], int Q[], double Dx[], int *do_recip, double Rs[], void *Numeric) { @@ -102,18 +229,45 @@ return umfpack_zi_get_numeric(Lp,Lj,Lx?&lx0_real:0,0,Up,Ui,Ux?&ux0_real:0,0,P,Q, Dx?&dx0_real:0,0,do_recip,Rs,Numeric); } +inline SuiteSparse_long umfpack_get_numeric(SuiteSparse_long Lp[], SuiteSparse_long Lj[], double Lx[], SuiteSparse_long Up[], SuiteSparse_long Ui[], double Ux[], + SuiteSparse_long P[], SuiteSparse_long Q[], double Dx[], SuiteSparse_long *do_recip, double Rs[], void *Numeric) +{ + return umfpack_dl_get_numeric(Lp,Lj,Lx,Up,Ui,Ux,P,Q,Dx,do_recip,Rs,Numeric); +} -inline int umfpack_get_determinant(double *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO]) +inline SuiteSparse_long umfpack_get_numeric(SuiteSparse_long Lp[], SuiteSparse_long Lj[], std::complex<double> Lx[], SuiteSparse_long Up[], SuiteSparse_long Ui[], std::complex<double> Ux[], + SuiteSparse_long P[], SuiteSparse_long Q[], std::complex<double> Dx[], SuiteSparse_long *do_recip, double Rs[], void *Numeric) +{ + double& lx0_real = numext::real_ref(Lx[0]); + double& ux0_real = numext::real_ref(Ux[0]); + double& dx0_real = numext::real_ref(Dx[0]); + return umfpack_zl_get_numeric(Lp,Lj,Lx?&lx0_real:0,0,Up,Ui,Ux?&ux0_real:0,0,P,Q, + Dx?&dx0_real:0,0,do_recip,Rs,Numeric); +} + +// Get Determinant +inline int umfpack_get_determinant(double *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO], int) { return umfpack_di_get_determinant(Mx,Ex,NumericHandle,User_Info); } -inline int umfpack_get_determinant(std::complex<double> *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO]) +inline int umfpack_get_determinant(std::complex<double> *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO], int) { double& mx_real = numext::real_ref(*Mx); return umfpack_zi_get_determinant(&mx_real,0,Ex,NumericHandle,User_Info); } +inline SuiteSparse_long umfpack_get_determinant(double *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO], SuiteSparse_long) +{ + return umfpack_dl_get_determinant(Mx,Ex,NumericHandle,User_Info); +} + +inline SuiteSparse_long umfpack_get_determinant(std::complex<double> *Mx, double *Ex, void *NumericHandle, double User_Info [UMFPACK_INFO], SuiteSparse_long) +{ + double& mx_real = numext::real_ref(*Mx); + return umfpack_zl_get_determinant(&mx_real,0,Ex,NumericHandle,User_Info); +} + /** \ingroup UmfPackSupport_Module * \brief A sparse LU factorization and solver based on UmfPack @@ -146,7 +300,7 @@ typedef Matrix<int, 1, MatrixType::ColsAtCompileTime> IntRowVectorType; typedef Matrix<int, MatrixType::RowsAtCompileTime, 1> IntColVectorType; typedef SparseMatrix<Scalar> LUMatrixType; - typedef SparseMatrix<Scalar,ColMajor,int> UmfpackMatrixType; + typedef SparseMatrix<Scalar,ColMajor,StorageIndex> UmfpackMatrixType; typedef Ref<const UmfpackMatrixType, StandardCompressedFormat> UmfpackMatrixRef; enum { ColsAtCompileTime = MatrixType::ColsAtCompileTime, @@ -156,6 +310,7 @@ public: typedef Array<double, UMFPACK_CONTROL, 1> UmfpackControl; + typedef Array<double, UMFPACK_INFO, 1> UmfpackInfo; UmfPackLU() : m_dummy(0,0), mp_matrix(m_dummy) @@ -173,8 +328,8 @@ ~UmfPackLU() { - if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar()); - if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar()); + if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar(), StorageIndex()); + if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar(), StorageIndex()); } inline Index rows() const { return mp_matrix.rows(); } @@ -182,7 +337,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, + * \returns \c Success if computation was successful, * \c NumericalIssue if the matrix.appears to be negative. */ ComputationInfo info() const @@ -215,15 +370,15 @@ return m_q; } - /** Computes the sparse Cholesky decomposition of \a matrix + /** Computes the sparse Cholesky decomposition of \a matrix * Note that the matrix should be column-major, and in compressed format for best performance. * \sa SparseMatrix::makeCompressed(). */ template<typename InputMatrixType> void compute(const InputMatrixType& matrix) { - if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar()); - if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar()); + if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar(),StorageIndex()); + if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar(),StorageIndex()); grab(matrix.derived()); analyzePattern_impl(); factorize_impl(); @@ -238,9 +393,9 @@ template<typename InputMatrixType> void analyzePattern(const InputMatrixType& matrix) { - if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar()); - if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar()); - + if(m_symbolic) umfpack_free_symbolic(&m_symbolic,Scalar(),StorageIndex()); + if(m_numeric) umfpack_free_numeric(&m_numeric,Scalar(),StorageIndex()); + grab(matrix.derived()); analyzePattern_impl(); @@ -267,7 +422,7 @@ { return m_control; } - + /** Provides access to the control settings array used by UmfPack. * * If this array contains NaN's, the default values are used. @@ -278,7 +433,7 @@ { return m_control; } - + /** Performs a numeric decomposition of \a matrix * * The given matrix must has the same sparcity than the matrix on which the pattern anylysis has been performed. @@ -290,13 +445,41 @@ { eigen_assert(m_analysisIsOk && "UmfPackLU: you must first call analyzePattern()"); if(m_numeric) - umfpack_free_numeric(&m_numeric,Scalar()); + umfpack_free_numeric(&m_numeric,Scalar(),StorageIndex()); grab(matrix.derived()); - + factorize_impl(); } + /** Prints the current UmfPack control settings. + * + * \sa umfpackControl() + */ + void printUmfpackControl() + { + umfpack_report_control(m_control.data(), Scalar(),StorageIndex()); + } + + /** Prints statistics collected by UmfPack. + * + * \sa analyzePattern(), compute() + */ + void printUmfpackInfo() + { + eigen_assert(m_analysisIsOk && "UmfPackLU: you must first call analyzePattern()"); + umfpack_report_info(m_control.data(), m_umfpackInfo.data(), Scalar(),StorageIndex()); + } + + /** Prints the status of the previous factorization operation performed by UmfPack (symbolic or numerical factorization). + * + * \sa analyzePattern(), compute() + */ + void printUmfpackStatus() { + eigen_assert(m_analysisIsOk && "UmfPackLU: you must first call analyzePattern()"); + umfpack_report_status(m_control.data(), m_fact_errorCode, Scalar(),StorageIndex()); + } + /** \internal */ template<typename BDerived,typename XDerived> bool _solve_impl(const MatrixBase<BDerived> &b, MatrixBase<XDerived> &x) const; @@ -314,41 +497,42 @@ m_numeric = 0; m_symbolic = 0; m_extractedDataAreDirty = true; + + umfpack_defaults(m_control.data(), Scalar(),StorageIndex()); } - + void analyzePattern_impl() { - umfpack_defaults(m_control.data(), Scalar()); - int errorCode = 0; - errorCode = umfpack_symbolic(internal::convert_index<int>(mp_matrix.rows()), - internal::convert_index<int>(mp_matrix.cols()), - mp_matrix.outerIndexPtr(), mp_matrix.innerIndexPtr(), mp_matrix.valuePtr(), - &m_symbolic, m_control.data(), 0); + m_fact_errorCode = umfpack_symbolic(internal::convert_index<StorageIndex>(mp_matrix.rows()), + internal::convert_index<StorageIndex>(mp_matrix.cols()), + mp_matrix.outerIndexPtr(), mp_matrix.innerIndexPtr(), mp_matrix.valuePtr(), + &m_symbolic, m_control.data(), m_umfpackInfo.data()); m_isInitialized = true; - m_info = errorCode ? InvalidInput : Success; + m_info = m_fact_errorCode ? InvalidInput : Success; m_analysisIsOk = true; m_factorizationIsOk = false; m_extractedDataAreDirty = true; } - + void factorize_impl() { + m_fact_errorCode = umfpack_numeric(mp_matrix.outerIndexPtr(), mp_matrix.innerIndexPtr(), mp_matrix.valuePtr(), - m_symbolic, &m_numeric, m_control.data(), 0); + m_symbolic, &m_numeric, m_control.data(), m_umfpackInfo.data()); m_info = m_fact_errorCode == UMFPACK_OK ? Success : NumericalIssue; m_factorizationIsOk = true; m_extractedDataAreDirty = true; } - + template<typename MatrixDerived> void grab(const EigenBase<MatrixDerived> &A) { mp_matrix.~UmfpackMatrixRef(); ::new (&mp_matrix) UmfpackMatrixRef(A.derived()); } - + void grab(const UmfpackMatrixRef &A) { if(&(A.derived()) != &mp_matrix) @@ -357,19 +541,20 @@ ::new (&mp_matrix) UmfpackMatrixRef(A); } } - + // cached data to reduce reallocation, etc. mutable LUMatrixType m_l; - int m_fact_errorCode; + StorageIndex m_fact_errorCode; UmfpackControl m_control; - + mutable UmfpackInfo m_umfpackInfo; + mutable LUMatrixType m_u; mutable IntColVectorType m_p; mutable IntRowVectorType m_q; UmfpackMatrixType m_dummy; UmfpackMatrixRef mp_matrix; - + void* m_numeric; void* m_symbolic; @@ -377,9 +562,9 @@ int m_factorizationIsOk; int m_analysisIsOk; mutable bool m_extractedDataAreDirty; - + private: - UmfPackLU(UmfPackLU& ) { } + UmfPackLU(const UmfPackLU& ) { } }; @@ -389,7 +574,7 @@ if (m_extractedDataAreDirty) { // get size of the data - int lnz, unz, rows, cols, nz_udiag; + StorageIndex lnz, unz, rows, cols, nz_udiag; umfpack_get_lunz(&lnz, &unz, &rows, &cols, &nz_udiag, m_numeric, Scalar()); // allocate data @@ -415,7 +600,7 @@ typename UmfPackLU<MatrixType>::Scalar UmfPackLU<MatrixType>::determinant() const { Scalar det; - umfpack_get_determinant(&det, 0, m_numeric, 0); + umfpack_get_determinant(&det, 0, m_numeric, 0, StorageIndex()); return det; } @@ -427,8 +612,7 @@ eigen_assert((BDerived::Flags&RowMajorBit)==0 && "UmfPackLU backend does not support non col-major rhs yet"); eigen_assert((XDerived::Flags&RowMajorBit)==0 && "UmfPackLU backend does not support non col-major result yet"); eigen_assert(b.derived().data() != x.derived().data() && " Umfpack does not support inplace solve"); - - int errorCode; + Scalar* x_ptr = 0; Matrix<Scalar,Dynamic,1> x_tmp; if(x.innerStride()!=1) @@ -440,9 +624,10 @@ { if(x.innerStride()==1) x_ptr = &x.col(j).coeffRef(0); - errorCode = umfpack_solve(UMFPACK_A, - mp_matrix.outerIndexPtr(), mp_matrix.innerIndexPtr(), mp_matrix.valuePtr(), - x_ptr, &b.const_cast_derived().col(j).coeffRef(0), m_numeric, m_control.data(), 0); + StorageIndex errorCode = umfpack_solve(UMFPACK_A, + mp_matrix.outerIndexPtr(), mp_matrix.innerIndexPtr(), mp_matrix.valuePtr(), + x_ptr, &b.const_cast_derived().col(j).coeffRef(0), + m_numeric, m_control.data(), m_umfpackInfo.data()); if(x.innerStride()!=1) x.col(j) = x_tmp; if (errorCode!=0)
diff --git a/Eigen/src/misc/CMakeLists.txt b/Eigen/src/misc/CMakeLists.txt deleted file mode 100644 index a58ffb7..0000000 --- a/Eigen/src/misc/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_misc_SRCS "*.h") - -INSTALL(FILES - ${Eigen_misc_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/misc COMPONENT Devel - )
diff --git a/Eigen/src/misc/RealSvd2x2.h b/Eigen/src/misc/RealSvd2x2.h new file mode 100644 index 0000000..abb4d3c --- /dev/null +++ b/Eigen/src/misc/RealSvd2x2.h
@@ -0,0 +1,55 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2009-2010 Benoit Jacob <jacob.benoit.1@gmail.com> +// Copyright (C) 2013-2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_REALSVD2X2_H +#define EIGEN_REALSVD2X2_H + +namespace Eigen { + +namespace internal { + +template<typename MatrixType, typename RealScalar, typename Index> +void real_2x2_jacobi_svd(const MatrixType& matrix, Index p, Index q, + JacobiRotation<RealScalar> *j_left, + JacobiRotation<RealScalar> *j_right) +{ + using std::sqrt; + using std::abs; + Matrix<RealScalar,2,2> m; + m << numext::real(matrix.coeff(p,p)), numext::real(matrix.coeff(p,q)), + numext::real(matrix.coeff(q,p)), numext::real(matrix.coeff(q,q)); + JacobiRotation<RealScalar> rot1; + RealScalar t = m.coeff(0,0) + m.coeff(1,1); + RealScalar d = m.coeff(1,0) - m.coeff(0,1); + + if(abs(d) < (std::numeric_limits<RealScalar>::min)()) + { + rot1.s() = RealScalar(0); + rot1.c() = RealScalar(1); + } + else + { + // If d!=0, then t/d cannot overflow because the magnitude of the + // entries forming d are not too small compared to the ones forming t. + RealScalar u = t / d; + RealScalar tmp = sqrt(RealScalar(1) + numext::abs2(u)); + rot1.s() = RealScalar(1) / tmp; + rot1.c() = u / tmp; + } + m.applyOnTheLeft(0,1,rot1); + j_right->makeJacobi(m,0,1); + *j_left = rot1 * j_right->transpose(); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_REALSVD2X2_H
diff --git a/Eigen/src/misc/lapacke.h b/Eigen/src/misc/lapacke.h new file mode 100755 index 0000000..3d8e24f --- /dev/null +++ b/Eigen/src/misc/lapacke.h
@@ -0,0 +1,16292 @@ +/***************************************************************************** + Copyright (c) 2010, Intel Corp. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. +****************************************************************************** +* Contents: Native C interface to LAPACK +* Author: Intel Corporation +* Generated November, 2011 +*****************************************************************************/ + +#ifndef _MKL_LAPACKE_H_ + +#ifndef _LAPACKE_H_ +#define _LAPACKE_H_ + +/* +* Turn on HAVE_LAPACK_CONFIG_H to redefine C-LAPACK datatypes +*/ +#ifdef HAVE_LAPACK_CONFIG_H +#include "lapacke_config.h" +#endif + +#include <stdlib.h> + +#ifndef lapack_int +#define lapack_int int +#endif + +#ifndef lapack_logical +#define lapack_logical lapack_int +#endif + +/* Complex types are structures equivalent to the +* Fortran complex types COMPLEX(4) and COMPLEX(8). +* +* One can also redefine the types with his own types +* for example by including in the code definitions like +* +* #define lapack_complex_float std::complex<float> +* #define lapack_complex_double std::complex<double> +* +* or define these types in the command line: +* +* -Dlapack_complex_float="std::complex<float>" +* -Dlapack_complex_double="std::complex<double>" +*/ + +#ifndef LAPACK_COMPLEX_CUSTOM + +/* Complex type (single precision) */ +#ifndef lapack_complex_float +#include <complex.h> +#define lapack_complex_float float _Complex +#endif + +#ifndef lapack_complex_float_real +#define lapack_complex_float_real(z) (creal(z)) +#endif + +#ifndef lapack_complex_float_imag +#define lapack_complex_float_imag(z) (cimag(z)) +#endif + +lapack_complex_float lapack_make_complex_float( float re, float im ); + +/* Complex type (double precision) */ +#ifndef lapack_complex_double +#include <complex.h> +#define lapack_complex_double double _Complex +#endif + +#ifndef lapack_complex_double_real +#define lapack_complex_double_real(z) (creal(z)) +#endif + +#ifndef lapack_complex_double_imag +#define lapack_complex_double_imag(z) (cimag(z)) +#endif + +lapack_complex_double lapack_make_complex_double( double re, double im ); + +#endif + + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef LAPACKE_malloc +#define LAPACKE_malloc( size ) malloc( size ) +#endif +#ifndef LAPACKE_free +#define LAPACKE_free( p ) free( p ) +#endif + +#define LAPACK_C2INT( x ) (lapack_int)(*((float*)&x )) +#define LAPACK_Z2INT( x ) (lapack_int)(*((double*)&x )) + +#define LAPACK_ROW_MAJOR 101 +#define LAPACK_COL_MAJOR 102 + +#define LAPACK_WORK_MEMORY_ERROR -1010 +#define LAPACK_TRANSPOSE_MEMORY_ERROR -1011 + +/* Callback logical functions of one, two, or three arguments are used +* to select eigenvalues to sort to the top left of the Schur form. +* The value is selected if function returns TRUE (non-zero). */ + +typedef lapack_logical (*LAPACK_S_SELECT2) ( const float*, const float* ); +typedef lapack_logical (*LAPACK_S_SELECT3) + ( const float*, const float*, const float* ); +typedef lapack_logical (*LAPACK_D_SELECT2) ( const double*, const double* ); +typedef lapack_logical (*LAPACK_D_SELECT3) + ( const double*, const double*, const double* ); + +typedef lapack_logical (*LAPACK_C_SELECT1) ( const lapack_complex_float* ); +typedef lapack_logical (*LAPACK_C_SELECT2) + ( const lapack_complex_float*, const lapack_complex_float* ); +typedef lapack_logical (*LAPACK_Z_SELECT1) ( const lapack_complex_double* ); +typedef lapack_logical (*LAPACK_Z_SELECT2) + ( const lapack_complex_double*, const lapack_complex_double* ); + +#include "lapacke_mangling.h" + +#define LAPACK_lsame LAPACK_GLOBAL(lsame,LSAME) +lapack_logical LAPACK_lsame( char* ca, char* cb, + lapack_int lca, lapack_int lcb ); + +/* C-LAPACK function prototypes */ + +lapack_int LAPACKE_sbdsdc( int matrix_order, char uplo, char compq, + lapack_int n, float* d, float* e, float* u, + lapack_int ldu, float* vt, lapack_int ldvt, float* q, + lapack_int* iq ); +lapack_int LAPACKE_dbdsdc( int matrix_order, char uplo, char compq, + lapack_int n, double* d, double* e, double* u, + lapack_int ldu, double* vt, lapack_int ldvt, + double* q, lapack_int* iq ); + +lapack_int LAPACKE_sbdsqr( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + float* d, float* e, float* vt, lapack_int ldvt, + float* u, lapack_int ldu, float* c, lapack_int ldc ); +lapack_int LAPACKE_dbdsqr( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + double* d, double* e, double* vt, lapack_int ldvt, + double* u, lapack_int ldu, double* c, + lapack_int ldc ); +lapack_int LAPACKE_cbdsqr( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + float* d, float* e, lapack_complex_float* vt, + lapack_int ldvt, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* c, + lapack_int ldc ); +lapack_int LAPACKE_zbdsqr( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + double* d, double* e, lapack_complex_double* vt, + lapack_int ldvt, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* c, + lapack_int ldc ); + +lapack_int LAPACKE_sdisna( char job, lapack_int m, lapack_int n, const float* d, + float* sep ); +lapack_int LAPACKE_ddisna( char job, lapack_int m, lapack_int n, + const double* d, double* sep ); + +lapack_int LAPACKE_sgbbrd( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, float* ab, lapack_int ldab, float* d, + float* e, float* q, lapack_int ldq, float* pt, + lapack_int ldpt, float* c, lapack_int ldc ); +lapack_int LAPACKE_dgbbrd( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, double* ab, lapack_int ldab, + double* d, double* e, double* q, lapack_int ldq, + double* pt, lapack_int ldpt, double* c, + lapack_int ldc ); +lapack_int LAPACKE_cgbbrd( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, lapack_complex_float* ab, + lapack_int ldab, float* d, float* e, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* pt, lapack_int ldpt, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zgbbrd( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, lapack_complex_double* ab, + lapack_int ldab, double* d, double* e, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* pt, lapack_int ldpt, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_sgbcon( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, const lapack_int* ipiv, float anorm, + float* rcond ); +lapack_int LAPACKE_dgbcon( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, const lapack_int* ipiv, + double anorm, double* rcond ); +lapack_int LAPACKE_cgbcon( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zgbcon( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, lapack_int ldab, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_sgbequ( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, float* r, float* c, float* rowcnd, + float* colcnd, float* amax ); +lapack_int LAPACKE_dgbequ( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); +lapack_int LAPACKE_cgbequ( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, lapack_int ldab, + float* r, float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_zgbequ( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, lapack_int ldab, + double* r, double* c, double* rowcnd, double* colcnd, + double* amax ); + +lapack_int LAPACKE_sgbequb( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, float* r, float* c, float* rowcnd, + float* colcnd, float* amax ); +lapack_int LAPACKE_dgbequb( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); +lapack_int LAPACKE_cgbequb( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, lapack_int ldab, + float* r, float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_zgbequb( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, lapack_int ldab, + double* r, double* c, double* rowcnd, + double* colcnd, double* amax ); + +lapack_int LAPACKE_sgbrfs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const float* ab, lapack_int ldab, const float* afb, + lapack_int ldafb, const lapack_int* ipiv, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_dgbrfs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const double* ab, lapack_int ldab, const double* afb, + lapack_int ldafb, const lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr ); +lapack_int LAPACKE_cgbrfs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* afb, lapack_int ldafb, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zgbrfs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + const lapack_complex_double* afb, lapack_int ldafb, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sgbrfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const float* ab, lapack_int ldab, + const float* afb, lapack_int ldafb, + const lapack_int* ipiv, const float* r, + const float* c, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dgbrfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const double* ab, lapack_int ldab, + const double* afb, lapack_int ldafb, + const lapack_int* ipiv, const double* r, + const double* c, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); +lapack_int LAPACKE_cgbrfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const lapack_complex_float* ab, + lapack_int ldab, const lapack_complex_float* afb, + lapack_int ldafb, const lapack_int* ipiv, + const float* r, const float* c, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_zgbrfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const lapack_complex_double* ab, + lapack_int ldab, const lapack_complex_double* afb, + lapack_int ldafb, const lapack_int* ipiv, + const double* r, const double* c, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); + +lapack_int LAPACKE_sgbsv( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, float* ab, + lapack_int ldab, lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgbsv( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, double* ab, + lapack_int ldab, lapack_int* ipiv, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cgbsv( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgbsv( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sgbsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, float* ab, lapack_int ldab, + float* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, float* r, float* c, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* rpivot ); +lapack_int LAPACKE_dgbsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, double* ab, lapack_int ldab, + double* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, double* r, double* c, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* rpivot ); +lapack_int LAPACKE_cgbsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + float* r, float* c, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, float* rpivot ); +lapack_int LAPACKE_zgbsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + double* r, double* c, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, double* rpivot ); + +lapack_int LAPACKE_sgbsvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, float* ab, lapack_int ldab, + float* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, float* r, float* c, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dgbsvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, double* ab, lapack_int ldab, + double* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, double* r, double* c, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); +lapack_int LAPACKE_cgbsvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + float* r, float* c, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_zgbsvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + double* r, double* c, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); + +lapack_int LAPACKE_sgbtrf( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, float* ab, + lapack_int ldab, lapack_int* ipiv ); +lapack_int LAPACKE_dgbtrf( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, double* ab, + lapack_int ldab, lapack_int* ipiv ); +lapack_int LAPACKE_cgbtrf( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + lapack_complex_float* ab, lapack_int ldab, + lapack_int* ipiv ); +lapack_int LAPACKE_zgbtrf( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + lapack_complex_double* ab, lapack_int ldab, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgbtrs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const float* ab, lapack_int ldab, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dgbtrs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const double* ab, lapack_int ldab, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_cgbtrs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgbtrs( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sgebak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const float* scale, + lapack_int m, float* v, lapack_int ldv ); +lapack_int LAPACKE_dgebak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const double* scale, + lapack_int m, double* v, lapack_int ldv ); +lapack_int LAPACKE_cgebak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const float* scale, + lapack_int m, lapack_complex_float* v, + lapack_int ldv ); +lapack_int LAPACKE_zgebak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const double* scale, + lapack_int m, lapack_complex_double* v, + lapack_int ldv ); + +lapack_int LAPACKE_sgebal( int matrix_order, char job, lapack_int n, float* a, + lapack_int lda, lapack_int* ilo, lapack_int* ihi, + float* scale ); +lapack_int LAPACKE_dgebal( int matrix_order, char job, lapack_int n, double* a, + lapack_int lda, lapack_int* ilo, lapack_int* ihi, + double* scale ); +lapack_int LAPACKE_cgebal( int matrix_order, char job, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ilo, lapack_int* ihi, float* scale ); +lapack_int LAPACKE_zgebal( int matrix_order, char job, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ilo, lapack_int* ihi, double* scale ); + +lapack_int LAPACKE_sgebrd( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* d, float* e, + float* tauq, float* taup ); +lapack_int LAPACKE_dgebrd( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* d, double* e, + double* tauq, double* taup ); +lapack_int LAPACKE_cgebrd( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, float* d, + float* e, lapack_complex_float* tauq, + lapack_complex_float* taup ); +lapack_int LAPACKE_zgebrd( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, double* d, + double* e, lapack_complex_double* tauq, + lapack_complex_double* taup ); + +lapack_int LAPACKE_sgecon( int matrix_order, char norm, lapack_int n, + const float* a, lapack_int lda, float anorm, + float* rcond ); +lapack_int LAPACKE_dgecon( int matrix_order, char norm, lapack_int n, + const double* a, lapack_int lda, double anorm, + double* rcond ); +lapack_int LAPACKE_cgecon( int matrix_order, char norm, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float anorm, float* rcond ); +lapack_int LAPACKE_zgecon( int matrix_order, char norm, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double anorm, double* rcond ); + +lapack_int LAPACKE_sgeequ( int matrix_order, lapack_int m, lapack_int n, + const float* a, lapack_int lda, float* r, float* c, + float* rowcnd, float* colcnd, float* amax ); +lapack_int LAPACKE_dgeequ( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, double* r, + double* c, double* rowcnd, double* colcnd, + double* amax ); +lapack_int LAPACKE_cgeequ( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* r, float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_zgeequ( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* r, double* c, double* rowcnd, double* colcnd, + double* amax ); + +lapack_int LAPACKE_sgeequb( int matrix_order, lapack_int m, lapack_int n, + const float* a, lapack_int lda, float* r, float* c, + float* rowcnd, float* colcnd, float* amax ); +lapack_int LAPACKE_dgeequb( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, double* r, + double* c, double* rowcnd, double* colcnd, + double* amax ); +lapack_int LAPACKE_cgeequb( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* r, float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_zgeequb( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* r, double* c, double* rowcnd, + double* colcnd, double* amax ); + +lapack_int LAPACKE_sgees( int matrix_order, char jobvs, char sort, + LAPACK_S_SELECT2 select, lapack_int n, float* a, + lapack_int lda, lapack_int* sdim, float* wr, + float* wi, float* vs, lapack_int ldvs ); +lapack_int LAPACKE_dgees( int matrix_order, char jobvs, char sort, + LAPACK_D_SELECT2 select, lapack_int n, double* a, + lapack_int lda, lapack_int* sdim, double* wr, + double* wi, double* vs, lapack_int ldvs ); +lapack_int LAPACKE_cgees( int matrix_order, char jobvs, char sort, + LAPACK_C_SELECT1 select, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* sdim, lapack_complex_float* w, + lapack_complex_float* vs, lapack_int ldvs ); +lapack_int LAPACKE_zgees( int matrix_order, char jobvs, char sort, + LAPACK_Z_SELECT1 select, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* sdim, lapack_complex_double* w, + lapack_complex_double* vs, lapack_int ldvs ); + +lapack_int LAPACKE_sgeesx( int matrix_order, char jobvs, char sort, + LAPACK_S_SELECT2 select, char sense, lapack_int n, + float* a, lapack_int lda, lapack_int* sdim, + float* wr, float* wi, float* vs, lapack_int ldvs, + float* rconde, float* rcondv ); +lapack_int LAPACKE_dgeesx( int matrix_order, char jobvs, char sort, + LAPACK_D_SELECT2 select, char sense, lapack_int n, + double* a, lapack_int lda, lapack_int* sdim, + double* wr, double* wi, double* vs, lapack_int ldvs, + double* rconde, double* rcondv ); +lapack_int LAPACKE_cgeesx( int matrix_order, char jobvs, char sort, + LAPACK_C_SELECT1 select, char sense, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* sdim, lapack_complex_float* w, + lapack_complex_float* vs, lapack_int ldvs, + float* rconde, float* rcondv ); +lapack_int LAPACKE_zgeesx( int matrix_order, char jobvs, char sort, + LAPACK_Z_SELECT1 select, char sense, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* sdim, lapack_complex_double* w, + lapack_complex_double* vs, lapack_int ldvs, + double* rconde, double* rcondv ); + +lapack_int LAPACKE_sgeev( int matrix_order, char jobvl, char jobvr, + lapack_int n, float* a, lapack_int lda, float* wr, + float* wi, float* vl, lapack_int ldvl, float* vr, + lapack_int ldvr ); +lapack_int LAPACKE_dgeev( int matrix_order, char jobvl, char jobvr, + lapack_int n, double* a, lapack_int lda, double* wr, + double* wi, double* vl, lapack_int ldvl, double* vr, + lapack_int ldvr ); +lapack_int LAPACKE_cgeev( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_float* a, lapack_int lda, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int ldvl, lapack_complex_float* vr, + lapack_int ldvr ); +lapack_int LAPACKE_zgeev( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* w, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr ); + +lapack_int LAPACKE_sgeevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, float* a, + lapack_int lda, float* wr, float* wi, float* vl, + lapack_int ldvl, float* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, float* scale, + float* abnrm, float* rconde, float* rcondv ); +lapack_int LAPACKE_dgeevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, double* a, + lapack_int lda, double* wr, double* wi, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, double* scale, + double* abnrm, double* rconde, double* rcondv ); +lapack_int LAPACKE_cgeevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int ldvl, lapack_complex_float* vr, + lapack_int ldvr, lapack_int* ilo, lapack_int* ihi, + float* scale, float* abnrm, float* rconde, + float* rcondv ); +lapack_int LAPACKE_zgeevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* w, lapack_complex_double* vl, + lapack_int ldvl, lapack_complex_double* vr, + lapack_int ldvr, lapack_int* ilo, lapack_int* ihi, + double* scale, double* abnrm, double* rconde, + double* rcondv ); + +lapack_int LAPACKE_sgehrd( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, float* a, lapack_int lda, + float* tau ); +lapack_int LAPACKE_dgehrd( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, double* a, lapack_int lda, + double* tau ); +lapack_int LAPACKE_cgehrd( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* tau ); +lapack_int LAPACKE_zgehrd( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* tau ); + +lapack_int LAPACKE_sgejsv( int matrix_order, char joba, char jobu, char jobv, + char jobr, char jobt, char jobp, lapack_int m, + lapack_int n, float* a, lapack_int lda, float* sva, + float* u, lapack_int ldu, float* v, lapack_int ldv, + float* stat, lapack_int* istat ); +lapack_int LAPACKE_dgejsv( int matrix_order, char joba, char jobu, char jobv, + char jobr, char jobt, char jobp, lapack_int m, + lapack_int n, double* a, lapack_int lda, double* sva, + double* u, lapack_int ldu, double* v, lapack_int ldv, + double* stat, lapack_int* istat ); + +lapack_int LAPACKE_sgelq2( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgelq2( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgelq2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgelq2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgelqf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgelqf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgelqf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgelqf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgels( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* b, lapack_int ldb ); +lapack_int LAPACKE_dgels( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* b, lapack_int ldb ); +lapack_int LAPACKE_cgels( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zgels( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sgelsd( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_dgelsd( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* s, double rcond, + lapack_int* rank ); +lapack_int LAPACKE_cgelsd( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_zgelsd( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* s, double rcond, + lapack_int* rank ); + +lapack_int LAPACKE_sgelss( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_dgelss( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* s, double rcond, + lapack_int* rank ); +lapack_int LAPACKE_cgelss( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_zgelss( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* s, double rcond, + lapack_int* rank ); + +lapack_int LAPACKE_sgelsy( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, float* b, + lapack_int ldb, lapack_int* jpvt, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_dgelsy( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, lapack_int* jpvt, + double rcond, lapack_int* rank ); +lapack_int LAPACKE_cgelsy( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_int* jpvt, float rcond, + lapack_int* rank ); +lapack_int LAPACKE_zgelsy( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_int* jpvt, double rcond, + lapack_int* rank ); + +lapack_int LAPACKE_sgeqlf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgeqlf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgeqlf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqlf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgeqp3( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* jpvt, + float* tau ); +lapack_int LAPACKE_dgeqp3( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* jpvt, + double* tau ); +lapack_int LAPACKE_cgeqp3( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqp3( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_double* tau ); + +lapack_int LAPACKE_sgeqpf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* jpvt, + float* tau ); +lapack_int LAPACKE_dgeqpf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* jpvt, + double* tau ); +lapack_int LAPACKE_cgeqpf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqpf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_double* tau ); + +lapack_int LAPACKE_sgeqr2( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgeqr2( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgeqr2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqr2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgeqrf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgeqrf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgeqrf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqrf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgeqrfp( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgeqrfp( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgeqrfp( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgeqrfp( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgerfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dgerfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr ); +lapack_int LAPACKE_cgerfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zgerfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sgerfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* r, + const float* c, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dgerfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* r, + const double* c, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); +lapack_int LAPACKE_cgerfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* r, + const float* c, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_zgerfsx( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* r, + const double* c, const lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); + +lapack_int LAPACKE_sgerqf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dgerqf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_cgerqf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_zgerqf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_sgesdd( int matrix_order, char jobz, lapack_int m, + lapack_int n, float* a, lapack_int lda, float* s, + float* u, lapack_int ldu, float* vt, + lapack_int ldvt ); +lapack_int LAPACKE_dgesdd( int matrix_order, char jobz, lapack_int m, + lapack_int n, double* a, lapack_int lda, double* s, + double* u, lapack_int ldu, double* vt, + lapack_int ldvt ); +lapack_int LAPACKE_cgesdd( int matrix_order, char jobz, lapack_int m, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float* s, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* vt, + lapack_int ldvt ); +lapack_int LAPACKE_zgesdd( int matrix_order, char jobz, lapack_int m, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double* s, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* vt, + lapack_int ldvt ); + +lapack_int LAPACKE_sgesv( int matrix_order, lapack_int n, lapack_int nrhs, + float* a, lapack_int lda, lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgesv( int matrix_order, lapack_int n, lapack_int nrhs, + double* a, lapack_int lda, lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cgesv( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgesv( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); +lapack_int LAPACKE_dsgesv( int matrix_order, lapack_int n, lapack_int nrhs, + double* a, lapack_int lda, lapack_int* ipiv, + double* b, lapack_int ldb, double* x, lapack_int ldx, + lapack_int* iter ); +lapack_int LAPACKE_zcgesv( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, lapack_int* iter ); + +lapack_int LAPACKE_sgesvd( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, float* a, lapack_int lda, + float* s, float* u, lapack_int ldu, float* vt, + lapack_int ldvt, float* superb ); +lapack_int LAPACKE_dgesvd( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, double* a, + lapack_int lda, double* s, double* u, lapack_int ldu, + double* vt, lapack_int ldvt, double* superb ); +lapack_int LAPACKE_cgesvd( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, lapack_complex_float* a, + lapack_int lda, float* s, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* vt, + lapack_int ldvt, float* superb ); +lapack_int LAPACKE_zgesvd( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, lapack_complex_double* a, + lapack_int lda, double* s, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* vt, + lapack_int ldvt, double* superb ); + +lapack_int LAPACKE_sgesvj( int matrix_order, char joba, char jobu, char jobv, + lapack_int m, lapack_int n, float* a, lapack_int lda, + float* sva, lapack_int mv, float* v, lapack_int ldv, + float* stat ); +lapack_int LAPACKE_dgesvj( int matrix_order, char joba, char jobu, char jobv, + lapack_int m, lapack_int n, double* a, + lapack_int lda, double* sva, lapack_int mv, + double* v, lapack_int ldv, double* stat ); + +lapack_int LAPACKE_sgesvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + float* b, lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* rpivot ); +lapack_int LAPACKE_dgesvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* rpivot ); +lapack_int LAPACKE_cgesvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* rpivot ); +lapack_int LAPACKE_zgesvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* rpivot ); + +lapack_int LAPACKE_sgesvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + float* b, lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dgesvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); +lapack_int LAPACKE_cgesvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_zgesvxx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); + +lapack_int LAPACKE_sgetf2( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_dgetf2( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_cgetf2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zgetf2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgetrf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_dgetrf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_cgetrf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zgetrf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgetri( int matrix_order, lapack_int n, float* a, + lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_dgetri( int matrix_order, lapack_int n, double* a, + lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_cgetri( int matrix_order, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zgetri( int matrix_order, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); + +lapack_int LAPACKE_sgetrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dgetrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_cgetrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zgetrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sggbak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const float* lscale, + const float* rscale, lapack_int m, float* v, + lapack_int ldv ); +lapack_int LAPACKE_dggbak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const double* lscale, + const double* rscale, lapack_int m, double* v, + lapack_int ldv ); +lapack_int LAPACKE_cggbak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const float* lscale, + const float* rscale, lapack_int m, + lapack_complex_float* v, lapack_int ldv ); +lapack_int LAPACKE_zggbak( int matrix_order, char job, char side, lapack_int n, + lapack_int ilo, lapack_int ihi, const double* lscale, + const double* rscale, lapack_int m, + lapack_complex_double* v, lapack_int ldv ); + +lapack_int LAPACKE_sggbal( int matrix_order, char job, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale ); +lapack_int LAPACKE_dggbal( int matrix_order, char job, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, double* lscale, + double* rscale ); +lapack_int LAPACKE_cggbal( int matrix_order, char job, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale ); +lapack_int LAPACKE_zggbal( int matrix_order, char job, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, double* lscale, + double* rscale ); + +lapack_int LAPACKE_sgges( int matrix_order, char jobvsl, char jobvsr, char sort, + LAPACK_S_SELECT3 selctg, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + lapack_int* sdim, float* alphar, float* alphai, + float* beta, float* vsl, lapack_int ldvsl, float* vsr, + lapack_int ldvsr ); +lapack_int LAPACKE_dgges( int matrix_order, char jobvsl, char jobvsr, char sort, + LAPACK_D_SELECT3 selctg, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + lapack_int* sdim, double* alphar, double* alphai, + double* beta, double* vsl, lapack_int ldvsl, + double* vsr, lapack_int ldvsr ); +lapack_int LAPACKE_cgges( int matrix_order, char jobvsl, char jobvsr, char sort, + LAPACK_C_SELECT2 selctg, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_int* sdim, lapack_complex_float* alpha, + lapack_complex_float* beta, lapack_complex_float* vsl, + lapack_int ldvsl, lapack_complex_float* vsr, + lapack_int ldvsr ); +lapack_int LAPACKE_zgges( int matrix_order, char jobvsl, char jobvsr, char sort, + LAPACK_Z_SELECT2 selctg, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_int* sdim, lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int ldvsl, + lapack_complex_double* vsr, lapack_int ldvsr ); + +lapack_int LAPACKE_sggesx( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_S_SELECT3 selctg, char sense, + lapack_int n, float* a, lapack_int lda, float* b, + lapack_int ldb, lapack_int* sdim, float* alphar, + float* alphai, float* beta, float* vsl, + lapack_int ldvsl, float* vsr, lapack_int ldvsr, + float* rconde, float* rcondv ); +lapack_int LAPACKE_dggesx( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_D_SELECT3 selctg, char sense, + lapack_int n, double* a, lapack_int lda, double* b, + lapack_int ldb, lapack_int* sdim, double* alphar, + double* alphai, double* beta, double* vsl, + lapack_int ldvsl, double* vsr, lapack_int ldvsr, + double* rconde, double* rcondv ); +lapack_int LAPACKE_cggesx( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_C_SELECT2 selctg, char sense, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_int* sdim, + lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* vsl, lapack_int ldvsl, + lapack_complex_float* vsr, lapack_int ldvsr, + float* rconde, float* rcondv ); +lapack_int LAPACKE_zggesx( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_Z_SELECT2 selctg, char sense, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_int* sdim, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int ldvsl, + lapack_complex_double* vsr, lapack_int ldvsr, + double* rconde, double* rcondv ); + +lapack_int LAPACKE_sggev( int matrix_order, char jobvl, char jobvr, + lapack_int n, float* a, lapack_int lda, float* b, + lapack_int ldb, float* alphar, float* alphai, + float* beta, float* vl, lapack_int ldvl, float* vr, + lapack_int ldvr ); +lapack_int LAPACKE_dggev( int matrix_order, char jobvl, char jobvr, + lapack_int n, double* a, lapack_int lda, double* b, + lapack_int ldb, double* alphar, double* alphai, + double* beta, double* vl, lapack_int ldvl, double* vr, + lapack_int ldvr ); +lapack_int LAPACKE_cggev( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* alpha, + lapack_complex_float* beta, lapack_complex_float* vl, + lapack_int ldvl, lapack_complex_float* vr, + lapack_int ldvr ); +lapack_int LAPACKE_zggev( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr ); + +lapack_int LAPACKE_sggevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* alphar, float* alphai, float* beta, float* vl, + lapack_int ldvl, float* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale, float* abnrm, float* bbnrm, + float* rconde, float* rcondv ); +lapack_int LAPACKE_dggevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* alphar, double* alphai, double* beta, + double* vl, lapack_int ldvl, double* vr, + lapack_int ldvr, lapack_int* ilo, lapack_int* ihi, + double* lscale, double* rscale, double* abnrm, + double* bbnrm, double* rconde, double* rcondv ); +lapack_int LAPACKE_cggevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* alpha, + lapack_complex_float* beta, lapack_complex_float* vl, + lapack_int ldvl, lapack_complex_float* vr, + lapack_int ldvr, lapack_int* ilo, lapack_int* ihi, + float* lscale, float* rscale, float* abnrm, + float* bbnrm, float* rconde, float* rcondv ); +lapack_int LAPACKE_zggevx( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, double* lscale, + double* rscale, double* abnrm, double* bbnrm, + double* rconde, double* rcondv ); + +lapack_int LAPACKE_sggglm( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, float* a, lapack_int lda, float* b, + lapack_int ldb, float* d, float* x, float* y ); +lapack_int LAPACKE_dggglm( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, double* a, lapack_int lda, double* b, + lapack_int ldb, double* d, double* x, double* y ); +lapack_int LAPACKE_cggglm( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* d, + lapack_complex_float* x, lapack_complex_float* y ); +lapack_int LAPACKE_zggglm( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* d, + lapack_complex_double* x, lapack_complex_double* y ); + +lapack_int LAPACKE_sgghrd( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + float* a, lapack_int lda, float* b, lapack_int ldb, + float* q, lapack_int ldq, float* z, lapack_int ldz ); +lapack_int LAPACKE_dgghrd( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + double* a, lapack_int lda, double* b, lapack_int ldb, + double* q, lapack_int ldq, double* z, + lapack_int ldz ); +lapack_int LAPACKE_cgghrd( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zgghrd( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_sgglse( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, float* a, lapack_int lda, float* b, + lapack_int ldb, float* c, float* d, float* x ); +lapack_int LAPACKE_dgglse( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, double* a, lapack_int lda, double* b, + lapack_int ldb, double* c, double* d, double* x ); +lapack_int LAPACKE_cgglse( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* c, + lapack_complex_float* d, lapack_complex_float* x ); +lapack_int LAPACKE_zgglse( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* c, + lapack_complex_double* d, lapack_complex_double* x ); + +lapack_int LAPACKE_sggqrf( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, float* a, lapack_int lda, float* taua, + float* b, lapack_int ldb, float* taub ); +lapack_int LAPACKE_dggqrf( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, double* a, lapack_int lda, + double* taua, double* b, lapack_int ldb, + double* taub ); +lapack_int LAPACKE_cggqrf( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* taua, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* taub ); +lapack_int LAPACKE_zggqrf( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* taua, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* taub ); + +lapack_int LAPACKE_sggrqf( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, float* a, lapack_int lda, float* taua, + float* b, lapack_int ldb, float* taub ); +lapack_int LAPACKE_dggrqf( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, double* a, lapack_int lda, + double* taua, double* b, lapack_int ldb, + double* taub ); +lapack_int LAPACKE_cggrqf( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* taua, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* taub ); +lapack_int LAPACKE_zggrqf( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* taua, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* taub ); + +lapack_int LAPACKE_sggsvd( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int n, lapack_int p, + lapack_int* k, lapack_int* l, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* alpha, float* beta, float* u, lapack_int ldu, + float* v, lapack_int ldv, float* q, lapack_int ldq, + lapack_int* iwork ); +lapack_int LAPACKE_dggsvd( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int n, lapack_int p, + lapack_int* k, lapack_int* l, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* alpha, double* beta, double* u, + lapack_int ldu, double* v, lapack_int ldv, double* q, + lapack_int ldq, lapack_int* iwork ); +lapack_int LAPACKE_cggsvd( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int n, lapack_int p, + lapack_int* k, lapack_int* l, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + float* alpha, float* beta, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* v, + lapack_int ldv, lapack_complex_float* q, + lapack_int ldq, lapack_int* iwork ); +lapack_int LAPACKE_zggsvd( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int n, lapack_int p, + lapack_int* k, lapack_int* l, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double* alpha, double* beta, + lapack_complex_double* u, lapack_int ldu, + lapack_complex_double* v, lapack_int ldv, + lapack_complex_double* q, lapack_int ldq, + lapack_int* iwork ); + +lapack_int LAPACKE_sggsvp( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, float tola, + float tolb, lapack_int* k, lapack_int* l, float* u, + lapack_int ldu, float* v, lapack_int ldv, float* q, + lapack_int ldq ); +lapack_int LAPACKE_dggsvp( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double tola, double tolb, lapack_int* k, + lapack_int* l, double* u, lapack_int ldu, double* v, + lapack_int ldv, double* q, lapack_int ldq ); +lapack_int LAPACKE_cggsvp( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, float tola, + float tolb, lapack_int* k, lapack_int* l, + lapack_complex_float* u, lapack_int ldu, + lapack_complex_float* v, lapack_int ldv, + lapack_complex_float* q, lapack_int ldq ); +lapack_int LAPACKE_zggsvp( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double tola, double tolb, lapack_int* k, + lapack_int* l, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* v, + lapack_int ldv, lapack_complex_double* q, + lapack_int ldq ); + +lapack_int LAPACKE_sgtcon( char norm, lapack_int n, const float* dl, + const float* d, const float* du, const float* du2, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_dgtcon( char norm, lapack_int n, const double* dl, + const double* d, const double* du, const double* du2, + const lapack_int* ipiv, double anorm, + double* rcond ); +lapack_int LAPACKE_cgtcon( char norm, lapack_int n, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zgtcon( char norm, lapack_int n, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_sgtrfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* dl, const float* d, + const float* du, const float* dlf, const float* df, + const float* duf, const float* du2, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dgtrfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* dl, const double* d, + const double* du, const double* dlf, + const double* df, const double* duf, + const double* du2, const lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr ); +lapack_int LAPACKE_cgtrfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* dlf, + const lapack_complex_float* df, + const lapack_complex_float* duf, + const lapack_complex_float* du2, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zgtrfs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* dlf, + const lapack_complex_double* df, + const lapack_complex_double* duf, + const lapack_complex_double* du2, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sgtsv( int matrix_order, lapack_int n, lapack_int nrhs, + float* dl, float* d, float* du, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgtsv( int matrix_order, lapack_int n, lapack_int nrhs, + double* dl, double* d, double* du, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cgtsv( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_float* dl, lapack_complex_float* d, + lapack_complex_float* du, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgtsv( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* dl, lapack_complex_double* d, + lapack_complex_double* du, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sgtsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, const float* dl, + const float* d, const float* du, float* dlf, + float* df, float* duf, float* du2, lapack_int* ipiv, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dgtsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, const double* dl, + const double* d, const double* du, double* dlf, + double* df, double* duf, double* du2, + lapack_int* ipiv, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* ferr, double* berr ); +lapack_int LAPACKE_cgtsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + lapack_complex_float* dlf, lapack_complex_float* df, + lapack_complex_float* duf, lapack_complex_float* du2, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_zgtsvx( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + lapack_complex_double* dlf, + lapack_complex_double* df, + lapack_complex_double* duf, + lapack_complex_double* du2, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_sgttrf( lapack_int n, float* dl, float* d, float* du, + float* du2, lapack_int* ipiv ); +lapack_int LAPACKE_dgttrf( lapack_int n, double* dl, double* d, double* du, + double* du2, lapack_int* ipiv ); +lapack_int LAPACKE_cgttrf( lapack_int n, lapack_complex_float* dl, + lapack_complex_float* d, lapack_complex_float* du, + lapack_complex_float* du2, lapack_int* ipiv ); +lapack_int LAPACKE_zgttrf( lapack_int n, lapack_complex_double* dl, + lapack_complex_double* d, lapack_complex_double* du, + lapack_complex_double* du2, lapack_int* ipiv ); + +lapack_int LAPACKE_sgttrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* dl, const float* d, + const float* du, const float* du2, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dgttrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* dl, const double* d, + const double* du, const double* du2, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_cgttrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgttrs( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_chbev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, lapack_complex_float* ab, + lapack_int ldab, float* w, lapack_complex_float* z, + lapack_int ldz ); +lapack_int LAPACKE_zhbev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, lapack_complex_double* ab, + lapack_int ldab, double* w, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_chbevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, lapack_complex_float* ab, + lapack_int ldab, float* w, lapack_complex_float* z, + lapack_int ldz ); +lapack_int LAPACKE_zhbevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, lapack_complex_double* ab, + lapack_int ldab, double* w, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_chbevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* q, lapack_int ldq, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* ifail ); +lapack_int LAPACKE_zhbevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* q, lapack_int ldq, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_chbgst( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* bb, lapack_int ldbb, + lapack_complex_float* x, lapack_int ldx ); +lapack_int LAPACKE_zhbgst( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + const lapack_complex_double* bb, lapack_int ldbb, + lapack_complex_double* x, lapack_int ldx ); + +lapack_int LAPACKE_chbgv( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* bb, lapack_int ldbb, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhbgv( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* bb, lapack_int ldbb, double* w, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chbgvd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* bb, lapack_int ldbb, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhbgvd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* bb, lapack_int ldbb, + double* w, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_chbgvx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* bb, lapack_int ldbb, + lapack_complex_float* q, lapack_int ldq, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* ifail ); +lapack_int LAPACKE_zhbgvx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* bb, lapack_int ldbb, + lapack_complex_double* q, lapack_int ldq, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_chbtrd( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int kd, lapack_complex_float* ab, + lapack_int ldab, float* d, float* e, + lapack_complex_float* q, lapack_int ldq ); +lapack_int LAPACKE_zhbtrd( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int kd, lapack_complex_double* ab, + lapack_int ldab, double* d, double* e, + lapack_complex_double* q, lapack_int ldq ); + +lapack_int LAPACKE_checon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zhecon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_cheequb( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zheequb( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_cheev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, float* w ); +lapack_int LAPACKE_zheev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, double* w ); + +lapack_int LAPACKE_cheevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, float* w ); +lapack_int LAPACKE_zheevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double* w ); + +lapack_int LAPACKE_cheevr( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_int* isuppz ); +lapack_int LAPACKE_zheevr( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, lapack_int ldz, + lapack_int* isuppz ); + +lapack_int LAPACKE_cheevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_zheevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_chegst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhegst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_chegv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* w ); +lapack_int LAPACKE_zhegv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* w ); + +lapack_int LAPACKE_chegvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* w ); +lapack_int LAPACKE_zhegvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* w ); + +lapack_int LAPACKE_chegvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* ifail ); +lapack_int LAPACKE_zhegvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_cherfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zherfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_cherfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* s, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_zherfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* s, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); + +lapack_int LAPACKE_chesv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zhesv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_chesvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* af, + lapack_int ldaf, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zhesvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* af, + lapack_int ldaf, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_chesvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_zhesvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); + +lapack_int LAPACKE_chetrd( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, float* d, + float* e, lapack_complex_float* tau ); +lapack_int LAPACKE_zhetrd( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, double* d, + double* e, lapack_complex_double* tau ); + +lapack_int LAPACKE_chetrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zhetrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_chetri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zhetri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); + +lapack_int LAPACKE_chetrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zhetrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_chfrk( int matrix_order, char transr, char uplo, char trans, + lapack_int n, lapack_int k, float alpha, + const lapack_complex_float* a, lapack_int lda, + float beta, lapack_complex_float* c ); +lapack_int LAPACKE_zhfrk( int matrix_order, char transr, char uplo, char trans, + lapack_int n, lapack_int k, double alpha, + const lapack_complex_double* a, lapack_int lda, + double beta, lapack_complex_double* c ); + +lapack_int LAPACKE_shgeqz( int matrix_order, char job, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + float* h, lapack_int ldh, float* t, lapack_int ldt, + float* alphar, float* alphai, float* beta, float* q, + lapack_int ldq, float* z, lapack_int ldz ); +lapack_int LAPACKE_dhgeqz( int matrix_order, char job, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + double* h, lapack_int ldh, double* t, lapack_int ldt, + double* alphar, double* alphai, double* beta, + double* q, lapack_int ldq, double* z, + lapack_int ldz ); +lapack_int LAPACKE_chgeqz( int matrix_order, char job, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_float* h, lapack_int ldh, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* alpha, + lapack_complex_float* beta, lapack_complex_float* q, + lapack_int ldq, lapack_complex_float* z, + lapack_int ldz ); +lapack_int LAPACKE_zhgeqz( int matrix_order, char job, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_double* h, lapack_int ldh, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chpcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zhpcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_chpev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_float* ap, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhpev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_double* ap, double* w, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chpevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_float* ap, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhpevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_complex_double* ap, double* w, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chpevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_float* ap, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* ifail ); +lapack_int LAPACKE_zhpevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_complex_double* ap, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_chpgst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_float* ap, + const lapack_complex_float* bp ); +lapack_int LAPACKE_zhpgst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_double* ap, + const lapack_complex_double* bp ); + +lapack_int LAPACKE_chpgv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_float* ap, + lapack_complex_float* bp, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhpgv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_double* ap, + lapack_complex_double* bp, double* w, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chpgvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_float* ap, + lapack_complex_float* bp, float* w, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zhpgvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_double* ap, + lapack_complex_double* bp, double* w, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_chpgvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_float* ap, lapack_complex_float* bp, + float vl, float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_zhpgvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_double* ap, lapack_complex_double* bp, + double vl, double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_chprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zhprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_chpsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhpsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_chpsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + lapack_complex_float* afp, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zhpsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + lapack_complex_double* afp, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_chptrd( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, float* d, float* e, + lapack_complex_float* tau ); +lapack_int LAPACKE_zhptrd( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, double* d, double* e, + lapack_complex_double* tau ); + +lapack_int LAPACKE_chptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, lapack_int* ipiv ); +lapack_int LAPACKE_zhptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, lapack_int* ipiv ); + +lapack_int LAPACKE_chptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, const lapack_int* ipiv ); +lapack_int LAPACKE_zhptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, const lapack_int* ipiv ); + +lapack_int LAPACKE_chptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_shsein( int matrix_order, char job, char eigsrc, char initv, + lapack_logical* select, lapack_int n, const float* h, + lapack_int ldh, float* wr, const float* wi, + float* vl, lapack_int ldvl, float* vr, + lapack_int ldvr, lapack_int mm, lapack_int* m, + lapack_int* ifaill, lapack_int* ifailr ); +lapack_int LAPACKE_dhsein( int matrix_order, char job, char eigsrc, char initv, + lapack_logical* select, lapack_int n, + const double* h, lapack_int ldh, double* wr, + const double* wi, double* vl, lapack_int ldvl, + double* vr, lapack_int ldvr, lapack_int mm, + lapack_int* m, lapack_int* ifaill, + lapack_int* ifailr ); +lapack_int LAPACKE_chsein( int matrix_order, char job, char eigsrc, char initv, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* h, lapack_int ldh, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int ldvl, lapack_complex_float* vr, + lapack_int ldvr, lapack_int mm, lapack_int* m, + lapack_int* ifaill, lapack_int* ifailr ); +lapack_int LAPACKE_zhsein( int matrix_order, char job, char eigsrc, char initv, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* h, lapack_int ldh, + lapack_complex_double* w, lapack_complex_double* vl, + lapack_int ldvl, lapack_complex_double* vr, + lapack_int ldvr, lapack_int mm, lapack_int* m, + lapack_int* ifaill, lapack_int* ifailr ); + +lapack_int LAPACKE_shseqr( int matrix_order, char job, char compz, lapack_int n, + lapack_int ilo, lapack_int ihi, float* h, + lapack_int ldh, float* wr, float* wi, float* z, + lapack_int ldz ); +lapack_int LAPACKE_dhseqr( int matrix_order, char job, char compz, lapack_int n, + lapack_int ilo, lapack_int ihi, double* h, + lapack_int ldh, double* wr, double* wi, double* z, + lapack_int ldz ); +lapack_int LAPACKE_chseqr( int matrix_order, char job, char compz, lapack_int n, + lapack_int ilo, lapack_int ihi, + lapack_complex_float* h, lapack_int ldh, + lapack_complex_float* w, lapack_complex_float* z, + lapack_int ldz ); +lapack_int LAPACKE_zhseqr( int matrix_order, char job, char compz, lapack_int n, + lapack_int ilo, lapack_int ihi, + lapack_complex_double* h, lapack_int ldh, + lapack_complex_double* w, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_clacgv( lapack_int n, lapack_complex_float* x, + lapack_int incx ); +lapack_int LAPACKE_zlacgv( lapack_int n, lapack_complex_double* x, + lapack_int incx ); + +lapack_int LAPACKE_slacpy( int matrix_order, char uplo, lapack_int m, + lapack_int n, const float* a, lapack_int lda, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dlacpy( int matrix_order, char uplo, lapack_int m, + lapack_int n, const double* a, lapack_int lda, double* b, + lapack_int ldb ); +lapack_int LAPACKE_clacpy( int matrix_order, char uplo, lapack_int m, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zlacpy( int matrix_order, char uplo, lapack_int m, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_zlag2c( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_float* sa, lapack_int ldsa ); + +lapack_int LAPACKE_slag2d( int matrix_order, lapack_int m, lapack_int n, + const float* sa, lapack_int ldsa, double* a, + lapack_int lda ); + +lapack_int LAPACKE_dlag2s( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, float* sa, + lapack_int ldsa ); + +lapack_int LAPACKE_clag2z( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* sa, lapack_int ldsa, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_slagge( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* d, + float* a, lapack_int lda, lapack_int* iseed ); +lapack_int LAPACKE_dlagge( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* d, + double* a, lapack_int lda, lapack_int* iseed ); +lapack_int LAPACKE_clagge( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* d, + lapack_complex_float* a, lapack_int lda, + lapack_int* iseed ); +lapack_int LAPACKE_zlagge( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* d, + lapack_complex_double* a, lapack_int lda, + lapack_int* iseed ); + +float LAPACKE_slamch( char cmach ); +double LAPACKE_dlamch( char cmach ); + +float LAPACKE_slange( int matrix_order, char norm, lapack_int m, + lapack_int n, const float* a, lapack_int lda ); +double LAPACKE_dlange( int matrix_order, char norm, lapack_int m, + lapack_int n, const double* a, lapack_int lda ); +float LAPACKE_clange( int matrix_order, char norm, lapack_int m, + lapack_int n, const lapack_complex_float* a, + lapack_int lda ); +double LAPACKE_zlange( int matrix_order, char norm, lapack_int m, + lapack_int n, const lapack_complex_double* a, + lapack_int lda ); + +float LAPACKE_clanhe( int matrix_order, char norm, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda ); +double LAPACKE_zlanhe( int matrix_order, char norm, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda ); + +float LAPACKE_slansy( int matrix_order, char norm, char uplo, lapack_int n, + const float* a, lapack_int lda ); +double LAPACKE_dlansy( int matrix_order, char norm, char uplo, lapack_int n, + const double* a, lapack_int lda ); +float LAPACKE_clansy( int matrix_order, char norm, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda ); +double LAPACKE_zlansy( int matrix_order, char norm, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda ); + +float LAPACKE_slantr( int matrix_order, char norm, char uplo, char diag, + lapack_int m, lapack_int n, const float* a, + lapack_int lda ); +double LAPACKE_dlantr( int matrix_order, char norm, char uplo, char diag, + lapack_int m, lapack_int n, const double* a, + lapack_int lda ); +float LAPACKE_clantr( int matrix_order, char norm, char uplo, char diag, + lapack_int m, lapack_int n, const lapack_complex_float* a, + lapack_int lda ); +double LAPACKE_zlantr( int matrix_order, char norm, char uplo, char diag, + lapack_int m, lapack_int n, const lapack_complex_double* a, + lapack_int lda ); + + +lapack_int LAPACKE_slarfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, const float* v, lapack_int ldv, + const float* t, lapack_int ldt, float* c, + lapack_int ldc ); +lapack_int LAPACKE_dlarfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, const double* v, lapack_int ldv, + const double* t, lapack_int ldt, double* c, + lapack_int ldc ); +lapack_int LAPACKE_clarfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, const lapack_complex_float* v, + lapack_int ldv, const lapack_complex_float* t, + lapack_int ldt, lapack_complex_float* c, + lapack_int ldc ); +lapack_int LAPACKE_zlarfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, const lapack_complex_double* v, + lapack_int ldv, const lapack_complex_double* t, + lapack_int ldt, lapack_complex_double* c, + lapack_int ldc ); + +lapack_int LAPACKE_slarfg( lapack_int n, float* alpha, float* x, + lapack_int incx, float* tau ); +lapack_int LAPACKE_dlarfg( lapack_int n, double* alpha, double* x, + lapack_int incx, double* tau ); +lapack_int LAPACKE_clarfg( lapack_int n, lapack_complex_float* alpha, + lapack_complex_float* x, lapack_int incx, + lapack_complex_float* tau ); +lapack_int LAPACKE_zlarfg( lapack_int n, lapack_complex_double* alpha, + lapack_complex_double* x, lapack_int incx, + lapack_complex_double* tau ); + +lapack_int LAPACKE_slarft( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, const float* v, + lapack_int ldv, const float* tau, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dlarft( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, const double* v, + lapack_int ldv, const double* tau, double* t, + lapack_int ldt ); +lapack_int LAPACKE_clarft( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* tau, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zlarft( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* tau, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_slarfx( int matrix_order, char side, lapack_int m, + lapack_int n, const float* v, float tau, float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_dlarfx( int matrix_order, char side, lapack_int m, + lapack_int n, const double* v, double tau, double* c, + lapack_int ldc, double* work ); +lapack_int LAPACKE_clarfx( int matrix_order, char side, lapack_int m, + lapack_int n, const lapack_complex_float* v, + lapack_complex_float tau, lapack_complex_float* c, + lapack_int ldc, lapack_complex_float* work ); +lapack_int LAPACKE_zlarfx( int matrix_order, char side, lapack_int m, + lapack_int n, const lapack_complex_double* v, + lapack_complex_double tau, lapack_complex_double* c, + lapack_int ldc, lapack_complex_double* work ); + +lapack_int LAPACKE_slarnv( lapack_int idist, lapack_int* iseed, lapack_int n, + float* x ); +lapack_int LAPACKE_dlarnv( lapack_int idist, lapack_int* iseed, lapack_int n, + double* x ); +lapack_int LAPACKE_clarnv( lapack_int idist, lapack_int* iseed, lapack_int n, + lapack_complex_float* x ); +lapack_int LAPACKE_zlarnv( lapack_int idist, lapack_int* iseed, lapack_int n, + lapack_complex_double* x ); + +lapack_int LAPACKE_slaset( int matrix_order, char uplo, lapack_int m, + lapack_int n, float alpha, float beta, float* a, + lapack_int lda ); +lapack_int LAPACKE_dlaset( int matrix_order, char uplo, lapack_int m, + lapack_int n, double alpha, double beta, double* a, + lapack_int lda ); +lapack_int LAPACKE_claset( int matrix_order, char uplo, lapack_int m, + lapack_int n, lapack_complex_float alpha, + lapack_complex_float beta, lapack_complex_float* a, + lapack_int lda ); +lapack_int LAPACKE_zlaset( int matrix_order, char uplo, lapack_int m, + lapack_int n, lapack_complex_double alpha, + lapack_complex_double beta, lapack_complex_double* a, + lapack_int lda ); + +lapack_int LAPACKE_slasrt( char id, lapack_int n, float* d ); +lapack_int LAPACKE_dlasrt( char id, lapack_int n, double* d ); + +lapack_int LAPACKE_slaswp( int matrix_order, lapack_int n, float* a, + lapack_int lda, lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); +lapack_int LAPACKE_dlaswp( int matrix_order, lapack_int n, double* a, + lapack_int lda, lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); +lapack_int LAPACKE_claswp( int matrix_order, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int k1, lapack_int k2, const lapack_int* ipiv, + lapack_int incx ); +lapack_int LAPACKE_zlaswp( int matrix_order, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int k1, lapack_int k2, const lapack_int* ipiv, + lapack_int incx ); + +lapack_int LAPACKE_slatms( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, float* d, + lapack_int mode, float cond, float dmax, + lapack_int kl, lapack_int ku, char pack, float* a, + lapack_int lda ); +lapack_int LAPACKE_dlatms( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, double* d, + lapack_int mode, double cond, double dmax, + lapack_int kl, lapack_int ku, char pack, double* a, + lapack_int lda ); +lapack_int LAPACKE_clatms( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, float* d, + lapack_int mode, float cond, float dmax, + lapack_int kl, lapack_int ku, char pack, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zlatms( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, double* d, + lapack_int mode, double cond, double dmax, + lapack_int kl, lapack_int ku, char pack, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_slauum( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda ); +lapack_int LAPACKE_dlauum( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda ); +lapack_int LAPACKE_clauum( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zlauum( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_sopgtr( int matrix_order, char uplo, lapack_int n, + const float* ap, const float* tau, float* q, + lapack_int ldq ); +lapack_int LAPACKE_dopgtr( int matrix_order, char uplo, lapack_int n, + const double* ap, const double* tau, double* q, + lapack_int ldq ); + +lapack_int LAPACKE_sopmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, const float* ap, + const float* tau, float* c, lapack_int ldc ); +lapack_int LAPACKE_dopmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, const double* ap, + const double* tau, double* c, lapack_int ldc ); + +lapack_int LAPACKE_sorgbr( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorgbr( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, double* a, + lapack_int lda, const double* tau ); + +lapack_int LAPACKE_sorghr( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorghr( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, double* a, lapack_int lda, + const double* tau ); + +lapack_int LAPACKE_sorglq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorglq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau ); + +lapack_int LAPACKE_sorgql( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorgql( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau ); + +lapack_int LAPACKE_sorgqr( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorgqr( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau ); + +lapack_int LAPACKE_sorgrq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau ); +lapack_int LAPACKE_dorgrq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau ); + +lapack_int LAPACKE_sorgtr( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, const float* tau ); +lapack_int LAPACKE_dorgtr( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda, const double* tau ); + +lapack_int LAPACKE_sormbr( int matrix_order, char vect, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, const float* tau, + float* c, lapack_int ldc ); +lapack_int LAPACKE_dormbr( int matrix_order, char vect, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, const double* tau, + double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormhr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc ); +lapack_int LAPACKE_dormhr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormlq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, const float* tau, + float* c, lapack_int ldc ); +lapack_int LAPACKE_dormlq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, const double* tau, + double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormql( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, const float* tau, + float* c, lapack_int ldc ); +lapack_int LAPACKE_dormql( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, const double* tau, + double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormqr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, const float* tau, + float* c, lapack_int ldc ); +lapack_int LAPACKE_dormqr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, const double* tau, + double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormrq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, const float* tau, + float* c, lapack_int ldc ); +lapack_int LAPACKE_dormrq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, const double* tau, + double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormrz( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc ); +lapack_int LAPACKE_dormrz( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc ); + +lapack_int LAPACKE_sormtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, const float* a, + lapack_int lda, const float* tau, float* c, + lapack_int ldc ); +lapack_int LAPACKE_dormtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, const double* a, + lapack_int lda, const double* tau, double* c, + lapack_int ldc ); + +lapack_int LAPACKE_spbcon( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const float* ab, lapack_int ldab, + float anorm, float* rcond ); +lapack_int LAPACKE_dpbcon( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const double* ab, lapack_int ldab, + double anorm, double* rcond ); +lapack_int LAPACKE_cpbcon( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_float* ab, + lapack_int ldab, float anorm, float* rcond ); +lapack_int LAPACKE_zpbcon( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_double* ab, + lapack_int ldab, double anorm, double* rcond ); + +lapack_int LAPACKE_spbequ( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const float* ab, lapack_int ldab, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_dpbequ( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const double* ab, lapack_int ldab, + double* s, double* scond, double* amax ); +lapack_int LAPACKE_cpbequ( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_float* ab, + lapack_int ldab, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_zpbequ( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_double* ab, + lapack_int ldab, double* s, double* scond, + double* amax ); + +lapack_int LAPACKE_spbrfs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const float* ab, + lapack_int ldab, const float* afb, lapack_int ldafb, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_dpbrfs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const double* ab, + lapack_int ldab, const double* afb, lapack_int ldafb, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr ); +lapack_int LAPACKE_cpbrfs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* afb, lapack_int ldafb, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zpbrfs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + const lapack_complex_double* afb, lapack_int ldafb, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_spbstf( int matrix_order, char uplo, lapack_int n, + lapack_int kb, float* bb, lapack_int ldbb ); +lapack_int LAPACKE_dpbstf( int matrix_order, char uplo, lapack_int n, + lapack_int kb, double* bb, lapack_int ldbb ); +lapack_int LAPACKE_cpbstf( int matrix_order, char uplo, lapack_int n, + lapack_int kb, lapack_complex_float* bb, + lapack_int ldbb ); +lapack_int LAPACKE_zpbstf( int matrix_order, char uplo, lapack_int n, + lapack_int kb, lapack_complex_double* bb, + lapack_int ldbb ); + +lapack_int LAPACKE_spbsv( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, float* ab, + lapack_int ldab, float* b, lapack_int ldb ); +lapack_int LAPACKE_dpbsv( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, double* ab, + lapack_int ldab, double* b, lapack_int ldb ); +lapack_int LAPACKE_cpbsv( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpbsv( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spbsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, float* ab, + lapack_int ldab, float* afb, lapack_int ldafb, + char* equed, float* s, float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dpbsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, double* ab, + lapack_int ldab, double* afb, lapack_int ldafb, + char* equed, double* s, double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* ferr, double* berr ); +lapack_int LAPACKE_cpbsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* afb, lapack_int ldafb, + char* equed, float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_zpbsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* afb, lapack_int ldafb, + char* equed, double* s, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr ); + +lapack_int LAPACKE_spbtrf( int matrix_order, char uplo, lapack_int n, + lapack_int kd, float* ab, lapack_int ldab ); +lapack_int LAPACKE_dpbtrf( int matrix_order, char uplo, lapack_int n, + lapack_int kd, double* ab, lapack_int ldab ); +lapack_int LAPACKE_cpbtrf( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_complex_float* ab, + lapack_int ldab ); +lapack_int LAPACKE_zpbtrf( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_complex_double* ab, + lapack_int ldab ); + +lapack_int LAPACKE_spbtrs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const float* ab, + lapack_int ldab, float* b, lapack_int ldb ); +lapack_int LAPACKE_dpbtrs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const double* ab, + lapack_int ldab, double* b, lapack_int ldb ); +lapack_int LAPACKE_cpbtrs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpbtrs( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spftrf( int matrix_order, char transr, char uplo, + lapack_int n, float* a ); +lapack_int LAPACKE_dpftrf( int matrix_order, char transr, char uplo, + lapack_int n, double* a ); +lapack_int LAPACKE_cpftrf( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_float* a ); +lapack_int LAPACKE_zpftrf( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_double* a ); + +lapack_int LAPACKE_spftri( int matrix_order, char transr, char uplo, + lapack_int n, float* a ); +lapack_int LAPACKE_dpftri( int matrix_order, char transr, char uplo, + lapack_int n, double* a ); +lapack_int LAPACKE_cpftri( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_float* a ); +lapack_int LAPACKE_zpftri( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_double* a ); + +lapack_int LAPACKE_spftrs( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, const float* a, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dpftrs( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, const double* a, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cpftrs( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpftrs( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spocon( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float anorm, + float* rcond ); +lapack_int LAPACKE_dpocon( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double anorm, + double* rcond ); +lapack_int LAPACKE_cpocon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float anorm, float* rcond ); +lapack_int LAPACKE_zpocon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double anorm, double* rcond ); + +lapack_int LAPACKE_spoequ( int matrix_order, lapack_int n, const float* a, + lapack_int lda, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dpoequ( int matrix_order, lapack_int n, const double* a, + lapack_int lda, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cpoequ( int matrix_order, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zpoequ( int matrix_order, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_spoequb( int matrix_order, lapack_int n, const float* a, + lapack_int lda, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dpoequb( int matrix_order, lapack_int n, const double* a, + lapack_int lda, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cpoequb( int matrix_order, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zpoequb( int matrix_order, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_sporfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dporfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const double* af, lapack_int ldaf, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr ); +lapack_int LAPACKE_cporfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_zporfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* ferr, double* berr ); + +lapack_int LAPACKE_sporfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, lapack_int ldaf, + const float* s, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dporfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, lapack_int ldaf, + const double* s, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); +lapack_int LAPACKE_cporfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, lapack_int ldaf, + const float* s, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_zporfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, lapack_int ldaf, + const double* s, const lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); + +lapack_int LAPACKE_sposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); +lapack_int LAPACKE_dsposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* x, lapack_int ldx, + lapack_int* iter ); +lapack_int LAPACKE_zcposv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, lapack_int* iter ); + +lapack_int LAPACKE_sposvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, float* af, + lapack_int ldaf, char* equed, float* s, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_dposvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* af, lapack_int ldaf, char* equed, double* s, + double* b, lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); +lapack_int LAPACKE_cposvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* af, + lapack_int ldaf, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zposvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* af, + lapack_int ldaf, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_sposvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + char* equed, float* s, float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_dposvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + char* equed, double* s, double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); +lapack_int LAPACKE_cposvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + char* equed, float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_zposvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + char* equed, double* s, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); + +lapack_int LAPACKE_spotrf( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda ); +lapack_int LAPACKE_dpotrf( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda ); +lapack_int LAPACKE_cpotrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zpotrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_spotri( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda ); +lapack_int LAPACKE_dpotri( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda ); +lapack_int LAPACKE_cpotri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zpotri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_spotrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dpotrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cpotrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zpotrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sppcon( int matrix_order, char uplo, lapack_int n, + const float* ap, float anorm, float* rcond ); +lapack_int LAPACKE_dppcon( int matrix_order, char uplo, lapack_int n, + const double* ap, double anorm, double* rcond ); +lapack_int LAPACKE_cppcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, float anorm, + float* rcond ); +lapack_int LAPACKE_zppcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, double anorm, + double* rcond ); + +lapack_int LAPACKE_sppequ( int matrix_order, char uplo, lapack_int n, + const float* ap, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dppequ( int matrix_order, char uplo, lapack_int n, + const double* ap, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cppequ( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, float* s, + float* scond, float* amax ); +lapack_int LAPACKE_zppequ( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, double* s, + double* scond, double* amax ); + +lapack_int LAPACKE_spprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, const float* afp, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_dpprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, const double* afp, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr ); +lapack_int LAPACKE_cpprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zpprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sppsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* ap, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dppsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* ap, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cppsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zppsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sppsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, float* ap, float* afp, char* equed, + float* s, float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dppsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, double* ap, double* afp, + char* equed, double* s, double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* ferr, double* berr ); +lapack_int LAPACKE_cppsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_complex_float* afp, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zppsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_complex_double* afp, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_spptrf( int matrix_order, char uplo, lapack_int n, + float* ap ); +lapack_int LAPACKE_dpptrf( int matrix_order, char uplo, lapack_int n, + double* ap ); +lapack_int LAPACKE_cpptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap ); +lapack_int LAPACKE_zpptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap ); + +lapack_int LAPACKE_spptri( int matrix_order, char uplo, lapack_int n, + float* ap ); +lapack_int LAPACKE_dpptri( int matrix_order, char uplo, lapack_int n, + double* ap ); +lapack_int LAPACKE_cpptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap ); +lapack_int LAPACKE_zpptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap ); + +lapack_int LAPACKE_spptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dpptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cpptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spstrf( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, lapack_int* piv, lapack_int* rank, + float tol ); +lapack_int LAPACKE_dpstrf( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda, lapack_int* piv, lapack_int* rank, + double tol ); +lapack_int LAPACKE_cpstrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* piv, lapack_int* rank, float tol ); +lapack_int LAPACKE_zpstrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* piv, lapack_int* rank, double tol ); + +lapack_int LAPACKE_sptcon( lapack_int n, const float* d, const float* e, + float anorm, float* rcond ); +lapack_int LAPACKE_dptcon( lapack_int n, const double* d, const double* e, + double anorm, double* rcond ); +lapack_int LAPACKE_cptcon( lapack_int n, const float* d, + const lapack_complex_float* e, float anorm, + float* rcond ); +lapack_int LAPACKE_zptcon( lapack_int n, const double* d, + const lapack_complex_double* e, double anorm, + double* rcond ); + +lapack_int LAPACKE_spteqr( int matrix_order, char compz, lapack_int n, float* d, + float* e, float* z, lapack_int ldz ); +lapack_int LAPACKE_dpteqr( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz ); +lapack_int LAPACKE_cpteqr( int matrix_order, char compz, lapack_int n, float* d, + float* e, lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zpteqr( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_sptrfs( int matrix_order, lapack_int n, lapack_int nrhs, + const float* d, const float* e, const float* df, + const float* ef, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_dptrfs( int matrix_order, lapack_int n, lapack_int nrhs, + const double* d, const double* e, const double* df, + const double* ef, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* ferr, + double* berr ); +lapack_int LAPACKE_cptrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, const float* df, + const lapack_complex_float* ef, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zptrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, const double* df, + const lapack_complex_double* ef, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sptsv( int matrix_order, lapack_int n, lapack_int nrhs, + float* d, float* e, float* b, lapack_int ldb ); +lapack_int LAPACKE_dptsv( int matrix_order, lapack_int n, lapack_int nrhs, + double* d, double* e, double* b, lapack_int ldb ); +lapack_int LAPACKE_cptsv( int matrix_order, lapack_int n, lapack_int nrhs, + float* d, lapack_complex_float* e, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zptsv( int matrix_order, lapack_int n, lapack_int nrhs, + double* d, lapack_complex_double* e, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sptsvx( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const float* d, const float* e, + float* df, float* ef, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dptsvx( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const double* d, const double* e, + double* df, double* ef, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); +lapack_int LAPACKE_cptsvx( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, float* df, + lapack_complex_float* ef, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zptsvx( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, double* df, + lapack_complex_double* ef, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_spttrf( lapack_int n, float* d, float* e ); +lapack_int LAPACKE_dpttrf( lapack_int n, double* d, double* e ); +lapack_int LAPACKE_cpttrf( lapack_int n, float* d, lapack_complex_float* e ); +lapack_int LAPACKE_zpttrf( lapack_int n, double* d, lapack_complex_double* e ); + +lapack_int LAPACKE_spttrs( int matrix_order, lapack_int n, lapack_int nrhs, + const float* d, const float* e, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dpttrs( int matrix_order, lapack_int n, lapack_int nrhs, + const double* d, const double* e, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cpttrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpttrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_ssbev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, float* ab, lapack_int ldab, float* w, + float* z, lapack_int ldz ); +lapack_int LAPACKE_dsbev( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, double* ab, lapack_int ldab, double* w, + double* z, lapack_int ldz ); + +lapack_int LAPACKE_ssbevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, float* ab, lapack_int ldab, float* w, + float* z, lapack_int ldz ); +lapack_int LAPACKE_dsbevd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int kd, double* ab, lapack_int ldab, + double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_ssbevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int kd, float* ab, + lapack_int ldab, float* q, lapack_int ldq, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dsbevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int kd, double* ab, + lapack_int ldab, double* q, lapack_int ldq, + double vl, double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* ifail ); + +lapack_int LAPACKE_ssbgst( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, float* ab, + lapack_int ldab, const float* bb, lapack_int ldbb, + float* x, lapack_int ldx ); +lapack_int LAPACKE_dsbgst( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, double* ab, + lapack_int ldab, const double* bb, lapack_int ldbb, + double* x, lapack_int ldx ); + +lapack_int LAPACKE_ssbgv( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, float* ab, + lapack_int ldab, float* bb, lapack_int ldbb, float* w, + float* z, lapack_int ldz ); +lapack_int LAPACKE_dsbgv( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, double* ab, + lapack_int ldab, double* bb, lapack_int ldbb, + double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_ssbgvd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, float* ab, + lapack_int ldab, float* bb, lapack_int ldbb, + float* w, float* z, lapack_int ldz ); +lapack_int LAPACKE_dsbgvd( int matrix_order, char jobz, char uplo, lapack_int n, + lapack_int ka, lapack_int kb, double* ab, + lapack_int ldab, double* bb, lapack_int ldbb, + double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_ssbgvx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + float* ab, lapack_int ldab, float* bb, + lapack_int ldbb, float* q, lapack_int ldq, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dsbgvx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + double* ab, lapack_int ldab, double* bb, + lapack_int ldbb, double* q, lapack_int ldq, + double vl, double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* ifail ); + +lapack_int LAPACKE_ssbtrd( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int kd, float* ab, lapack_int ldab, float* d, + float* e, float* q, lapack_int ldq ); +lapack_int LAPACKE_dsbtrd( int matrix_order, char vect, char uplo, lapack_int n, + lapack_int kd, double* ab, lapack_int ldab, + double* d, double* e, double* q, lapack_int ldq ); + +lapack_int LAPACKE_ssfrk( int matrix_order, char transr, char uplo, char trans, + lapack_int n, lapack_int k, float alpha, + const float* a, lapack_int lda, float beta, + float* c ); +lapack_int LAPACKE_dsfrk( int matrix_order, char transr, char uplo, char trans, + lapack_int n, lapack_int k, double alpha, + const double* a, lapack_int lda, double beta, + double* c ); + +lapack_int LAPACKE_sspcon( int matrix_order, char uplo, lapack_int n, + const float* ap, const lapack_int* ipiv, float anorm, + float* rcond ); +lapack_int LAPACKE_dspcon( int matrix_order, char uplo, lapack_int n, + const double* ap, const lapack_int* ipiv, + double anorm, double* rcond ); +lapack_int LAPACKE_cspcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zspcon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_sspev( int matrix_order, char jobz, char uplo, lapack_int n, + float* ap, float* w, float* z, lapack_int ldz ); +lapack_int LAPACKE_dspev( int matrix_order, char jobz, char uplo, lapack_int n, + double* ap, double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sspevd( int matrix_order, char jobz, char uplo, lapack_int n, + float* ap, float* w, float* z, lapack_int ldz ); +lapack_int LAPACKE_dspevd( int matrix_order, char jobz, char uplo, lapack_int n, + double* ap, double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sspevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, float* ap, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dspevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, double* ap, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_sspgst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, float* ap, const float* bp ); +lapack_int LAPACKE_dspgst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, double* ap, const double* bp ); + +lapack_int LAPACKE_sspgv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* ap, float* bp, + float* w, float* z, lapack_int ldz ); +lapack_int LAPACKE_dspgv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* ap, double* bp, + double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sspgvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* ap, float* bp, + float* w, float* z, lapack_int ldz ); +lapack_int LAPACKE_dspgvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* ap, double* bp, + double* w, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sspgvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, float* ap, + float* bp, float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, float* w, + float* z, lapack_int ldz, lapack_int* ifail ); +lapack_int LAPACKE_dspgvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, double* ap, + double* bp, double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, double* z, lapack_int ldz, + lapack_int* ifail ); + +lapack_int LAPACKE_ssprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, const float* afp, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dsprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, const double* afp, + const lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr ); +lapack_int LAPACKE_csprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zsprfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_sspsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* ap, lapack_int* ipiv, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dspsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* ap, lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cspsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zspsv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sspsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, float* afp, + lapack_int* ipiv, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dspsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, double* afp, + lapack_int* ipiv, const double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* ferr, double* berr ); +lapack_int LAPACKE_cspsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + lapack_complex_float* afp, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zspsvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + lapack_complex_double* afp, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_ssptrd( int matrix_order, char uplo, lapack_int n, float* ap, + float* d, float* e, float* tau ); +lapack_int LAPACKE_dsptrd( int matrix_order, char uplo, lapack_int n, + double* ap, double* d, double* e, double* tau ); + +lapack_int LAPACKE_ssptrf( int matrix_order, char uplo, lapack_int n, float* ap, + lapack_int* ipiv ); +lapack_int LAPACKE_dsptrf( int matrix_order, char uplo, lapack_int n, + double* ap, lapack_int* ipiv ); +lapack_int LAPACKE_csptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, lapack_int* ipiv ); +lapack_int LAPACKE_zsptrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, lapack_int* ipiv ); + +lapack_int LAPACKE_ssptri( int matrix_order, char uplo, lapack_int n, float* ap, + const lapack_int* ipiv ); +lapack_int LAPACKE_dsptri( int matrix_order, char uplo, lapack_int n, + double* ap, const lapack_int* ipiv ); +lapack_int LAPACKE_csptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, const lapack_int* ipiv ); +lapack_int LAPACKE_zsptri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, const lapack_int* ipiv ); + +lapack_int LAPACKE_ssptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dsptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_csptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zsptrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* ap, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sstebz( char range, char order, lapack_int n, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + const float* d, const float* e, lapack_int* m, + lapack_int* nsplit, float* w, lapack_int* iblock, + lapack_int* isplit ); +lapack_int LAPACKE_dstebz( char range, char order, lapack_int n, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, const double* d, const double* e, + lapack_int* m, lapack_int* nsplit, double* w, + lapack_int* iblock, lapack_int* isplit ); + +lapack_int LAPACKE_sstedc( int matrix_order, char compz, lapack_int n, float* d, + float* e, float* z, lapack_int ldz ); +lapack_int LAPACKE_dstedc( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz ); +lapack_int LAPACKE_cstedc( int matrix_order, char compz, lapack_int n, float* d, + float* e, lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zstedc( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_sstegr( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* isuppz ); +lapack_int LAPACKE_dstegr( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* isuppz ); +lapack_int LAPACKE_cstegr( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* isuppz ); +lapack_int LAPACKE_zstegr( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* isuppz ); + +lapack_int LAPACKE_sstein( int matrix_order, lapack_int n, const float* d, + const float* e, lapack_int m, const float* w, + const lapack_int* iblock, const lapack_int* isplit, + float* z, lapack_int ldz, lapack_int* ifailv ); +lapack_int LAPACKE_dstein( int matrix_order, lapack_int n, const double* d, + const double* e, lapack_int m, const double* w, + const lapack_int* iblock, const lapack_int* isplit, + double* z, lapack_int ldz, lapack_int* ifailv ); +lapack_int LAPACKE_cstein( int matrix_order, lapack_int n, const float* d, + const float* e, lapack_int m, const float* w, + const lapack_int* iblock, const lapack_int* isplit, + lapack_complex_float* z, lapack_int ldz, + lapack_int* ifailv ); +lapack_int LAPACKE_zstein( int matrix_order, lapack_int n, const double* d, + const double* e, lapack_int m, const double* w, + const lapack_int* iblock, const lapack_int* isplit, + lapack_complex_double* z, lapack_int ldz, + lapack_int* ifailv ); + +lapack_int LAPACKE_sstemr( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, lapack_int* m, + float* w, float* z, lapack_int ldz, lapack_int nzc, + lapack_int* isuppz, lapack_logical* tryrac ); +lapack_int LAPACKE_dstemr( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + lapack_int* m, double* w, double* z, lapack_int ldz, + lapack_int nzc, lapack_int* isuppz, + lapack_logical* tryrac ); +lapack_int LAPACKE_cstemr( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, lapack_int* m, + float* w, lapack_complex_float* z, lapack_int ldz, + lapack_int nzc, lapack_int* isuppz, + lapack_logical* tryrac ); +lapack_int LAPACKE_zstemr( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int ldz, lapack_int nzc, lapack_int* isuppz, + lapack_logical* tryrac ); + +lapack_int LAPACKE_ssteqr( int matrix_order, char compz, lapack_int n, float* d, + float* e, float* z, lapack_int ldz ); +lapack_int LAPACKE_dsteqr( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz ); +lapack_int LAPACKE_csteqr( int matrix_order, char compz, lapack_int n, float* d, + float* e, lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zsteqr( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz ); + +lapack_int LAPACKE_ssterf( lapack_int n, float* d, float* e ); +lapack_int LAPACKE_dsterf( lapack_int n, double* d, double* e ); + +lapack_int LAPACKE_sstev( int matrix_order, char jobz, lapack_int n, float* d, + float* e, float* z, lapack_int ldz ); +lapack_int LAPACKE_dstev( int matrix_order, char jobz, lapack_int n, double* d, + double* e, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sstevd( int matrix_order, char jobz, lapack_int n, float* d, + float* e, float* z, lapack_int ldz ); +lapack_int LAPACKE_dstevd( int matrix_order, char jobz, lapack_int n, double* d, + double* e, double* z, lapack_int ldz ); + +lapack_int LAPACKE_sstevr( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* isuppz ); +lapack_int LAPACKE_dstevr( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* isuppz ); + +lapack_int LAPACKE_sstevx( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dstevx( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* ifail ); + +lapack_int LAPACKE_ssycon( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_dsycon( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond ); +lapack_int LAPACKE_csycon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, float* rcond ); +lapack_int LAPACKE_zsycon( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond ); + +lapack_int LAPACKE_ssyequb( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float* s, + float* scond, float* amax ); +lapack_int LAPACKE_dsyequb( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double* s, + double* scond, double* amax ); +lapack_int LAPACKE_csyequb( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zsyequb( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_ssyev( int matrix_order, char jobz, char uplo, lapack_int n, + float* a, lapack_int lda, float* w ); +lapack_int LAPACKE_dsyev( int matrix_order, char jobz, char uplo, lapack_int n, + double* a, lapack_int lda, double* w ); + +lapack_int LAPACKE_ssyevd( int matrix_order, char jobz, char uplo, lapack_int n, + float* a, lapack_int lda, float* w ); +lapack_int LAPACKE_dsyevd( int matrix_order, char jobz, char uplo, lapack_int n, + double* a, lapack_int lda, double* w ); + +lapack_int LAPACKE_ssyevr( int matrix_order, char jobz, char range, char uplo, + lapack_int n, float* a, lapack_int lda, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* isuppz ); +lapack_int LAPACKE_dsyevr( int matrix_order, char jobz, char range, char uplo, + lapack_int n, double* a, lapack_int lda, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* isuppz ); + +lapack_int LAPACKE_ssyevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, float* a, lapack_int lda, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dsyevx( int matrix_order, char jobz, char range, char uplo, + lapack_int n, double* a, lapack_int lda, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* ifail ); + +lapack_int LAPACKE_ssygst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, float* a, lapack_int lda, + const float* b, lapack_int ldb ); +lapack_int LAPACKE_dsygst( int matrix_order, lapack_int itype, char uplo, + lapack_int n, double* a, lapack_int lda, + const double* b, lapack_int ldb ); + +lapack_int LAPACKE_ssygv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* a, lapack_int lda, + float* b, lapack_int ldb, float* w ); +lapack_int LAPACKE_dsygv( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* a, lapack_int lda, + double* b, lapack_int ldb, double* w ); + +lapack_int LAPACKE_ssygvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* a, lapack_int lda, + float* b, lapack_int ldb, float* w ); +lapack_int LAPACKE_dsygvd( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* a, lapack_int lda, + double* b, lapack_int ldb, double* w ); + +lapack_int LAPACKE_ssygvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, float vl, + float vu, lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, lapack_int ldz, + lapack_int* ifail ); +lapack_int LAPACKE_dsygvx( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* ifail ); + +lapack_int LAPACKE_ssyrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dsyrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr ); +lapack_int LAPACKE_csyrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_zsyrfs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_ssyrfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* s, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dsyrfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* s, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); +lapack_int LAPACKE_csyrfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* s, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params ); +lapack_int LAPACKE_zsyrfsx( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, lapack_int ldaf, + const lapack_int* ipiv, const double* s, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params ); + +lapack_int LAPACKE_ssysv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dsysv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_csysv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zsysv( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_ssysvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + float* af, lapack_int ldaf, lapack_int* ipiv, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr ); +lapack_int LAPACKE_dsysvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + double* af, lapack_int ldaf, lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr ); +lapack_int LAPACKE_csysvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* af, + lapack_int ldaf, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr ); +lapack_int LAPACKE_zsysvx( int matrix_order, char fact, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* af, + lapack_int ldaf, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr ); + +lapack_int LAPACKE_ssysvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_dsysvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); +lapack_int LAPACKE_csysvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params ); +lapack_int LAPACKE_zsysvxx( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params ); + +lapack_int LAPACKE_ssytrd( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, float* d, float* e, float* tau ); +lapack_int LAPACKE_dsytrd( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda, double* d, double* e, double* tau ); + +lapack_int LAPACKE_ssytrf( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_dsytrf( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_csytrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zsytrf( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_ssytri( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_dsytri( int matrix_order, char uplo, lapack_int n, double* a, + lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_csytri( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zsytri( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); + +lapack_int LAPACKE_ssytrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_dsytrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_csytrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zsytrs( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stbcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, lapack_int kd, const float* ab, + lapack_int ldab, float* rcond ); +lapack_int LAPACKE_dtbcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, lapack_int kd, const double* ab, + lapack_int ldab, double* rcond ); +lapack_int LAPACKE_ctbcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, lapack_int kd, + const lapack_complex_float* ab, lapack_int ldab, + float* rcond ); +lapack_int LAPACKE_ztbcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, lapack_int kd, + const lapack_complex_double* ab, lapack_int ldab, + double* rcond ); + +lapack_int LAPACKE_stbrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const float* ab, lapack_int ldab, const float* b, + lapack_int ldb, const float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_dtbrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const double* ab, lapack_int ldab, const double* b, + lapack_int ldb, const double* x, lapack_int ldx, + double* ferr, double* berr ); +lapack_int LAPACKE_ctbrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_ztbrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_stbtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const float* ab, lapack_int ldab, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dtbtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const double* ab, lapack_int ldab, double* b, + lapack_int ldb ); +lapack_int LAPACKE_ctbtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztbtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stfsm( int matrix_order, char transr, char side, char uplo, + char trans, char diag, lapack_int m, lapack_int n, + float alpha, const float* a, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dtfsm( int matrix_order, char transr, char side, char uplo, + char trans, char diag, lapack_int m, lapack_int n, + double alpha, const double* a, double* b, + lapack_int ldb ); +lapack_int LAPACKE_ctfsm( int matrix_order, char transr, char side, char uplo, + char trans, char diag, lapack_int m, lapack_int n, + lapack_complex_float alpha, + const lapack_complex_float* a, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztfsm( int matrix_order, char transr, char side, char uplo, + char trans, char diag, lapack_int m, lapack_int n, + lapack_complex_double alpha, + const lapack_complex_double* a, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stftri( int matrix_order, char transr, char uplo, char diag, + lapack_int n, float* a ); +lapack_int LAPACKE_dtftri( int matrix_order, char transr, char uplo, char diag, + lapack_int n, double* a ); +lapack_int LAPACKE_ctftri( int matrix_order, char transr, char uplo, char diag, + lapack_int n, lapack_complex_float* a ); +lapack_int LAPACKE_ztftri( int matrix_order, char transr, char uplo, char diag, + lapack_int n, lapack_complex_double* a ); + +lapack_int LAPACKE_stfttp( int matrix_order, char transr, char uplo, + lapack_int n, const float* arf, float* ap ); +lapack_int LAPACKE_dtfttp( int matrix_order, char transr, char uplo, + lapack_int n, const double* arf, double* ap ); +lapack_int LAPACKE_ctfttp( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* arf, + lapack_complex_float* ap ); +lapack_int LAPACKE_ztfttp( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* arf, + lapack_complex_double* ap ); + +lapack_int LAPACKE_stfttr( int matrix_order, char transr, char uplo, + lapack_int n, const float* arf, float* a, + lapack_int lda ); +lapack_int LAPACKE_dtfttr( int matrix_order, char transr, char uplo, + lapack_int n, const double* arf, double* a, + lapack_int lda ); +lapack_int LAPACKE_ctfttr( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* arf, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_ztfttr( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* arf, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_stgevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const float* s, lapack_int lds, const float* p, + lapack_int ldp, float* vl, lapack_int ldvl, + float* vr, lapack_int ldvr, lapack_int mm, + lapack_int* m ); +lapack_int LAPACKE_dtgevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const double* s, lapack_int lds, const double* p, + lapack_int ldp, double* vl, lapack_int ldvl, + double* vr, lapack_int ldvr, lapack_int mm, + lapack_int* m ); +lapack_int LAPACKE_ctgevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* s, lapack_int lds, + const lapack_complex_float* p, lapack_int ldp, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ztgevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* s, lapack_int lds, + const lapack_complex_double* p, lapack_int ldp, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m ); + +lapack_int LAPACKE_stgexc( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, float* q, + lapack_int ldq, float* z, lapack_int ldz, + lapack_int* ifst, lapack_int* ilst ); +lapack_int LAPACKE_dtgexc( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, double* q, + lapack_int ldq, double* z, lapack_int ldz, + lapack_int* ifst, lapack_int* ilst ); +lapack_int LAPACKE_ctgexc( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz, + lapack_int ifst, lapack_int ilst ); +lapack_int LAPACKE_ztgexc( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz, + lapack_int ifst, lapack_int ilst ); + +lapack_int LAPACKE_stgsen( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* alphar, float* alphai, float* beta, float* q, + lapack_int ldq, float* z, lapack_int ldz, + lapack_int* m, float* pl, float* pr, float* dif ); +lapack_int LAPACKE_dtgsen( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + double* a, lapack_int lda, double* b, lapack_int ldb, + double* alphar, double* alphai, double* beta, + double* q, lapack_int ldq, double* z, lapack_int ldz, + lapack_int* m, double* pl, double* pr, double* dif ); +lapack_int LAPACKE_ctgsen( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* alpha, + lapack_complex_float* beta, lapack_complex_float* q, + lapack_int ldq, lapack_complex_float* z, + lapack_int ldz, lapack_int* m, float* pl, float* pr, + float* dif ); +lapack_int LAPACKE_ztgsen( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz, + lapack_int* m, double* pl, double* pr, double* dif ); + +lapack_int LAPACKE_stgsja( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_int k, lapack_int l, float* a, lapack_int lda, + float* b, lapack_int ldb, float tola, float tolb, + float* alpha, float* beta, float* u, lapack_int ldu, + float* v, lapack_int ldv, float* q, lapack_int ldq, + lapack_int* ncycle ); +lapack_int LAPACKE_dtgsja( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_int k, lapack_int l, double* a, + lapack_int lda, double* b, lapack_int ldb, + double tola, double tolb, double* alpha, + double* beta, double* u, lapack_int ldu, double* v, + lapack_int ldv, double* q, lapack_int ldq, + lapack_int* ncycle ); +lapack_int LAPACKE_ctgsja( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_int k, lapack_int l, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float tola, float tolb, float* alpha, + float* beta, lapack_complex_float* u, lapack_int ldu, + lapack_complex_float* v, lapack_int ldv, + lapack_complex_float* q, lapack_int ldq, + lapack_int* ncycle ); +lapack_int LAPACKE_ztgsja( int matrix_order, char jobu, char jobv, char jobq, + lapack_int m, lapack_int p, lapack_int n, + lapack_int k, lapack_int l, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double tola, double tolb, + double* alpha, double* beta, + lapack_complex_double* u, lapack_int ldu, + lapack_complex_double* v, lapack_int ldv, + lapack_complex_double* q, lapack_int ldq, + lapack_int* ncycle ); + +lapack_int LAPACKE_stgsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const float* a, lapack_int lda, const float* b, + lapack_int ldb, const float* vl, lapack_int ldvl, + const float* vr, lapack_int ldvr, float* s, + float* dif, lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_dtgsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const double* a, lapack_int lda, const double* b, + lapack_int ldb, const double* vl, lapack_int ldvl, + const double* vr, lapack_int ldvr, double* s, + double* dif, lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ctgsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* vl, lapack_int ldvl, + const lapack_complex_float* vr, lapack_int ldvr, + float* s, float* dif, lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ztgsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* vl, lapack_int ldvl, + const lapack_complex_double* vr, lapack_int ldvr, + double* s, double* dif, lapack_int mm, + lapack_int* m ); + +lapack_int LAPACKE_stgsyl( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, const float* a, + lapack_int lda, const float* b, lapack_int ldb, + float* c, lapack_int ldc, const float* d, + lapack_int ldd, const float* e, lapack_int lde, + float* f, lapack_int ldf, float* scale, float* dif ); +lapack_int LAPACKE_dtgsyl( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, const double* a, + lapack_int lda, const double* b, lapack_int ldb, + double* c, lapack_int ldc, const double* d, + lapack_int ldd, const double* e, lapack_int lde, + double* f, lapack_int ldf, double* scale, + double* dif ); +lapack_int LAPACKE_ctgsyl( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* c, lapack_int ldc, + const lapack_complex_float* d, lapack_int ldd, + const lapack_complex_float* e, lapack_int lde, + lapack_complex_float* f, lapack_int ldf, + float* scale, float* dif ); +lapack_int LAPACKE_ztgsyl( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* c, lapack_int ldc, + const lapack_complex_double* d, lapack_int ldd, + const lapack_complex_double* e, lapack_int lde, + lapack_complex_double* f, lapack_int ldf, + double* scale, double* dif ); + +lapack_int LAPACKE_stpcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const float* ap, float* rcond ); +lapack_int LAPACKE_dtpcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const double* ap, double* rcond ); +lapack_int LAPACKE_ctpcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const lapack_complex_float* ap, + float* rcond ); +lapack_int LAPACKE_ztpcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const lapack_complex_double* ap, + double* rcond ); + +lapack_int LAPACKE_stprfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const float* ap, + const float* b, lapack_int ldb, const float* x, + lapack_int ldx, float* ferr, float* berr ); +lapack_int LAPACKE_dtprfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const double* ap, + const double* b, lapack_int ldb, const double* x, + lapack_int ldx, double* ferr, double* berr ); +lapack_int LAPACKE_ctprfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_ztprfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_stptri( int matrix_order, char uplo, char diag, lapack_int n, + float* ap ); +lapack_int LAPACKE_dtptri( int matrix_order, char uplo, char diag, lapack_int n, + double* ap ); +lapack_int LAPACKE_ctptri( int matrix_order, char uplo, char diag, lapack_int n, + lapack_complex_float* ap ); +lapack_int LAPACKE_ztptri( int matrix_order, char uplo, char diag, lapack_int n, + lapack_complex_double* ap ); + +lapack_int LAPACKE_stptrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const float* ap, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dtptrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const double* ap, + double* b, lapack_int ldb ); +lapack_int LAPACKE_ctptrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztptrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stpttf( int matrix_order, char transr, char uplo, + lapack_int n, const float* ap, float* arf ); +lapack_int LAPACKE_dtpttf( int matrix_order, char transr, char uplo, + lapack_int n, const double* ap, double* arf ); +lapack_int LAPACKE_ctpttf( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* ap, + lapack_complex_float* arf ); +lapack_int LAPACKE_ztpttf( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* ap, + lapack_complex_double* arf ); + +lapack_int LAPACKE_stpttr( int matrix_order, char uplo, lapack_int n, + const float* ap, float* a, lapack_int lda ); +lapack_int LAPACKE_dtpttr( int matrix_order, char uplo, lapack_int n, + const double* ap, double* a, lapack_int lda ); +lapack_int LAPACKE_ctpttr( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_ztpttr( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_strcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const float* a, lapack_int lda, + float* rcond ); +lapack_int LAPACKE_dtrcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const double* a, lapack_int lda, + double* rcond ); +lapack_int LAPACKE_ctrcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, float* rcond ); +lapack_int LAPACKE_ztrcon( int matrix_order, char norm, char uplo, char diag, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, double* rcond ); + +lapack_int LAPACKE_strevc( int matrix_order, char side, char howmny, + lapack_logical* select, lapack_int n, const float* t, + lapack_int ldt, float* vl, lapack_int ldvl, + float* vr, lapack_int ldvr, lapack_int mm, + lapack_int* m ); +lapack_int LAPACKE_dtrevc( int matrix_order, char side, char howmny, + lapack_logical* select, lapack_int n, + const double* t, lapack_int ldt, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ctrevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ztrevc( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m ); + +lapack_int LAPACKE_strexc( int matrix_order, char compq, lapack_int n, float* t, + lapack_int ldt, float* q, lapack_int ldq, + lapack_int* ifst, lapack_int* ilst ); +lapack_int LAPACKE_dtrexc( int matrix_order, char compq, lapack_int n, + double* t, lapack_int ldt, double* q, lapack_int ldq, + lapack_int* ifst, lapack_int* ilst ); +lapack_int LAPACKE_ctrexc( int matrix_order, char compq, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* q, lapack_int ldq, + lapack_int ifst, lapack_int ilst ); +lapack_int LAPACKE_ztrexc( int matrix_order, char compq, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* q, lapack_int ldq, + lapack_int ifst, lapack_int ilst ); + +lapack_int LAPACKE_strrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* b, lapack_int ldb, + const float* x, lapack_int ldx, float* ferr, + float* berr ); +lapack_int LAPACKE_dtrrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* b, lapack_int ldb, + const double* x, lapack_int ldx, double* ferr, + double* berr ); +lapack_int LAPACKE_ctrrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr ); +lapack_int LAPACKE_ztrrfs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr ); + +lapack_int LAPACKE_strsen( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, float* t, + lapack_int ldt, float* q, lapack_int ldq, float* wr, + float* wi, lapack_int* m, float* s, float* sep ); +lapack_int LAPACKE_dtrsen( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + double* t, lapack_int ldt, double* q, lapack_int ldq, + double* wr, double* wi, lapack_int* m, double* s, + double* sep ); +lapack_int LAPACKE_ctrsen( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* w, lapack_int* m, float* s, + float* sep ); +lapack_int LAPACKE_ztrsen( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* w, lapack_int* m, double* s, + double* sep ); + +lapack_int LAPACKE_strsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const float* t, lapack_int ldt, const float* vl, + lapack_int ldvl, const float* vr, lapack_int ldvr, + float* s, float* sep, lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_dtrsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const double* t, lapack_int ldt, const double* vl, + lapack_int ldvl, const double* vr, lapack_int ldvr, + double* s, double* sep, lapack_int mm, + lapack_int* m ); +lapack_int LAPACKE_ctrsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* t, lapack_int ldt, + const lapack_complex_float* vl, lapack_int ldvl, + const lapack_complex_float* vr, lapack_int ldvr, + float* s, float* sep, lapack_int mm, lapack_int* m ); +lapack_int LAPACKE_ztrsna( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* t, lapack_int ldt, + const lapack_complex_double* vl, lapack_int ldvl, + const lapack_complex_double* vr, lapack_int ldvr, + double* s, double* sep, lapack_int mm, + lapack_int* m ); + +lapack_int LAPACKE_strsyl( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const float* a, lapack_int lda, const float* b, + lapack_int ldb, float* c, lapack_int ldc, + float* scale ); +lapack_int LAPACKE_dtrsyl( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const double* a, lapack_int lda, const double* b, + lapack_int ldb, double* c, lapack_int ldc, + double* scale ); +lapack_int LAPACKE_ctrsyl( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* c, lapack_int ldc, + float* scale ); +lapack_int LAPACKE_ztrsyl( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* c, lapack_int ldc, + double* scale ); + +lapack_int LAPACKE_strtri( int matrix_order, char uplo, char diag, lapack_int n, + float* a, lapack_int lda ); +lapack_int LAPACKE_dtrtri( int matrix_order, char uplo, char diag, lapack_int n, + double* a, lapack_int lda ); +lapack_int LAPACKE_ctrtri( int matrix_order, char uplo, char diag, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_ztrtri( int matrix_order, char uplo, char diag, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_strtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, float* b, lapack_int ldb ); +lapack_int LAPACKE_dtrtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, double* b, lapack_int ldb ); +lapack_int LAPACKE_ctrtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztrtrs( int matrix_order, char uplo, char trans, char diag, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_strttf( int matrix_order, char transr, char uplo, + lapack_int n, const float* a, lapack_int lda, + float* arf ); +lapack_int LAPACKE_dtrttf( int matrix_order, char transr, char uplo, + lapack_int n, const double* a, lapack_int lda, + double* arf ); +lapack_int LAPACKE_ctrttf( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* arf ); +lapack_int LAPACKE_ztrttf( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* arf ); + +lapack_int LAPACKE_strttp( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float* ap ); +lapack_int LAPACKE_dtrttp( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double* ap ); +lapack_int LAPACKE_ctrttp( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* ap ); +lapack_int LAPACKE_ztrttp( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* ap ); + +lapack_int LAPACKE_stzrzf( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau ); +lapack_int LAPACKE_dtzrzf( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau ); +lapack_int LAPACKE_ctzrzf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau ); +lapack_int LAPACKE_ztzrzf( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau ); + +lapack_int LAPACKE_cungbr( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zungbr( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cunghr( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zunghr( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cunglq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zunglq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cungql( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zungql( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cungqr( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zungqr( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cungrq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau ); +lapack_int LAPACKE_zungrq( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau ); + +lapack_int LAPACKE_cungtr( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau ); +lapack_int LAPACKE_zungtr( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau ); + +lapack_int LAPACKE_cunmbr( int matrix_order, char vect, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmbr( int matrix_order, char vect, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmhr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmhr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmlq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmlq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmql( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmql( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmqr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmqr( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmrq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmrq( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmrz( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmrz( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cunmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zunmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_cupgtr( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_complex_float* tau, + lapack_complex_float* q, lapack_int ldq ); +lapack_int LAPACKE_zupgtr( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_complex_double* tau, + lapack_complex_double* q, lapack_int ldq ); + +lapack_int LAPACKE_cupmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, + const lapack_complex_float* ap, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc ); +lapack_int LAPACKE_zupmtr( int matrix_order, char side, char uplo, char trans, + lapack_int m, lapack_int n, + const lapack_complex_double* ap, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc ); + +lapack_int LAPACKE_sbdsdc_work( int matrix_order, char uplo, char compq, + lapack_int n, float* d, float* e, float* u, + lapack_int ldu, float* vt, lapack_int ldvt, + float* q, lapack_int* iq, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dbdsdc_work( int matrix_order, char uplo, char compq, + lapack_int n, double* d, double* e, double* u, + lapack_int ldu, double* vt, lapack_int ldvt, + double* q, lapack_int* iq, double* work, + lapack_int* iwork ); + +lapack_int LAPACKE_sbdsqr_work( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + float* d, float* e, float* vt, lapack_int ldvt, + float* u, lapack_int ldu, float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_dbdsqr_work( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + double* d, double* e, double* vt, + lapack_int ldvt, double* u, lapack_int ldu, + double* c, lapack_int ldc, double* work ); +lapack_int LAPACKE_cbdsqr_work( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + float* d, float* e, lapack_complex_float* vt, + lapack_int ldvt, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_zbdsqr_work( int matrix_order, char uplo, lapack_int n, + lapack_int ncvt, lapack_int nru, lapack_int ncc, + double* d, double* e, lapack_complex_double* vt, + lapack_int ldvt, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* c, + lapack_int ldc, double* work ); + +lapack_int LAPACKE_sdisna_work( char job, lapack_int m, lapack_int n, + const float* d, float* sep ); +lapack_int LAPACKE_ddisna_work( char job, lapack_int m, lapack_int n, + const double* d, double* sep ); + +lapack_int LAPACKE_sgbbrd_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, float* ab, lapack_int ldab, + float* d, float* e, float* q, lapack_int ldq, + float* pt, lapack_int ldpt, float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_dgbbrd_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, double* ab, lapack_int ldab, + double* d, double* e, double* q, lapack_int ldq, + double* pt, lapack_int ldpt, double* c, + lapack_int ldc, double* work ); +lapack_int LAPACKE_cgbbrd_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, lapack_complex_float* ab, + lapack_int ldab, float* d, float* e, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* pt, lapack_int ldpt, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgbbrd_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int ncc, lapack_int kl, + lapack_int ku, lapack_complex_double* ab, + lapack_int ldab, double* d, double* e, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* pt, lapack_int ldpt, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgbcon_work( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, const lapack_int* ipiv, + float anorm, float* rcond, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgbcon_work( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, const lapack_int* ipiv, + double anorm, double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgbcon_work( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgbcon_work( int matrix_order, char norm, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, + lapack_int ldab, const lapack_int* ipiv, + double anorm, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgbequ_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, float* r, float* c, + float* rowcnd, float* colcnd, float* amax ); +lapack_int LAPACKE_dgbequ_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); +lapack_int LAPACKE_cgbequ_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, lapack_int ldab, + float* r, float* c, float* rowcnd, + float* colcnd, float* amax ); +lapack_int LAPACKE_zgbequ_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); + +lapack_int LAPACKE_sgbequb_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* ab, + lapack_int ldab, float* r, float* c, + float* rowcnd, float* colcnd, float* amax ); +lapack_int LAPACKE_dgbequb_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); +lapack_int LAPACKE_cgbequb_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_float* ab, + lapack_int ldab, float* r, float* c, + float* rowcnd, float* colcnd, float* amax ); +lapack_int LAPACKE_zgbequb_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + const lapack_complex_double* ab, + lapack_int ldab, double* r, double* c, + double* rowcnd, double* colcnd, double* amax ); + +lapack_int LAPACKE_sgbrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const float* ab, lapack_int ldab, + const float* afb, lapack_int ldafb, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgbrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const double* ab, lapack_int ldab, + const double* afb, lapack_int ldafb, + const lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgbrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* afb, + lapack_int ldafb, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgbrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, + const lapack_complex_double* afb, + lapack_int ldafb, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgbrfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const float* ab, + lapack_int ldab, const float* afb, + lapack_int ldafb, const lapack_int* ipiv, + const float* r, const float* c, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgbrfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, const double* ab, + lapack_int ldab, const double* afb, + lapack_int ldafb, const lapack_int* ipiv, + const double* r, const double* c, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgbrfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, + const lapack_complex_float* ab, + lapack_int ldab, + const lapack_complex_float* afb, + lapack_int ldafb, const lapack_int* ipiv, + const float* r, const float* c, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgbrfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, + const lapack_complex_double* afb, + lapack_int ldafb, const lapack_int* ipiv, + const double* r, const double* c, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sgbsv_work( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, float* ab, + lapack_int ldab, lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgbsv_work( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, double* ab, + lapack_int ldab, lapack_int* ipiv, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cgbsv_work( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgbsv_work( int matrix_order, lapack_int n, lapack_int kl, + lapack_int ku, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sgbsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, float* ab, lapack_int ldab, + float* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, float* r, float* c, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dgbsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, double* ab, lapack_int ldab, + double* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, double* r, double* c, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cgbsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + float* r, float* c, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgbsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* afb, + lapack_int ldafb, lapack_int* ipiv, char* equed, + double* r, double* c, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sgbsvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, float* ab, lapack_int ldab, + float* afb, lapack_int ldafb, lapack_int* ipiv, + char* equed, float* r, float* c, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgbsvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, double* ab, lapack_int ldab, + double* afb, lapack_int ldafb, + lapack_int* ipiv, char* equed, double* r, + double* c, double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgbsvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* afb, + lapack_int ldafb, lapack_int* ipiv, + char* equed, float* r, float* c, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgbsvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int kl, lapack_int ku, + lapack_int nrhs, lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* afb, + lapack_int ldafb, lapack_int* ipiv, + char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sgbtrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, float* ab, + lapack_int ldab, lapack_int* ipiv ); +lapack_int LAPACKE_dgbtrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, double* ab, + lapack_int ldab, lapack_int* ipiv ); +lapack_int LAPACKE_cgbtrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + lapack_complex_float* ab, lapack_int ldab, + lapack_int* ipiv ); +lapack_int LAPACKE_zgbtrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, + lapack_complex_double* ab, lapack_int ldab, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgbtrs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const float* ab, lapack_int ldab, + const lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgbtrs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const double* ab, lapack_int ldab, + const lapack_int* ipiv, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cgbtrs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgbtrs_work( int matrix_order, char trans, lapack_int n, + lapack_int kl, lapack_int ku, lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sgebak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const float* scale, lapack_int m, float* v, + lapack_int ldv ); +lapack_int LAPACKE_dgebak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const double* scale, lapack_int m, double* v, + lapack_int ldv ); +lapack_int LAPACKE_cgebak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const float* scale, lapack_int m, + lapack_complex_float* v, lapack_int ldv ); +lapack_int LAPACKE_zgebak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const double* scale, lapack_int m, + lapack_complex_double* v, lapack_int ldv ); + +lapack_int LAPACKE_sgebal_work( int matrix_order, char job, lapack_int n, + float* a, lapack_int lda, lapack_int* ilo, + lapack_int* ihi, float* scale ); +lapack_int LAPACKE_dgebal_work( int matrix_order, char job, lapack_int n, + double* a, lapack_int lda, lapack_int* ilo, + lapack_int* ihi, double* scale ); +lapack_int LAPACKE_cgebal_work( int matrix_order, char job, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ilo, lapack_int* ihi, + float* scale ); +lapack_int LAPACKE_zgebal_work( int matrix_order, char job, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ilo, lapack_int* ihi, + double* scale ); + +lapack_int LAPACKE_sgebrd_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* d, float* e, + float* tauq, float* taup, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dgebrd_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* d, double* e, + double* tauq, double* taup, double* work, + lapack_int lwork ); +lapack_int LAPACKE_cgebrd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + float* d, float* e, lapack_complex_float* tauq, + lapack_complex_float* taup, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgebrd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double* d, double* e, + lapack_complex_double* tauq, + lapack_complex_double* taup, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgecon_work( int matrix_order, char norm, lapack_int n, + const float* a, lapack_int lda, float anorm, + float* rcond, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dgecon_work( int matrix_order, char norm, lapack_int n, + const double* a, lapack_int lda, double anorm, + double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgecon_work( int matrix_order, char norm, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float anorm, float* rcond, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgecon_work( int matrix_order, char norm, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double anorm, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgeequ_work( int matrix_order, lapack_int m, lapack_int n, + const float* a, lapack_int lda, float* r, + float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_dgeequ_work( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, double* r, + double* c, double* rowcnd, double* colcnd, + double* amax ); +lapack_int LAPACKE_cgeequ_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* r, float* c, float* rowcnd, + float* colcnd, float* amax ); +lapack_int LAPACKE_zgeequ_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* r, double* c, double* rowcnd, + double* colcnd, double* amax ); + +lapack_int LAPACKE_sgeequb_work( int matrix_order, lapack_int m, lapack_int n, + const float* a, lapack_int lda, float* r, + float* c, float* rowcnd, float* colcnd, + float* amax ); +lapack_int LAPACKE_dgeequb_work( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, double* r, + double* c, double* rowcnd, double* colcnd, + double* amax ); +lapack_int LAPACKE_cgeequb_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* r, float* c, float* rowcnd, + float* colcnd, float* amax ); +lapack_int LAPACKE_zgeequb_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* r, double* c, double* rowcnd, + double* colcnd, double* amax ); + +lapack_int LAPACKE_sgees_work( int matrix_order, char jobvs, char sort, + LAPACK_S_SELECT2 select, lapack_int n, float* a, + lapack_int lda, lapack_int* sdim, float* wr, + float* wi, float* vs, lapack_int ldvs, + float* work, lapack_int lwork, + lapack_logical* bwork ); +lapack_int LAPACKE_dgees_work( int matrix_order, char jobvs, char sort, + LAPACK_D_SELECT2 select, lapack_int n, double* a, + lapack_int lda, lapack_int* sdim, double* wr, + double* wi, double* vs, lapack_int ldvs, + double* work, lapack_int lwork, + lapack_logical* bwork ); +lapack_int LAPACKE_cgees_work( int matrix_order, char jobvs, char sort, + LAPACK_C_SELECT1 select, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* sdim, lapack_complex_float* w, + lapack_complex_float* vs, lapack_int ldvs, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_logical* bwork ); +lapack_int LAPACKE_zgees_work( int matrix_order, char jobvs, char sort, + LAPACK_Z_SELECT1 select, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* sdim, lapack_complex_double* w, + lapack_complex_double* vs, lapack_int ldvs, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_logical* bwork ); + +lapack_int LAPACKE_sgeesx_work( int matrix_order, char jobvs, char sort, + LAPACK_S_SELECT2 select, char sense, + lapack_int n, float* a, lapack_int lda, + lapack_int* sdim, float* wr, float* wi, + float* vs, lapack_int ldvs, float* rconde, + float* rcondv, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork, + lapack_logical* bwork ); +lapack_int LAPACKE_dgeesx_work( int matrix_order, char jobvs, char sort, + LAPACK_D_SELECT2 select, char sense, + lapack_int n, double* a, lapack_int lda, + lapack_int* sdim, double* wr, double* wi, + double* vs, lapack_int ldvs, double* rconde, + double* rcondv, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork, + lapack_logical* bwork ); +lapack_int LAPACKE_cgeesx_work( int matrix_order, char jobvs, char sort, + LAPACK_C_SELECT1 select, char sense, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_int* sdim, + lapack_complex_float* w, + lapack_complex_float* vs, lapack_int ldvs, + float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_logical* bwork ); +lapack_int LAPACKE_zgeesx_work( int matrix_order, char jobvs, char sort, + LAPACK_Z_SELECT1 select, char sense, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_int* sdim, + lapack_complex_double* w, + lapack_complex_double* vs, lapack_int ldvs, + double* rconde, double* rcondv, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_logical* bwork ); + +lapack_int LAPACKE_sgeev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, float* a, lapack_int lda, + float* wr, float* wi, float* vl, lapack_int ldvl, + float* vr, lapack_int ldvr, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dgeev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, double* a, lapack_int lda, + double* wr, double* wi, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgeev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* w, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zgeev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* w, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_sgeevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, float* a, + lapack_int lda, float* wr, float* wi, float* vl, + lapack_int ldvl, float* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, float* scale, + float* abnrm, float* rconde, float* rcondv, + float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dgeevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, double* a, + lapack_int lda, double* wr, double* wi, + double* vl, lapack_int ldvl, double* vr, + lapack_int ldvr, lapack_int* ilo, + lapack_int* ihi, double* scale, double* abnrm, + double* rconde, double* rcondv, double* work, + lapack_int lwork, lapack_int* iwork ); +lapack_int LAPACKE_cgeevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* w, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, float* scale, + float* abnrm, float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zgeevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* w, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, double* scale, + double* abnrm, double* rconde, double* rcondv, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_sgehrd_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, float* a, lapack_int lda, + float* tau, float* work, lapack_int lwork ); +lapack_int LAPACKE_dgehrd_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, double* a, lapack_int lda, + double* tau, double* work, lapack_int lwork ); +lapack_int LAPACKE_cgehrd_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgehrd_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgejsv_work( int matrix_order, char joba, char jobu, + char jobv, char jobr, char jobt, char jobp, + lapack_int m, lapack_int n, float* a, + lapack_int lda, float* sva, float* u, + lapack_int ldu, float* v, lapack_int ldv, + float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dgejsv_work( int matrix_order, char joba, char jobu, + char jobv, char jobr, char jobt, char jobp, + lapack_int m, lapack_int n, double* a, + lapack_int lda, double* sva, double* u, + lapack_int ldu, double* v, lapack_int ldv, + double* work, lapack_int lwork, + lapack_int* iwork ); + +lapack_int LAPACKE_sgelq2_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work ); +lapack_int LAPACKE_dgelq2_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work ); +lapack_int LAPACKE_cgelq2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work ); +lapack_int LAPACKE_zgelq2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work ); + +lapack_int LAPACKE_sgelqf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgelqf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgelqf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgelqf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgels_work( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgels_work( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgels_work( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgels_work( int matrix_order, char trans, lapack_int m, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgelsd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + float* b, lapack_int ldb, float* s, float rcond, + lapack_int* rank, float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dgelsd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* s, + double rcond, lapack_int* rank, double* work, + lapack_int lwork, lapack_int* iwork ); +lapack_int LAPACKE_cgelsd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int* iwork ); +lapack_int LAPACKE_zgelsd_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* s, double rcond, + lapack_int* rank, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int* iwork ); + +lapack_int LAPACKE_sgelss_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + float* b, lapack_int ldb, float* s, float rcond, + lapack_int* rank, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dgelss_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* s, + double rcond, lapack_int* rank, double* work, + lapack_int lwork ); +lapack_int LAPACKE_cgelss_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* s, float rcond, + lapack_int* rank, lapack_complex_float* work, + lapack_int lwork, float* rwork ); +lapack_int LAPACKE_zgelss_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double* s, double rcond, + lapack_int* rank, lapack_complex_double* work, + lapack_int lwork, double* rwork ); + +lapack_int LAPACKE_sgelsy_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + float* b, lapack_int ldb, lapack_int* jpvt, + float rcond, lapack_int* rank, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dgelsy_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, lapack_int* jpvt, + double rcond, lapack_int* rank, double* work, + lapack_int lwork ); +lapack_int LAPACKE_cgelsy_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_int* jpvt, float rcond, + lapack_int* rank, lapack_complex_float* work, + lapack_int lwork, float* rwork ); +lapack_int LAPACKE_zgelsy_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_int* jpvt, double rcond, + lapack_int* rank, lapack_complex_double* work, + lapack_int lwork, double* rwork ); + +lapack_int LAPACKE_sgeqlf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgeqlf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgeqlf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgeqlf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgeqp3_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* jpvt, + float* tau, float* work, lapack_int lwork ); +lapack_int LAPACKE_dgeqp3_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* jpvt, + double* tau, double* work, lapack_int lwork ); +lapack_int LAPACKE_cgeqp3_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zgeqp3_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_sgeqpf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* jpvt, + float* tau, float* work ); +lapack_int LAPACKE_dgeqpf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* jpvt, + double* tau, double* work ); +lapack_int LAPACKE_cgeqpf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_float* tau, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgeqpf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* jpvt, lapack_complex_double* tau, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgeqr2_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work ); +lapack_int LAPACKE_dgeqr2_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work ); +lapack_int LAPACKE_cgeqr2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work ); +lapack_int LAPACKE_zgeqr2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work ); + +lapack_int LAPACKE_sgeqrf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgeqrf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgeqrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgeqrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgeqrfp_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgeqrfp_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgeqrfp_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgeqrfp_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sgerfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgerfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cgerfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgerfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgerfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, + lapack_int ldaf, const lapack_int* ipiv, + const float* r, const float* c, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgerfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* r, const double* c, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgerfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const float* r, const float* c, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgerfsx_work( int matrix_order, char trans, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* r, const double* c, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sgerqf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgerqf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgerqf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgerqf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgesdd_work( int matrix_order, char jobz, lapack_int m, + lapack_int n, float* a, lapack_int lda, + float* s, float* u, lapack_int ldu, float* vt, + lapack_int ldvt, float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dgesdd_work( int matrix_order, char jobz, lapack_int m, + lapack_int n, double* a, lapack_int lda, + double* s, double* u, lapack_int ldu, + double* vt, lapack_int ldvt, double* work, + lapack_int lwork, lapack_int* iwork ); +lapack_int LAPACKE_cgesdd_work( int matrix_order, char jobz, lapack_int m, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float* s, + lapack_complex_float* u, lapack_int ldu, + lapack_complex_float* vt, lapack_int ldvt, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int* iwork ); +lapack_int LAPACKE_zgesdd_work( int matrix_order, char jobz, lapack_int m, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double* s, + lapack_complex_double* u, lapack_int ldu, + lapack_complex_double* vt, lapack_int ldvt, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int* iwork ); + +lapack_int LAPACKE_sgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + float* a, lapack_int lda, lapack_int* ipiv, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + double* a, lapack_int lda, lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); +lapack_int LAPACKE_dsgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + double* a, lapack_int lda, lapack_int* ipiv, + double* b, lapack_int ldb, double* x, + lapack_int ldx, double* work, float* swork, + lapack_int* iter ); +lapack_int LAPACKE_zcgesv_work( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, lapack_complex_double* work, + lapack_complex_float* swork, double* rwork, + lapack_int* iter ); + +lapack_int LAPACKE_sgesvd_work( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, float* a, + lapack_int lda, float* s, float* u, + lapack_int ldu, float* vt, lapack_int ldvt, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgesvd_work( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, double* a, + lapack_int lda, double* s, double* u, + lapack_int ldu, double* vt, lapack_int ldvt, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgesvd_work( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + float* s, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* vt, + lapack_int ldvt, lapack_complex_float* work, + lapack_int lwork, float* rwork ); +lapack_int LAPACKE_zgesvd_work( int matrix_order, char jobu, char jobvt, + lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double* s, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* vt, + lapack_int ldvt, lapack_complex_double* work, + lapack_int lwork, double* rwork ); + +lapack_int LAPACKE_sgesvj_work( int matrix_order, char joba, char jobu, + char jobv, lapack_int m, lapack_int n, float* a, + lapack_int lda, float* sva, lapack_int mv, + float* v, lapack_int ldv, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dgesvj_work( int matrix_order, char joba, char jobu, + char jobv, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* sva, + lapack_int mv, double* v, lapack_int ldv, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgesvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, + float* c, float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dgesvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, + double* c, double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, double* work, lapack_int* iwork ); +lapack_int LAPACKE_cgesvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, + float* c, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zgesvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, + double* c, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sgesvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, + float* c, float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgesvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, + double* c, double* b, lapack_int ldb, + double* x, lapack_int ldx, double* rcond, + double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgesvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* r, + float* c, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgesvxx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* r, + double* c, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgetf2_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_dgetf2_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_cgetf2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zgetf2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgetrf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_dgetrf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, lapack_int* ipiv ); +lapack_int LAPACKE_cgetrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv ); +lapack_int LAPACKE_zgetrf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv ); + +lapack_int LAPACKE_sgetri_work( int matrix_order, lapack_int n, float* a, + lapack_int lda, const lapack_int* ipiv, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dgetri_work( int matrix_order, lapack_int n, double* a, + lapack_int lda, const lapack_int* ipiv, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cgetri_work( int matrix_order, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgetri_work( int matrix_order, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgetrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgetrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cgetrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zgetrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sggbak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const float* lscale, const float* rscale, + lapack_int m, float* v, lapack_int ldv ); +lapack_int LAPACKE_dggbak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const double* lscale, const double* rscale, + lapack_int m, double* v, lapack_int ldv ); +lapack_int LAPACKE_cggbak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const float* lscale, const float* rscale, + lapack_int m, lapack_complex_float* v, + lapack_int ldv ); +lapack_int LAPACKE_zggbak_work( int matrix_order, char job, char side, + lapack_int n, lapack_int ilo, lapack_int ihi, + const double* lscale, const double* rscale, + lapack_int m, lapack_complex_double* v, + lapack_int ldv ); + +lapack_int LAPACKE_sggbal_work( int matrix_order, char job, lapack_int n, + float* a, lapack_int lda, float* b, + lapack_int ldb, lapack_int* ilo, + lapack_int* ihi, float* lscale, float* rscale, + float* work ); +lapack_int LAPACKE_dggbal_work( int matrix_order, char job, lapack_int n, + double* a, lapack_int lda, double* b, + lapack_int ldb, lapack_int* ilo, + lapack_int* ihi, double* lscale, double* rscale, + double* work ); +lapack_int LAPACKE_cggbal_work( int matrix_order, char job, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale, float* work ); +lapack_int LAPACKE_zggbal_work( int matrix_order, char job, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_int* ilo, lapack_int* ihi, + double* lscale, double* rscale, double* work ); + +lapack_int LAPACKE_sgges_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_S_SELECT3 selctg, lapack_int n, + float* a, lapack_int lda, float* b, + lapack_int ldb, lapack_int* sdim, float* alphar, + float* alphai, float* beta, float* vsl, + lapack_int ldvsl, float* vsr, lapack_int ldvsr, + float* work, lapack_int lwork, + lapack_logical* bwork ); +lapack_int LAPACKE_dgges_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_D_SELECT3 selctg, lapack_int n, + double* a, lapack_int lda, double* b, + lapack_int ldb, lapack_int* sdim, double* alphar, + double* alphai, double* beta, double* vsl, + lapack_int ldvsl, double* vsr, lapack_int ldvsr, + double* work, lapack_int lwork, + lapack_logical* bwork ); +lapack_int LAPACKE_cgges_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_C_SELECT2 selctg, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_int* sdim, lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* vsl, lapack_int ldvsl, + lapack_complex_float* vsr, lapack_int ldvsr, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_logical* bwork ); +lapack_int LAPACKE_zgges_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_Z_SELECT2 selctg, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_int* sdim, lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int ldvsl, + lapack_complex_double* vsr, lapack_int ldvsr, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_logical* bwork ); + +lapack_int LAPACKE_sggesx_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_S_SELECT3 selctg, char sense, + lapack_int n, float* a, lapack_int lda, + float* b, lapack_int ldb, lapack_int* sdim, + float* alphar, float* alphai, float* beta, + float* vsl, lapack_int ldvsl, float* vsr, + lapack_int ldvsr, float* rconde, float* rcondv, + float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork, + lapack_logical* bwork ); +lapack_int LAPACKE_dggesx_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_D_SELECT3 selctg, char sense, + lapack_int n, double* a, lapack_int lda, + double* b, lapack_int ldb, lapack_int* sdim, + double* alphar, double* alphai, double* beta, + double* vsl, lapack_int ldvsl, double* vsr, + lapack_int ldvsr, double* rconde, + double* rcondv, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork, + lapack_logical* bwork ); +lapack_int LAPACKE_cggesx_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_C_SELECT2 selctg, char sense, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_int* sdim, + lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* vsl, lapack_int ldvsl, + lapack_complex_float* vsr, lapack_int ldvsr, + float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int* iwork, + lapack_int liwork, lapack_logical* bwork ); +lapack_int LAPACKE_zggesx_work( int matrix_order, char jobvsl, char jobvsr, + char sort, LAPACK_Z_SELECT2 selctg, char sense, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_int* sdim, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int ldvsl, + lapack_complex_double* vsr, lapack_int ldvsr, + double* rconde, double* rcondv, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int* iwork, + lapack_int liwork, lapack_logical* bwork ); + +lapack_int LAPACKE_sggev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, float* a, lapack_int lda, float* b, + lapack_int ldb, float* alphar, float* alphai, + float* beta, float* vl, lapack_int ldvl, + float* vr, lapack_int ldvr, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dggev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, double* a, lapack_int lda, + double* b, lapack_int ldb, double* alphar, + double* alphai, double* beta, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + double* work, lapack_int lwork ); +lapack_int LAPACKE_cggev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zggev_work( int matrix_order, char jobvl, char jobvr, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_sggevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* alphar, float* alphai, float* beta, + float* vl, lapack_int ldvl, float* vr, + lapack_int ldvr, lapack_int* ilo, + lapack_int* ihi, float* lscale, float* rscale, + float* abnrm, float* bbnrm, float* rconde, + float* rcondv, float* work, lapack_int lwork, + lapack_int* iwork, lapack_logical* bwork ); +lapack_int LAPACKE_dggevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* alphar, double* alphai, double* beta, + double* vl, lapack_int ldvl, double* vr, + lapack_int ldvr, lapack_int* ilo, + lapack_int* ihi, double* lscale, double* rscale, + double* abnrm, double* bbnrm, double* rconde, + double* rcondv, double* work, lapack_int lwork, + lapack_int* iwork, lapack_logical* bwork ); +lapack_int LAPACKE_cggevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale, float* abnrm, float* bbnrm, + float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int* iwork, + lapack_logical* bwork ); +lapack_int LAPACKE_zggevx_work( int matrix_order, char balanc, char jobvl, + char jobvr, char sense, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int* ilo, lapack_int* ihi, + double* lscale, double* rscale, double* abnrm, + double* bbnrm, double* rconde, double* rcondv, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int* iwork, + lapack_logical* bwork ); + +lapack_int LAPACKE_sggglm_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, float* a, lapack_int lda, + float* b, lapack_int ldb, float* d, float* x, + float* y, float* work, lapack_int lwork ); +lapack_int LAPACKE_dggglm_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, double* a, lapack_int lda, + double* b, lapack_int ldb, double* d, double* x, + double* y, double* work, lapack_int lwork ); +lapack_int LAPACKE_cggglm_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* d, + lapack_complex_float* x, + lapack_complex_float* y, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zggglm_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* d, + lapack_complex_double* x, + lapack_complex_double* y, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sgghrd_work( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + float* a, lapack_int lda, float* b, + lapack_int ldb, float* q, lapack_int ldq, + float* z, lapack_int ldz ); +lapack_int LAPACKE_dgghrd_work( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + double* a, lapack_int lda, double* b, + lapack_int ldb, double* q, lapack_int ldq, + double* z, lapack_int ldz ); +lapack_int LAPACKE_cgghrd_work( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz ); +lapack_int LAPACKE_zgghrd_work( int matrix_order, char compq, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz ); + +lapack_int LAPACKE_sgglse_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, float* a, lapack_int lda, + float* b, lapack_int ldb, float* c, float* d, + float* x, float* work, lapack_int lwork ); +lapack_int LAPACKE_dgglse_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, double* a, lapack_int lda, + double* b, lapack_int ldb, double* c, double* d, + double* x, double* work, lapack_int lwork ); +lapack_int LAPACKE_cgglse_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* c, + lapack_complex_float* d, + lapack_complex_float* x, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zgglse_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* c, + lapack_complex_double* d, + lapack_complex_double* x, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sggqrf_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, float* a, lapack_int lda, + float* taua, float* b, lapack_int ldb, + float* taub, float* work, lapack_int lwork ); +lapack_int LAPACKE_dggqrf_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, double* a, lapack_int lda, + double* taua, double* b, lapack_int ldb, + double* taub, double* work, lapack_int lwork ); +lapack_int LAPACKE_cggqrf_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* taua, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* taub, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zggqrf_work( int matrix_order, lapack_int n, lapack_int m, + lapack_int p, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* taua, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* taub, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sggrqf_work( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, float* a, lapack_int lda, + float* taua, float* b, lapack_int ldb, + float* taub, float* work, lapack_int lwork ); +lapack_int LAPACKE_dggrqf_work( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, double* a, lapack_int lda, + double* taua, double* b, lapack_int ldb, + double* taub, double* work, lapack_int lwork ); +lapack_int LAPACKE_cggrqf_work( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* taua, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* taub, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zggrqf_work( int matrix_order, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* taua, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* taub, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_sggsvd_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int n, + lapack_int p, lapack_int* k, lapack_int* l, + float* a, lapack_int lda, float* b, + lapack_int ldb, float* alpha, float* beta, + float* u, lapack_int ldu, float* v, + lapack_int ldv, float* q, lapack_int ldq, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dggsvd_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int n, + lapack_int p, lapack_int* k, lapack_int* l, + double* a, lapack_int lda, double* b, + lapack_int ldb, double* alpha, double* beta, + double* u, lapack_int ldu, double* v, + lapack_int ldv, double* q, lapack_int ldq, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cggsvd_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int n, + lapack_int p, lapack_int* k, lapack_int* l, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + float* alpha, float* beta, + lapack_complex_float* u, lapack_int ldu, + lapack_complex_float* v, lapack_int ldv, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* work, float* rwork, + lapack_int* iwork ); +lapack_int LAPACKE_zggsvd_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int n, + lapack_int p, lapack_int* k, lapack_int* l, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double* alpha, double* beta, + lapack_complex_double* u, lapack_int ldu, + lapack_complex_double* v, lapack_int ldv, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* work, double* rwork, + lapack_int* iwork ); + +lapack_int LAPACKE_sggsvp_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, float* a, lapack_int lda, + float* b, lapack_int ldb, float tola, + float tolb, lapack_int* k, lapack_int* l, + float* u, lapack_int ldu, float* v, + lapack_int ldv, float* q, lapack_int ldq, + lapack_int* iwork, float* tau, float* work ); +lapack_int LAPACKE_dggsvp_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, double* a, lapack_int lda, + double* b, lapack_int ldb, double tola, + double tolb, lapack_int* k, lapack_int* l, + double* u, lapack_int ldu, double* v, + lapack_int ldv, double* q, lapack_int ldq, + lapack_int* iwork, double* tau, double* work ); +lapack_int LAPACKE_cggsvp_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float tola, float tolb, + lapack_int* k, lapack_int* l, + lapack_complex_float* u, lapack_int ldu, + lapack_complex_float* v, lapack_int ldv, + lapack_complex_float* q, lapack_int ldq, + lapack_int* iwork, float* rwork, + lapack_complex_float* tau, + lapack_complex_float* work ); +lapack_int LAPACKE_zggsvp_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, double tola, double tolb, + lapack_int* k, lapack_int* l, + lapack_complex_double* u, lapack_int ldu, + lapack_complex_double* v, lapack_int ldv, + lapack_complex_double* q, lapack_int ldq, + lapack_int* iwork, double* rwork, + lapack_complex_double* tau, + lapack_complex_double* work ); + +lapack_int LAPACKE_sgtcon_work( char norm, lapack_int n, const float* dl, + const float* d, const float* du, + const float* du2, const lapack_int* ipiv, + float anorm, float* rcond, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgtcon_work( char norm, lapack_int n, const double* dl, + const double* d, const double* du, + const double* du2, const lapack_int* ipiv, + double anorm, double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgtcon_work( char norm, lapack_int n, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work ); +lapack_int LAPACKE_zgtcon_work( char norm, lapack_int n, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, + const lapack_int* ipiv, double anorm, + double* rcond, lapack_complex_double* work ); + +lapack_int LAPACKE_sgtrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* dl, + const float* d, const float* du, + const float* dlf, const float* df, + const float* duf, const float* du2, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dgtrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* dl, + const double* d, const double* du, + const double* dlf, const double* df, + const double* duf, const double* du2, + const lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cgtrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* dlf, + const lapack_complex_float* df, + const lapack_complex_float* duf, + const lapack_complex_float* du2, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgtrfs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* dlf, + const lapack_complex_double* df, + const lapack_complex_double* duf, + const lapack_complex_double* du2, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgtsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + float* dl, float* d, float* du, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dgtsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + double* dl, double* d, double* du, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cgtsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_float* dl, + lapack_complex_float* d, + lapack_complex_float* du, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zgtsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + lapack_complex_double* dl, + lapack_complex_double* d, + lapack_complex_double* du, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sgtsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, const float* dl, + const float* d, const float* du, float* dlf, + float* df, float* duf, float* du2, + lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dgtsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, const double* dl, + const double* d, const double* du, double* dlf, + double* df, double* duf, double* du2, + lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cgtsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + lapack_complex_float* dlf, + lapack_complex_float* df, + lapack_complex_float* duf, + lapack_complex_float* du2, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zgtsvx_work( int matrix_order, char fact, char trans, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + lapack_complex_double* dlf, + lapack_complex_double* df, + lapack_complex_double* duf, + lapack_complex_double* du2, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sgttrf_work( lapack_int n, float* dl, float* d, float* du, + float* du2, lapack_int* ipiv ); +lapack_int LAPACKE_dgttrf_work( lapack_int n, double* dl, double* d, double* du, + double* du2, lapack_int* ipiv ); +lapack_int LAPACKE_cgttrf_work( lapack_int n, lapack_complex_float* dl, + lapack_complex_float* d, + lapack_complex_float* du, + lapack_complex_float* du2, lapack_int* ipiv ); +lapack_int LAPACKE_zgttrf_work( lapack_int n, lapack_complex_double* dl, + lapack_complex_double* d, + lapack_complex_double* du, + lapack_complex_double* du2, lapack_int* ipiv ); + +lapack_int LAPACKE_sgttrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const float* dl, + const float* d, const float* du, + const float* du2, const lapack_int* ipiv, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dgttrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const double* dl, + const double* d, const double* du, + const double* du2, const lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cgttrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zgttrs_work( int matrix_order, char trans, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, + const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_chbev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_float* ab, lapack_int ldab, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zhbev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_double* ab, lapack_int ldab, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chbevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_float* ab, lapack_int ldab, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zhbevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_double* ab, lapack_int ldab, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_chbevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int kd, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* q, lapack_int ldq, + float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + float* rwork, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_zhbevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int kd, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* q, lapack_int ldq, + double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + double* rwork, lapack_int* iwork, + lapack_int* ifail ); + +lapack_int LAPACKE_chbgst_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* bb, lapack_int ldbb, + lapack_complex_float* x, lapack_int ldx, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zhbgst_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + const lapack_complex_double* bb, + lapack_int ldbb, lapack_complex_double* x, + lapack_int ldx, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chbgv_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* bb, lapack_int ldbb, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zhbgv_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* bb, lapack_int ldbb, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chbgvd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* bb, lapack_int ldbb, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zhbgvd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* bb, lapack_int ldbb, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_chbgvx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int ka, + lapack_int kb, lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* bb, + lapack_int ldbb, lapack_complex_float* q, + lapack_int ldq, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_zhbgvx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int ka, + lapack_int kb, lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* bb, + lapack_int ldbb, lapack_complex_double* q, + lapack_int ldq, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_chbtrd_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_float* ab, lapack_int ldab, + float* d, float* e, lapack_complex_float* q, + lapack_int ldq, lapack_complex_float* work ); +lapack_int LAPACKE_zhbtrd_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int kd, + lapack_complex_double* ab, lapack_int ldab, + double* d, double* e, lapack_complex_double* q, + lapack_int ldq, lapack_complex_double* work ); + +lapack_int LAPACKE_checon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work ); +lapack_int LAPACKE_zhecon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond, lapack_complex_double* work ); + +lapack_int LAPACKE_cheequb_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax, + lapack_complex_float* work ); +lapack_int LAPACKE_zheequb_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax, + lapack_complex_double* work ); + +lapack_int LAPACKE_cheev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float* w, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zheev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double* w, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_cheevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, float* w, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_zheevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, double* w, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_cheevr_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_int* isuppz, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_zheevr_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_int* isuppz, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_cheevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_zheevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_chegst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhegst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_chegv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb, float* w, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zhegv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double* w, lapack_complex_double* work, + lapack_int lwork, double* rwork ); + +lapack_int LAPACKE_chegvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + float* w, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zhegvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double* w, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_chegvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_zhegvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_cherfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zherfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_cherfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const float* s, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zherfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* s, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chesv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zhesv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_chesvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + lapack_int lwork, float* rwork ); +lapack_int LAPACKE_zhesvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_chesvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zhesvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chetrd_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + float* d, float* e, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zhetrd_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + double* d, double* e, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_chetrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_float* work, + lapack_int lwork ); +lapack_int LAPACKE_zhetrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* work, + lapack_int lwork ); + +lapack_int LAPACKE_chetri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work ); +lapack_int LAPACKE_zhetri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work ); + +lapack_int LAPACKE_chetrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zhetrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_chfrk_work( int matrix_order, char transr, char uplo, + char trans, lapack_int n, lapack_int k, + float alpha, const lapack_complex_float* a, + lapack_int lda, float beta, + lapack_complex_float* c ); +lapack_int LAPACKE_zhfrk_work( int matrix_order, char transr, char uplo, + char trans, lapack_int n, lapack_int k, + double alpha, const lapack_complex_double* a, + lapack_int lda, double beta, + lapack_complex_double* c ); + +lapack_int LAPACKE_shgeqz_work( int matrix_order, char job, char compq, + char compz, lapack_int n, lapack_int ilo, + lapack_int ihi, float* h, lapack_int ldh, + float* t, lapack_int ldt, float* alphar, + float* alphai, float* beta, float* q, + lapack_int ldq, float* z, lapack_int ldz, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dhgeqz_work( int matrix_order, char job, char compq, + char compz, lapack_int n, lapack_int ilo, + lapack_int ihi, double* h, lapack_int ldh, + double* t, lapack_int ldt, double* alphar, + double* alphai, double* beta, double* q, + lapack_int ldq, double* z, lapack_int ldz, + double* work, lapack_int lwork ); +lapack_int LAPACKE_chgeqz_work( int matrix_order, char job, char compq, + char compz, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_float* h, + lapack_int ldh, lapack_complex_float* t, + lapack_int ldt, lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, lapack_int lwork, + float* rwork ); +lapack_int LAPACKE_zhgeqz_work( int matrix_order, char job, char compq, + char compz, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_double* h, + lapack_int ldh, lapack_complex_double* t, + lapack_int ldt, lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_chpcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work ); +lapack_int LAPACKE_zhpcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_int* ipiv, double anorm, + double* rcond, lapack_complex_double* work ); + +lapack_int LAPACKE_chpev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_float* ap, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zhpev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_double* ap, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_chpevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_float* ap, + float* w, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zhpevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_complex_double* ap, + double* w, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_chpevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_float* ap, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_zhpevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, + lapack_complex_double* ap, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_chpgst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_float* ap, + const lapack_complex_float* bp ); +lapack_int LAPACKE_zhpgst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, lapack_complex_double* ap, + const lapack_complex_double* bp ); + +lapack_int LAPACKE_chpgv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_float* ap, + lapack_complex_float* bp, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zhpgv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_double* ap, + lapack_complex_double* bp, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_chpgvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_float* ap, + lapack_complex_float* bp, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_zhpgvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, + lapack_complex_double* ap, + lapack_complex_double* bp, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int lrwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_chpgvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_float* ap, + lapack_complex_float* bp, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_zhpgvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, + lapack_complex_double* ap, + lapack_complex_double* bp, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_chprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zhprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_chpsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhpsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_chpsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + lapack_complex_float* afp, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zhpsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + lapack_complex_double* afp, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_chptrd_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, float* d, float* e, + lapack_complex_float* tau ); +lapack_int LAPACKE_zhptrd_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, double* d, double* e, + lapack_complex_double* tau ); + +lapack_int LAPACKE_chptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, lapack_int* ipiv ); +lapack_int LAPACKE_zhptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, lapack_int* ipiv ); + +lapack_int LAPACKE_chptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, + const lapack_int* ipiv, + lapack_complex_float* work ); +lapack_int LAPACKE_zhptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, + const lapack_int* ipiv, + lapack_complex_double* work ); + +lapack_int LAPACKE_chptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zhptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_shsein_work( int matrix_order, char job, char eigsrc, + char initv, lapack_logical* select, + lapack_int n, const float* h, lapack_int ldh, + float* wr, const float* wi, float* vl, + lapack_int ldvl, float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, float* work, + lapack_int* ifaill, lapack_int* ifailr ); +lapack_int LAPACKE_dhsein_work( int matrix_order, char job, char eigsrc, + char initv, lapack_logical* select, + lapack_int n, const double* h, lapack_int ldh, + double* wr, const double* wi, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, double* work, + lapack_int* ifaill, lapack_int* ifailr ); +lapack_int LAPACKE_chsein_work( int matrix_order, char job, char eigsrc, + char initv, const lapack_logical* select, + lapack_int n, const lapack_complex_float* h, + lapack_int ldh, lapack_complex_float* w, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_float* work, float* rwork, + lapack_int* ifaill, lapack_int* ifailr ); +lapack_int LAPACKE_zhsein_work( int matrix_order, char job, char eigsrc, + char initv, const lapack_logical* select, + lapack_int n, const lapack_complex_double* h, + lapack_int ldh, lapack_complex_double* w, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_double* work, double* rwork, + lapack_int* ifaill, lapack_int* ifailr ); + +lapack_int LAPACKE_shseqr_work( int matrix_order, char job, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + float* h, lapack_int ldh, float* wr, float* wi, + float* z, lapack_int ldz, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dhseqr_work( int matrix_order, char job, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + double* h, lapack_int ldh, double* wr, + double* wi, double* z, lapack_int ldz, + double* work, lapack_int lwork ); +lapack_int LAPACKE_chseqr_work( int matrix_order, char job, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_float* h, lapack_int ldh, + lapack_complex_float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zhseqr_work( int matrix_order, char job, char compz, + lapack_int n, lapack_int ilo, lapack_int ihi, + lapack_complex_double* h, lapack_int ldh, + lapack_complex_double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_clacgv_work( lapack_int n, lapack_complex_float* x, + lapack_int incx ); +lapack_int LAPACKE_zlacgv_work( lapack_int n, lapack_complex_double* x, + lapack_int incx ); + +lapack_int LAPACKE_slacpy_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, const float* a, lapack_int lda, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dlacpy_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, const double* a, lapack_int lda, + double* b, lapack_int ldb ); +lapack_int LAPACKE_clacpy_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zlacpy_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_zlag2c_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_float* sa, lapack_int ldsa ); + +lapack_int LAPACKE_slag2d_work( int matrix_order, lapack_int m, lapack_int n, + const float* sa, lapack_int ldsa, double* a, + lapack_int lda ); + +lapack_int LAPACKE_dlag2s_work( int matrix_order, lapack_int m, lapack_int n, + const double* a, lapack_int lda, float* sa, + lapack_int ldsa ); + +lapack_int LAPACKE_clag2z_work( int matrix_order, lapack_int m, lapack_int n, + const lapack_complex_float* sa, lapack_int ldsa, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_slagge_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* d, + float* a, lapack_int lda, lapack_int* iseed, + float* work ); +lapack_int LAPACKE_dlagge_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* d, + double* a, lapack_int lda, lapack_int* iseed, + double* work ); +lapack_int LAPACKE_clagge_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const float* d, + lapack_complex_float* a, lapack_int lda, + lapack_int* iseed, lapack_complex_float* work ); +lapack_int LAPACKE_zlagge_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int kl, lapack_int ku, const double* d, + lapack_complex_double* a, lapack_int lda, + lapack_int* iseed, + lapack_complex_double* work ); + +lapack_int LAPACKE_claghe_work( int matrix_order, lapack_int n, lapack_int k, + const float* d, lapack_complex_float* a, + lapack_int lda, lapack_int* iseed, + lapack_complex_float* work ); +lapack_int LAPACKE_zlaghe_work( int matrix_order, lapack_int n, lapack_int k, + const double* d, lapack_complex_double* a, + lapack_int lda, lapack_int* iseed, + lapack_complex_double* work ); + +lapack_int LAPACKE_slagsy_work( int matrix_order, lapack_int n, lapack_int k, + const float* d, float* a, lapack_int lda, + lapack_int* iseed, float* work ); +lapack_int LAPACKE_dlagsy_work( int matrix_order, lapack_int n, lapack_int k, + const double* d, double* a, lapack_int lda, + lapack_int* iseed, double* work ); +lapack_int LAPACKE_clagsy_work( int matrix_order, lapack_int n, lapack_int k, + const float* d, lapack_complex_float* a, + lapack_int lda, lapack_int* iseed, + lapack_complex_float* work ); +lapack_int LAPACKE_zlagsy_work( int matrix_order, lapack_int n, lapack_int k, + const double* d, lapack_complex_double* a, + lapack_int lda, lapack_int* iseed, + lapack_complex_double* work ); + +lapack_int LAPACKE_slapmr_work( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, float* x, + lapack_int ldx, lapack_int* k ); +lapack_int LAPACKE_dlapmr_work( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, double* x, + lapack_int ldx, lapack_int* k ); +lapack_int LAPACKE_clapmr_work( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, + lapack_complex_float* x, lapack_int ldx, + lapack_int* k ); +lapack_int LAPACKE_zlapmr_work( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, + lapack_complex_double* x, lapack_int ldx, + lapack_int* k ); + +lapack_int LAPACKE_slartgp_work( float f, float g, float* cs, float* sn, + float* r ); +lapack_int LAPACKE_dlartgp_work( double f, double g, double* cs, double* sn, + double* r ); + +lapack_int LAPACKE_slartgs_work( float x, float y, float sigma, float* cs, + float* sn ); +lapack_int LAPACKE_dlartgs_work( double x, double y, double sigma, double* cs, + double* sn ); + +float LAPACKE_slapy2_work( float x, float y ); +double LAPACKE_dlapy2_work( double x, double y ); + +float LAPACKE_slapy3_work( float x, float y, float z ); +double LAPACKE_dlapy3_work( double x, double y, double z ); + +float LAPACKE_slamch_work( char cmach ); +double LAPACKE_dlamch_work( char cmach ); + +float LAPACKE_slange_work( int matrix_order, char norm, lapack_int m, + lapack_int n, const float* a, lapack_int lda, + float* work ); +double LAPACKE_dlange_work( int matrix_order, char norm, lapack_int m, + lapack_int n, const double* a, lapack_int lda, + double* work ); +float LAPACKE_clange_work( int matrix_order, char norm, lapack_int m, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, float* work ); +double LAPACKE_zlange_work( int matrix_order, char norm, lapack_int m, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, double* work ); + +float LAPACKE_clanhe_work( int matrix_order, char norm, char uplo, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, float* work ); +double LAPACKE_zlanhe_work( int matrix_order, char norm, char uplo, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, double* work ); + +float LAPACKE_slansy_work( int matrix_order, char norm, char uplo, + lapack_int n, const float* a, lapack_int lda, + float* work ); +double LAPACKE_dlansy_work( int matrix_order, char norm, char uplo, + lapack_int n, const double* a, lapack_int lda, + double* work ); +float LAPACKE_clansy_work( int matrix_order, char norm, char uplo, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, float* work ); +double LAPACKE_zlansy_work( int matrix_order, char norm, char uplo, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, double* work ); + +float LAPACKE_slantr_work( int matrix_order, char norm, char uplo, + char diag, lapack_int m, lapack_int n, const float* a, + lapack_int lda, float* work ); +double LAPACKE_dlantr_work( int matrix_order, char norm, char uplo, + char diag, lapack_int m, lapack_int n, + const double* a, lapack_int lda, double* work ); +float LAPACKE_clantr_work( int matrix_order, char norm, char uplo, + char diag, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* work ); +double LAPACKE_zlantr_work( int matrix_order, char norm, char uplo, + char diag, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* work ); + +lapack_int LAPACKE_slarfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, const float* v, + lapack_int ldv, const float* t, lapack_int ldt, + float* c, lapack_int ldc, float* work, + lapack_int ldwork ); +lapack_int LAPACKE_dlarfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, const double* v, + lapack_int ldv, const double* t, lapack_int ldt, + double* c, lapack_int ldc, double* work, + lapack_int ldwork ); +lapack_int LAPACKE_clarfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int ldwork ); +lapack_int LAPACKE_zlarfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, + lapack_int ldwork ); + +lapack_int LAPACKE_slarfg_work( lapack_int n, float* alpha, float* x, + lapack_int incx, float* tau ); +lapack_int LAPACKE_dlarfg_work( lapack_int n, double* alpha, double* x, + lapack_int incx, double* tau ); +lapack_int LAPACKE_clarfg_work( lapack_int n, lapack_complex_float* alpha, + lapack_complex_float* x, lapack_int incx, + lapack_complex_float* tau ); +lapack_int LAPACKE_zlarfg_work( lapack_int n, lapack_complex_double* alpha, + lapack_complex_double* x, lapack_int incx, + lapack_complex_double* tau ); + +lapack_int LAPACKE_slarft_work( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, const float* v, + lapack_int ldv, const float* tau, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dlarft_work( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, const double* v, + lapack_int ldv, const double* tau, double* t, + lapack_int ldt ); +lapack_int LAPACKE_clarft_work( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* tau, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zlarft_work( int matrix_order, char direct, char storev, + lapack_int n, lapack_int k, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* tau, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_slarfx_work( int matrix_order, char side, lapack_int m, + lapack_int n, const float* v, float tau, + float* c, lapack_int ldc, float* work ); +lapack_int LAPACKE_dlarfx_work( int matrix_order, char side, lapack_int m, + lapack_int n, const double* v, double tau, + double* c, lapack_int ldc, double* work ); +lapack_int LAPACKE_clarfx_work( int matrix_order, char side, lapack_int m, + lapack_int n, const lapack_complex_float* v, + lapack_complex_float tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work ); +lapack_int LAPACKE_zlarfx_work( int matrix_order, char side, lapack_int m, + lapack_int n, const lapack_complex_double* v, + lapack_complex_double tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work ); + +lapack_int LAPACKE_slarnv_work( lapack_int idist, lapack_int* iseed, + lapack_int n, float* x ); +lapack_int LAPACKE_dlarnv_work( lapack_int idist, lapack_int* iseed, + lapack_int n, double* x ); +lapack_int LAPACKE_clarnv_work( lapack_int idist, lapack_int* iseed, + lapack_int n, lapack_complex_float* x ); +lapack_int LAPACKE_zlarnv_work( lapack_int idist, lapack_int* iseed, + lapack_int n, lapack_complex_double* x ); + +lapack_int LAPACKE_slaset_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, float alpha, float beta, float* a, + lapack_int lda ); +lapack_int LAPACKE_dlaset_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, double alpha, double beta, + double* a, lapack_int lda ); +lapack_int LAPACKE_claset_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, lapack_complex_float alpha, + lapack_complex_float beta, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zlaset_work( int matrix_order, char uplo, lapack_int m, + lapack_int n, lapack_complex_double alpha, + lapack_complex_double beta, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_slasrt_work( char id, lapack_int n, float* d ); +lapack_int LAPACKE_dlasrt_work( char id, lapack_int n, double* d ); + +lapack_int LAPACKE_slaswp_work( int matrix_order, lapack_int n, float* a, + lapack_int lda, lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); +lapack_int LAPACKE_dlaswp_work( int matrix_order, lapack_int n, double* a, + lapack_int lda, lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); +lapack_int LAPACKE_claswp_work( int matrix_order, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); +lapack_int LAPACKE_zlaswp_work( int matrix_order, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int k1, lapack_int k2, + const lapack_int* ipiv, lapack_int incx ); + +lapack_int LAPACKE_slatms_work( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, + float* d, lapack_int mode, float cond, + float dmax, lapack_int kl, lapack_int ku, + char pack, float* a, lapack_int lda, + float* work ); +lapack_int LAPACKE_dlatms_work( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, + double* d, lapack_int mode, double cond, + double dmax, lapack_int kl, lapack_int ku, + char pack, double* a, lapack_int lda, + double* work ); +lapack_int LAPACKE_clatms_work( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, + float* d, lapack_int mode, float cond, + float dmax, lapack_int kl, lapack_int ku, + char pack, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* work ); +lapack_int LAPACKE_zlatms_work( int matrix_order, lapack_int m, lapack_int n, + char dist, lapack_int* iseed, char sym, + double* d, lapack_int mode, double cond, + double dmax, lapack_int kl, lapack_int ku, + char pack, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* work ); + +lapack_int LAPACKE_slauum_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda ); +lapack_int LAPACKE_dlauum_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda ); +lapack_int LAPACKE_clauum_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zlauum_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_sopgtr_work( int matrix_order, char uplo, lapack_int n, + const float* ap, const float* tau, float* q, + lapack_int ldq, float* work ); +lapack_int LAPACKE_dopgtr_work( int matrix_order, char uplo, lapack_int n, + const double* ap, const double* tau, double* q, + lapack_int ldq, double* work ); + +lapack_int LAPACKE_sopmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const float* ap, const float* tau, float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_dopmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const double* ap, const double* tau, double* c, + lapack_int ldc, double* work ); + +lapack_int LAPACKE_sorgbr_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, float* a, + lapack_int lda, const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorgbr_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, double* a, + lapack_int lda, const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorghr_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, float* a, lapack_int lda, + const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorghr_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, double* a, lapack_int lda, + const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorglq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorglq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorgql_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorgql_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorgqr_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorgqr_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorgrq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, float* a, lapack_int lda, + const float* tau, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dorgrq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, double* a, lapack_int lda, + const double* tau, double* work, + lapack_int lwork ); + +lapack_int LAPACKE_sorgtr_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, const float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dorgtr_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, const double* tau, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormbr_work( int matrix_order, char vect, char side, + char trans, lapack_int m, lapack_int n, + lapack_int k, const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormbr_work( int matrix_order, char vect, char side, + char trans, lapack_int m, lapack_int n, + lapack_int k, const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormhr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormhr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormlq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormlq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormql_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormql_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormqr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormqr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormrq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormrq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormrz_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormrz_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_sormtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const float* a, lapack_int lda, + const float* tau, float* c, lapack_int ldc, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dormtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const double* a, lapack_int lda, + const double* tau, double* c, lapack_int ldc, + double* work, lapack_int lwork ); + +lapack_int LAPACKE_spbcon_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const float* ab, lapack_int ldab, + float anorm, float* rcond, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dpbcon_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const double* ab, + lapack_int ldab, double anorm, double* rcond, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cpbcon_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_float* ab, + lapack_int ldab, float anorm, float* rcond, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zpbcon_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_double* ab, + lapack_int ldab, double anorm, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_spbequ_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const float* ab, lapack_int ldab, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_dpbequ_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const double* ab, + lapack_int ldab, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cpbequ_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_float* ab, + lapack_int ldab, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_zpbequ_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, const lapack_complex_double* ab, + lapack_int ldab, double* s, double* scond, + double* amax ); + +lapack_int LAPACKE_spbrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const float* ab, + lapack_int ldab, const float* afb, + lapack_int ldafb, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dpbrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const double* ab, lapack_int ldab, + const double* afb, lapack_int ldafb, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cpbrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + const lapack_complex_float* afb, + lapack_int ldafb, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zpbrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, + const lapack_complex_double* afb, + lapack_int ldafb, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_spbstf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kb, float* bb, lapack_int ldbb ); +lapack_int LAPACKE_dpbstf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kb, double* bb, lapack_int ldbb ); +lapack_int LAPACKE_cpbstf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kb, lapack_complex_float* bb, + lapack_int ldbb ); +lapack_int LAPACKE_zpbstf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kb, lapack_complex_double* bb, + lapack_int ldbb ); + +lapack_int LAPACKE_spbsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, float* ab, + lapack_int ldab, float* b, lapack_int ldb ); +lapack_int LAPACKE_dpbsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, double* ab, + lapack_int ldab, double* b, lapack_int ldb ); +lapack_int LAPACKE_cpbsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpbsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spbsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int kd, lapack_int nrhs, + float* ab, lapack_int ldab, float* afb, + lapack_int ldafb, char* equed, float* s, + float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dpbsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int kd, lapack_int nrhs, + double* ab, lapack_int ldab, double* afb, + lapack_int ldafb, char* equed, double* s, + double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, double* work, lapack_int* iwork ); +lapack_int LAPACKE_cpbsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int kd, lapack_int nrhs, + lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* afb, lapack_int ldafb, + char* equed, float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zpbsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int kd, lapack_int nrhs, + lapack_complex_double* ab, lapack_int ldab, + lapack_complex_double* afb, lapack_int ldafb, + char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_spbtrf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, float* ab, lapack_int ldab ); +lapack_int LAPACKE_dpbtrf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, double* ab, lapack_int ldab ); +lapack_int LAPACKE_cpbtrf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_complex_float* ab, + lapack_int ldab ); +lapack_int LAPACKE_zpbtrf_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_complex_double* ab, + lapack_int ldab ); + +lapack_int LAPACKE_spbtrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, const float* ab, + lapack_int ldab, float* b, lapack_int ldb ); +lapack_int LAPACKE_dpbtrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const double* ab, lapack_int ldab, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cpbtrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_float* ab, lapack_int ldab, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpbtrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int kd, lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_spftrf_work( int matrix_order, char transr, char uplo, + lapack_int n, float* a ); +lapack_int LAPACKE_dpftrf_work( int matrix_order, char transr, char uplo, + lapack_int n, double* a ); +lapack_int LAPACKE_cpftrf_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_float* a ); +lapack_int LAPACKE_zpftrf_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_double* a ); + +lapack_int LAPACKE_spftri_work( int matrix_order, char transr, char uplo, + lapack_int n, float* a ); +lapack_int LAPACKE_dpftri_work( int matrix_order, char transr, char uplo, + lapack_int n, double* a ); +lapack_int LAPACKE_cpftri_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_float* a ); +lapack_int LAPACKE_zpftri_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_complex_double* a ); + +lapack_int LAPACKE_spftrs_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, const float* a, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dpftrs_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, const double* a, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cpftrs_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpftrs_work( int matrix_order, char transr, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spocon_work( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float anorm, + float* rcond, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dpocon_work( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double anorm, + double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cpocon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float anorm, float* rcond, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zpocon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double anorm, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_spoequ_work( int matrix_order, lapack_int n, const float* a, + lapack_int lda, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dpoequ_work( int matrix_order, lapack_int n, const double* a, + lapack_int lda, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cpoequ_work( int matrix_order, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zpoequ_work( int matrix_order, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_spoequb_work( int matrix_order, lapack_int n, const float* a, + lapack_int lda, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dpoequb_work( int matrix_order, lapack_int n, + const double* a, lapack_int lda, double* s, + double* scond, double* amax ); +lapack_int LAPACKE_cpoequb_work( int matrix_order, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax ); +lapack_int LAPACKE_zpoequb_work( int matrix_order, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax ); + +lapack_int LAPACKE_sporfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dporfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cporfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zporfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sporfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, + lapack_int ldaf, const float* s, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dporfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const double* s, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cporfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, + lapack_int ldaf, const float* s, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zporfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, + lapack_int ldaf, const double* s, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); +lapack_int LAPACKE_dsposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + double* b, lapack_int ldb, double* x, + lapack_int ldx, double* work, float* swork, + lapack_int* iter ); +lapack_int LAPACKE_zcposv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, lapack_complex_double* work, + lapack_complex_float* swork, double* rwork, + lapack_int* iter ); + +lapack_int LAPACKE_sposvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + char* equed, float* s, float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dposvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + char* equed, double* s, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cposvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + char* equed, float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zposvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sposvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + char* equed, float* s, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dposvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + char* equed, double* s, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cposvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + char* equed, float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zposvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_spotrf_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda ); +lapack_int LAPACKE_dpotrf_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda ); +lapack_int LAPACKE_cpotrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zpotrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_spotri_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda ); +lapack_int LAPACKE_dpotri_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda ); +lapack_int LAPACKE_cpotri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zpotri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_spotrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dpotrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, double* b, lapack_int ldb ); +lapack_int LAPACKE_cpotrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zpotrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sppcon_work( int matrix_order, char uplo, lapack_int n, + const float* ap, float anorm, float* rcond, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dppcon_work( int matrix_order, char uplo, lapack_int n, + const double* ap, double anorm, double* rcond, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cppcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, float anorm, + float* rcond, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zppcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, double anorm, + double* rcond, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_sppequ_work( int matrix_order, char uplo, lapack_int n, + const float* ap, float* s, float* scond, + float* amax ); +lapack_int LAPACKE_dppequ_work( int matrix_order, char uplo, lapack_int n, + const double* ap, double* s, double* scond, + double* amax ); +lapack_int LAPACKE_cppequ_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, float* s, + float* scond, float* amax ); +lapack_int LAPACKE_zppequ_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, double* s, + double* scond, double* amax ); + +lapack_int LAPACKE_spprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, + const float* afp, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dpprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, + const double* afp, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cpprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zpprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sppsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* ap, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dppsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* ap, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cppsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zppsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sppsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* ap, + float* afp, char* equed, float* s, float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dppsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* ap, + double* afp, char* equed, double* s, double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cppsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* ap, + lapack_complex_float* afp, char* equed, + float* s, lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zppsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* ap, + lapack_complex_double* afp, char* equed, + double* s, lapack_complex_double* b, + lapack_int ldb, lapack_complex_double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_spptrf_work( int matrix_order, char uplo, lapack_int n, + float* ap ); +lapack_int LAPACKE_dpptrf_work( int matrix_order, char uplo, lapack_int n, + double* ap ); +lapack_int LAPACKE_cpptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap ); +lapack_int LAPACKE_zpptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap ); + +lapack_int LAPACKE_spptri_work( int matrix_order, char uplo, lapack_int n, + float* ap ); +lapack_int LAPACKE_dpptri_work( int matrix_order, char uplo, lapack_int n, + double* ap ); +lapack_int LAPACKE_cpptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap ); +lapack_int LAPACKE_zpptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap ); + +lapack_int LAPACKE_spptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dpptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cpptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_spstrf_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, lapack_int* piv, + lapack_int* rank, float tol, float* work ); +lapack_int LAPACKE_dpstrf_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, lapack_int* piv, + lapack_int* rank, double tol, double* work ); +lapack_int LAPACKE_cpstrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* piv, lapack_int* rank, float tol, + float* work ); +lapack_int LAPACKE_zpstrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* piv, lapack_int* rank, double tol, + double* work ); + +lapack_int LAPACKE_sptcon_work( lapack_int n, const float* d, const float* e, + float anorm, float* rcond, float* work ); +lapack_int LAPACKE_dptcon_work( lapack_int n, const double* d, const double* e, + double anorm, double* rcond, double* work ); +lapack_int LAPACKE_cptcon_work( lapack_int n, const float* d, + const lapack_complex_float* e, float anorm, + float* rcond, float* work ); +lapack_int LAPACKE_zptcon_work( lapack_int n, const double* d, + const lapack_complex_double* e, double anorm, + double* rcond, double* work ); + +lapack_int LAPACKE_spteqr_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, float* z, lapack_int ldz, + float* work ); +lapack_int LAPACKE_dpteqr_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz, + double* work ); +lapack_int LAPACKE_cpteqr_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, lapack_complex_float* z, + lapack_int ldz, float* work ); +lapack_int LAPACKE_zpteqr_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz, double* work ); + +lapack_int LAPACKE_sptrfs_work( int matrix_order, lapack_int n, lapack_int nrhs, + const float* d, const float* e, const float* df, + const float* ef, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* ferr, + float* berr, float* work ); +lapack_int LAPACKE_dptrfs_work( int matrix_order, lapack_int n, lapack_int nrhs, + const double* d, const double* e, + const double* df, const double* ef, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr, + double* work ); +lapack_int LAPACKE_cptrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, const float* df, + const lapack_complex_float* ef, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zptrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, + const double* df, + const lapack_complex_double* ef, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sptsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + float* d, float* e, float* b, lapack_int ldb ); +lapack_int LAPACKE_dptsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + double* d, double* e, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cptsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + float* d, lapack_complex_float* e, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zptsv_work( int matrix_order, lapack_int n, lapack_int nrhs, + double* d, lapack_complex_double* e, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sptsvx_work( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const float* d, const float* e, + float* df, float* ef, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work ); +lapack_int LAPACKE_dptsvx_work( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const double* d, + const double* e, double* df, double* ef, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* ferr, + double* berr, double* work ); +lapack_int LAPACKE_cptsvx_work( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, float* df, + lapack_complex_float* ef, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zptsvx_work( int matrix_order, char fact, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, double* df, + lapack_complex_double* ef, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_spttrf_work( lapack_int n, float* d, float* e ); +lapack_int LAPACKE_dpttrf_work( lapack_int n, double* d, double* e ); +lapack_int LAPACKE_cpttrf_work( lapack_int n, float* d, + lapack_complex_float* e ); +lapack_int LAPACKE_zpttrf_work( lapack_int n, double* d, + lapack_complex_double* e ); + +lapack_int LAPACKE_spttrs_work( int matrix_order, lapack_int n, lapack_int nrhs, + const float* d, const float* e, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dpttrs_work( int matrix_order, lapack_int n, lapack_int nrhs, + const double* d, const double* e, double* b, + lapack_int ldb ); +lapack_int LAPACKE_cpttrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* d, + const lapack_complex_float* e, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zpttrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* d, + const lapack_complex_double* e, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_ssbev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, float* ab, + lapack_int ldab, float* w, float* z, + lapack_int ldz, float* work ); +lapack_int LAPACKE_dsbev_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, double* ab, + lapack_int ldab, double* w, double* z, + lapack_int ldz, double* work ); + +lapack_int LAPACKE_ssbevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, float* ab, + lapack_int ldab, float* w, float* z, + lapack_int ldz, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dsbevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int kd, double* ab, + lapack_int ldab, double* w, double* z, + lapack_int ldz, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_ssbevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int kd, + float* ab, lapack_int ldab, float* q, + lapack_int ldq, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, + lapack_int ldz, float* work, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_dsbevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int kd, + double* ab, lapack_int ldab, double* q, + lapack_int ldq, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, double* z, + lapack_int ldz, double* work, lapack_int* iwork, + lapack_int* ifail ); + +lapack_int LAPACKE_ssbgst_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + float* ab, lapack_int ldab, const float* bb, + lapack_int ldbb, float* x, lapack_int ldx, + float* work ); +lapack_int LAPACKE_dsbgst_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + double* ab, lapack_int ldab, const double* bb, + lapack_int ldbb, double* x, lapack_int ldx, + double* work ); + +lapack_int LAPACKE_ssbgv_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + float* ab, lapack_int ldab, float* bb, + lapack_int ldbb, float* w, float* z, + lapack_int ldz, float* work ); +lapack_int LAPACKE_dsbgv_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + double* ab, lapack_int ldab, double* bb, + lapack_int ldbb, double* w, double* z, + lapack_int ldz, double* work ); + +lapack_int LAPACKE_ssbgvd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + float* ab, lapack_int ldab, float* bb, + lapack_int ldbb, float* w, float* z, + lapack_int ldz, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dsbgvd_work( int matrix_order, char jobz, char uplo, + lapack_int n, lapack_int ka, lapack_int kb, + double* ab, lapack_int ldab, double* bb, + lapack_int ldbb, double* w, double* z, + lapack_int ldz, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_ssbgvx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int ka, + lapack_int kb, float* ab, lapack_int ldab, + float* bb, lapack_int ldbb, float* q, + lapack_int ldq, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, + lapack_int ldz, float* work, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_dsbgvx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, lapack_int ka, + lapack_int kb, double* ab, lapack_int ldab, + double* bb, lapack_int ldbb, double* q, + lapack_int ldq, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, double* z, + lapack_int ldz, double* work, lapack_int* iwork, + lapack_int* ifail ); + +lapack_int LAPACKE_ssbtrd_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int kd, float* ab, + lapack_int ldab, float* d, float* e, float* q, + lapack_int ldq, float* work ); +lapack_int LAPACKE_dsbtrd_work( int matrix_order, char vect, char uplo, + lapack_int n, lapack_int kd, double* ab, + lapack_int ldab, double* d, double* e, + double* q, lapack_int ldq, double* work ); + +lapack_int LAPACKE_ssfrk_work( int matrix_order, char transr, char uplo, + char trans, lapack_int n, lapack_int k, + float alpha, const float* a, lapack_int lda, + float beta, float* c ); +lapack_int LAPACKE_dsfrk_work( int matrix_order, char transr, char uplo, + char trans, lapack_int n, lapack_int k, + double alpha, const double* a, lapack_int lda, + double beta, double* c ); + +lapack_int LAPACKE_sspcon_work( int matrix_order, char uplo, lapack_int n, + const float* ap, const lapack_int* ipiv, + float anorm, float* rcond, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dspcon_work( int matrix_order, char uplo, lapack_int n, + const double* ap, const lapack_int* ipiv, + double anorm, double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_cspcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work ); +lapack_int LAPACKE_zspcon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_int* ipiv, double anorm, + double* rcond, lapack_complex_double* work ); + +lapack_int LAPACKE_sspev_work( int matrix_order, char jobz, char uplo, + lapack_int n, float* ap, float* w, float* z, + lapack_int ldz, float* work ); +lapack_int LAPACKE_dspev_work( int matrix_order, char jobz, char uplo, + lapack_int n, double* ap, double* w, double* z, + lapack_int ldz, double* work ); + +lapack_int LAPACKE_sspevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, float* ap, float* w, float* z, + lapack_int ldz, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dspevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, double* ap, double* w, double* z, + lapack_int ldz, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_sspevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, float* ap, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, float* z, + lapack_int ldz, float* work, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_dspevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, double* ap, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + double* z, lapack_int ldz, double* work, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_sspgst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, float* ap, const float* bp ); +lapack_int LAPACKE_dspgst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, double* ap, const double* bp ); + +lapack_int LAPACKE_sspgv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* ap, float* bp, + float* w, float* z, lapack_int ldz, + float* work ); +lapack_int LAPACKE_dspgv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* ap, double* bp, + double* w, double* z, lapack_int ldz, + double* work ); + +lapack_int LAPACKE_sspgvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* ap, float* bp, + float* w, float* z, lapack_int ldz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_dspgvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* ap, double* bp, + double* w, double* z, lapack_int ldz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_sspgvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, float* ap, + float* bp, float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, float* z, lapack_int ldz, float* work, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_dspgvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, double* ap, + double* bp, double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, double* z, lapack_int ldz, + double* work, lapack_int* iwork, + lapack_int* ifail ); + +lapack_int LAPACKE_ssprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, + const float* afp, const lapack_int* ipiv, + const float* b, lapack_int ldb, float* x, + lapack_int ldx, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dsprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, + const double* afp, const lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_csprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zsprfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_sspsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* ap, lapack_int* ipiv, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dspsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* ap, lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_cspsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* ap, + lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zspsv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* ap, + lapack_int* ipiv, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_sspsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, const float* ap, + float* afp, lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dspsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, const double* ap, + double* afp, lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_cspsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + lapack_complex_float* afp, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zspsvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + lapack_complex_double* afp, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_ssptrd_work( int matrix_order, char uplo, lapack_int n, + float* ap, float* d, float* e, float* tau ); +lapack_int LAPACKE_dsptrd_work( int matrix_order, char uplo, lapack_int n, + double* ap, double* d, double* e, double* tau ); + +lapack_int LAPACKE_ssptrf_work( int matrix_order, char uplo, lapack_int n, + float* ap, lapack_int* ipiv ); +lapack_int LAPACKE_dsptrf_work( int matrix_order, char uplo, lapack_int n, + double* ap, lapack_int* ipiv ); +lapack_int LAPACKE_csptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, lapack_int* ipiv ); +lapack_int LAPACKE_zsptrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, lapack_int* ipiv ); + +lapack_int LAPACKE_ssptri_work( int matrix_order, char uplo, lapack_int n, + float* ap, const lapack_int* ipiv, + float* work ); +lapack_int LAPACKE_dsptri_work( int matrix_order, char uplo, lapack_int n, + double* ap, const lapack_int* ipiv, + double* work ); +lapack_int LAPACKE_csptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* ap, + const lapack_int* ipiv, + lapack_complex_float* work ); +lapack_int LAPACKE_zsptri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* ap, + const lapack_int* ipiv, + lapack_complex_double* work ); + +lapack_int LAPACKE_ssptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* ap, + const lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dsptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* ap, + const lapack_int* ipiv, double* b, + lapack_int ldb ); +lapack_int LAPACKE_csptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_zsptrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_sstebz_work( char range, char order, lapack_int n, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, const float* d, const float* e, + lapack_int* m, lapack_int* nsplit, float* w, + lapack_int* iblock, lapack_int* isplit, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dstebz_work( char range, char order, lapack_int n, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, const double* d, const double* e, + lapack_int* m, lapack_int* nsplit, double* w, + lapack_int* iblock, lapack_int* isplit, + double* work, lapack_int* iwork ); + +lapack_int LAPACKE_sstedc_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, float* z, lapack_int ldz, + float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dstedc_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_cstedc_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, lapack_complex_float* z, + lapack_int ldz, lapack_complex_float* work, + lapack_int lwork, float* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zstedc_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz, lapack_complex_double* work, + lapack_int lwork, double* rwork, + lapack_int lrwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_sstegr_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, float* z, + lapack_int ldz, lapack_int* isuppz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_dstegr_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + double* z, lapack_int ldz, lapack_int* isuppz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_cstegr_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_int* isuppz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zstegr_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int* isuppz, double* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_sstein_work( int matrix_order, lapack_int n, const float* d, + const float* e, lapack_int m, const float* w, + const lapack_int* iblock, + const lapack_int* isplit, float* z, + lapack_int ldz, float* work, lapack_int* iwork, + lapack_int* ifailv ); +lapack_int LAPACKE_dstein_work( int matrix_order, lapack_int n, const double* d, + const double* e, lapack_int m, const double* w, + const lapack_int* iblock, + const lapack_int* isplit, double* z, + lapack_int ldz, double* work, lapack_int* iwork, + lapack_int* ifailv ); +lapack_int LAPACKE_cstein_work( int matrix_order, lapack_int n, const float* d, + const float* e, lapack_int m, const float* w, + const lapack_int* iblock, + const lapack_int* isplit, + lapack_complex_float* z, lapack_int ldz, + float* work, lapack_int* iwork, + lapack_int* ifailv ); +lapack_int LAPACKE_zstein_work( int matrix_order, lapack_int n, const double* d, + const double* e, lapack_int m, const double* w, + const lapack_int* iblock, + const lapack_int* isplit, + lapack_complex_double* z, lapack_int ldz, + double* work, lapack_int* iwork, + lapack_int* ifailv ); + +lapack_int LAPACKE_sstemr_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + lapack_int* m, float* w, float* z, + lapack_int ldz, lapack_int nzc, + lapack_int* isuppz, lapack_logical* tryrac, + float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dstemr_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int nzc, + lapack_int* isuppz, lapack_logical* tryrac, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_cstemr_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + lapack_int* m, float* w, + lapack_complex_float* z, lapack_int ldz, + lapack_int nzc, lapack_int* isuppz, + lapack_logical* tryrac, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_zstemr_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + lapack_int* m, double* w, + lapack_complex_double* z, lapack_int ldz, + lapack_int nzc, lapack_int* isuppz, + lapack_logical* tryrac, double* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_ssteqr_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, float* z, lapack_int ldz, + float* work ); +lapack_int LAPACKE_dsteqr_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz, + double* work ); +lapack_int LAPACKE_csteqr_work( int matrix_order, char compz, lapack_int n, + float* d, float* e, lapack_complex_float* z, + lapack_int ldz, float* work ); +lapack_int LAPACKE_zsteqr_work( int matrix_order, char compz, lapack_int n, + double* d, double* e, lapack_complex_double* z, + lapack_int ldz, double* work ); + +lapack_int LAPACKE_ssterf_work( lapack_int n, float* d, float* e ); +lapack_int LAPACKE_dsterf_work( lapack_int n, double* d, double* e ); + +lapack_int LAPACKE_sstev_work( int matrix_order, char jobz, lapack_int n, + float* d, float* e, float* z, lapack_int ldz, + float* work ); +lapack_int LAPACKE_dstev_work( int matrix_order, char jobz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz, + double* work ); + +lapack_int LAPACKE_sstevd_work( int matrix_order, char jobz, lapack_int n, + float* d, float* e, float* z, lapack_int ldz, + float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dstevd_work( int matrix_order, char jobz, lapack_int n, + double* d, double* e, double* z, lapack_int ldz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_sstevr_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, float* z, + lapack_int ldz, lapack_int* isuppz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_dstevr_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + double* z, lapack_int ldz, lapack_int* isuppz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_sstevx_work( int matrix_order, char jobz, char range, + lapack_int n, float* d, float* e, float vl, + float vu, lapack_int il, lapack_int iu, + float abstol, lapack_int* m, float* w, float* z, + lapack_int ldz, float* work, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_dstevx_work( int matrix_order, char jobz, char range, + lapack_int n, double* d, double* e, double vl, + double vu, lapack_int il, lapack_int iu, + double abstol, lapack_int* m, double* w, + double* z, lapack_int ldz, double* work, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_ssycon_work( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, + float* rcond, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dsycon_work( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_csycon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, float anorm, + float* rcond, lapack_complex_float* work ); +lapack_int LAPACKE_zsycon_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, double anorm, + double* rcond, lapack_complex_double* work ); + +lapack_int LAPACKE_ssyequb_work( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float* s, + float* scond, float* amax, float* work ); +lapack_int LAPACKE_dsyequb_work( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double* s, + double* scond, double* amax, double* work ); +lapack_int LAPACKE_csyequb_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* s, float* scond, float* amax, + lapack_complex_float* work ); +lapack_int LAPACKE_zsyequb_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* s, double* scond, double* amax, + lapack_complex_double* work ); + +lapack_int LAPACKE_ssyev_work( int matrix_order, char jobz, char uplo, + lapack_int n, float* a, lapack_int lda, float* w, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dsyev_work( int matrix_order, char jobz, char uplo, + lapack_int n, double* a, lapack_int lda, + double* w, double* work, lapack_int lwork ); + +lapack_int LAPACKE_ssyevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, float* a, lapack_int lda, + float* w, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dsyevd_work( int matrix_order, char jobz, char uplo, + lapack_int n, double* a, lapack_int lda, + double* w, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_ssyevr_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, float* a, + lapack_int lda, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, + lapack_int ldz, lapack_int* isuppz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_dsyevr_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, double* a, + lapack_int lda, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, double* z, + lapack_int ldz, lapack_int* isuppz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_ssyevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, float* a, + lapack_int lda, float vl, float vu, + lapack_int il, lapack_int iu, float abstol, + lapack_int* m, float* w, float* z, + lapack_int ldz, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int* ifail ); +lapack_int LAPACKE_dsyevx_work( int matrix_order, char jobz, char range, + char uplo, lapack_int n, double* a, + lapack_int lda, double vl, double vu, + lapack_int il, lapack_int iu, double abstol, + lapack_int* m, double* w, double* z, + lapack_int ldz, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_ssygst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, float* a, lapack_int lda, + const float* b, lapack_int ldb ); +lapack_int LAPACKE_dsygst_work( int matrix_order, lapack_int itype, char uplo, + lapack_int n, double* a, lapack_int lda, + const double* b, lapack_int ldb ); + +lapack_int LAPACKE_ssygv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* w, float* work, lapack_int lwork ); +lapack_int LAPACKE_dsygv_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* w, double* work, lapack_int lwork ); + +lapack_int LAPACKE_ssygvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* w, float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dsygvd_work( int matrix_order, lapack_int itype, char jobz, + char uplo, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* w, double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); + +lapack_int LAPACKE_ssygvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float vl, float vu, lapack_int il, + lapack_int iu, float abstol, lapack_int* m, + float* w, float* z, lapack_int ldz, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int* ifail ); +lapack_int LAPACKE_dsygvx_work( int matrix_order, lapack_int itype, char jobz, + char range, char uplo, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double vl, double vu, lapack_int il, + lapack_int iu, double abstol, lapack_int* m, + double* w, double* z, lapack_int ldz, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int* ifail ); + +lapack_int LAPACKE_ssyrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const float* af, lapack_int ldaf, + const lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dsyrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* b, lapack_int ldb, double* x, + lapack_int ldx, double* ferr, double* berr, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_csyrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_zsyrfs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_ssyrfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, const float* af, + lapack_int ldaf, const lapack_int* ipiv, + const float* s, const float* b, lapack_int ldb, + float* x, lapack_int ldx, float* rcond, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dsyrfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, const double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* s, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_csyrfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* af, + lapack_int ldaf, const lapack_int* ipiv, + const float* s, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zsyrfsx_work( int matrix_order, char uplo, char equed, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* af, + lapack_int ldaf, const lapack_int* ipiv, + const double* s, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_ssysv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, float* a, lapack_int lda, + lapack_int* ipiv, float* b, lapack_int ldb, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dsysv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, double* a, lapack_int lda, + lapack_int* ipiv, double* b, lapack_int ldb, + double* work, lapack_int lwork ); +lapack_int LAPACKE_csysv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_float* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zsysv_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, lapack_complex_double* a, + lapack_int lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_ssysvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, const float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, const float* b, + lapack_int ldb, float* x, lapack_int ldx, + float* rcond, float* ferr, float* berr, + float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dsysvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, const double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, const double* b, + lapack_int ldb, double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + double* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_csysvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int ldb, lapack_complex_float* x, + lapack_int ldx, float* rcond, float* ferr, + float* berr, lapack_complex_float* work, + lapack_int lwork, float* rwork ); +lapack_int LAPACKE_zsysvx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, lapack_int lwork, + double* rwork ); + +lapack_int LAPACKE_ssysvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, float* a, + lapack_int lda, float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, + float* b, lapack_int ldb, float* x, + lapack_int ldx, float* rcond, float* rpvgrw, + float* berr, lapack_int n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int nparams, float* params, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dsysvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, double* a, + lapack_int lda, double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, + double* b, lapack_int ldb, double* x, + lapack_int ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int nparams, double* params, + double* work, lapack_int* iwork ); +lapack_int LAPACKE_csysvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* x, lapack_int ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int nparams, + float* params, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_zsysvxx_work( int matrix_order, char fact, char uplo, + lapack_int n, lapack_int nrhs, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* af, lapack_int ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* x, lapack_int ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int nparams, + double* params, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_ssytrd_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, float* d, float* e, + float* tau, float* work, lapack_int lwork ); +lapack_int LAPACKE_dsytrd_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, double* d, double* e, + double* tau, double* work, lapack_int lwork ); + +lapack_int LAPACKE_ssytrf_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, lapack_int* ipiv, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dsytrf_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, lapack_int* ipiv, + double* work, lapack_int lwork ); +lapack_int LAPACKE_csytrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_float* work, + lapack_int lwork ); +lapack_int LAPACKE_zsytrf_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_int* ipiv, lapack_complex_double* work, + lapack_int lwork ); + +lapack_int LAPACKE_ssytri_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, + const lapack_int* ipiv, float* work ); +lapack_int LAPACKE_dsytri_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, + const lapack_int* ipiv, double* work ); +lapack_int LAPACKE_csytri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work ); +lapack_int LAPACKE_zsytri_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work ); + +lapack_int LAPACKE_ssytrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const lapack_int* ipiv, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dsytrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const lapack_int* ipiv, + double* b, lapack_int ldb ); +lapack_int LAPACKE_csytrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_zsytrs_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stbcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, lapack_int kd, + const float* ab, lapack_int ldab, float* rcond, + float* work, lapack_int* iwork ); +lapack_int LAPACKE_dtbcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, lapack_int kd, + const double* ab, lapack_int ldab, + double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_ctbcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, lapack_int kd, + const lapack_complex_float* ab, lapack_int ldab, + float* rcond, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_ztbcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, lapack_int kd, + const lapack_complex_double* ab, + lapack_int ldab, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_stbrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const float* ab, + lapack_int ldab, const float* b, lapack_int ldb, + const float* x, lapack_int ldx, float* ferr, + float* berr, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dtbrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const double* ab, + lapack_int ldab, const double* b, + lapack_int ldb, const double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_ctbrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const lapack_complex_float* ab, + lapack_int ldab, const lapack_complex_float* b, + lapack_int ldb, const lapack_complex_float* x, + lapack_int ldx, float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztbrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, const lapack_complex_double* b, + lapack_int ldb, const lapack_complex_double* x, + lapack_int ldx, double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_stbtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const float* ab, + lapack_int ldab, float* b, lapack_int ldb ); +lapack_int LAPACKE_dtbtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const double* ab, + lapack_int ldab, double* b, lapack_int ldb ); +lapack_int LAPACKE_ctbtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, const lapack_complex_float* ab, + lapack_int ldab, lapack_complex_float* b, + lapack_int ldb ); +lapack_int LAPACKE_ztbtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int kd, + lapack_int nrhs, + const lapack_complex_double* ab, + lapack_int ldab, lapack_complex_double* b, + lapack_int ldb ); + +lapack_int LAPACKE_stfsm_work( int matrix_order, char transr, char side, + char uplo, char trans, char diag, lapack_int m, + lapack_int n, float alpha, const float* a, + float* b, lapack_int ldb ); +lapack_int LAPACKE_dtfsm_work( int matrix_order, char transr, char side, + char uplo, char trans, char diag, lapack_int m, + lapack_int n, double alpha, const double* a, + double* b, lapack_int ldb ); +lapack_int LAPACKE_ctfsm_work( int matrix_order, char transr, char side, + char uplo, char trans, char diag, lapack_int m, + lapack_int n, lapack_complex_float alpha, + const lapack_complex_float* a, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztfsm_work( int matrix_order, char transr, char side, + char uplo, char trans, char diag, lapack_int m, + lapack_int n, lapack_complex_double alpha, + const lapack_complex_double* a, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stftri_work( int matrix_order, char transr, char uplo, + char diag, lapack_int n, float* a ); +lapack_int LAPACKE_dtftri_work( int matrix_order, char transr, char uplo, + char diag, lapack_int n, double* a ); +lapack_int LAPACKE_ctftri_work( int matrix_order, char transr, char uplo, + char diag, lapack_int n, + lapack_complex_float* a ); +lapack_int LAPACKE_ztftri_work( int matrix_order, char transr, char uplo, + char diag, lapack_int n, + lapack_complex_double* a ); + +lapack_int LAPACKE_stfttp_work( int matrix_order, char transr, char uplo, + lapack_int n, const float* arf, float* ap ); +lapack_int LAPACKE_dtfttp_work( int matrix_order, char transr, char uplo, + lapack_int n, const double* arf, double* ap ); +lapack_int LAPACKE_ctfttp_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* arf, + lapack_complex_float* ap ); +lapack_int LAPACKE_ztfttp_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* arf, + lapack_complex_double* ap ); + +lapack_int LAPACKE_stfttr_work( int matrix_order, char transr, char uplo, + lapack_int n, const float* arf, float* a, + lapack_int lda ); +lapack_int LAPACKE_dtfttr_work( int matrix_order, char transr, char uplo, + lapack_int n, const double* arf, double* a, + lapack_int lda ); +lapack_int LAPACKE_ctfttr_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* arf, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_ztfttr_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* arf, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_stgevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const float* s, lapack_int lds, const float* p, + lapack_int ldp, float* vl, lapack_int ldvl, + float* vr, lapack_int ldvr, lapack_int mm, + lapack_int* m, float* work ); +lapack_int LAPACKE_dtgevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const double* s, lapack_int lds, + const double* p, lapack_int ldp, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, double* work ); +lapack_int LAPACKE_ctgevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* s, lapack_int lds, + const lapack_complex_float* p, lapack_int ldp, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztgevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* s, lapack_int lds, + const lapack_complex_double* p, lapack_int ldp, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_stgexc_work( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, float* a, + lapack_int lda, float* b, lapack_int ldb, + float* q, lapack_int ldq, float* z, + lapack_int ldz, lapack_int* ifst, + lapack_int* ilst, float* work, + lapack_int lwork ); +lapack_int LAPACKE_dtgexc_work( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* q, lapack_int ldq, double* z, + lapack_int ldz, lapack_int* ifst, + lapack_int* ilst, double* work, + lapack_int lwork ); +lapack_int LAPACKE_ctgexc_work( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz, + lapack_int ifst, lapack_int ilst ); +lapack_int LAPACKE_ztgexc_work( int matrix_order, lapack_logical wantq, + lapack_logical wantz, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz, + lapack_int ifst, lapack_int ilst ); + +lapack_int LAPACKE_stgsen_work( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + float* a, lapack_int lda, float* b, + lapack_int ldb, float* alphar, float* alphai, + float* beta, float* q, lapack_int ldq, float* z, + lapack_int ldz, lapack_int* m, float* pl, + float* pr, float* dif, float* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); +lapack_int LAPACKE_dtgsen_work( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + double* a, lapack_int lda, double* b, + lapack_int ldb, double* alphar, double* alphai, + double* beta, double* q, lapack_int ldq, + double* z, lapack_int ldz, lapack_int* m, + double* pl, double* pr, double* dif, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_ctgsen_work( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* alpha, + lapack_complex_float* beta, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* z, lapack_int ldz, + lapack_int* m, float* pl, float* pr, float* dif, + lapack_complex_float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_ztgsen_work( int matrix_order, lapack_int ijob, + lapack_logical wantq, lapack_logical wantz, + const lapack_logical* select, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* alpha, + lapack_complex_double* beta, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* z, lapack_int ldz, + lapack_int* m, double* pl, double* pr, + double* dif, lapack_complex_double* work, + lapack_int lwork, lapack_int* iwork, + lapack_int liwork ); + +lapack_int LAPACKE_stgsja_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_int k, lapack_int l, + float* a, lapack_int lda, float* b, + lapack_int ldb, float tola, float tolb, + float* alpha, float* beta, float* u, + lapack_int ldu, float* v, lapack_int ldv, + float* q, lapack_int ldq, float* work, + lapack_int* ncycle ); +lapack_int LAPACKE_dtgsja_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_int k, lapack_int l, + double* a, lapack_int lda, double* b, + lapack_int ldb, double tola, double tolb, + double* alpha, double* beta, double* u, + lapack_int ldu, double* v, lapack_int ldv, + double* q, lapack_int ldq, double* work, + lapack_int* ncycle ); +lapack_int LAPACKE_ctgsja_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_int k, lapack_int l, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + float tola, float tolb, float* alpha, + float* beta, lapack_complex_float* u, + lapack_int ldu, lapack_complex_float* v, + lapack_int ldv, lapack_complex_float* q, + lapack_int ldq, lapack_complex_float* work, + lapack_int* ncycle ); +lapack_int LAPACKE_ztgsja_work( int matrix_order, char jobu, char jobv, + char jobq, lapack_int m, lapack_int p, + lapack_int n, lapack_int k, lapack_int l, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + double tola, double tolb, double* alpha, + double* beta, lapack_complex_double* u, + lapack_int ldu, lapack_complex_double* v, + lapack_int ldv, lapack_complex_double* q, + lapack_int ldq, lapack_complex_double* work, + lapack_int* ncycle ); + +lapack_int LAPACKE_stgsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const float* a, lapack_int lda, const float* b, + lapack_int ldb, const float* vl, + lapack_int ldvl, const float* vr, + lapack_int ldvr, float* s, float* dif, + lapack_int mm, lapack_int* m, float* work, + lapack_int lwork, lapack_int* iwork ); +lapack_int LAPACKE_dtgsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const double* a, lapack_int lda, + const double* b, lapack_int ldb, + const double* vl, lapack_int ldvl, + const double* vr, lapack_int ldvr, double* s, + double* dif, lapack_int mm, lapack_int* m, + double* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_ctgsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* vl, lapack_int ldvl, + const lapack_complex_float* vr, lapack_int ldvr, + float* s, float* dif, lapack_int mm, + lapack_int* m, lapack_complex_float* work, + lapack_int lwork, lapack_int* iwork ); +lapack_int LAPACKE_ztgsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* vl, + lapack_int ldvl, + const lapack_complex_double* vr, + lapack_int ldvr, double* s, double* dif, + lapack_int mm, lapack_int* m, + lapack_complex_double* work, lapack_int lwork, + lapack_int* iwork ); + +lapack_int LAPACKE_stgsyl_work( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, const float* a, + lapack_int lda, const float* b, lapack_int ldb, + float* c, lapack_int ldc, const float* d, + lapack_int ldd, const float* e, lapack_int lde, + float* f, lapack_int ldf, float* scale, + float* dif, float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dtgsyl_work( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, const double* a, + lapack_int lda, const double* b, lapack_int ldb, + double* c, lapack_int ldc, const double* d, + lapack_int ldd, const double* e, lapack_int lde, + double* f, lapack_int ldf, double* scale, + double* dif, double* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_ctgsyl_work( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* c, lapack_int ldc, + const lapack_complex_float* d, lapack_int ldd, + const lapack_complex_float* e, lapack_int lde, + lapack_complex_float* f, lapack_int ldf, + float* scale, float* dif, + lapack_complex_float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_ztgsyl_work( int matrix_order, char trans, lapack_int ijob, + lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* c, lapack_int ldc, + const lapack_complex_double* d, lapack_int ldd, + const lapack_complex_double* e, lapack_int lde, + lapack_complex_double* f, lapack_int ldf, + double* scale, double* dif, + lapack_complex_double* work, lapack_int lwork, + lapack_int* iwork ); + +lapack_int LAPACKE_stpcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, const float* ap, + float* rcond, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dtpcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, const double* ap, + double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_ctpcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, + const lapack_complex_float* ap, float* rcond, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztpcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, + const lapack_complex_double* ap, double* rcond, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_stprfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const float* ap, const float* b, lapack_int ldb, + const float* x, lapack_int ldx, float* ferr, + float* berr, float* work, lapack_int* iwork ); +lapack_int LAPACKE_dtprfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const double* ap, const double* b, + lapack_int ldb, const double* x, lapack_int ldx, + double* ferr, double* berr, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_ctprfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztprfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_stptri_work( int matrix_order, char uplo, char diag, + lapack_int n, float* ap ); +lapack_int LAPACKE_dtptri_work( int matrix_order, char uplo, char diag, + lapack_int n, double* ap ); +lapack_int LAPACKE_ctptri_work( int matrix_order, char uplo, char diag, + lapack_int n, lapack_complex_float* ap ); +lapack_int LAPACKE_ztptri_work( int matrix_order, char uplo, char diag, + lapack_int n, lapack_complex_double* ap ); + +lapack_int LAPACKE_stptrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const float* ap, float* b, lapack_int ldb ); +lapack_int LAPACKE_dtptrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const double* ap, double* b, lapack_int ldb ); +lapack_int LAPACKE_ctptrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_float* ap, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztptrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_double* ap, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_stpttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const float* ap, float* arf ); +lapack_int LAPACKE_dtpttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const double* ap, double* arf ); +lapack_int LAPACKE_ctpttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* ap, + lapack_complex_float* arf ); +lapack_int LAPACKE_ztpttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* ap, + lapack_complex_double* arf ); + +lapack_int LAPACKE_stpttr_work( int matrix_order, char uplo, lapack_int n, + const float* ap, float* a, lapack_int lda ); +lapack_int LAPACKE_dtpttr_work( int matrix_order, char uplo, lapack_int n, + const double* ap, double* a, lapack_int lda ); +lapack_int LAPACKE_ctpttr_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_ztpttr_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_strcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, const float* a, + lapack_int lda, float* rcond, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dtrcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, const double* a, + lapack_int lda, double* rcond, double* work, + lapack_int* iwork ); +lapack_int LAPACKE_ctrcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + float* rcond, lapack_complex_float* work, + float* rwork ); +lapack_int LAPACKE_ztrcon_work( int matrix_order, char norm, char uplo, + char diag, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + double* rcond, lapack_complex_double* work, + double* rwork ); + +lapack_int LAPACKE_strevc_work( int matrix_order, char side, char howmny, + lapack_logical* select, lapack_int n, + const float* t, lapack_int ldt, float* vl, + lapack_int ldvl, float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, float* work ); +lapack_int LAPACKE_dtrevc_work( int matrix_order, char side, char howmny, + lapack_logical* select, lapack_int n, + const double* t, lapack_int ldt, double* vl, + lapack_int ldvl, double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, double* work ); +lapack_int LAPACKE_ctrevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* vl, lapack_int ldvl, + lapack_complex_float* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztrevc_work( int matrix_order, char side, char howmny, + const lapack_logical* select, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* vl, lapack_int ldvl, + lapack_complex_double* vr, lapack_int ldvr, + lapack_int mm, lapack_int* m, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_strexc_work( int matrix_order, char compq, lapack_int n, + float* t, lapack_int ldt, float* q, + lapack_int ldq, lapack_int* ifst, + lapack_int* ilst, float* work ); +lapack_int LAPACKE_dtrexc_work( int matrix_order, char compq, lapack_int n, + double* t, lapack_int ldt, double* q, + lapack_int ldq, lapack_int* ifst, + lapack_int* ilst, double* work ); +lapack_int LAPACKE_ctrexc_work( int matrix_order, char compq, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* q, lapack_int ldq, + lapack_int ifst, lapack_int ilst ); +lapack_int LAPACKE_ztrexc_work( int matrix_order, char compq, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* q, lapack_int ldq, + lapack_int ifst, lapack_int ilst ); + +lapack_int LAPACKE_strrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const float* a, lapack_int lda, const float* b, + lapack_int ldb, const float* x, lapack_int ldx, + float* ferr, float* berr, float* work, + lapack_int* iwork ); +lapack_int LAPACKE_dtrrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const double* a, lapack_int lda, + const double* b, lapack_int ldb, + const double* x, lapack_int ldx, double* ferr, + double* berr, double* work, lapack_int* iwork ); +lapack_int LAPACKE_ctrrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + const lapack_complex_float* x, lapack_int ldx, + float* ferr, float* berr, + lapack_complex_float* work, float* rwork ); +lapack_int LAPACKE_ztrrfs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + const lapack_complex_double* x, lapack_int ldx, + double* ferr, double* berr, + lapack_complex_double* work, double* rwork ); + +lapack_int LAPACKE_strsen_work( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + float* t, lapack_int ldt, float* q, + lapack_int ldq, float* wr, float* wi, + lapack_int* m, float* s, float* sep, + float* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_dtrsen_work( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + double* t, lapack_int ldt, double* q, + lapack_int ldq, double* wr, double* wi, + lapack_int* m, double* s, double* sep, + double* work, lapack_int lwork, + lapack_int* iwork, lapack_int liwork ); +lapack_int LAPACKE_ctrsen_work( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* w, lapack_int* m, + float* s, float* sep, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_ztrsen_work( int matrix_order, char job, char compq, + const lapack_logical* select, lapack_int n, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* w, lapack_int* m, + double* s, double* sep, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_strsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const float* t, lapack_int ldt, const float* vl, + lapack_int ldvl, const float* vr, + lapack_int ldvr, float* s, float* sep, + lapack_int mm, lapack_int* m, float* work, + lapack_int ldwork, lapack_int* iwork ); +lapack_int LAPACKE_dtrsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const double* t, lapack_int ldt, + const double* vl, lapack_int ldvl, + const double* vr, lapack_int ldvr, double* s, + double* sep, lapack_int mm, lapack_int* m, + double* work, lapack_int ldwork, + lapack_int* iwork ); +lapack_int LAPACKE_ctrsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_float* t, lapack_int ldt, + const lapack_complex_float* vl, lapack_int ldvl, + const lapack_complex_float* vr, lapack_int ldvr, + float* s, float* sep, lapack_int mm, + lapack_int* m, lapack_complex_float* work, + lapack_int ldwork, float* rwork ); +lapack_int LAPACKE_ztrsna_work( int matrix_order, char job, char howmny, + const lapack_logical* select, lapack_int n, + const lapack_complex_double* t, lapack_int ldt, + const lapack_complex_double* vl, + lapack_int ldvl, + const lapack_complex_double* vr, + lapack_int ldvr, double* s, double* sep, + lapack_int mm, lapack_int* m, + lapack_complex_double* work, lapack_int ldwork, + double* rwork ); + +lapack_int LAPACKE_strsyl_work( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const float* a, lapack_int lda, const float* b, + lapack_int ldb, float* c, lapack_int ldc, + float* scale ); +lapack_int LAPACKE_dtrsyl_work( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const double* a, lapack_int lda, + const double* b, lapack_int ldb, double* c, + lapack_int ldc, double* scale ); +lapack_int LAPACKE_ctrsyl_work( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* c, lapack_int ldc, + float* scale ); +lapack_int LAPACKE_ztrsyl_work( int matrix_order, char trana, char tranb, + lapack_int isgn, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* c, lapack_int ldc, + double* scale ); + +lapack_int LAPACKE_strtri_work( int matrix_order, char uplo, char diag, + lapack_int n, float* a, lapack_int lda ); +lapack_int LAPACKE_dtrtri_work( int matrix_order, char uplo, char diag, + lapack_int n, double* a, lapack_int lda ); +lapack_int LAPACKE_ctrtri_work( int matrix_order, char uplo, char diag, + lapack_int n, lapack_complex_float* a, + lapack_int lda ); +lapack_int LAPACKE_ztrtri_work( int matrix_order, char uplo, char diag, + lapack_int n, lapack_complex_double* a, + lapack_int lda ); + +lapack_int LAPACKE_strtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const float* a, lapack_int lda, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dtrtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const double* a, lapack_int lda, double* b, + lapack_int ldb ); +lapack_int LAPACKE_ctrtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztrtrs_work( int matrix_order, char uplo, char trans, + char diag, lapack_int n, lapack_int nrhs, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_strttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const float* a, lapack_int lda, + float* arf ); +lapack_int LAPACKE_dtrttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const double* a, lapack_int lda, + double* arf ); +lapack_int LAPACKE_ctrttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_float* a, + lapack_int lda, lapack_complex_float* arf ); +lapack_int LAPACKE_ztrttf_work( int matrix_order, char transr, char uplo, + lapack_int n, const lapack_complex_double* a, + lapack_int lda, lapack_complex_double* arf ); + +lapack_int LAPACKE_strttp_work( int matrix_order, char uplo, lapack_int n, + const float* a, lapack_int lda, float* ap ); +lapack_int LAPACKE_dtrttp_work( int matrix_order, char uplo, lapack_int n, + const double* a, lapack_int lda, double* ap ); +lapack_int LAPACKE_ctrttp_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + lapack_complex_float* ap ); +lapack_int LAPACKE_ztrttp_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + lapack_complex_double* ap ); + +lapack_int LAPACKE_stzrzf_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* tau, + float* work, lapack_int lwork ); +lapack_int LAPACKE_dtzrzf_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* tau, + double* work, lapack_int lwork ); +lapack_int LAPACKE_ctzrzf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_ztzrzf_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cungbr_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, + lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zungbr_work( int matrix_order, char vect, lapack_int m, + lapack_int n, lapack_int k, + lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunghr_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunghr_work( int matrix_order, lapack_int n, lapack_int ilo, + lapack_int ihi, lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunglq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunglq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cungql_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zungql_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cungqr_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zungqr_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cungrq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zungrq_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int k, lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cungtr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zungtr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmbr_work( int matrix_order, char vect, char side, + char trans, lapack_int m, lapack_int n, + lapack_int k, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmbr_work( int matrix_order, char vect, char side, + char trans, lapack_int m, lapack_int n, + lapack_int k, const lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmhr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmhr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int ilo, + lapack_int ihi, const lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmlq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmlq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmql_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmql_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmqr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmqr_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmrq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmrq_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmrz_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const lapack_complex_float* a, + lapack_int lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmrz_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, const lapack_complex_double* a, + lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cunmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const lapack_complex_float* a, lapack_int lda, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_zunmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const lapack_complex_double* a, lapack_int lda, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work, lapack_int lwork ); + +lapack_int LAPACKE_cupgtr_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_float* ap, + const lapack_complex_float* tau, + lapack_complex_float* q, lapack_int ldq, + lapack_complex_float* work ); +lapack_int LAPACKE_zupgtr_work( int matrix_order, char uplo, lapack_int n, + const lapack_complex_double* ap, + const lapack_complex_double* tau, + lapack_complex_double* q, lapack_int ldq, + lapack_complex_double* work ); + +lapack_int LAPACKE_cupmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const lapack_complex_float* ap, + const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int ldc, + lapack_complex_float* work ); +lapack_int LAPACKE_zupmtr_work( int matrix_order, char side, char uplo, + char trans, lapack_int m, lapack_int n, + const lapack_complex_double* ap, + const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int ldc, + lapack_complex_double* work ); + +lapack_int LAPACKE_claghe( int matrix_order, lapack_int n, lapack_int k, + const float* d, lapack_complex_float* a, + lapack_int lda, lapack_int* iseed ); +lapack_int LAPACKE_zlaghe( int matrix_order, lapack_int n, lapack_int k, + const double* d, lapack_complex_double* a, + lapack_int lda, lapack_int* iseed ); + +lapack_int LAPACKE_slagsy( int matrix_order, lapack_int n, lapack_int k, + const float* d, float* a, lapack_int lda, + lapack_int* iseed ); +lapack_int LAPACKE_dlagsy( int matrix_order, lapack_int n, lapack_int k, + const double* d, double* a, lapack_int lda, + lapack_int* iseed ); +lapack_int LAPACKE_clagsy( int matrix_order, lapack_int n, lapack_int k, + const float* d, lapack_complex_float* a, + lapack_int lda, lapack_int* iseed ); +lapack_int LAPACKE_zlagsy( int matrix_order, lapack_int n, lapack_int k, + const double* d, lapack_complex_double* a, + lapack_int lda, lapack_int* iseed ); + +lapack_int LAPACKE_slapmr( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, float* x, lapack_int ldx, + lapack_int* k ); +lapack_int LAPACKE_dlapmr( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, double* x, + lapack_int ldx, lapack_int* k ); +lapack_int LAPACKE_clapmr( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, lapack_complex_float* x, + lapack_int ldx, lapack_int* k ); +lapack_int LAPACKE_zlapmr( int matrix_order, lapack_logical forwrd, + lapack_int m, lapack_int n, lapack_complex_double* x, + lapack_int ldx, lapack_int* k ); + + +float LAPACKE_slapy2( float x, float y ); +double LAPACKE_dlapy2( double x, double y ); + +float LAPACKE_slapy3( float x, float y, float z ); +double LAPACKE_dlapy3( double x, double y, double z ); + +lapack_int LAPACKE_slartgp( float f, float g, float* cs, float* sn, float* r ); +lapack_int LAPACKE_dlartgp( double f, double g, double* cs, double* sn, + double* r ); + +lapack_int LAPACKE_slartgs( float x, float y, float sigma, float* cs, + float* sn ); +lapack_int LAPACKE_dlartgs( double x, double y, double sigma, double* cs, + double* sn ); + + +//LAPACK 3.3.0 +lapack_int LAPACKE_cbbcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, lapack_int m, + lapack_int p, lapack_int q, float* theta, float* phi, + lapack_complex_float* u1, lapack_int ldu1, + lapack_complex_float* u2, lapack_int ldu2, + lapack_complex_float* v1t, lapack_int ldv1t, + lapack_complex_float* v2t, lapack_int ldv2t, + float* b11d, float* b11e, float* b12d, float* b12e, + float* b21d, float* b21e, float* b22d, float* b22e ); +lapack_int LAPACKE_cbbcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + lapack_int m, lapack_int p, lapack_int q, + float* theta, float* phi, + lapack_complex_float* u1, lapack_int ldu1, + lapack_complex_float* u2, lapack_int ldu2, + lapack_complex_float* v1t, lapack_int ldv1t, + lapack_complex_float* v2t, lapack_int ldv2t, + float* b11d, float* b11e, float* b12d, + float* b12e, float* b21d, float* b21e, + float* b22d, float* b22e, float* rwork, + lapack_int lrwork ); +lapack_int LAPACKE_cheswapr( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_cheswapr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_chetri2( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_chetri2_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_chetri2x( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, lapack_int nb ); +lapack_int LAPACKE_chetri2x_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int nb ); +lapack_int LAPACKE_chetrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_chetrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work ); +lapack_int LAPACKE_csyconv( int matrix_order, char uplo, char way, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_csyconv_work( int matrix_order, char uplo, char way, + lapack_int n, lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* work ); +lapack_int LAPACKE_csyswapr( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_csyswapr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_csytri2( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_csytri2_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_csytri2x( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, lapack_int nb ); +lapack_int LAPACKE_csytri2x_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int nb ); +lapack_int LAPACKE_csytrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_csytrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_float* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work ); +lapack_int LAPACKE_cunbdb( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_float* x11, lapack_int ldx11, + lapack_complex_float* x12, lapack_int ldx12, + lapack_complex_float* x21, lapack_int ldx21, + lapack_complex_float* x22, lapack_int ldx22, + float* theta, float* phi, + lapack_complex_float* taup1, + lapack_complex_float* taup2, + lapack_complex_float* tauq1, + lapack_complex_float* tauq2 ); +lapack_int LAPACKE_cunbdb_work( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_float* x11, lapack_int ldx11, + lapack_complex_float* x12, lapack_int ldx12, + lapack_complex_float* x21, lapack_int ldx21, + lapack_complex_float* x22, lapack_int ldx22, + float* theta, float* phi, + lapack_complex_float* taup1, + lapack_complex_float* taup2, + lapack_complex_float* tauq1, + lapack_complex_float* tauq2, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_cuncsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_float* x11, lapack_int ldx11, + lapack_complex_float* x12, lapack_int ldx12, + lapack_complex_float* x21, lapack_int ldx21, + lapack_complex_float* x22, lapack_int ldx22, + float* theta, lapack_complex_float* u1, + lapack_int ldu1, lapack_complex_float* u2, + lapack_int ldu2, lapack_complex_float* v1t, + lapack_int ldv1t, lapack_complex_float* v2t, + lapack_int ldv2t ); +lapack_int LAPACKE_cuncsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + char signs, lapack_int m, lapack_int p, + lapack_int q, lapack_complex_float* x11, + lapack_int ldx11, lapack_complex_float* x12, + lapack_int ldx12, lapack_complex_float* x21, + lapack_int ldx21, lapack_complex_float* x22, + lapack_int ldx22, float* theta, + lapack_complex_float* u1, lapack_int ldu1, + lapack_complex_float* u2, lapack_int ldu2, + lapack_complex_float* v1t, lapack_int ldv1t, + lapack_complex_float* v2t, lapack_int ldv2t, + lapack_complex_float* work, lapack_int lwork, + float* rwork, lapack_int lrwork, + lapack_int* iwork ); +lapack_int LAPACKE_dbbcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, lapack_int m, + lapack_int p, lapack_int q, double* theta, + double* phi, double* u1, lapack_int ldu1, double* u2, + lapack_int ldu2, double* v1t, lapack_int ldv1t, + double* v2t, lapack_int ldv2t, double* b11d, + double* b11e, double* b12d, double* b12e, + double* b21d, double* b21e, double* b22d, + double* b22e ); +lapack_int LAPACKE_dbbcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + lapack_int m, lapack_int p, lapack_int q, + double* theta, double* phi, double* u1, + lapack_int ldu1, double* u2, lapack_int ldu2, + double* v1t, lapack_int ldv1t, double* v2t, + lapack_int ldv2t, double* b11d, double* b11e, + double* b12d, double* b12e, double* b21d, + double* b21e, double* b22d, double* b22e, + double* work, lapack_int lwork ); +lapack_int LAPACKE_dorbdb( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + double* x11, lapack_int ldx11, double* x12, + lapack_int ldx12, double* x21, lapack_int ldx21, + double* x22, lapack_int ldx22, double* theta, + double* phi, double* taup1, double* taup2, + double* tauq1, double* tauq2 ); +lapack_int LAPACKE_dorbdb_work( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + double* x11, lapack_int ldx11, double* x12, + lapack_int ldx12, double* x21, lapack_int ldx21, + double* x22, lapack_int ldx22, double* theta, + double* phi, double* taup1, double* taup2, + double* tauq1, double* tauq2, double* work, + lapack_int lwork ); +lapack_int LAPACKE_dorcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + double* x11, lapack_int ldx11, double* x12, + lapack_int ldx12, double* x21, lapack_int ldx21, + double* x22, lapack_int ldx22, double* theta, + double* u1, lapack_int ldu1, double* u2, + lapack_int ldu2, double* v1t, lapack_int ldv1t, + double* v2t, lapack_int ldv2t ); +lapack_int LAPACKE_dorcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + char signs, lapack_int m, lapack_int p, + lapack_int q, double* x11, lapack_int ldx11, + double* x12, lapack_int ldx12, double* x21, + lapack_int ldx21, double* x22, lapack_int ldx22, + double* theta, double* u1, lapack_int ldu1, + double* u2, lapack_int ldu2, double* v1t, + lapack_int ldv1t, double* v2t, lapack_int ldv2t, + double* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_dsyconv( int matrix_order, char uplo, char way, lapack_int n, + double* a, lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_dsyconv_work( int matrix_order, char uplo, char way, + lapack_int n, double* a, lapack_int lda, + const lapack_int* ipiv, double* work ); +lapack_int LAPACKE_dsyswapr( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int i1, lapack_int i2 ); +lapack_int LAPACKE_dsyswapr_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int i1, lapack_int i2 ); +lapack_int LAPACKE_dsytri2( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_dsytri2_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int lwork ); +lapack_int LAPACKE_dsytri2x( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, const lapack_int* ipiv, + lapack_int nb ); +lapack_int LAPACKE_dsytri2x_work( int matrix_order, char uplo, lapack_int n, + double* a, lapack_int lda, + const lapack_int* ipiv, double* work, + lapack_int nb ); +lapack_int LAPACKE_dsytrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, lapack_int lda, + const lapack_int* ipiv, double* b, lapack_int ldb ); +lapack_int LAPACKE_dsytrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const double* a, + lapack_int lda, const lapack_int* ipiv, + double* b, lapack_int ldb, double* work ); +lapack_int LAPACKE_sbbcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, lapack_int m, + lapack_int p, lapack_int q, float* theta, float* phi, + float* u1, lapack_int ldu1, float* u2, + lapack_int ldu2, float* v1t, lapack_int ldv1t, + float* v2t, lapack_int ldv2t, float* b11d, + float* b11e, float* b12d, float* b12e, float* b21d, + float* b21e, float* b22d, float* b22e ); +lapack_int LAPACKE_sbbcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + lapack_int m, lapack_int p, lapack_int q, + float* theta, float* phi, float* u1, + lapack_int ldu1, float* u2, lapack_int ldu2, + float* v1t, lapack_int ldv1t, float* v2t, + lapack_int ldv2t, float* b11d, float* b11e, + float* b12d, float* b12e, float* b21d, + float* b21e, float* b22d, float* b22e, + float* work, lapack_int lwork ); +lapack_int LAPACKE_sorbdb( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, float* x11, + lapack_int ldx11, float* x12, lapack_int ldx12, + float* x21, lapack_int ldx21, float* x22, + lapack_int ldx22, float* theta, float* phi, + float* taup1, float* taup2, float* tauq1, + float* tauq2 ); +lapack_int LAPACKE_sorbdb_work( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + float* x11, lapack_int ldx11, float* x12, + lapack_int ldx12, float* x21, lapack_int ldx21, + float* x22, lapack_int ldx22, float* theta, + float* phi, float* taup1, float* taup2, + float* tauq1, float* tauq2, float* work, + lapack_int lwork ); +lapack_int LAPACKE_sorcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, float* x11, + lapack_int ldx11, float* x12, lapack_int ldx12, + float* x21, lapack_int ldx21, float* x22, + lapack_int ldx22, float* theta, float* u1, + lapack_int ldu1, float* u2, lapack_int ldu2, + float* v1t, lapack_int ldv1t, float* v2t, + lapack_int ldv2t ); +lapack_int LAPACKE_sorcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + char signs, lapack_int m, lapack_int p, + lapack_int q, float* x11, lapack_int ldx11, + float* x12, lapack_int ldx12, float* x21, + lapack_int ldx21, float* x22, lapack_int ldx22, + float* theta, float* u1, lapack_int ldu1, + float* u2, lapack_int ldu2, float* v1t, + lapack_int ldv1t, float* v2t, lapack_int ldv2t, + float* work, lapack_int lwork, + lapack_int* iwork ); +lapack_int LAPACKE_ssyconv( int matrix_order, char uplo, char way, lapack_int n, + float* a, lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_ssyconv_work( int matrix_order, char uplo, char way, + lapack_int n, float* a, lapack_int lda, + const lapack_int* ipiv, float* work ); +lapack_int LAPACKE_ssyswapr( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int i1, lapack_int i2 ); +lapack_int LAPACKE_ssyswapr_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int i1, lapack_int i2 ); +lapack_int LAPACKE_ssytri2( int matrix_order, char uplo, lapack_int n, float* a, + lapack_int lda, const lapack_int* ipiv ); +lapack_int LAPACKE_ssytri2_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int lwork ); +lapack_int LAPACKE_ssytri2x( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, const lapack_int* ipiv, + lapack_int nb ); +lapack_int LAPACKE_ssytri2x_work( int matrix_order, char uplo, lapack_int n, + float* a, lapack_int lda, + const lapack_int* ipiv, float* work, + lapack_int nb ); +lapack_int LAPACKE_ssytrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, lapack_int lda, + const lapack_int* ipiv, float* b, lapack_int ldb ); +lapack_int LAPACKE_ssytrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const float* a, + lapack_int lda, const lapack_int* ipiv, + float* b, lapack_int ldb, float* work ); +lapack_int LAPACKE_zbbcsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, lapack_int m, + lapack_int p, lapack_int q, double* theta, + double* phi, lapack_complex_double* u1, + lapack_int ldu1, lapack_complex_double* u2, + lapack_int ldu2, lapack_complex_double* v1t, + lapack_int ldv1t, lapack_complex_double* v2t, + lapack_int ldv2t, double* b11d, double* b11e, + double* b12d, double* b12e, double* b21d, + double* b21e, double* b22d, double* b22e ); +lapack_int LAPACKE_zbbcsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + lapack_int m, lapack_int p, lapack_int q, + double* theta, double* phi, + lapack_complex_double* u1, lapack_int ldu1, + lapack_complex_double* u2, lapack_int ldu2, + lapack_complex_double* v1t, lapack_int ldv1t, + lapack_complex_double* v2t, lapack_int ldv2t, + double* b11d, double* b11e, double* b12d, + double* b12e, double* b21d, double* b21e, + double* b22d, double* b22e, double* rwork, + lapack_int lrwork ); +lapack_int LAPACKE_zheswapr( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_zheswapr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_zhetri2( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zhetri2_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int lwork ); +lapack_int LAPACKE_zhetri2x( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, lapack_int nb ); +lapack_int LAPACKE_zhetri2x_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int nb ); +lapack_int LAPACKE_zhetrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); +lapack_int LAPACKE_zhetrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work ); +lapack_int LAPACKE_zsyconv( int matrix_order, char uplo, char way, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zsyconv_work( int matrix_order, char uplo, char way, + lapack_int n, lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* work ); +lapack_int LAPACKE_zsyswapr( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_zsyswapr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int i1, + lapack_int i2 ); +lapack_int LAPACKE_zsytri2( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv ); +lapack_int LAPACKE_zsytri2_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int lwork ); +lapack_int LAPACKE_zsytri2x( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, lapack_int nb ); +lapack_int LAPACKE_zsytri2x_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double* a, lapack_int lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int nb ); +lapack_int LAPACKE_zsytrs2( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb ); +lapack_int LAPACKE_zsytrs2_work( int matrix_order, char uplo, lapack_int n, + lapack_int nrhs, const lapack_complex_double* a, + lapack_int lda, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work ); +lapack_int LAPACKE_zunbdb( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_double* x11, lapack_int ldx11, + lapack_complex_double* x12, lapack_int ldx12, + lapack_complex_double* x21, lapack_int ldx21, + lapack_complex_double* x22, lapack_int ldx22, + double* theta, double* phi, + lapack_complex_double* taup1, + lapack_complex_double* taup2, + lapack_complex_double* tauq1, + lapack_complex_double* tauq2 ); +lapack_int LAPACKE_zunbdb_work( int matrix_order, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_double* x11, lapack_int ldx11, + lapack_complex_double* x12, lapack_int ldx12, + lapack_complex_double* x21, lapack_int ldx21, + lapack_complex_double* x22, lapack_int ldx22, + double* theta, double* phi, + lapack_complex_double* taup1, + lapack_complex_double* taup2, + lapack_complex_double* tauq1, + lapack_complex_double* tauq2, + lapack_complex_double* work, lapack_int lwork ); +lapack_int LAPACKE_zuncsd( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, char signs, + lapack_int m, lapack_int p, lapack_int q, + lapack_complex_double* x11, lapack_int ldx11, + lapack_complex_double* x12, lapack_int ldx12, + lapack_complex_double* x21, lapack_int ldx21, + lapack_complex_double* x22, lapack_int ldx22, + double* theta, lapack_complex_double* u1, + lapack_int ldu1, lapack_complex_double* u2, + lapack_int ldu2, lapack_complex_double* v1t, + lapack_int ldv1t, lapack_complex_double* v2t, + lapack_int ldv2t ); +lapack_int LAPACKE_zuncsd_work( int matrix_order, char jobu1, char jobu2, + char jobv1t, char jobv2t, char trans, + char signs, lapack_int m, lapack_int p, + lapack_int q, lapack_complex_double* x11, + lapack_int ldx11, lapack_complex_double* x12, + lapack_int ldx12, lapack_complex_double* x21, + lapack_int ldx21, lapack_complex_double* x22, + lapack_int ldx22, double* theta, + lapack_complex_double* u1, lapack_int ldu1, + lapack_complex_double* u2, lapack_int ldu2, + lapack_complex_double* v1t, lapack_int ldv1t, + lapack_complex_double* v2t, lapack_int ldv2t, + lapack_complex_double* work, lapack_int lwork, + double* rwork, lapack_int lrwork, + lapack_int* iwork ); +//LAPACK 3.4.0 +lapack_int LAPACKE_sgemqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const float* v, lapack_int ldv, + const float* t, lapack_int ldt, float* c, + lapack_int ldc ); +lapack_int LAPACKE_dgemqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const double* v, lapack_int ldv, + const double* t, lapack_int ldt, double* c, + lapack_int ldc ); +lapack_int LAPACKE_cgemqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const lapack_complex_float* v, + lapack_int ldv, const lapack_complex_float* t, + lapack_int ldt, lapack_complex_float* c, + lapack_int ldc ); +lapack_int LAPACKE_zgemqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const lapack_complex_double* v, + lapack_int ldv, const lapack_complex_double* t, + lapack_int ldt, lapack_complex_double* c, + lapack_int ldc ); + +lapack_int LAPACKE_sgeqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, float* a, lapack_int lda, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dgeqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, double* a, lapack_int lda, double* t, + lapack_int ldt ); +lapack_int LAPACKE_cgeqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* t, + lapack_int ldt ); +lapack_int LAPACKE_zgeqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* t, + lapack_int ldt ); + +lapack_int LAPACKE_sgeqrt2( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dgeqrt2( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* t, + lapack_int ldt ); +lapack_int LAPACKE_cgeqrt2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zgeqrt2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_sgeqrt3( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dgeqrt3( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* t, + lapack_int ldt ); +lapack_int LAPACKE_cgeqrt3( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zgeqrt3( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_stpmqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, const float* v, + lapack_int ldv, const float* t, lapack_int ldt, + float* a, lapack_int lda, float* b, + lapack_int ldb ); +lapack_int LAPACKE_dtpmqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, const double* v, + lapack_int ldv, const double* t, lapack_int ldt, + double* a, lapack_int lda, double* b, + lapack_int ldb ); +lapack_int LAPACKE_ctpmqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb ); +lapack_int LAPACKE_ztpmqrt( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb ); + +lapack_int LAPACKE_dtpqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, double* a, + lapack_int lda, double* b, lapack_int ldb, double* t, + lapack_int ldt ); +lapack_int LAPACKE_ctpqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* t, + lapack_complex_float* b, lapack_int ldb, + lapack_int ldt ); +lapack_int LAPACKE_ztpqrt( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_stpqrt2( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* b, lapack_int ldb, + float* t, lapack_int ldt ); +lapack_int LAPACKE_dtpqrt2( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* b, + lapack_int ldb, double* t, lapack_int ldt ); +lapack_int LAPACKE_ctpqrt2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_ztpqrt2( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_stprfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, lapack_int l, const float* v, + lapack_int ldv, const float* t, lapack_int ldt, + float* a, lapack_int lda, float* b, lapack_int ldb, + lapack_int myldwork ); +lapack_int LAPACKE_dtprfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, lapack_int l, const double* v, + lapack_int ldv, const double* t, lapack_int ldt, + double* a, lapack_int lda, double* b, lapack_int ldb, + lapack_int myldwork ); +lapack_int LAPACKE_ctprfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, lapack_int l, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_int myldwork ); +lapack_int LAPACKE_ztprfb( int matrix_order, char side, char trans, char direct, + char storev, lapack_int m, lapack_int n, + lapack_int k, lapack_int l, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_int myldwork ); + +lapack_int LAPACKE_sgemqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const float* v, lapack_int ldv, + const float* t, lapack_int ldt, float* c, + lapack_int ldc, float* work ); +lapack_int LAPACKE_dgemqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const double* v, lapack_int ldv, + const double* t, lapack_int ldt, double* c, + lapack_int ldc, double* work ); +lapack_int LAPACKE_cgemqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const lapack_complex_float* v, + lapack_int ldv, const lapack_complex_float* t, + lapack_int ldt, lapack_complex_float* c, + lapack_int ldc, lapack_complex_float* work ); +lapack_int LAPACKE_zgemqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int nb, const lapack_complex_double* v, + lapack_int ldv, const lapack_complex_double* t, + lapack_int ldt, lapack_complex_double* c, + lapack_int ldc, lapack_complex_double* work ); + +lapack_int LAPACKE_sgeqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, float* a, lapack_int lda, + float* t, lapack_int ldt, float* work ); +lapack_int LAPACKE_dgeqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, double* a, lapack_int lda, + double* t, lapack_int ldt, double* work ); +lapack_int LAPACKE_cgeqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, lapack_complex_float* a, + lapack_int lda, lapack_complex_float* t, + lapack_int ldt, lapack_complex_float* work ); +lapack_int LAPACKE_zgeqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int nb, lapack_complex_double* a, + lapack_int lda, lapack_complex_double* t, + lapack_int ldt, lapack_complex_double* work ); + +lapack_int LAPACKE_sgeqrt2_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dgeqrt2_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* t, + lapack_int ldt ); +lapack_int LAPACKE_cgeqrt2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zgeqrt2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_sgeqrt3_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* t, + lapack_int ldt ); +lapack_int LAPACKE_dgeqrt3_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* t, + lapack_int ldt ); +lapack_int LAPACKE_cgeqrt3_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_zgeqrt3_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_stpmqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, const float* v, + lapack_int ldv, const float* t, lapack_int ldt, + float* a, lapack_int lda, float* b, + lapack_int ldb, float* work ); +lapack_int LAPACKE_dtpmqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, const double* v, + lapack_int ldv, const double* t, + lapack_int ldt, double* a, lapack_int lda, + double* b, lapack_int ldb, double* work ); +lapack_int LAPACKE_ctpmqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* work ); +lapack_int LAPACKE_ztpmqrt_work( int matrix_order, char side, char trans, + lapack_int m, lapack_int n, lapack_int k, + lapack_int l, lapack_int nb, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* work ); + +lapack_int LAPACKE_dtpqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, double* a, + lapack_int lda, double* b, lapack_int ldb, + double* t, lapack_int ldt, double* work ); +lapack_int LAPACKE_ctpqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* t, + lapack_complex_float* b, lapack_int ldb, + lapack_int ldt, lapack_complex_float* work ); +lapack_int LAPACKE_ztpqrt_work( int matrix_order, lapack_int m, lapack_int n, + lapack_int l, lapack_int nb, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* work ); + +lapack_int LAPACKE_stpqrt2_work( int matrix_order, lapack_int m, lapack_int n, + float* a, lapack_int lda, float* b, + lapack_int ldb, float* t, lapack_int ldt ); +lapack_int LAPACKE_dtpqrt2_work( int matrix_order, lapack_int m, lapack_int n, + double* a, lapack_int lda, double* b, + lapack_int ldb, double* t, lapack_int ldt ); +lapack_int LAPACKE_ctpqrt2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + lapack_complex_float* t, lapack_int ldt ); +lapack_int LAPACKE_ztpqrt2_work( int matrix_order, lapack_int m, lapack_int n, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + lapack_complex_double* t, lapack_int ldt ); + +lapack_int LAPACKE_stprfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, lapack_int l, + const float* v, lapack_int ldv, const float* t, + lapack_int ldt, float* a, lapack_int lda, + float* b, lapack_int ldb, const float* mywork, + lapack_int myldwork ); +lapack_int LAPACKE_dtprfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, lapack_int l, + const double* v, lapack_int ldv, + const double* t, lapack_int ldt, double* a, + lapack_int lda, double* b, lapack_int ldb, + const double* mywork, lapack_int myldwork ); +lapack_int LAPACKE_ctprfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, lapack_int l, + const lapack_complex_float* v, lapack_int ldv, + const lapack_complex_float* t, lapack_int ldt, + lapack_complex_float* a, lapack_int lda, + lapack_complex_float* b, lapack_int ldb, + const float* mywork, lapack_int myldwork ); +lapack_int LAPACKE_ztprfb_work( int matrix_order, char side, char trans, + char direct, char storev, lapack_int m, + lapack_int n, lapack_int k, lapack_int l, + const lapack_complex_double* v, lapack_int ldv, + const lapack_complex_double* t, lapack_int ldt, + lapack_complex_double* a, lapack_int lda, + lapack_complex_double* b, lapack_int ldb, + const double* mywork, lapack_int myldwork ); +//LAPACK 3.X.X +lapack_int LAPACKE_csyr( int matrix_order, char uplo, lapack_int n, + lapack_complex_float alpha, + const lapack_complex_float* x, lapack_int incx, + lapack_complex_float* a, lapack_int lda ); +lapack_int LAPACKE_zsyr( int matrix_order, char uplo, lapack_int n, + lapack_complex_double alpha, + const lapack_complex_double* x, lapack_int incx, + lapack_complex_double* a, lapack_int lda ); + +lapack_int LAPACKE_csyr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_float alpha, + const lapack_complex_float* x, + lapack_int incx, lapack_complex_float* a, + lapack_int lda ); +lapack_int LAPACKE_zsyr_work( int matrix_order, char uplo, lapack_int n, + lapack_complex_double alpha, + const lapack_complex_double* x, + lapack_int incx, lapack_complex_double* a, + lapack_int lda ); + + + +#define LAPACK_sgetrf LAPACK_GLOBAL(sgetrf,SGETRF) +#define LAPACK_dgetrf LAPACK_GLOBAL(dgetrf,DGETRF) +#define LAPACK_cgetrf LAPACK_GLOBAL(cgetrf,CGETRF) +#define LAPACK_zgetrf LAPACK_GLOBAL(zgetrf,ZGETRF) +#define LAPACK_sgbtrf LAPACK_GLOBAL(sgbtrf,SGBTRF) +#define LAPACK_dgbtrf LAPACK_GLOBAL(dgbtrf,DGBTRF) +#define LAPACK_cgbtrf LAPACK_GLOBAL(cgbtrf,CGBTRF) +#define LAPACK_zgbtrf LAPACK_GLOBAL(zgbtrf,ZGBTRF) +#define LAPACK_sgttrf LAPACK_GLOBAL(sgttrf,SGTTRF) +#define LAPACK_dgttrf LAPACK_GLOBAL(dgttrf,DGTTRF) +#define LAPACK_cgttrf LAPACK_GLOBAL(cgttrf,CGTTRF) +#define LAPACK_zgttrf LAPACK_GLOBAL(zgttrf,ZGTTRF) +#define LAPACK_spotrf LAPACK_GLOBAL(spotrf,SPOTRF) +#define LAPACK_dpotrf LAPACK_GLOBAL(dpotrf,DPOTRF) +#define LAPACK_cpotrf LAPACK_GLOBAL(cpotrf,CPOTRF) +#define LAPACK_zpotrf LAPACK_GLOBAL(zpotrf,ZPOTRF) +#define LAPACK_dpstrf LAPACK_GLOBAL(dpstrf,DPSTRF) +#define LAPACK_spstrf LAPACK_GLOBAL(spstrf,SPSTRF) +#define LAPACK_zpstrf LAPACK_GLOBAL(zpstrf,ZPSTRF) +#define LAPACK_cpstrf LAPACK_GLOBAL(cpstrf,CPSTRF) +#define LAPACK_dpftrf LAPACK_GLOBAL(dpftrf,DPFTRF) +#define LAPACK_spftrf LAPACK_GLOBAL(spftrf,SPFTRF) +#define LAPACK_zpftrf LAPACK_GLOBAL(zpftrf,ZPFTRF) +#define LAPACK_cpftrf LAPACK_GLOBAL(cpftrf,CPFTRF) +#define LAPACK_spptrf LAPACK_GLOBAL(spptrf,SPPTRF) +#define LAPACK_dpptrf LAPACK_GLOBAL(dpptrf,DPPTRF) +#define LAPACK_cpptrf LAPACK_GLOBAL(cpptrf,CPPTRF) +#define LAPACK_zpptrf LAPACK_GLOBAL(zpptrf,ZPPTRF) +#define LAPACK_spbtrf LAPACK_GLOBAL(spbtrf,SPBTRF) +#define LAPACK_dpbtrf LAPACK_GLOBAL(dpbtrf,DPBTRF) +#define LAPACK_cpbtrf LAPACK_GLOBAL(cpbtrf,CPBTRF) +#define LAPACK_zpbtrf LAPACK_GLOBAL(zpbtrf,ZPBTRF) +#define LAPACK_spttrf LAPACK_GLOBAL(spttrf,SPTTRF) +#define LAPACK_dpttrf LAPACK_GLOBAL(dpttrf,DPTTRF) +#define LAPACK_cpttrf LAPACK_GLOBAL(cpttrf,CPTTRF) +#define LAPACK_zpttrf LAPACK_GLOBAL(zpttrf,ZPTTRF) +#define LAPACK_ssytrf LAPACK_GLOBAL(ssytrf,SSYTRF) +#define LAPACK_dsytrf LAPACK_GLOBAL(dsytrf,DSYTRF) +#define LAPACK_csytrf LAPACK_GLOBAL(csytrf,CSYTRF) +#define LAPACK_zsytrf LAPACK_GLOBAL(zsytrf,ZSYTRF) +#define LAPACK_chetrf LAPACK_GLOBAL(chetrf,CHETRF) +#define LAPACK_zhetrf LAPACK_GLOBAL(zhetrf,ZHETRF) +#define LAPACK_ssptrf LAPACK_GLOBAL(ssptrf,SSPTRF) +#define LAPACK_dsptrf LAPACK_GLOBAL(dsptrf,DSPTRF) +#define LAPACK_csptrf LAPACK_GLOBAL(csptrf,CSPTRF) +#define LAPACK_zsptrf LAPACK_GLOBAL(zsptrf,ZSPTRF) +#define LAPACK_chptrf LAPACK_GLOBAL(chptrf,CHPTRF) +#define LAPACK_zhptrf LAPACK_GLOBAL(zhptrf,ZHPTRF) +#define LAPACK_sgetrs LAPACK_GLOBAL(sgetrs,SGETRS) +#define LAPACK_dgetrs LAPACK_GLOBAL(dgetrs,DGETRS) +#define LAPACK_cgetrs LAPACK_GLOBAL(cgetrs,CGETRS) +#define LAPACK_zgetrs LAPACK_GLOBAL(zgetrs,ZGETRS) +#define LAPACK_sgbtrs LAPACK_GLOBAL(sgbtrs,SGBTRS) +#define LAPACK_dgbtrs LAPACK_GLOBAL(dgbtrs,DGBTRS) +#define LAPACK_cgbtrs LAPACK_GLOBAL(cgbtrs,CGBTRS) +#define LAPACK_zgbtrs LAPACK_GLOBAL(zgbtrs,ZGBTRS) +#define LAPACK_sgttrs LAPACK_GLOBAL(sgttrs,SGTTRS) +#define LAPACK_dgttrs LAPACK_GLOBAL(dgttrs,DGTTRS) +#define LAPACK_cgttrs LAPACK_GLOBAL(cgttrs,CGTTRS) +#define LAPACK_zgttrs LAPACK_GLOBAL(zgttrs,ZGTTRS) +#define LAPACK_spotrs LAPACK_GLOBAL(spotrs,SPOTRS) +#define LAPACK_dpotrs LAPACK_GLOBAL(dpotrs,DPOTRS) +#define LAPACK_cpotrs LAPACK_GLOBAL(cpotrs,CPOTRS) +#define LAPACK_zpotrs LAPACK_GLOBAL(zpotrs,ZPOTRS) +#define LAPACK_dpftrs LAPACK_GLOBAL(dpftrs,DPFTRS) +#define LAPACK_spftrs LAPACK_GLOBAL(spftrs,SPFTRS) +#define LAPACK_zpftrs LAPACK_GLOBAL(zpftrs,ZPFTRS) +#define LAPACK_cpftrs LAPACK_GLOBAL(cpftrs,CPFTRS) +#define LAPACK_spptrs LAPACK_GLOBAL(spptrs,SPPTRS) +#define LAPACK_dpptrs LAPACK_GLOBAL(dpptrs,DPPTRS) +#define LAPACK_cpptrs LAPACK_GLOBAL(cpptrs,CPPTRS) +#define LAPACK_zpptrs LAPACK_GLOBAL(zpptrs,ZPPTRS) +#define LAPACK_spbtrs LAPACK_GLOBAL(spbtrs,SPBTRS) +#define LAPACK_dpbtrs LAPACK_GLOBAL(dpbtrs,DPBTRS) +#define LAPACK_cpbtrs LAPACK_GLOBAL(cpbtrs,CPBTRS) +#define LAPACK_zpbtrs LAPACK_GLOBAL(zpbtrs,ZPBTRS) +#define LAPACK_spttrs LAPACK_GLOBAL(spttrs,SPTTRS) +#define LAPACK_dpttrs LAPACK_GLOBAL(dpttrs,DPTTRS) +#define LAPACK_cpttrs LAPACK_GLOBAL(cpttrs,CPTTRS) +#define LAPACK_zpttrs LAPACK_GLOBAL(zpttrs,ZPTTRS) +#define LAPACK_ssytrs LAPACK_GLOBAL(ssytrs,SSYTRS) +#define LAPACK_dsytrs LAPACK_GLOBAL(dsytrs,DSYTRS) +#define LAPACK_csytrs LAPACK_GLOBAL(csytrs,CSYTRS) +#define LAPACK_zsytrs LAPACK_GLOBAL(zsytrs,ZSYTRS) +#define LAPACK_chetrs LAPACK_GLOBAL(chetrs,CHETRS) +#define LAPACK_zhetrs LAPACK_GLOBAL(zhetrs,ZHETRS) +#define LAPACK_ssptrs LAPACK_GLOBAL(ssptrs,SSPTRS) +#define LAPACK_dsptrs LAPACK_GLOBAL(dsptrs,DSPTRS) +#define LAPACK_csptrs LAPACK_GLOBAL(csptrs,CSPTRS) +#define LAPACK_zsptrs LAPACK_GLOBAL(zsptrs,ZSPTRS) +#define LAPACK_chptrs LAPACK_GLOBAL(chptrs,CHPTRS) +#define LAPACK_zhptrs LAPACK_GLOBAL(zhptrs,ZHPTRS) +#define LAPACK_strtrs LAPACK_GLOBAL(strtrs,STRTRS) +#define LAPACK_dtrtrs LAPACK_GLOBAL(dtrtrs,DTRTRS) +#define LAPACK_ctrtrs LAPACK_GLOBAL(ctrtrs,CTRTRS) +#define LAPACK_ztrtrs LAPACK_GLOBAL(ztrtrs,ZTRTRS) +#define LAPACK_stptrs LAPACK_GLOBAL(stptrs,STPTRS) +#define LAPACK_dtptrs LAPACK_GLOBAL(dtptrs,DTPTRS) +#define LAPACK_ctptrs LAPACK_GLOBAL(ctptrs,CTPTRS) +#define LAPACK_ztptrs LAPACK_GLOBAL(ztptrs,ZTPTRS) +#define LAPACK_stbtrs LAPACK_GLOBAL(stbtrs,STBTRS) +#define LAPACK_dtbtrs LAPACK_GLOBAL(dtbtrs,DTBTRS) +#define LAPACK_ctbtrs LAPACK_GLOBAL(ctbtrs,CTBTRS) +#define LAPACK_ztbtrs LAPACK_GLOBAL(ztbtrs,ZTBTRS) +#define LAPACK_sgecon LAPACK_GLOBAL(sgecon,SGECON) +#define LAPACK_dgecon LAPACK_GLOBAL(dgecon,DGECON) +#define LAPACK_cgecon LAPACK_GLOBAL(cgecon,CGECON) +#define LAPACK_zgecon LAPACK_GLOBAL(zgecon,ZGECON) +#define LAPACK_sgbcon LAPACK_GLOBAL(sgbcon,SGBCON) +#define LAPACK_dgbcon LAPACK_GLOBAL(dgbcon,DGBCON) +#define LAPACK_cgbcon LAPACK_GLOBAL(cgbcon,CGBCON) +#define LAPACK_zgbcon LAPACK_GLOBAL(zgbcon,ZGBCON) +#define LAPACK_sgtcon LAPACK_GLOBAL(sgtcon,SGTCON) +#define LAPACK_dgtcon LAPACK_GLOBAL(dgtcon,DGTCON) +#define LAPACK_cgtcon LAPACK_GLOBAL(cgtcon,CGTCON) +#define LAPACK_zgtcon LAPACK_GLOBAL(zgtcon,ZGTCON) +#define LAPACK_spocon LAPACK_GLOBAL(spocon,SPOCON) +#define LAPACK_dpocon LAPACK_GLOBAL(dpocon,DPOCON) +#define LAPACK_cpocon LAPACK_GLOBAL(cpocon,CPOCON) +#define LAPACK_zpocon LAPACK_GLOBAL(zpocon,ZPOCON) +#define LAPACK_sppcon LAPACK_GLOBAL(sppcon,SPPCON) +#define LAPACK_dppcon LAPACK_GLOBAL(dppcon,DPPCON) +#define LAPACK_cppcon LAPACK_GLOBAL(cppcon,CPPCON) +#define LAPACK_zppcon LAPACK_GLOBAL(zppcon,ZPPCON) +#define LAPACK_spbcon LAPACK_GLOBAL(spbcon,SPBCON) +#define LAPACK_dpbcon LAPACK_GLOBAL(dpbcon,DPBCON) +#define LAPACK_cpbcon LAPACK_GLOBAL(cpbcon,CPBCON) +#define LAPACK_zpbcon LAPACK_GLOBAL(zpbcon,ZPBCON) +#define LAPACK_sptcon LAPACK_GLOBAL(sptcon,SPTCON) +#define LAPACK_dptcon LAPACK_GLOBAL(dptcon,DPTCON) +#define LAPACK_cptcon LAPACK_GLOBAL(cptcon,CPTCON) +#define LAPACK_zptcon LAPACK_GLOBAL(zptcon,ZPTCON) +#define LAPACK_ssycon LAPACK_GLOBAL(ssycon,SSYCON) +#define LAPACK_dsycon LAPACK_GLOBAL(dsycon,DSYCON) +#define LAPACK_csycon LAPACK_GLOBAL(csycon,CSYCON) +#define LAPACK_zsycon LAPACK_GLOBAL(zsycon,ZSYCON) +#define LAPACK_checon LAPACK_GLOBAL(checon,CHECON) +#define LAPACK_zhecon LAPACK_GLOBAL(zhecon,ZHECON) +#define LAPACK_sspcon LAPACK_GLOBAL(sspcon,SSPCON) +#define LAPACK_dspcon LAPACK_GLOBAL(dspcon,DSPCON) +#define LAPACK_cspcon LAPACK_GLOBAL(cspcon,CSPCON) +#define LAPACK_zspcon LAPACK_GLOBAL(zspcon,ZSPCON) +#define LAPACK_chpcon LAPACK_GLOBAL(chpcon,CHPCON) +#define LAPACK_zhpcon LAPACK_GLOBAL(zhpcon,ZHPCON) +#define LAPACK_strcon LAPACK_GLOBAL(strcon,STRCON) +#define LAPACK_dtrcon LAPACK_GLOBAL(dtrcon,DTRCON) +#define LAPACK_ctrcon LAPACK_GLOBAL(ctrcon,CTRCON) +#define LAPACK_ztrcon LAPACK_GLOBAL(ztrcon,ZTRCON) +#define LAPACK_stpcon LAPACK_GLOBAL(stpcon,STPCON) +#define LAPACK_dtpcon LAPACK_GLOBAL(dtpcon,DTPCON) +#define LAPACK_ctpcon LAPACK_GLOBAL(ctpcon,CTPCON) +#define LAPACK_ztpcon LAPACK_GLOBAL(ztpcon,ZTPCON) +#define LAPACK_stbcon LAPACK_GLOBAL(stbcon,STBCON) +#define LAPACK_dtbcon LAPACK_GLOBAL(dtbcon,DTBCON) +#define LAPACK_ctbcon LAPACK_GLOBAL(ctbcon,CTBCON) +#define LAPACK_ztbcon LAPACK_GLOBAL(ztbcon,ZTBCON) +#define LAPACK_sgerfs LAPACK_GLOBAL(sgerfs,SGERFS) +#define LAPACK_dgerfs LAPACK_GLOBAL(dgerfs,DGERFS) +#define LAPACK_cgerfs LAPACK_GLOBAL(cgerfs,CGERFS) +#define LAPACK_zgerfs LAPACK_GLOBAL(zgerfs,ZGERFS) +#define LAPACK_dgerfsx LAPACK_GLOBAL(dgerfsx,DGERFSX) +#define LAPACK_sgerfsx LAPACK_GLOBAL(sgerfsx,SGERFSX) +#define LAPACK_zgerfsx LAPACK_GLOBAL(zgerfsx,ZGERFSX) +#define LAPACK_cgerfsx LAPACK_GLOBAL(cgerfsx,CGERFSX) +#define LAPACK_sgbrfs LAPACK_GLOBAL(sgbrfs,SGBRFS) +#define LAPACK_dgbrfs LAPACK_GLOBAL(dgbrfs,DGBRFS) +#define LAPACK_cgbrfs LAPACK_GLOBAL(cgbrfs,CGBRFS) +#define LAPACK_zgbrfs LAPACK_GLOBAL(zgbrfs,ZGBRFS) +#define LAPACK_dgbrfsx LAPACK_GLOBAL(dgbrfsx,DGBRFSX) +#define LAPACK_sgbrfsx LAPACK_GLOBAL(sgbrfsx,SGBRFSX) +#define LAPACK_zgbrfsx LAPACK_GLOBAL(zgbrfsx,ZGBRFSX) +#define LAPACK_cgbrfsx LAPACK_GLOBAL(cgbrfsx,CGBRFSX) +#define LAPACK_sgtrfs LAPACK_GLOBAL(sgtrfs,SGTRFS) +#define LAPACK_dgtrfs LAPACK_GLOBAL(dgtrfs,DGTRFS) +#define LAPACK_cgtrfs LAPACK_GLOBAL(cgtrfs,CGTRFS) +#define LAPACK_zgtrfs LAPACK_GLOBAL(zgtrfs,ZGTRFS) +#define LAPACK_sporfs LAPACK_GLOBAL(sporfs,SPORFS) +#define LAPACK_dporfs LAPACK_GLOBAL(dporfs,DPORFS) +#define LAPACK_cporfs LAPACK_GLOBAL(cporfs,CPORFS) +#define LAPACK_zporfs LAPACK_GLOBAL(zporfs,ZPORFS) +#define LAPACK_dporfsx LAPACK_GLOBAL(dporfsx,DPORFSX) +#define LAPACK_sporfsx LAPACK_GLOBAL(sporfsx,SPORFSX) +#define LAPACK_zporfsx LAPACK_GLOBAL(zporfsx,ZPORFSX) +#define LAPACK_cporfsx LAPACK_GLOBAL(cporfsx,CPORFSX) +#define LAPACK_spprfs LAPACK_GLOBAL(spprfs,SPPRFS) +#define LAPACK_dpprfs LAPACK_GLOBAL(dpprfs,DPPRFS) +#define LAPACK_cpprfs LAPACK_GLOBAL(cpprfs,CPPRFS) +#define LAPACK_zpprfs LAPACK_GLOBAL(zpprfs,ZPPRFS) +#define LAPACK_spbrfs LAPACK_GLOBAL(spbrfs,SPBRFS) +#define LAPACK_dpbrfs LAPACK_GLOBAL(dpbrfs,DPBRFS) +#define LAPACK_cpbrfs LAPACK_GLOBAL(cpbrfs,CPBRFS) +#define LAPACK_zpbrfs LAPACK_GLOBAL(zpbrfs,ZPBRFS) +#define LAPACK_sptrfs LAPACK_GLOBAL(sptrfs,SPTRFS) +#define LAPACK_dptrfs LAPACK_GLOBAL(dptrfs,DPTRFS) +#define LAPACK_cptrfs LAPACK_GLOBAL(cptrfs,CPTRFS) +#define LAPACK_zptrfs LAPACK_GLOBAL(zptrfs,ZPTRFS) +#define LAPACK_ssyrfs LAPACK_GLOBAL(ssyrfs,SSYRFS) +#define LAPACK_dsyrfs LAPACK_GLOBAL(dsyrfs,DSYRFS) +#define LAPACK_csyrfs LAPACK_GLOBAL(csyrfs,CSYRFS) +#define LAPACK_zsyrfs LAPACK_GLOBAL(zsyrfs,ZSYRFS) +#define LAPACK_dsyrfsx LAPACK_GLOBAL(dsyrfsx,DSYRFSX) +#define LAPACK_ssyrfsx LAPACK_GLOBAL(ssyrfsx,SSYRFSX) +#define LAPACK_zsyrfsx LAPACK_GLOBAL(zsyrfsx,ZSYRFSX) +#define LAPACK_csyrfsx LAPACK_GLOBAL(csyrfsx,CSYRFSX) +#define LAPACK_cherfs LAPACK_GLOBAL(cherfs,CHERFS) +#define LAPACK_zherfs LAPACK_GLOBAL(zherfs,ZHERFS) +#define LAPACK_zherfsx LAPACK_GLOBAL(zherfsx,ZHERFSX) +#define LAPACK_cherfsx LAPACK_GLOBAL(cherfsx,CHERFSX) +#define LAPACK_ssprfs LAPACK_GLOBAL(ssprfs,SSPRFS) +#define LAPACK_dsprfs LAPACK_GLOBAL(dsprfs,DSPRFS) +#define LAPACK_csprfs LAPACK_GLOBAL(csprfs,CSPRFS) +#define LAPACK_zsprfs LAPACK_GLOBAL(zsprfs,ZSPRFS) +#define LAPACK_chprfs LAPACK_GLOBAL(chprfs,CHPRFS) +#define LAPACK_zhprfs LAPACK_GLOBAL(zhprfs,ZHPRFS) +#define LAPACK_strrfs LAPACK_GLOBAL(strrfs,STRRFS) +#define LAPACK_dtrrfs LAPACK_GLOBAL(dtrrfs,DTRRFS) +#define LAPACK_ctrrfs LAPACK_GLOBAL(ctrrfs,CTRRFS) +#define LAPACK_ztrrfs LAPACK_GLOBAL(ztrrfs,ZTRRFS) +#define LAPACK_stprfs LAPACK_GLOBAL(stprfs,STPRFS) +#define LAPACK_dtprfs LAPACK_GLOBAL(dtprfs,DTPRFS) +#define LAPACK_ctprfs LAPACK_GLOBAL(ctprfs,CTPRFS) +#define LAPACK_ztprfs LAPACK_GLOBAL(ztprfs,ZTPRFS) +#define LAPACK_stbrfs LAPACK_GLOBAL(stbrfs,STBRFS) +#define LAPACK_dtbrfs LAPACK_GLOBAL(dtbrfs,DTBRFS) +#define LAPACK_ctbrfs LAPACK_GLOBAL(ctbrfs,CTBRFS) +#define LAPACK_ztbrfs LAPACK_GLOBAL(ztbrfs,ZTBRFS) +#define LAPACK_sgetri LAPACK_GLOBAL(sgetri,SGETRI) +#define LAPACK_dgetri LAPACK_GLOBAL(dgetri,DGETRI) +#define LAPACK_cgetri LAPACK_GLOBAL(cgetri,CGETRI) +#define LAPACK_zgetri LAPACK_GLOBAL(zgetri,ZGETRI) +#define LAPACK_spotri LAPACK_GLOBAL(spotri,SPOTRI) +#define LAPACK_dpotri LAPACK_GLOBAL(dpotri,DPOTRI) +#define LAPACK_cpotri LAPACK_GLOBAL(cpotri,CPOTRI) +#define LAPACK_zpotri LAPACK_GLOBAL(zpotri,ZPOTRI) +#define LAPACK_dpftri LAPACK_GLOBAL(dpftri,DPFTRI) +#define LAPACK_spftri LAPACK_GLOBAL(spftri,SPFTRI) +#define LAPACK_zpftri LAPACK_GLOBAL(zpftri,ZPFTRI) +#define LAPACK_cpftri LAPACK_GLOBAL(cpftri,CPFTRI) +#define LAPACK_spptri LAPACK_GLOBAL(spptri,SPPTRI) +#define LAPACK_dpptri LAPACK_GLOBAL(dpptri,DPPTRI) +#define LAPACK_cpptri LAPACK_GLOBAL(cpptri,CPPTRI) +#define LAPACK_zpptri LAPACK_GLOBAL(zpptri,ZPPTRI) +#define LAPACK_ssytri LAPACK_GLOBAL(ssytri,SSYTRI) +#define LAPACK_dsytri LAPACK_GLOBAL(dsytri,DSYTRI) +#define LAPACK_csytri LAPACK_GLOBAL(csytri,CSYTRI) +#define LAPACK_zsytri LAPACK_GLOBAL(zsytri,ZSYTRI) +#define LAPACK_chetri LAPACK_GLOBAL(chetri,CHETRI) +#define LAPACK_zhetri LAPACK_GLOBAL(zhetri,ZHETRI) +#define LAPACK_ssptri LAPACK_GLOBAL(ssptri,SSPTRI) +#define LAPACK_dsptri LAPACK_GLOBAL(dsptri,DSPTRI) +#define LAPACK_csptri LAPACK_GLOBAL(csptri,CSPTRI) +#define LAPACK_zsptri LAPACK_GLOBAL(zsptri,ZSPTRI) +#define LAPACK_chptri LAPACK_GLOBAL(chptri,CHPTRI) +#define LAPACK_zhptri LAPACK_GLOBAL(zhptri,ZHPTRI) +#define LAPACK_strtri LAPACK_GLOBAL(strtri,STRTRI) +#define LAPACK_dtrtri LAPACK_GLOBAL(dtrtri,DTRTRI) +#define LAPACK_ctrtri LAPACK_GLOBAL(ctrtri,CTRTRI) +#define LAPACK_ztrtri LAPACK_GLOBAL(ztrtri,ZTRTRI) +#define LAPACK_dtftri LAPACK_GLOBAL(dtftri,DTFTRI) +#define LAPACK_stftri LAPACK_GLOBAL(stftri,STFTRI) +#define LAPACK_ztftri LAPACK_GLOBAL(ztftri,ZTFTRI) +#define LAPACK_ctftri LAPACK_GLOBAL(ctftri,CTFTRI) +#define LAPACK_stptri LAPACK_GLOBAL(stptri,STPTRI) +#define LAPACK_dtptri LAPACK_GLOBAL(dtptri,DTPTRI) +#define LAPACK_ctptri LAPACK_GLOBAL(ctptri,CTPTRI) +#define LAPACK_ztptri LAPACK_GLOBAL(ztptri,ZTPTRI) +#define LAPACK_sgeequ LAPACK_GLOBAL(sgeequ,SGEEQU) +#define LAPACK_dgeequ LAPACK_GLOBAL(dgeequ,DGEEQU) +#define LAPACK_cgeequ LAPACK_GLOBAL(cgeequ,CGEEQU) +#define LAPACK_zgeequ LAPACK_GLOBAL(zgeequ,ZGEEQU) +#define LAPACK_dgeequb LAPACK_GLOBAL(dgeequb,DGEEQUB) +#define LAPACK_sgeequb LAPACK_GLOBAL(sgeequb,SGEEQUB) +#define LAPACK_zgeequb LAPACK_GLOBAL(zgeequb,ZGEEQUB) +#define LAPACK_cgeequb LAPACK_GLOBAL(cgeequb,CGEEQUB) +#define LAPACK_sgbequ LAPACK_GLOBAL(sgbequ,SGBEQU) +#define LAPACK_dgbequ LAPACK_GLOBAL(dgbequ,DGBEQU) +#define LAPACK_cgbequ LAPACK_GLOBAL(cgbequ,CGBEQU) +#define LAPACK_zgbequ LAPACK_GLOBAL(zgbequ,ZGBEQU) +#define LAPACK_dgbequb LAPACK_GLOBAL(dgbequb,DGBEQUB) +#define LAPACK_sgbequb LAPACK_GLOBAL(sgbequb,SGBEQUB) +#define LAPACK_zgbequb LAPACK_GLOBAL(zgbequb,ZGBEQUB) +#define LAPACK_cgbequb LAPACK_GLOBAL(cgbequb,CGBEQUB) +#define LAPACK_spoequ LAPACK_GLOBAL(spoequ,SPOEQU) +#define LAPACK_dpoequ LAPACK_GLOBAL(dpoequ,DPOEQU) +#define LAPACK_cpoequ LAPACK_GLOBAL(cpoequ,CPOEQU) +#define LAPACK_zpoequ LAPACK_GLOBAL(zpoequ,ZPOEQU) +#define LAPACK_dpoequb LAPACK_GLOBAL(dpoequb,DPOEQUB) +#define LAPACK_spoequb LAPACK_GLOBAL(spoequb,SPOEQUB) +#define LAPACK_zpoequb LAPACK_GLOBAL(zpoequb,ZPOEQUB) +#define LAPACK_cpoequb LAPACK_GLOBAL(cpoequb,CPOEQUB) +#define LAPACK_sppequ LAPACK_GLOBAL(sppequ,SPPEQU) +#define LAPACK_dppequ LAPACK_GLOBAL(dppequ,DPPEQU) +#define LAPACK_cppequ LAPACK_GLOBAL(cppequ,CPPEQU) +#define LAPACK_zppequ LAPACK_GLOBAL(zppequ,ZPPEQU) +#define LAPACK_spbequ LAPACK_GLOBAL(spbequ,SPBEQU) +#define LAPACK_dpbequ LAPACK_GLOBAL(dpbequ,DPBEQU) +#define LAPACK_cpbequ LAPACK_GLOBAL(cpbequ,CPBEQU) +#define LAPACK_zpbequ LAPACK_GLOBAL(zpbequ,ZPBEQU) +#define LAPACK_dsyequb LAPACK_GLOBAL(dsyequb,DSYEQUB) +#define LAPACK_ssyequb LAPACK_GLOBAL(ssyequb,SSYEQUB) +#define LAPACK_zsyequb LAPACK_GLOBAL(zsyequb,ZSYEQUB) +#define LAPACK_csyequb LAPACK_GLOBAL(csyequb,CSYEQUB) +#define LAPACK_zheequb LAPACK_GLOBAL(zheequb,ZHEEQUB) +#define LAPACK_cheequb LAPACK_GLOBAL(cheequb,CHEEQUB) +#define LAPACK_sgesv LAPACK_GLOBAL(sgesv,SGESV) +#define LAPACK_dgesv LAPACK_GLOBAL(dgesv,DGESV) +#define LAPACK_cgesv LAPACK_GLOBAL(cgesv,CGESV) +#define LAPACK_zgesv LAPACK_GLOBAL(zgesv,ZGESV) +#define LAPACK_dsgesv LAPACK_GLOBAL(dsgesv,DSGESV) +#define LAPACK_zcgesv LAPACK_GLOBAL(zcgesv,ZCGESV) +#define LAPACK_sgesvx LAPACK_GLOBAL(sgesvx,SGESVX) +#define LAPACK_dgesvx LAPACK_GLOBAL(dgesvx,DGESVX) +#define LAPACK_cgesvx LAPACK_GLOBAL(cgesvx,CGESVX) +#define LAPACK_zgesvx LAPACK_GLOBAL(zgesvx,ZGESVX) +#define LAPACK_dgesvxx LAPACK_GLOBAL(dgesvxx,DGESVXX) +#define LAPACK_sgesvxx LAPACK_GLOBAL(sgesvxx,SGESVXX) +#define LAPACK_zgesvxx LAPACK_GLOBAL(zgesvxx,ZGESVXX) +#define LAPACK_cgesvxx LAPACK_GLOBAL(cgesvxx,CGESVXX) +#define LAPACK_sgbsv LAPACK_GLOBAL(sgbsv,SGBSV) +#define LAPACK_dgbsv LAPACK_GLOBAL(dgbsv,DGBSV) +#define LAPACK_cgbsv LAPACK_GLOBAL(cgbsv,CGBSV) +#define LAPACK_zgbsv LAPACK_GLOBAL(zgbsv,ZGBSV) +#define LAPACK_sgbsvx LAPACK_GLOBAL(sgbsvx,SGBSVX) +#define LAPACK_dgbsvx LAPACK_GLOBAL(dgbsvx,DGBSVX) +#define LAPACK_cgbsvx LAPACK_GLOBAL(cgbsvx,CGBSVX) +#define LAPACK_zgbsvx LAPACK_GLOBAL(zgbsvx,ZGBSVX) +#define LAPACK_dgbsvxx LAPACK_GLOBAL(dgbsvxx,DGBSVXX) +#define LAPACK_sgbsvxx LAPACK_GLOBAL(sgbsvxx,SGBSVXX) +#define LAPACK_zgbsvxx LAPACK_GLOBAL(zgbsvxx,ZGBSVXX) +#define LAPACK_cgbsvxx LAPACK_GLOBAL(cgbsvxx,CGBSVXX) +#define LAPACK_sgtsv LAPACK_GLOBAL(sgtsv,SGTSV) +#define LAPACK_dgtsv LAPACK_GLOBAL(dgtsv,DGTSV) +#define LAPACK_cgtsv LAPACK_GLOBAL(cgtsv,CGTSV) +#define LAPACK_zgtsv LAPACK_GLOBAL(zgtsv,ZGTSV) +#define LAPACK_sgtsvx LAPACK_GLOBAL(sgtsvx,SGTSVX) +#define LAPACK_dgtsvx LAPACK_GLOBAL(dgtsvx,DGTSVX) +#define LAPACK_cgtsvx LAPACK_GLOBAL(cgtsvx,CGTSVX) +#define LAPACK_zgtsvx LAPACK_GLOBAL(zgtsvx,ZGTSVX) +#define LAPACK_sposv LAPACK_GLOBAL(sposv,SPOSV) +#define LAPACK_dposv LAPACK_GLOBAL(dposv,DPOSV) +#define LAPACK_cposv LAPACK_GLOBAL(cposv,CPOSV) +#define LAPACK_zposv LAPACK_GLOBAL(zposv,ZPOSV) +#define LAPACK_dsposv LAPACK_GLOBAL(dsposv,DSPOSV) +#define LAPACK_zcposv LAPACK_GLOBAL(zcposv,ZCPOSV) +#define LAPACK_sposvx LAPACK_GLOBAL(sposvx,SPOSVX) +#define LAPACK_dposvx LAPACK_GLOBAL(dposvx,DPOSVX) +#define LAPACK_cposvx LAPACK_GLOBAL(cposvx,CPOSVX) +#define LAPACK_zposvx LAPACK_GLOBAL(zposvx,ZPOSVX) +#define LAPACK_dposvxx LAPACK_GLOBAL(dposvxx,DPOSVXX) +#define LAPACK_sposvxx LAPACK_GLOBAL(sposvxx,SPOSVXX) +#define LAPACK_zposvxx LAPACK_GLOBAL(zposvxx,ZPOSVXX) +#define LAPACK_cposvxx LAPACK_GLOBAL(cposvxx,CPOSVXX) +#define LAPACK_sppsv LAPACK_GLOBAL(sppsv,SPPSV) +#define LAPACK_dppsv LAPACK_GLOBAL(dppsv,DPPSV) +#define LAPACK_cppsv LAPACK_GLOBAL(cppsv,CPPSV) +#define LAPACK_zppsv LAPACK_GLOBAL(zppsv,ZPPSV) +#define LAPACK_sppsvx LAPACK_GLOBAL(sppsvx,SPPSVX) +#define LAPACK_dppsvx LAPACK_GLOBAL(dppsvx,DPPSVX) +#define LAPACK_cppsvx LAPACK_GLOBAL(cppsvx,CPPSVX) +#define LAPACK_zppsvx LAPACK_GLOBAL(zppsvx,ZPPSVX) +#define LAPACK_spbsv LAPACK_GLOBAL(spbsv,SPBSV) +#define LAPACK_dpbsv LAPACK_GLOBAL(dpbsv,DPBSV) +#define LAPACK_cpbsv LAPACK_GLOBAL(cpbsv,CPBSV) +#define LAPACK_zpbsv LAPACK_GLOBAL(zpbsv,ZPBSV) +#define LAPACK_spbsvx LAPACK_GLOBAL(spbsvx,SPBSVX) +#define LAPACK_dpbsvx LAPACK_GLOBAL(dpbsvx,DPBSVX) +#define LAPACK_cpbsvx LAPACK_GLOBAL(cpbsvx,CPBSVX) +#define LAPACK_zpbsvx LAPACK_GLOBAL(zpbsvx,ZPBSVX) +#define LAPACK_sptsv LAPACK_GLOBAL(sptsv,SPTSV) +#define LAPACK_dptsv LAPACK_GLOBAL(dptsv,DPTSV) +#define LAPACK_cptsv LAPACK_GLOBAL(cptsv,CPTSV) +#define LAPACK_zptsv LAPACK_GLOBAL(zptsv,ZPTSV) +#define LAPACK_sptsvx LAPACK_GLOBAL(sptsvx,SPTSVX) +#define LAPACK_dptsvx LAPACK_GLOBAL(dptsvx,DPTSVX) +#define LAPACK_cptsvx LAPACK_GLOBAL(cptsvx,CPTSVX) +#define LAPACK_zptsvx LAPACK_GLOBAL(zptsvx,ZPTSVX) +#define LAPACK_ssysv LAPACK_GLOBAL(ssysv,SSYSV) +#define LAPACK_dsysv LAPACK_GLOBAL(dsysv,DSYSV) +#define LAPACK_csysv LAPACK_GLOBAL(csysv,CSYSV) +#define LAPACK_zsysv LAPACK_GLOBAL(zsysv,ZSYSV) +#define LAPACK_ssysvx LAPACK_GLOBAL(ssysvx,SSYSVX) +#define LAPACK_dsysvx LAPACK_GLOBAL(dsysvx,DSYSVX) +#define LAPACK_csysvx LAPACK_GLOBAL(csysvx,CSYSVX) +#define LAPACK_zsysvx LAPACK_GLOBAL(zsysvx,ZSYSVX) +#define LAPACK_dsysvxx LAPACK_GLOBAL(dsysvxx,DSYSVXX) +#define LAPACK_ssysvxx LAPACK_GLOBAL(ssysvxx,SSYSVXX) +#define LAPACK_zsysvxx LAPACK_GLOBAL(zsysvxx,ZSYSVXX) +#define LAPACK_csysvxx LAPACK_GLOBAL(csysvxx,CSYSVXX) +#define LAPACK_chesv LAPACK_GLOBAL(chesv,CHESV) +#define LAPACK_zhesv LAPACK_GLOBAL(zhesv,ZHESV) +#define LAPACK_chesvx LAPACK_GLOBAL(chesvx,CHESVX) +#define LAPACK_zhesvx LAPACK_GLOBAL(zhesvx,ZHESVX) +#define LAPACK_zhesvxx LAPACK_GLOBAL(zhesvxx,ZHESVXX) +#define LAPACK_chesvxx LAPACK_GLOBAL(chesvxx,CHESVXX) +#define LAPACK_sspsv LAPACK_GLOBAL(sspsv,SSPSV) +#define LAPACK_dspsv LAPACK_GLOBAL(dspsv,DSPSV) +#define LAPACK_cspsv LAPACK_GLOBAL(cspsv,CSPSV) +#define LAPACK_zspsv LAPACK_GLOBAL(zspsv,ZSPSV) +#define LAPACK_sspsvx LAPACK_GLOBAL(sspsvx,SSPSVX) +#define LAPACK_dspsvx LAPACK_GLOBAL(dspsvx,DSPSVX) +#define LAPACK_cspsvx LAPACK_GLOBAL(cspsvx,CSPSVX) +#define LAPACK_zspsvx LAPACK_GLOBAL(zspsvx,ZSPSVX) +#define LAPACK_chpsv LAPACK_GLOBAL(chpsv,CHPSV) +#define LAPACK_zhpsv LAPACK_GLOBAL(zhpsv,ZHPSV) +#define LAPACK_chpsvx LAPACK_GLOBAL(chpsvx,CHPSVX) +#define LAPACK_zhpsvx LAPACK_GLOBAL(zhpsvx,ZHPSVX) +#define LAPACK_sgeqrf LAPACK_GLOBAL(sgeqrf,SGEQRF) +#define LAPACK_dgeqrf LAPACK_GLOBAL(dgeqrf,DGEQRF) +#define LAPACK_cgeqrf LAPACK_GLOBAL(cgeqrf,CGEQRF) +#define LAPACK_zgeqrf LAPACK_GLOBAL(zgeqrf,ZGEQRF) +#define LAPACK_sgeqpf LAPACK_GLOBAL(sgeqpf,SGEQPF) +#define LAPACK_dgeqpf LAPACK_GLOBAL(dgeqpf,DGEQPF) +#define LAPACK_cgeqpf LAPACK_GLOBAL(cgeqpf,CGEQPF) +#define LAPACK_zgeqpf LAPACK_GLOBAL(zgeqpf,ZGEQPF) +#define LAPACK_sgeqp3 LAPACK_GLOBAL(sgeqp3,SGEQP3) +#define LAPACK_dgeqp3 LAPACK_GLOBAL(dgeqp3,DGEQP3) +#define LAPACK_cgeqp3 LAPACK_GLOBAL(cgeqp3,CGEQP3) +#define LAPACK_zgeqp3 LAPACK_GLOBAL(zgeqp3,ZGEQP3) +#define LAPACK_sorgqr LAPACK_GLOBAL(sorgqr,SORGQR) +#define LAPACK_dorgqr LAPACK_GLOBAL(dorgqr,DORGQR) +#define LAPACK_sormqr LAPACK_GLOBAL(sormqr,SORMQR) +#define LAPACK_dormqr LAPACK_GLOBAL(dormqr,DORMQR) +#define LAPACK_cungqr LAPACK_GLOBAL(cungqr,CUNGQR) +#define LAPACK_zungqr LAPACK_GLOBAL(zungqr,ZUNGQR) +#define LAPACK_cunmqr LAPACK_GLOBAL(cunmqr,CUNMQR) +#define LAPACK_zunmqr LAPACK_GLOBAL(zunmqr,ZUNMQR) +#define LAPACK_sgelqf LAPACK_GLOBAL(sgelqf,SGELQF) +#define LAPACK_dgelqf LAPACK_GLOBAL(dgelqf,DGELQF) +#define LAPACK_cgelqf LAPACK_GLOBAL(cgelqf,CGELQF) +#define LAPACK_zgelqf LAPACK_GLOBAL(zgelqf,ZGELQF) +#define LAPACK_sorglq LAPACK_GLOBAL(sorglq,SORGLQ) +#define LAPACK_dorglq LAPACK_GLOBAL(dorglq,DORGLQ) +#define LAPACK_sormlq LAPACK_GLOBAL(sormlq,SORMLQ) +#define LAPACK_dormlq LAPACK_GLOBAL(dormlq,DORMLQ) +#define LAPACK_cunglq LAPACK_GLOBAL(cunglq,CUNGLQ) +#define LAPACK_zunglq LAPACK_GLOBAL(zunglq,ZUNGLQ) +#define LAPACK_cunmlq LAPACK_GLOBAL(cunmlq,CUNMLQ) +#define LAPACK_zunmlq LAPACK_GLOBAL(zunmlq,ZUNMLQ) +#define LAPACK_sgeqlf LAPACK_GLOBAL(sgeqlf,SGEQLF) +#define LAPACK_dgeqlf LAPACK_GLOBAL(dgeqlf,DGEQLF) +#define LAPACK_cgeqlf LAPACK_GLOBAL(cgeqlf,CGEQLF) +#define LAPACK_zgeqlf LAPACK_GLOBAL(zgeqlf,ZGEQLF) +#define LAPACK_sorgql LAPACK_GLOBAL(sorgql,SORGQL) +#define LAPACK_dorgql LAPACK_GLOBAL(dorgql,DORGQL) +#define LAPACK_cungql LAPACK_GLOBAL(cungql,CUNGQL) +#define LAPACK_zungql LAPACK_GLOBAL(zungql,ZUNGQL) +#define LAPACK_sormql LAPACK_GLOBAL(sormql,SORMQL) +#define LAPACK_dormql LAPACK_GLOBAL(dormql,DORMQL) +#define LAPACK_cunmql LAPACK_GLOBAL(cunmql,CUNMQL) +#define LAPACK_zunmql LAPACK_GLOBAL(zunmql,ZUNMQL) +#define LAPACK_sgerqf LAPACK_GLOBAL(sgerqf,SGERQF) +#define LAPACK_dgerqf LAPACK_GLOBAL(dgerqf,DGERQF) +#define LAPACK_cgerqf LAPACK_GLOBAL(cgerqf,CGERQF) +#define LAPACK_zgerqf LAPACK_GLOBAL(zgerqf,ZGERQF) +#define LAPACK_sorgrq LAPACK_GLOBAL(sorgrq,SORGRQ) +#define LAPACK_dorgrq LAPACK_GLOBAL(dorgrq,DORGRQ) +#define LAPACK_cungrq LAPACK_GLOBAL(cungrq,CUNGRQ) +#define LAPACK_zungrq LAPACK_GLOBAL(zungrq,ZUNGRQ) +#define LAPACK_sormrq LAPACK_GLOBAL(sormrq,SORMRQ) +#define LAPACK_dormrq LAPACK_GLOBAL(dormrq,DORMRQ) +#define LAPACK_cunmrq LAPACK_GLOBAL(cunmrq,CUNMRQ) +#define LAPACK_zunmrq LAPACK_GLOBAL(zunmrq,ZUNMRQ) +#define LAPACK_stzrzf LAPACK_GLOBAL(stzrzf,STZRZF) +#define LAPACK_dtzrzf LAPACK_GLOBAL(dtzrzf,DTZRZF) +#define LAPACK_ctzrzf LAPACK_GLOBAL(ctzrzf,CTZRZF) +#define LAPACK_ztzrzf LAPACK_GLOBAL(ztzrzf,ZTZRZF) +#define LAPACK_sormrz LAPACK_GLOBAL(sormrz,SORMRZ) +#define LAPACK_dormrz LAPACK_GLOBAL(dormrz,DORMRZ) +#define LAPACK_cunmrz LAPACK_GLOBAL(cunmrz,CUNMRZ) +#define LAPACK_zunmrz LAPACK_GLOBAL(zunmrz,ZUNMRZ) +#define LAPACK_sggqrf LAPACK_GLOBAL(sggqrf,SGGQRF) +#define LAPACK_dggqrf LAPACK_GLOBAL(dggqrf,DGGQRF) +#define LAPACK_cggqrf LAPACK_GLOBAL(cggqrf,CGGQRF) +#define LAPACK_zggqrf LAPACK_GLOBAL(zggqrf,ZGGQRF) +#define LAPACK_sggrqf LAPACK_GLOBAL(sggrqf,SGGRQF) +#define LAPACK_dggrqf LAPACK_GLOBAL(dggrqf,DGGRQF) +#define LAPACK_cggrqf LAPACK_GLOBAL(cggrqf,CGGRQF) +#define LAPACK_zggrqf LAPACK_GLOBAL(zggrqf,ZGGRQF) +#define LAPACK_sgebrd LAPACK_GLOBAL(sgebrd,SGEBRD) +#define LAPACK_dgebrd LAPACK_GLOBAL(dgebrd,DGEBRD) +#define LAPACK_cgebrd LAPACK_GLOBAL(cgebrd,CGEBRD) +#define LAPACK_zgebrd LAPACK_GLOBAL(zgebrd,ZGEBRD) +#define LAPACK_sgbbrd LAPACK_GLOBAL(sgbbrd,SGBBRD) +#define LAPACK_dgbbrd LAPACK_GLOBAL(dgbbrd,DGBBRD) +#define LAPACK_cgbbrd LAPACK_GLOBAL(cgbbrd,CGBBRD) +#define LAPACK_zgbbrd LAPACK_GLOBAL(zgbbrd,ZGBBRD) +#define LAPACK_sorgbr LAPACK_GLOBAL(sorgbr,SORGBR) +#define LAPACK_dorgbr LAPACK_GLOBAL(dorgbr,DORGBR) +#define LAPACK_sormbr LAPACK_GLOBAL(sormbr,SORMBR) +#define LAPACK_dormbr LAPACK_GLOBAL(dormbr,DORMBR) +#define LAPACK_cungbr LAPACK_GLOBAL(cungbr,CUNGBR) +#define LAPACK_zungbr LAPACK_GLOBAL(zungbr,ZUNGBR) +#define LAPACK_cunmbr LAPACK_GLOBAL(cunmbr,CUNMBR) +#define LAPACK_zunmbr LAPACK_GLOBAL(zunmbr,ZUNMBR) +#define LAPACK_sbdsqr LAPACK_GLOBAL(sbdsqr,SBDSQR) +#define LAPACK_dbdsqr LAPACK_GLOBAL(dbdsqr,DBDSQR) +#define LAPACK_cbdsqr LAPACK_GLOBAL(cbdsqr,CBDSQR) +#define LAPACK_zbdsqr LAPACK_GLOBAL(zbdsqr,ZBDSQR) +#define LAPACK_sbdsdc LAPACK_GLOBAL(sbdsdc,SBDSDC) +#define LAPACK_dbdsdc LAPACK_GLOBAL(dbdsdc,DBDSDC) +#define LAPACK_ssytrd LAPACK_GLOBAL(ssytrd,SSYTRD) +#define LAPACK_dsytrd LAPACK_GLOBAL(dsytrd,DSYTRD) +#define LAPACK_sorgtr LAPACK_GLOBAL(sorgtr,SORGTR) +#define LAPACK_dorgtr LAPACK_GLOBAL(dorgtr,DORGTR) +#define LAPACK_sormtr LAPACK_GLOBAL(sormtr,SORMTR) +#define LAPACK_dormtr LAPACK_GLOBAL(dormtr,DORMTR) +#define LAPACK_chetrd LAPACK_GLOBAL(chetrd,CHETRD) +#define LAPACK_zhetrd LAPACK_GLOBAL(zhetrd,ZHETRD) +#define LAPACK_cungtr LAPACK_GLOBAL(cungtr,CUNGTR) +#define LAPACK_zungtr LAPACK_GLOBAL(zungtr,ZUNGTR) +#define LAPACK_cunmtr LAPACK_GLOBAL(cunmtr,CUNMTR) +#define LAPACK_zunmtr LAPACK_GLOBAL(zunmtr,ZUNMTR) +#define LAPACK_ssptrd LAPACK_GLOBAL(ssptrd,SSPTRD) +#define LAPACK_dsptrd LAPACK_GLOBAL(dsptrd,DSPTRD) +#define LAPACK_sopgtr LAPACK_GLOBAL(sopgtr,SOPGTR) +#define LAPACK_dopgtr LAPACK_GLOBAL(dopgtr,DOPGTR) +#define LAPACK_sopmtr LAPACK_GLOBAL(sopmtr,SOPMTR) +#define LAPACK_dopmtr LAPACK_GLOBAL(dopmtr,DOPMTR) +#define LAPACK_chptrd LAPACK_GLOBAL(chptrd,CHPTRD) +#define LAPACK_zhptrd LAPACK_GLOBAL(zhptrd,ZHPTRD) +#define LAPACK_cupgtr LAPACK_GLOBAL(cupgtr,CUPGTR) +#define LAPACK_zupgtr LAPACK_GLOBAL(zupgtr,ZUPGTR) +#define LAPACK_cupmtr LAPACK_GLOBAL(cupmtr,CUPMTR) +#define LAPACK_zupmtr LAPACK_GLOBAL(zupmtr,ZUPMTR) +#define LAPACK_ssbtrd LAPACK_GLOBAL(ssbtrd,SSBTRD) +#define LAPACK_dsbtrd LAPACK_GLOBAL(dsbtrd,DSBTRD) +#define LAPACK_chbtrd LAPACK_GLOBAL(chbtrd,CHBTRD) +#define LAPACK_zhbtrd LAPACK_GLOBAL(zhbtrd,ZHBTRD) +#define LAPACK_ssterf LAPACK_GLOBAL(ssterf,SSTERF) +#define LAPACK_dsterf LAPACK_GLOBAL(dsterf,DSTERF) +#define LAPACK_ssteqr LAPACK_GLOBAL(ssteqr,SSTEQR) +#define LAPACK_dsteqr LAPACK_GLOBAL(dsteqr,DSTEQR) +#define LAPACK_csteqr LAPACK_GLOBAL(csteqr,CSTEQR) +#define LAPACK_zsteqr LAPACK_GLOBAL(zsteqr,ZSTEQR) +#define LAPACK_sstemr LAPACK_GLOBAL(sstemr,SSTEMR) +#define LAPACK_dstemr LAPACK_GLOBAL(dstemr,DSTEMR) +#define LAPACK_cstemr LAPACK_GLOBAL(cstemr,CSTEMR) +#define LAPACK_zstemr LAPACK_GLOBAL(zstemr,ZSTEMR) +#define LAPACK_sstedc LAPACK_GLOBAL(sstedc,SSTEDC) +#define LAPACK_dstedc LAPACK_GLOBAL(dstedc,DSTEDC) +#define LAPACK_cstedc LAPACK_GLOBAL(cstedc,CSTEDC) +#define LAPACK_zstedc LAPACK_GLOBAL(zstedc,ZSTEDC) +#define LAPACK_sstegr LAPACK_GLOBAL(sstegr,SSTEGR) +#define LAPACK_dstegr LAPACK_GLOBAL(dstegr,DSTEGR) +#define LAPACK_cstegr LAPACK_GLOBAL(cstegr,CSTEGR) +#define LAPACK_zstegr LAPACK_GLOBAL(zstegr,ZSTEGR) +#define LAPACK_spteqr LAPACK_GLOBAL(spteqr,SPTEQR) +#define LAPACK_dpteqr LAPACK_GLOBAL(dpteqr,DPTEQR) +#define LAPACK_cpteqr LAPACK_GLOBAL(cpteqr,CPTEQR) +#define LAPACK_zpteqr LAPACK_GLOBAL(zpteqr,ZPTEQR) +#define LAPACK_sstebz LAPACK_GLOBAL(sstebz,SSTEBZ) +#define LAPACK_dstebz LAPACK_GLOBAL(dstebz,DSTEBZ) +#define LAPACK_sstein LAPACK_GLOBAL(sstein,SSTEIN) +#define LAPACK_dstein LAPACK_GLOBAL(dstein,DSTEIN) +#define LAPACK_cstein LAPACK_GLOBAL(cstein,CSTEIN) +#define LAPACK_zstein LAPACK_GLOBAL(zstein,ZSTEIN) +#define LAPACK_sdisna LAPACK_GLOBAL(sdisna,SDISNA) +#define LAPACK_ddisna LAPACK_GLOBAL(ddisna,DDISNA) +#define LAPACK_ssygst LAPACK_GLOBAL(ssygst,SSYGST) +#define LAPACK_dsygst LAPACK_GLOBAL(dsygst,DSYGST) +#define LAPACK_chegst LAPACK_GLOBAL(chegst,CHEGST) +#define LAPACK_zhegst LAPACK_GLOBAL(zhegst,ZHEGST) +#define LAPACK_sspgst LAPACK_GLOBAL(sspgst,SSPGST) +#define LAPACK_dspgst LAPACK_GLOBAL(dspgst,DSPGST) +#define LAPACK_chpgst LAPACK_GLOBAL(chpgst,CHPGST) +#define LAPACK_zhpgst LAPACK_GLOBAL(zhpgst,ZHPGST) +#define LAPACK_ssbgst LAPACK_GLOBAL(ssbgst,SSBGST) +#define LAPACK_dsbgst LAPACK_GLOBAL(dsbgst,DSBGST) +#define LAPACK_chbgst LAPACK_GLOBAL(chbgst,CHBGST) +#define LAPACK_zhbgst LAPACK_GLOBAL(zhbgst,ZHBGST) +#define LAPACK_spbstf LAPACK_GLOBAL(spbstf,SPBSTF) +#define LAPACK_dpbstf LAPACK_GLOBAL(dpbstf,DPBSTF) +#define LAPACK_cpbstf LAPACK_GLOBAL(cpbstf,CPBSTF) +#define LAPACK_zpbstf LAPACK_GLOBAL(zpbstf,ZPBSTF) +#define LAPACK_sgehrd LAPACK_GLOBAL(sgehrd,SGEHRD) +#define LAPACK_dgehrd LAPACK_GLOBAL(dgehrd,DGEHRD) +#define LAPACK_cgehrd LAPACK_GLOBAL(cgehrd,CGEHRD) +#define LAPACK_zgehrd LAPACK_GLOBAL(zgehrd,ZGEHRD) +#define LAPACK_sorghr LAPACK_GLOBAL(sorghr,SORGHR) +#define LAPACK_dorghr LAPACK_GLOBAL(dorghr,DORGHR) +#define LAPACK_sormhr LAPACK_GLOBAL(sormhr,SORMHR) +#define LAPACK_dormhr LAPACK_GLOBAL(dormhr,DORMHR) +#define LAPACK_cunghr LAPACK_GLOBAL(cunghr,CUNGHR) +#define LAPACK_zunghr LAPACK_GLOBAL(zunghr,ZUNGHR) +#define LAPACK_cunmhr LAPACK_GLOBAL(cunmhr,CUNMHR) +#define LAPACK_zunmhr LAPACK_GLOBAL(zunmhr,ZUNMHR) +#define LAPACK_sgebal LAPACK_GLOBAL(sgebal,SGEBAL) +#define LAPACK_dgebal LAPACK_GLOBAL(dgebal,DGEBAL) +#define LAPACK_cgebal LAPACK_GLOBAL(cgebal,CGEBAL) +#define LAPACK_zgebal LAPACK_GLOBAL(zgebal,ZGEBAL) +#define LAPACK_sgebak LAPACK_GLOBAL(sgebak,SGEBAK) +#define LAPACK_dgebak LAPACK_GLOBAL(dgebak,DGEBAK) +#define LAPACK_cgebak LAPACK_GLOBAL(cgebak,CGEBAK) +#define LAPACK_zgebak LAPACK_GLOBAL(zgebak,ZGEBAK) +#define LAPACK_shseqr LAPACK_GLOBAL(shseqr,SHSEQR) +#define LAPACK_dhseqr LAPACK_GLOBAL(dhseqr,DHSEQR) +#define LAPACK_chseqr LAPACK_GLOBAL(chseqr,CHSEQR) +#define LAPACK_zhseqr LAPACK_GLOBAL(zhseqr,ZHSEQR) +#define LAPACK_shsein LAPACK_GLOBAL(shsein,SHSEIN) +#define LAPACK_dhsein LAPACK_GLOBAL(dhsein,DHSEIN) +#define LAPACK_chsein LAPACK_GLOBAL(chsein,CHSEIN) +#define LAPACK_zhsein LAPACK_GLOBAL(zhsein,ZHSEIN) +#define LAPACK_strevc LAPACK_GLOBAL(strevc,STREVC) +#define LAPACK_dtrevc LAPACK_GLOBAL(dtrevc,DTREVC) +#define LAPACK_ctrevc LAPACK_GLOBAL(ctrevc,CTREVC) +#define LAPACK_ztrevc LAPACK_GLOBAL(ztrevc,ZTREVC) +#define LAPACK_strsna LAPACK_GLOBAL(strsna,STRSNA) +#define LAPACK_dtrsna LAPACK_GLOBAL(dtrsna,DTRSNA) +#define LAPACK_ctrsna LAPACK_GLOBAL(ctrsna,CTRSNA) +#define LAPACK_ztrsna LAPACK_GLOBAL(ztrsna,ZTRSNA) +#define LAPACK_strexc LAPACK_GLOBAL(strexc,STREXC) +#define LAPACK_dtrexc LAPACK_GLOBAL(dtrexc,DTREXC) +#define LAPACK_ctrexc LAPACK_GLOBAL(ctrexc,CTREXC) +#define LAPACK_ztrexc LAPACK_GLOBAL(ztrexc,ZTREXC) +#define LAPACK_strsen LAPACK_GLOBAL(strsen,STRSEN) +#define LAPACK_dtrsen LAPACK_GLOBAL(dtrsen,DTRSEN) +#define LAPACK_ctrsen LAPACK_GLOBAL(ctrsen,CTRSEN) +#define LAPACK_ztrsen LAPACK_GLOBAL(ztrsen,ZTRSEN) +#define LAPACK_strsyl LAPACK_GLOBAL(strsyl,STRSYL) +#define LAPACK_dtrsyl LAPACK_GLOBAL(dtrsyl,DTRSYL) +#define LAPACK_ctrsyl LAPACK_GLOBAL(ctrsyl,CTRSYL) +#define LAPACK_ztrsyl LAPACK_GLOBAL(ztrsyl,ZTRSYL) +#define LAPACK_sgghrd LAPACK_GLOBAL(sgghrd,SGGHRD) +#define LAPACK_dgghrd LAPACK_GLOBAL(dgghrd,DGGHRD) +#define LAPACK_cgghrd LAPACK_GLOBAL(cgghrd,CGGHRD) +#define LAPACK_zgghrd LAPACK_GLOBAL(zgghrd,ZGGHRD) +#define LAPACK_sggbal LAPACK_GLOBAL(sggbal,SGGBAL) +#define LAPACK_dggbal LAPACK_GLOBAL(dggbal,DGGBAL) +#define LAPACK_cggbal LAPACK_GLOBAL(cggbal,CGGBAL) +#define LAPACK_zggbal LAPACK_GLOBAL(zggbal,ZGGBAL) +#define LAPACK_sggbak LAPACK_GLOBAL(sggbak,SGGBAK) +#define LAPACK_dggbak LAPACK_GLOBAL(dggbak,DGGBAK) +#define LAPACK_cggbak LAPACK_GLOBAL(cggbak,CGGBAK) +#define LAPACK_zggbak LAPACK_GLOBAL(zggbak,ZGGBAK) +#define LAPACK_shgeqz LAPACK_GLOBAL(shgeqz,SHGEQZ) +#define LAPACK_dhgeqz LAPACK_GLOBAL(dhgeqz,DHGEQZ) +#define LAPACK_chgeqz LAPACK_GLOBAL(chgeqz,CHGEQZ) +#define LAPACK_zhgeqz LAPACK_GLOBAL(zhgeqz,ZHGEQZ) +#define LAPACK_stgevc LAPACK_GLOBAL(stgevc,STGEVC) +#define LAPACK_dtgevc LAPACK_GLOBAL(dtgevc,DTGEVC) +#define LAPACK_ctgevc LAPACK_GLOBAL(ctgevc,CTGEVC) +#define LAPACK_ztgevc LAPACK_GLOBAL(ztgevc,ZTGEVC) +#define LAPACK_stgexc LAPACK_GLOBAL(stgexc,STGEXC) +#define LAPACK_dtgexc LAPACK_GLOBAL(dtgexc,DTGEXC) +#define LAPACK_ctgexc LAPACK_GLOBAL(ctgexc,CTGEXC) +#define LAPACK_ztgexc LAPACK_GLOBAL(ztgexc,ZTGEXC) +#define LAPACK_stgsen LAPACK_GLOBAL(stgsen,STGSEN) +#define LAPACK_dtgsen LAPACK_GLOBAL(dtgsen,DTGSEN) +#define LAPACK_ctgsen LAPACK_GLOBAL(ctgsen,CTGSEN) +#define LAPACK_ztgsen LAPACK_GLOBAL(ztgsen,ZTGSEN) +#define LAPACK_stgsyl LAPACK_GLOBAL(stgsyl,STGSYL) +#define LAPACK_dtgsyl LAPACK_GLOBAL(dtgsyl,DTGSYL) +#define LAPACK_ctgsyl LAPACK_GLOBAL(ctgsyl,CTGSYL) +#define LAPACK_ztgsyl LAPACK_GLOBAL(ztgsyl,ZTGSYL) +#define LAPACK_stgsna LAPACK_GLOBAL(stgsna,STGSNA) +#define LAPACK_dtgsna LAPACK_GLOBAL(dtgsna,DTGSNA) +#define LAPACK_ctgsna LAPACK_GLOBAL(ctgsna,CTGSNA) +#define LAPACK_ztgsna LAPACK_GLOBAL(ztgsna,ZTGSNA) +#define LAPACK_sggsvp LAPACK_GLOBAL(sggsvp,SGGSVP) +#define LAPACK_dggsvp LAPACK_GLOBAL(dggsvp,DGGSVP) +#define LAPACK_cggsvp LAPACK_GLOBAL(cggsvp,CGGSVP) +#define LAPACK_zggsvp LAPACK_GLOBAL(zggsvp,ZGGSVP) +#define LAPACK_stgsja LAPACK_GLOBAL(stgsja,STGSJA) +#define LAPACK_dtgsja LAPACK_GLOBAL(dtgsja,DTGSJA) +#define LAPACK_ctgsja LAPACK_GLOBAL(ctgsja,CTGSJA) +#define LAPACK_ztgsja LAPACK_GLOBAL(ztgsja,ZTGSJA) +#define LAPACK_sgels LAPACK_GLOBAL(sgels,SGELS) +#define LAPACK_dgels LAPACK_GLOBAL(dgels,DGELS) +#define LAPACK_cgels LAPACK_GLOBAL(cgels,CGELS) +#define LAPACK_zgels LAPACK_GLOBAL(zgels,ZGELS) +#define LAPACK_sgelsy LAPACK_GLOBAL(sgelsy,SGELSY) +#define LAPACK_dgelsy LAPACK_GLOBAL(dgelsy,DGELSY) +#define LAPACK_cgelsy LAPACK_GLOBAL(cgelsy,CGELSY) +#define LAPACK_zgelsy LAPACK_GLOBAL(zgelsy,ZGELSY) +#define LAPACK_sgelss LAPACK_GLOBAL(sgelss,SGELSS) +#define LAPACK_dgelss LAPACK_GLOBAL(dgelss,DGELSS) +#define LAPACK_cgelss LAPACK_GLOBAL(cgelss,CGELSS) +#define LAPACK_zgelss LAPACK_GLOBAL(zgelss,ZGELSS) +#define LAPACK_sgelsd LAPACK_GLOBAL(sgelsd,SGELSD) +#define LAPACK_dgelsd LAPACK_GLOBAL(dgelsd,DGELSD) +#define LAPACK_cgelsd LAPACK_GLOBAL(cgelsd,CGELSD) +#define LAPACK_zgelsd LAPACK_GLOBAL(zgelsd,ZGELSD) +#define LAPACK_sgglse LAPACK_GLOBAL(sgglse,SGGLSE) +#define LAPACK_dgglse LAPACK_GLOBAL(dgglse,DGGLSE) +#define LAPACK_cgglse LAPACK_GLOBAL(cgglse,CGGLSE) +#define LAPACK_zgglse LAPACK_GLOBAL(zgglse,ZGGLSE) +#define LAPACK_sggglm LAPACK_GLOBAL(sggglm,SGGGLM) +#define LAPACK_dggglm LAPACK_GLOBAL(dggglm,DGGGLM) +#define LAPACK_cggglm LAPACK_GLOBAL(cggglm,CGGGLM) +#define LAPACK_zggglm LAPACK_GLOBAL(zggglm,ZGGGLM) +#define LAPACK_ssyev LAPACK_GLOBAL(ssyev,SSYEV) +#define LAPACK_dsyev LAPACK_GLOBAL(dsyev,DSYEV) +#define LAPACK_cheev LAPACK_GLOBAL(cheev,CHEEV) +#define LAPACK_zheev LAPACK_GLOBAL(zheev,ZHEEV) +#define LAPACK_ssyevd LAPACK_GLOBAL(ssyevd,SSYEVD) +#define LAPACK_dsyevd LAPACK_GLOBAL(dsyevd,DSYEVD) +#define LAPACK_cheevd LAPACK_GLOBAL(cheevd,CHEEVD) +#define LAPACK_zheevd LAPACK_GLOBAL(zheevd,ZHEEVD) +#define LAPACK_ssyevx LAPACK_GLOBAL(ssyevx,SSYEVX) +#define LAPACK_dsyevx LAPACK_GLOBAL(dsyevx,DSYEVX) +#define LAPACK_cheevx LAPACK_GLOBAL(cheevx,CHEEVX) +#define LAPACK_zheevx LAPACK_GLOBAL(zheevx,ZHEEVX) +#define LAPACK_ssyevr LAPACK_GLOBAL(ssyevr,SSYEVR) +#define LAPACK_dsyevr LAPACK_GLOBAL(dsyevr,DSYEVR) +#define LAPACK_cheevr LAPACK_GLOBAL(cheevr,CHEEVR) +#define LAPACK_zheevr LAPACK_GLOBAL(zheevr,ZHEEVR) +#define LAPACK_sspev LAPACK_GLOBAL(sspev,SSPEV) +#define LAPACK_dspev LAPACK_GLOBAL(dspev,DSPEV) +#define LAPACK_chpev LAPACK_GLOBAL(chpev,CHPEV) +#define LAPACK_zhpev LAPACK_GLOBAL(zhpev,ZHPEV) +#define LAPACK_sspevd LAPACK_GLOBAL(sspevd,SSPEVD) +#define LAPACK_dspevd LAPACK_GLOBAL(dspevd,DSPEVD) +#define LAPACK_chpevd LAPACK_GLOBAL(chpevd,CHPEVD) +#define LAPACK_zhpevd LAPACK_GLOBAL(zhpevd,ZHPEVD) +#define LAPACK_sspevx LAPACK_GLOBAL(sspevx,SSPEVX) +#define LAPACK_dspevx LAPACK_GLOBAL(dspevx,DSPEVX) +#define LAPACK_chpevx LAPACK_GLOBAL(chpevx,CHPEVX) +#define LAPACK_zhpevx LAPACK_GLOBAL(zhpevx,ZHPEVX) +#define LAPACK_ssbev LAPACK_GLOBAL(ssbev,SSBEV) +#define LAPACK_dsbev LAPACK_GLOBAL(dsbev,DSBEV) +#define LAPACK_chbev LAPACK_GLOBAL(chbev,CHBEV) +#define LAPACK_zhbev LAPACK_GLOBAL(zhbev,ZHBEV) +#define LAPACK_ssbevd LAPACK_GLOBAL(ssbevd,SSBEVD) +#define LAPACK_dsbevd LAPACK_GLOBAL(dsbevd,DSBEVD) +#define LAPACK_chbevd LAPACK_GLOBAL(chbevd,CHBEVD) +#define LAPACK_zhbevd LAPACK_GLOBAL(zhbevd,ZHBEVD) +#define LAPACK_ssbevx LAPACK_GLOBAL(ssbevx,SSBEVX) +#define LAPACK_dsbevx LAPACK_GLOBAL(dsbevx,DSBEVX) +#define LAPACK_chbevx LAPACK_GLOBAL(chbevx,CHBEVX) +#define LAPACK_zhbevx LAPACK_GLOBAL(zhbevx,ZHBEVX) +#define LAPACK_sstev LAPACK_GLOBAL(sstev,SSTEV) +#define LAPACK_dstev LAPACK_GLOBAL(dstev,DSTEV) +#define LAPACK_sstevd LAPACK_GLOBAL(sstevd,SSTEVD) +#define LAPACK_dstevd LAPACK_GLOBAL(dstevd,DSTEVD) +#define LAPACK_sstevx LAPACK_GLOBAL(sstevx,SSTEVX) +#define LAPACK_dstevx LAPACK_GLOBAL(dstevx,DSTEVX) +#define LAPACK_sstevr LAPACK_GLOBAL(sstevr,SSTEVR) +#define LAPACK_dstevr LAPACK_GLOBAL(dstevr,DSTEVR) +#define LAPACK_sgees LAPACK_GLOBAL(sgees,SGEES) +#define LAPACK_dgees LAPACK_GLOBAL(dgees,DGEES) +#define LAPACK_cgees LAPACK_GLOBAL(cgees,CGEES) +#define LAPACK_zgees LAPACK_GLOBAL(zgees,ZGEES) +#define LAPACK_sgeesx LAPACK_GLOBAL(sgeesx,SGEESX) +#define LAPACK_dgeesx LAPACK_GLOBAL(dgeesx,DGEESX) +#define LAPACK_cgeesx LAPACK_GLOBAL(cgeesx,CGEESX) +#define LAPACK_zgeesx LAPACK_GLOBAL(zgeesx,ZGEESX) +#define LAPACK_sgeev LAPACK_GLOBAL(sgeev,SGEEV) +#define LAPACK_dgeev LAPACK_GLOBAL(dgeev,DGEEV) +#define LAPACK_cgeev LAPACK_GLOBAL(cgeev,CGEEV) +#define LAPACK_zgeev LAPACK_GLOBAL(zgeev,ZGEEV) +#define LAPACK_sgeevx LAPACK_GLOBAL(sgeevx,SGEEVX) +#define LAPACK_dgeevx LAPACK_GLOBAL(dgeevx,DGEEVX) +#define LAPACK_cgeevx LAPACK_GLOBAL(cgeevx,CGEEVX) +#define LAPACK_zgeevx LAPACK_GLOBAL(zgeevx,ZGEEVX) +#define LAPACK_sgesvd LAPACK_GLOBAL(sgesvd,SGESVD) +#define LAPACK_dgesvd LAPACK_GLOBAL(dgesvd,DGESVD) +#define LAPACK_cgesvd LAPACK_GLOBAL(cgesvd,CGESVD) +#define LAPACK_zgesvd LAPACK_GLOBAL(zgesvd,ZGESVD) +#define LAPACK_sgesdd LAPACK_GLOBAL(sgesdd,SGESDD) +#define LAPACK_dgesdd LAPACK_GLOBAL(dgesdd,DGESDD) +#define LAPACK_cgesdd LAPACK_GLOBAL(cgesdd,CGESDD) +#define LAPACK_zgesdd LAPACK_GLOBAL(zgesdd,ZGESDD) +#define LAPACK_dgejsv LAPACK_GLOBAL(dgejsv,DGEJSV) +#define LAPACK_sgejsv LAPACK_GLOBAL(sgejsv,SGEJSV) +#define LAPACK_dgesvj LAPACK_GLOBAL(dgesvj,DGESVJ) +#define LAPACK_sgesvj LAPACK_GLOBAL(sgesvj,SGESVJ) +#define LAPACK_sggsvd LAPACK_GLOBAL(sggsvd,SGGSVD) +#define LAPACK_dggsvd LAPACK_GLOBAL(dggsvd,DGGSVD) +#define LAPACK_cggsvd LAPACK_GLOBAL(cggsvd,CGGSVD) +#define LAPACK_zggsvd LAPACK_GLOBAL(zggsvd,ZGGSVD) +#define LAPACK_ssygv LAPACK_GLOBAL(ssygv,SSYGV) +#define LAPACK_dsygv LAPACK_GLOBAL(dsygv,DSYGV) +#define LAPACK_chegv LAPACK_GLOBAL(chegv,CHEGV) +#define LAPACK_zhegv LAPACK_GLOBAL(zhegv,ZHEGV) +#define LAPACK_ssygvd LAPACK_GLOBAL(ssygvd,SSYGVD) +#define LAPACK_dsygvd LAPACK_GLOBAL(dsygvd,DSYGVD) +#define LAPACK_chegvd LAPACK_GLOBAL(chegvd,CHEGVD) +#define LAPACK_zhegvd LAPACK_GLOBAL(zhegvd,ZHEGVD) +#define LAPACK_ssygvx LAPACK_GLOBAL(ssygvx,SSYGVX) +#define LAPACK_dsygvx LAPACK_GLOBAL(dsygvx,DSYGVX) +#define LAPACK_chegvx LAPACK_GLOBAL(chegvx,CHEGVX) +#define LAPACK_zhegvx LAPACK_GLOBAL(zhegvx,ZHEGVX) +#define LAPACK_sspgv LAPACK_GLOBAL(sspgv,SSPGV) +#define LAPACK_dspgv LAPACK_GLOBAL(dspgv,DSPGV) +#define LAPACK_chpgv LAPACK_GLOBAL(chpgv,CHPGV) +#define LAPACK_zhpgv LAPACK_GLOBAL(zhpgv,ZHPGV) +#define LAPACK_sspgvd LAPACK_GLOBAL(sspgvd,SSPGVD) +#define LAPACK_dspgvd LAPACK_GLOBAL(dspgvd,DSPGVD) +#define LAPACK_chpgvd LAPACK_GLOBAL(chpgvd,CHPGVD) +#define LAPACK_zhpgvd LAPACK_GLOBAL(zhpgvd,ZHPGVD) +#define LAPACK_sspgvx LAPACK_GLOBAL(sspgvx,SSPGVX) +#define LAPACK_dspgvx LAPACK_GLOBAL(dspgvx,DSPGVX) +#define LAPACK_chpgvx LAPACK_GLOBAL(chpgvx,CHPGVX) +#define LAPACK_zhpgvx LAPACK_GLOBAL(zhpgvx,ZHPGVX) +#define LAPACK_ssbgv LAPACK_GLOBAL(ssbgv,SSBGV) +#define LAPACK_dsbgv LAPACK_GLOBAL(dsbgv,DSBGV) +#define LAPACK_chbgv LAPACK_GLOBAL(chbgv,CHBGV) +#define LAPACK_zhbgv LAPACK_GLOBAL(zhbgv,ZHBGV) +#define LAPACK_ssbgvd LAPACK_GLOBAL(ssbgvd,SSBGVD) +#define LAPACK_dsbgvd LAPACK_GLOBAL(dsbgvd,DSBGVD) +#define LAPACK_chbgvd LAPACK_GLOBAL(chbgvd,CHBGVD) +#define LAPACK_zhbgvd LAPACK_GLOBAL(zhbgvd,ZHBGVD) +#define LAPACK_ssbgvx LAPACK_GLOBAL(ssbgvx,SSBGVX) +#define LAPACK_dsbgvx LAPACK_GLOBAL(dsbgvx,DSBGVX) +#define LAPACK_chbgvx LAPACK_GLOBAL(chbgvx,CHBGVX) +#define LAPACK_zhbgvx LAPACK_GLOBAL(zhbgvx,ZHBGVX) +#define LAPACK_sgges LAPACK_GLOBAL(sgges,SGGES) +#define LAPACK_dgges LAPACK_GLOBAL(dgges,DGGES) +#define LAPACK_cgges LAPACK_GLOBAL(cgges,CGGES) +#define LAPACK_zgges LAPACK_GLOBAL(zgges,ZGGES) +#define LAPACK_sggesx LAPACK_GLOBAL(sggesx,SGGESX) +#define LAPACK_dggesx LAPACK_GLOBAL(dggesx,DGGESX) +#define LAPACK_cggesx LAPACK_GLOBAL(cggesx,CGGESX) +#define LAPACK_zggesx LAPACK_GLOBAL(zggesx,ZGGESX) +#define LAPACK_sggev LAPACK_GLOBAL(sggev,SGGEV) +#define LAPACK_dggev LAPACK_GLOBAL(dggev,DGGEV) +#define LAPACK_cggev LAPACK_GLOBAL(cggev,CGGEV) +#define LAPACK_zggev LAPACK_GLOBAL(zggev,ZGGEV) +#define LAPACK_sggevx LAPACK_GLOBAL(sggevx,SGGEVX) +#define LAPACK_dggevx LAPACK_GLOBAL(dggevx,DGGEVX) +#define LAPACK_cggevx LAPACK_GLOBAL(cggevx,CGGEVX) +#define LAPACK_zggevx LAPACK_GLOBAL(zggevx,ZGGEVX) +#define LAPACK_dsfrk LAPACK_GLOBAL(dsfrk,DSFRK) +#define LAPACK_ssfrk LAPACK_GLOBAL(ssfrk,SSFRK) +#define LAPACK_zhfrk LAPACK_GLOBAL(zhfrk,ZHFRK) +#define LAPACK_chfrk LAPACK_GLOBAL(chfrk,CHFRK) +#define LAPACK_dtfsm LAPACK_GLOBAL(dtfsm,DTFSM) +#define LAPACK_stfsm LAPACK_GLOBAL(stfsm,STFSM) +#define LAPACK_ztfsm LAPACK_GLOBAL(ztfsm,ZTFSM) +#define LAPACK_ctfsm LAPACK_GLOBAL(ctfsm,CTFSM) +#define LAPACK_dtfttp LAPACK_GLOBAL(dtfttp,DTFTTP) +#define LAPACK_stfttp LAPACK_GLOBAL(stfttp,STFTTP) +#define LAPACK_ztfttp LAPACK_GLOBAL(ztfttp,ZTFTTP) +#define LAPACK_ctfttp LAPACK_GLOBAL(ctfttp,CTFTTP) +#define LAPACK_dtfttr LAPACK_GLOBAL(dtfttr,DTFTTR) +#define LAPACK_stfttr LAPACK_GLOBAL(stfttr,STFTTR) +#define LAPACK_ztfttr LAPACK_GLOBAL(ztfttr,ZTFTTR) +#define LAPACK_ctfttr LAPACK_GLOBAL(ctfttr,CTFTTR) +#define LAPACK_dtpttf LAPACK_GLOBAL(dtpttf,DTPTTF) +#define LAPACK_stpttf LAPACK_GLOBAL(stpttf,STPTTF) +#define LAPACK_ztpttf LAPACK_GLOBAL(ztpttf,ZTPTTF) +#define LAPACK_ctpttf LAPACK_GLOBAL(ctpttf,CTPTTF) +#define LAPACK_dtpttr LAPACK_GLOBAL(dtpttr,DTPTTR) +#define LAPACK_stpttr LAPACK_GLOBAL(stpttr,STPTTR) +#define LAPACK_ztpttr LAPACK_GLOBAL(ztpttr,ZTPTTR) +#define LAPACK_ctpttr LAPACK_GLOBAL(ctpttr,CTPTTR) +#define LAPACK_dtrttf LAPACK_GLOBAL(dtrttf,DTRTTF) +#define LAPACK_strttf LAPACK_GLOBAL(strttf,STRTTF) +#define LAPACK_ztrttf LAPACK_GLOBAL(ztrttf,ZTRTTF) +#define LAPACK_ctrttf LAPACK_GLOBAL(ctrttf,CTRTTF) +#define LAPACK_dtrttp LAPACK_GLOBAL(dtrttp,DTRTTP) +#define LAPACK_strttp LAPACK_GLOBAL(strttp,STRTTP) +#define LAPACK_ztrttp LAPACK_GLOBAL(ztrttp,ZTRTTP) +#define LAPACK_ctrttp LAPACK_GLOBAL(ctrttp,CTRTTP) +#define LAPACK_sgeqrfp LAPACK_GLOBAL(sgeqrfp,SGEQRFP) +#define LAPACK_dgeqrfp LAPACK_GLOBAL(dgeqrfp,DGEQRFP) +#define LAPACK_cgeqrfp LAPACK_GLOBAL(cgeqrfp,CGEQRFP) +#define LAPACK_zgeqrfp LAPACK_GLOBAL(zgeqrfp,ZGEQRFP) +#define LAPACK_clacgv LAPACK_GLOBAL(clacgv,CLACGV) +#define LAPACK_zlacgv LAPACK_GLOBAL(zlacgv,ZLACGV) +#define LAPACK_slarnv LAPACK_GLOBAL(slarnv,SLARNV) +#define LAPACK_dlarnv LAPACK_GLOBAL(dlarnv,DLARNV) +#define LAPACK_clarnv LAPACK_GLOBAL(clarnv,CLARNV) +#define LAPACK_zlarnv LAPACK_GLOBAL(zlarnv,ZLARNV) +#define LAPACK_sgeqr2 LAPACK_GLOBAL(sgeqr2,SGEQR2) +#define LAPACK_dgeqr2 LAPACK_GLOBAL(dgeqr2,DGEQR2) +#define LAPACK_cgeqr2 LAPACK_GLOBAL(cgeqr2,CGEQR2) +#define LAPACK_zgeqr2 LAPACK_GLOBAL(zgeqr2,ZGEQR2) +#define LAPACK_slacpy LAPACK_GLOBAL(slacpy,SLACPY) +#define LAPACK_dlacpy LAPACK_GLOBAL(dlacpy,DLACPY) +#define LAPACK_clacpy LAPACK_GLOBAL(clacpy,CLACPY) +#define LAPACK_zlacpy LAPACK_GLOBAL(zlacpy,ZLACPY) +#define LAPACK_sgetf2 LAPACK_GLOBAL(sgetf2,SGETF2) +#define LAPACK_dgetf2 LAPACK_GLOBAL(dgetf2,DGETF2) +#define LAPACK_cgetf2 LAPACK_GLOBAL(cgetf2,CGETF2) +#define LAPACK_zgetf2 LAPACK_GLOBAL(zgetf2,ZGETF2) +#define LAPACK_slaswp LAPACK_GLOBAL(slaswp,SLASWP) +#define LAPACK_dlaswp LAPACK_GLOBAL(dlaswp,DLASWP) +#define LAPACK_claswp LAPACK_GLOBAL(claswp,CLASWP) +#define LAPACK_zlaswp LAPACK_GLOBAL(zlaswp,ZLASWP) +#define LAPACK_slange LAPACK_GLOBAL(slange,SLANGE) +#define LAPACK_dlange LAPACK_GLOBAL(dlange,DLANGE) +#define LAPACK_clange LAPACK_GLOBAL(clange,CLANGE) +#define LAPACK_zlange LAPACK_GLOBAL(zlange,ZLANGE) +#define LAPACK_clanhe LAPACK_GLOBAL(clanhe,CLANHE) +#define LAPACK_zlanhe LAPACK_GLOBAL(zlanhe,ZLANHE) +#define LAPACK_slansy LAPACK_GLOBAL(slansy,SLANSY) +#define LAPACK_dlansy LAPACK_GLOBAL(dlansy,DLANSY) +#define LAPACK_clansy LAPACK_GLOBAL(clansy,CLANSY) +#define LAPACK_zlansy LAPACK_GLOBAL(zlansy,ZLANSY) +#define LAPACK_slantr LAPACK_GLOBAL(slantr,SLANTR) +#define LAPACK_dlantr LAPACK_GLOBAL(dlantr,DLANTR) +#define LAPACK_clantr LAPACK_GLOBAL(clantr,CLANTR) +#define LAPACK_zlantr LAPACK_GLOBAL(zlantr,ZLANTR) +#define LAPACK_slamch LAPACK_GLOBAL(slamch,SLAMCH) +#define LAPACK_dlamch LAPACK_GLOBAL(dlamch,DLAMCH) +#define LAPACK_sgelq2 LAPACK_GLOBAL(sgelq2,SGELQ2) +#define LAPACK_dgelq2 LAPACK_GLOBAL(dgelq2,DGELQ2) +#define LAPACK_cgelq2 LAPACK_GLOBAL(cgelq2,CGELQ2) +#define LAPACK_zgelq2 LAPACK_GLOBAL(zgelq2,ZGELQ2) +#define LAPACK_slarfb LAPACK_GLOBAL(slarfb,SLARFB) +#define LAPACK_dlarfb LAPACK_GLOBAL(dlarfb,DLARFB) +#define LAPACK_clarfb LAPACK_GLOBAL(clarfb,CLARFB) +#define LAPACK_zlarfb LAPACK_GLOBAL(zlarfb,ZLARFB) +#define LAPACK_slarfg LAPACK_GLOBAL(slarfg,SLARFG) +#define LAPACK_dlarfg LAPACK_GLOBAL(dlarfg,DLARFG) +#define LAPACK_clarfg LAPACK_GLOBAL(clarfg,CLARFG) +#define LAPACK_zlarfg LAPACK_GLOBAL(zlarfg,ZLARFG) +#define LAPACK_slarft LAPACK_GLOBAL(slarft,SLARFT) +#define LAPACK_dlarft LAPACK_GLOBAL(dlarft,DLARFT) +#define LAPACK_clarft LAPACK_GLOBAL(clarft,CLARFT) +#define LAPACK_zlarft LAPACK_GLOBAL(zlarft,ZLARFT) +#define LAPACK_slarfx LAPACK_GLOBAL(slarfx,SLARFX) +#define LAPACK_dlarfx LAPACK_GLOBAL(dlarfx,DLARFX) +#define LAPACK_clarfx LAPACK_GLOBAL(clarfx,CLARFX) +#define LAPACK_zlarfx LAPACK_GLOBAL(zlarfx,ZLARFX) +#define LAPACK_slatms LAPACK_GLOBAL(slatms,SLATMS) +#define LAPACK_dlatms LAPACK_GLOBAL(dlatms,DLATMS) +#define LAPACK_clatms LAPACK_GLOBAL(clatms,CLATMS) +#define LAPACK_zlatms LAPACK_GLOBAL(zlatms,ZLATMS) +#define LAPACK_slag2d LAPACK_GLOBAL(slag2d,SLAG2D) +#define LAPACK_dlag2s LAPACK_GLOBAL(dlag2s,DLAG2S) +#define LAPACK_clag2z LAPACK_GLOBAL(clag2z,CLAG2Z) +#define LAPACK_zlag2c LAPACK_GLOBAL(zlag2c,ZLAG2C) +#define LAPACK_slauum LAPACK_GLOBAL(slauum,SLAUUM) +#define LAPACK_dlauum LAPACK_GLOBAL(dlauum,DLAUUM) +#define LAPACK_clauum LAPACK_GLOBAL(clauum,CLAUUM) +#define LAPACK_zlauum LAPACK_GLOBAL(zlauum,ZLAUUM) +#define LAPACK_slagge LAPACK_GLOBAL(slagge,SLAGGE) +#define LAPACK_dlagge LAPACK_GLOBAL(dlagge,DLAGGE) +#define LAPACK_clagge LAPACK_GLOBAL(clagge,CLAGGE) +#define LAPACK_zlagge LAPACK_GLOBAL(zlagge,ZLAGGE) +#define LAPACK_slaset LAPACK_GLOBAL(slaset,SLASET) +#define LAPACK_dlaset LAPACK_GLOBAL(dlaset,DLASET) +#define LAPACK_claset LAPACK_GLOBAL(claset,CLASET) +#define LAPACK_zlaset LAPACK_GLOBAL(zlaset,ZLASET) +#define LAPACK_slasrt LAPACK_GLOBAL(slasrt,SLASRT) +#define LAPACK_dlasrt LAPACK_GLOBAL(dlasrt,DLASRT) +#define LAPACK_slagsy LAPACK_GLOBAL(slagsy,SLAGSY) +#define LAPACK_dlagsy LAPACK_GLOBAL(dlagsy,DLAGSY) +#define LAPACK_clagsy LAPACK_GLOBAL(clagsy,CLAGSY) +#define LAPACK_zlagsy LAPACK_GLOBAL(zlagsy,ZLAGSY) +#define LAPACK_claghe LAPACK_GLOBAL(claghe,CLAGHE) +#define LAPACK_zlaghe LAPACK_GLOBAL(zlaghe,ZLAGHE) +#define LAPACK_slapmr LAPACK_GLOBAL(slapmr,SLAPMR) +#define LAPACK_dlapmr LAPACK_GLOBAL(dlapmr,DLAPMR) +#define LAPACK_clapmr LAPACK_GLOBAL(clapmr,CLAPMR) +#define LAPACK_zlapmr LAPACK_GLOBAL(zlapmr,ZLAPMR) +#define LAPACK_slapy2 LAPACK_GLOBAL(slapy2,SLAPY2) +#define LAPACK_dlapy2 LAPACK_GLOBAL(dlapy2,DLAPY2) +#define LAPACK_slapy3 LAPACK_GLOBAL(slapy3,SLAPY3) +#define LAPACK_dlapy3 LAPACK_GLOBAL(dlapy3,DLAPY3) +#define LAPACK_slartgp LAPACK_GLOBAL(slartgp,SLARTGP) +#define LAPACK_dlartgp LAPACK_GLOBAL(dlartgp,DLARTGP) +#define LAPACK_slartgs LAPACK_GLOBAL(slartgs,SLARTGS) +#define LAPACK_dlartgs LAPACK_GLOBAL(dlartgs,DLARTGS) +// LAPACK 3.3.0 +#define LAPACK_cbbcsd LAPACK_GLOBAL(cbbcsd,CBBCSD) +#define LAPACK_cheswapr LAPACK_GLOBAL(cheswapr,CHESWAPR) +#define LAPACK_chetri2 LAPACK_GLOBAL(chetri2,CHETRI2) +#define LAPACK_chetri2x LAPACK_GLOBAL(chetri2x,CHETRI2X) +#define LAPACK_chetrs2 LAPACK_GLOBAL(chetrs2,CHETRS2) +#define LAPACK_csyconv LAPACK_GLOBAL(csyconv,CSYCONV) +#define LAPACK_csyswapr LAPACK_GLOBAL(csyswapr,CSYSWAPR) +#define LAPACK_csytri2 LAPACK_GLOBAL(csytri2,CSYTRI2) +#define LAPACK_csytri2x LAPACK_GLOBAL(csytri2x,CSYTRI2X) +#define LAPACK_csytrs2 LAPACK_GLOBAL(csytrs2,CSYTRS2) +#define LAPACK_cunbdb LAPACK_GLOBAL(cunbdb,CUNBDB) +#define LAPACK_cuncsd LAPACK_GLOBAL(cuncsd,CUNCSD) +#define LAPACK_dbbcsd LAPACK_GLOBAL(dbbcsd,DBBCSD) +#define LAPACK_dorbdb LAPACK_GLOBAL(dorbdb,DORBDB) +#define LAPACK_dorcsd LAPACK_GLOBAL(dorcsd,DORCSD) +#define LAPACK_dsyconv LAPACK_GLOBAL(dsyconv,DSYCONV) +#define LAPACK_dsyswapr LAPACK_GLOBAL(dsyswapr,DSYSWAPR) +#define LAPACK_dsytri2 LAPACK_GLOBAL(dsytri2,DSYTRI2) +#define LAPACK_dsytri2x LAPACK_GLOBAL(dsytri2x,DSYTRI2X) +#define LAPACK_dsytrs2 LAPACK_GLOBAL(dsytrs2,DSYTRS2) +#define LAPACK_sbbcsd LAPACK_GLOBAL(sbbcsd,SBBCSD) +#define LAPACK_sorbdb LAPACK_GLOBAL(sorbdb,SORBDB) +#define LAPACK_sorcsd LAPACK_GLOBAL(sorcsd,SORCSD) +#define LAPACK_ssyconv LAPACK_GLOBAL(ssyconv,SSYCONV) +#define LAPACK_ssyswapr LAPACK_GLOBAL(ssyswapr,SSYSWAPR) +#define LAPACK_ssytri2 LAPACK_GLOBAL(ssytri2,SSYTRI2) +#define LAPACK_ssytri2x LAPACK_GLOBAL(ssytri2x,SSYTRI2X) +#define LAPACK_ssytrs2 LAPACK_GLOBAL(ssytrs2,SSYTRS2) +#define LAPACK_zbbcsd LAPACK_GLOBAL(zbbcsd,ZBBCSD) +#define LAPACK_zheswapr LAPACK_GLOBAL(zheswapr,ZHESWAPR) +#define LAPACK_zhetri2 LAPACK_GLOBAL(zhetri2,ZHETRI2) +#define LAPACK_zhetri2x LAPACK_GLOBAL(zhetri2x,ZHETRI2X) +#define LAPACK_zhetrs2 LAPACK_GLOBAL(zhetrs2,ZHETRS2) +#define LAPACK_zsyconv LAPACK_GLOBAL(zsyconv,ZSYCONV) +#define LAPACK_zsyswapr LAPACK_GLOBAL(zsyswapr,ZSYSWAPR) +#define LAPACK_zsytri2 LAPACK_GLOBAL(zsytri2,ZSYTRI2) +#define LAPACK_zsytri2x LAPACK_GLOBAL(zsytri2x,ZSYTRI2X) +#define LAPACK_zsytrs2 LAPACK_GLOBAL(zsytrs2,ZSYTRS2) +#define LAPACK_zunbdb LAPACK_GLOBAL(zunbdb,ZUNBDB) +#define LAPACK_zuncsd LAPACK_GLOBAL(zuncsd,ZUNCSD) +// LAPACK 3.4.0 +#define LAPACK_sgemqrt LAPACK_GLOBAL(sgemqrt,SGEMQRT) +#define LAPACK_dgemqrt LAPACK_GLOBAL(dgemqrt,DGEMQRT) +#define LAPACK_cgemqrt LAPACK_GLOBAL(cgemqrt,CGEMQRT) +#define LAPACK_zgemqrt LAPACK_GLOBAL(zgemqrt,ZGEMQRT) +#define LAPACK_sgeqrt LAPACK_GLOBAL(sgeqrt,SGEQRT) +#define LAPACK_dgeqrt LAPACK_GLOBAL(dgeqrt,DGEQRT) +#define LAPACK_cgeqrt LAPACK_GLOBAL(cgeqrt,CGEQRT) +#define LAPACK_zgeqrt LAPACK_GLOBAL(zgeqrt,ZGEQRT) +#define LAPACK_sgeqrt2 LAPACK_GLOBAL(sgeqrt2,SGEQRT2) +#define LAPACK_dgeqrt2 LAPACK_GLOBAL(dgeqrt2,DGEQRT2) +#define LAPACK_cgeqrt2 LAPACK_GLOBAL(cgeqrt2,CGEQRT2) +#define LAPACK_zgeqrt2 LAPACK_GLOBAL(zgeqrt2,ZGEQRT2) +#define LAPACK_sgeqrt3 LAPACK_GLOBAL(sgeqrt3,SGEQRT3) +#define LAPACK_dgeqrt3 LAPACK_GLOBAL(dgeqrt3,DGEQRT3) +#define LAPACK_cgeqrt3 LAPACK_GLOBAL(cgeqrt3,CGEQRT3) +#define LAPACK_zgeqrt3 LAPACK_GLOBAL(zgeqrt3,ZGEQRT3) +#define LAPACK_stpmqrt LAPACK_GLOBAL(stpmqrt,STPMQRT) +#define LAPACK_dtpmqrt LAPACK_GLOBAL(dtpmqrt,DTPMQRT) +#define LAPACK_ctpmqrt LAPACK_GLOBAL(ctpmqrt,CTPMQRT) +#define LAPACK_ztpmqrt LAPACK_GLOBAL(ztpmqrt,ZTPMQRT) +#define LAPACK_dtpqrt LAPACK_GLOBAL(dtpqrt,DTPQRT) +#define LAPACK_ctpqrt LAPACK_GLOBAL(ctpqrt,CTPQRT) +#define LAPACK_ztpqrt LAPACK_GLOBAL(ztpqrt,ZTPQRT) +#define LAPACK_stpqrt2 LAPACK_GLOBAL(stpqrt2,STPQRT2) +#define LAPACK_dtpqrt2 LAPACK_GLOBAL(dtpqrt2,DTPQRT2) +#define LAPACK_ctpqrt2 LAPACK_GLOBAL(ctpqrt2,CTPQRT2) +#define LAPACK_ztpqrt2 LAPACK_GLOBAL(ztpqrt2,ZTPQRT2) +#define LAPACK_stprfb LAPACK_GLOBAL(stprfb,STPRFB) +#define LAPACK_dtprfb LAPACK_GLOBAL(dtprfb,DTPRFB) +#define LAPACK_ctprfb LAPACK_GLOBAL(ctprfb,CTPRFB) +#define LAPACK_ztprfb LAPACK_GLOBAL(ztprfb,ZTPRFB) +// LAPACK 3.X.X +#define LAPACK_csyr LAPACK_GLOBAL(csyr,CSYR) +#define LAPACK_zsyr LAPACK_GLOBAL(zsyr,ZSYR) + + +void LAPACK_sgetrf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_dgetrf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_cgetrf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* ipiv, lapack_int *info ); +void LAPACK_zgetrf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, lapack_int *info ); +void LAPACK_sgbtrf( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, float* ab, lapack_int* ldab, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_dgbtrf( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, double* ab, lapack_int* ldab, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_cgbtrf( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_complex_float* ab, lapack_int* ldab, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_zgbtrf( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_complex_double* ab, lapack_int* ldab, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_sgttrf( lapack_int* n, float* dl, float* d, float* du, float* du2, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_dgttrf( lapack_int* n, double* dl, double* d, double* du, + double* du2, lapack_int* ipiv, lapack_int *info ); +void LAPACK_cgttrf( lapack_int* n, lapack_complex_float* dl, + lapack_complex_float* d, lapack_complex_float* du, + lapack_complex_float* du2, lapack_int* ipiv, + lapack_int *info ); +void LAPACK_zgttrf( lapack_int* n, lapack_complex_double* dl, + lapack_complex_double* d, lapack_complex_double* du, + lapack_complex_double* du2, lapack_int* ipiv, + lapack_int *info ); +void LAPACK_spotrf( char* uplo, lapack_int* n, float* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dpotrf( char* uplo, lapack_int* n, double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_cpotrf( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_zpotrf( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_dpstrf( char* uplo, lapack_int* n, double* a, lapack_int* lda, + lapack_int* piv, lapack_int* rank, double* tol, + double* work, lapack_int *info ); +void LAPACK_spstrf( char* uplo, lapack_int* n, float* a, lapack_int* lda, + lapack_int* piv, lapack_int* rank, float* tol, float* work, + lapack_int *info ); +void LAPACK_zpstrf( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* piv, lapack_int* rank, + double* tol, double* work, lapack_int *info ); +void LAPACK_cpstrf( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* piv, lapack_int* rank, + float* tol, float* work, lapack_int *info ); +void LAPACK_dpftrf( char* transr, char* uplo, lapack_int* n, double* a, + lapack_int *info ); +void LAPACK_spftrf( char* transr, char* uplo, lapack_int* n, float* a, + lapack_int *info ); +void LAPACK_zpftrf( char* transr, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int *info ); +void LAPACK_cpftrf( char* transr, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int *info ); +void LAPACK_spptrf( char* uplo, lapack_int* n, float* ap, lapack_int *info ); +void LAPACK_dpptrf( char* uplo, lapack_int* n, double* ap, lapack_int *info ); +void LAPACK_cpptrf( char* uplo, lapack_int* n, lapack_complex_float* ap, + lapack_int *info ); +void LAPACK_zpptrf( char* uplo, lapack_int* n, lapack_complex_double* ap, + lapack_int *info ); +void LAPACK_spbtrf( char* uplo, lapack_int* n, lapack_int* kd, float* ab, + lapack_int* ldab, lapack_int *info ); +void LAPACK_dpbtrf( char* uplo, lapack_int* n, lapack_int* kd, double* ab, + lapack_int* ldab, lapack_int *info ); +void LAPACK_cpbtrf( char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_float* ab, lapack_int* ldab, + lapack_int *info ); +void LAPACK_zpbtrf( char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_double* ab, lapack_int* ldab, + lapack_int *info ); +void LAPACK_spttrf( lapack_int* n, float* d, float* e, lapack_int *info ); +void LAPACK_dpttrf( lapack_int* n, double* d, double* e, lapack_int *info ); +void LAPACK_cpttrf( lapack_int* n, float* d, lapack_complex_float* e, + lapack_int *info ); +void LAPACK_zpttrf( lapack_int* n, double* d, lapack_complex_double* e, + lapack_int *info ); +void LAPACK_ssytrf( char* uplo, lapack_int* n, float* a, lapack_int* lda, + lapack_int* ipiv, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dsytrf( char* uplo, lapack_int* n, double* a, lapack_int* lda, + lapack_int* ipiv, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_csytrf( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* ipiv, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zsytrf( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_chetrf( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* ipiv, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zhetrf( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ssptrf( char* uplo, lapack_int* n, float* ap, lapack_int* ipiv, + lapack_int *info ); +void LAPACK_dsptrf( char* uplo, lapack_int* n, double* ap, lapack_int* ipiv, + lapack_int *info ); +void LAPACK_csptrf( char* uplo, lapack_int* n, lapack_complex_float* ap, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_zsptrf( char* uplo, lapack_int* n, lapack_complex_double* ap, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_chptrf( char* uplo, lapack_int* n, lapack_complex_float* ap, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_zhptrf( char* uplo, lapack_int* n, lapack_complex_double* ap, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_sgetrs( char* trans, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, const lapack_int* ipiv, + float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_dgetrs( char* trans, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const lapack_int* ipiv, + double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_cgetrs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zgetrs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_sgbtrs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const float* ab, lapack_int* ldab, + const lapack_int* ipiv, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dgbtrs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const double* ab, lapack_int* ldab, + const lapack_int* ipiv, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cgbtrs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const lapack_complex_float* ab, + lapack_int* ldab, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zgbtrs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const lapack_complex_double* ab, + lapack_int* ldab, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_sgttrs( char* trans, lapack_int* n, lapack_int* nrhs, + const float* dl, const float* d, const float* du, + const float* du2, const lapack_int* ipiv, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dgttrs( char* trans, lapack_int* n, lapack_int* nrhs, + const double* dl, const double* d, const double* du, + const double* du2, const lapack_int* ipiv, double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_cgttrs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zgttrs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_spotrs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* a, + lapack_int* lda, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dpotrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_cpotrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zpotrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dpftrs( char* transr, char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_spftrs( char* transr, char* uplo, lapack_int* n, lapack_int* nrhs, + const float* a, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zpftrs( char* transr, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_cpftrs( char* transr, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_spptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const float* ap, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dpptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* ap, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cpptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zpptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_spbtrs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const float* ab, lapack_int* ldab, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dpbtrs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const double* ab, lapack_int* ldab, double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_cpbtrs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zpbtrs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_spttrs( lapack_int* n, lapack_int* nrhs, const float* d, + const float* e, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dpttrs( lapack_int* n, lapack_int* nrhs, const double* d, + const double* e, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cpttrs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* d, + const lapack_complex_float* e, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zpttrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* d, const lapack_complex_double* e, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_ssytrs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* a, + lapack_int* lda, const lapack_int* ipiv, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dsytrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const lapack_int* ipiv, + double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_csytrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zsytrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_chetrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zhetrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_ssptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const float* ap, const lapack_int* ipiv, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dsptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* ap, const lapack_int* ipiv, double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_csptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zsptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_chptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zhptrs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_strtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const float* a, lapack_int* lda, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dtrtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const double* a, lapack_int* lda, + double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_ctrtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_ztrtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_stptrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const float* ap, float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dtptrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const double* ap, double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_ctptrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* ap, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_ztptrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_double* ap, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_stbtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, const float* ab, + lapack_int* ldab, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dtbtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, const double* ab, + lapack_int* ldab, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_ctbtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, + const lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_ztbtrs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, + const lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_sgecon( char* norm, lapack_int* n, const float* a, lapack_int* lda, + float* anorm, float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgecon( char* norm, lapack_int* n, const double* a, lapack_int* lda, + double* anorm, double* rcond, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cgecon( char* norm, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* anorm, float* rcond, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zgecon( char* norm, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* anorm, double* rcond, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sgbcon( char* norm, lapack_int* n, lapack_int* kl, lapack_int* ku, + const float* ab, lapack_int* ldab, const lapack_int* ipiv, + float* anorm, float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgbcon( char* norm, lapack_int* n, lapack_int* kl, lapack_int* ku, + const double* ab, lapack_int* ldab, const lapack_int* ipiv, + double* anorm, double* rcond, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cgbcon( char* norm, lapack_int* n, lapack_int* kl, lapack_int* ku, + const lapack_complex_float* ab, lapack_int* ldab, + const lapack_int* ipiv, float* anorm, float* rcond, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zgbcon( char* norm, lapack_int* n, lapack_int* kl, lapack_int* ku, + const lapack_complex_double* ab, lapack_int* ldab, + const lapack_int* ipiv, double* anorm, double* rcond, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sgtcon( char* norm, lapack_int* n, const float* dl, const float* d, + const float* du, const float* du2, const lapack_int* ipiv, + float* anorm, float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgtcon( char* norm, lapack_int* n, const double* dl, + const double* d, const double* du, const double* du2, + const lapack_int* ipiv, double* anorm, double* rcond, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cgtcon( char* norm, lapack_int* n, const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* du2, const lapack_int* ipiv, + float* anorm, float* rcond, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zgtcon( char* norm, lapack_int* n, const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* du2, const lapack_int* ipiv, + double* anorm, double* rcond, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_spocon( char* uplo, lapack_int* n, const float* a, lapack_int* lda, + float* anorm, float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dpocon( char* uplo, lapack_int* n, const double* a, lapack_int* lda, + double* anorm, double* rcond, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cpocon( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* anorm, float* rcond, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zpocon( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* anorm, double* rcond, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sppcon( char* uplo, lapack_int* n, const float* ap, float* anorm, + float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dppcon( char* uplo, lapack_int* n, const double* ap, double* anorm, + double* rcond, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cppcon( char* uplo, lapack_int* n, const lapack_complex_float* ap, + float* anorm, float* rcond, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zppcon( char* uplo, lapack_int* n, const lapack_complex_double* ap, + double* anorm, double* rcond, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_spbcon( char* uplo, lapack_int* n, lapack_int* kd, const float* ab, + lapack_int* ldab, float* anorm, float* rcond, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dpbcon( char* uplo, lapack_int* n, lapack_int* kd, const double* ab, + lapack_int* ldab, double* anorm, double* rcond, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cpbcon( char* uplo, lapack_int* n, lapack_int* kd, + const lapack_complex_float* ab, lapack_int* ldab, + float* anorm, float* rcond, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zpbcon( char* uplo, lapack_int* n, lapack_int* kd, + const lapack_complex_double* ab, lapack_int* ldab, + double* anorm, double* rcond, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_sptcon( lapack_int* n, const float* d, const float* e, float* anorm, + float* rcond, float* work, lapack_int *info ); +void LAPACK_dptcon( lapack_int* n, const double* d, const double* e, + double* anorm, double* rcond, double* work, + lapack_int *info ); +void LAPACK_cptcon( lapack_int* n, const float* d, + const lapack_complex_float* e, float* anorm, float* rcond, + float* work, lapack_int *info ); +void LAPACK_zptcon( lapack_int* n, const double* d, + const lapack_complex_double* e, double* anorm, + double* rcond, double* work, lapack_int *info ); +void LAPACK_ssycon( char* uplo, lapack_int* n, const float* a, lapack_int* lda, + const lapack_int* ipiv, float* anorm, float* rcond, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dsycon( char* uplo, lapack_int* n, const double* a, lapack_int* lda, + const lapack_int* ipiv, double* anorm, double* rcond, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_csycon( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, float* anorm, + float* rcond, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zsycon( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, const lapack_int* ipiv, double* anorm, + double* rcond, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_checon( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, float* anorm, + float* rcond, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zhecon( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, const lapack_int* ipiv, double* anorm, + double* rcond, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_sspcon( char* uplo, lapack_int* n, const float* ap, + const lapack_int* ipiv, float* anorm, float* rcond, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dspcon( char* uplo, lapack_int* n, const double* ap, + const lapack_int* ipiv, double* anorm, double* rcond, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cspcon( char* uplo, lapack_int* n, const lapack_complex_float* ap, + const lapack_int* ipiv, float* anorm, float* rcond, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zspcon( char* uplo, lapack_int* n, const lapack_complex_double* ap, + const lapack_int* ipiv, double* anorm, double* rcond, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_chpcon( char* uplo, lapack_int* n, const lapack_complex_float* ap, + const lapack_int* ipiv, float* anorm, float* rcond, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zhpcon( char* uplo, lapack_int* n, const lapack_complex_double* ap, + const lapack_int* ipiv, double* anorm, double* rcond, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_strcon( char* norm, char* uplo, char* diag, lapack_int* n, + const float* a, lapack_int* lda, float* rcond, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dtrcon( char* norm, char* uplo, char* diag, lapack_int* n, + const double* a, lapack_int* lda, double* rcond, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_ctrcon( char* norm, char* uplo, char* diag, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, + float* rcond, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztrcon( char* norm, char* uplo, char* diag, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, + double* rcond, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_stpcon( char* norm, char* uplo, char* diag, lapack_int* n, + const float* ap, float* rcond, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dtpcon( char* norm, char* uplo, char* diag, lapack_int* n, + const double* ap, double* rcond, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ctpcon( char* norm, char* uplo, char* diag, lapack_int* n, + const lapack_complex_float* ap, float* rcond, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztpcon( char* norm, char* uplo, char* diag, lapack_int* n, + const lapack_complex_double* ap, double* rcond, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_stbcon( char* norm, char* uplo, char* diag, lapack_int* n, + lapack_int* kd, const float* ab, lapack_int* ldab, + float* rcond, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dtbcon( char* norm, char* uplo, char* diag, lapack_int* n, + lapack_int* kd, const double* ab, lapack_int* ldab, + double* rcond, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_ctbcon( char* norm, char* uplo, char* diag, lapack_int* n, + lapack_int* kd, const lapack_complex_float* ab, + lapack_int* ldab, float* rcond, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_ztbcon( char* norm, char* uplo, char* diag, lapack_int* n, + lapack_int* kd, const lapack_complex_double* ab, + lapack_int* ldab, double* rcond, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sgerfs( char* trans, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, const float* af, + lapack_int* ldaf, const lapack_int* ipiv, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* ferr, + float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgerfs( char* trans, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const lapack_int* ipiv, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cgerfs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zgerfs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_dgerfsx( char* trans, char* equed, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const lapack_int* ipiv, const double* r, + const double* c, const double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* rcond, double* berr, + lapack_int* n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int* nparams, double* params, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_sgerfsx( char* trans, char* equed, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, const float* af, + lapack_int* ldaf, const lapack_int* ipiv, const float* r, + const float* c, const float* b, lapack_int* ldb, float* x, + lapack_int* ldx, float* rcond, float* berr, + lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_zgerfsx( char* trans, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const double* r, const double* c, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cgerfsx( char* trans, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const float* r, const float* c, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* berr, lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sgbrfs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const float* ab, lapack_int* ldab, + const float* afb, lapack_int* ldafb, const lapack_int* ipiv, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgbrfs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const double* ab, lapack_int* ldab, + const double* afb, lapack_int* ldafb, + const lapack_int* ipiv, const double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* ferr, double* berr, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cgbrfs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const lapack_complex_float* ab, + lapack_int* ldab, const lapack_complex_float* afb, + lapack_int* ldafb, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zgbrfs( char* trans, lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, const lapack_complex_double* ab, + lapack_int* ldab, const lapack_complex_double* afb, + lapack_int* ldafb, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_dgbrfsx( char* trans, char* equed, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, const double* ab, + lapack_int* ldab, const double* afb, lapack_int* ldafb, + const lapack_int* ipiv, const double* r, const double* c, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* rcond, double* berr, + lapack_int* n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int* nparams, double* params, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_sgbrfsx( char* trans, char* equed, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, const float* ab, + lapack_int* ldab, const float* afb, lapack_int* ldafb, + const lapack_int* ipiv, const float* r, const float* c, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* rcond, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_zgbrfsx( char* trans, char* equed, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, + const lapack_complex_double* ab, lapack_int* ldab, + const lapack_complex_double* afb, lapack_int* ldafb, + const lapack_int* ipiv, const double* r, const double* c, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cgbrfsx( char* trans, char* equed, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, + const lapack_complex_float* ab, lapack_int* ldab, + const lapack_complex_float* afb, lapack_int* ldafb, + const lapack_int* ipiv, const float* r, const float* c, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* berr, lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sgtrfs( char* trans, lapack_int* n, lapack_int* nrhs, + const float* dl, const float* d, const float* du, + const float* dlf, const float* df, const float* duf, + const float* du2, const lapack_int* ipiv, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* ferr, + float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgtrfs( char* trans, lapack_int* n, lapack_int* nrhs, + const double* dl, const double* d, const double* du, + const double* dlf, const double* df, const double* duf, + const double* du2, const lapack_int* ipiv, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cgtrfs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, + const lapack_complex_float* dlf, + const lapack_complex_float* df, + const lapack_complex_float* duf, + const lapack_complex_float* du2, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zgtrfs( char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, + const lapack_complex_double* dlf, + const lapack_complex_double* df, + const lapack_complex_double* duf, + const lapack_complex_double* du2, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sporfs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* a, + lapack_int* lda, const float* af, lapack_int* ldaf, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dporfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* ferr, double* berr, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cporfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zporfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_dporfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const double* s, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* rcond, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_sporfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, const float* af, + lapack_int* ldaf, const float* s, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* berr, lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_zporfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const double* s, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cporfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const float* s, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_spprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const float* ap, const float* afp, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* ferr, + float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dpprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* ap, const double* afp, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cpprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, + const lapack_complex_float* afp, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zpprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_spbrfs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const float* ab, lapack_int* ldab, const float* afb, + lapack_int* ldafb, const float* b, lapack_int* ldb, + float* x, lapack_int* ldx, float* ferr, float* berr, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dpbrfs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const double* ab, lapack_int* ldab, const double* afb, + lapack_int* ldafb, const double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* ferr, double* berr, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cpbrfs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const lapack_complex_float* ab, lapack_int* ldab, + const lapack_complex_float* afb, lapack_int* ldafb, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zpbrfs( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + const lapack_complex_double* ab, lapack_int* ldab, + const lapack_complex_double* afb, lapack_int* ldafb, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sptrfs( lapack_int* n, lapack_int* nrhs, const float* d, + const float* e, const float* df, const float* ef, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* ferr, float* berr, float* work, lapack_int *info ); +void LAPACK_dptrfs( lapack_int* n, lapack_int* nrhs, const double* d, + const double* e, const double* df, const double* ef, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* ferr, double* berr, double* work, + lapack_int *info ); +void LAPACK_cptrfs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* d, + const lapack_complex_float* e, const float* df, + const lapack_complex_float* ef, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zptrfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* d, const lapack_complex_double* e, + const double* df, const lapack_complex_double* ef, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_ssyrfs( char* uplo, lapack_int* n, lapack_int* nrhs, const float* a, + lapack_int* lda, const float* af, lapack_int* ldaf, + const lapack_int* ipiv, const float* b, lapack_int* ldb, + float* x, lapack_int* ldx, float* ferr, float* berr, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dsyrfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const lapack_int* ipiv, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_csyrfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zsyrfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_dsyrfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, const double* af, + lapack_int* ldaf, const lapack_int* ipiv, const double* s, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* rcond, double* berr, + lapack_int* n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int* nparams, double* params, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_ssyrfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, const float* af, + lapack_int* ldaf, const lapack_int* ipiv, const float* s, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* rcond, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_zsyrfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const double* s, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_csyrfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const float* s, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* berr, lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_cherfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zherfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_zherfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* af, lapack_int* ldaf, + const lapack_int* ipiv, const double* s, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cherfsx( char* uplo, char* equed, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* af, lapack_int* ldaf, + const lapack_int* ipiv, const float* s, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* berr, lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ssprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const float* ap, const float* afp, const lapack_int* ipiv, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dsprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const double* ap, const double* afp, const lapack_int* ipiv, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* ferr, double* berr, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_csprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, + const lapack_complex_float* afp, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zsprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_chprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, + const lapack_complex_float* afp, const lapack_int* ipiv, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zhprfs( char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, + const lapack_complex_double* afp, const lapack_int* ipiv, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* ferr, + double* berr, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_strrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const float* a, lapack_int* lda, + const float* b, lapack_int* ldb, const float* x, + lapack_int* ldx, float* ferr, float* berr, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dtrrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const double* a, lapack_int* lda, + const double* b, lapack_int* ldb, const double* x, + lapack_int* ldx, double* ferr, double* berr, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ctrrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* b, + lapack_int* ldb, const lapack_complex_float* x, + lapack_int* ldx, float* ferr, float* berr, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztrrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* b, + lapack_int* ldb, const lapack_complex_double* x, + lapack_int* ldx, double* ferr, double* berr, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_stprfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const float* ap, const float* b, + lapack_int* ldb, const float* x, lapack_int* ldx, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dtprfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const double* ap, const double* b, + lapack_int* ldb, const double* x, lapack_int* ldx, + double* ferr, double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_ctprfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* ap, + const lapack_complex_float* b, lapack_int* ldb, + const lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztprfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* nrhs, const lapack_complex_double* ap, + const lapack_complex_double* b, lapack_int* ldb, + const lapack_complex_double* x, lapack_int* ldx, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_stbrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, const float* ab, + lapack_int* ldab, const float* b, lapack_int* ldb, + const float* x, lapack_int* ldx, float* ferr, float* berr, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dtbrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, const double* ab, + lapack_int* ldab, const double* b, lapack_int* ldb, + const double* x, lapack_int* ldx, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_ctbrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, + const lapack_complex_float* ab, lapack_int* ldab, + const lapack_complex_float* b, lapack_int* ldb, + const lapack_complex_float* x, lapack_int* ldx, float* ferr, + float* berr, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztbrfs( char* uplo, char* trans, char* diag, lapack_int* n, + lapack_int* kd, lapack_int* nrhs, + const lapack_complex_double* ab, lapack_int* ldab, + const lapack_complex_double* b, lapack_int* ldb, + const lapack_complex_double* x, lapack_int* ldx, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_sgetri( lapack_int* n, float* a, lapack_int* lda, + const lapack_int* ipiv, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgetri( lapack_int* n, double* a, lapack_int* lda, + const lapack_int* ipiv, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgetri( lapack_int* n, lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zgetri( lapack_int* n, lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_spotri( char* uplo, lapack_int* n, float* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dpotri( char* uplo, lapack_int* n, double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_cpotri( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_zpotri( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_dpftri( char* transr, char* uplo, lapack_int* n, double* a, + lapack_int *info ); +void LAPACK_spftri( char* transr, char* uplo, lapack_int* n, float* a, + lapack_int *info ); +void LAPACK_zpftri( char* transr, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int *info ); +void LAPACK_cpftri( char* transr, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int *info ); +void LAPACK_spptri( char* uplo, lapack_int* n, float* ap, lapack_int *info ); +void LAPACK_dpptri( char* uplo, lapack_int* n, double* ap, lapack_int *info ); +void LAPACK_cpptri( char* uplo, lapack_int* n, lapack_complex_float* ap, + lapack_int *info ); +void LAPACK_zpptri( char* uplo, lapack_int* n, lapack_complex_double* ap, + lapack_int *info ); +void LAPACK_ssytri( char* uplo, lapack_int* n, float* a, lapack_int* lda, + const lapack_int* ipiv, float* work, lapack_int *info ); +void LAPACK_dsytri( char* uplo, lapack_int* n, double* a, lapack_int* lda, + const lapack_int* ipiv, double* work, lapack_int *info ); +void LAPACK_csytri( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zsytri( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_chetri( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zhetri( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_ssptri( char* uplo, lapack_int* n, float* ap, + const lapack_int* ipiv, float* work, lapack_int *info ); +void LAPACK_dsptri( char* uplo, lapack_int* n, double* ap, + const lapack_int* ipiv, double* work, lapack_int *info ); +void LAPACK_csptri( char* uplo, lapack_int* n, lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zsptri( char* uplo, lapack_int* n, lapack_complex_double* ap, + const lapack_int* ipiv, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_chptri( char* uplo, lapack_int* n, lapack_complex_float* ap, + const lapack_int* ipiv, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zhptri( char* uplo, lapack_int* n, lapack_complex_double* ap, + const lapack_int* ipiv, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_strtri( char* uplo, char* diag, lapack_int* n, float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_dtrtri( char* uplo, char* diag, lapack_int* n, double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_ctrtri( char* uplo, char* diag, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_ztrtri( char* uplo, char* diag, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dtftri( char* transr, char* uplo, char* diag, lapack_int* n, + double* a, lapack_int *info ); +void LAPACK_stftri( char* transr, char* uplo, char* diag, lapack_int* n, + float* a, lapack_int *info ); +void LAPACK_ztftri( char* transr, char* uplo, char* diag, lapack_int* n, + lapack_complex_double* a, lapack_int *info ); +void LAPACK_ctftri( char* transr, char* uplo, char* diag, lapack_int* n, + lapack_complex_float* a, lapack_int *info ); +void LAPACK_stptri( char* uplo, char* diag, lapack_int* n, float* ap, + lapack_int *info ); +void LAPACK_dtptri( char* uplo, char* diag, lapack_int* n, double* ap, + lapack_int *info ); +void LAPACK_ctptri( char* uplo, char* diag, lapack_int* n, + lapack_complex_float* ap, lapack_int *info ); +void LAPACK_ztptri( char* uplo, char* diag, lapack_int* n, + lapack_complex_double* ap, lapack_int *info ); +void LAPACK_sgeequ( lapack_int* m, lapack_int* n, const float* a, + lapack_int* lda, float* r, float* c, float* rowcnd, + float* colcnd, float* amax, lapack_int *info ); +void LAPACK_dgeequ( lapack_int* m, lapack_int* n, const double* a, + lapack_int* lda, double* r, double* c, double* rowcnd, + double* colcnd, double* amax, lapack_int *info ); +void LAPACK_cgeequ( lapack_int* m, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* r, float* c, float* rowcnd, + float* colcnd, float* amax, lapack_int *info ); +void LAPACK_zgeequ( lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, double* r, + double* c, double* rowcnd, double* colcnd, double* amax, + lapack_int *info ); +void LAPACK_dgeequb( lapack_int* m, lapack_int* n, const double* a, + lapack_int* lda, double* r, double* c, double* rowcnd, + double* colcnd, double* amax, lapack_int *info ); +void LAPACK_sgeequb( lapack_int* m, lapack_int* n, const float* a, + lapack_int* lda, float* r, float* c, float* rowcnd, + float* colcnd, float* amax, lapack_int *info ); +void LAPACK_zgeequb( lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, double* r, + double* c, double* rowcnd, double* colcnd, double* amax, + lapack_int *info ); +void LAPACK_cgeequb( lapack_int* m, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, float* r, + float* c, float* rowcnd, float* colcnd, float* amax, + lapack_int *info ); +void LAPACK_sgbequ( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const float* ab, lapack_int* ldab, float* r, + float* c, float* rowcnd, float* colcnd, float* amax, + lapack_int *info ); +void LAPACK_dgbequ( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const double* ab, lapack_int* ldab, + double* r, double* c, double* rowcnd, double* colcnd, + double* amax, lapack_int *info ); +void LAPACK_cgbequ( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const lapack_complex_float* ab, + lapack_int* ldab, float* r, float* c, float* rowcnd, + float* colcnd, float* amax, lapack_int *info ); +void LAPACK_zgbequ( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const lapack_complex_double* ab, + lapack_int* ldab, double* r, double* c, double* rowcnd, + double* colcnd, double* amax, lapack_int *info ); +void LAPACK_dgbequb( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const double* ab, lapack_int* ldab, + double* r, double* c, double* rowcnd, double* colcnd, + double* amax, lapack_int *info ); +void LAPACK_sgbequb( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const float* ab, lapack_int* ldab, + float* r, float* c, float* rowcnd, float* colcnd, + float* amax, lapack_int *info ); +void LAPACK_zgbequb( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const lapack_complex_double* ab, + lapack_int* ldab, double* r, double* c, double* rowcnd, + double* colcnd, double* amax, lapack_int *info ); +void LAPACK_cgbequb( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const lapack_complex_float* ab, + lapack_int* ldab, float* r, float* c, float* rowcnd, + float* colcnd, float* amax, lapack_int *info ); +void LAPACK_spoequ( lapack_int* n, const float* a, lapack_int* lda, float* s, + float* scond, float* amax, lapack_int *info ); +void LAPACK_dpoequ( lapack_int* n, const double* a, lapack_int* lda, double* s, + double* scond, double* amax, lapack_int *info ); +void LAPACK_cpoequ( lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* s, float* scond, float* amax, + lapack_int *info ); +void LAPACK_zpoequ( lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* s, double* scond, double* amax, + lapack_int *info ); +void LAPACK_dpoequb( lapack_int* n, const double* a, lapack_int* lda, double* s, + double* scond, double* amax, lapack_int *info ); +void LAPACK_spoequb( lapack_int* n, const float* a, lapack_int* lda, float* s, + float* scond, float* amax, lapack_int *info ); +void LAPACK_zpoequb( lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* s, double* scond, double* amax, + lapack_int *info ); +void LAPACK_cpoequb( lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* s, float* scond, float* amax, + lapack_int *info ); +void LAPACK_sppequ( char* uplo, lapack_int* n, const float* ap, float* s, + float* scond, float* amax, lapack_int *info ); +void LAPACK_dppequ( char* uplo, lapack_int* n, const double* ap, double* s, + double* scond, double* amax, lapack_int *info ); +void LAPACK_cppequ( char* uplo, lapack_int* n, const lapack_complex_float* ap, + float* s, float* scond, float* amax, lapack_int *info ); +void LAPACK_zppequ( char* uplo, lapack_int* n, const lapack_complex_double* ap, + double* s, double* scond, double* amax, lapack_int *info ); +void LAPACK_spbequ( char* uplo, lapack_int* n, lapack_int* kd, const float* ab, + lapack_int* ldab, float* s, float* scond, float* amax, + lapack_int *info ); +void LAPACK_dpbequ( char* uplo, lapack_int* n, lapack_int* kd, const double* ab, + lapack_int* ldab, double* s, double* scond, double* amax, + lapack_int *info ); +void LAPACK_cpbequ( char* uplo, lapack_int* n, lapack_int* kd, + const lapack_complex_float* ab, lapack_int* ldab, float* s, + float* scond, float* amax, lapack_int *info ); +void LAPACK_zpbequ( char* uplo, lapack_int* n, lapack_int* kd, + const lapack_complex_double* ab, lapack_int* ldab, + double* s, double* scond, double* amax, lapack_int *info ); +void LAPACK_dsyequb( char* uplo, lapack_int* n, const double* a, + lapack_int* lda, double* s, double* scond, double* amax, + double* work, lapack_int *info ); +void LAPACK_ssyequb( char* uplo, lapack_int* n, const float* a, lapack_int* lda, + float* s, float* scond, float* amax, float* work, + lapack_int *info ); +void LAPACK_zsyequb( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* s, double* scond, double* amax, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_csyequb( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* s, float* scond, float* amax, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zheequb( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, double* s, double* scond, double* amax, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_cheequb( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, float* s, float* scond, float* amax, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_sgesv( lapack_int* n, lapack_int* nrhs, float* a, lapack_int* lda, + lapack_int* ipiv, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dgesv( lapack_int* n, lapack_int* nrhs, double* a, lapack_int* lda, + lapack_int* ipiv, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cgesv( lapack_int* n, lapack_int* nrhs, lapack_complex_float* a, + lapack_int* lda, lapack_int* ipiv, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zgesv( lapack_int* n, lapack_int* nrhs, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_dsgesv( lapack_int* n, lapack_int* nrhs, double* a, lapack_int* lda, + lapack_int* ipiv, double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* work, float* swork, + lapack_int* iter, lapack_int *info ); +void LAPACK_zcgesv( lapack_int* n, lapack_int* nrhs, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + lapack_complex_double* work, lapack_complex_float* swork, + double* rwork, lapack_int* iter, lapack_int *info ); +void LAPACK_sgesvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgesvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int* ldb, double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cgesvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zgesvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_dgesvxx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int* ldb, double* x, lapack_int* ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int* n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int* nparams, double* params, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_sgesvxx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_zgesvxx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cgesvxx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* r, float* c, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sgbsv( lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, float* ab, lapack_int* ldab, + lapack_int* ipiv, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dgbsv( lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, double* ab, lapack_int* ldab, + lapack_int* ipiv, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cgbsv( lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, lapack_complex_float* ab, lapack_int* ldab, + lapack_int* ipiv, lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zgbsv( lapack_int* n, lapack_int* kl, lapack_int* ku, + lapack_int* nrhs, lapack_complex_double* ab, + lapack_int* ldab, lapack_int* ipiv, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_sgbsvx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, float* ab, + lapack_int* ldab, float* afb, lapack_int* ldafb, + lapack_int* ipiv, char* equed, float* r, float* c, float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgbsvx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, double* ab, + lapack_int* ldab, double* afb, lapack_int* ldafb, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int* ldb, double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cgbsvx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, lapack_complex_float* ab, + lapack_int* ldab, lapack_complex_float* afb, + lapack_int* ldafb, lapack_int* ipiv, char* equed, float* r, + float* c, lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zgbsvx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, lapack_complex_double* ab, + lapack_int* ldab, lapack_complex_double* afb, + lapack_int* ldafb, lapack_int* ipiv, char* equed, double* r, + double* c, lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_dgbsvxx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, double* ab, + lapack_int* ldab, double* afb, lapack_int* ldafb, + lapack_int* ipiv, char* equed, double* r, double* c, + double* b, lapack_int* ldb, double* x, lapack_int* ldx, + double* rcond, double* rpvgrw, double* berr, + lapack_int* n_err_bnds, double* err_bnds_norm, + double* err_bnds_comp, lapack_int* nparams, double* params, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_sgbsvxx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, float* ab, + lapack_int* ldab, float* afb, lapack_int* ldafb, + lapack_int* ipiv, char* equed, float* r, float* c, + float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* rcond, float* rpvgrw, float* berr, + lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_zgbsvxx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, + lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* afb, lapack_int* ldafb, + lapack_int* ipiv, char* equed, double* r, double* c, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cgbsvxx( char* fact, char* trans, lapack_int* n, lapack_int* kl, + lapack_int* ku, lapack_int* nrhs, lapack_complex_float* ab, + lapack_int* ldab, lapack_complex_float* afb, + lapack_int* ldafb, lapack_int* ipiv, char* equed, float* r, + float* c, lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sgtsv( lapack_int* n, lapack_int* nrhs, float* dl, float* d, + float* du, float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_dgtsv( lapack_int* n, lapack_int* nrhs, double* dl, double* d, + double* du, double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_cgtsv( lapack_int* n, lapack_int* nrhs, lapack_complex_float* dl, + lapack_complex_float* d, lapack_complex_float* du, + lapack_complex_float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_zgtsv( lapack_int* n, lapack_int* nrhs, lapack_complex_double* dl, + lapack_complex_double* d, lapack_complex_double* du, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_sgtsvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + const float* dl, const float* d, const float* du, + float* dlf, float* df, float* duf, float* du2, + lapack_int* ipiv, const float* b, lapack_int* ldb, float* x, + lapack_int* ldx, float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dgtsvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + const double* dl, const double* d, const double* du, + double* dlf, double* df, double* duf, double* du2, + lapack_int* ipiv, const double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* rcond, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cgtsvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* dl, + const lapack_complex_float* d, + const lapack_complex_float* du, lapack_complex_float* dlf, + lapack_complex_float* df, lapack_complex_float* duf, + lapack_complex_float* du2, lapack_int* ipiv, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zgtsvx( char* fact, char* trans, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* dl, + const lapack_complex_double* d, + const lapack_complex_double* du, lapack_complex_double* dlf, + lapack_complex_double* df, lapack_complex_double* duf, + lapack_complex_double* du2, lapack_int* ipiv, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_sposv( char* uplo, lapack_int* n, lapack_int* nrhs, float* a, + lapack_int* lda, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dposv( char* uplo, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cposv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_zposv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dsposv( char* uplo, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* work, float* swork, + lapack_int* iter, lapack_int *info ); +void LAPACK_zcposv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, + lapack_complex_double* work, lapack_complex_float* swork, + double* rwork, lapack_int* iter, lapack_int *info ); +void LAPACK_sposvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* af, lapack_int* ldaf, + char* equed, float* s, float* b, lapack_int* ldb, float* x, + lapack_int* ldx, float* rcond, float* ferr, float* berr, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dposvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* af, lapack_int* ldaf, + char* equed, double* s, double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* rcond, double* ferr, + double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cposvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, char* equed, + float* s, lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zposvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, char* equed, + double* s, lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_dposvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* af, lapack_int* ldaf, + char* equed, double* s, double* b, lapack_int* ldb, + double* x, lapack_int* ldx, double* rcond, double* rpvgrw, + double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_sposvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* af, lapack_int* ldaf, + char* equed, float* s, float* b, lapack_int* ldb, float* x, + lapack_int* ldx, float* rcond, float* rpvgrw, float* berr, + lapack_int* n_err_bnds, float* err_bnds_norm, + float* err_bnds_comp, lapack_int* nparams, float* params, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_zposvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, char* equed, + double* s, lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_cposvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, char* equed, + float* s, lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sppsv( char* uplo, lapack_int* n, lapack_int* nrhs, float* ap, + float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_dppsv( char* uplo, lapack_int* n, lapack_int* nrhs, double* ap, + double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_cppsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* ap, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zppsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* ap, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_sppsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + float* ap, float* afp, char* equed, float* s, float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dppsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + double* ap, double* afp, char* equed, double* s, double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cppsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* ap, lapack_complex_float* afp, + char* equed, float* s, lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zppsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* ap, lapack_complex_double* afp, + char* equed, double* s, lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_spbsv( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + float* ab, lapack_int* ldab, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dpbsv( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + double* ab, lapack_int* ldab, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cpbsv( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_zpbsv( char* uplo, lapack_int* n, lapack_int* kd, lapack_int* nrhs, + lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_spbsvx( char* fact, char* uplo, lapack_int* n, lapack_int* kd, + lapack_int* nrhs, float* ab, lapack_int* ldab, float* afb, + lapack_int* ldafb, char* equed, float* s, float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dpbsvx( char* fact, char* uplo, lapack_int* n, lapack_int* kd, + lapack_int* nrhs, double* ab, lapack_int* ldab, double* afb, + lapack_int* ldafb, char* equed, double* s, double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_cpbsvx( char* fact, char* uplo, lapack_int* n, lapack_int* kd, + lapack_int* nrhs, lapack_complex_float* ab, + lapack_int* ldab, lapack_complex_float* afb, + lapack_int* ldafb, char* equed, float* s, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zpbsvx( char* fact, char* uplo, lapack_int* n, lapack_int* kd, + lapack_int* nrhs, lapack_complex_double* ab, + lapack_int* ldab, lapack_complex_double* afb, + lapack_int* ldafb, char* equed, double* s, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_sptsv( lapack_int* n, lapack_int* nrhs, float* d, float* e, + float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_dptsv( lapack_int* n, lapack_int* nrhs, double* d, double* e, + double* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_cptsv( lapack_int* n, lapack_int* nrhs, float* d, + lapack_complex_float* e, lapack_complex_float* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_zptsv( lapack_int* n, lapack_int* nrhs, double* d, + lapack_complex_double* e, lapack_complex_double* b, + lapack_int* ldb, lapack_int *info ); +void LAPACK_sptsvx( char* fact, lapack_int* n, lapack_int* nrhs, const float* d, + const float* e, float* df, float* ef, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int *info ); +void LAPACK_dptsvx( char* fact, lapack_int* n, lapack_int* nrhs, + const double* d, const double* e, double* df, double* ef, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* rcond, double* ferr, double* berr, + double* work, lapack_int *info ); +void LAPACK_cptsvx( char* fact, lapack_int* n, lapack_int* nrhs, const float* d, + const lapack_complex_float* e, float* df, + lapack_complex_float* ef, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zptsvx( char* fact, lapack_int* n, lapack_int* nrhs, + const double* d, const lapack_complex_double* e, double* df, + lapack_complex_double* ef, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_ssysv( char* uplo, lapack_int* n, lapack_int* nrhs, float* a, + lapack_int* lda, lapack_int* ipiv, float* b, lapack_int* ldb, + float* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_dsysv( char* uplo, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, lapack_int* ipiv, double* b, + lapack_int* ldb, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_csysv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zsysv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ssysvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const float* a, lapack_int* lda, float* af, + lapack_int* ldaf, lapack_int* ipiv, const float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* ferr, float* berr, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dsysvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const double* a, lapack_int* lda, double* af, + lapack_int* ldaf, lapack_int* ipiv, const double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* rcond, + double* ferr, double* berr, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_csysvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zsysvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_dsysvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* s, double* b, + lapack_int* ldb, double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ssysvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* s, float* b, + lapack_int* ldb, float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_zsysvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_csysvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_chesv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zhesv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_chesvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zhesvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_zhesvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, double* s, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* x, lapack_int* ldx, double* rcond, + double* rpvgrw, double* berr, lapack_int* n_err_bnds, + double* err_bnds_norm, double* err_bnds_comp, + lapack_int* nparams, double* params, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_chesvxx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* af, lapack_int* ldaf, + lapack_int* ipiv, char* equed, float* s, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* x, lapack_int* ldx, float* rcond, + float* rpvgrw, float* berr, lapack_int* n_err_bnds, + float* err_bnds_norm, float* err_bnds_comp, + lapack_int* nparams, float* params, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_sspsv( char* uplo, lapack_int* n, lapack_int* nrhs, float* ap, + lapack_int* ipiv, float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dspsv( char* uplo, lapack_int* n, lapack_int* nrhs, double* ap, + lapack_int* ipiv, double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_cspsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* ap, lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_zspsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* ap, lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_sspsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const float* ap, float* afp, lapack_int* ipiv, + const float* b, lapack_int* ldb, float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, float* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dspsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const double* ap, double* afp, lapack_int* ipiv, + const double* b, lapack_int* ldb, double* x, + lapack_int* ldx, double* rcond, double* ferr, double* berr, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cspsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, lapack_complex_float* afp, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zspsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, lapack_complex_double* afp, + lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_chpsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* ap, lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, lapack_int *info ); +void LAPACK_zhpsv( char* uplo, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* ap, lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_chpsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_float* ap, lapack_complex_float* afp, + lapack_int* ipiv, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* x, lapack_int* ldx, + float* rcond, float* ferr, float* berr, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zhpsvx( char* fact, char* uplo, lapack_int* n, lapack_int* nrhs, + const lapack_complex_double* ap, lapack_complex_double* afp, + lapack_int* ipiv, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* x, lapack_int* ldx, + double* rcond, double* ferr, double* berr, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sgeqrf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgeqrf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgeqrf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgeqrf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgeqpf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + lapack_int* jpvt, float* tau, float* work, + lapack_int *info ); +void LAPACK_dgeqpf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + lapack_int* jpvt, double* tau, double* work, + lapack_int *info ); +void LAPACK_cgeqpf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* jpvt, + lapack_complex_float* tau, lapack_complex_float* work, + float* rwork, lapack_int *info ); +void LAPACK_zgeqpf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* jpvt, + lapack_complex_double* tau, lapack_complex_double* work, + double* rwork, lapack_int *info ); +void LAPACK_sgeqp3( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + lapack_int* jpvt, float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dgeqp3( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + lapack_int* jpvt, double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cgeqp3( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* jpvt, + lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int *info ); +void LAPACK_zgeqp3( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* jpvt, + lapack_complex_double* tau, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int *info ); +void LAPACK_sorgqr( lapack_int* m, lapack_int* n, lapack_int* k, float* a, + lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorgqr( lapack_int* m, lapack_int* n, lapack_int* k, double* a, + lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sormqr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const float* a, lapack_int* lda, + const float* tau, float* c, lapack_int* ldc, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dormqr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const double* a, lapack_int* lda, + const double* tau, double* c, lapack_int* ldc, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cungqr( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zungqr( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmqr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmqr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgelqf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgelqf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgelqf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgelqf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sorglq( lapack_int* m, lapack_int* n, lapack_int* k, float* a, + lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorglq( lapack_int* m, lapack_int* n, lapack_int* k, double* a, + lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sormlq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const float* a, lapack_int* lda, + const float* tau, float* c, lapack_int* ldc, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dormlq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const double* a, lapack_int* lda, + const double* tau, double* c, lapack_int* ldc, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cunglq( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zunglq( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmlq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmlq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgeqlf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgeqlf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgeqlf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgeqlf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sorgql( lapack_int* m, lapack_int* n, lapack_int* k, float* a, + lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorgql( lapack_int* m, lapack_int* n, lapack_int* k, double* a, + lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cungql( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zungql( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sormql( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const float* a, lapack_int* lda, + const float* tau, float* c, lapack_int* ldc, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dormql( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const double* a, lapack_int* lda, + const double* tau, double* c, lapack_int* ldc, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cunmql( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmql( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgerqf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgerqf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgerqf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgerqf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sorgrq( lapack_int* m, lapack_int* n, lapack_int* k, float* a, + lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorgrq( lapack_int* m, lapack_int* n, lapack_int* k, double* a, + lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cungrq( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zungrq( lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sormrq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const float* a, lapack_int* lda, + const float* tau, float* c, lapack_int* ldc, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dormrq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const double* a, lapack_int* lda, + const double* tau, double* c, lapack_int* ldc, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cunmrq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmrq( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_stzrzf( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dtzrzf( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ctzrzf( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ztzrzf( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sormrz( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, const float* a, + lapack_int* lda, const float* tau, float* c, + lapack_int* ldc, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dormrz( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, const double* a, + lapack_int* lda, const double* tau, double* c, + lapack_int* ldc, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmrz( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmrz( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, lapack_complex_double* c, + lapack_int* ldc, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sggqrf( lapack_int* n, lapack_int* m, lapack_int* p, float* a, + lapack_int* lda, float* taua, float* b, lapack_int* ldb, + float* taub, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dggqrf( lapack_int* n, lapack_int* m, lapack_int* p, double* a, + lapack_int* lda, double* taua, double* b, lapack_int* ldb, + double* taub, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cggqrf( lapack_int* n, lapack_int* m, lapack_int* p, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* taua, lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* taub, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zggqrf( lapack_int* n, lapack_int* m, lapack_int* p, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* taua, lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* taub, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sggrqf( lapack_int* m, lapack_int* p, lapack_int* n, float* a, + lapack_int* lda, float* taua, float* b, lapack_int* ldb, + float* taub, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dggrqf( lapack_int* m, lapack_int* p, lapack_int* n, double* a, + lapack_int* lda, double* taua, double* b, lapack_int* ldb, + double* taub, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cggrqf( lapack_int* m, lapack_int* p, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* taua, lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* taub, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zggrqf( lapack_int* m, lapack_int* p, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* taua, lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* taub, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgebrd( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* d, float* e, float* tauq, float* taup, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dgebrd( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* d, double* e, double* tauq, double* taup, + double* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_cgebrd( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, float* d, float* e, + lapack_complex_float* tauq, lapack_complex_float* taup, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgebrd( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, double* d, double* e, + lapack_complex_double* tauq, lapack_complex_double* taup, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgbbrd( char* vect, lapack_int* m, lapack_int* n, lapack_int* ncc, + lapack_int* kl, lapack_int* ku, float* ab, lapack_int* ldab, + float* d, float* e, float* q, lapack_int* ldq, float* pt, + lapack_int* ldpt, float* c, lapack_int* ldc, float* work, + lapack_int *info ); +void LAPACK_dgbbrd( char* vect, lapack_int* m, lapack_int* n, lapack_int* ncc, + lapack_int* kl, lapack_int* ku, double* ab, + lapack_int* ldab, double* d, double* e, double* q, + lapack_int* ldq, double* pt, lapack_int* ldpt, double* c, + lapack_int* ldc, double* work, lapack_int *info ); +void LAPACK_cgbbrd( char* vect, lapack_int* m, lapack_int* n, lapack_int* ncc, + lapack_int* kl, lapack_int* ku, lapack_complex_float* ab, + lapack_int* ldab, float* d, float* e, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* pt, lapack_int* ldpt, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zgbbrd( char* vect, lapack_int* m, lapack_int* n, lapack_int* ncc, + lapack_int* kl, lapack_int* ku, lapack_complex_double* ab, + lapack_int* ldab, double* d, double* e, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* pt, lapack_int* ldpt, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sorgbr( char* vect, lapack_int* m, lapack_int* n, lapack_int* k, + float* a, lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorgbr( char* vect, lapack_int* m, lapack_int* n, lapack_int* k, + double* a, lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sormbr( char* vect, char* side, char* trans, lapack_int* m, + lapack_int* n, lapack_int* k, const float* a, + lapack_int* lda, const float* tau, float* c, + lapack_int* ldc, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dormbr( char* vect, char* side, char* trans, lapack_int* m, + lapack_int* n, lapack_int* k, const double* a, + lapack_int* lda, const double* tau, double* c, + lapack_int* ldc, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cungbr( char* vect, lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zungbr( char* vect, lapack_int* m, lapack_int* n, lapack_int* k, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmbr( char* vect, char* side, char* trans, lapack_int* m, + lapack_int* n, lapack_int* k, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmbr( char* vect, char* side, char* trans, lapack_int* m, + lapack_int* n, lapack_int* k, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, lapack_complex_double* c, + lapack_int* ldc, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sbdsqr( char* uplo, lapack_int* n, lapack_int* ncvt, + lapack_int* nru, lapack_int* ncc, float* d, float* e, + float* vt, lapack_int* ldvt, float* u, lapack_int* ldu, + float* c, lapack_int* ldc, float* work, lapack_int *info ); +void LAPACK_dbdsqr( char* uplo, lapack_int* n, lapack_int* ncvt, + lapack_int* nru, lapack_int* ncc, double* d, double* e, + double* vt, lapack_int* ldvt, double* u, lapack_int* ldu, + double* c, lapack_int* ldc, double* work, + lapack_int *info ); +void LAPACK_cbdsqr( char* uplo, lapack_int* n, lapack_int* ncvt, + lapack_int* nru, lapack_int* ncc, float* d, float* e, + lapack_complex_float* vt, lapack_int* ldvt, + lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* c, lapack_int* ldc, float* work, + lapack_int *info ); +void LAPACK_zbdsqr( char* uplo, lapack_int* n, lapack_int* ncvt, + lapack_int* nru, lapack_int* ncc, double* d, double* e, + lapack_complex_double* vt, lapack_int* ldvt, + lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* c, lapack_int* ldc, double* work, + lapack_int *info ); +void LAPACK_sbdsdc( char* uplo, char* compq, lapack_int* n, float* d, float* e, + float* u, lapack_int* ldu, float* vt, lapack_int* ldvt, + float* q, lapack_int* iq, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dbdsdc( char* uplo, char* compq, lapack_int* n, double* d, + double* e, double* u, lapack_int* ldu, double* vt, + lapack_int* ldvt, double* q, lapack_int* iq, double* work, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ssytrd( char* uplo, lapack_int* n, float* a, lapack_int* lda, + float* d, float* e, float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dsytrd( char* uplo, lapack_int* n, double* a, lapack_int* lda, + double* d, double* e, double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sorgtr( char* uplo, lapack_int* n, float* a, lapack_int* lda, + const float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dorgtr( char* uplo, lapack_int* n, double* a, lapack_int* lda, + const double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sormtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const float* a, lapack_int* lda, + const float* tau, float* c, lapack_int* ldc, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dormtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const double* a, lapack_int* lda, + const double* tau, double* c, lapack_int* ldc, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_chetrd( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, float* d, float* e, + lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zhetrd( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, double* d, double* e, + lapack_complex_double* tau, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cungtr( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zungtr( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zunmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ssptrd( char* uplo, lapack_int* n, float* ap, float* d, float* e, + float* tau, lapack_int *info ); +void LAPACK_dsptrd( char* uplo, lapack_int* n, double* ap, double* d, double* e, + double* tau, lapack_int *info ); +void LAPACK_sopgtr( char* uplo, lapack_int* n, const float* ap, + const float* tau, float* q, lapack_int* ldq, float* work, + lapack_int *info ); +void LAPACK_dopgtr( char* uplo, lapack_int* n, const double* ap, + const double* tau, double* q, lapack_int* ldq, double* work, + lapack_int *info ); +void LAPACK_sopmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const float* ap, const float* tau, float* c, + lapack_int* ldc, float* work, lapack_int *info ); +void LAPACK_dopmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const double* ap, const double* tau, + double* c, lapack_int* ldc, double* work, + lapack_int *info ); +void LAPACK_chptrd( char* uplo, lapack_int* n, lapack_complex_float* ap, + float* d, float* e, lapack_complex_float* tau, + lapack_int *info ); +void LAPACK_zhptrd( char* uplo, lapack_int* n, lapack_complex_double* ap, + double* d, double* e, lapack_complex_double* tau, + lapack_int *info ); +void LAPACK_cupgtr( char* uplo, lapack_int* n, const lapack_complex_float* ap, + const lapack_complex_float* tau, lapack_complex_float* q, + lapack_int* ldq, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zupgtr( char* uplo, lapack_int* n, const lapack_complex_double* ap, + const lapack_complex_double* tau, lapack_complex_double* q, + lapack_int* ldq, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_cupmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const lapack_complex_float* ap, + const lapack_complex_float* tau, lapack_complex_float* c, + lapack_int* ldc, lapack_complex_float* work, + lapack_int *info ); +void LAPACK_zupmtr( char* side, char* uplo, char* trans, lapack_int* m, + lapack_int* n, const lapack_complex_double* ap, + const lapack_complex_double* tau, lapack_complex_double* c, + lapack_int* ldc, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_ssbtrd( char* vect, char* uplo, lapack_int* n, lapack_int* kd, + float* ab, lapack_int* ldab, float* d, float* e, float* q, + lapack_int* ldq, float* work, lapack_int *info ); +void LAPACK_dsbtrd( char* vect, char* uplo, lapack_int* n, lapack_int* kd, + double* ab, lapack_int* ldab, double* d, double* e, + double* q, lapack_int* ldq, double* work, + lapack_int *info ); +void LAPACK_chbtrd( char* vect, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_float* ab, lapack_int* ldab, float* d, + float* e, lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zhbtrd( char* vect, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_double* ab, lapack_int* ldab, double* d, + double* e, lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_ssterf( lapack_int* n, float* d, float* e, lapack_int *info ); +void LAPACK_dsterf( lapack_int* n, double* d, double* e, lapack_int *info ); +void LAPACK_ssteqr( char* compz, lapack_int* n, float* d, float* e, float* z, + lapack_int* ldz, float* work, lapack_int *info ); +void LAPACK_dsteqr( char* compz, lapack_int* n, double* d, double* e, double* z, + lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_csteqr( char* compz, lapack_int* n, float* d, float* e, + lapack_complex_float* z, lapack_int* ldz, float* work, + lapack_int *info ); +void LAPACK_zsteqr( char* compz, lapack_int* n, double* d, double* e, + lapack_complex_double* z, lapack_int* ldz, double* work, + lapack_int *info ); +void LAPACK_sstemr( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + lapack_int* m, float* w, float* z, lapack_int* ldz, + lapack_int* nzc, lapack_int* isuppz, lapack_logical* tryrac, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_dstemr( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, lapack_int* m, double* w, double* z, + lapack_int* ldz, lapack_int* nzc, lapack_int* isuppz, + lapack_logical* tryrac, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_cstemr( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_int* nzc, lapack_int* isuppz, + lapack_logical* tryrac, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_zstemr( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int* ldz, lapack_int* nzc, + lapack_int* isuppz, lapack_logical* tryrac, double* work, + lapack_int* lwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_sstedc( char* compz, lapack_int* n, float* d, float* e, float* z, + lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dstedc( char* compz, lapack_int* n, double* d, double* e, double* z, + lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_cstedc( char* compz, lapack_int* n, float* d, float* e, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zstedc( char* compz, lapack_int* n, double* d, double* e, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_sstegr( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, float* z, + lapack_int* ldz, lapack_int* isuppz, float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_dstegr( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, lapack_int* isuppz, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_cstegr( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_int* isuppz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_zstegr( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_int* isuppz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_spteqr( char* compz, lapack_int* n, float* d, float* e, float* z, + lapack_int* ldz, float* work, lapack_int *info ); +void LAPACK_dpteqr( char* compz, lapack_int* n, double* d, double* e, double* z, + lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_cpteqr( char* compz, lapack_int* n, float* d, float* e, + lapack_complex_float* z, lapack_int* ldz, float* work, + lapack_int *info ); +void LAPACK_zpteqr( char* compz, lapack_int* n, double* d, double* e, + lapack_complex_double* z, lapack_int* ldz, double* work, + lapack_int *info ); +void LAPACK_sstebz( char* range, char* order, lapack_int* n, float* vl, + float* vu, lapack_int* il, lapack_int* iu, float* abstol, + const float* d, const float* e, lapack_int* m, + lapack_int* nsplit, float* w, lapack_int* iblock, + lapack_int* isplit, float* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dstebz( char* range, char* order, lapack_int* n, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + const double* d, const double* e, lapack_int* m, + lapack_int* nsplit, double* w, lapack_int* iblock, + lapack_int* isplit, double* work, lapack_int* iwork, + lapack_int *info ); +void LAPACK_sstein( lapack_int* n, const float* d, const float* e, + lapack_int* m, const float* w, const lapack_int* iblock, + const lapack_int* isplit, float* z, lapack_int* ldz, + float* work, lapack_int* iwork, lapack_int* ifailv, + lapack_int *info ); +void LAPACK_dstein( lapack_int* n, const double* d, const double* e, + lapack_int* m, const double* w, const lapack_int* iblock, + const lapack_int* isplit, double* z, lapack_int* ldz, + double* work, lapack_int* iwork, lapack_int* ifailv, + lapack_int *info ); +void LAPACK_cstein( lapack_int* n, const float* d, const float* e, + lapack_int* m, const float* w, const lapack_int* iblock, + const lapack_int* isplit, lapack_complex_float* z, + lapack_int* ldz, float* work, lapack_int* iwork, + lapack_int* ifailv, lapack_int *info ); +void LAPACK_zstein( lapack_int* n, const double* d, const double* e, + lapack_int* m, const double* w, const lapack_int* iblock, + const lapack_int* isplit, lapack_complex_double* z, + lapack_int* ldz, double* work, lapack_int* iwork, + lapack_int* ifailv, lapack_int *info ); +void LAPACK_sdisna( char* job, lapack_int* m, lapack_int* n, const float* d, + float* sep, lapack_int *info ); +void LAPACK_ddisna( char* job, lapack_int* m, lapack_int* n, const double* d, + double* sep, lapack_int *info ); +void LAPACK_ssygst( lapack_int* itype, char* uplo, lapack_int* n, float* a, + lapack_int* lda, const float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_dsygst( lapack_int* itype, char* uplo, lapack_int* n, double* a, + lapack_int* lda, const double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_chegst( lapack_int* itype, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_zhegst( lapack_int* itype, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* b, lapack_int* ldb, + lapack_int *info ); +void LAPACK_sspgst( lapack_int* itype, char* uplo, lapack_int* n, float* ap, + const float* bp, lapack_int *info ); +void LAPACK_dspgst( lapack_int* itype, char* uplo, lapack_int* n, double* ap, + const double* bp, lapack_int *info ); +void LAPACK_chpgst( lapack_int* itype, char* uplo, lapack_int* n, + lapack_complex_float* ap, const lapack_complex_float* bp, + lapack_int *info ); +void LAPACK_zhpgst( lapack_int* itype, char* uplo, lapack_int* n, + lapack_complex_double* ap, const lapack_complex_double* bp, + lapack_int *info ); +void LAPACK_ssbgst( char* vect, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, float* ab, lapack_int* ldab, + const float* bb, lapack_int* ldbb, float* x, + lapack_int* ldx, float* work, lapack_int *info ); +void LAPACK_dsbgst( char* vect, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, double* ab, lapack_int* ldab, + const double* bb, lapack_int* ldbb, double* x, + lapack_int* ldx, double* work, lapack_int *info ); +void LAPACK_chbgst( char* vect, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_float* ab, lapack_int* ldab, + const lapack_complex_float* bb, lapack_int* ldbb, + lapack_complex_float* x, lapack_int* ldx, + lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zhbgst( char* vect, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_double* ab, lapack_int* ldab, + const lapack_complex_double* bb, lapack_int* ldbb, + lapack_complex_double* x, lapack_int* ldx, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_spbstf( char* uplo, lapack_int* n, lapack_int* kb, float* bb, + lapack_int* ldbb, lapack_int *info ); +void LAPACK_dpbstf( char* uplo, lapack_int* n, lapack_int* kb, double* bb, + lapack_int* ldbb, lapack_int *info ); +void LAPACK_cpbstf( char* uplo, lapack_int* n, lapack_int* kb, + lapack_complex_float* bb, lapack_int* ldbb, + lapack_int *info ); +void LAPACK_zpbstf( char* uplo, lapack_int* n, lapack_int* kb, + lapack_complex_double* bb, lapack_int* ldbb, + lapack_int *info ); +void LAPACK_sgehrd( lapack_int* n, lapack_int* ilo, lapack_int* ihi, float* a, + lapack_int* lda, float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgehrd( lapack_int* n, lapack_int* ilo, lapack_int* ihi, double* a, + lapack_int* lda, double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cgehrd( lapack_int* n, lapack_int* ilo, lapack_int* ihi, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zgehrd( lapack_int* n, lapack_int* ilo, lapack_int* ihi, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* tau, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sorghr( lapack_int* n, lapack_int* ilo, lapack_int* ihi, float* a, + lapack_int* lda, const float* tau, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dorghr( lapack_int* n, lapack_int* ilo, lapack_int* ihi, double* a, + lapack_int* lda, const double* tau, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sormhr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, const float* a, + lapack_int* lda, const float* tau, float* c, + lapack_int* ldc, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dormhr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, const double* a, + lapack_int* lda, const double* tau, double* c, + lapack_int* ldc, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunghr( lapack_int* n, lapack_int* ilo, lapack_int* ihi, + lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zunghr( lapack_int* n, lapack_int* ilo, lapack_int* ihi, + lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cunmhr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* tau, lapack_complex_float* c, + lapack_int* ldc, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zunmhr( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* tau, lapack_complex_double* c, + lapack_int* ldc, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sgebal( char* job, lapack_int* n, float* a, lapack_int* lda, + lapack_int* ilo, lapack_int* ihi, float* scale, + lapack_int *info ); +void LAPACK_dgebal( char* job, lapack_int* n, double* a, lapack_int* lda, + lapack_int* ilo, lapack_int* ihi, double* scale, + lapack_int *info ); +void LAPACK_cgebal( char* job, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* ilo, lapack_int* ihi, + float* scale, lapack_int *info ); +void LAPACK_zgebal( char* job, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* ilo, lapack_int* ihi, + double* scale, lapack_int *info ); +void LAPACK_sgebak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const float* scale, lapack_int* m, + float* v, lapack_int* ldv, lapack_int *info ); +void LAPACK_dgebak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const double* scale, lapack_int* m, + double* v, lapack_int* ldv, lapack_int *info ); +void LAPACK_cgebak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const float* scale, lapack_int* m, + lapack_complex_float* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_zgebak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const double* scale, lapack_int* m, + lapack_complex_double* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_shseqr( char* job, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, float* h, lapack_int* ldh, float* wr, + float* wi, float* z, lapack_int* ldz, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dhseqr( char* job, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, double* h, lapack_int* ldh, double* wr, + double* wi, double* z, lapack_int* ldz, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_chseqr( char* job, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, lapack_complex_float* h, lapack_int* ldh, + lapack_complex_float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zhseqr( char* job, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, lapack_complex_double* h, lapack_int* ldh, + lapack_complex_double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_shsein( char* job, char* eigsrc, char* initv, + lapack_logical* select, lapack_int* n, const float* h, + lapack_int* ldh, float* wr, const float* wi, float* vl, + lapack_int* ldvl, float* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, float* work, + lapack_int* ifaill, lapack_int* ifailr, lapack_int *info ); +void LAPACK_dhsein( char* job, char* eigsrc, char* initv, + lapack_logical* select, lapack_int* n, const double* h, + lapack_int* ldh, double* wr, const double* wi, double* vl, + lapack_int* ldvl, double* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, double* work, + lapack_int* ifaill, lapack_int* ifailr, lapack_int *info ); +void LAPACK_chsein( char* job, char* eigsrc, char* initv, + const lapack_logical* select, lapack_int* n, + const lapack_complex_float* h, lapack_int* ldh, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int* ldvl, lapack_complex_float* vr, + lapack_int* ldvr, lapack_int* mm, lapack_int* m, + lapack_complex_float* work, float* rwork, + lapack_int* ifaill, lapack_int* ifailr, lapack_int *info ); +void LAPACK_zhsein( char* job, char* eigsrc, char* initv, + const lapack_logical* select, lapack_int* n, + const lapack_complex_double* h, lapack_int* ldh, + lapack_complex_double* w, lapack_complex_double* vl, + lapack_int* ldvl, lapack_complex_double* vr, + lapack_int* ldvr, lapack_int* mm, lapack_int* m, + lapack_complex_double* work, double* rwork, + lapack_int* ifaill, lapack_int* ifailr, lapack_int *info ); +void LAPACK_strevc( char* side, char* howmny, lapack_logical* select, + lapack_int* n, const float* t, lapack_int* ldt, float* vl, + lapack_int* ldvl, float* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, float* work, + lapack_int *info ); +void LAPACK_dtrevc( char* side, char* howmny, lapack_logical* select, + lapack_int* n, const double* t, lapack_int* ldt, double* vl, + lapack_int* ldvl, double* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, double* work, + lapack_int *info ); +void LAPACK_ctrevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* vl, lapack_int* ldvl, + lapack_complex_float* vr, lapack_int* ldvr, lapack_int* mm, + lapack_int* m, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztrevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* vl, lapack_int* ldvl, + lapack_complex_double* vr, lapack_int* ldvr, lapack_int* mm, + lapack_int* m, lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_strsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const float* t, lapack_int* ldt, + const float* vl, lapack_int* ldvl, const float* vr, + lapack_int* ldvr, float* s, float* sep, lapack_int* mm, + lapack_int* m, float* work, lapack_int* ldwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dtrsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const double* t, lapack_int* ldt, + const double* vl, lapack_int* ldvl, const double* vr, + lapack_int* ldvr, double* s, double* sep, lapack_int* mm, + lapack_int* m, double* work, lapack_int* ldwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ctrsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_float* t, + lapack_int* ldt, const lapack_complex_float* vl, + lapack_int* ldvl, const lapack_complex_float* vr, + lapack_int* ldvr, float* s, float* sep, lapack_int* mm, + lapack_int* m, lapack_complex_float* work, + lapack_int* ldwork, float* rwork, lapack_int *info ); +void LAPACK_ztrsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_double* t, + lapack_int* ldt, const lapack_complex_double* vl, + lapack_int* ldvl, const lapack_complex_double* vr, + lapack_int* ldvr, double* s, double* sep, lapack_int* mm, + lapack_int* m, lapack_complex_double* work, + lapack_int* ldwork, double* rwork, lapack_int *info ); +void LAPACK_strexc( char* compq, lapack_int* n, float* t, lapack_int* ldt, + float* q, lapack_int* ldq, lapack_int* ifst, + lapack_int* ilst, float* work, lapack_int *info ); +void LAPACK_dtrexc( char* compq, lapack_int* n, double* t, lapack_int* ldt, + double* q, lapack_int* ldq, lapack_int* ifst, + lapack_int* ilst, double* work, lapack_int *info ); +void LAPACK_ctrexc( char* compq, lapack_int* n, lapack_complex_float* t, + lapack_int* ldt, lapack_complex_float* q, lapack_int* ldq, + lapack_int* ifst, lapack_int* ilst, lapack_int *info ); +void LAPACK_ztrexc( char* compq, lapack_int* n, lapack_complex_double* t, + lapack_int* ldt, lapack_complex_double* q, lapack_int* ldq, + lapack_int* ifst, lapack_int* ilst, lapack_int *info ); +void LAPACK_strsen( char* job, char* compq, const lapack_logical* select, + lapack_int* n, float* t, lapack_int* ldt, float* q, + lapack_int* ldq, float* wr, float* wi, lapack_int* m, + float* s, float* sep, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dtrsen( char* job, char* compq, const lapack_logical* select, + lapack_int* n, double* t, lapack_int* ldt, double* q, + lapack_int* ldq, double* wr, double* wi, lapack_int* m, + double* s, double* sep, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_ctrsen( char* job, char* compq, const lapack_logical* select, + lapack_int* n, lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* w, lapack_int* m, float* s, + float* sep, lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_ztrsen( char* job, char* compq, const lapack_logical* select, + lapack_int* n, lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* w, lapack_int* m, double* s, + double* sep, lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_strsyl( char* trana, char* tranb, lapack_int* isgn, lapack_int* m, + lapack_int* n, const float* a, lapack_int* lda, + const float* b, lapack_int* ldb, float* c, lapack_int* ldc, + float* scale, lapack_int *info ); +void LAPACK_dtrsyl( char* trana, char* tranb, lapack_int* isgn, lapack_int* m, + lapack_int* n, const double* a, lapack_int* lda, + const double* b, lapack_int* ldb, double* c, + lapack_int* ldc, double* scale, lapack_int *info ); +void LAPACK_ctrsyl( char* trana, char* tranb, lapack_int* isgn, lapack_int* m, + lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* b, + lapack_int* ldb, lapack_complex_float* c, lapack_int* ldc, + float* scale, lapack_int *info ); +void LAPACK_ztrsyl( char* trana, char* tranb, lapack_int* isgn, lapack_int* m, + lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* b, + lapack_int* ldb, lapack_complex_double* c, lapack_int* ldc, + double* scale, lapack_int *info ); +void LAPACK_sgghrd( char* compq, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, float* a, lapack_int* lda, float* b, + lapack_int* ldb, float* q, lapack_int* ldq, float* z, + lapack_int* ldz, lapack_int *info ); +void LAPACK_dgghrd( char* compq, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, double* a, lapack_int* lda, double* b, + lapack_int* ldb, double* q, lapack_int* ldq, double* z, + lapack_int* ldz, lapack_int *info ); +void LAPACK_cgghrd( char* compq, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* z, lapack_int* ldz, + lapack_int *info ); +void LAPACK_zgghrd( char* compq, char* compz, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* z, lapack_int* ldz, + lapack_int *info ); +void LAPACK_sggbal( char* job, lapack_int* n, float* a, lapack_int* lda, + float* b, lapack_int* ldb, lapack_int* ilo, lapack_int* ihi, + float* lscale, float* rscale, float* work, + lapack_int *info ); +void LAPACK_dggbal( char* job, lapack_int* n, double* a, lapack_int* lda, + double* b, lapack_int* ldb, lapack_int* ilo, + lapack_int* ihi, double* lscale, double* rscale, + double* work, lapack_int *info ); +void LAPACK_cggbal( char* job, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* b, lapack_int* ldb, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale, float* work, lapack_int *info ); +void LAPACK_zggbal( char* job, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* b, lapack_int* ldb, + lapack_int* ilo, lapack_int* ihi, double* lscale, + double* rscale, double* work, lapack_int *info ); +void LAPACK_sggbak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const float* lscale, const float* rscale, + lapack_int* m, float* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_dggbak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const double* lscale, const double* rscale, + lapack_int* m, double* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_cggbak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const float* lscale, const float* rscale, + lapack_int* m, lapack_complex_float* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_zggbak( char* job, char* side, lapack_int* n, lapack_int* ilo, + lapack_int* ihi, const double* lscale, const double* rscale, + lapack_int* m, lapack_complex_double* v, lapack_int* ldv, + lapack_int *info ); +void LAPACK_shgeqz( char* job, char* compq, char* compz, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, float* h, lapack_int* ldh, + float* t, lapack_int* ldt, float* alphar, float* alphai, + float* beta, float* q, lapack_int* ldq, float* z, + lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dhgeqz( char* job, char* compq, char* compz, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, double* h, + lapack_int* ldh, double* t, lapack_int* ldt, double* alphar, + double* alphai, double* beta, double* q, lapack_int* ldq, + double* z, lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_chgeqz( char* job, char* compq, char* compz, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, lapack_complex_float* h, + lapack_int* ldh, lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zhgeqz( char* job, char* compq, char* compz, lapack_int* n, + lapack_int* ilo, lapack_int* ihi, lapack_complex_double* h, + lapack_int* ldh, lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_stgevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, const float* s, lapack_int* lds, + const float* p, lapack_int* ldp, float* vl, + lapack_int* ldvl, float* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, float* work, + lapack_int *info ); +void LAPACK_dtgevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, const double* s, lapack_int* lds, + const double* p, lapack_int* ldp, double* vl, + lapack_int* ldvl, double* vr, lapack_int* ldvr, + lapack_int* mm, lapack_int* m, double* work, + lapack_int *info ); +void LAPACK_ctgevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_float* s, + lapack_int* lds, const lapack_complex_float* p, + lapack_int* ldp, lapack_complex_float* vl, lapack_int* ldvl, + lapack_complex_float* vr, lapack_int* ldvr, lapack_int* mm, + lapack_int* m, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_ztgevc( char* side, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_double* s, + lapack_int* lds, const lapack_complex_double* p, + lapack_int* ldp, lapack_complex_double* vl, + lapack_int* ldvl, lapack_complex_double* vr, + lapack_int* ldvr, lapack_int* mm, lapack_int* m, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_stgexc( lapack_logical* wantq, lapack_logical* wantz, lapack_int* n, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* q, lapack_int* ldq, float* z, lapack_int* ldz, + lapack_int* ifst, lapack_int* ilst, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dtgexc( lapack_logical* wantq, lapack_logical* wantz, lapack_int* n, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* q, lapack_int* ldq, double* z, lapack_int* ldz, + lapack_int* ifst, lapack_int* ilst, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_ctgexc( lapack_logical* wantq, lapack_logical* wantz, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* z, lapack_int* ldz, lapack_int* ifst, + lapack_int* ilst, lapack_int *info ); +void LAPACK_ztgexc( lapack_logical* wantq, lapack_logical* wantz, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* z, lapack_int* ldz, lapack_int* ifst, + lapack_int* ilst, lapack_int *info ); +void LAPACK_stgsen( lapack_int* ijob, lapack_logical* wantq, + lapack_logical* wantz, const lapack_logical* select, + lapack_int* n, float* a, lapack_int* lda, float* b, + lapack_int* ldb, float* alphar, float* alphai, float* beta, + float* q, lapack_int* ldq, float* z, lapack_int* ldz, + lapack_int* m, float* pl, float* pr, float* dif, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_dtgsen( lapack_int* ijob, lapack_logical* wantq, + lapack_logical* wantz, const lapack_logical* select, + lapack_int* n, double* a, lapack_int* lda, double* b, + lapack_int* ldb, double* alphar, double* alphai, + double* beta, double* q, lapack_int* ldq, double* z, + lapack_int* ldz, lapack_int* m, double* pl, double* pr, + double* dif, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_ctgsen( lapack_int* ijob, lapack_logical* wantq, + lapack_logical* wantz, const lapack_logical* select, + lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* z, lapack_int* ldz, lapack_int* m, + float* pl, float* pr, float* dif, + lapack_complex_float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_ztgsen( lapack_int* ijob, lapack_logical* wantq, + lapack_logical* wantz, const lapack_logical* select, + lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* z, lapack_int* ldz, lapack_int* m, + double* pl, double* pr, double* dif, + lapack_complex_double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_stgsyl( char* trans, lapack_int* ijob, lapack_int* m, lapack_int* n, + const float* a, lapack_int* lda, const float* b, + lapack_int* ldb, float* c, lapack_int* ldc, const float* d, + lapack_int* ldd, const float* e, lapack_int* lde, float* f, + lapack_int* ldf, float* scale, float* dif, float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_dtgsyl( char* trans, lapack_int* ijob, lapack_int* m, lapack_int* n, + const double* a, lapack_int* lda, const double* b, + lapack_int* ldb, double* c, lapack_int* ldc, + const double* d, lapack_int* ldd, const double* e, + lapack_int* lde, double* f, lapack_int* ldf, double* scale, + double* dif, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ctgsyl( char* trans, lapack_int* ijob, lapack_int* m, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, + const lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* c, lapack_int* ldc, + const lapack_complex_float* d, lapack_int* ldd, + const lapack_complex_float* e, lapack_int* lde, + lapack_complex_float* f, lapack_int* ldf, float* scale, + float* dif, lapack_complex_float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ztgsyl( char* trans, lapack_int* ijob, lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, + const lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* c, lapack_int* ldc, + const lapack_complex_double* d, lapack_int* ldd, + const lapack_complex_double* e, lapack_int* lde, + lapack_complex_double* f, lapack_int* ldf, double* scale, + double* dif, lapack_complex_double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_stgsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const float* a, lapack_int* lda, + const float* b, lapack_int* ldb, const float* vl, + lapack_int* ldvl, const float* vr, lapack_int* ldvr, + float* s, float* dif, lapack_int* mm, lapack_int* m, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dtgsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const double* a, lapack_int* lda, + const double* b, lapack_int* ldb, const double* vl, + lapack_int* ldvl, const double* vr, lapack_int* ldvr, + double* s, double* dif, lapack_int* mm, lapack_int* m, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int *info ); +void LAPACK_ctgsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, const lapack_complex_float* b, + lapack_int* ldb, const lapack_complex_float* vl, + lapack_int* ldvl, const lapack_complex_float* vr, + lapack_int* ldvr, float* s, float* dif, lapack_int* mm, + lapack_int* m, lapack_complex_float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_ztgsna( char* job, char* howmny, const lapack_logical* select, + lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, const lapack_complex_double* b, + lapack_int* ldb, const lapack_complex_double* vl, + lapack_int* ldvl, const lapack_complex_double* vr, + lapack_int* ldvr, double* s, double* dif, lapack_int* mm, + lapack_int* m, lapack_complex_double* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_sggsvp( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, float* a, lapack_int* lda, + float* b, lapack_int* ldb, float* tola, float* tolb, + lapack_int* k, lapack_int* l, float* u, lapack_int* ldu, + float* v, lapack_int* ldv, float* q, lapack_int* ldq, + lapack_int* iwork, float* tau, float* work, + lapack_int *info ); +void LAPACK_dggsvp( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, double* a, lapack_int* lda, + double* b, lapack_int* ldb, double* tola, double* tolb, + lapack_int* k, lapack_int* l, double* u, lapack_int* ldu, + double* v, lapack_int* ldv, double* q, lapack_int* ldq, + lapack_int* iwork, double* tau, double* work, + lapack_int *info ); +void LAPACK_cggsvp( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* b, lapack_int* ldb, + float* tola, float* tolb, lapack_int* k, lapack_int* l, + lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* v, lapack_int* ldv, + lapack_complex_float* q, lapack_int* ldq, lapack_int* iwork, + float* rwork, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zggsvp( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* b, lapack_int* ldb, + double* tola, double* tolb, lapack_int* k, lapack_int* l, + lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* v, lapack_int* ldv, + lapack_complex_double* q, lapack_int* ldq, + lapack_int* iwork, double* rwork, + lapack_complex_double* tau, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_stgsja( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_int* k, lapack_int* l, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* tola, float* tolb, float* alpha, float* beta, + float* u, lapack_int* ldu, float* v, lapack_int* ldv, + float* q, lapack_int* ldq, float* work, lapack_int* ncycle, + lapack_int *info ); +void LAPACK_dtgsja( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_int* k, lapack_int* l, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* tola, double* tolb, double* alpha, double* beta, + double* u, lapack_int* ldu, double* v, lapack_int* ldv, + double* q, lapack_int* ldq, double* work, + lapack_int* ncycle, lapack_int *info ); +void LAPACK_ctgsja( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_int* k, lapack_int* l, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* tola, + float* tolb, float* alpha, float* beta, + lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* v, lapack_int* ldv, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* work, lapack_int* ncycle, + lapack_int *info ); +void LAPACK_ztgsja( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* p, lapack_int* n, lapack_int* k, lapack_int* l, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* tola, + double* tolb, double* alpha, double* beta, + lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* v, lapack_int* ldv, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* work, lapack_int* ncycle, + lapack_int *info ); +void LAPACK_sgels( char* trans, lapack_int* m, lapack_int* n, lapack_int* nrhs, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_dgels( char* trans, lapack_int* m, lapack_int* n, lapack_int* nrhs, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_cgels( char* trans, lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgels( char* trans, lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_sgelsy( lapack_int* m, lapack_int* n, lapack_int* nrhs, float* a, + lapack_int* lda, float* b, lapack_int* ldb, + lapack_int* jpvt, float* rcond, lapack_int* rank, + float* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_dgelsy( lapack_int* m, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, double* b, lapack_int* ldb, + lapack_int* jpvt, double* rcond, lapack_int* rank, + double* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_cgelsy( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, lapack_int* jpvt, + float* rcond, lapack_int* rank, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int *info ); +void LAPACK_zgelsy( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, lapack_int* jpvt, + double* rcond, lapack_int* rank, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_sgelss( lapack_int* m, lapack_int* n, lapack_int* nrhs, float* a, + lapack_int* lda, float* b, lapack_int* ldb, float* s, + float* rcond, lapack_int* rank, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dgelss( lapack_int* m, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* s, + double* rcond, lapack_int* rank, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cgelss( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* s, + float* rcond, lapack_int* rank, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int *info ); +void LAPACK_zgelss( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* s, + double* rcond, lapack_int* rank, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_sgelsd( lapack_int* m, lapack_int* n, lapack_int* nrhs, float* a, + lapack_int* lda, float* b, lapack_int* ldb, float* s, + float* rcond, lapack_int* rank, float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_dgelsd( lapack_int* m, lapack_int* n, lapack_int* nrhs, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* s, + double* rcond, lapack_int* rank, double* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_cgelsd( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* s, + float* rcond, lapack_int* rank, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int* iwork, + lapack_int *info ); +void LAPACK_zgelsd( lapack_int* m, lapack_int* n, lapack_int* nrhs, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* s, + double* rcond, lapack_int* rank, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_sgglse( lapack_int* m, lapack_int* n, lapack_int* p, float* a, + lapack_int* lda, float* b, lapack_int* ldb, float* c, + float* d, float* x, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgglse( lapack_int* m, lapack_int* n, lapack_int* p, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* c, + double* d, double* x, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgglse( lapack_int* m, lapack_int* n, lapack_int* p, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* c, lapack_complex_float* d, + lapack_complex_float* x, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zgglse( lapack_int* m, lapack_int* n, lapack_int* p, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* c, lapack_complex_double* d, + lapack_complex_double* x, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sggglm( lapack_int* n, lapack_int* m, lapack_int* p, float* a, + lapack_int* lda, float* b, lapack_int* ldb, float* d, + float* x, float* y, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dggglm( lapack_int* n, lapack_int* m, lapack_int* p, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* d, + double* x, double* y, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cggglm( lapack_int* n, lapack_int* m, lapack_int* p, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* d, lapack_complex_float* x, + lapack_complex_float* y, lapack_complex_float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_zggglm( lapack_int* n, lapack_int* m, lapack_int* p, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* d, lapack_complex_double* x, + lapack_complex_double* y, lapack_complex_double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_ssyev( char* jobz, char* uplo, lapack_int* n, float* a, + lapack_int* lda, float* w, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dsyev( char* jobz, char* uplo, lapack_int* n, double* a, + lapack_int* lda, double* w, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cheev( char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* w, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zheev( char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* w, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_ssyevd( char* jobz, char* uplo, lapack_int* n, float* a, + lapack_int* lda, float* w, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dsyevd( char* jobz, char* uplo, lapack_int* n, double* a, + lapack_int* lda, double* w, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_cheevd( char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* w, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zheevd( char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* w, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_ssyevx( char* jobz, char* range, char* uplo, lapack_int* n, + float* a, lapack_int* lda, float* vl, float* vu, + lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, float* z, lapack_int* ldz, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_dsyevx( char* jobz, char* range, char* uplo, lapack_int* n, + double* a, lapack_int* lda, double* vl, double* vu, + lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, double* z, lapack_int* ldz, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_cheevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* vl, + float* vu, lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_zheevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_ssyevr( char* jobz, char* range, char* uplo, lapack_int* n, + float* a, lapack_int* lda, float* vl, float* vu, + lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, float* z, lapack_int* ldz, + lapack_int* isuppz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dsyevr( char* jobz, char* range, char* uplo, lapack_int* n, + double* a, lapack_int* lda, double* vl, double* vu, + lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, double* z, lapack_int* ldz, + lapack_int* isuppz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_cheevr( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* vl, + float* vu, lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_int* isuppz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zheevr( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_int* isuppz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_sspev( char* jobz, char* uplo, lapack_int* n, float* ap, float* w, + float* z, lapack_int* ldz, float* work, lapack_int *info ); +void LAPACK_dspev( char* jobz, char* uplo, lapack_int* n, double* ap, double* w, + double* z, lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_chpev( char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* ap, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, float* rwork, + lapack_int *info ); +void LAPACK_zhpev( char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* ap, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sspevd( char* jobz, char* uplo, lapack_int* n, float* ap, float* w, + float* z, lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dspevd( char* jobz, char* uplo, lapack_int* n, double* ap, + double* w, double* z, lapack_int* ldz, double* work, + lapack_int* lwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_chpevd( char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* ap, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int* lrwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_zhpevd( char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* ap, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_sspevx( char* jobz, char* range, char* uplo, lapack_int* n, + float* ap, float* vl, float* vu, lapack_int* il, + lapack_int* iu, float* abstol, lapack_int* m, float* w, + float* z, lapack_int* ldz, float* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_dspevx( char* jobz, char* range, char* uplo, lapack_int* n, + double* ap, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, double* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_chpevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_float* ap, float* vl, float* vu, + lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_zhpevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_complex_double* ap, double* vl, double* vu, + lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_ssbev( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + float* ab, lapack_int* ldab, float* w, float* z, + lapack_int* ldz, float* work, lapack_int *info ); +void LAPACK_dsbev( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + double* ab, lapack_int* ldab, double* w, double* z, + lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_chbev( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_float* ab, lapack_int* ldab, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, float* rwork, lapack_int *info ); +void LAPACK_zhbev( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_double* ab, lapack_int* ldab, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_ssbevd( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + float* ab, lapack_int* ldab, float* w, float* z, + lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dsbevd( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + double* ab, lapack_int* ldab, double* w, double* z, + lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_chbevd( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_float* ab, lapack_int* ldab, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zhbevd( char* jobz, char* uplo, lapack_int* n, lapack_int* kd, + lapack_complex_double* ab, lapack_int* ldab, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_ssbevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* kd, float* ab, lapack_int* ldab, float* q, + lapack_int* ldq, float* vl, float* vu, lapack_int* il, + lapack_int* iu, float* abstol, lapack_int* m, float* w, + float* z, lapack_int* ldz, float* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_dsbevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* kd, double* ab, lapack_int* ldab, double* q, + lapack_int* ldq, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, double* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_chbevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* kd, lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* q, lapack_int* ldq, float* vl, + float* vu, lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_zhbevx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* kd, lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* q, lapack_int* ldq, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_sstev( char* jobz, lapack_int* n, float* d, float* e, float* z, + lapack_int* ldz, float* work, lapack_int *info ); +void LAPACK_dstev( char* jobz, lapack_int* n, double* d, double* e, double* z, + lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_sstevd( char* jobz, lapack_int* n, float* d, float* e, float* z, + lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_dstevd( char* jobz, lapack_int* n, double* d, double* e, double* z, + lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_sstevx( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, float* z, + lapack_int* ldz, float* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_dstevx( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, double* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_sstevr( char* jobz, char* range, lapack_int* n, float* d, float* e, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, float* z, + lapack_int* ldz, lapack_int* isuppz, float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_dstevr( char* jobz, char* range, lapack_int* n, double* d, + double* e, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, lapack_int* isuppz, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_sgees( char* jobvs, char* sort, LAPACK_S_SELECT2 select, + lapack_int* n, float* a, lapack_int* lda, lapack_int* sdim, + float* wr, float* wi, float* vs, lapack_int* ldvs, + float* work, lapack_int* lwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_dgees( char* jobvs, char* sort, LAPACK_D_SELECT2 select, + lapack_int* n, double* a, lapack_int* lda, lapack_int* sdim, + double* wr, double* wi, double* vs, lapack_int* ldvs, + double* work, lapack_int* lwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_cgees( char* jobvs, char* sort, LAPACK_C_SELECT1 select, + lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_int* sdim, lapack_complex_float* w, + lapack_complex_float* vs, lapack_int* ldvs, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_logical* bwork, lapack_int *info ); +void LAPACK_zgees( char* jobvs, char* sort, LAPACK_Z_SELECT1 select, + lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_int* sdim, lapack_complex_double* w, + lapack_complex_double* vs, lapack_int* ldvs, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_logical* bwork, lapack_int *info ); +void LAPACK_sgeesx( char* jobvs, char* sort, LAPACK_S_SELECT2 select, + char* sense, lapack_int* n, float* a, lapack_int* lda, + lapack_int* sdim, float* wr, float* wi, float* vs, + lapack_int* ldvs, float* rconde, float* rcondv, float* work, + lapack_int* lwork, lapack_int* iwork, lapack_int* liwork, + lapack_logical* bwork, lapack_int *info ); +void LAPACK_dgeesx( char* jobvs, char* sort, LAPACK_D_SELECT2 select, + char* sense, lapack_int* n, double* a, lapack_int* lda, + lapack_int* sdim, double* wr, double* wi, double* vs, + lapack_int* ldvs, double* rconde, double* rcondv, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_cgeesx( char* jobvs, char* sort, LAPACK_C_SELECT1 select, + char* sense, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* sdim, lapack_complex_float* w, + lapack_complex_float* vs, lapack_int* ldvs, float* rconde, + float* rcondv, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_zgeesx( char* jobvs, char* sort, LAPACK_Z_SELECT1 select, + char* sense, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* sdim, lapack_complex_double* w, + lapack_complex_double* vs, lapack_int* ldvs, double* rconde, + double* rcondv, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_sgeev( char* jobvl, char* jobvr, lapack_int* n, float* a, + lapack_int* lda, float* wr, float* wi, float* vl, + lapack_int* ldvl, float* vr, lapack_int* ldvr, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dgeev( char* jobvl, char* jobvr, lapack_int* n, double* a, + lapack_int* lda, double* wr, double* wi, double* vl, + lapack_int* ldvl, double* vr, lapack_int* ldvr, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cgeev( char* jobvl, char* jobvr, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int* ldvl, lapack_complex_float* vr, lapack_int* ldvr, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zgeev( char* jobvl, char* jobvr, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* w, lapack_complex_double* vl, + lapack_int* ldvl, lapack_complex_double* vr, + lapack_int* ldvr, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int *info ); +void LAPACK_sgeevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, float* a, lapack_int* lda, float* wr, + float* wi, float* vl, lapack_int* ldvl, float* vr, + lapack_int* ldvr, lapack_int* ilo, lapack_int* ihi, + float* scale, float* abnrm, float* rconde, float* rcondv, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int *info ); +void LAPACK_dgeevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, double* a, lapack_int* lda, double* wr, + double* wi, double* vl, lapack_int* ldvl, double* vr, + lapack_int* ldvr, lapack_int* ilo, lapack_int* ihi, + double* scale, double* abnrm, double* rconde, + double* rcondv, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_cgeevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* w, lapack_complex_float* vl, + lapack_int* ldvl, lapack_complex_float* vr, + lapack_int* ldvr, lapack_int* ilo, lapack_int* ihi, + float* scale, float* abnrm, float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zgeevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* w, lapack_complex_double* vl, + lapack_int* ldvl, lapack_complex_double* vr, + lapack_int* ldvr, lapack_int* ilo, lapack_int* ihi, + double* scale, double* abnrm, double* rconde, + double* rcondv, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int *info ); +void LAPACK_sgesvd( char* jobu, char* jobvt, lapack_int* m, lapack_int* n, + float* a, lapack_int* lda, float* s, float* u, + lapack_int* ldu, float* vt, lapack_int* ldvt, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_dgesvd( char* jobu, char* jobvt, lapack_int* m, lapack_int* n, + double* a, lapack_int* lda, double* s, double* u, + lapack_int* ldu, double* vt, lapack_int* ldvt, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cgesvd( char* jobu, char* jobvt, lapack_int* m, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* s, + lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* vt, lapack_int* ldvt, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zgesvd( char* jobu, char* jobvt, lapack_int* m, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* s, + lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* vt, lapack_int* ldvt, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_sgesdd( char* jobz, lapack_int* m, lapack_int* n, float* a, + lapack_int* lda, float* s, float* u, lapack_int* ldu, + float* vt, lapack_int* ldvt, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dgesdd( char* jobz, lapack_int* m, lapack_int* n, double* a, + lapack_int* lda, double* s, double* u, lapack_int* ldu, + double* vt, lapack_int* ldvt, double* work, + lapack_int* lwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_cgesdd( char* jobz, lapack_int* m, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, float* s, + lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* vt, lapack_int* ldvt, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_zgesdd( char* jobz, lapack_int* m, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, double* s, + lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* vt, lapack_int* ldvt, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* iwork, lapack_int *info ); +void LAPACK_dgejsv( char* joba, char* jobu, char* jobv, char* jobr, char* jobt, + char* jobp, lapack_int* m, lapack_int* n, double* a, + lapack_int* lda, double* sva, double* u, lapack_int* ldu, + double* v, lapack_int* ldv, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_sgejsv( char* joba, char* jobu, char* jobv, char* jobr, char* jobt, + char* jobp, lapack_int* m, lapack_int* n, float* a, + lapack_int* lda, float* sva, float* u, lapack_int* ldu, + float* v, lapack_int* ldv, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_dgesvj( char* joba, char* jobu, char* jobv, lapack_int* m, + lapack_int* n, double* a, lapack_int* lda, double* sva, + lapack_int* mv, double* v, lapack_int* ldv, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sgesvj( char* joba, char* jobu, char* jobv, lapack_int* m, + lapack_int* n, float* a, lapack_int* lda, float* sva, + lapack_int* mv, float* v, lapack_int* ldv, float* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_sggsvd( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* n, lapack_int* p, lapack_int* k, lapack_int* l, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* alpha, float* beta, float* u, lapack_int* ldu, + float* v, lapack_int* ldv, float* q, lapack_int* ldq, + float* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_dggsvd( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* n, lapack_int* p, lapack_int* k, lapack_int* l, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* alpha, double* beta, double* u, lapack_int* ldu, + double* v, lapack_int* ldv, double* q, lapack_int* ldq, + double* work, lapack_int* iwork, lapack_int *info ); +void LAPACK_cggsvd( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* n, lapack_int* p, lapack_int* k, lapack_int* l, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* alpha, + float* beta, lapack_complex_float* u, lapack_int* ldu, + lapack_complex_float* v, lapack_int* ldv, + lapack_complex_float* q, lapack_int* ldq, + lapack_complex_float* work, float* rwork, lapack_int* iwork, + lapack_int *info ); +void LAPACK_zggsvd( char* jobu, char* jobv, char* jobq, lapack_int* m, + lapack_int* n, lapack_int* p, lapack_int* k, lapack_int* l, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* alpha, + double* beta, lapack_complex_double* u, lapack_int* ldu, + lapack_complex_double* v, lapack_int* ldv, + lapack_complex_double* q, lapack_int* ldq, + lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int *info ); +void LAPACK_ssygv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* w, float* work, lapack_int* lwork, lapack_int *info ); +void LAPACK_dsygv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* w, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_chegv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* w, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zhegv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* w, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_ssygvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + float* w, float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_dsygvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* w, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_chegvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* w, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zhegvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* w, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_ssygvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, float* a, lapack_int* lda, float* b, + lapack_int* ldb, float* vl, float* vu, lapack_int* il, + lapack_int* iu, float* abstol, lapack_int* m, float* w, + float* z, lapack_int* ldz, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_dsygvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, double* a, lapack_int* lda, double* b, + lapack_int* ldb, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_chegvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, float* vl, + float* vu, lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_zhegvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_sspgv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + float* ap, float* bp, float* w, float* z, lapack_int* ldz, + float* work, lapack_int *info ); +void LAPACK_dspgv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + double* ap, double* bp, double* w, double* z, + lapack_int* ldz, double* work, lapack_int *info ); +void LAPACK_chpgv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* ap, lapack_complex_float* bp, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, float* rwork, lapack_int *info ); +void LAPACK_zhpgv( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* ap, lapack_complex_double* bp, + double* w, lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_sspgvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + float* ap, float* bp, float* w, float* z, lapack_int* ldz, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_dspgvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + double* ap, double* bp, double* w, double* z, + lapack_int* ldz, double* work, lapack_int* lwork, + lapack_int* iwork, lapack_int* liwork, lapack_int *info ); +void LAPACK_chpgvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_float* ap, lapack_complex_float* bp, + float* w, lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zhpgvd( lapack_int* itype, char* jobz, char* uplo, lapack_int* n, + lapack_complex_double* ap, lapack_complex_double* bp, + double* w, lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_sspgvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, float* ap, float* bp, float* vl, float* vu, + lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, float* z, lapack_int* ldz, + float* work, lapack_int* iwork, lapack_int* ifail, + lapack_int *info ); +void LAPACK_dspgvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, double* ap, double* bp, double* vl, + double* vu, lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, double* z, lapack_int* ldz, + double* work, lapack_int* iwork, lapack_int* ifail, + lapack_int *info ); +void LAPACK_chpgvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, lapack_complex_float* ap, + lapack_complex_float* bp, float* vl, float* vu, + lapack_int* il, lapack_int* iu, float* abstol, + lapack_int* m, float* w, lapack_complex_float* z, + lapack_int* ldz, lapack_complex_float* work, float* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_zhpgvx( lapack_int* itype, char* jobz, char* range, char* uplo, + lapack_int* n, lapack_complex_double* ap, + lapack_complex_double* bp, double* vl, double* vu, + lapack_int* il, lapack_int* iu, double* abstol, + lapack_int* m, double* w, lapack_complex_double* z, + lapack_int* ldz, lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_ssbgv( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, float* ab, lapack_int* ldab, float* bb, + lapack_int* ldbb, float* w, float* z, lapack_int* ldz, + float* work, lapack_int *info ); +void LAPACK_dsbgv( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, double* ab, lapack_int* ldab, double* bb, + lapack_int* ldbb, double* w, double* z, lapack_int* ldz, + double* work, lapack_int *info ); +void LAPACK_chbgv( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* bb, lapack_int* ldbb, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, float* rwork, lapack_int *info ); +void LAPACK_zhbgv( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* bb, lapack_int* ldbb, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, double* rwork, + lapack_int *info ); +void LAPACK_ssbgvd( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, float* ab, lapack_int* ldab, float* bb, + lapack_int* ldbb, float* w, float* z, lapack_int* ldz, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_dsbgvd( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, double* ab, lapack_int* ldab, double* bb, + lapack_int* ldbb, double* w, double* z, lapack_int* ldz, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_chbgvd( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_float* ab, lapack_int* ldab, + lapack_complex_float* bb, lapack_int* ldbb, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* lrwork, lapack_int* iwork, lapack_int* liwork, + lapack_int *info ); +void LAPACK_zhbgvd( char* jobz, char* uplo, lapack_int* n, lapack_int* ka, + lapack_int* kb, lapack_complex_double* ab, lapack_int* ldab, + lapack_complex_double* bb, lapack_int* ldbb, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, lapack_int* iwork, + lapack_int* liwork, lapack_int *info ); +void LAPACK_ssbgvx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* ka, lapack_int* kb, float* ab, lapack_int* ldab, + float* bb, lapack_int* ldbb, float* q, lapack_int* ldq, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, float* z, + lapack_int* ldz, float* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_dsbgvx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* ka, lapack_int* kb, double* ab, + lapack_int* ldab, double* bb, lapack_int* ldbb, double* q, + lapack_int* ldq, double* vl, double* vu, lapack_int* il, + lapack_int* iu, double* abstol, lapack_int* m, double* w, + double* z, lapack_int* ldz, double* work, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_chbgvx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* ka, lapack_int* kb, lapack_complex_float* ab, + lapack_int* ldab, lapack_complex_float* bb, + lapack_int* ldbb, lapack_complex_float* q, lapack_int* ldq, + float* vl, float* vu, lapack_int* il, lapack_int* iu, + float* abstol, lapack_int* m, float* w, + lapack_complex_float* z, lapack_int* ldz, + lapack_complex_float* work, float* rwork, lapack_int* iwork, + lapack_int* ifail, lapack_int *info ); +void LAPACK_zhbgvx( char* jobz, char* range, char* uplo, lapack_int* n, + lapack_int* ka, lapack_int* kb, lapack_complex_double* ab, + lapack_int* ldab, lapack_complex_double* bb, + lapack_int* ldbb, lapack_complex_double* q, lapack_int* ldq, + double* vl, double* vu, lapack_int* il, lapack_int* iu, + double* abstol, lapack_int* m, double* w, + lapack_complex_double* z, lapack_int* ldz, + lapack_complex_double* work, double* rwork, + lapack_int* iwork, lapack_int* ifail, lapack_int *info ); +void LAPACK_sgges( char* jobvsl, char* jobvsr, char* sort, + LAPACK_S_SELECT3 selctg, lapack_int* n, float* a, + lapack_int* lda, float* b, lapack_int* ldb, lapack_int* sdim, + float* alphar, float* alphai, float* beta, float* vsl, + lapack_int* ldvsl, float* vsr, lapack_int* ldvsr, + float* work, lapack_int* lwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_dgges( char* jobvsl, char* jobvsr, char* sort, + LAPACK_D_SELECT3 selctg, lapack_int* n, double* a, + lapack_int* lda, double* b, lapack_int* ldb, + lapack_int* sdim, double* alphar, double* alphai, + double* beta, double* vsl, lapack_int* ldvsl, double* vsr, + lapack_int* ldvsr, double* work, lapack_int* lwork, + lapack_logical* bwork, lapack_int *info ); +void LAPACK_cgges( char* jobvsl, char* jobvsr, char* sort, + LAPACK_C_SELECT2 selctg, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, lapack_int* sdim, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* vsl, lapack_int* ldvsl, + lapack_complex_float* vsr, lapack_int* ldvsr, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_logical* bwork, lapack_int *info ); +void LAPACK_zgges( char* jobvsl, char* jobvsr, char* sort, + LAPACK_Z_SELECT2 selctg, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, lapack_int* sdim, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int* ldvsl, + lapack_complex_double* vsr, lapack_int* ldvsr, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_logical* bwork, lapack_int *info ); +void LAPACK_sggesx( char* jobvsl, char* jobvsr, char* sort, + LAPACK_S_SELECT3 selctg, char* sense, lapack_int* n, + float* a, lapack_int* lda, float* b, lapack_int* ldb, + lapack_int* sdim, float* alphar, float* alphai, float* beta, + float* vsl, lapack_int* ldvsl, float* vsr, + lapack_int* ldvsr, float* rconde, float* rcondv, + float* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_dggesx( char* jobvsl, char* jobvsr, char* sort, + LAPACK_D_SELECT3 selctg, char* sense, lapack_int* n, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + lapack_int* sdim, double* alphar, double* alphai, + double* beta, double* vsl, lapack_int* ldvsl, double* vsr, + lapack_int* ldvsr, double* rconde, double* rcondv, + double* work, lapack_int* lwork, lapack_int* iwork, + lapack_int* liwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_cggesx( char* jobvsl, char* jobvsr, char* sort, + LAPACK_C_SELECT2 selctg, char* sense, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, lapack_int* sdim, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* vsl, lapack_int* ldvsl, + lapack_complex_float* vsr, lapack_int* ldvsr, float* rconde, + float* rcondv, lapack_complex_float* work, + lapack_int* lwork, float* rwork, lapack_int* iwork, + lapack_int* liwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_zggesx( char* jobvsl, char* jobvsr, char* sort, + LAPACK_Z_SELECT2 selctg, char* sense, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, lapack_int* sdim, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* vsl, lapack_int* ldvsl, + lapack_complex_double* vsr, lapack_int* ldvsr, + double* rconde, double* rcondv, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int* iwork, + lapack_int* liwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_sggev( char* jobvl, char* jobvr, lapack_int* n, float* a, + lapack_int* lda, float* b, lapack_int* ldb, float* alphar, + float* alphai, float* beta, float* vl, lapack_int* ldvl, + float* vr, lapack_int* ldvr, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dggev( char* jobvl, char* jobvr, lapack_int* n, double* a, + lapack_int* lda, double* b, lapack_int* ldb, double* alphar, + double* alphai, double* beta, double* vl, lapack_int* ldvl, + double* vr, lapack_int* ldvr, double* work, + lapack_int* lwork, lapack_int *info ); +void LAPACK_cggev( char* jobvl, char* jobvr, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* vl, lapack_int* ldvl, + lapack_complex_float* vr, lapack_int* ldvr, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int *info ); +void LAPACK_zggev( char* jobvl, char* jobvr, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int* ldvl, + lapack_complex_double* vr, lapack_int* ldvr, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int *info ); +void LAPACK_sggevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, float* a, lapack_int* lda, float* b, + lapack_int* ldb, float* alphar, float* alphai, float* beta, + float* vl, lapack_int* ldvl, float* vr, lapack_int* ldvr, + lapack_int* ilo, lapack_int* ihi, float* lscale, + float* rscale, float* abnrm, float* bbnrm, float* rconde, + float* rcondv, float* work, lapack_int* lwork, + lapack_int* iwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_dggevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, double* a, lapack_int* lda, double* b, + lapack_int* ldb, double* alphar, double* alphai, + double* beta, double* vl, lapack_int* ldvl, double* vr, + lapack_int* ldvr, lapack_int* ilo, lapack_int* ihi, + double* lscale, double* rscale, double* abnrm, + double* bbnrm, double* rconde, double* rcondv, double* work, + lapack_int* lwork, lapack_int* iwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_cggevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* vl, lapack_int* ldvl, + lapack_complex_float* vr, lapack_int* ldvr, lapack_int* ilo, + lapack_int* ihi, float* lscale, float* rscale, float* abnrm, + float* bbnrm, float* rconde, float* rcondv, + lapack_complex_float* work, lapack_int* lwork, float* rwork, + lapack_int* iwork, lapack_logical* bwork, + lapack_int *info ); +void LAPACK_zggevx( char* balanc, char* jobvl, char* jobvr, char* sense, + lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* vl, lapack_int* ldvl, + lapack_complex_double* vr, lapack_int* ldvr, + lapack_int* ilo, lapack_int* ihi, double* lscale, + double* rscale, double* abnrm, double* bbnrm, + double* rconde, double* rcondv, lapack_complex_double* work, + lapack_int* lwork, double* rwork, lapack_int* iwork, + lapack_logical* bwork, lapack_int *info ); +void LAPACK_dsfrk( char* transr, char* uplo, char* trans, lapack_int* n, + lapack_int* k, double* alpha, const double* a, + lapack_int* lda, double* beta, double* c ); +void LAPACK_ssfrk( char* transr, char* uplo, char* trans, lapack_int* n, + lapack_int* k, float* alpha, const float* a, lapack_int* lda, + float* beta, float* c ); +void LAPACK_zhfrk( char* transr, char* uplo, char* trans, lapack_int* n, + lapack_int* k, double* alpha, const lapack_complex_double* a, + lapack_int* lda, double* beta, lapack_complex_double* c ); +void LAPACK_chfrk( char* transr, char* uplo, char* trans, lapack_int* n, + lapack_int* k, float* alpha, const lapack_complex_float* a, + lapack_int* lda, float* beta, lapack_complex_float* c ); +void LAPACK_dtfsm( char* transr, char* side, char* uplo, char* trans, + char* diag, lapack_int* m, lapack_int* n, double* alpha, + const double* a, double* b, lapack_int* ldb ); +void LAPACK_stfsm( char* transr, char* side, char* uplo, char* trans, + char* diag, lapack_int* m, lapack_int* n, float* alpha, + const float* a, float* b, lapack_int* ldb ); +void LAPACK_ztfsm( char* transr, char* side, char* uplo, char* trans, + char* diag, lapack_int* m, lapack_int* n, + lapack_complex_double* alpha, const lapack_complex_double* a, + lapack_complex_double* b, lapack_int* ldb ); +void LAPACK_ctfsm( char* transr, char* side, char* uplo, char* trans, + char* diag, lapack_int* m, lapack_int* n, + lapack_complex_float* alpha, const lapack_complex_float* a, + lapack_complex_float* b, lapack_int* ldb ); +void LAPACK_dtfttp( char* transr, char* uplo, lapack_int* n, const double* arf, + double* ap, lapack_int *info ); +void LAPACK_stfttp( char* transr, char* uplo, lapack_int* n, const float* arf, + float* ap, lapack_int *info ); +void LAPACK_ztfttp( char* transr, char* uplo, lapack_int* n, + const lapack_complex_double* arf, lapack_complex_double* ap, + lapack_int *info ); +void LAPACK_ctfttp( char* transr, char* uplo, lapack_int* n, + const lapack_complex_float* arf, lapack_complex_float* ap, + lapack_int *info ); +void LAPACK_dtfttr( char* transr, char* uplo, lapack_int* n, const double* arf, + double* a, lapack_int* lda, lapack_int *info ); +void LAPACK_stfttr( char* transr, char* uplo, lapack_int* n, const float* arf, + float* a, lapack_int* lda, lapack_int *info ); +void LAPACK_ztfttr( char* transr, char* uplo, lapack_int* n, + const lapack_complex_double* arf, lapack_complex_double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_ctfttr( char* transr, char* uplo, lapack_int* n, + const lapack_complex_float* arf, lapack_complex_float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_dtpttf( char* transr, char* uplo, lapack_int* n, const double* ap, + double* arf, lapack_int *info ); +void LAPACK_stpttf( char* transr, char* uplo, lapack_int* n, const float* ap, + float* arf, lapack_int *info ); +void LAPACK_ztpttf( char* transr, char* uplo, lapack_int* n, + const lapack_complex_double* ap, lapack_complex_double* arf, + lapack_int *info ); +void LAPACK_ctpttf( char* transr, char* uplo, lapack_int* n, + const lapack_complex_float* ap, lapack_complex_float* arf, + lapack_int *info ); +void LAPACK_dtpttr( char* uplo, lapack_int* n, const double* ap, double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_stpttr( char* uplo, lapack_int* n, const float* ap, float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_ztpttr( char* uplo, lapack_int* n, const lapack_complex_double* ap, + lapack_complex_double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_ctpttr( char* uplo, lapack_int* n, const lapack_complex_float* ap, + lapack_complex_float* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dtrttf( char* transr, char* uplo, lapack_int* n, const double* a, + lapack_int* lda, double* arf, lapack_int *info ); +void LAPACK_strttf( char* transr, char* uplo, lapack_int* n, const float* a, + lapack_int* lda, float* arf, lapack_int *info ); +void LAPACK_ztrttf( char* transr, char* uplo, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* arf, lapack_int *info ); +void LAPACK_ctrttf( char* transr, char* uplo, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* arf, lapack_int *info ); +void LAPACK_dtrttp( char* uplo, lapack_int* n, const double* a, lapack_int* lda, + double* ap, lapack_int *info ); +void LAPACK_strttp( char* uplo, lapack_int* n, const float* a, lapack_int* lda, + float* ap, lapack_int *info ); +void LAPACK_ztrttp( char* uplo, lapack_int* n, const lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* ap, + lapack_int *info ); +void LAPACK_ctrttp( char* uplo, lapack_int* n, const lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* ap, + lapack_int *info ); +void LAPACK_sgeqrfp( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_dgeqrfp( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_cgeqrfp( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_zgeqrfp( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int* lwork, + lapack_int *info ); +void LAPACK_clacgv( lapack_int* n, lapack_complex_float* x, lapack_int* incx ); +void LAPACK_zlacgv( lapack_int* n, lapack_complex_double* x, lapack_int* incx ); +void LAPACK_slarnv( lapack_int* idist, lapack_int* iseed, lapack_int* n, + float* x ); +void LAPACK_dlarnv( lapack_int* idist, lapack_int* iseed, lapack_int* n, + double* x ); +void LAPACK_clarnv( lapack_int* idist, lapack_int* iseed, lapack_int* n, + lapack_complex_float* x ); +void LAPACK_zlarnv( lapack_int* idist, lapack_int* iseed, lapack_int* n, + lapack_complex_double* x ); +void LAPACK_sgeqr2( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int *info ); +void LAPACK_dgeqr2( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int *info ); +void LAPACK_cgeqr2( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zgeqr2( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_slacpy( char* uplo, lapack_int* m, lapack_int* n, const float* a, + lapack_int* lda, float* b, lapack_int* ldb ); +void LAPACK_dlacpy( char* uplo, lapack_int* m, lapack_int* n, const double* a, + lapack_int* lda, double* b, lapack_int* ldb ); +void LAPACK_clacpy( char* uplo, lapack_int* m, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb ); +void LAPACK_zlacpy( char* uplo, lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb ); +void LAPACK_sgetf2( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_dgetf2( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + lapack_int* ipiv, lapack_int *info ); +void LAPACK_cgetf2( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int* ipiv, lapack_int *info ); +void LAPACK_zgetf2( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int* ipiv, lapack_int *info ); +void LAPACK_slaswp( lapack_int* n, float* a, lapack_int* lda, lapack_int* k1, + lapack_int* k2, const lapack_int* ipiv, lapack_int* incx ); +void LAPACK_dlaswp( lapack_int* n, double* a, lapack_int* lda, lapack_int* k1, + lapack_int* k2, const lapack_int* ipiv, lapack_int* incx ); +void LAPACK_claswp( lapack_int* n, lapack_complex_float* a, lapack_int* lda, + lapack_int* k1, lapack_int* k2, const lapack_int* ipiv, + lapack_int* incx ); +void LAPACK_zlaswp( lapack_int* n, lapack_complex_double* a, lapack_int* lda, + lapack_int* k1, lapack_int* k2, const lapack_int* ipiv, + lapack_int* incx ); +float LAPACK_slange( char* norm, lapack_int* m, lapack_int* n, const float* a, + lapack_int* lda, float* work ); +double LAPACK_dlange( char* norm, lapack_int* m, lapack_int* n, const double* a, + lapack_int* lda, double* work ); +float LAPACK_clange( char* norm, lapack_int* m, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, float* work ); +double LAPACK_zlange( char* norm, lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, double* work ); +float LAPACK_clanhe( char* norm, char* uplo, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, float* work ); +double LAPACK_zlanhe( char* norm, char* uplo, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, double* work ); +float LAPACK_slansy( char* norm, char* uplo, lapack_int* n, const float* a, + lapack_int* lda, float* work ); +double LAPACK_dlansy( char* norm, char* uplo, lapack_int* n, const double* a, + lapack_int* lda, double* work ); +float LAPACK_clansy( char* norm, char* uplo, lapack_int* n, + const lapack_complex_float* a, lapack_int* lda, float* work ); +double LAPACK_zlansy( char* norm, char* uplo, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, double* work ); +float LAPACK_slantr( char* norm, char* uplo, char* diag, lapack_int* m, + lapack_int* n, const float* a, lapack_int* lda, float* work ); +double LAPACK_dlantr( char* norm, char* uplo, char* diag, lapack_int* m, + lapack_int* n, const double* a, lapack_int* lda, double* work ); +float LAPACK_clantr( char* norm, char* uplo, char* diag, lapack_int* m, + lapack_int* n, const lapack_complex_float* a, lapack_int* lda, + float* work ); +double LAPACK_zlantr( char* norm, char* uplo, char* diag, lapack_int* m, + lapack_int* n, const lapack_complex_double* a, lapack_int* lda, + double* work ); +float LAPACK_slamch( char* cmach ); +double LAPACK_dlamch( char* cmach ); +void LAPACK_sgelq2( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* tau, float* work, lapack_int *info ); +void LAPACK_dgelq2( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* tau, double* work, lapack_int *info ); +void LAPACK_cgelq2( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* tau, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zgelq2( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* tau, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_slarfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, const float* v, + lapack_int* ldv, const float* t, lapack_int* ldt, float* c, + lapack_int* ldc, float* work, lapack_int* ldwork ); +void LAPACK_dlarfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, + const double* v, lapack_int* ldv, const double* t, + lapack_int* ldt, double* c, lapack_int* ldc, double* work, + lapack_int* ldwork ); +void LAPACK_clarfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, + const lapack_complex_float* v, lapack_int* ldv, + const lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int* ldwork ); +void LAPACK_zlarfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, + const lapack_complex_double* v, lapack_int* ldv, + const lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int* ldwork ); +void LAPACK_slarfg( lapack_int* n, float* alpha, float* x, lapack_int* incx, + float* tau ); +void LAPACK_dlarfg( lapack_int* n, double* alpha, double* x, lapack_int* incx, + double* tau ); +void LAPACK_clarfg( lapack_int* n, lapack_complex_float* alpha, + lapack_complex_float* x, lapack_int* incx, + lapack_complex_float* tau ); +void LAPACK_zlarfg( lapack_int* n, lapack_complex_double* alpha, + lapack_complex_double* x, lapack_int* incx, + lapack_complex_double* tau ); +void LAPACK_slarft( char* direct, char* storev, lapack_int* n, lapack_int* k, + const float* v, lapack_int* ldv, const float* tau, float* t, + lapack_int* ldt ); +void LAPACK_dlarft( char* direct, char* storev, lapack_int* n, lapack_int* k, + const double* v, lapack_int* ldv, const double* tau, + double* t, lapack_int* ldt ); +void LAPACK_clarft( char* direct, char* storev, lapack_int* n, lapack_int* k, + const lapack_complex_float* v, lapack_int* ldv, + const lapack_complex_float* tau, lapack_complex_float* t, + lapack_int* ldt ); +void LAPACK_zlarft( char* direct, char* storev, lapack_int* n, lapack_int* k, + const lapack_complex_double* v, lapack_int* ldv, + const lapack_complex_double* tau, lapack_complex_double* t, + lapack_int* ldt ); +void LAPACK_slarfx( char* side, lapack_int* m, lapack_int* n, const float* v, + float* tau, float* c, lapack_int* ldc, float* work ); +void LAPACK_dlarfx( char* side, lapack_int* m, lapack_int* n, const double* v, + double* tau, double* c, lapack_int* ldc, double* work ); +void LAPACK_clarfx( char* side, lapack_int* m, lapack_int* n, + const lapack_complex_float* v, lapack_complex_float* tau, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work ); +void LAPACK_zlarfx( char* side, lapack_int* m, lapack_int* n, + const lapack_complex_double* v, lapack_complex_double* tau, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work ); +void LAPACK_slatms( lapack_int* m, lapack_int* n, char* dist, lapack_int* iseed, + char* sym, float* d, lapack_int* mode, float* cond, + float* dmax, lapack_int* kl, lapack_int* ku, char* pack, + float* a, lapack_int* lda, float* work, lapack_int *info ); +void LAPACK_dlatms( lapack_int* m, lapack_int* n, char* dist, lapack_int* iseed, + char* sym, double* d, lapack_int* mode, double* cond, + double* dmax, lapack_int* kl, lapack_int* ku, char* pack, + double* a, lapack_int* lda, double* work, + lapack_int *info ); +void LAPACK_clatms( lapack_int* m, lapack_int* n, char* dist, lapack_int* iseed, + char* sym, float* d, lapack_int* mode, float* cond, + float* dmax, lapack_int* kl, lapack_int* ku, char* pack, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zlatms( lapack_int* m, lapack_int* n, char* dist, lapack_int* iseed, + char* sym, double* d, lapack_int* mode, double* cond, + double* dmax, lapack_int* kl, lapack_int* ku, char* pack, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_slag2d( lapack_int* m, lapack_int* n, const float* sa, + lapack_int* ldsa, double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dlag2s( lapack_int* m, lapack_int* n, const double* a, + lapack_int* lda, float* sa, lapack_int* ldsa, + lapack_int *info ); +void LAPACK_clag2z( lapack_int* m, lapack_int* n, + const lapack_complex_float* sa, lapack_int* ldsa, + lapack_complex_double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_zlag2c( lapack_int* m, lapack_int* n, + const lapack_complex_double* a, lapack_int* lda, + lapack_complex_float* sa, lapack_int* ldsa, + lapack_int *info ); +void LAPACK_slauum( char* uplo, lapack_int* n, float* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_dlauum( char* uplo, lapack_int* n, double* a, lapack_int* lda, + lapack_int *info ); +void LAPACK_clauum( char* uplo, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_zlauum( char* uplo, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_int *info ); +void LAPACK_slagge( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const float* d, float* a, lapack_int* lda, + lapack_int* iseed, float* work, lapack_int *info ); +void LAPACK_dlagge( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const double* d, double* a, lapack_int* lda, + lapack_int* iseed, double* work, lapack_int *info ); +void LAPACK_clagge( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const float* d, lapack_complex_float* a, + lapack_int* lda, lapack_int* iseed, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zlagge( lapack_int* m, lapack_int* n, lapack_int* kl, + lapack_int* ku, const double* d, lapack_complex_double* a, + lapack_int* lda, lapack_int* iseed, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_slaset( char* uplo, lapack_int* m, lapack_int* n, float* alpha, + float* beta, float* a, lapack_int* lda ); +void LAPACK_dlaset( char* uplo, lapack_int* m, lapack_int* n, double* alpha, + double* beta, double* a, lapack_int* lda ); +void LAPACK_claset( char* uplo, lapack_int* m, lapack_int* n, + lapack_complex_float* alpha, lapack_complex_float* beta, + lapack_complex_float* a, lapack_int* lda ); +void LAPACK_zlaset( char* uplo, lapack_int* m, lapack_int* n, + lapack_complex_double* alpha, lapack_complex_double* beta, + lapack_complex_double* a, lapack_int* lda ); +void LAPACK_slasrt( char* id, lapack_int* n, float* d, lapack_int *info ); +void LAPACK_dlasrt( char* id, lapack_int* n, double* d, lapack_int *info ); +void LAPACK_claghe( lapack_int* n, lapack_int* k, const float* d, + lapack_complex_float* a, lapack_int* lda, lapack_int* iseed, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zlaghe( lapack_int* n, lapack_int* k, const double* d, + lapack_complex_double* a, lapack_int* lda, + lapack_int* iseed, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_slagsy( lapack_int* n, lapack_int* k, const float* d, float* a, + lapack_int* lda, lapack_int* iseed, float* work, + lapack_int *info ); +void LAPACK_dlagsy( lapack_int* n, lapack_int* k, const double* d, double* a, + lapack_int* lda, lapack_int* iseed, double* work, + lapack_int *info ); +void LAPACK_clagsy( lapack_int* n, lapack_int* k, const float* d, + lapack_complex_float* a, lapack_int* lda, lapack_int* iseed, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zlagsy( lapack_int* n, lapack_int* k, const double* d, + lapack_complex_double* a, lapack_int* lda, + lapack_int* iseed, lapack_complex_double* work, + lapack_int *info ); +void LAPACK_slapmr( lapack_logical* forwrd, lapack_int* m, lapack_int* n, + float* x, lapack_int* ldx, lapack_int* k ); +void LAPACK_dlapmr( lapack_logical* forwrd, lapack_int* m, lapack_int* n, + double* x, lapack_int* ldx, lapack_int* k ); +void LAPACK_clapmr( lapack_logical* forwrd, lapack_int* m, lapack_int* n, + lapack_complex_float* x, lapack_int* ldx, lapack_int* k ); +void LAPACK_zlapmr( lapack_logical* forwrd, lapack_int* m, lapack_int* n, + lapack_complex_double* x, lapack_int* ldx, lapack_int* k ); +float LAPACK_slapy2( float* x, float* y ); +double LAPACK_dlapy2( double* x, double* y ); +float LAPACK_slapy3( float* x, float* y, float* z ); +double LAPACK_dlapy3( double* x, double* y, double* z ); +void LAPACK_slartgp( float* f, float* g, float* cs, float* sn, float* r ); +void LAPACK_dlartgp( double* f, double* g, double* cs, double* sn, double* r ); +void LAPACK_slartgs( float* x, float* y, float* sigma, float* cs, float* sn ); +void LAPACK_dlartgs( double* x, double* y, double* sigma, double* cs, + double* sn ); +// LAPACK 3.3.0 +void LAPACK_cbbcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + lapack_int* m, lapack_int* p, lapack_int* q, + float* theta, float* phi, + lapack_complex_float* u1, lapack_int* ldu1, + lapack_complex_float* u2, lapack_int* ldu2, + lapack_complex_float* v1t, lapack_int* ldv1t, + lapack_complex_float* v2t, lapack_int* ldv2t, + float* b11d, float* b11e, float* b12d, + float* b12e, float* b21d, float* b21e, + float* b22d, float* b22e, float* rwork, + lapack_int* lrwork , lapack_int *info ); +void LAPACK_cheswapr( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* i1, + lapack_int* i2 ); +void LAPACK_chetri2( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_chetri2x( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int* nb , lapack_int *info ); +void LAPACK_chetrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work , lapack_int *info ); +void LAPACK_csyconv( char* uplo, char* way, + lapack_int* n, lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_float* work , lapack_int *info ); +void LAPACK_csyswapr( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* i1, + lapack_int* i2 ); +void LAPACK_csytri2( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_csytri2x( char* uplo, lapack_int* n, + lapack_complex_float* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int* nb , lapack_int *info ); +void LAPACK_csytrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, const lapack_complex_float* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work , lapack_int *info ); +void LAPACK_cunbdb( char* trans, char* signs, + lapack_int* m, lapack_int* p, lapack_int* q, + lapack_complex_float* x11, lapack_int* ldx11, + lapack_complex_float* x12, lapack_int* ldx12, + lapack_complex_float* x21, lapack_int* ldx21, + lapack_complex_float* x22, lapack_int* ldx22, + float* theta, float* phi, + lapack_complex_float* taup1, + lapack_complex_float* taup2, + lapack_complex_float* tauq1, + lapack_complex_float* tauq2, + lapack_complex_float* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_cuncsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + char* signs, lapack_int* m, lapack_int* p, + lapack_int* q, lapack_complex_float* x11, + lapack_int* ldx11, lapack_complex_float* x12, + lapack_int* ldx12, lapack_complex_float* x21, + lapack_int* ldx21, lapack_complex_float* x22, + lapack_int* ldx22, float* theta, + lapack_complex_float* u1, lapack_int* ldu1, + lapack_complex_float* u2, lapack_int* ldu2, + lapack_complex_float* v1t, lapack_int* ldv1t, + lapack_complex_float* v2t, lapack_int* ldv2t, + lapack_complex_float* work, lapack_int* lwork, + float* rwork, lapack_int* lrwork, + lapack_int* iwork , lapack_int *info ); +void LAPACK_dbbcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + lapack_int* m, lapack_int* p, lapack_int* q, + double* theta, double* phi, double* u1, + lapack_int* ldu1, double* u2, lapack_int* ldu2, + double* v1t, lapack_int* ldv1t, double* v2t, + lapack_int* ldv2t, double* b11d, double* b11e, + double* b12d, double* b12e, double* b21d, + double* b21e, double* b22d, double* b22e, + double* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_dorbdb( char* trans, char* signs, + lapack_int* m, lapack_int* p, lapack_int* q, + double* x11, lapack_int* ldx11, double* x12, + lapack_int* ldx12, double* x21, lapack_int* ldx21, + double* x22, lapack_int* ldx22, double* theta, + double* phi, double* taup1, double* taup2, + double* tauq1, double* tauq2, double* work, + lapack_int* lwork , lapack_int *info ); +void LAPACK_dorcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + char* signs, lapack_int* m, lapack_int* p, + lapack_int* q, double* x11, lapack_int* ldx11, + double* x12, lapack_int* ldx12, double* x21, + lapack_int* ldx21, double* x22, lapack_int* ldx22, + double* theta, double* u1, lapack_int* ldu1, + double* u2, lapack_int* ldu2, double* v1t, + lapack_int* ldv1t, double* v2t, lapack_int* ldv2t, + double* work, lapack_int* lwork, + lapack_int* iwork , lapack_int *info ); +void LAPACK_dsyconv( char* uplo, char* way, + lapack_int* n, double* a, lapack_int* lda, + const lapack_int* ipiv, double* work , lapack_int *info ); +void LAPACK_dsyswapr( char* uplo, lapack_int* n, + double* a, lapack_int* i1, lapack_int* i2 ); +void LAPACK_dsytri2( char* uplo, lapack_int* n, + double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_dsytri2x( char* uplo, lapack_int* n, + double* a, lapack_int* lda, + const lapack_int* ipiv, double* work, + lapack_int* nb , lapack_int *info ); +void LAPACK_dsytrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, const double* a, + lapack_int* lda, const lapack_int* ipiv, + double* b, lapack_int* ldb, double* work , lapack_int *info ); +void LAPACK_sbbcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + lapack_int* m, lapack_int* p, lapack_int* q, + float* theta, float* phi, float* u1, + lapack_int* ldu1, float* u2, lapack_int* ldu2, + float* v1t, lapack_int* ldv1t, float* v2t, + lapack_int* ldv2t, float* b11d, float* b11e, + float* b12d, float* b12e, float* b21d, + float* b21e, float* b22d, float* b22e, + float* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_sorbdb( char* trans, char* signs, + lapack_int* m, lapack_int* p, lapack_int* q, + float* x11, lapack_int* ldx11, float* x12, + lapack_int* ldx12, float* x21, lapack_int* ldx21, + float* x22, lapack_int* ldx22, float* theta, + float* phi, float* taup1, float* taup2, + float* tauq1, float* tauq2, float* work, + lapack_int* lwork , lapack_int *info ); +void LAPACK_sorcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + char* signs, lapack_int* m, lapack_int* p, + lapack_int* q, float* x11, lapack_int* ldx11, + float* x12, lapack_int* ldx12, float* x21, + lapack_int* ldx21, float* x22, lapack_int* ldx22, + float* theta, float* u1, lapack_int* ldu1, + float* u2, lapack_int* ldu2, float* v1t, + lapack_int* ldv1t, float* v2t, lapack_int* ldv2t, + float* work, lapack_int* lwork, + lapack_int* iwork , lapack_int *info ); +void LAPACK_ssyconv( char* uplo, char* way, + lapack_int* n, float* a, lapack_int* lda, + const lapack_int* ipiv, float* work , lapack_int *info ); +void LAPACK_ssyswapr( char* uplo, lapack_int* n, + float* a, lapack_int* i1, lapack_int* i2 ); +void LAPACK_ssytri2( char* uplo, lapack_int* n, + float* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_float* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_ssytri2x( char* uplo, lapack_int* n, + float* a, lapack_int* lda, + const lapack_int* ipiv, float* work, + lapack_int* nb , lapack_int *info ); +void LAPACK_ssytrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, const float* a, + lapack_int* lda, const lapack_int* ipiv, + float* b, lapack_int* ldb, float* work , lapack_int *info ); +void LAPACK_zbbcsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + lapack_int* m, lapack_int* p, lapack_int* q, + double* theta, double* phi, + lapack_complex_double* u1, lapack_int* ldu1, + lapack_complex_double* u2, lapack_int* ldu2, + lapack_complex_double* v1t, lapack_int* ldv1t, + lapack_complex_double* v2t, lapack_int* ldv2t, + double* b11d, double* b11e, double* b12d, + double* b12e, double* b21d, double* b21e, + double* b22d, double* b22e, double* rwork, + lapack_int* lrwork , lapack_int *info ); +void LAPACK_zheswapr( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* i1, + lapack_int* i2 ); +void LAPACK_zhetri2( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_zhetri2x( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int* nb , lapack_int *info ); +void LAPACK_zhetrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work , lapack_int *info ); +void LAPACK_zsyconv( char* uplo, char* way, + lapack_int* n, lapack_complex_double* a, + lapack_int* lda, const lapack_int* ipiv, + lapack_complex_double* work , lapack_int *info ); +void LAPACK_zsyswapr( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* i1, + lapack_int* i2 ); +void LAPACK_zsytri2( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_zsytri2x( char* uplo, lapack_int* n, + lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* work, lapack_int* nb , lapack_int *info ); +void LAPACK_zsytrs2( char* uplo, lapack_int* n, + lapack_int* nrhs, + const lapack_complex_double* a, lapack_int* lda, + const lapack_int* ipiv, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work , lapack_int *info ); +void LAPACK_zunbdb( char* trans, char* signs, + lapack_int* m, lapack_int* p, lapack_int* q, + lapack_complex_double* x11, lapack_int* ldx11, + lapack_complex_double* x12, lapack_int* ldx12, + lapack_complex_double* x21, lapack_int* ldx21, + lapack_complex_double* x22, lapack_int* ldx22, + double* theta, double* phi, + lapack_complex_double* taup1, + lapack_complex_double* taup2, + lapack_complex_double* tauq1, + lapack_complex_double* tauq2, + lapack_complex_double* work, lapack_int* lwork , lapack_int *info ); +void LAPACK_zuncsd( char* jobu1, char* jobu2, + char* jobv1t, char* jobv2t, char* trans, + char* signs, lapack_int* m, lapack_int* p, + lapack_int* q, lapack_complex_double* x11, + lapack_int* ldx11, lapack_complex_double* x12, + lapack_int* ldx12, lapack_complex_double* x21, + lapack_int* ldx21, lapack_complex_double* x22, + lapack_int* ldx22, double* theta, + lapack_complex_double* u1, lapack_int* ldu1, + lapack_complex_double* u2, lapack_int* ldu2, + lapack_complex_double* v1t, lapack_int* ldv1t, + lapack_complex_double* v2t, lapack_int* ldv2t, + lapack_complex_double* work, lapack_int* lwork, + double* rwork, lapack_int* lrwork, + lapack_int* iwork , lapack_int *info ); +// LAPACK 3.4.0 +void LAPACK_sgemqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* nb, const float* v, + lapack_int* ldv, const float* t, lapack_int* ldt, float* c, + lapack_int* ldc, float* work, lapack_int *info ); +void LAPACK_dgemqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* nb, const double* v, + lapack_int* ldv, const double* t, lapack_int* ldt, + double* c, lapack_int* ldc, double* work, + lapack_int *info ); +void LAPACK_cgemqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* nb, + const lapack_complex_float* v, lapack_int* ldv, + const lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* c, lapack_int* ldc, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zgemqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* nb, + const lapack_complex_double* v, lapack_int* ldv, + const lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* c, lapack_int* ldc, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_sgeqrt( lapack_int* m, lapack_int* n, lapack_int* nb, float* a, + lapack_int* lda, float* t, lapack_int* ldt, float* work, + lapack_int *info ); +void LAPACK_dgeqrt( lapack_int* m, lapack_int* n, lapack_int* nb, double* a, + lapack_int* lda, double* t, lapack_int* ldt, double* work, + lapack_int *info ); +void LAPACK_cgeqrt( lapack_int* m, lapack_int* n, lapack_int* nb, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_zgeqrt( lapack_int* m, lapack_int* n, lapack_int* nb, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_sgeqrt2( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* t, lapack_int* ldt, lapack_int *info ); +void LAPACK_dgeqrt2( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* t, lapack_int* ldt, lapack_int *info ); +void LAPACK_cgeqrt2( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_zgeqrt2( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_sgeqrt3( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* t, lapack_int* ldt, lapack_int *info ); +void LAPACK_dgeqrt3( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* t, lapack_int* ldt, lapack_int *info ); +void LAPACK_cgeqrt3( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_zgeqrt3( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_stpmqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, lapack_int* nb, + const float* v, lapack_int* ldv, const float* t, + lapack_int* ldt, float* a, lapack_int* lda, float* b, + lapack_int* ldb, float* work, lapack_int *info ); +void LAPACK_dtpmqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, lapack_int* nb, + const double* v, lapack_int* ldv, const double* t, + lapack_int* ldt, double* a, lapack_int* lda, double* b, + lapack_int* ldb, double* work, lapack_int *info ); +void LAPACK_ctpmqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, lapack_int* nb, + const lapack_complex_float* v, lapack_int* ldv, + const lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_ztpmqrt( char* side, char* trans, lapack_int* m, lapack_int* n, + lapack_int* k, lapack_int* l, lapack_int* nb, + const lapack_complex_double* v, lapack_int* ldv, + const lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_dtpqrt( lapack_int* m, lapack_int* n, lapack_int* l, lapack_int* nb, + double* a, lapack_int* lda, double* b, lapack_int* ldb, + double* t, lapack_int* ldt, double* work, + lapack_int *info ); +void LAPACK_ctpqrt( lapack_int* m, lapack_int* n, lapack_int* l, lapack_int* nb, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* t, lapack_complex_float* b, + lapack_int* ldb, lapack_int* ldt, + lapack_complex_float* work, lapack_int *info ); +void LAPACK_ztpqrt( lapack_int* m, lapack_int* n, lapack_int* l, lapack_int* nb, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* work, lapack_int *info ); +void LAPACK_stpqrt2( lapack_int* m, lapack_int* n, float* a, lapack_int* lda, + float* b, lapack_int* ldb, float* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_dtpqrt2( lapack_int* m, lapack_int* n, double* a, lapack_int* lda, + double* b, lapack_int* ldb, double* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_ctpqrt2( lapack_int* m, lapack_int* n, lapack_complex_float* a, + lapack_int* lda, lapack_complex_float* b, lapack_int* ldb, + lapack_complex_float* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_ztpqrt2( lapack_int* m, lapack_int* n, lapack_complex_double* a, + lapack_int* lda, lapack_complex_double* b, lapack_int* ldb, + lapack_complex_double* t, lapack_int* ldt, + lapack_int *info ); +void LAPACK_stprfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, lapack_int* l, + const float* v, lapack_int* ldv, const float* t, + lapack_int* ldt, float* a, lapack_int* lda, float* b, + lapack_int* ldb, const float* mywork, + lapack_int* myldwork ); +void LAPACK_dtprfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, lapack_int* l, + const double* v, lapack_int* ldv, const double* t, + lapack_int* ldt, double* a, lapack_int* lda, double* b, + lapack_int* ldb, const double* mywork, + lapack_int* myldwork ); +void LAPACK_ctprfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, lapack_int* l, + const lapack_complex_float* v, lapack_int* ldv, + const lapack_complex_float* t, lapack_int* ldt, + lapack_complex_float* a, lapack_int* lda, + lapack_complex_float* b, lapack_int* ldb, + const float* mywork, lapack_int* myldwork ); +void LAPACK_ztprfb( char* side, char* trans, char* direct, char* storev, + lapack_int* m, lapack_int* n, lapack_int* k, lapack_int* l, + const lapack_complex_double* v, lapack_int* ldv, + const lapack_complex_double* t, lapack_int* ldt, + lapack_complex_double* a, lapack_int* lda, + lapack_complex_double* b, lapack_int* ldb, + const double* mywork, lapack_int* myldwork ); +// LAPACK 3.X.X +void LAPACK_csyr( char* uplo, lapack_int* n, lapack_complex_float* alpha, + const lapack_complex_float* x, lapack_int* incx, + lapack_complex_float* a, lapack_int* lda ); +void LAPACK_zsyr( char* uplo, lapack_int* n, lapack_complex_double* alpha, + const lapack_complex_double* x, lapack_int* incx, + lapack_complex_double* a, lapack_int* lda ); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _LAPACKE_H_ */ + +#endif /* _MKL_LAPACKE_H_ */
diff --git a/Eigen/src/misc/lapacke_mangling.h b/Eigen/src/misc/lapacke_mangling.h new file mode 100644 index 0000000..6211fd1 --- /dev/null +++ b/Eigen/src/misc/lapacke_mangling.h
@@ -0,0 +1,17 @@ +#ifndef LAPACK_HEADER_INCLUDED +#define LAPACK_HEADER_INCLUDED + +#ifndef LAPACK_GLOBAL +#if defined(LAPACK_GLOBAL_PATTERN_LC) || defined(ADD_) +#define LAPACK_GLOBAL(lcname,UCNAME) lcname##_ +#elif defined(LAPACK_GLOBAL_PATTERN_UC) || defined(UPPER) +#define LAPACK_GLOBAL(lcname,UCNAME) UCNAME +#elif defined(LAPACK_GLOBAL_PATTERN_MC) || defined(NOCHANGE) +#define LAPACK_GLOBAL(lcname,UCNAME) lcname +#else +#define LAPACK_GLOBAL(lcname,UCNAME) lcname##_ +#endif +#endif + +#endif +
diff --git a/Eigen/src/plugins/ArrayCwiseBinaryOps.h b/Eigen/src/plugins/ArrayCwiseBinaryOps.h index 5694592..577196c 100644 --- a/Eigen/src/plugins/ArrayCwiseBinaryOps.h +++ b/Eigen/src/plugins/ArrayCwiseBinaryOps.h
@@ -1,13 +1,14 @@ + /** \returns an expression of the coefficient wise product of \c *this and \a other * * \sa MatrixBase::cwiseProduct */ template<typename OtherDerived> EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const EIGEN_CWISE_PRODUCT_RETURN_TYPE(Derived,OtherDerived) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product) operator*(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const { - return EIGEN_CWISE_PRODUCT_RETURN_TYPE(Derived,OtherDerived)(derived(), other.derived()); + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product)(derived(), other.derived()); } /** \returns an expression of the coefficient wise quotient of \c *this and \a other @@ -16,10 +17,10 @@ */ template<typename OtherDerived> EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const Derived, const OtherDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_quotient_op<Scalar,typename OtherDerived::Scalar>, const Derived, const OtherDerived> operator/(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const { - return CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); + return CwiseBinaryOp<internal::scalar_quotient_op<Scalar,typename OtherDerived::Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); } /** \returns an expression of the coefficient-wise min of \c *this and \a other @@ -29,14 +30,14 @@ * * \sa max() */ -EIGEN_MAKE_CWISE_BINARY_OP(min,internal::scalar_min_op) +EIGEN_MAKE_CWISE_BINARY_OP(min,min) /** \returns an expression of the coefficient-wise min of \c *this and scalar \a other * * \sa max() */ EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar>, const Derived, +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar,Scalar>, const Derived, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> > #ifdef EIGEN_PARSED_BY_DOXYGEN min @@ -55,14 +56,14 @@ * * \sa min() */ -EIGEN_MAKE_CWISE_BINARY_OP(max,internal::scalar_max_op) +EIGEN_MAKE_CWISE_BINARY_OP(max,max) /** \returns an expression of the coefficient-wise max of \c *this and scalar \a other * * \sa min() */ EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar>, const Derived, +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar,Scalar>, const Derived, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> > #ifdef EIGEN_PARSED_BY_DOXYGEN max @@ -81,27 +82,38 @@ * Example: \include Cwise_array_power_array.cpp * Output: \verbinclude Cwise_array_power_array.out */ -template<typename ExponentDerived> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -const CwiseBinaryOp<internal::scalar_binary_pow_op<Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived> -pow(const ArrayBase<ExponentDerived>& exponents) const -{ - return CwiseBinaryOp<internal::scalar_binary_pow_op<Scalar, typename ExponentDerived::Scalar>, const Derived, const ExponentDerived>( - this->derived(), - exponents.derived() - ); -} +EIGEN_MAKE_CWISE_BINARY_OP(pow,pow) + +#ifndef EIGEN_PARSED_BY_DOXYGEN +EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(pow,pow) +#else +/** \returns an expression of the coefficients of \c *this rasied to the constant power \a exponent + * + * \tparam T is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression. + * + * This function computes the coefficient-wise power. The function MatrixBase::pow() in the + * unsupported module MatrixFunctions computes the matrix power. + * + * Example: \include Cwise_pow.cpp + * Output: \verbinclude Cwise_pow.out + * + * \sa ArrayBase::pow(ArrayBase), square(), cube(), exp(), log() + */ +template<typename T> +const CwiseBinaryOp<internal::scalar_pow_op<Scalar,T>,Derived,Constant<T> > pow(const T& exponent) const; +#endif + // TODO code generating macros could be moved to Macros.h and could include generation of documentation #define EIGEN_MAKE_CWISE_COMP_OP(OP, COMPARATOR) \ template<typename OtherDerived> \ -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const OtherDerived> \ +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_cmp_op<Scalar, typename OtherDerived::Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const OtherDerived> \ OP(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const \ { \ - return CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const OtherDerived>(derived(), other.derived()); \ + return CwiseBinaryOp<internal::scalar_cmp_op<Scalar, typename OtherDerived::Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const OtherDerived>(derived(), other.derived()); \ }\ -typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> > Cmp ## COMPARATOR ## ReturnType; \ -typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_ ## COMPARATOR>, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject>, const Derived > RCmp ## COMPARATOR ## ReturnType; \ +typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar,Scalar, internal::cmp_ ## COMPARATOR>, const Derived, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> > Cmp ## COMPARATOR ## ReturnType; \ +typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar,Scalar, internal::cmp_ ## COMPARATOR>, const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject>, const Derived > RCmp ## COMPARATOR ## ReturnType; \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Cmp ## COMPARATOR ## ReturnType \ OP(const Scalar& s) const { \ return this->OP(Derived::PlainObject::Constant(rows(), cols(), s)); \ @@ -113,10 +125,10 @@ #define EIGEN_MAKE_CWISE_COMP_R_OP(OP, R_OP, RCOMPARATOR) \ template<typename OtherDerived> \ -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_##RCOMPARATOR>, const OtherDerived, const Derived> \ +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_cmp_op<typename OtherDerived::Scalar, Scalar, internal::cmp_##RCOMPARATOR>, const OtherDerived, const Derived> \ OP(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const \ { \ - return CwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_##RCOMPARATOR>, const OtherDerived, const Derived>(other.derived(), derived()); \ + return CwiseBinaryOp<internal::scalar_cmp_op<typename OtherDerived::Scalar, Scalar, internal::cmp_##RCOMPARATOR>, const OtherDerived, const Derived>(other.derived(), derived()); \ } \ EIGEN_DEVICE_FUNC \ inline const RCmp ## RCOMPARATOR ## ReturnType \ @@ -199,86 +211,63 @@ #undef EIGEN_MAKE_CWISE_COMP_R_OP // scalar addition - +#ifndef EIGEN_PARSED_BY_DOXYGEN +EIGEN_MAKE_SCALAR_BINARY_OP(operator+,sum) +#else /** \returns an expression of \c *this with each coeff incremented by the constant \a scalar * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + * * Example: \include Cwise_plus.cpp * Output: \verbinclude Cwise_plus.out * * \sa operator+=(), operator-() */ -EIGEN_DEVICE_FUNC -inline const CwiseUnaryOp<internal::scalar_add_op<Scalar>, const Derived> -operator+(const Scalar& scalar) const -{ - return CwiseUnaryOp<internal::scalar_add_op<Scalar>, const Derived>(derived(), internal::scalar_add_op<Scalar>(scalar)); -} +template<typename T> +const CwiseBinaryOp<internal::scalar_sum_op<Scalar,T>,Derived,Constant<T> > operator+(const T& scalar) const; +/** \returns an expression of \a expr with each coeff incremented by the constant \a scalar + * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + */ +template<typename T> friend +const CwiseBinaryOp<internal::scalar_sum_op<T,Scalar>,Constant<T>,Derived> operator+(const T& scalar, const StorageBaseType& expr); +#endif -EIGEN_DEVICE_FUNC -friend inline const CwiseUnaryOp<internal::scalar_add_op<Scalar>, const Derived> -operator+(const Scalar& scalar,const EIGEN_CURRENT_STORAGE_BASE_CLASS<Derived>& other) -{ - return other + scalar; -} - +#ifndef EIGEN_PARSED_BY_DOXYGEN +EIGEN_MAKE_SCALAR_BINARY_OP(operator-,difference) +#else /** \returns an expression of \c *this with each coeff decremented by the constant \a scalar * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + * * Example: \include Cwise_minus.cpp * Output: \verbinclude Cwise_minus.out * - * \sa operator+(), operator-=() + * \sa operator+=(), operator-() */ -EIGEN_DEVICE_FUNC -inline const CwiseUnaryOp<internal::scalar_sub_op<Scalar>, const Derived> -operator-(const Scalar& scalar) const -{ - return CwiseUnaryOp<internal::scalar_sub_op<Scalar>, const Derived>(derived(), internal::scalar_sub_op<Scalar>(scalar));; -} - -EIGEN_DEVICE_FUNC -friend inline const CwiseUnaryOp<internal::scalar_rsub_op<Scalar>, const Derived> -operator-(const Scalar& scalar,const EIGEN_CURRENT_STORAGE_BASE_CLASS<Derived>& other) -{ - return CwiseUnaryOp<internal::scalar_rsub_op<Scalar>, const Derived>(other.derived(), internal::scalar_rsub_op<Scalar>(scalar));; -} - -/** \returns an expression of the coefficient-wise && operator of *this and \a other +template<typename T> +const CwiseBinaryOp<internal::scalar_difference_op<Scalar,T>,Derived,Constant<T> > operator-(const T& scalar) const; +/** \returns an expression of the constant matrix of value \a scalar decremented by the coefficients of \a expr * - * \warning this operator is for expression of bool only. - * - * Example: \include Cwise_boolean_and.cpp - * Output: \verbinclude Cwise_boolean_and.out - * - * \sa operator||(), select() + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. */ -template<typename OtherDerived> -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp<internal::scalar_boolean_and_op, const Derived, const OtherDerived> -operator&&(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const -{ - EIGEN_STATIC_ASSERT((internal::is_same<bool,Scalar>::value && internal::is_same<bool,typename OtherDerived::Scalar>::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); - return CwiseBinaryOp<internal::scalar_boolean_and_op, const Derived, const OtherDerived>(derived(),other.derived()); -} +template<typename T> friend +const CwiseBinaryOp<internal::scalar_difference_op<T,Scalar>,Constant<T>,Derived> operator-(const T& scalar, const StorageBaseType& expr); +#endif -/** \returns an expression of the coefficient-wise || operator of *this and \a other - * - * \warning this operator is for expression of bool only. - * - * Example: \include Cwise_boolean_or.cpp - * Output: \verbinclude Cwise_boolean_or.out - * - * \sa operator&&(), select() - */ -template<typename OtherDerived> -EIGEN_DEVICE_FUNC -inline const CwiseBinaryOp<internal::scalar_boolean_or_op, const Derived, const OtherDerived> -operator||(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const -{ - EIGEN_STATIC_ASSERT((internal::is_same<bool,Scalar>::value && internal::is_same<bool,typename OtherDerived::Scalar>::value), - THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); - return CwiseBinaryOp<internal::scalar_boolean_or_op, const Derived, const OtherDerived>(derived(),other.derived()); -} + +#ifndef EIGEN_PARSED_BY_DOXYGEN + EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(operator/,quotient) +#else + /** + * \brief Component-wise division of the scalar \a s by array elements of \a a. + * + * \tparam Scalar is the scalar type of \a x. It must be compatible with the scalar type of the given array expression (\c Derived::Scalar). + */ + template<typename T> friend + inline const CwiseBinaryOp<internal::scalar_quotient_op<T,Scalar>,Constant<T>,Derived> + operator/(const T& s,const StorageBaseType& a); +#endif /** \returns an expression of the coefficient-wise ^ operator of *this and \a other * @@ -298,3 +287,46 @@ THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); return CwiseBinaryOp<internal::scalar_boolean_xor_op, const Derived, const OtherDerived>(derived(),other.derived()); } + +// NOTE disabled until we agree on argument order +#if 0 +/** \cpp11 \returns an expression of the coefficient-wise polygamma function. + * + * \specialfunctions_module + * + * It returns the \a n -th derivative of the digamma(psi) evaluated at \c *this. + * + * \warning Be careful with the order of the parameters: x.polygamma(n) is equivalent to polygamma(n,x) + * + * \sa Eigen::polygamma() + */ +template<typename DerivedN> +inline const CwiseBinaryOp<internal::scalar_polygamma_op<Scalar>, const DerivedN, const Derived> +polygamma(const EIGEN_CURRENT_STORAGE_BASE_CLASS<DerivedN> &n) const +{ + return CwiseBinaryOp<internal::scalar_polygamma_op<Scalar>, const DerivedN, const Derived>(n.derived(), this->derived()); +} +#endif + +/** \returns an expression of the coefficient-wise zeta function. + * + * \specialfunctions_module + * + * It returns the Riemann zeta function of two arguments \c *this and \a q: + * + * \param *this is the exponent, it must be > 1 + * \param q is the shift, it must be > 0 + * + * \note This function supports only float and double scalar types. To support other scalar types, the user has + * to provide implementations of zeta(T,T) for any scalar type T to be supported. + * + * This method is an alias for zeta(*this,q); + * + * \sa Eigen::zeta() + */ +template<typename DerivedQ> +inline const CwiseBinaryOp<internal::scalar_zeta_op<Scalar>, const Derived, const DerivedQ> +zeta(const EIGEN_CURRENT_STORAGE_BASE_CLASS<DerivedQ> &q) const +{ + return CwiseBinaryOp<internal::scalar_zeta_op<Scalar>, const Derived, const DerivedQ>(this->derived(), q.derived()); +}
diff --git a/Eigen/src/plugins/ArrayCwiseUnaryOps.h b/Eigen/src/plugins/ArrayCwiseUnaryOps.h index 56c7117..2f99ee0 100644 --- a/Eigen/src/plugins/ArrayCwiseUnaryOps.h +++ b/Eigen/src/plugins/ArrayCwiseUnaryOps.h
@@ -10,7 +10,9 @@ typedef CwiseUnaryOp<internal::scalar_boolean_not_op<Scalar>, const Derived> BooleanNotReturnType; typedef CwiseUnaryOp<internal::scalar_exp_op<Scalar>, const Derived> ExpReturnType; +typedef CwiseUnaryOp<internal::scalar_expm1_op<Scalar>, const Derived> Expm1ReturnType; typedef CwiseUnaryOp<internal::scalar_log_op<Scalar>, const Derived> LogReturnType; +typedef CwiseUnaryOp<internal::scalar_log1p_op<Scalar>, const Derived> Log1pReturnType; typedef CwiseUnaryOp<internal::scalar_log10_op<Scalar>, const Derived> Log10ReturnType; typedef CwiseUnaryOp<internal::scalar_cos_op<Scalar>, const Derived> CosReturnType; typedef CwiseUnaryOp<internal::scalar_sin_op<Scalar>, const Derived> SinReturnType; @@ -19,15 +21,14 @@ typedef CwiseUnaryOp<internal::scalar_asin_op<Scalar>, const Derived> AsinReturnType; typedef CwiseUnaryOp<internal::scalar_atan_op<Scalar>, const Derived> AtanReturnType; typedef CwiseUnaryOp<internal::scalar_tanh_op<Scalar>, const Derived> TanhReturnType; +typedef CwiseUnaryOp<internal::scalar_logistic_op<Scalar>, const Derived> LogisticReturnType; typedef CwiseUnaryOp<internal::scalar_sinh_op<Scalar>, const Derived> SinhReturnType; +#if EIGEN_HAS_CXX11_MATH +typedef CwiseUnaryOp<internal::scalar_atanh_op<Scalar>, const Derived> AtanhReturnType; +typedef CwiseUnaryOp<internal::scalar_asinh_op<Scalar>, const Derived> AsinhReturnType; +typedef CwiseUnaryOp<internal::scalar_acosh_op<Scalar>, const Derived> AcoshReturnType; +#endif typedef CwiseUnaryOp<internal::scalar_cosh_op<Scalar>, const Derived> CoshReturnType; -typedef CwiseUnaryOp<internal::scalar_lgamma_op<Scalar>, const Derived> LgammaReturnType; -typedef CwiseUnaryOp<internal::scalar_digamma_op<Scalar>, const Derived> DigammaReturnType; -typedef CwiseUnaryOp<internal::scalar_zeta_op<Scalar>, const Derived> ZetaReturnType; -typedef CwiseUnaryOp<internal::scalar_polygamma_op<Scalar>, const Derived> PolygammaReturnType; -typedef CwiseUnaryOp<internal::scalar_erf_op<Scalar>, const Derived> ErfReturnType; -typedef CwiseUnaryOp<internal::scalar_erfc_op<Scalar>, const Derived> ErfcReturnType; -typedef CwiseUnaryOp<internal::scalar_pow_op<Scalar>, const Derived> PowReturnType; typedef CwiseUnaryOp<internal::scalar_square_op<Scalar>, const Derived> SquareReturnType; typedef CwiseUnaryOp<internal::scalar_cube_op<Scalar>, const Derived> CubeReturnType; typedef CwiseUnaryOp<internal::scalar_round_op<Scalar>, const Derived> RoundReturnType; @@ -42,7 +43,7 @@ * Example: \include Cwise_abs.cpp * Output: \verbinclude Cwise_abs.out * - * \sa abs2() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_abs">Math functions</a>, abs2() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const AbsReturnType @@ -70,7 +71,7 @@ * Example: \include Cwise_abs2.cpp * Output: \verbinclude Cwise_abs2.out * - * \sa abs(), square() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_abs2">Math functions</a>, abs(), square() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Abs2ReturnType @@ -87,7 +88,7 @@ * Example: \include Cwise_exp.cpp * Output: \verbinclude Cwise_exp.out * - * \sa pow(), log(), sin(), cos() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_exp">Math functions</a>, pow(), log(), sin(), cos() */ EIGEN_DEVICE_FUNC inline const ExpReturnType @@ -96,6 +97,20 @@ return ExpReturnType(derived()); } +/** \returns an expression of the coefficient-wise exponential of *this minus 1. + * + * In exact arithmetic, \c x.expm1() is equivalent to \c x.exp() - 1, + * however, with finite precision, this function is much more accurate when \c x is close to zero. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_expm1">Math functions</a>, exp() + */ +EIGEN_DEVICE_FUNC +inline const Expm1ReturnType +expm1() const +{ + return Expm1ReturnType(derived()); +} + /** \returns an expression of the coefficient-wise logarithm of *this. * * This function computes the coefficient-wise logarithm. The function MatrixBase::log() in the @@ -104,7 +119,7 @@ * Example: \include Cwise_log.cpp * Output: \verbinclude Cwise_log.out * - * \sa exp() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_log">Math functions</a>, log() */ EIGEN_DEVICE_FUNC inline const LogReturnType @@ -113,6 +128,20 @@ return LogReturnType(derived()); } +/** \returns an expression of the coefficient-wise logarithm of 1 plus \c *this. + * + * In exact arithmetic, \c x.log() is equivalent to \c (x+1).log(), + * however, with finite precision, this function is much more accurate when \c x is close to zero. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_log1p">Math functions</a>, log() + */ +EIGEN_DEVICE_FUNC +inline const Log1pReturnType +log1p() const +{ + return Log1pReturnType(derived()); +} + /** \returns an expression of the coefficient-wise base-10 logarithm of *this. * * This function computes the coefficient-wise base-10 logarithm. @@ -120,7 +149,7 @@ * Example: \include Cwise_log10.cpp * Output: \verbinclude Cwise_log10.out * - * \sa log() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_log10">Math functions</a>, log() */ EIGEN_DEVICE_FUNC inline const Log10ReturnType @@ -137,7 +166,7 @@ * Example: \include Cwise_sqrt.cpp * Output: \verbinclude Cwise_sqrt.out * - * \sa pow(), square() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_sqrt">Math functions</a>, pow(), square() */ EIGEN_DEVICE_FUNC inline const SqrtReturnType @@ -187,7 +216,7 @@ * Example: \include Cwise_cos.cpp * Output: \verbinclude Cwise_cos.out * - * \sa sin(), acos() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_cos">Math functions</a>, sin(), acos() */ EIGEN_DEVICE_FUNC inline const CosReturnType @@ -205,7 +234,7 @@ * Example: \include Cwise_sin.cpp * Output: \verbinclude Cwise_sin.out * - * \sa cos(), asin() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_sin">Math functions</a>, cos(), asin() */ EIGEN_DEVICE_FUNC inline const SinReturnType @@ -219,7 +248,7 @@ * Example: \include Cwise_tan.cpp * Output: \verbinclude Cwise_tan.out * - * \sa cos(), sin() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_tan">Math functions</a>, cos(), sin() */ EIGEN_DEVICE_FUNC inline const TanReturnType @@ -233,8 +262,9 @@ * Example: \include Cwise_atan.cpp * Output: \verbinclude Cwise_atan.out * - * \sa tan(), asin(), acos() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_atan">Math functions</a>, tan(), asin(), acos() */ +EIGEN_DEVICE_FUNC inline const AtanReturnType atan() const { @@ -246,7 +276,7 @@ * Example: \include Cwise_acos.cpp * Output: \verbinclude Cwise_acos.out * - * \sa cos(), asin() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_acos">Math functions</a>, cos(), asin() */ EIGEN_DEVICE_FUNC inline const AcosReturnType @@ -260,7 +290,7 @@ * Example: \include Cwise_asin.cpp * Output: \verbinclude Cwise_asin.out * - * \sa sin(), acos() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_asin">Math functions</a>, sin(), acos() */ EIGEN_DEVICE_FUNC inline const AsinReturnType @@ -274,8 +304,9 @@ * Example: \include Cwise_tanh.cpp * Output: \verbinclude Cwise_tanh.out * - * \sa tan(), sinh(), cosh() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_tanh">Math functions</a>, tan(), sinh(), cosh() */ +EIGEN_DEVICE_FUNC inline const TanhReturnType tanh() const { @@ -287,8 +318,9 @@ * Example: \include Cwise_sinh.cpp * Output: \verbinclude Cwise_sinh.out * - * \sa sin(), tanh(), cosh() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_sinh">Math functions</a>, sin(), tanh(), cosh() */ +EIGEN_DEVICE_FUNC inline const SinhReturnType sinh() const { @@ -300,98 +332,58 @@ * Example: \include Cwise_cosh.cpp * Output: \verbinclude Cwise_cosh.out * - * \sa tan(), sinh(), cosh() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_cosh">Math functions</a>, tanh(), sinh(), cosh() */ +EIGEN_DEVICE_FUNC inline const CoshReturnType cosh() const { return CoshReturnType(derived()); } -/** \returns an expression of the coefficient-wise ln(|gamma(*this)|). - * - * Example: \include Cwise_lgamma.cpp - * Output: \verbinclude Cwise_lgamma.out - * - * \sa cos(), sin(), tan() - */ -inline const LgammaReturnType -lgamma() const -{ - return LgammaReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise digamma (psi, derivative of lgamma). - * - * \sa cos(), sin(), tan() - */ -inline const DigammaReturnType -digamma() const -{ - return DigammaReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise zeta function. - */ -inline const ZetaReturnType -zeta() const -{ - return ZetaReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise polygamma function. - */ -inline const PolygammaReturnType -polygamma() const -{ - return PolygammaReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise Gauss error - * function of *this. - * - * Example: \include Cwise_erf.cpp - * Output: \verbinclude Cwise_erf.out - * - * \sa cos(), sin(), tan() - */ -inline const ErfReturnType -erf() const -{ - return ErfReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise Complementary error - * function of *this. - * - * Example: \include Cwise_erfc.cpp - * Output: \verbinclude Cwise_erfc.out - * - * \sa cos(), sin(), tan() - */ -inline const ErfcReturnType -erfc() const -{ - return ErfcReturnType(derived()); -} - -/** \returns an expression of the coefficient-wise power of *this to the given exponent. +#if EIGEN_HAS_CXX11_MATH +/** \returns an expression of the coefficient-wise inverse hyperbolic tan of *this. * - * This function computes the coefficient-wise power. The function MatrixBase::pow() in the - * unsupported module MatrixFunctions computes the matrix power. - * - * Example: \include Cwise_pow.cpp - * Output: \verbinclude Cwise_pow.out - * - * \sa exp(), log() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_atanh">Math functions</a>, atanh(), asinh(), acosh() */ EIGEN_DEVICE_FUNC -inline const PowReturnType -pow(const Scalar& exponent) const +inline const AtanhReturnType +atanh() const { - return PowReturnType(derived(), internal::scalar_pow_op<Scalar>(exponent)); + return AtanhReturnType(derived()); } +/** \returns an expression of the coefficient-wise inverse hyperbolic sin of *this. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_asinh">Math functions</a>, atanh(), asinh(), acosh() + */ +EIGEN_DEVICE_FUNC +inline const AsinhReturnType +asinh() const +{ + return AsinhReturnType(derived()); +} + +/** \returns an expression of the coefficient-wise inverse hyperbolic cos of *this. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_acosh">Math functions</a>, atanh(), asinh(), acosh() + */ +EIGEN_DEVICE_FUNC +inline const AcoshReturnType +acosh() const +{ + return AcoshReturnType(derived()); +} +#endif + +/** \returns an expression of the coefficient-wise logistic of *this. + */ +EIGEN_DEVICE_FUNC +inline const LogisticReturnType +logistic() const +{ + return LogisticReturnType(derived()); +} /** \returns an expression of the coefficient-wise inverse of *this. * @@ -412,7 +404,7 @@ * Example: \include Cwise_square.cpp * Output: \verbinclude Cwise_square.out * - * \sa operator/(), operator*(), abs2() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_squareE">Math functions</a>, abs2(), cube(), pow() */ EIGEN_DEVICE_FUNC inline const SquareReturnType @@ -426,7 +418,7 @@ * Example: \include Cwise_cube.cpp * Output: \verbinclude Cwise_cube.out * - * \sa square(), pow() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_cube">Math functions</a>, square(), pow() */ EIGEN_DEVICE_FUNC inline const CubeReturnType @@ -440,8 +432,9 @@ * Example: \include Cwise_round.cpp * Output: \verbinclude Cwise_round.out * - * \sa ceil(), floor() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_round">Math functions</a>, ceil(), floor() */ +EIGEN_DEVICE_FUNC inline const RoundReturnType round() const { @@ -453,8 +446,9 @@ * Example: \include Cwise_floor.cpp * Output: \verbinclude Cwise_floor.out * - * \sa ceil(), round() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_floor">Math functions</a>, ceil(), round() */ +EIGEN_DEVICE_FUNC inline const FloorReturnType floor() const { @@ -466,8 +460,9 @@ * Example: \include Cwise_ceil.cpp * Output: \verbinclude Cwise_ceil.out * - * \sa floor(), round() + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_ceil">Math functions</a>, floor(), round() */ +EIGEN_DEVICE_FUNC inline const CeilReturnType ceil() const { @@ -481,6 +476,7 @@ * * \sa isfinite(), isinf() */ +EIGEN_DEVICE_FUNC inline const IsNaNReturnType isNaN() const { @@ -494,6 +490,7 @@ * * \sa isnan(), isfinite() */ +EIGEN_DEVICE_FUNC inline const IsInfReturnType isInf() const { @@ -507,6 +504,7 @@ * * \sa isnan(), isinf() */ +EIGEN_DEVICE_FUNC inline const IsFiniteReturnType isFinite() const { @@ -530,3 +528,90 @@ THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); return BooleanNotReturnType(derived()); } + + +// --- SpecialFunctions module --- + +typedef CwiseUnaryOp<internal::scalar_lgamma_op<Scalar>, const Derived> LgammaReturnType; +typedef CwiseUnaryOp<internal::scalar_digamma_op<Scalar>, const Derived> DigammaReturnType; +typedef CwiseUnaryOp<internal::scalar_erf_op<Scalar>, const Derived> ErfReturnType; +typedef CwiseUnaryOp<internal::scalar_erfc_op<Scalar>, const Derived> ErfcReturnType; + +/** \cpp11 \returns an expression of the coefficient-wise ln(|gamma(*this)|). + * + * \specialfunctions_module + * + * Example: \include Cwise_lgamma.cpp + * Output: \verbinclude Cwise_lgamma.out + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of lgamma(T) for any scalar + * type T to be supported. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_lgamma">Math functions</a>, digamma() + */ +EIGEN_DEVICE_FUNC +inline const LgammaReturnType +lgamma() const +{ + return LgammaReturnType(derived()); +} + +/** \returns an expression of the coefficient-wise digamma (psi, derivative of lgamma). + * + * \specialfunctions_module + * + * \note This function supports only float and double scalar types. To support other scalar types, + * the user has to provide implementations of digamma(T) for any scalar + * type T to be supported. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_digamma">Math functions</a>, Eigen::digamma(), Eigen::polygamma(), lgamma() + */ +EIGEN_DEVICE_FUNC +inline const DigammaReturnType +digamma() const +{ + return DigammaReturnType(derived()); +} + +/** \cpp11 \returns an expression of the coefficient-wise Gauss error + * function of *this. + * + * \specialfunctions_module + * + * Example: \include Cwise_erf.cpp + * Output: \verbinclude Cwise_erf.out + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of erf(T) for any scalar + * type T to be supported. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_erf">Math functions</a>, erfc() + */ +EIGEN_DEVICE_FUNC +inline const ErfReturnType +erf() const +{ + return ErfReturnType(derived()); +} + +/** \cpp11 \returns an expression of the coefficient-wise Complementary error + * function of *this. + * + * \specialfunctions_module + * + * Example: \include Cwise_erfc.cpp + * Output: \verbinclude Cwise_erfc.out + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of erfc(T) for any scalar + * type T to be supported. + * + * \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_erfc">Math functions</a>, erf() + */ +EIGEN_DEVICE_FUNC +inline const ErfcReturnType +erfc() const +{ + return ErfcReturnType(derived()); +}
diff --git a/Eigen/src/plugins/BlockMethods.h b/Eigen/src/plugins/BlockMethods.h index 632094e..935a604 100644 --- a/Eigen/src/plugins/BlockMethods.h +++ b/Eigen/src/plugins/BlockMethods.h
@@ -10,28 +10,28 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN -/** \internal expression type of a column */ +/// \internal expression type of a column */ typedef Block<Derived, internal::traits<Derived>::RowsAtCompileTime, 1, !IsRowMajor> ColXpr; typedef const Block<const Derived, internal::traits<Derived>::RowsAtCompileTime, 1, !IsRowMajor> ConstColXpr; -/** \internal expression type of a row */ +/// \internal expression type of a row */ typedef Block<Derived, 1, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> RowXpr; typedef const Block<const Derived, 1, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> ConstRowXpr; -/** \internal expression type of a block of whole columns */ +/// \internal expression type of a block of whole columns */ typedef Block<Derived, internal::traits<Derived>::RowsAtCompileTime, Dynamic, !IsRowMajor> ColsBlockXpr; typedef const Block<const Derived, internal::traits<Derived>::RowsAtCompileTime, Dynamic, !IsRowMajor> ConstColsBlockXpr; -/** \internal expression type of a block of whole rows */ +/// \internal expression type of a block of whole rows */ typedef Block<Derived, Dynamic, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> RowsBlockXpr; typedef const Block<const Derived, Dynamic, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> ConstRowsBlockXpr; -/** \internal expression type of a block of whole columns */ +/// \internal expression type of a block of whole columns */ template<int N> struct NColsBlockXpr { typedef Block<Derived, internal::traits<Derived>::RowsAtCompileTime, N, !IsRowMajor> Type; }; template<int N> struct ConstNColsBlockXpr { typedef const Block<const Derived, internal::traits<Derived>::RowsAtCompileTime, N, !IsRowMajor> Type; }; -/** \internal expression type of a block of whole rows */ +/// \internal expression type of a block of whole rows */ template<int N> struct NRowsBlockXpr { typedef Block<Derived, N, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> Type; }; template<int N> struct ConstNRowsBlockXpr { typedef const Block<const Derived, N, internal::traits<Derived>::ColsAtCompileTime, IsRowMajor> Type; }; -/** \internal expression of a block */ +/// \internal expression of a block */ typedef Block<Derived> BlockXpr; typedef const Block<const Derived> ConstBlockXpr; -/** \internal expression of a block of fixed sizes */ +/// \internal expression of a block of fixed sizes */ template<int Rows, int Cols> struct FixedBlockXpr { typedef Block<Derived,Rows,Cols> Type; }; template<int Rows, int Cols> struct ConstFixedBlockXpr { typedef Block<const Derived,Rows,Cols> Type; }; @@ -40,961 +40,1404 @@ template<int Size> struct FixedSegmentReturnType { typedef VectorBlock<Derived, Size> Type; }; template<int Size> struct ConstFixedSegmentReturnType { typedef const VectorBlock<const Derived, Size> Type; }; +/// \internal inner-vector +typedef Block<Derived,IsRowMajor?1:Dynamic,IsRowMajor?Dynamic:1,true> InnerVectorReturnType; +typedef Block<const Derived,IsRowMajor?1:Dynamic,IsRowMajor?Dynamic:1,true> ConstInnerVectorReturnType; + +/// \internal set of inner-vectors +typedef Block<Derived,Dynamic,Dynamic,true> InnerVectorsReturnType; +typedef Block<const Derived,Dynamic,Dynamic,true> ConstInnerVectorsReturnType; + #endif // not EIGEN_PARSED_BY_DOXYGEN -/** \returns a dynamic-size expression of a block in *this. - * - * \param startRow the first row in the block - * \param startCol the first column in the block - * \param blockRows the number of rows in the block - * \param blockCols the number of columns in the block - * - * Example: \include MatrixBase_block_int_int_int_int.cpp - * Output: \verbinclude MatrixBase_block_int_int_int_int.out - * - * \note Even though the returned expression has dynamic size, in the case - * when it is applied to a fixed-size matrix, it inherits a fixed maximal size, - * which means that evaluating it does not cause a dynamic memory allocation. - * - * \sa class Block, block(Index,Index) - */ -EIGEN_DEVICE_FUNC -inline BlockXpr block(Index startRow, Index startCol, Index blockRows, Index blockCols) +/// \returns an expression of a block in \c *this with either dynamic or fixed sizes. +/// +/// \param startRow the first row in the block +/// \param startCol the first column in the block +/// \param blockRows number of rows in the block, specified at either run-time or compile-time +/// \param blockCols number of columns in the block, specified at either run-time or compile-time +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example using runtime (aka dynamic) sizes: \include MatrixBase_block_int_int_int_int.cpp +/// Output: \verbinclude MatrixBase_block_int_int_int_int.out +/// +/// \newin{3.4}: +/// +/// The number of rows \a blockRows and columns \a blockCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. In the later case, \c n plays the role of a runtime fallback value in case \c N equals Eigen::Dynamic. +/// Here is an example with a fixed number of rows \c NRows and dynamic number of columns \c cols: +/// \code +/// mat.block(i,j,fix<NRows>,cols) +/// \endcode +/// +/// This function thus fully covers the features offered by the following overloads block<NRows,NCols>(Index, Index), +/// and block<NRows,NCols>(Index, Index, Index, Index) that are thus obsolete. Indeed, this generic version avoids +/// redundancy, it preserves the argument order, and prevents the need to rely on the template keyword in templated code. +/// +/// but with less redundancy and more consistency as it does not modify the argument order +/// and seamlessly enable hybrid fixed/dynamic sizes. +/// +/// \note Even in the case that the returned expression has dynamic size, in the case +/// when it is applied to a fixed-size matrix, it inherits a fixed maximal size, +/// which means that evaluating it does not cause a dynamic memory allocation. +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block, fix, fix<N>(int) +/// +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename FixedBlockXpr<...,...>::Type +#endif +block(Index startRow, Index startCol, NRowsType blockRows, NColsType blockCols) { - return BlockXpr(derived(), startRow, startCol, blockRows, blockCols); + return typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type( + derived(), startRow, startCol, internal::get_runtime_value(blockRows), internal::get_runtime_value(blockCols)); } -/** This is the const version of block(Index,Index,Index,Index). */ -EIGEN_DEVICE_FUNC -inline const ConstBlockXpr block(Index startRow, Index startCol, Index blockRows, Index blockCols) const +/// This is the const version of block(Index,Index,NRowsType,NColsType) +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstFixedBlockXpr<...,...>::Type +#endif +block(Index startRow, Index startCol, NRowsType blockRows, NColsType blockCols) const { - return ConstBlockXpr(derived(), startRow, startCol, blockRows, blockCols); + return typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type( + derived(), startRow, startCol, internal::get_runtime_value(blockRows), internal::get_runtime_value(blockCols)); } - -/** \returns a dynamic-size expression of a top-right corner of *this. - * - * \param cRows the number of rows in the corner - * \param cCols the number of columns in the corner - * - * Example: \include MatrixBase_topRightCorner_int_int.cpp - * Output: \verbinclude MatrixBase_topRightCorner_int_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline BlockXpr topRightCorner(Index cRows, Index cCols) +/// \returns a expression of a top-right corner of \c *this with either dynamic or fixed sizes. +/// +/// \param cRows the number of rows in the corner +/// \param cCols the number of columns in the corner +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example with dynamic sizes: \include MatrixBase_topRightCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_topRightCorner_int_int.out +/// +/// The number of rows \a blockRows and columns \a blockCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename FixedBlockXpr<...,...>::Type +#endif +topRightCorner(NRowsType cRows, NColsType cCols) { - return BlockXpr(derived(), 0, cols() - cCols, cRows, cCols); + return typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, cols() - internal::get_runtime_value(cCols), internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** This is the const version of topRightCorner(Index, Index).*/ -EIGEN_DEVICE_FUNC -inline const ConstBlockXpr topRightCorner(Index cRows, Index cCols) const +/// This is the const version of topRightCorner(NRowsType, NColsType). +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstFixedBlockXpr<...,...>::Type +#endif +topRightCorner(NRowsType cRows, NColsType cCols) const { - return ConstBlockXpr(derived(), 0, cols() - cCols, cRows, cCols); + return typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, cols() - internal::get_runtime_value(cCols), internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** \returns an expression of a fixed-size top-right corner of *this. - * - * \tparam CRows the number of rows in the corner - * \tparam CCols the number of columns in the corner - * - * Example: \include MatrixBase_template_int_int_topRightCorner.cpp - * Output: \verbinclude MatrixBase_template_int_int_topRightCorner.out - * - * \sa class Block, block<int,int>(Index,Index) - */ +/// \returns an expression of a fixed-size top-right corner of \c *this. +/// +/// \tparam CRows the number of rows in the corner +/// \tparam CCols the number of columns in the corner +/// +/// Example: \include MatrixBase_template_int_int_topRightCorner.cpp +/// Output: \verbinclude MatrixBase_template_int_int_topRightCorner.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block, block<int,int>(Index,Index) +/// template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline typename FixedBlockXpr<CRows,CCols>::Type topRightCorner() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type topRightCorner() { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), 0, cols() - CCols); } -/** This is the const version of topRightCorner<int, int>().*/ +/// This is the const version of topRightCorner<int, int>(). template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type topRightCorner() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type topRightCorner() const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), 0, cols() - CCols); } -/** \returns an expression of a top-right corner of *this. - * - * \tparam CRows number of rows in corner as specified at compile-time - * \tparam CCols number of columns in corner as specified at compile-time - * \param cRows number of rows in corner as specified at run-time - * \param cCols number of columns in corner as specified at run-time - * - * This function is mainly useful for corners where the number of rows is specified at compile-time - * and the number of columns is specified at run-time, or vice versa. The compile-time and run-time - * information should not contradict. In other words, \a cRows should equal \a CRows unless - * \a CRows is \a Dynamic, and the same for the number of columns. - * - * Example: \include MatrixBase_template_int_int_topRightCorner_int_int.cpp - * Output: \verbinclude MatrixBase_template_int_int_topRightCorner_int_int.out - * - * \sa class Block - */ +/// \returns an expression of a top-right corner of \c *this. +/// +/// \tparam CRows number of rows in corner as specified at compile-time +/// \tparam CCols number of columns in corner as specified at compile-time +/// \param cRows number of rows in corner as specified at run-time +/// \param cCols number of columns in corner as specified at run-time +/// +/// This function is mainly useful for corners where the number of rows is specified at compile-time +/// and the number of columns is specified at run-time, or vice versa. The compile-time and run-time +/// information should not contradict. In other words, \a cRows should equal \a CRows unless +/// \a CRows is \a Dynamic, and the same for the number of columns. +/// +/// Example: \include MatrixBase_template_int_int_topRightCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_template_int_int_topRightCorner_int_int.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block +/// template<int CRows, int CCols> -inline typename FixedBlockXpr<CRows,CCols>::Type topRightCorner(Index cRows, Index cCols) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type topRightCorner(Index cRows, Index cCols) { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), 0, cols() - cCols, cRows, cCols); } -/** This is the const version of topRightCorner<int, int>(Index, Index).*/ +/// This is the const version of topRightCorner<int, int>(Index, Index). template<int CRows, int CCols> -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type topRightCorner(Index cRows, Index cCols) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type topRightCorner(Index cRows, Index cCols) const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), 0, cols() - cCols, cRows, cCols); } -/** \returns a dynamic-size expression of a top-left corner of *this. - * - * \param cRows the number of rows in the corner - * \param cCols the number of columns in the corner - * - * Example: \include MatrixBase_topLeftCorner_int_int.cpp - * Output: \verbinclude MatrixBase_topLeftCorner_int_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline BlockXpr topLeftCorner(Index cRows, Index cCols) +/// \returns an expression of a top-left corner of \c *this with either dynamic or fixed sizes. +/// +/// \param cRows the number of rows in the corner +/// \param cCols the number of columns in the corner +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include MatrixBase_topLeftCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_topLeftCorner_int_int.out +/// +/// The number of rows \a blockRows and columns \a blockCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename FixedBlockXpr<...,...>::Type +#endif +topLeftCorner(NRowsType cRows, NColsType cCols) { - return BlockXpr(derived(), 0, 0, cRows, cCols); + return typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, 0, internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** This is the const version of topLeftCorner(Index, Index).*/ -EIGEN_DEVICE_FUNC -inline const ConstBlockXpr topLeftCorner(Index cRows, Index cCols) const +/// This is the const version of topLeftCorner(Index, Index). +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstFixedBlockXpr<...,...>::Type +#endif +topLeftCorner(NRowsType cRows, NColsType cCols) const { - return ConstBlockXpr(derived(), 0, 0, cRows, cCols); + return typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, 0, internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** \returns an expression of a fixed-size top-left corner of *this. - * - * The template parameters CRows and CCols are the number of rows and columns in the corner. - * - * Example: \include MatrixBase_template_int_int_topLeftCorner.cpp - * Output: \verbinclude MatrixBase_template_int_int_topLeftCorner.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns an expression of a fixed-size top-left corner of \c *this. +/// +/// The template parameters CRows and CCols are the number of rows and columns in the corner. +/// +/// Example: \include MatrixBase_template_int_int_topLeftCorner.cpp +/// Output: \verbinclude MatrixBase_template_int_int_topLeftCorner.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline typename FixedBlockXpr<CRows,CCols>::Type topLeftCorner() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type topLeftCorner() { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), 0, 0); } -/** This is the const version of topLeftCorner<int, int>().*/ +/// This is the const version of topLeftCorner<int, int>(). template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type topLeftCorner() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type topLeftCorner() const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), 0, 0); } -/** \returns an expression of a top-left corner of *this. - * - * \tparam CRows number of rows in corner as specified at compile-time - * \tparam CCols number of columns in corner as specified at compile-time - * \param cRows number of rows in corner as specified at run-time - * \param cCols number of columns in corner as specified at run-time - * - * This function is mainly useful for corners where the number of rows is specified at compile-time - * and the number of columns is specified at run-time, or vice versa. The compile-time and run-time - * information should not contradict. In other words, \a cRows should equal \a CRows unless - * \a CRows is \a Dynamic, and the same for the number of columns. - * - * Example: \include MatrixBase_template_int_int_topLeftCorner_int_int.cpp - * Output: \verbinclude MatrixBase_template_int_int_topLeftCorner_int_int.out - * - * \sa class Block - */ +/// \returns an expression of a top-left corner of \c *this. +/// +/// \tparam CRows number of rows in corner as specified at compile-time +/// \tparam CCols number of columns in corner as specified at compile-time +/// \param cRows number of rows in corner as specified at run-time +/// \param cCols number of columns in corner as specified at run-time +/// +/// This function is mainly useful for corners where the number of rows is specified at compile-time +/// and the number of columns is specified at run-time, or vice versa. The compile-time and run-time +/// information should not contradict. In other words, \a cRows should equal \a CRows unless +/// \a CRows is \a Dynamic, and the same for the number of columns. +/// +/// Example: \include MatrixBase_template_int_int_topLeftCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_template_int_int_topLeftCorner_int_int.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block +/// template<int CRows, int CCols> -inline typename FixedBlockXpr<CRows,CCols>::Type topLeftCorner(Index cRows, Index cCols) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type topLeftCorner(Index cRows, Index cCols) { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), 0, 0, cRows, cCols); } -/** This is the const version of topLeftCorner<int, int>(Index, Index).*/ +/// This is the const version of topLeftCorner<int, int>(Index, Index). template<int CRows, int CCols> -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type topLeftCorner(Index cRows, Index cCols) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type topLeftCorner(Index cRows, Index cCols) const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), 0, 0, cRows, cCols); } -/** \returns a dynamic-size expression of a bottom-right corner of *this. - * - * \param cRows the number of rows in the corner - * \param cCols the number of columns in the corner - * - * Example: \include MatrixBase_bottomRightCorner_int_int.cpp - * Output: \verbinclude MatrixBase_bottomRightCorner_int_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline BlockXpr bottomRightCorner(Index cRows, Index cCols) +/// \returns an expression of a bottom-right corner of \c *this with either dynamic or fixed sizes. +/// +/// \param cRows the number of rows in the corner +/// \param cCols the number of columns in the corner +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include MatrixBase_bottomRightCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_bottomRightCorner_int_int.out +/// +/// The number of rows \a blockRows and columns \a blockCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename FixedBlockXpr<...,...>::Type +#endif +bottomRightCorner(NRowsType cRows, NColsType cCols) { - return BlockXpr(derived(), rows() - cRows, cols() - cCols, cRows, cCols); + return typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(cRows), cols() - internal::get_runtime_value(cCols), + internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** This is the const version of bottomRightCorner(Index, Index).*/ -EIGEN_DEVICE_FUNC -inline const ConstBlockXpr bottomRightCorner(Index cRows, Index cCols) const +/// This is the const version of bottomRightCorner(NRowsType, NColsType). +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstFixedBlockXpr<...,...>::Type +#endif +bottomRightCorner(NRowsType cRows, NColsType cCols) const { - return ConstBlockXpr(derived(), rows() - cRows, cols() - cCols, cRows, cCols); + return typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(cRows), cols() - internal::get_runtime_value(cCols), + internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** \returns an expression of a fixed-size bottom-right corner of *this. - * - * The template parameters CRows and CCols are the number of rows and columns in the corner. - * - * Example: \include MatrixBase_template_int_int_bottomRightCorner.cpp - * Output: \verbinclude MatrixBase_template_int_int_bottomRightCorner.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns an expression of a fixed-size bottom-right corner of \c *this. +/// +/// The template parameters CRows and CCols are the number of rows and columns in the corner. +/// +/// Example: \include MatrixBase_template_int_int_bottomRightCorner.cpp +/// Output: \verbinclude MatrixBase_template_int_int_bottomRightCorner.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline typename FixedBlockXpr<CRows,CCols>::Type bottomRightCorner() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type bottomRightCorner() { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), rows() - CRows, cols() - CCols); } -/** This is the const version of bottomRightCorner<int, int>().*/ +/// This is the const version of bottomRightCorner<int, int>(). template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomRightCorner() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomRightCorner() const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), rows() - CRows, cols() - CCols); } -/** \returns an expression of a bottom-right corner of *this. - * - * \tparam CRows number of rows in corner as specified at compile-time - * \tparam CCols number of columns in corner as specified at compile-time - * \param cRows number of rows in corner as specified at run-time - * \param cCols number of columns in corner as specified at run-time - * - * This function is mainly useful for corners where the number of rows is specified at compile-time - * and the number of columns is specified at run-time, or vice versa. The compile-time and run-time - * information should not contradict. In other words, \a cRows should equal \a CRows unless - * \a CRows is \a Dynamic, and the same for the number of columns. - * - * Example: \include MatrixBase_template_int_int_bottomRightCorner_int_int.cpp - * Output: \verbinclude MatrixBase_template_int_int_bottomRightCorner_int_int.out - * - * \sa class Block - */ +/// \returns an expression of a bottom-right corner of \c *this. +/// +/// \tparam CRows number of rows in corner as specified at compile-time +/// \tparam CCols number of columns in corner as specified at compile-time +/// \param cRows number of rows in corner as specified at run-time +/// \param cCols number of columns in corner as specified at run-time +/// +/// This function is mainly useful for corners where the number of rows is specified at compile-time +/// and the number of columns is specified at run-time, or vice versa. The compile-time and run-time +/// information should not contradict. In other words, \a cRows should equal \a CRows unless +/// \a CRows is \a Dynamic, and the same for the number of columns. +/// +/// Example: \include MatrixBase_template_int_int_bottomRightCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_template_int_int_bottomRightCorner_int_int.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block +/// template<int CRows, int CCols> -inline typename FixedBlockXpr<CRows,CCols>::Type bottomRightCorner(Index cRows, Index cCols) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type bottomRightCorner(Index cRows, Index cCols) { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), rows() - cRows, cols() - cCols, cRows, cCols); } -/** This is the const version of bottomRightCorner<int, int>(Index, Index).*/ +/// This is the const version of bottomRightCorner<int, int>(Index, Index). template<int CRows, int CCols> -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomRightCorner(Index cRows, Index cCols) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomRightCorner(Index cRows, Index cCols) const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), rows() - cRows, cols() - cCols, cRows, cCols); } -/** \returns a dynamic-size expression of a bottom-left corner of *this. - * - * \param cRows the number of rows in the corner - * \param cCols the number of columns in the corner - * - * Example: \include MatrixBase_bottomLeftCorner_int_int.cpp - * Output: \verbinclude MatrixBase_bottomLeftCorner_int_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline BlockXpr bottomLeftCorner(Index cRows, Index cCols) +/// \returns an expression of a bottom-left corner of \c *this with either dynamic or fixed sizes. +/// +/// \param cRows the number of rows in the corner +/// \param cCols the number of columns in the corner +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include MatrixBase_bottomLeftCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_bottomLeftCorner_int_int.out +/// +/// The number of rows \a blockRows and columns \a blockCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename FixedBlockXpr<...,...>::Type +#endif +bottomLeftCorner(NRowsType cRows, NColsType cCols) { - return BlockXpr(derived(), rows() - cRows, 0, cRows, cCols); + return typename FixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(cRows), 0, + internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** This is the const version of bottomLeftCorner(Index, Index).*/ -EIGEN_DEVICE_FUNC -inline const ConstBlockXpr bottomLeftCorner(Index cRows, Index cCols) const +/// This is the const version of bottomLeftCorner(NRowsType, NColsType). +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type +#else +typename ConstFixedBlockXpr<...,...>::Type +#endif +bottomLeftCorner(NRowsType cRows, NColsType cCols) const { - return ConstBlockXpr(derived(), rows() - cRows, 0, cRows, cCols); + return typename ConstFixedBlockXpr<internal::get_fixed_value<NRowsType>::value,internal::get_fixed_value<NColsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(cRows), 0, + internal::get_runtime_value(cRows), internal::get_runtime_value(cCols)); } -/** \returns an expression of a fixed-size bottom-left corner of *this. - * - * The template parameters CRows and CCols are the number of rows and columns in the corner. - * - * Example: \include MatrixBase_template_int_int_bottomLeftCorner.cpp - * Output: \verbinclude MatrixBase_template_int_int_bottomLeftCorner.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns an expression of a fixed-size bottom-left corner of \c *this. +/// +/// The template parameters CRows and CCols are the number of rows and columns in the corner. +/// +/// Example: \include MatrixBase_template_int_int_bottomLeftCorner.cpp +/// Output: \verbinclude MatrixBase_template_int_int_bottomLeftCorner.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline typename FixedBlockXpr<CRows,CCols>::Type bottomLeftCorner() +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type bottomLeftCorner() { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), rows() - CRows, 0); } -/** This is the const version of bottomLeftCorner<int, int>().*/ +/// This is the const version of bottomLeftCorner<int, int>(). template<int CRows, int CCols> -EIGEN_DEVICE_FUNC -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomLeftCorner() const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomLeftCorner() const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), rows() - CRows, 0); } -/** \returns an expression of a bottom-left corner of *this. - * - * \tparam CRows number of rows in corner as specified at compile-time - * \tparam CCols number of columns in corner as specified at compile-time - * \param cRows number of rows in corner as specified at run-time - * \param cCols number of columns in corner as specified at run-time - * - * This function is mainly useful for corners where the number of rows is specified at compile-time - * and the number of columns is specified at run-time, or vice versa. The compile-time and run-time - * information should not contradict. In other words, \a cRows should equal \a CRows unless - * \a CRows is \a Dynamic, and the same for the number of columns. - * - * Example: \include MatrixBase_template_int_int_bottomLeftCorner_int_int.cpp - * Output: \verbinclude MatrixBase_template_int_int_bottomLeftCorner_int_int.out - * - * \sa class Block - */ +/// \returns an expression of a bottom-left corner of \c *this. +/// +/// \tparam CRows number of rows in corner as specified at compile-time +/// \tparam CCols number of columns in corner as specified at compile-time +/// \param cRows number of rows in corner as specified at run-time +/// \param cCols number of columns in corner as specified at run-time +/// +/// This function is mainly useful for corners where the number of rows is specified at compile-time +/// and the number of columns is specified at run-time, or vice versa. The compile-time and run-time +/// information should not contradict. In other words, \a cRows should equal \a CRows unless +/// \a CRows is \a Dynamic, and the same for the number of columns. +/// +/// Example: \include MatrixBase_template_int_int_bottomLeftCorner_int_int.cpp +/// Output: \verbinclude MatrixBase_template_int_int_bottomLeftCorner_int_int.out +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa class Block +/// template<int CRows, int CCols> -inline typename FixedBlockXpr<CRows,CCols>::Type bottomLeftCorner(Index cRows, Index cCols) +EIGEN_STRONG_INLINE +typename FixedBlockXpr<CRows,CCols>::Type bottomLeftCorner(Index cRows, Index cCols) { return typename FixedBlockXpr<CRows,CCols>::Type(derived(), rows() - cRows, 0, cRows, cCols); } -/** This is the const version of bottomLeftCorner<int, int>(Index, Index).*/ +/// This is the const version of bottomLeftCorner<int, int>(Index, Index). template<int CRows, int CCols> -inline const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomLeftCorner(Index cRows, Index cCols) const +EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<CRows,CCols>::Type bottomLeftCorner(Index cRows, Index cCols) const { return typename ConstFixedBlockXpr<CRows,CCols>::Type(derived(), rows() - cRows, 0, cRows, cCols); } -/** \returns a block consisting of the top rows of *this. - * - * \param n the number of rows in the block - * - * Example: \include MatrixBase_topRows_int.cpp - * Output: \verbinclude MatrixBase_topRows_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline RowsBlockXpr topRows(Index n) +/// \returns a block consisting of the top rows of \c *this. +/// +/// \param n the number of rows in the block +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// +/// Example: \include MatrixBase_topRows_int.cpp +/// Output: \verbinclude MatrixBase_topRows_int.out +/// +/// The number of rows \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +typename NRowsBlockXpr<...>::Type +#endif +topRows(NRowsType n) { - return RowsBlockXpr(derived(), 0, 0, n, cols()); + return typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), 0, 0, internal::get_runtime_value(n), cols()); } -/** This is the const version of topRows(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstRowsBlockXpr topRows(Index n) const +/// This is the const version of topRows(NRowsType). +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +const typename ConstNRowsBlockXpr<...>::Type +#endif +topRows(NRowsType n) const { - return ConstRowsBlockXpr(derived(), 0, 0, n, cols()); + return typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), 0, 0, internal::get_runtime_value(n), cols()); } -/** \returns a block consisting of the top rows of *this. - * - * \tparam N the number of rows in the block as specified at compile-time - * \param n the number of rows in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_topRows.cpp - * Output: \verbinclude MatrixBase_template_int_topRows.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of the top rows of \c *this. +/// +/// \tparam N the number of rows in the block as specified at compile-time +/// \param n the number of rows in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_topRows.cpp +/// Output: \verbinclude MatrixBase_template_int_topRows.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NRowsBlockXpr<N>::Type topRows(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NRowsBlockXpr<N>::Type topRows(Index n = N) { return typename NRowsBlockXpr<N>::Type(derived(), 0, 0, n, cols()); } -/** This is the const version of topRows<int>().*/ +/// This is the const version of topRows<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNRowsBlockXpr<N>::Type topRows(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNRowsBlockXpr<N>::Type topRows(Index n = N) const { return typename ConstNRowsBlockXpr<N>::Type(derived(), 0, 0, n, cols()); } -/** \returns a block consisting of the bottom rows of *this. - * - * \param n the number of rows in the block - * - * Example: \include MatrixBase_bottomRows_int.cpp - * Output: \verbinclude MatrixBase_bottomRows_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline RowsBlockXpr bottomRows(Index n) +/// \returns a block consisting of the bottom rows of \c *this. +/// +/// \param n the number of rows in the block +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// +/// Example: \include MatrixBase_bottomRows_int.cpp +/// Output: \verbinclude MatrixBase_bottomRows_int.out +/// +/// The number of rows \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +typename NRowsBlockXpr<...>::Type +#endif +bottomRows(NRowsType n) { - return RowsBlockXpr(derived(), rows() - n, 0, n, cols()); + return typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(n), 0, internal::get_runtime_value(n), cols()); } -/** This is the const version of bottomRows(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstRowsBlockXpr bottomRows(Index n) const +/// This is the const version of bottomRows(NRowsType). +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +const typename ConstNRowsBlockXpr<...>::Type +#endif +bottomRows(NRowsType n) const { - return ConstRowsBlockXpr(derived(), rows() - n, 0, n, cols()); + return typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), rows() - internal::get_runtime_value(n), 0, internal::get_runtime_value(n), cols()); } -/** \returns a block consisting of the bottom rows of *this. - * - * \tparam N the number of rows in the block as specified at compile-time - * \param n the number of rows in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_bottomRows.cpp - * Output: \verbinclude MatrixBase_template_int_bottomRows.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of the bottom rows of \c *this. +/// +/// \tparam N the number of rows in the block as specified at compile-time +/// \param n the number of rows in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_bottomRows.cpp +/// Output: \verbinclude MatrixBase_template_int_bottomRows.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NRowsBlockXpr<N>::Type bottomRows(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NRowsBlockXpr<N>::Type bottomRows(Index n = N) { return typename NRowsBlockXpr<N>::Type(derived(), rows() - n, 0, n, cols()); } -/** This is the const version of bottomRows<int>().*/ +/// This is the const version of bottomRows<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNRowsBlockXpr<N>::Type bottomRows(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNRowsBlockXpr<N>::Type bottomRows(Index n = N) const { return typename ConstNRowsBlockXpr<N>::Type(derived(), rows() - n, 0, n, cols()); } -/** \returns a block consisting of a range of rows of *this. - * - * \param startRow the index of the first row in the block - * \param n the number of rows in the block - * - * Example: \include DenseBase_middleRows_int.cpp - * Output: \verbinclude DenseBase_middleRows_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline RowsBlockXpr middleRows(Index startRow, Index n) +/// \returns a block consisting of a range of rows of \c *this. +/// +/// \param startRow the index of the first row in the block +/// \param n the number of rows in the block +/// \tparam NRowsType the type of the value handling the number of rows in the block, typically Index. +/// +/// Example: \include DenseBase_middleRows_int.cpp +/// Output: \verbinclude DenseBase_middleRows_int.out +/// +/// The number of rows \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +typename NRowsBlockXpr<...>::Type +#endif +middleRows(Index startRow, NRowsType n) { - return RowsBlockXpr(derived(), startRow, 0, n, cols()); + return typename NRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), startRow, 0, internal::get_runtime_value(n), cols()); } -/** This is the const version of middleRows(Index,Index).*/ -EIGEN_DEVICE_FUNC -inline ConstRowsBlockXpr middleRows(Index startRow, Index n) const +/// This is the const version of middleRows(Index,NRowsType). +template<typename NRowsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type +#else +const typename ConstNRowsBlockXpr<...>::Type +#endif +middleRows(Index startRow, NRowsType n) const { - return ConstRowsBlockXpr(derived(), startRow, 0, n, cols()); + return typename ConstNRowsBlockXpr<internal::get_fixed_value<NRowsType>::value>::Type + (derived(), startRow, 0, internal::get_runtime_value(n), cols()); } -/** \returns a block consisting of a range of rows of *this. - * - * \tparam N the number of rows in the block as specified at compile-time - * \param startRow the index of the first row in the block - * \param n the number of rows in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include DenseBase_template_int_middleRows.cpp - * Output: \verbinclude DenseBase_template_int_middleRows.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of a range of rows of \c *this. +/// +/// \tparam N the number of rows in the block as specified at compile-time +/// \param startRow the index of the first row in the block +/// \param n the number of rows in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include DenseBase_template_int_middleRows.cpp +/// Output: \verbinclude DenseBase_template_int_middleRows.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NRowsBlockXpr<N>::Type middleRows(Index startRow, Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NRowsBlockXpr<N>::Type middleRows(Index startRow, Index n = N) { return typename NRowsBlockXpr<N>::Type(derived(), startRow, 0, n, cols()); } -/** This is the const version of middleRows<int>().*/ +/// This is the const version of middleRows<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNRowsBlockXpr<N>::Type middleRows(Index startRow, Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNRowsBlockXpr<N>::Type middleRows(Index startRow, Index n = N) const { return typename ConstNRowsBlockXpr<N>::Type(derived(), startRow, 0, n, cols()); } -/** \returns a block consisting of the left columns of *this. - * - * \param n the number of columns in the block - * - * Example: \include MatrixBase_leftCols_int.cpp - * Output: \verbinclude MatrixBase_leftCols_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline ColsBlockXpr leftCols(Index n) +/// \returns a block consisting of the left columns of \c *this. +/// +/// \param n the number of columns in the block +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include MatrixBase_leftCols_int.cpp +/// Output: \verbinclude MatrixBase_leftCols_int.out +/// +/// The number of columns \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +typename NColsBlockXpr<...>::Type +#endif +leftCols(NColsType n) { - return ColsBlockXpr(derived(), 0, 0, rows(), n); + return typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, 0, rows(), internal::get_runtime_value(n)); } -/** This is the const version of leftCols(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstColsBlockXpr leftCols(Index n) const +/// This is the const version of leftCols(NColsType). +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstNColsBlockXpr<...>::Type +#endif +leftCols(NColsType n) const { - return ConstColsBlockXpr(derived(), 0, 0, rows(), n); + return typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, 0, rows(), internal::get_runtime_value(n)); } -/** \returns a block consisting of the left columns of *this. - * - * \tparam N the number of columns in the block as specified at compile-time - * \param n the number of columns in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_leftCols.cpp - * Output: \verbinclude MatrixBase_template_int_leftCols.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of the left columns of \c *this. +/// +/// \tparam N the number of columns in the block as specified at compile-time +/// \param n the number of columns in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_leftCols.cpp +/// Output: \verbinclude MatrixBase_template_int_leftCols.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NColsBlockXpr<N>::Type leftCols(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NColsBlockXpr<N>::Type leftCols(Index n = N) { return typename NColsBlockXpr<N>::Type(derived(), 0, 0, rows(), n); } -/** This is the const version of leftCols<int>().*/ +/// This is the const version of leftCols<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNColsBlockXpr<N>::Type leftCols(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNColsBlockXpr<N>::Type leftCols(Index n = N) const { return typename ConstNColsBlockXpr<N>::Type(derived(), 0, 0, rows(), n); } -/** \returns a block consisting of the right columns of *this. - * - * \param n the number of columns in the block - * - * Example: \include MatrixBase_rightCols_int.cpp - * Output: \verbinclude MatrixBase_rightCols_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline ColsBlockXpr rightCols(Index n) +/// \returns a block consisting of the right columns of \c *this. +/// +/// \param n the number of columns in the block +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include MatrixBase_rightCols_int.cpp +/// Output: \verbinclude MatrixBase_rightCols_int.out +/// +/// The number of columns \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +typename NColsBlockXpr<...>::Type +#endif +rightCols(NColsType n) { - return ColsBlockXpr(derived(), 0, cols() - n, rows(), n); + return typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, cols() - internal::get_runtime_value(n), rows(), internal::get_runtime_value(n)); } -/** This is the const version of rightCols(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstColsBlockXpr rightCols(Index n) const +/// This is the const version of rightCols(NColsType). +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstNColsBlockXpr<...>::Type +#endif +rightCols(NColsType n) const { - return ConstColsBlockXpr(derived(), 0, cols() - n, rows(), n); + return typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, cols() - internal::get_runtime_value(n), rows(), internal::get_runtime_value(n)); } -/** \returns a block consisting of the right columns of *this. - * - * \tparam N the number of columns in the block as specified at compile-time - * \param n the number of columns in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_rightCols.cpp - * Output: \verbinclude MatrixBase_template_int_rightCols.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of the right columns of \c *this. +/// +/// \tparam N the number of columns in the block as specified at compile-time +/// \param n the number of columns in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_rightCols.cpp +/// Output: \verbinclude MatrixBase_template_int_rightCols.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NColsBlockXpr<N>::Type rightCols(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NColsBlockXpr<N>::Type rightCols(Index n = N) { return typename NColsBlockXpr<N>::Type(derived(), 0, cols() - n, rows(), n); } -/** This is the const version of rightCols<int>().*/ +/// This is the const version of rightCols<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNColsBlockXpr<N>::Type rightCols(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNColsBlockXpr<N>::Type rightCols(Index n = N) const { return typename ConstNColsBlockXpr<N>::Type(derived(), 0, cols() - n, rows(), n); } -/** \returns a block consisting of a range of columns of *this. - * - * \param startCol the index of the first column in the block - * \param numCols the number of columns in the block - * - * Example: \include DenseBase_middleCols_int.cpp - * Output: \verbinclude DenseBase_middleCols_int.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ -EIGEN_DEVICE_FUNC -inline ColsBlockXpr middleCols(Index startCol, Index numCols) +/// \returns a block consisting of a range of columns of \c *this. +/// +/// \param startCol the index of the first column in the block +/// \param numCols the number of columns in the block +/// \tparam NColsType the type of the value handling the number of columns in the block, typically Index. +/// +/// Example: \include DenseBase_middleCols_int.cpp +/// Output: \verbinclude DenseBase_middleCols_int.out +/// +/// The number of columns \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +typename NColsBlockXpr<...>::Type +#endif +middleCols(Index startCol, NColsType numCols) { - return ColsBlockXpr(derived(), 0, startCol, rows(), numCols); + return typename NColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, startCol, rows(), internal::get_runtime_value(numCols)); } -/** This is the const version of middleCols(Index,Index).*/ -EIGEN_DEVICE_FUNC -inline ConstColsBlockXpr middleCols(Index startCol, Index numCols) const +/// This is the const version of middleCols(Index,NColsType). +template<typename NColsType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type +#else +const typename ConstNColsBlockXpr<...>::Type +#endif +middleCols(Index startCol, NColsType numCols) const { - return ConstColsBlockXpr(derived(), 0, startCol, rows(), numCols); + return typename ConstNColsBlockXpr<internal::get_fixed_value<NColsType>::value>::Type + (derived(), 0, startCol, rows(), internal::get_runtime_value(numCols)); } -/** \returns a block consisting of a range of columns of *this. - * - * \tparam N the number of columns in the block as specified at compile-time - * \param startCol the index of the first column in the block - * \param n the number of columns in the block as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include DenseBase_template_int_middleCols.cpp - * Output: \verbinclude DenseBase_template_int_middleCols.out - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a block consisting of a range of columns of \c *this. +/// +/// \tparam N the number of columns in the block as specified at compile-time +/// \param startCol the index of the first column in the block +/// \param n the number of columns in the block as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include DenseBase_template_int_middleCols.cpp +/// Output: \verbinclude DenseBase_template_int_middleCols.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename NColsBlockXpr<N>::Type middleCols(Index startCol, Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename NColsBlockXpr<N>::Type middleCols(Index startCol, Index n = N) { return typename NColsBlockXpr<N>::Type(derived(), 0, startCol, rows(), n); } -/** This is the const version of middleCols<int>().*/ +/// This is the const version of middleCols<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstNColsBlockXpr<N>::Type middleCols(Index startCol, Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstNColsBlockXpr<N>::Type middleCols(Index startCol, Index n = N) const { return typename ConstNColsBlockXpr<N>::Type(derived(), 0, startCol, rows(), n); } -/** \returns a fixed-size expression of a block in *this. - * - * The template parameters \a NRows and \a NCols are the number of - * rows and columns in the block. - * - * \param startRow the first row in the block - * \param startCol the first column in the block - * - * Example: \include MatrixBase_block_int_int.cpp - * Output: \verbinclude MatrixBase_block_int_int.out - * - * \note since block is a templated member, the keyword template has to be used - * if the matrix type is also a template parameter: \code m.template block<3,3>(1,1); \endcode - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns a fixed-size expression of a block of \c *this. +/// +/// The template parameters \a NRows and \a NCols are the number of +/// rows and columns in the block. +/// +/// \param startRow the first row in the block +/// \param startCol the first column in the block +/// +/// Example: \include MatrixBase_block_int_int.cpp +/// Output: \verbinclude MatrixBase_block_int_int.out +/// +/// \note The usage of of this overload is discouraged from %Eigen 3.4, better used the generic +/// block(Index,Index,NRowsType,NColsType), here is the one-to-one equivalence: +/// \code +/// mat.template block<NRows,NCols>(i,j) <--> mat.block(i,j,fix<NRows>,fix<NCols>) +/// \endcode +/// +/// \note since block is a templated member, the keyword template has to be used +/// if the matrix type is also a template parameter: \code m.template block<3,3>(1,1); \endcode +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int NRows, int NCols> -EIGEN_DEVICE_FUNC -inline typename FixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol) { return typename FixedBlockXpr<NRows,NCols>::Type(derived(), startRow, startCol); } -/** This is the const version of block<>(Index, Index). */ +/// This is the const version of block<>(Index, Index). */ template<int NRows, int NCols> -EIGEN_DEVICE_FUNC -inline const typename ConstFixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol) const { return typename ConstFixedBlockXpr<NRows,NCols>::Type(derived(), startRow, startCol); } -/** \returns an expression of a block in *this. - * - * \tparam NRows number of rows in block as specified at compile-time - * \tparam NCols number of columns in block as specified at compile-time - * \param startRow the first row in the block - * \param startCol the first column in the block - * \param blockRows number of rows in block as specified at run-time - * \param blockCols number of columns in block as specified at run-time - * - * This function is mainly useful for blocks where the number of rows is specified at compile-time - * and the number of columns is specified at run-time, or vice versa. The compile-time and run-time - * information should not contradict. In other words, \a blockRows should equal \a NRows unless - * \a NRows is \a Dynamic, and the same for the number of columns. - * - * Example: \include MatrixBase_template_int_int_block_int_int_int_int.cpp - * Output: \verbinclude MatrixBase_template_int_int_block_int_int_int_int.cpp - * - * \sa class Block, block(Index,Index,Index,Index) - */ +/// \returns an expression of a block of \c *this. +/// +/// \tparam NRows number of rows in block as specified at compile-time +/// \tparam NCols number of columns in block as specified at compile-time +/// \param startRow the first row in the block +/// \param startCol the first column in the block +/// \param blockRows number of rows in block as specified at run-time +/// \param blockCols number of columns in block as specified at run-time +/// +/// This function is mainly useful for blocks where the number of rows is specified at compile-time +/// and the number of columns is specified at run-time, or vice versa. The compile-time and run-time +/// information should not contradict. In other words, \a blockRows should equal \a NRows unless +/// \a NRows is \a Dynamic, and the same for the number of columns. +/// +/// Example: \include MatrixBase_template_int_int_block_int_int_int_int.cpp +/// Output: \verbinclude MatrixBase_template_int_int_block_int_int_int_int.out +/// +/// \note The usage of of this overload is discouraged from %Eigen 3.4, better used the generic +/// block(Index,Index,NRowsType,NColsType), here is the one-to-one complete equivalence: +/// \code +/// mat.template block<NRows,NCols>(i,j,rows,cols) <--> mat.block(i,j,fix<NRows>(rows),fix<NCols>(cols)) +/// \endcode +/// If we known that, e.g., NRows==Dynamic and NCols!=Dynamic, then the equivalence becomes: +/// \code +/// mat.template block<Dynamic,NCols>(i,j,rows,NCols) <--> mat.block(i,j,rows,fix<NCols>) +/// \endcode +/// +EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL +/// +/// \sa block(Index,Index,NRowsType,NColsType), class Block +/// template<int NRows, int NCols> -inline typename FixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol, +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol, Index blockRows, Index blockCols) { return typename FixedBlockXpr<NRows,NCols>::Type(derived(), startRow, startCol, blockRows, blockCols); } -/** This is the const version of block<>(Index, Index, Index, Index). */ +/// This is the const version of block<>(Index, Index, Index, Index). template<int NRows, int NCols> -inline const typename ConstFixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol, +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const typename ConstFixedBlockXpr<NRows,NCols>::Type block(Index startRow, Index startCol, Index blockRows, Index blockCols) const { return typename ConstFixedBlockXpr<NRows,NCols>::Type(derived(), startRow, startCol, blockRows, blockCols); } -/** \returns an expression of the \a i-th column of *this. Note that the numbering starts at 0. - * - * Example: \include MatrixBase_col.cpp - * Output: \verbinclude MatrixBase_col.out - * +/// \returns an expression of the \a i-th column of \c *this. Note that the numbering starts at 0. +/// +/// Example: \include MatrixBase_col.cpp +/// Output: \verbinclude MatrixBase_col.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(column-major) +/** * \sa row(), class Block */ -EIGEN_DEVICE_FUNC -inline ColXpr col(Index i) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +ColXpr col(Index i) { return ColXpr(derived(), i); } -/** This is the const version of col(). */ -EIGEN_DEVICE_FUNC -inline ConstColXpr col(Index i) const +/// This is the const version of col(). +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +ConstColXpr col(Index i) const { return ConstColXpr(derived(), i); } -/** \returns an expression of the \a i-th row of *this. Note that the numbering starts at 0. - * - * Example: \include MatrixBase_row.cpp - * Output: \verbinclude MatrixBase_row.out - * +/// \returns an expression of the \a i-th row of \c *this. Note that the numbering starts at 0. +/// +/// Example: \include MatrixBase_row.cpp +/// Output: \verbinclude MatrixBase_row.out +/// +EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(row-major) +/** * \sa col(), class Block */ -EIGEN_DEVICE_FUNC -inline RowXpr row(Index i) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +RowXpr row(Index i) { return RowXpr(derived(), i); } -/** This is the const version of row(). */ -EIGEN_DEVICE_FUNC -inline ConstRowXpr row(Index i) const +/// This is the const version of row(). */ +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +ConstRowXpr row(Index i) const { return ConstRowXpr(derived(), i); } -/** \returns a dynamic-size expression of a segment (i.e. a vector block) in *this. - * - * \only_for_vectors - * - * \param start the first coefficient in the segment - * \param n the number of coefficients in the segment - * - * Example: \include MatrixBase_segment_int_int.cpp - * Output: \verbinclude MatrixBase_segment_int_int.out - * - * \note Even though the returned expression has dynamic size, in the case - * when it is applied to a fixed-size vector, it inherits a fixed maximal size, - * which means that evaluating it does not cause a dynamic memory allocation. - * - * \sa class Block, segment(Index) - */ -EIGEN_DEVICE_FUNC -inline SegmentReturnType segment(Index start, Index n) +/// \returns an expression of a segment (i.e. a vector block) in \c *this with either dynamic or fixed sizes. +/// +/// \only_for_vectors +/// +/// \param start the first coefficient in the segment +/// \param n the number of coefficients in the segment +/// \tparam NType the type of the value handling the number of coefficients in the segment, typically Index. +/// +/// Example: \include MatrixBase_segment_int_int.cpp +/// Output: \verbinclude MatrixBase_segment_int_int.out +/// +/// The number of coefficients \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +/// \note Even in the case that the returned expression has dynamic size, in the case +/// when it is applied to a fixed-size vector, it inherits a fixed maximal size, +/// which means that evaluating it does not cause a dynamic memory allocation. +/// +/// \sa block(Index,Index,NRowsType,NColsType), fix<N>, fix<N>(int), class Block +/// +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +typename FixedSegmentReturnType<...>::Type +#endif +segment(Index start, NType n) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return SegmentReturnType(derived(), start, n); + return typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), start, internal::get_runtime_value(n)); } -/** This is the const version of segment(Index,Index).*/ -EIGEN_DEVICE_FUNC -inline ConstSegmentReturnType segment(Index start, Index n) const +/// This is the const version of segment(Index,NType). +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +const typename ConstFixedSegmentReturnType<...>::Type +#endif +segment(Index start, NType n) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return ConstSegmentReturnType(derived(), start, n); + return typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), start, internal::get_runtime_value(n)); } -/** \returns a dynamic-size expression of the first coefficients of *this. - * - * \only_for_vectors - * - * \param n the number of coefficients in the segment - * - * Example: \include MatrixBase_start_int.cpp - * Output: \verbinclude MatrixBase_start_int.out - * - * \note Even though the returned expression has dynamic size, in the case - * when it is applied to a fixed-size vector, it inherits a fixed maximal size, - * which means that evaluating it does not cause a dynamic memory allocation. - * - * \sa class Block, block(Index,Index) - */ -EIGEN_DEVICE_FUNC -inline SegmentReturnType head(Index n) +/// \returns an expression of the first coefficients of \c *this with either dynamic or fixed sizes. +/// +/// \only_for_vectors +/// +/// \param n the number of coefficients in the segment +/// \tparam NType the type of the value handling the number of coefficients in the segment, typically Index. +/// +/// Example: \include MatrixBase_start_int.cpp +/// Output: \verbinclude MatrixBase_start_int.out +/// +/// The number of coefficients \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +/// \note Even in the case that the returned expression has dynamic size, in the case +/// when it is applied to a fixed-size vector, it inherits a fixed maximal size, +/// which means that evaluating it does not cause a dynamic memory allocation. +/// +/// \sa class Block, block(Index,Index) +/// +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +typename FixedSegmentReturnType<...>::Type +#endif +head(NType n) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return SegmentReturnType(derived(), 0, n); + return typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), 0, internal::get_runtime_value(n)); } -/** This is the const version of head(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstSegmentReturnType head(Index n) const +/// This is the const version of head(NType). +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +const typename ConstFixedSegmentReturnType<...>::Type +#endif +head(NType n) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return ConstSegmentReturnType(derived(), 0, n); + return typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), 0, internal::get_runtime_value(n)); } -/** \returns a dynamic-size expression of the last coefficients of *this. - * - * \only_for_vectors - * - * \param n the number of coefficients in the segment - * - * Example: \include MatrixBase_end_int.cpp - * Output: \verbinclude MatrixBase_end_int.out - * - * \note Even though the returned expression has dynamic size, in the case - * when it is applied to a fixed-size vector, it inherits a fixed maximal size, - * which means that evaluating it does not cause a dynamic memory allocation. - * - * \sa class Block, block(Index,Index) - */ -EIGEN_DEVICE_FUNC -inline SegmentReturnType tail(Index n) +/// \returns an expression of a last coefficients of \c *this with either dynamic or fixed sizes. +/// +/// \only_for_vectors +/// +/// \param n the number of coefficients in the segment +/// \tparam NType the type of the value handling the number of coefficients in the segment, typically Index. +/// +/// Example: \include MatrixBase_end_int.cpp +/// Output: \verbinclude MatrixBase_end_int.out +/// +/// The number of coefficients \a n can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. +/// See \link block(Index,Index,NRowsType,NColsType) block() \endlink for the details. +/// +/// \note Even in the case that the returned expression has dynamic size, in the case +/// when it is applied to a fixed-size vector, it inherits a fixed maximal size, +/// which means that evaluating it does not cause a dynamic memory allocation. +/// +/// \sa class Block, block(Index,Index) +/// +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +typename FixedSegmentReturnType<...>::Type +#endif +tail(NType n) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return SegmentReturnType(derived(), this->size() - n, n); + return typename FixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), this->size() - internal::get_runtime_value(n), internal::get_runtime_value(n)); } -/** This is the const version of tail(Index).*/ -EIGEN_DEVICE_FUNC -inline ConstSegmentReturnType tail(Index n) const +/// This is the const version of tail(Index). +template<typename NType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +#ifndef EIGEN_PARSED_BY_DOXYGEN +const typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type +#else +const typename ConstFixedSegmentReturnType<...>::Type +#endif +tail(NType n) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - return ConstSegmentReturnType(derived(), this->size() - n, n); + return typename ConstFixedSegmentReturnType<internal::get_fixed_value<NType>::value>::Type + (derived(), this->size() - internal::get_runtime_value(n), internal::get_runtime_value(n)); } -/** \returns a fixed-size expression of a segment (i.e. a vector block) in \c *this - * - * \only_for_vectors - * - * \tparam N the number of coefficients in the segment as specified at compile-time - * \param start the index of the first element in the segment - * \param n the number of coefficients in the segment as specified at compile-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_segment.cpp - * Output: \verbinclude MatrixBase_template_int_segment.out - * - * \sa class Block - */ +/// \returns a fixed-size expression of a segment (i.e. a vector block) in \c *this +/// +/// \only_for_vectors +/// +/// \tparam N the number of coefficients in the segment as specified at compile-time +/// \param start the index of the first element in the segment +/// \param n the number of coefficients in the segment as specified at compile-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_segment.cpp +/// Output: \verbinclude MatrixBase_template_int_segment.out +/// +/// \sa segment(Index,NType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename FixedSegmentReturnType<N>::Type segment(Index start, Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedSegmentReturnType<N>::Type segment(Index start, Index n = N) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename FixedSegmentReturnType<N>::Type(derived(), start, n); } -/** This is the const version of segment<int>(Index).*/ +/// This is the const version of segment<int>(Index). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstFixedSegmentReturnType<N>::Type segment(Index start, Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstFixedSegmentReturnType<N>::Type segment(Index start, Index n = N) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename ConstFixedSegmentReturnType<N>::Type(derived(), start, n); } -/** \returns a fixed-size expression of the first coefficients of *this. - * - * \only_for_vectors - * - * \tparam N the number of coefficients in the segment as specified at compile-time - * \param n the number of coefficients in the segment as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_start.cpp - * Output: \verbinclude MatrixBase_template_int_start.out - * - * \sa class Block - */ +/// \returns a fixed-size expression of the first coefficients of \c *this. +/// +/// \only_for_vectors +/// +/// \tparam N the number of coefficients in the segment as specified at compile-time +/// \param n the number of coefficients in the segment as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_start.cpp +/// Output: \verbinclude MatrixBase_template_int_start.out +/// +/// \sa head(NType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename FixedSegmentReturnType<N>::Type head(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedSegmentReturnType<N>::Type head(Index n = N) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename FixedSegmentReturnType<N>::Type(derived(), 0, n); } -/** This is the const version of head<int>().*/ +/// This is the const version of head<int>(). template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstFixedSegmentReturnType<N>::Type head(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstFixedSegmentReturnType<N>::Type head(Index n = N) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename ConstFixedSegmentReturnType<N>::Type(derived(), 0, n); } -/** \returns a fixed-size expression of the last coefficients of *this. - * - * \only_for_vectors - * - * \tparam N the number of coefficients in the segment as specified at compile-time - * \param n the number of coefficients in the segment as specified at run-time - * - * The compile-time and run-time information should not contradict. In other words, - * \a n should equal \a N unless \a N is \a Dynamic. - * - * Example: \include MatrixBase_template_int_end.cpp - * Output: \verbinclude MatrixBase_template_int_end.out - * - * \sa class Block - */ +/// \returns a fixed-size expression of the last coefficients of \c *this. +/// +/// \only_for_vectors +/// +/// \tparam N the number of coefficients in the segment as specified at compile-time +/// \param n the number of coefficients in the segment as specified at run-time +/// +/// The compile-time and run-time information should not contradict. In other words, +/// \a n should equal \a N unless \a N is \a Dynamic. +/// +/// Example: \include MatrixBase_template_int_end.cpp +/// Output: \verbinclude MatrixBase_template_int_end.out +/// +/// \sa tail(NType), class Block +/// template<int N> -EIGEN_DEVICE_FUNC -inline typename FixedSegmentReturnType<N>::Type tail(Index n = N) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename FixedSegmentReturnType<N>::Type tail(Index n = N) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename FixedSegmentReturnType<N>::Type(derived(), size() - n); } -/** This is the const version of tail<int>.*/ +/// This is the const version of tail<int>. template<int N> -EIGEN_DEVICE_FUNC -inline typename ConstFixedSegmentReturnType<N>::Type tail(Index n = N) const +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename ConstFixedSegmentReturnType<N>::Type tail(Index n = N) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return typename ConstFixedSegmentReturnType<N>::Type(derived(), size() - n); } + +/// \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this +/// is col-major (resp. row-major). +/// +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +InnerVectorReturnType innerVector(Index outer) +{ return InnerVectorReturnType(derived(), outer); } + +/// \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this +/// is col-major (resp. row-major). Read-only. +/// +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const ConstInnerVectorReturnType innerVector(Index outer) const +{ return ConstInnerVectorReturnType(derived(), outer); } + +/// \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this +/// is col-major (resp. row-major). +/// +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +InnerVectorsReturnType +innerVectors(Index outerStart, Index outerSize) +{ + return Block<Derived,Dynamic,Dynamic,true>(derived(), + IsRowMajor ? outerStart : 0, IsRowMajor ? 0 : outerStart, + IsRowMajor ? outerSize : rows(), IsRowMajor ? cols() : outerSize); + +} + +/// \returns the \a outer -th column (resp. row) of the matrix \c *this if \c *this +/// is col-major (resp. row-major). Read-only. +/// +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +const ConstInnerVectorsReturnType +innerVectors(Index outerStart, Index outerSize) const +{ + return Block<const Derived,Dynamic,Dynamic,true>(derived(), + IsRowMajor ? outerStart : 0, IsRowMajor ? 0 : outerStart, + IsRowMajor ? outerSize : rows(), IsRowMajor ? cols() : outerSize); + +} + +/** \returns the i-th subvector (column or vector) according to the \c Direction + * \sa subVectors() + */ +template<DirectionType Direction> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename internal::conditional<Direction==Vertical,ColXpr,RowXpr>::type +subVector(Index i) +{ + return typename internal::conditional<Direction==Vertical,ColXpr,RowXpr>::type(derived(),i); +} + +/** This is the const version of subVector(Index) */ +template<DirectionType Direction> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +typename internal::conditional<Direction==Vertical,ConstColXpr,ConstRowXpr>::type +subVector(Index i) const +{ + return typename internal::conditional<Direction==Vertical,ConstColXpr,ConstRowXpr>::type(derived(),i); +} + +/** \returns the number of subvectors (rows or columns) in the direction \c Direction + * \sa subVector(Index) + */ +template<DirectionType Direction> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Index subVectors() const +{ return (Direction==Vertical)?cols():rows(); } +
diff --git a/Eigen/src/plugins/CMakeLists.txt b/Eigen/src/plugins/CMakeLists.txt deleted file mode 100644 index 1a1d3ff..0000000 --- a/Eigen/src/plugins/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_plugins_SRCS "*.h") - -INSTALL(FILES - ${Eigen_plugins_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/Eigen/src/plugins COMPONENT Devel - )
diff --git a/Eigen/src/plugins/CommonCwiseBinaryOps.h b/Eigen/src/plugins/CommonCwiseBinaryOps.h index a8fa287..8b6730e 100644 --- a/Eigen/src/plugins/CommonCwiseBinaryOps.h +++ b/Eigen/src/plugins/CommonCwiseBinaryOps.h
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> // // This Source Code Form is subject to the terms of the Mozilla @@ -16,7 +16,7 @@ * * \sa class CwiseBinaryOp, operator-=() */ -EIGEN_MAKE_CWISE_BINARY_OP(operator-,internal::scalar_difference_op) +EIGEN_MAKE_CWISE_BINARY_OP(operator-,difference) /** \returns an expression of the sum of \c *this and \a other * @@ -24,7 +24,7 @@ * * \sa class CwiseBinaryOp, operator+=() */ -EIGEN_MAKE_CWISE_BINARY_OP(operator+,internal::scalar_sum_op) +EIGEN_MAKE_CWISE_BINARY_OP(operator+,sum) /** \returns an expression of a custom coefficient-wise operator \a func of *this and \a other * @@ -45,3 +45,71 @@ return CwiseBinaryOp<CustomBinaryOp, const Derived, const OtherDerived>(derived(), other.derived(), func); } + +#ifndef EIGEN_PARSED_BY_DOXYGEN +EIGEN_MAKE_SCALAR_BINARY_OP(operator*,product) +#else +/** \returns an expression of \c *this scaled by the scalar factor \a scalar + * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + */ +template<typename T> +const CwiseBinaryOp<internal::scalar_product_op<Scalar,T>,Derived,Constant<T> > operator*(const T& scalar) const; +/** \returns an expression of \a expr scaled by the scalar factor \a scalar + * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + */ +template<typename T> friend +const CwiseBinaryOp<internal::scalar_product_op<T,Scalar>,Constant<T>,Derived> operator*(const T& scalar, const StorageBaseType& expr); +#endif + + + +#ifndef EIGEN_PARSED_BY_DOXYGEN +EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(operator/,quotient) +#else +/** \returns an expression of \c *this divided by the scalar value \a scalar + * + * \tparam T is the scalar type of \a scalar. It must be compatible with the scalar type of the given expression. + */ +template<typename T> +const CwiseBinaryOp<internal::scalar_quotient_op<Scalar,T>,Derived,Constant<T> > operator/(const T& scalar) const; +#endif + +/** \returns an expression of the coefficient-wise boolean \b and operator of \c *this and \a other + * + * \warning this operator is for expression of bool only. + * + * Example: \include Cwise_boolean_and.cpp + * Output: \verbinclude Cwise_boolean_and.out + * + * \sa operator||(), select() + */ +template<typename OtherDerived> +EIGEN_DEVICE_FUNC +inline const CwiseBinaryOp<internal::scalar_boolean_and_op, const Derived, const OtherDerived> +operator&&(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const +{ + EIGEN_STATIC_ASSERT((internal::is_same<bool,Scalar>::value && internal::is_same<bool,typename OtherDerived::Scalar>::value), + THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); + return CwiseBinaryOp<internal::scalar_boolean_and_op, const Derived, const OtherDerived>(derived(),other.derived()); +} + +/** \returns an expression of the coefficient-wise boolean \b or operator of \c *this and \a other + * + * \warning this operator is for expression of bool only. + * + * Example: \include Cwise_boolean_or.cpp + * Output: \verbinclude Cwise_boolean_or.out + * + * \sa operator&&(), select() + */ +template<typename OtherDerived> +EIGEN_DEVICE_FUNC +inline const CwiseBinaryOp<internal::scalar_boolean_or_op, const Derived, const OtherDerived> +operator||(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const +{ + EIGEN_STATIC_ASSERT((internal::is_same<bool,Scalar>::value && internal::is_same<bool,typename OtherDerived::Scalar>::value), + THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL); + return CwiseBinaryOp<internal::scalar_boolean_or_op, const Derived, const OtherDerived>(derived(),other.derived()); +}
diff --git a/Eigen/src/plugins/CommonCwiseUnaryOps.h b/Eigen/src/plugins/CommonCwiseUnaryOps.h index 050bce0..5418dc4 100644 --- a/Eigen/src/plugins/CommonCwiseUnaryOps.h +++ b/Eigen/src/plugins/CommonCwiseUnaryOps.h
@@ -12,12 +12,6 @@ #ifndef EIGEN_PARSED_BY_DOXYGEN -/** \internal Represents a scalar multiple of an expression */ -typedef CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Derived> ScalarMultipleReturnType; -typedef CwiseUnaryOp<internal::scalar_multiple2_op<Scalar,std::complex<Scalar> >, const Derived> ScalarComplexMultipleReturnType; - -/** \internal Represents a quotient of an expression by a scalar*/ -typedef CwiseUnaryOp<internal::scalar_quotient1_op<Scalar>, const Derived> ScalarQuotient1ReturnType; /** \internal the return type of conjugate() */ typedef typename internal::conditional<NumTraits<Scalar>::IsComplex, const CwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const Derived>, @@ -39,65 +33,29 @@ typedef CwiseUnaryView<internal::scalar_imag_ref_op<Scalar>, Derived> NonConstImagReturnType; typedef CwiseUnaryOp<internal::scalar_opposite_op<Scalar>, const Derived> NegativeReturnType; -//typedef CwiseUnaryOp<internal::scalar_quotient1_op<Scalar>, const Derived> #endif // not EIGEN_PARSED_BY_DOXYGEN -/** \returns an expression of the opposite of \c *this - */ +/// \returns an expression of the opposite of \c *this +/// +EIGEN_DOC_UNARY_ADDONS(operator-,opposite) +/// EIGEN_DEVICE_FUNC inline const NegativeReturnType operator-() const { return NegativeReturnType(derived()); } -/** \returns an expression of \c *this scaled by the scalar factor \a scalar */ -EIGEN_DEVICE_FUNC -inline const ScalarMultipleReturnType -operator*(const Scalar& scalar) const -{ - return ScalarMultipleReturnType(derived(), internal::scalar_multiple_op<Scalar>(scalar)); -} - -#ifdef EIGEN_PARSED_BY_DOXYGEN -const ScalarMultipleReturnType operator*(const RealScalar& scalar) const; -#endif - -/** \returns an expression of \c *this divided by the scalar value \a scalar */ -EIGEN_DEVICE_FUNC -inline const ScalarQuotient1ReturnType -operator/(const Scalar& scalar) const -{ - return ScalarQuotient1ReturnType(derived(), internal::scalar_quotient1_op<Scalar>(scalar)); -} - -/** Overloaded for efficient real matrix times complex scalar value */ -EIGEN_DEVICE_FUNC -inline const ScalarComplexMultipleReturnType -operator*(const std::complex<Scalar>& scalar) const -{ - return ScalarComplexMultipleReturnType(derived(), internal::scalar_multiple2_op<Scalar,std::complex<Scalar> >(scalar)); -} - -EIGEN_DEVICE_FUNC -inline friend const ScalarMultipleReturnType -operator*(const Scalar& scalar, const StorageBaseType& matrix) -{ return matrix*scalar; } - -EIGEN_DEVICE_FUNC -inline friend const CwiseUnaryOp<internal::scalar_multiple2_op<Scalar,std::complex<Scalar> >, const Derived> -operator*(const std::complex<Scalar>& scalar, const StorageBaseType& matrix) -{ return matrix*scalar; } - - template<class NewType> struct CastXpr { typedef typename internal::cast_return_type<Derived,const CwiseUnaryOp<internal::scalar_cast_op<Scalar, NewType>, const Derived> >::type Type; }; -/** \returns an expression of *this with the \a Scalar type casted to - * \a NewScalar. - * - * The template parameter \a NewScalar is the type we are casting the scalars to. - * - * \sa class CwiseUnaryOp - */ +/// \returns an expression of \c *this with the \a Scalar type casted to +/// \a NewScalar. +/// +/// The template parameter \a NewScalar is the type we are casting the scalars to. +/// +EIGEN_DOC_UNARY_ADDONS(cast,conversion function) +/// +/// \sa class CwiseUnaryOp +/// template<typename NewType> EIGEN_DEVICE_FUNC typename CastXpr<NewType>::Type @@ -106,9 +64,11 @@ return typename CastXpr<NewType>::Type(derived()); } -/** \returns an expression of the complex conjugate of \c *this. - * - * \sa adjoint() */ +/// \returns an expression of the complex conjugate of \c *this. +/// +EIGEN_DOC_UNARY_ADDONS(conjugate,complex conjugate) +/// +/// \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_conj">Math functions</a>, MatrixBase::adjoint() EIGEN_DEVICE_FUNC inline ConjugateReturnType conjugate() const @@ -116,39 +76,59 @@ return ConjugateReturnType(derived()); } -/** \returns a read-only expression of the real part of \c *this. - * - * \sa imag() */ +/// \returns an expression of the complex conjugate of \c *this if Cond==true, returns derived() otherwise. +/// +EIGEN_DOC_UNARY_ADDONS(conjugate,complex conjugate) +/// +/// \sa conjugate() +template<bool Cond> +EIGEN_DEVICE_FUNC +inline typename internal::conditional<Cond,ConjugateReturnType,const Derived&>::type +conjugateIf() const +{ + typedef typename internal::conditional<Cond,ConjugateReturnType,const Derived&>::type ReturnType; + return ReturnType(derived()); +} + +/// \returns a read-only expression of the real part of \c *this. +/// +EIGEN_DOC_UNARY_ADDONS(real,real part function) +/// +/// \sa imag() EIGEN_DEVICE_FUNC inline RealReturnType real() const { return RealReturnType(derived()); } -/** \returns an read-only expression of the imaginary part of \c *this. - * - * \sa real() */ +/// \returns an read-only expression of the imaginary part of \c *this. +/// +EIGEN_DOC_UNARY_ADDONS(imag,imaginary part function) +/// +/// \sa real() EIGEN_DEVICE_FUNC inline const ImagReturnType imag() const { return ImagReturnType(derived()); } -/** \brief Apply a unary operator coefficient-wise - * \param[in] func Functor implementing the unary operator - * \tparam CustomUnaryOp Type of \a func - * \returns An expression of a custom coefficient-wise unary operator \a func of *this - * - * The function \c ptr_fun() from the C++ standard library can be used to make functors out of normal functions. - * - * Example: - * \include class_CwiseUnaryOp_ptrfun.cpp - * Output: \verbinclude class_CwiseUnaryOp_ptrfun.out - * - * Genuine functors allow for more possibilities, for instance it may contain a state. - * - * Example: - * \include class_CwiseUnaryOp.cpp - * Output: \verbinclude class_CwiseUnaryOp.out - * - * \sa class CwiseUnaryOp, class CwiseBinaryOp - */ +/// \brief Apply a unary operator coefficient-wise +/// \param[in] func Functor implementing the unary operator +/// \tparam CustomUnaryOp Type of \a func +/// \returns An expression of a custom coefficient-wise unary operator \a func of *this +/// +/// The function \c ptr_fun() from the C++ standard library can be used to make functors out of normal functions. +/// +/// Example: +/// \include class_CwiseUnaryOp_ptrfun.cpp +/// Output: \verbinclude class_CwiseUnaryOp_ptrfun.out +/// +/// Genuine functors allow for more possibilities, for instance it may contain a state. +/// +/// Example: +/// \include class_CwiseUnaryOp.cpp +/// Output: \verbinclude class_CwiseUnaryOp.out +/// +EIGEN_DOC_UNARY_ADDONS(unaryExpr,unary function) +/// +/// \sa unaryViewExpr, binaryExpr, class CwiseUnaryOp +/// template<typename CustomUnaryOp> EIGEN_DEVICE_FUNC inline const CwiseUnaryOp<CustomUnaryOp, const Derived> @@ -157,17 +137,19 @@ return CwiseUnaryOp<CustomUnaryOp, const Derived>(derived(), func); } -/** \returns an expression of a custom coefficient-wise unary operator \a func of *this - * - * The template parameter \a CustomUnaryOp is the type of the functor - * of the custom unary operator. - * - * Example: - * \include class_CwiseUnaryOp.cpp - * Output: \verbinclude class_CwiseUnaryOp.out - * - * \sa class CwiseUnaryOp, class CwiseBinaryOp - */ +/// \returns an expression of a custom coefficient-wise unary operator \a func of *this +/// +/// The template parameter \a CustomUnaryOp is the type of the functor +/// of the custom unary operator. +/// +/// Example: +/// \include class_CwiseUnaryOp.cpp +/// Output: \verbinclude class_CwiseUnaryOp.out +/// +EIGEN_DOC_UNARY_ADDONS(unaryViewExpr,unary function) +/// +/// \sa unaryExpr, binaryExpr class CwiseUnaryOp +/// template<typename CustomViewOp> EIGEN_DEVICE_FUNC inline const CwiseUnaryView<CustomViewOp, const Derived> @@ -176,16 +158,20 @@ return CwiseUnaryView<CustomViewOp, const Derived>(derived(), func); } -/** \returns a non const expression of the real part of \c *this. - * - * \sa imag() */ +/// \returns a non const expression of the real part of \c *this. +/// +EIGEN_DOC_UNARY_ADDONS(real,real part function) +/// +/// \sa imag() EIGEN_DEVICE_FUNC inline NonConstRealReturnType real() { return NonConstRealReturnType(derived()); } -/** \returns a non const expression of the imaginary part of \c *this. - * - * \sa real() */ +/// \returns a non const expression of the imaginary part of \c *this. +/// +EIGEN_DOC_UNARY_ADDONS(imag,imaginary part function) +/// +/// \sa real() EIGEN_DEVICE_FUNC inline NonConstImagReturnType imag() { return NonConstImagReturnType(derived()); }
diff --git a/Eigen/src/plugins/IndexedViewMethods.h b/Eigen/src/plugins/IndexedViewMethods.h new file mode 100644 index 0000000..5bfb19a --- /dev/null +++ b/Eigen/src/plugins/IndexedViewMethods.h
@@ -0,0 +1,262 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if !defined(EIGEN_PARSED_BY_DOXYGEN) + +// This file is automatically included twice to generate const and non-const versions + +#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS +#define EIGEN_INDEXED_VIEW_METHOD_CONST const +#define EIGEN_INDEXED_VIEW_METHOD_TYPE ConstIndexedViewType +#else +#define EIGEN_INDEXED_VIEW_METHOD_CONST +#define EIGEN_INDEXED_VIEW_METHOD_TYPE IndexedViewType +#endif + +#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS +protected: + +// define some aliases to ease readability + +template<typename Indices> +struct IvcRowType : public internal::IndexedViewCompatibleType<Indices,RowsAtCompileTime> {}; + +template<typename Indices> +struct IvcColType : public internal::IndexedViewCompatibleType<Indices,ColsAtCompileTime> {}; + +template<typename Indices> +struct IvcType : public internal::IndexedViewCompatibleType<Indices,SizeAtCompileTime> {}; + +typedef typename internal::IndexedViewCompatibleType<Index,1>::type IvcIndex; + +template<typename Indices> +typename IvcRowType<Indices>::type +ivcRow(const Indices& indices) const { + return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic<Index,RowsAtCompileTime>(derived().rows()),Specialized); +} + +template<typename Indices> +typename IvcColType<Indices>::type +ivcCol(const Indices& indices) const { + return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic<Index,ColsAtCompileTime>(derived().cols()),Specialized); +} + +template<typename Indices> +typename IvcColType<Indices>::type +ivcSize(const Indices& indices) const { + return internal::makeIndexedViewCompatible(indices, internal::variable_if_dynamic<Index,SizeAtCompileTime>(derived().size()),Specialized); +} + +public: + +#endif + +template<typename RowIndices, typename ColIndices> +struct EIGEN_INDEXED_VIEW_METHOD_TYPE { + typedef IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived, + typename IvcRowType<RowIndices>::type, + typename IvcColType<ColIndices>::type> type; +}; + +// This is the generic version + +template<typename RowIndices, typename ColIndices> +typename internal::enable_if<internal::valid_indexed_view_overload<RowIndices,ColIndices>::value + && internal::traits<typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type>::ReturnAsIndexedView, + typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type >::type +operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type + (derived(), ivcRow(rowIndices), ivcCol(colIndices)); +} + +// The following overload returns a Block<> object + +template<typename RowIndices, typename ColIndices> +typename internal::enable_if<internal::valid_indexed_view_overload<RowIndices,ColIndices>::value + && internal::traits<typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type>::ReturnAsBlock, + typename internal::traits<typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type>::BlockType>::type +operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + typedef typename internal::traits<typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type>::BlockType BlockType; + typename IvcRowType<RowIndices>::type actualRowIndices = ivcRow(rowIndices); + typename IvcColType<ColIndices>::type actualColIndices = ivcCol(colIndices); + return BlockType(derived(), + internal::first(actualRowIndices), + internal::first(actualColIndices), + internal::size(actualRowIndices), + internal::size(actualColIndices)); +} + +// The following overload returns a Scalar + +template<typename RowIndices, typename ColIndices> +typename internal::enable_if<internal::valid_indexed_view_overload<RowIndices,ColIndices>::value + && internal::traits<typename EIGEN_INDEXED_VIEW_METHOD_TYPE<RowIndices,ColIndices>::type>::ReturnAsScalar, + CoeffReturnType >::type +operator()(const RowIndices& rowIndices, const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return Base::operator()(internal::eval_expr_given_size(rowIndices,rows()),internal::eval_expr_given_size(colIndices,cols())); +} + +#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE + +// The following three overloads are needed to handle raw Index[N] arrays. + +template<typename RowIndicesT, std::size_t RowIndicesN, typename ColIndices> +IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const RowIndicesT (&)[RowIndicesN],typename IvcColType<ColIndices>::type> +operator()(const RowIndicesT (&rowIndices)[RowIndicesN], const ColIndices& colIndices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const RowIndicesT (&)[RowIndicesN],typename IvcColType<ColIndices>::type> + (derived(), rowIndices, ivcCol(colIndices)); +} + +template<typename RowIndices, typename ColIndicesT, std::size_t ColIndicesN> +IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,typename IvcRowType<RowIndices>::type, const ColIndicesT (&)[ColIndicesN]> +operator()(const RowIndices& rowIndices, const ColIndicesT (&colIndices)[ColIndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,typename IvcRowType<RowIndices>::type,const ColIndicesT (&)[ColIndicesN]> + (derived(), ivcRow(rowIndices), colIndices); +} + +template<typename RowIndicesT, std::size_t RowIndicesN, typename ColIndicesT, std::size_t ColIndicesN> +IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const RowIndicesT (&)[RowIndicesN], const ColIndicesT (&)[ColIndicesN]> +operator()(const RowIndicesT (&rowIndices)[RowIndicesN], const ColIndicesT (&colIndices)[ColIndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const RowIndicesT (&)[RowIndicesN],const ColIndicesT (&)[ColIndicesN]> + (derived(), rowIndices, colIndices); +} + +#endif // EIGEN_HAS_STATIC_ARRAY_TEMPLATE + +// Overloads for 1D vectors/arrays + +template<typename Indices> +typename internal::enable_if< + IsRowMajor && (!(internal::get_compile_time_incr<typename IvcType<Indices>::type>::value==1 || internal::is_valid_index_type<Indices>::value)), + IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,IvcIndex,typename IvcType<Indices>::type> >::type +operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,IvcIndex,typename IvcType<Indices>::type> + (derived(), IvcIndex(0), ivcCol(indices)); +} + +template<typename Indices> +typename internal::enable_if< + (!IsRowMajor) && (!(internal::get_compile_time_incr<typename IvcType<Indices>::type>::value==1 || internal::is_valid_index_type<Indices>::value)), + IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,typename IvcType<Indices>::type,IvcIndex> >::type +operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,typename IvcType<Indices>::type,IvcIndex> + (derived(), ivcRow(indices), IvcIndex(0)); +} + +template<typename Indices> +typename internal::enable_if< + (internal::get_compile_time_incr<typename IvcType<Indices>::type>::value==1) && (!internal::is_valid_index_type<Indices>::value) && (!symbolic::is_symbolic<Indices>::value), + VectorBlock<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,internal::array_size<Indices>::value> >::type +operator()(const Indices& indices) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + typename IvcType<Indices>::type actualIndices = ivcSize(indices); + return VectorBlock<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,internal::array_size<Indices>::value> + (derived(), internal::first(actualIndices), internal::size(actualIndices)); +} + +template<typename IndexType> +typename internal::enable_if<symbolic::is_symbolic<IndexType>::value, CoeffReturnType >::type +operator()(const IndexType& id) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + return Base::operator()(internal::eval_expr_given_size(id,size())); +} + +#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE + +template<typename IndicesT, std::size_t IndicesN> +typename internal::enable_if<IsRowMajor, + IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,IvcIndex,const IndicesT (&)[IndicesN]> >::type +operator()(const IndicesT (&indices)[IndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,IvcIndex,const IndicesT (&)[IndicesN]> + (derived(), IvcIndex(0), indices); +} + +template<typename IndicesT, std::size_t IndicesN> +typename internal::enable_if<!IsRowMajor, + IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const IndicesT (&)[IndicesN],IvcIndex> >::type +operator()(const IndicesT (&indices)[IndicesN]) EIGEN_INDEXED_VIEW_METHOD_CONST +{ + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + return IndexedView<EIGEN_INDEXED_VIEW_METHOD_CONST Derived,const IndicesT (&)[IndicesN],IvcIndex> + (derived(), indices, IvcIndex(0)); +} + +#endif // EIGEN_HAS_STATIC_ARRAY_TEMPLATE + +#undef EIGEN_INDEXED_VIEW_METHOD_CONST +#undef EIGEN_INDEXED_VIEW_METHOD_TYPE + +#ifndef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS +#define EIGEN_INDEXED_VIEW_METHOD_2ND_PASS +#include "IndexedViewMethods.h" +#undef EIGEN_INDEXED_VIEW_METHOD_2ND_PASS +#endif + +#else // EIGEN_PARSED_BY_DOXYGEN + +/** + * \returns a generic submatrix view defined by the rows and columns indexed \a rowIndices and \a colIndices respectively. + * + * Each parameter must either be: + * - An integer indexing a single row or column + * - Eigen::all indexing the full set of respective rows or columns in increasing order + * - An ArithmeticSequence as returned by the Eigen::seq and Eigen::seqN functions + * - Any %Eigen's vector/array of integers or expressions + * - Plain C arrays: \c int[N] + * - And more generally any type exposing the following two member functions: + * \code + * <integral type> operator[](<integral type>) const; + * <integral type> size() const; + * \endcode + * where \c <integral \c type> stands for any integer type compatible with Eigen::Index (i.e. \c std::ptrdiff_t). + * + * The last statement implies compatibility with \c std::vector, \c std::valarray, \c std::array, many of the Range-v3's ranges, etc. + * + * If the submatrix can be represented using a starting position \c (i,j) and positive sizes \c (rows,columns), then this + * method will returns a Block object after extraction of the relevant information from the passed arguments. This is the case + * when all arguments are either: + * - An integer + * - Eigen::all + * - An ArithmeticSequence with compile-time increment strictly equal to 1, as returned by Eigen::seq(a,b), and Eigen::seqN(a,N). + * + * Otherwise a more general IndexedView<Derived,RowIndices',ColIndices'> object will be returned, after conversion of the inputs + * to more suitable types \c RowIndices' and \c ColIndices'. + * + * For 1D vectors and arrays, you better use the operator()(const Indices&) overload, which behave the same way but taking a single parameter. + * + * See also this <a href="https://stackoverflow.com/questions/46110917/eigen-replicate-items-along-one-dimension-without-useless-allocations">question</a> and its answer for an example of how to duplicate coefficients. + * + * \sa operator()(const Indices&), class Block, class IndexedView, DenseBase::block(Index,Index,Index,Index) + */ +template<typename RowIndices, typename ColIndices> +IndexedView_or_Block +operator()(const RowIndices& rowIndices, const ColIndices& colIndices); + +/** This is an overload of operator()(const RowIndices&, const ColIndices&) for 1D vectors or arrays + * + * \only_for_vectors + */ +template<typename Indices> +IndexedView_or_VectorBlock +operator()(const Indices& indices); + +#endif // EIGEN_PARSED_BY_DOXYGEN
diff --git a/Eigen/src/plugins/MatrixCwiseBinaryOps.h b/Eigen/src/plugins/MatrixCwiseBinaryOps.h index 6dd2e11..f1084ab 100644 --- a/Eigen/src/plugins/MatrixCwiseBinaryOps.h +++ b/Eigen/src/plugins/MatrixCwiseBinaryOps.h
@@ -19,10 +19,10 @@ */ template<typename OtherDerived> EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const EIGEN_CWISE_PRODUCT_RETURN_TYPE(Derived,OtherDerived) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product) cwiseProduct(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const { - return EIGEN_CWISE_PRODUCT_RETURN_TYPE(Derived,OtherDerived)(derived(), other.derived()); + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,product)(derived(), other.derived()); } /** \returns an expression of the coefficient-wise == operator of *this and \a other @@ -74,10 +74,10 @@ */ template<typename OtherDerived> EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar>, const Derived, const OtherDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar,Scalar>, const Derived, const OtherDerived> cwiseMin(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const { - return CwiseBinaryOp<internal::scalar_min_op<Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); + return CwiseBinaryOp<internal::scalar_min_op<Scalar,Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); } /** \returns an expression of the coefficient-wise min of *this and scalar \a other @@ -85,7 +85,7 @@ * \sa class CwiseBinaryOp, min() */ EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar>, const Derived, const ConstantReturnType> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_min_op<Scalar,Scalar>, const Derived, const ConstantReturnType> cwiseMin(const Scalar &other) const { return cwiseMin(Derived::Constant(rows(), cols(), other)); @@ -100,10 +100,10 @@ */ template<typename OtherDerived> EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar>, const Derived, const OtherDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar,Scalar>, const Derived, const OtherDerived> cwiseMax(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const { - return CwiseBinaryOp<internal::scalar_max_op<Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); + return CwiseBinaryOp<internal::scalar_max_op<Scalar,Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); } /** \returns an expression of the coefficient-wise max of *this and scalar \a other @@ -111,7 +111,7 @@ * \sa class CwiseBinaryOp, min() */ EIGEN_DEVICE_FUNC -EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar>, const Derived, const ConstantReturnType> +EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_max_op<Scalar,Scalar>, const Derived, const ConstantReturnType> cwiseMax(const Scalar &other) const { return cwiseMax(Derived::Constant(rows(), cols(), other)); @@ -133,7 +133,7 @@ return CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const Derived, const OtherDerived>(derived(), other.derived()); } -typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar,internal::cmp_EQ>, const Derived, const ConstantReturnType> CwiseScalarEqualReturnType; +typedef CwiseBinaryOp<internal::scalar_cmp_op<Scalar,Scalar,internal::cmp_EQ>, const Derived, const ConstantReturnType> CwiseScalarEqualReturnType; /** \returns an expression of the coefficient-wise == operator of \c *this and a scalar \a s * @@ -148,5 +148,5 @@ inline const CwiseScalarEqualReturnType cwiseEqual(const Scalar& s) const { - return CwiseScalarEqualReturnType(derived(), Derived::Constant(rows(), cols(), s), internal::scalar_cmp_op<Scalar,internal::cmp_EQ>()); + return CwiseScalarEqualReturnType(derived(), Derived::Constant(rows(), cols(), s), internal::scalar_cmp_op<Scalar,Scalar,internal::cmp_EQ>()); }
diff --git a/Eigen/src/plugins/MatrixCwiseUnaryOps.h b/Eigen/src/plugins/MatrixCwiseUnaryOps.h index e16bb37..b1be3d5 100644 --- a/Eigen/src/plugins/MatrixCwiseUnaryOps.h +++ b/Eigen/src/plugins/MatrixCwiseUnaryOps.h
@@ -11,63 +11,75 @@ // This file is included into the body of the base classes supporting matrix specific coefficient-wise functions. // This include MatrixBase and SparseMatrixBase. + typedef CwiseUnaryOp<internal::scalar_abs_op<Scalar>, const Derived> CwiseAbsReturnType; typedef CwiseUnaryOp<internal::scalar_abs2_op<Scalar>, const Derived> CwiseAbs2ReturnType; typedef CwiseUnaryOp<internal::scalar_sqrt_op<Scalar>, const Derived> CwiseSqrtReturnType; typedef CwiseUnaryOp<internal::scalar_sign_op<Scalar>, const Derived> CwiseSignReturnType; typedef CwiseUnaryOp<internal::scalar_inverse_op<Scalar>, const Derived> CwiseInverseReturnType; -/** \returns an expression of the coefficient-wise absolute value of \c *this - * - * Example: \include MatrixBase_cwiseAbs.cpp - * Output: \verbinclude MatrixBase_cwiseAbs.out - * - * \sa cwiseAbs2() - */ +/// \returns an expression of the coefficient-wise absolute value of \c *this +/// +/// Example: \include MatrixBase_cwiseAbs.cpp +/// Output: \verbinclude MatrixBase_cwiseAbs.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseAbs,absolute value) +/// +/// \sa cwiseAbs2() +/// EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseAbsReturnType cwiseAbs() const { return CwiseAbsReturnType(derived()); } -/** \returns an expression of the coefficient-wise squared absolute value of \c *this - * - * Example: \include MatrixBase_cwiseAbs2.cpp - * Output: \verbinclude MatrixBase_cwiseAbs2.out - * - * \sa cwiseAbs() - */ +/// \returns an expression of the coefficient-wise squared absolute value of \c *this +/// +/// Example: \include MatrixBase_cwiseAbs2.cpp +/// Output: \verbinclude MatrixBase_cwiseAbs2.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseAbs2,squared absolute value) +/// +/// \sa cwiseAbs() +/// EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CwiseAbs2ReturnType cwiseAbs2() const { return CwiseAbs2ReturnType(derived()); } -/** \returns an expression of the coefficient-wise square root of *this. - * - * Example: \include MatrixBase_cwiseSqrt.cpp - * Output: \verbinclude MatrixBase_cwiseSqrt.out - * - * \sa cwisePow(), cwiseSquare() - */ +/// \returns an expression of the coefficient-wise square root of *this. +/// +/// Example: \include MatrixBase_cwiseSqrt.cpp +/// Output: \verbinclude MatrixBase_cwiseSqrt.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseSqrt,square-root) +/// +/// \sa cwisePow(), cwiseSquare() +/// EIGEN_DEVICE_FUNC inline const CwiseSqrtReturnType cwiseSqrt() const { return CwiseSqrtReturnType(derived()); } -/** \returns an expression of the coefficient-wise signum of *this. - * - * Example: \include MatrixBase_cwiseSign.cpp - * Output: \verbinclude MatrixBase_cwiseSign.out - * - */ +/// \returns an expression of the coefficient-wise signum of *this. +/// +/// Example: \include MatrixBase_cwiseSign.cpp +/// Output: \verbinclude MatrixBase_cwiseSign.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseSign,sign function) +/// EIGEN_DEVICE_FUNC inline const CwiseSignReturnType cwiseSign() const { return CwiseSignReturnType(derived()); } -/** \returns an expression of the coefficient-wise inverse of *this. - * - * Example: \include MatrixBase_cwiseInverse.cpp - * Output: \verbinclude MatrixBase_cwiseInverse.out - * - * \sa cwiseProduct() - */ +/// \returns an expression of the coefficient-wise inverse of *this. +/// +/// Example: \include MatrixBase_cwiseInverse.cpp +/// Output: \verbinclude MatrixBase_cwiseInverse.out +/// +EIGEN_DOC_UNARY_ADDONS(cwiseInverse,inverse) +/// +/// \sa cwiseProduct() +/// EIGEN_DEVICE_FUNC inline const CwiseInverseReturnType cwiseInverse() const { return CwiseInverseReturnType(derived()); } + +
diff --git a/Eigen/src/plugins/ReshapedMethods.h b/Eigen/src/plugins/ReshapedMethods.h new file mode 100644 index 0000000..482a6b0 --- /dev/null +++ b/Eigen/src/plugins/ReshapedMethods.h
@@ -0,0 +1,149 @@ + +#ifdef EIGEN_PARSED_BY_DOXYGEN + +/// \returns an expression of \c *this with reshaped sizes. +/// +/// \param nRows the number of rows in the reshaped expression, specified at either run-time or compile-time, or AutoSize +/// \param nCols the number of columns in the reshaped expression, specified at either run-time or compile-time, or AutoSize +/// \tparam Order specifies whether the coefficients should be processed in column-major-order (ColMajor), in row-major-order (RowMajor), +/// or follows the \em natural order of the nested expression (AutoOrder). The default is ColMajor. +/// \tparam NRowsType the type of the value handling the number of rows, typically Index. +/// \tparam NColsType the type of the value handling the number of columns, typically Index. +/// +/// Dynamic size example: \include MatrixBase_reshaped_int_int.cpp +/// Output: \verbinclude MatrixBase_reshaped_int_int.out +/// +/// The number of rows \a nRows and columns \a nCols can also be specified at compile-time by passing Eigen::fix<N>, +/// or Eigen::fix<N>(n) as arguments. In the later case, \c n plays the role of a runtime fallback value in case \c N equals Eigen::Dynamic. +/// Here is an example with a fixed number of rows and columns: +/// \include MatrixBase_reshaped_fixed.cpp +/// Output: \verbinclude MatrixBase_reshaped_fixed.out +/// +/// Finally, one of the sizes parameter can be automatically deduced from the other one by passing AutoSize as in the following example: +/// \include MatrixBase_reshaped_auto.cpp +/// Output: \verbinclude MatrixBase_reshaped_auto.out +/// AutoSize does preserve compile-time sizes when possible, i.e., when the sizes of the input are known at compile time \b and +/// that the other size is passed at compile-time using Eigen::fix<N> as above. +/// +/// \sa class Reshaped, fix, fix<N>(int) +/// +template<int Order = ColMajor, typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC +inline Reshaped<Derived,...> +reshaped(NRowsType nRows, NColsType nCols); + +/// This is the const version of reshaped(NRowsType,NColsType). +template<int Order = ColMajor, typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC +inline const Reshaped<const Derived,...> +reshaped(NRowsType nRows, NColsType nCols) const; + +/// \returns an expression of \c *this with columns (or rows) stacked to a linear column vector +/// +/// \tparam Order specifies whether the coefficients should be processed in column-major-order (ColMajor), in row-major-order (RowMajor), +/// or follows the \em natural order of the nested expression (AutoOrder). The default is ColMajor. +/// +/// This overloads is essentially a shortcut for `A.reshaped<Order>(AutoSize,fix<1>)`. +/// +/// - If `Order==ColMajor` (the default), then it returns a column-vector from the stacked columns of \c *this. +/// - If `Order==RowMajor`, then it returns a column-vector from the stacked rows of \c *this. +/// - If `Order==AutoOrder`, then it returns a column-vector with elements stacked following the storage order of \c *this. +/// This mode is the recommended one when the particular ordering of the element is not relevant. +/// +/// Example: +/// \include MatrixBase_reshaped_to_vector.cpp +/// Output: \verbinclude MatrixBase_reshaped_to_vector.out +/// +/// If you want more control, you can still fall back to reshaped(NRowsType,NColsType). +/// +/// \sa reshaped(NRowsType,NColsType), class Reshaped +/// +template<int Order = ColMajor> +EIGEN_DEVICE_FUNC +inline Reshaped<Derived,...> +reshaped(); + +/// This is the const version of reshaped(). +template<int Order = ColMajor> +EIGEN_DEVICE_FUNC +inline const Reshaped<const Derived,...> +reshaped() const; + +#else + +// This file is automatically included twice to generate const and non-const versions + +#ifndef EIGEN_RESHAPED_METHOD_2ND_PASS +#define EIGEN_RESHAPED_METHOD_CONST const +#else +#define EIGEN_RESHAPED_METHOD_CONST +#endif + +#ifndef EIGEN_RESHAPED_METHOD_2ND_PASS + +// This part is included once + +#endif + +template<typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC +inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, + internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value> +reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST +{ + return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, + internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value> + (derived(), + internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()), + internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size())); +} + +template<int Order, typename NRowsType, typename NColsType> +EIGEN_DEVICE_FUNC +inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, + internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_order<Flags,Order>::value> +reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST +{ + return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, + internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value, + internal::get_compiletime_reshape_order<Flags,Order>::value> + (derived(), + internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()), + internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size())); +} + +// Views as linear vectors + +EIGEN_DEVICE_FUNC +inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,SizeAtCompileTime,1> +reshaped() EIGEN_RESHAPED_METHOD_CONST +{ + return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,SizeAtCompileTime,1>(derived(),size(),1); +} + +template<int Order> +EIGEN_DEVICE_FUNC +inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, + internal::get_compiletime_reshape_order<Flags,Order>::value> +reshaped() EIGEN_RESHAPED_METHOD_CONST +{ + EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER); + return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, + internal::get_compiletime_reshape_order<Flags,Order>::value> + (derived(), size(), 1); +} + +#undef EIGEN_RESHAPED_METHOD_CONST + +#ifndef EIGEN_RESHAPED_METHOD_2ND_PASS +#define EIGEN_RESHAPED_METHOD_2ND_PASS +#include "ReshapedMethods.h" +#undef EIGEN_RESHAPED_METHOD_2ND_PASS +#endif + +#endif // EIGEN_PARSED_BY_DOXYGEN
diff --git a/README.md b/README.md index 04fa63f..99c9e29 100644 --- a/README.md +++ b/README.md
@@ -1,3 +1,7 @@ -**Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms.** - -For more information go to http://eigen.tuxfamily.org/. +**Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms.** + +For more information go to http://eigen.tuxfamily.org/. + +For ***pull request*** please only use the official repository at https://bitbucket.org/eigen/eigen. + +For ***bug reports*** and ***feature requests*** go to http://eigen.tuxfamily.org/bz.
diff --git a/bench/BenchTimer.h b/bench/BenchTimer.h index ea28496..8a0dbbe 100644 --- a/bench/BenchTimer.h +++ b/bench/BenchTimer.h
@@ -28,11 +28,15 @@ #endif static void escape(void *p) { +#if EIGEN_COMP_GNUC || EIGEN_COMP_CLANG asm volatile("" : : "g"(p) : "memory"); +#endif } static void clobber() { +#if EIGEN_COMP_GNUC || EIGEN_COMP_CLANG asm volatile("" : : : "memory"); +#endif } #include <Eigen/Core>
diff --git a/bench/analyze-blocking-sizes.cpp b/bench/analyze-blocking-sizes.cpp index d563a1d..6bc4aca 100644 --- a/bench/analyze-blocking-sizes.cpp +++ b/bench/analyze-blocking-sizes.cpp
@@ -825,7 +825,7 @@ } for (int i = 1; i < argc; i++) { bool arg_handled = false; - // Step 1. Try to match action invokation names. + // Step 1. Try to match action invocation names. for (auto it = available_actions.begin(); it != available_actions.end(); ++it) { if (!strcmp(argv[i], (*it)->invokation_name())) { if (!action) {
diff --git a/bench/benchCholesky.cpp b/bench/benchCholesky.cpp index 42b3e12..9a8e7cf 100644 --- a/bench/benchCholesky.cpp +++ b/bench/benchCholesky.cpp
@@ -31,7 +31,7 @@ int rows = m.rows(); int cols = m.cols(); - int cost = 0; + double cost = 0; for (int j=0; j<rows; ++j) { int r = std::max(rows - j -1,0); @@ -78,10 +78,10 @@ else std::cout << "fixed "; std::cout << covMat.rows() << " \t" - << (timerNoSqrt.value() * REPEAT) / repeats << "s " - << "(" << 1e-6 * cost*repeats/timerNoSqrt.value() << " MFLOPS)\t" - << (timerSqrt.value() * REPEAT) / repeats << "s " - << "(" << 1e-6 * cost*repeats/timerSqrt.value() << " MFLOPS)\n"; + << (timerNoSqrt.best()) / repeats << "s " + << "(" << 1e-9 * cost*repeats/timerNoSqrt.best() << " GFLOPS)\t" + << (timerSqrt.best()) / repeats << "s " + << "(" << 1e-9 * cost*repeats/timerSqrt.best() << " GFLOPS)\n"; #ifdef BENCH_GSL @@ -119,13 +119,13 @@ int main(int argc, char* argv[]) { - const int dynsizes[] = {4,6,8,16,24,32,49,64,128,256,512,900,0}; - std::cout << "size no sqrt standard"; + const int dynsizes[] = {4,6,8,16,24,32,49,64,128,256,512,900,1500,0}; + std::cout << "size LDLT LLT"; // #ifdef BENCH_GSL // std::cout << " GSL (standard + double + ATLAS) "; // #endif std::cout << "\n"; - for (uint i=0; dynsizes[i]>0; ++i) + for (int i=0; dynsizes[i]>0; ++i) benchLLT(Matrix<Scalar,Dynamic,Dynamic>(dynsizes[i],dynsizes[i])); benchLLT(Matrix<Scalar,2,2>());
diff --git a/bench/bench_gemm.cpp b/bench/bench_gemm.cpp index 8528c55..78ca1cd 100644 --- a/bench/bench_gemm.cpp +++ b/bench/bench_gemm.cpp
@@ -11,8 +11,9 @@ // #include <iostream> -#include <Eigen/Core> #include <bench/BenchTimer.h> +#include <Eigen/Core> + using namespace std; using namespace Eigen; @@ -30,10 +31,22 @@ #define SCALARB SCALAR #endif +#ifdef ROWMAJ_A +const int opt_A = RowMajor; +#else +const int opt_A = ColMajor; +#endif + +#ifdef ROWMAJ_B +const int opt_B = RowMajor; +#else +const int opt_B = ColMajor; +#endif + typedef SCALAR Scalar; typedef NumTraits<Scalar>::Real RealScalar; -typedef Matrix<SCALARA,Dynamic,Dynamic> A; -typedef Matrix<SCALARB,Dynamic,Dynamic> B; +typedef Matrix<SCALARA,Dynamic,Dynamic,opt_A> A; +typedef Matrix<SCALARB,Dynamic,Dynamic,opt_B> B; typedef Matrix<Scalar,Dynamic,Dynamic> C; typedef Matrix<RealScalar,Dynamic,Dynamic> M; @@ -58,45 +71,61 @@ static char right = 'R'; static int intone = 1; -void blas_gemm(const MatrixXf& a, const MatrixXf& b, MatrixXf& c) +#ifdef ROWMAJ_A +const char transA = trans; +#else +const char transA = notrans; +#endif + +#ifdef ROWMAJ_B +const char transB = trans; +#else +const char transB = notrans; +#endif + +template<typename A,typename B> +void blas_gemm(const A& a, const B& b, MatrixXf& c) { int M = c.rows(); int N = c.cols(); int K = a.cols(); - int lda = a.rows(); int ldb = b.rows(); int ldc = c.rows(); + int lda = a.outerStride(); int ldb = b.outerStride(); int ldc = c.rows(); - sgemm_(¬rans,¬rans,&M,&N,&K,&fone, + sgemm_(&transA,&transB,&M,&N,&K,&fone, const_cast<float*>(a.data()),&lda, const_cast<float*>(b.data()),&ldb,&fone, c.data(),&ldc); } -EIGEN_DONT_INLINE void blas_gemm(const MatrixXd& a, const MatrixXd& b, MatrixXd& c) +template<typename A,typename B> +void blas_gemm(const A& a, const B& b, MatrixXd& c) { int M = c.rows(); int N = c.cols(); int K = a.cols(); - int lda = a.rows(); int ldb = b.rows(); int ldc = c.rows(); + int lda = a.outerStride(); int ldb = b.outerStride(); int ldc = c.rows(); - dgemm_(¬rans,¬rans,&M,&N,&K,&done, + dgemm_(&transA,&transB,&M,&N,&K,&done, const_cast<double*>(a.data()),&lda, const_cast<double*>(b.data()),&ldb,&done, c.data(),&ldc); } -void blas_gemm(const MatrixXcf& a, const MatrixXcf& b, MatrixXcf& c) +template<typename A,typename B> +void blas_gemm(const A& a, const B& b, MatrixXcf& c) { int M = c.rows(); int N = c.cols(); int K = a.cols(); - int lda = a.rows(); int ldb = b.rows(); int ldc = c.rows(); + int lda = a.outerStride(); int ldb = b.outerStride(); int ldc = c.rows(); - cgemm_(¬rans,¬rans,&M,&N,&K,(float*)&cfone, + cgemm_(&transA,&transB,&M,&N,&K,(float*)&cfone, const_cast<float*>((const float*)a.data()),&lda, const_cast<float*>((const float*)b.data()),&ldb,(float*)&cfone, (float*)c.data(),&ldc); } -void blas_gemm(const MatrixXcd& a, const MatrixXcd& b, MatrixXcd& c) +template<typename A,typename B> +void blas_gemm(const A& a, const B& b, MatrixXcd& c) { int M = c.rows(); int N = c.cols(); int K = a.cols(); - int lda = a.rows(); int ldb = b.rows(); int ldc = c.rows(); + int lda = a.outerStride(); int ldb = b.outerStride(); int ldc = c.rows(); - zgemm_(¬rans,¬rans,&M,&N,&K,(double*)&cdone, + zgemm_(&transA,&transB,&M,&N,&K,(double*)&cdone, const_cast<double*>((const double*)a.data()),&lda, const_cast<double*>((const double*)b.data()),&ldb,(double*)&cdone, (double*)c.data(),&ldc); @@ -112,6 +141,7 @@ cr.noalias() -= ai * bi; ci.noalias() += ar * bi; ci.noalias() += ai * br; + // [cr ci] += [ar ai] * br + [-ai ar] * bi } void matlab_real_cplx(const M& a, const M& br, const M& bi, M& cr, M& ci) @@ -126,10 +156,12 @@ ci.noalias() += ai * b; } + + template<typename A, typename B, typename C> EIGEN_DONT_INLINE void gemm(const A& a, const B& b, C& c) { - c.noalias() += a * b; + c.noalias() += a * b; } int main(int argc, char ** argv) @@ -179,8 +211,8 @@ } else if(argv[i][1]=='t') { + tries = atoi(argv[++i]); ++i; - tries = atoi(argv[i++]); } else if(argv[i][1]=='p') { @@ -216,7 +248,7 @@ std::cout << "Matrix sizes = " << m << "x" << p << " * " << p << "x" << n << "\n"; std::ptrdiff_t mc(m), nc(n), kc(p); internal::computeProductBlockingSizes<Scalar,Scalar>(kc, mc, nc); - std::cout << "blocking size (mc x kc) = " << mc << " x " << kc << "\n"; + std::cout << "blocking size (mc x kc) = " << mc << " x " << kc << " x " << nc << "\n"; C r = c; @@ -240,7 +272,7 @@ blas_gemm(a,b,r); c.noalias() += a * b; if(!r.isApprox(c)) { - std::cout << r - c << "\n"; + std::cout << (r - c).norm()/r.norm() << "\n"; std::cerr << "Warning, your product is crap!\n\n"; } #else @@ -249,7 +281,7 @@ gemm(a,b,c); r.noalias() += a.cast<Scalar>() .lazyProduct( b.cast<Scalar>() ); if(!r.isApprox(c)) { - std::cout << r - c << "\n"; + std::cout << (r - c).norm()/r.norm() << "\n"; std::cerr << "Warning, your product is crap!\n\n"; } } @@ -263,6 +295,9 @@ std::cout << "blas real " << tblas.best(REAL_TIMER)/rep << "s \t" << (double(m)*n*p*rep*2/tblas.best(REAL_TIMER))*1e-9 << " GFLOPS \t(" << tblas.total(REAL_TIMER) << "s)\n"; #endif + // warm start + if(b.norm()+a.norm()==123.554) std::cout << "\n"; + BenchTimer tmt; c = rc; BENCH(tmt, tries, rep, gemm(a,b,c)); @@ -285,11 +320,11 @@ if(1.*m*n*p<30*30*30) { - BenchTimer tmt; - c = rc; - BENCH(tmt, tries, rep, c.noalias()+=a.lazyProduct(b)); - std::cout << "lazy cpu " << tmt.best(CPU_TIMER)/rep << "s \t" << (double(m)*n*p*rep*2/tmt.best(CPU_TIMER))*1e-9 << " GFLOPS \t(" << tmt.total(CPU_TIMER) << "s)\n"; - std::cout << "lazy real " << tmt.best(REAL_TIMER)/rep << "s \t" << (double(m)*n*p*rep*2/tmt.best(REAL_TIMER))*1e-9 << " GFLOPS \t(" << tmt.total(REAL_TIMER) << "s)\n"; + BenchTimer tmt; + c = rc; + BENCH(tmt, tries, rep, c.noalias()+=a.lazyProduct(b)); + std::cout << "lazy cpu " << tmt.best(CPU_TIMER)/rep << "s \t" << (double(m)*n*p*rep*2/tmt.best(CPU_TIMER))*1e-9 << " GFLOPS \t(" << tmt.total(CPU_TIMER) << "s)\n"; + std::cout << "lazy real " << tmt.best(REAL_TIMER)/rep << "s \t" << (double(m)*n*p*rep*2/tmt.best(REAL_TIMER))*1e-9 << " GFLOPS \t(" << tmt.total(REAL_TIMER) << "s)\n"; } #ifdef DECOUPLED
diff --git a/bench/bench_norm.cpp b/bench/bench_norm.cpp index 129afcf..a861539 100644 --- a/bench/bench_norm.cpp +++ b/bench/bench_norm.cpp
@@ -134,7 +134,7 @@ iexp = - ((iemax+it)/2); s2m = std::pow(ibeta,iexp); // scaling factor for upper range - overfl = rbig*s2m; // overfow boundary for abig + overfl = rbig*s2m; // overflow boundary for abig eps = std::pow(ibeta, 1-it); relerr = std::sqrt(eps); // tolerance for neglecting asml abig = 1.0/eps - 1.0;
diff --git a/bench/btl/README b/bench/btl/README index f3f5fb3..ebed889 100644 --- a/bench/btl/README +++ b/bench/btl/README
@@ -36,7 +36,7 @@ You can also select a given set of actions defining the environment variable BTL_CONFIG this way: BTL_CONFIG="-a action1{:action2}*" ctest -V -An exemple: +An example: BTL_CONFIG="-a axpy:vector_matrix:trisolve:ata" ctest -V -R eigen2 Finally, if bench results already exist (the bench*.dat files) then they merges by keeping the best for each matrix size. If you want to overwrite the previous ones you can simply add the "--overwrite" option:
diff --git a/bench/btl/actions/basic_actions.hh b/bench/btl/actions/basic_actions.hh index a3333ea..62442f0 100644 --- a/bench/btl/actions/basic_actions.hh +++ b/bench/btl/actions/basic_actions.hh
@@ -6,7 +6,7 @@ #include "action_atv_product.hh" #include "action_matrix_matrix_product.hh" -// #include "action_ata_product.hh" +#include "action_ata_product.hh" #include "action_aat_product.hh" #include "action_trisolve.hh"
diff --git a/bench/btl/generic_bench/bench.hh b/bench/btl/generic_bench/bench.hh index 7b7b951..0732940 100644 --- a/bench/btl/generic_bench/bench.hh +++ b/bench/btl/generic_bench/bench.hh
@@ -159,7 +159,7 @@ // bench<Mixed_Perf_Analyzer,Action>(size_min,size_max,nb_point); - // Only for small problem size. Otherwize it will be too long + // Only for small problem size. Otherwise it will be too long // bench<X86_Perf_Analyzer,Action>(size_min,size_max,nb_point); // bench<STL_Perf_Analyzer,Action>(size_min,size_max,nb_point);
diff --git a/bench/btl/generic_bench/utils/size_log.hh b/bench/btl/generic_bench/utils/size_log.hh index 13a3da7..68945e7 100644 --- a/bench/btl/generic_bench/utils/size_log.hh +++ b/bench/btl/generic_bench/utils/size_log.hh
@@ -23,7 +23,7 @@ #include "math.h" // The Vector class must satisfy the following part of STL vector concept : // resize() method -// [] operator for seting element +// [] operator for setting element // the vector element are int compatible. template<class Vector> void size_log(const int nb_point, const int size_min, const int size_max, Vector & X)
diff --git a/bench/btl/generic_bench/utils/xy_file.hh b/bench/btl/generic_bench/utils/xy_file.hh index 4571bed..0492faf 100644 --- a/bench/btl/generic_bench/utils/xy_file.hh +++ b/bench/btl/generic_bench/utils/xy_file.hh
@@ -55,7 +55,7 @@ // The Vector class must satisfy the following part of STL vector concept : // resize() method -// [] operator for seting element +// [] operator for setting element // the vector element must have the << operator define using namespace std;
diff --git a/bench/btl/libs/BLAS/blas_interface_impl.hh b/bench/btl/libs/BLAS/blas_interface_impl.hh index fc4ba2a..9e0a649 100644 --- a/bench/btl/libs/BLAS/blas_interface_impl.hh +++ b/bench/btl/libs/BLAS/blas_interface_impl.hh
@@ -46,9 +46,9 @@ BLAS_FUNC(gemm)(¬rans,¬rans,&N,&N,&N,&fone,A,&N,B,&N,&fzero,X,&N); } -// static inline void ata_product(gene_matrix & A, gene_matrix & X, int N){ -// ssyrk_(&lower,&trans,&N,&N,&fone,A,&N,&fzero,X,&N); -// } + static inline void ata_product(gene_matrix & A, gene_matrix & X, int N){ + BLAS_FUNC(syrk)(&lower,&trans,&N,&N,&fone,A,&N,&fzero,X,&N); + } static inline void aat_product(gene_matrix & A, gene_matrix & X, int N){ BLAS_FUNC(syrk)(&lower,¬rans,&N,&N,&fone,A,&N,&fzero,X,&N);
diff --git a/bench/btl/libs/BLAS/main.cpp b/bench/btl/libs/BLAS/main.cpp index 564d55e..fd99149 100644 --- a/bench/btl/libs/BLAS/main.cpp +++ b/bench/btl/libs/BLAS/main.cpp
@@ -48,7 +48,7 @@ bench<Action_rot<blas_interface<REAL_TYPE> > >(MIN_AXPY,MAX_AXPY,NB_POINT); bench<Action_matrix_matrix_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); -// bench<Action_ata_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); + bench<Action_ata_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); bench<Action_aat_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); bench<Action_trisolve<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
diff --git a/bench/btl/libs/STL/STL_interface.hh b/bench/btl/libs/STL/STL_interface.hh index ef4cc92..16658c4 100644 --- a/bench/btl/libs/STL/STL_interface.hh +++ b/bench/btl/libs/STL/STL_interface.hh
@@ -78,18 +78,18 @@ cible[i][j]=source[i][j]; } -// static inline void ata_product(const gene_matrix & A, gene_matrix & X, int N) -// { -// real somme; -// for (int j=0;j<N;j++){ -// for (int i=0;i<N;i++){ -// somme=0.0; -// for (int k=0;k<N;k++) -// somme += A[i][k]*A[j][k]; -// X[j][i]=somme; -// } -// } -// } + static inline void ata_product(const gene_matrix & A, gene_matrix & X, int N) + { + real somme; + for (int j=0;j<N;j++){ + for (int i=0;i<N;i++){ + somme=0.0; + for (int k=0;k<N;k++) + somme += A[i][k]*A[j][k]; + X[j][i]=somme; + } + } + } static inline void aat_product(const gene_matrix & A, gene_matrix & X, int N) {
diff --git a/bench/btl/libs/blaze/CMakeLists.txt b/bench/btl/libs/blaze/CMakeLists.txt index f8b1b2e..e99a085 100644 --- a/bench/btl/libs/blaze/CMakeLists.txt +++ b/bench/btl/libs/blaze/CMakeLists.txt
@@ -1,10 +1,13 @@ find_package(BLAZE) -find_package(Boost) +find_package(Boost COMPONENTS system) if (BLAZE_FOUND AND Boost_FOUND) include_directories(${BLAZE_INCLUDE_DIR} ${Boost_INCLUDE_DIRS}) btl_add_bench(btl_blaze main.cpp) + # Note: The newest blaze version requires C++14. + # Ideally, we should set this depending on the version of Blaze we found + set_property(TARGET btl_blaze PROPERTY CXX_STANDARD 14) if(BUILD_btl_blaze) - target_link_libraries(btl_blaze ${Boost_LIBRARIES} ${Boost_system_LIBRARY} /opt/local/lib/libboost_system-mt.a ) + target_link_libraries(btl_blaze ${Boost_LIBRARIES}) endif() endif ()
diff --git a/bench/btl/libs/blaze/blaze_interface.hh b/bench/btl/libs/blaze/blaze_interface.hh index ee15239..7b418f6 100644 --- a/bench/btl/libs/blaze/blaze_interface.hh +++ b/bench/btl/libs/blaze/blaze_interface.hh
@@ -20,6 +20,7 @@ #include <blaze/Math.h> #include <blaze/Blaze.h> +#include <Eigen/Core> // using namespace blaze; #include <vector> @@ -80,35 +81,35 @@ } } - static inline void matrix_matrix_product(const gene_matrix & A, const gene_matrix & B, gene_matrix & X, int N){ + static EIGEN_DONT_INLINE void matrix_matrix_product(const gene_matrix & A, const gene_matrix & B, gene_matrix & X, int N){ X = (A*B); } - static inline void transposed_matrix_matrix_product(const gene_matrix & A, const gene_matrix & B, gene_matrix & X, int N){ + static EIGEN_DONT_INLINE void transposed_matrix_matrix_product(const gene_matrix & A, const gene_matrix & B, gene_matrix & X, int N){ X = (trans(A)*trans(B)); } - static inline void ata_product(const gene_matrix & A, gene_matrix & X, int N){ + static EIGEN_DONT_INLINE void ata_product(const gene_matrix & A, gene_matrix & X, int N){ X = (trans(A)*A); } - static inline void aat_product(const gene_matrix & A, gene_matrix & X, int N){ + static EIGEN_DONT_INLINE void aat_product(const gene_matrix & A, gene_matrix & X, int N){ X = (A*trans(A)); } - static inline void matrix_vector_product(gene_matrix & A, gene_vector & B, gene_vector & X, int N){ + static EIGEN_DONT_INLINE void matrix_vector_product(gene_matrix & A, gene_vector & B, gene_vector & X, int N){ X = (A*B); } - static inline void atv_product(gene_matrix & A, gene_vector & B, gene_vector & X, int N){ + static EIGEN_DONT_INLINE void atv_product(gene_matrix & A, gene_vector & B, gene_vector & X, int N){ X = (trans(A)*B); } - static inline void axpy(const real coef, const gene_vector & X, gene_vector & Y, int N){ + static EIGEN_DONT_INLINE void axpy(const real coef, const gene_vector & X, gene_vector & Y, int N){ Y += coef * X; } - static inline void axpby(real a, const gene_vector & X, real b, gene_vector & Y, int N){ + static EIGEN_DONT_INLINE void axpby(real a, const gene_vector & X, real b, gene_vector & Y, int N){ Y = a*X + b*Y; }
diff --git a/bench/btl/libs/blaze/main.cpp b/bench/btl/libs/blaze/main.cpp index 80e8f4e..ccae0cb 100644 --- a/bench/btl/libs/blaze/main.cpp +++ b/bench/btl/libs/blaze/main.cpp
@@ -30,9 +30,9 @@ bench<Action_matrix_vector_product<blaze_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT); bench<Action_atv_product<blaze_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT); -// bench<Action_matrix_matrix_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); -// bench<Action_ata_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); -// bench<Action_aat_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); + bench<Action_matrix_matrix_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); + bench<Action_ata_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); + bench<Action_aat_product<blaze_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); return 0; }
diff --git a/bench/btl/libs/eigen3/eigen3_interface.hh b/bench/btl/libs/eigen3/eigen3_interface.hh index b821fd7..2e302d0 100644 --- a/bench/btl/libs/eigen3/eigen3_interface.hh +++ b/bench/btl/libs/eigen3/eigen3_interface.hh
@@ -92,9 +92,11 @@ X.noalias() = A.transpose()*B.transpose(); } -// static inline void ata_product(const gene_matrix & A, gene_matrix & X, int /*N*/){ -// X.noalias() = A.transpose()*A; -// } + static inline void ata_product(const gene_matrix & A, gene_matrix & X, int /*N*/){ + //X.noalias() = A.transpose()*A; + X.template triangularView<Lower>().setZero(); + X.template selfadjointView<Lower>().rankUpdate(A.transpose()); + } static inline void aat_product(const gene_matrix & A, gene_matrix & X, int /*N*/){ X.template triangularView<Lower>().setZero();
diff --git a/bench/btl/libs/eigen3/main_matmat.cpp b/bench/btl/libs/eigen3/main_matmat.cpp index 926fa2b..052810a 100644 --- a/bench/btl/libs/eigen3/main_matmat.cpp +++ b/bench/btl/libs/eigen3/main_matmat.cpp
@@ -25,7 +25,7 @@ int main() { bench<Action_matrix_matrix_product<eigen3_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); -// bench<Action_ata_product<eigen3_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); + bench<Action_ata_product<eigen3_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); bench<Action_aat_product<eigen3_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT); bench<Action_trmm<eigen3_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
diff --git a/bench/btl/libs/ublas/ublas_interface.hh b/bench/btl/libs/ublas/ublas_interface.hh index 95cad51..f59b7cf 100644 --- a/bench/btl/libs/ublas/ublas_interface.hh +++ b/bench/btl/libs/ublas/ublas_interface.hh
@@ -100,7 +100,7 @@ Y+=coef*X; } - // alias free assignements + // alias free assignments static inline void matrix_vector_product(gene_matrix & A, gene_vector & B, gene_vector & X, int N){ X.assign(prod(A,B));
diff --git a/bench/dense_solvers.cpp b/bench/dense_solvers.cpp index aa4ff01..24343dc 100644 --- a/bench/dense_solvers.cpp +++ b/bench/dense_solvers.cpp
@@ -2,47 +2,74 @@ #include "BenchTimer.h" #include <Eigen/Dense> #include <map> +#include <vector> #include <string> +#include <sstream> using namespace Eigen; -std::map<std::string,Array<float,1,4> > results; +std::map<std::string,Array<float,1,8,DontAlign|RowMajor> > results; +std::vector<std::string> labels; +std::vector<Array2i> sizes; + +template<typename Solver,typename MatrixType> +EIGEN_DONT_INLINE +void compute_norm_equation(Solver &solver, const MatrixType &A) { + if(A.rows()!=A.cols()) + solver.compute(A.transpose()*A); + else + solver.compute(A); +} + +template<typename Solver,typename MatrixType> +EIGEN_DONT_INLINE +void compute(Solver &solver, const MatrixType &A) { + solver.compute(A); +} template<typename Scalar,int Size> -void bench(int id, int size = Size) +void bench(int id, int rows, int size = Size) { - typedef Matrix<Scalar,Size,Size> Mat; - Mat A(size,size); + typedef Matrix<Scalar,Dynamic,Size> Mat; + typedef Matrix<Scalar,Dynamic,Dynamic> MatDyn; + typedef Matrix<Scalar,Size,Size> MatSquare; + Mat A(rows,size); A.setRandom(); - A = A*A.adjoint(); + if(rows==size) + A = A*A.adjoint(); BenchTimer t_llt, t_ldlt, t_lu, t_fplu, t_qr, t_cpqr, t_cod, t_fpqr, t_jsvd, t_bdcsvd; + + int svd_opt = ComputeThinU|ComputeThinV; - int tries = 3; + int tries = 5; int rep = 1000/size; if(rep==0) rep = 1; // rep = rep*rep; - LLT<Mat> llt(A); - LDLT<Mat> ldlt(A); - PartialPivLU<Mat> lu(A); - FullPivLU<Mat> fplu(A); - HouseholderQR<Mat> qr(A); - ColPivHouseholderQR<Mat> cpqr(A); - CompleteOrthogonalDecomposition<Mat> cod(A); - FullPivHouseholderQR<Mat> fpqr(A); - JacobiSVD<Mat> jsvd(A.rows(),A.cols()); - BDCSVD<Mat> bdcsvd(A.rows(),A.cols()); + LLT<MatSquare> llt(size); + LDLT<MatSquare> ldlt(size); + PartialPivLU<MatSquare> lu(size); + FullPivLU<MatSquare> fplu(size,size); + HouseholderQR<Mat> qr(A.rows(),A.cols()); + ColPivHouseholderQR<Mat> cpqr(A.rows(),A.cols()); + CompleteOrthogonalDecomposition<Mat> cod(A.rows(),A.cols()); + FullPivHouseholderQR<Mat> fpqr(A.rows(),A.cols()); + JacobiSVD<MatDyn> jsvd(A.rows(),A.cols()); + BDCSVD<MatDyn> bdcsvd(A.rows(),A.cols()); - BENCH(t_llt, tries, rep, llt.compute(A)); - BENCH(t_ldlt, tries, rep, ldlt.compute(A)); - BENCH(t_lu, tries, rep, lu.compute(A)); - BENCH(t_fplu, tries, rep, fplu.compute(A)); - BENCH(t_qr, tries, rep, qr.compute(A)); - BENCH(t_cpqr, tries, rep, cpqr.compute(A)); - BENCH(t_cod, tries, rep, cod.compute(A)); - BENCH(t_fpqr, tries, rep, fpqr.compute(A)); + BENCH(t_llt, tries, rep, compute_norm_equation(llt,A)); + BENCH(t_ldlt, tries, rep, compute_norm_equation(ldlt,A)); + BENCH(t_lu, tries, rep, compute_norm_equation(lu,A)); + if(size<=1000) + BENCH(t_fplu, tries, rep, compute_norm_equation(fplu,A)); + BENCH(t_qr, tries, rep, compute(qr,A)); + BENCH(t_cpqr, tries, rep, compute(cpqr,A)); + BENCH(t_cod, tries, rep, compute(cod,A)); + if(size*rows<=10000000) + BENCH(t_fpqr, tries, rep, compute(fpqr,A)); if(size<500) // JacobiSVD is really too slow for too large matrices - BENCH(t_jsvd, tries, rep, jsvd.compute(A,ComputeFullU|ComputeFullV)); - BENCH(t_bdcsvd, tries, rep, bdcsvd.compute(A,ComputeFullU|ComputeFullV)); + BENCH(t_jsvd, tries, rep, jsvd.compute(A,svd_opt)); +// if(size*rows<=20000000) + BENCH(t_bdcsvd, tries, rep, bdcsvd.compute(A,svd_opt)); results["LLT"][id] = t_llt.best(); results["LDLT"][id] = t_ldlt.best(); @@ -52,33 +79,108 @@ results["ColPivHouseholderQR"][id] = t_cpqr.best(); results["CompleteOrthogonalDecomposition"][id] = t_cod.best(); results["FullPivHouseholderQR"][id] = t_fpqr.best(); - results["JacobiSVD"][id] = size<500 ? t_jsvd.best() : 0; + results["JacobiSVD"][id] = t_jsvd.best(); results["BDCSVD"][id] = t_bdcsvd.best(); } + int main() { + labels.push_back("LLT"); + labels.push_back("LDLT"); + labels.push_back("PartialPivLU"); + labels.push_back("FullPivLU"); + labels.push_back("HouseholderQR"); + labels.push_back("ColPivHouseholderQR"); + labels.push_back("CompleteOrthogonalDecomposition"); + labels.push_back("FullPivHouseholderQR"); + labels.push_back("JacobiSVD"); + labels.push_back("BDCSVD"); + + for(int i=0; i<labels.size(); ++i) + results[labels[i]].fill(-1); + const int small = 8; - const int medium = 100; - const int large = 1000; - const int xl = 4000; - - bench<float,small>(0); - bench<float,Dynamic>(1,medium); - bench<float,Dynamic>(2,large); - bench<float,Dynamic>(3,xl); - - IOFormat fmt(3, 0, " \t", "\n", "", ""); - - std::cout << "solver/size " << small << "\t" << medium << "\t" << large << "\t" << xl << "\n"; - std::cout << "LLT (ms) " << (results["LLT"]/1000.).format(fmt) << "\n"; - std::cout << "LDLT (%) " << (results["LDLT"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "PartialPivLU (%) " << (results["PartialPivLU"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "FullPivLU (%) " << (results["FullPivLU"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "HouseholderQR (%) " << (results["HouseholderQR"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "ColPivHouseholderQR (%) " << (results["ColPivHouseholderQR"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "CompleteOrthogonalDecomposition (%) " << (results["CompleteOrthogonalDecomposition"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "FullPivHouseholderQR (%) " << (results["FullPivHouseholderQR"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "JacobiSVD (%) " << (results["JacobiSVD"]/results["LLT"]).format(fmt) << "\n"; - std::cout << "BDCSVD (%) " << (results["BDCSVD"]/results["LLT"]).format(fmt) << "\n"; + sizes.push_back(Array2i(small,small)); + sizes.push_back(Array2i(100,100)); + sizes.push_back(Array2i(1000,1000)); + sizes.push_back(Array2i(4000,4000)); + sizes.push_back(Array2i(10000,small)); + sizes.push_back(Array2i(10000,100)); + sizes.push_back(Array2i(10000,1000)); + sizes.push_back(Array2i(10000,4000)); + + using namespace std; + + for(int k=0; k<sizes.size(); ++k) + { + cout << sizes[k](0) << "x" << sizes[k](1) << "...\n"; + bench<float,Dynamic>(k,sizes[k](0),sizes[k](1)); + } + + cout.width(32); + cout << "solver/size"; + cout << " "; + for(int k=0; k<sizes.size(); ++k) + { + std::stringstream ss; + ss << sizes[k](0) << "x" << sizes[k](1); + cout.width(10); cout << ss.str(); cout << " "; + } + cout << endl; + + + for(int i=0; i<labels.size(); ++i) + { + cout.width(32); cout << labels[i]; cout << " "; + ArrayXf r = (results[labels[i]]*100000.f).floor()/100.f; + for(int k=0; k<sizes.size(); ++k) + { + cout.width(10); + if(r(k)>=1e6) cout << "-"; + else cout << r(k); + cout << " "; + } + cout << endl; + } + + // HTML output + cout << "<table class=\"manual\">" << endl; + cout << "<tr><th>solver/size</th>" << endl; + for(int k=0; k<sizes.size(); ++k) + cout << " <th>" << sizes[k](0) << "x" << sizes[k](1) << "</th>"; + cout << "</tr>" << endl; + for(int i=0; i<labels.size(); ++i) + { + cout << "<tr"; + if(i%2==1) cout << " class=\"alt\""; + cout << "><td>" << labels[i] << "</td>"; + ArrayXf r = (results[labels[i]]*100000.f).floor()/100.f; + for(int k=0; k<sizes.size(); ++k) + { + if(r(k)>=1e6) cout << "<td>-</td>"; + else + { + cout << "<td>" << r(k); + if(i>0) + cout << " (x" << numext::round(10.f*results[labels[i]](k)/results["LLT"](k))/10.f << ")"; + if(i<4 && sizes[k](0)!=sizes[k](1)) + cout << " <sup><a href=\"#note_ls\">*</a></sup>"; + cout << "</td>"; + } + } + cout << "</tr>" << endl; + } + cout << "</table>" << endl; + +// cout << "LLT (ms) " << (results["LLT"]*1000.).format(fmt) << "\n"; +// cout << "LDLT (%) " << (results["LDLT"]/results["LLT"]).format(fmt) << "\n"; +// cout << "PartialPivLU (%) " << (results["PartialPivLU"]/results["LLT"]).format(fmt) << "\n"; +// cout << "FullPivLU (%) " << (results["FullPivLU"]/results["LLT"]).format(fmt) << "\n"; +// cout << "HouseholderQR (%) " << (results["HouseholderQR"]/results["LLT"]).format(fmt) << "\n"; +// cout << "ColPivHouseholderQR (%) " << (results["ColPivHouseholderQR"]/results["LLT"]).format(fmt) << "\n"; +// cout << "CompleteOrthogonalDecomposition (%) " << (results["CompleteOrthogonalDecomposition"]/results["LLT"]).format(fmt) << "\n"; +// cout << "FullPivHouseholderQR (%) " << (results["FullPivHouseholderQR"]/results["LLT"]).format(fmt) << "\n"; +// cout << "JacobiSVD (%) " << (results["JacobiSVD"]/results["LLT"]).format(fmt) << "\n"; +// cout << "BDCSVD (%) " << (results["BDCSVD"]/results["LLT"]).format(fmt) << "\n"; }
diff --git a/bench/eig33.cpp b/bench/eig33.cpp index 47947a9..f003d8a 100644 --- a/bench/eig33.cpp +++ b/bench/eig33.cpp
@@ -101,7 +101,7 @@ computeRoots(scaledMat,evals); // compute the eigen vectors - // **here we assume 3 differents eigenvalues** + // **here we assume 3 different eigenvalues** // "optimized version" which appears to be slower with gcc! // Vector base;
diff --git a/bench/perf_monitoring/changesets.txt b/bench/perf_monitoring/changesets.txt new file mode 100644 index 0000000..c6b3645 --- /dev/null +++ b/bench/perf_monitoring/changesets.txt
@@ -0,0 +1,93 @@ +#3.0.1 +#3.1.1 +#3.2.0 +3.2.4 +#5745:37f59e65eb6c +5891:d8652709345d # introduce AVX +#5893:24b4dc92c6d3 # merge +5895:997c2ef9fc8b # introduce FMA +#5904:e1eafd14eaa1 # complex and AVX +5908:f8ee3c721251 # improve packing with ptranspose +#5921:ca808bb456b0 # merge +#5927:8b1001f9e3ac +5937:5a4ca1ad8c53 # New gebp kernel: up to 3 packets x 4 register-level blocks +#5949:f3488f4e45b2 # merge +#5969:e09031dccfd9 # Disable 3pX4 kernel on Altivec +#5992:4a429f5e0483 # merge +before-evaluators +#6334:f6a45e5b8b7c # Implement evaluator for sparse outer products +#6639:c9121c60b5c7 +#6655:06f163b5221f # Properly detect FMA support on ARM +#6677:700e023044e7 # FMA has been wrongly disabled +#6681:11d31dafb0e3 +#6699:5e6e8e10aad1 # merge default to tensors +#6726:ff2d2388e7b9 # merge default to tensors +#6742:0cbd6195e829 # merge default to tensors +#6747:853d2bafeb8f # Generalized the gebp apis +6765:71584fd55762 # Made the blocking computation aware of the l3 cache;<br/> Also optimized the blocking parameters to take<br/> into account the number of threads used for a computation. +6781:9cc5a931b2c6 # generalized gemv +6792:f6e1daab600a # ensured that contractions that can be reduced to a matrix vector product +#6844:039efd86b75c # merge tensor +6845:7333ed40c6ef # change prefetching in gebp +#6856:b5be5e10eb7f # merge index conversion +6893:c3a64aba7c70 # clean blocking size computation +6899:877facace746 # rotating kernel for ARM only +#6904:c250623ae9fa # result_of +6921:915f1b1fc158 # fix prefetching change for ARM +6923:9ff25f6dacc6 # prefetching +6933:52572e60b5d3 # blocking size strategy +6937:c8c042f286b2 # avoid redundant pack_rhs +6981:7e5d6f78da59 # dynamic loop swapping +6984:45f26866c091 # rm dynamic loop swapping,<br/> adjust lhs's micro panel height to fully exploit L1 cache +6986:a675d05b6f8f # blocking heuristic:<br/> block on the rhs in L1 if the lhs fit in L1. +7013:f875e75f07e5 # organize a little our default cache sizes,<br/> and use a saner default L1 outside of x86 (10% faster on Nexus 5) +7015:8aad8f35c955 # Refactor computeProductBlockingSizes to make room<br/> for the possibility of using lookup tables +7016:a58d253e8c91 # Polish lookup tables generation +7018:9b27294a8186 # actual_panel_rows computation should always be resilient<br/> to parameters not consistent with the known L1 cache size, see comment +7019:c758b1e2c073 # Provide a empirical lookup table for blocking sizes measured on a Nexus 5.<br/> Only for float, only for Android on ARM 32bit for now. +7085:627e039fba68 # Bug 986: add support for coefficient-based<br/> product with 0 depth. +7098:b6f1db9cf9ec # Bug 992: don't select a 3p GEMM path with non-SIMD scalar types. +7591:09a8e2186610 # 3.3-alpha1 +7650:b0f3c8f43025 # help clang inlining +7708:dfc6ab9d9458 # Improve numerical accuracy in LLT and triangular solve<br/> by using true scalar divisions (instead of x * (1/y)) +#8744:74b789ada92a # Improved the matrix multiplication blocking in the case<br/> where mr is not a power of 2 (e.g on Haswell CPUs) +8789:efcb912e4356 # Made the index type a template parameter to evaluateProductBlockingSizes.<br/> Use numext::mini and numext::maxi instead of <br/> std::min/std::max to compute blocking sizes. +8972:81d53c711775 # Don't optimize the processing of the last rows of<br/> a matrix matrix product in cases that violate<br/> the assumptions made by the optimized code path. +8985:d935df21a082 # Remove the rotating kernel. +8988:6c2dc56e73b3 # Bug 256: enable vectorization with unaligned loads/stores. +9148:b8b8c421e36c # Relax mixing-type constraints for binary coeff-wise operators +9174:d228bc282ac9 # merge +9175:abc7a3600098 # Include the cost of stores in unrolling +9212:c90098affa7b # Fix perf regression introduced in changeset 8aad8f35c955 +9213:9f1c14e4694b # Fix perf regression in dgemm introduced by changeset 81d53c711775 +9361:69d418c06999 # 3.3-beta2 +9445:f27ff0ad77a3 # Optimize expression matching 'd?=a-b*c' as 'd?=a; d?=b*c;' +9583:bef509908b9d # 3.3-rc1 +9593:2f24280cf59a # Bug 1311: fix alignment logic in some cases<br/> of (scalar*small).lazyProduct(small) +9722:040d861b88b5 # Disabled part of the matrix matrix peeling code<br/> that's incompatible with 512 bit registers +9792:26667be4f70b # 3.3.0 +9891:41260bdfc23b # Fix a performance regression in (mat*mat)*vec<br/> for which mat*mat was evaluated multiple times. +9942:b1d3eba60130 # Operators += and -= do not resize! +9943:79bb9887afd4 # Ease compiler generating clean and efficient code in mat*vec +9946:2213991340ea # Complete rewrite of column-major-matrix * vector product<br/> to deliver higher performance of modern CPU. +9955:630471c3298c # Improve performance of row-major-dense-matrix * vector products<br/> for recent CPUs. +9975:2eeed9de710c # Revert vec/y to vec*(1/y) in row-major TRSM +10442:e3f17da72a40 # Bug 1435: fix aliasing issue in exressions like: A = C - B*A; +10735:6913f0cf7d06 # Adds missing EIGEN_STRONG_INLINE to support MSVC<br/> properly inlining small vector calculations +10943:4db388d946bd # Bug 1562: optimize evaluation of small products<br/> of the form s*A*B by rewriting them as: s*(A.lazyProduct(B))<br/> to save a costly temporary.<br/> Measured speedup from 2x to 5x. +10961:5007ff66c9f6 # Introduce the macro ei_declare_local_nested_eval to<br/> help allocating on the stack local temporaries via alloca,<br/> and let outer-products makes a good use of it. +11083:30a528a984bb # Bug 1578: Improve prefetching in matrix multiplication on MIPS. +11533:71609c41e9f8 # PR 526: Speed up multiplication of small, dynamically sized matrices +11545:6d348dc9b092 # Vectorize row-by-row gebp loop iterations on 16 packets as well +11579:efda481cbd7a # Bug 1624: improve matrix-matrix product on ARM 64, 20% speedup +11606:b8d3f548a9d9 # do not read buffers out of bounds +11638:22f9cc0079bd # Implement AVX512 vectorization of std::complex<float/double> +11642:9f52fde03483 # Bug 1636: fix gemm performance issue with gcc>=6 and no FMA +11648:81172653b67b # Bug 1515: disable gebp's 3pX4 micro kernel<br/> for MSVC<=19.14 because of register spilling. +11654:b81188e099f3 # fix EIGEN_GEBP_2PX4_SPILLING_WORKAROUND<br/> for non vectorized type, and non x86/64 target +11664:71546f1a9f0c # enable spilling workaround on architectures with SSE/AVX +11669:b500fef42ced # Artificially increase l1-blocking size for AVX512.<br/> +10% speedup with current kernels. +11683:2ea2960f1c7f # Make code compile again for older compilers. +11753:556fb4ceb654 # Bug: 1633: refactor gebp kernel and optimize for neon +11761:cefc1ba05596 # Bug 1661: fix regression in GEBP and AVX512 +11763:1e41e70fe97b # GEBP: cleanup logic to choose between<br/> a 4 packets of 1 packet (=209bf81aa3f3+fix) \ No newline at end of file
diff --git a/bench/perf_monitoring/gemm.cpp b/bench/perf_monitoring/gemm.cpp new file mode 100644 index 0000000..804139d --- /dev/null +++ b/bench/perf_monitoring/gemm.cpp
@@ -0,0 +1,12 @@ +#include "gemm_common.h" + +EIGEN_DONT_INLINE +void gemm(const Mat &A, const Mat &B, Mat &C) +{ + C.noalias() += A * B; +} + +int main(int argc, char **argv) +{ + return main_gemm(argc, argv, gemm); +}
diff --git a/bench/perf_monitoring/gemm/changesets.txt b/bench/perf_monitoring/gemm/changesets.txt deleted file mode 100644 index fb3e48e..0000000 --- a/bench/perf_monitoring/gemm/changesets.txt +++ /dev/null
@@ -1,47 +0,0 @@ -#3.0.1 -#3.1.1 -#3.2.0 -3.2.4 -#5745:37f59e65eb6c -5891:d8652709345d # introduce AVX -#5893:24b4dc92c6d3 # merge -5895:997c2ef9fc8b # introduce FMA -#5904:e1eafd14eaa1 # complex and AVX -5908:f8ee3c721251 # improve packing with ptranspose -#5921:ca808bb456b0 # merge -#5927:8b1001f9e3ac -5937:5a4ca1ad8c53 # New gebp kernel handling up to 3 packets x 4 register-level blocks -#5949:f3488f4e45b2 # merge -#5969:e09031dccfd9 # Disable 3pX4 kernel on Altivec -#5992:4a429f5e0483 # merge -before-evaluators -#6334:f6a45e5b8b7c # Implement evaluator for sparse outer products -#6639:c9121c60b5c7 -#6655:06f163b5221f # Properly detect FMA support on ARM -#6677:700e023044e7 # FMA has been wrongly disabled -#6681:11d31dafb0e3 -#6699:5e6e8e10aad1 # merge default to tensors -#6726:ff2d2388e7b9 # merge default to tensors -#6742:0cbd6195e829 # merge default to tensors -#6747:853d2bafeb8f # Generalized the gebp apis -6765:71584fd55762 # Made the blocking computation aware of the l3 cache; Also optimized the blocking parameters to take into account the number of threads used for a computation -#6781:9cc5a931b2c6 # generalized gemv -#6792:f6e1daab600a # ensured that contractions that can be reduced to a matrix vector product -#6844:039efd86b75c # merge tensor -6845:7333ed40c6ef # change prefetching in gebp -#6856:b5be5e10eb7f # merge index conversion -#6893:c3a64aba7c70 # clean blocking size computation -#6898:6fb31ebe6492 # rotating kernel for ARM -6899:877facace746 # rotating kernel for ARM only -#6904:c250623ae9fa # result_of -6921:915f1b1fc158 # fix prefetching change for ARM -6923:9ff25f6dacc6 # prefetching -6933:52572e60b5d3 # blocking size strategy -6937:c8c042f286b2 # avoid redundant pack_rhs -6981:7e5d6f78da59 # dynamic loop swapping -6984:45f26866c091 # rm dynamic loop swapping, adjust lhs's micro panel height to fully exploit L1 cache -6986:a675d05b6f8f # blocking heuristic: block on the rhs in L1 if the lhs fit in L1. -7013:f875e75f07e5 # organize a little our default cache sizes, and use a saner default L1 outside of x86 (10% faster on Nexus 5) -7591:09a8e2186610 # 3.3-alpha1 -7650:b0f3c8f43025 # help clang inlining -
diff --git a/bench/perf_monitoring/gemm/make_plot.sh b/bench/perf_monitoring/gemm/make_plot.sh deleted file mode 100755 index 4d60535..0000000 --- a/bench/perf_monitoring/gemm/make_plot.sh +++ /dev/null
@@ -1,38 +0,0 @@ -#!/bin/bash - -# base name of the bench -# it reads $1.out -# and generates $1.pdf -WHAT=$1 -bench=$2 - -header="rev " -while read line -do - if [ ! -z '$line' ]; then - header="$header \"$line\"" - fi -done < $bench"_settings.txt" - -echo $header > $WHAT.out.header -cat $WHAT.out >> $WHAT.out.header - - -echo "set title '$WHAT'" > $WHAT.gnuplot -echo "set key autotitle columnhead outside " >> $WHAT.gnuplot -echo "set xtics rotate 1" >> $WHAT.gnuplot - -echo "set term pdf color rounded enhanced fontscale 0.35 size 7in,5in" >> $WHAT.gnuplot -echo set output "'"$WHAT.pdf"'" >> $WHAT.gnuplot - -col=`cat settings.txt | wc -l` -echo "plot for [col=2:$col+1] '$WHAT.out.header' using 0:col:xticlabels(1) with lines" >> $WHAT.gnuplot -echo " " >> $WHAT.gnuplot - -gnuplot -persist < $WHAT.gnuplot - -# generate a png file -# convert -background white -density 120 -rotate 90 -resize 800 +dither -colors 256 -quality 0 $WHAT.ps -background white -flatten .$WHAT.png - -# clean -rm $WHAT.out.header $WHAT.gnuplot \ No newline at end of file
diff --git a/bench/perf_monitoring/gemm/run.sh b/bench/perf_monitoring/gemm/run.sh deleted file mode 100755 index bfb4ecf..0000000 --- a/bench/perf_monitoring/gemm/run.sh +++ /dev/null
@@ -1,156 +0,0 @@ -#!/bin/bash - -# ./run.sh gemm -# ./run.sh lazy_gemm - -# Examples of environment variables to be set: -# PREFIX="haswell-fma-" -# CXX_FLAGS="-mfma" - -# Options: -# -up : enforce the recomputation of existing data, and keep best results as a merging strategy -# -s : recompute selected changesets only and keep bests - -bench=$1 - -if echo "$*" | grep '\-up' > /dev/null; then - update=true -else - update=false -fi - -if echo "$*" | grep '\-s' > /dev/null; then - selected=true -else - selected=false -fi - -global_args="$*" - -if [ $selected == true ]; then - echo "Recompute selected changesets only and keep bests" -elif [ $update == true ]; then - echo "(Re-)Compute all changesets and keep bests" -else - echo "Skip previously computed changesets" -fi - - - -if [ ! -d "eigen_src" ]; then - hg clone https://bitbucket.org/eigen/eigen eigen_src -else - cd eigen_src - hg pull -u - cd .. -fi - -if [ ! -z '$CXX' ]; then - CXX=g++ -fi - -function make_backup -{ - if [ -f "$1.out" ]; then - mv "$1.out" "$1.backup" - fi -} - -function merge -{ - count1=`echo $1 | wc -w` - count2=`echo $2 | wc -w` - - if [ $count1 == $count2 ]; then - a=( $1 ); b=( $2 ) - res="" - for (( i=0 ; i<$count1 ; i++ )); do - ai=${a[$i]}; bi=${b[$i]} - tmp=`echo "if ($ai > $bi) $ai else $bi " | bc -l` - res="$res $tmp" - done - echo $res - - else - echo $1 - fi -} - -function test_current -{ - rev=$1 - scalar=$2 - name=$3 - - prev="" - if [ -e "$name.backup" ]; then - prev=`grep $rev "$name.backup" | cut -c 14-` - fi - res=$prev - count_rev=`echo $prev | wc -w` - count_ref=`cat $bench"_settings.txt" | wc -l` - if echo "$global_args" | grep "$rev" > /dev/null; then - rev_found=true - else - rev_found=false - fi -# echo $update et $selected et $rev_found because $rev et "$global_args" -# echo $count_rev et $count_ref - if [ $update == true ] || [ $count_rev != $count_ref ] || ([ $selected == true ] && [ $rev_found == true ]); then - if $CXX -O2 -DNDEBUG -march=native $CXX_FLAGS -I eigen_src $bench.cpp -DSCALAR=$scalar -o $name; then - curr=`./$name` - if [ $count_rev == $count_ref ]; then - echo "merge previous $prev" - echo "with new $curr" - else - echo "got $curr" - fi - res=`merge "$curr" "$prev"` -# echo $res - echo "$rev $res" >> $name.out - else - echo "Compilation failed, skip rev $rev" - fi - else - echo "Skip existing results for $rev / $name" - echo "$rev $res" >> $name.out - fi -} - -make_backup $PREFIX"s"$bench -make_backup $PREFIX"d"$bench -make_backup $PREFIX"c"$bench - -cut -f1 -d"#" < changesets.txt | grep -E '[[:alnum:]]' | while read rev -do - if [ ! -z '$rev' ]; then - echo "Testing rev $rev" - cd eigen_src - hg up -C $rev > /dev/null - actual_rev=`hg identify | cut -f1 -d' '` - cd .. - - test_current $actual_rev float $PREFIX"s"$bench - test_current $actual_rev double $PREFIX"d"$bench - test_current $actual_rev "std::complex<double>" $PREFIX"c"$bench - fi - -done - -echo "Float:" -cat $PREFIX"s"$bench.out" -echo "" - -echo "Double:" -cat $PREFIX"d"$bench.out" -echo "" - -echo "Complex:" -cat $PREFIX"c"$bench.out" -echo "" - -./make_plot.sh $PREFIX"s"$bench $bench -./make_plot.sh $PREFIX"d"$bench $bench -./make_plot.sh $PREFIX"c"$bench $bench - -
diff --git a/bench/perf_monitoring/gemm/gemm.cpp b/bench/perf_monitoring/gemm_common.h similarity index 66% rename from bench/perf_monitoring/gemm/gemm.cpp rename to bench/perf_monitoring/gemm_common.h index 614bd47..30dbc0d 100644 --- a/bench/perf_monitoring/gemm/gemm.cpp +++ b/bench/perf_monitoring/gemm_common.h
@@ -1,8 +1,9 @@ #include <iostream> #include <fstream> #include <vector> -#include <Eigen/Core> -#include "../../BenchTimer.h" +#include <string> +#include "eigen_src/Eigen/Core" +#include "../BenchTimer.h" using namespace Eigen; #ifndef SCALAR @@ -13,14 +14,9 @@ typedef Matrix<Scalar,Dynamic,Dynamic> Mat; +template<typename Func> EIGEN_DONT_INLINE -void gemm(const Mat &A, const Mat &B, Mat &C) -{ - C.noalias() += A * B; -} - -EIGEN_DONT_INLINE -double bench(long m, long n, long k) +double bench(long m, long n, long k, const Func& f) { Mat A(m,k); Mat B(k,n); @@ -44,21 +40,25 @@ long rep = std::max(1., std::min(100., up/flops) ); long tries = std::max(tm0, std::min(tm1, up/flops) ); - BENCH(t, tries, rep, gemm(A,B,C)); + BENCH(t, tries, rep, f(A,B,C)); return 1e-9 * rep * flops / t.best(); } -int main(int argc, char **argv) +template<typename Func> +int main_gemm(int argc, char **argv, const Func& f) { std::vector<double> results; - std::ifstream settings("gemm_settings.txt"); + std::string filename = std::string("gemm_settings.txt"); + if(argc>1) + filename = std::string(argv[1]); + std::ifstream settings(filename); long m, n, k; while(settings >> m >> n >> k) { //std::cerr << " Testing " << m << " " << n << " " << k << std::endl; - results.push_back( bench(m, n, k) ); + results.push_back( bench(m, n, k, f) ); } std::cout << RowVectorXd::Map(results.data(), results.size());
diff --git a/bench/perf_monitoring/gemm/gemm_settings.txt b/bench/perf_monitoring/gemm_settings.txt similarity index 100% rename from bench/perf_monitoring/gemm/gemm_settings.txt rename to bench/perf_monitoring/gemm_settings.txt
diff --git a/bench/perf_monitoring/gemm_square_settings.txt b/bench/perf_monitoring/gemm_square_settings.txt new file mode 100644 index 0000000..98474d1 --- /dev/null +++ b/bench/perf_monitoring/gemm_square_settings.txt
@@ -0,0 +1,11 @@ +8 8 8 +9 9 9 +12 12 12 +15 15 15 +16 16 16 +24 24 24 +102 102 102 +239 239 239 +240 240 240 +2400 2400 2400 +2463 2463 2463
diff --git a/bench/perf_monitoring/gemv.cpp b/bench/perf_monitoring/gemv.cpp new file mode 100644 index 0000000..82e5ab9 --- /dev/null +++ b/bench/perf_monitoring/gemv.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void gemv(const Mat &A, const Vec &B, Vec &C) +{ + C.noalias() += A * B; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, gemv); +}
diff --git a/bench/perf_monitoring/gemv_common.h b/bench/perf_monitoring/gemv_common.h new file mode 100644 index 0000000..cc32577 --- /dev/null +++ b/bench/perf_monitoring/gemv_common.h
@@ -0,0 +1,69 @@ +#include <iostream> +#include <fstream> +#include <vector> +#include <string> +#include <functional> +#include "eigen_src/Eigen/Core" +#include "../BenchTimer.h" +using namespace Eigen; + +#ifndef SCALAR +#error SCALAR must be defined +#endif + +typedef SCALAR Scalar; + +typedef Matrix<Scalar,Dynamic,Dynamic> Mat; +typedef Matrix<Scalar,Dynamic,1> Vec; + +template<typename Func> +EIGEN_DONT_INLINE +double bench(long m, long n, Func &f) +{ + Mat A(m,n); + Vec B(n); + Vec C(m); + A.setRandom(); + B.setRandom(); + C.setRandom(); + + BenchTimer t; + + double up = 1e8/sizeof(Scalar); + double tm0 = 4, tm1 = 10; + if(NumTraits<Scalar>::IsComplex) + { + up /= 4; + tm0 = 2; + tm1 = 4; + } + + double flops = 2. * m * n; + long rep = std::max(1., std::min(100., up/flops) ); + long tries = std::max(tm0, std::min(tm1, up/flops) ); + + BENCH(t, tries, rep, f(A,B,C)); + + return 1e-9 * rep * flops / t.best(); +} + +template<typename Func> +int main_gemv(int argc, char **argv, Func& f) +{ + std::vector<double> results; + + std::string filename = std::string("gemv_settings.txt"); + if(argc>1) + filename = std::string(argv[1]); + std::ifstream settings(filename); + long m, n; + while(settings >> m >> n) + { + //std::cerr << " Testing " << m << " " << n << std::endl; + results.push_back( bench(m, n, f) ); + } + + std::cout << RowVectorXd::Map(results.data(), results.size()); + + return 0; +}
diff --git a/bench/perf_monitoring/gemv_settings.txt b/bench/perf_monitoring/gemv_settings.txt new file mode 100644 index 0000000..21a5ee0 --- /dev/null +++ b/bench/perf_monitoring/gemv_settings.txt
@@ -0,0 +1,11 @@ +8 8 +9 9 +24 24 +239 239 +240 240 +2400 24 +24 2400 +24 240 +2400 2400 +4800 23 +23 4800
diff --git a/bench/perf_monitoring/gemv_square_settings.txt b/bench/perf_monitoring/gemv_square_settings.txt new file mode 100644 index 0000000..5165759 --- /dev/null +++ b/bench/perf_monitoring/gemv_square_settings.txt
@@ -0,0 +1,13 @@ +8 8 +9 9 +12 12 +15 15 +16 16 +24 24 +53 53 +74 74 +102 102 +239 239 +240 240 +2400 2400 +2463 2463
diff --git a/bench/perf_monitoring/gemvt.cpp b/bench/perf_monitoring/gemvt.cpp new file mode 100644 index 0000000..fe94576 --- /dev/null +++ b/bench/perf_monitoring/gemvt.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void gemv(const Mat &A, Vec &B, const Vec &C) +{ + B.noalias() += A.transpose() * C; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, gemv); +}
diff --git a/bench/perf_monitoring/gemm/lazy_gemm.cpp b/bench/perf_monitoring/lazy_gemm.cpp similarity index 87% rename from bench/perf_monitoring/gemm/lazy_gemm.cpp rename to bench/perf_monitoring/lazy_gemm.cpp index b443218..7733060 100644 --- a/bench/perf_monitoring/gemm/lazy_gemm.cpp +++ b/bench/perf_monitoring/lazy_gemm.cpp
@@ -12,12 +12,13 @@ typedef SCALAR Scalar; template<typename MatA, typename MatB, typename MatC> -inline void lazy_gemm(const MatA &A, const MatB &B, MatC &C) +EIGEN_DONT_INLINE +void lazy_gemm(const MatA &A, const MatB &B, MatC &C) { - escape((void*)A.data()); - escape((void*)B.data()); +// escape((void*)A.data()); +// escape((void*)B.data()); C.noalias() += A.lazyProduct(B); - escape((void*)C.data()); +// escape((void*)C.data()); } template<int m, int n, int k, int TA> @@ -83,7 +84,10 @@ { std::vector<double> results; - std::ifstream settings("lazy_gemm_settings.txt"); + std::string filename = std::string("lazy_gemm_settings.txt"); + if(argc>1) + filename = std::string(argv[1]); + std::ifstream settings(filename); long m, n, k, t; while(settings >> m >> n >> k >> t) {
diff --git a/bench/perf_monitoring/gemm/lazy_gemm_settings.txt b/bench/perf_monitoring/lazy_gemm_settings.txt similarity index 100% rename from bench/perf_monitoring/gemm/lazy_gemm_settings.txt rename to bench/perf_monitoring/lazy_gemm_settings.txt
diff --git a/bench/perf_monitoring/llt.cpp b/bench/perf_monitoring/llt.cpp new file mode 100644 index 0000000..d55b7d8 --- /dev/null +++ b/bench/perf_monitoring/llt.cpp
@@ -0,0 +1,15 @@ +#include "gemm_common.h" +#include <Eigen/Cholesky> + +EIGEN_DONT_INLINE +void llt(const Mat &A, const Mat &B, Mat &C) +{ + C = A; + C.diagonal().array() += 1000; + Eigen::internal::llt_inplace<Mat::Scalar, Lower>::blocked(C); +} + +int main(int argc, char **argv) +{ + return main_gemm(argc, argv, llt); +}
diff --git a/bench/perf_monitoring/make_plot.sh b/bench/perf_monitoring/make_plot.sh new file mode 100755 index 0000000..65aaf66 --- /dev/null +++ b/bench/perf_monitoring/make_plot.sh
@@ -0,0 +1,112 @@ +#!/bin/bash + +# base name of the bench +# it reads $1.out +# and generates $1.pdf +WHAT=$1 +bench=$2 +settings_file=$3 + +header="rev " +while read line +do + if [ ! -z '$line' ]; then + header="$header \"$line\"" + fi +done < $settings_file + +echo $header > $WHAT.out.header +cat $WHAT.out >> $WHAT.out.header + + +echo "set title '$WHAT'" > $WHAT.gnuplot +echo "set key autotitle columnhead outside " >> $WHAT.gnuplot +echo "set xtics rotate 1" >> $WHAT.gnuplot + +echo "set term pdf color rounded enhanced fontscale 0.35 size 7in,5in" >> $WHAT.gnuplot +echo set output "'"$WHAT.pdf"'" >> $WHAT.gnuplot + +col=`cat $settings_file | wc -l` +echo "plot for [col=2:$col+1] '$WHAT.out.header' using 0:col:xticlabels(1) with lines" >> $WHAT.gnuplot +echo " " >> $WHAT.gnuplot + +gnuplot -persist < $WHAT.gnuplot + +# generate a png file (thumbnail) +convert -colors 256 -background white -density 300 -resize 300 -quality 0 $WHAT.pdf -background white -flatten $WHAT.png + +# clean +rm $WHAT.out.header $WHAT.gnuplot + + +# generate html/svg graph + +echo " " > $WHAT.html +cat resources/chart_header.html > $WHAT.html +echo 'var customSettings = {"TITLE":"","SUBTITLE":"","XLABEL":"","YLABEL":""};' >> $WHAT.html +# 'data' is an array of datasets (i.e. curves), each of which is an object of the form +# { +# key: <name of the curve>, +# color: <optional color of the curve>, +# values: [{ +# r: <revision number>, +# v: <GFlops> +# }] +# } +echo 'var data = [' >> $WHAT.html + +col=2 +while read line +do + if [ ! -z '$line' ]; then + header="$header \"$line\"" + echo '{"key":"'$line'","values":[' >> $WHAT.html + i=0 + while read line2 + do + if [ ! -z "$line2" ]; then + val=`echo $line2 | cut -s -f $col -d ' '` + if [ -n "$val" ]; then # skip build failures + echo '{"r":'$i',"v":'$val'},' >> $WHAT.html + fi + fi + ((i++)) + done < $WHAT.out + echo ']},' >> $WHAT.html + fi + ((col++)) +done < $settings_file +echo '];' >> $WHAT.html + +echo 'var changesets = [' >> $WHAT.html +while read line2 +do + if [ ! -z '$line2' ]; then + echo '"'`echo $line2 | cut -f 1 -d ' '`'",' >> $WHAT.html + fi +done < $WHAT.out +echo '];' >> $WHAT.html + +echo 'var changesets_details = [' >> $WHAT.html +while read line2 +do + if [ ! -z '$line2' ]; then + num=`echo "$line2" | cut -f 1 -d ' '` + comment=`grep ":$num" changesets.txt | cut -f 2 -d '#'` + echo '"'"$comment"'",' >> $WHAT.html + fi +done < $WHAT.out +echo '];' >> $WHAT.html + +echo 'var changesets_count = [' >> $WHAT.html +i=0 +while read line2 +do + if [ ! -z '$line2' ]; then + echo $i ',' >> $WHAT.html + fi + ((i++)) +done < $WHAT.out +echo '];' >> $WHAT.html + +cat resources/chart_footer.html >> $WHAT.html
diff --git a/bench/perf_monitoring/resources/chart_footer.html b/bench/perf_monitoring/resources/chart_footer.html new file mode 100644 index 0000000..e8ef0a2 --- /dev/null +++ b/bench/perf_monitoring/resources/chart_footer.html
@@ -0,0 +1,41 @@ + /* setup the chart and its options */ + var chart = nv.models.lineChart() + .color(d3.scale.category10().range()) + .margin({left: 75, bottom: 100}) + .forceX([0]).forceY([0]); + + chart.x(function(datum){ return datum.r; }) + .xAxis.options({ + axisLabel: customSettings.XLABEL || 'Changeset', + tickFormat: d3.format('.0f') + }); + chart.xAxis + .tickValues(changesets_count) + .tickFormat(function(d){return changesets[d]}) + .rotateLabels(-90); + + chart.y(function(datum){ return datum.v; }) + .yAxis.options({ + axisLabel: customSettings.YLABEL || 'GFlops'/*, + tickFormat: function(val){ return d3.format('.0f')(val) + ' GFlops'; }*/ + }); + + chart.tooltip.headerFormatter(function(d) { return changesets[d] + + ' <p style="font-weight:normal;text-align: left;">' + + changesets_details[d] + "</p>"; }); + + //chart.useInteractiveGuideline(true); + d3.select('#chart').datum(data).call(chart); + var plot = d3.select('#chart > g'); + + /* setup the title */ + plot.append('text') + .style('font-size', '24px') + .attr('text-anchor', 'middle').attr('x', '50%').attr('y', '20px') + .text(customSettings.TITLE || ''); + + /* ensure the chart is responsive */ + nv.utils.windowResize(chart.update); + </script> + </body> +</html> \ No newline at end of file
diff --git a/bench/perf_monitoring/resources/chart_header.html b/bench/perf_monitoring/resources/chart_header.html new file mode 100644 index 0000000..bb9ddff --- /dev/null +++ b/bench/perf_monitoring/resources/chart_header.html
@@ -0,0 +1,46 @@ + +<!DOCTYPE html> +<html> + <head> + <meta charset='utf-8'> + <style>.nvd3 .nv-axis line,.nvd3 .nv-axis path{fill:none;shape-rendering:crispEdges}.nv-brush .extent,.nvd3 .background path,.nvd3 .nv-axis line,.nvd3 .nv-axis path{shape-rendering:crispEdges}.nv-distx,.nv-disty,.nv-noninteractive,.nvd3 .nv-axis,.nvd3.nv-pie .nv-label,.nvd3.nv-sparklineplus g.nv-hoverValue{pointer-events:none}.nvtooltip,svg.nvd3-svg{display:block;-webkit-touch-callout:none;-khtml-user-select:none}.nvd3 .nv-axis{opacity:1}.nvd3 .nv-axis.nv-disabled,.nvd3 .nv-controlsWrap .nv-legend .nv-check-box .nv-check{opacity:0}.nvd3 .nv-axis path{stroke:#000;stroke-opacity:.75}.nvd3 .nv-axis path.domain{stroke-opacity:.75}.nvd3 .nv-axis.nv-x path.domain{stroke-opacity:0}.nvd3 .nv-axis line{stroke:#e5e5e5}.nvd3 .nv-axis .zero line, .nvd3 .nv-axis line.zero{stroke-opacity:.75}.nvd3 .nv-axis .nv-axisMaxMin text{font-weight:700}.nvd3 .x .nv-axis .nv-axisMaxMin text,.nvd3 .x2 .nv-axis .nv-axisMaxMin text,.nvd3 .x3 .nv-axis .nv-axisMaxMin text{text-anchor:middle}.nvd3 .nv-bars rect{fill-opacity:.75;transition:fill-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear}.nvd3 .nv-bars rect.hover{fill-opacity:1}.nvd3 .nv-bars .hover rect{fill:#add8e6}.nvd3 .nv-bars text{fill:transparent}.nvd3 .nv-bars .hover text{fill:rgba(0,0,0,1)}.nvd3 .nv-discretebar .nv-groups rect,.nvd3 .nv-multibar .nv-groups rect,.nvd3 .nv-multibarHorizontal .nv-groups rect{stroke-opacity:0;transition:fill-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear}.nvd3 .nv-candlestickBar .nv-ticks rect:hover,.nvd3 .nv-discretebar .nv-groups rect:hover,.nvd3 .nv-multibar .nv-groups rect:hover,.nvd3 .nv-multibarHorizontal .nv-groups rect:hover{fill-opacity:1}.nvd3 .nv-discretebar .nv-groups text,.nvd3 .nv-multibarHorizontal .nv-groups text{font-weight:700;fill:rgba(0,0,0,1);stroke:transparent}.nvd3 .nv-boxplot circle{fill-opacity:.5}.nvd3 .nv-boxplot circle:hover,.nvd3 .nv-boxplot rect:hover{fill-opacity:1}.nvd3 line.nv-boxplot-median{stroke:#000}.nv-boxplot-tick:hover{stroke-width:2.5px}.nvd3.nv-bullet{font:10px sans-serif}.nvd3.nv-bullet .nv-measure{fill-opacity:.8}.nvd3.nv-bullet .nv-measure:hover{fill-opacity:1}.nvd3.nv-bullet .nv-marker{stroke:#000;stroke-width:2px}.nvd3.nv-bullet .nv-markerTriangle{stroke:#000;fill:#fff;stroke-width:1.5px}.nvd3.nv-bullet .nv-markerLine{stroke:#000;stroke-width:1.5px}.nvd3.nv-bullet .nv-tick line{stroke:#666;stroke-width:.5px}.nvd3.nv-bullet .nv-range.nv-s0{fill:#eee}.nvd3.nv-bullet .nv-range.nv-s1{fill:#ddd}.nvd3.nv-bullet .nv-range.nv-s2{fill:#ccc}.nvd3.nv-bullet .nv-title{font-size:14px;font-weight:700}.nvd3.nv-bullet .nv-subtitle{fill:#999}.nvd3.nv-bullet .nv-range{fill:#bababa;fill-opacity:.4}.nvd3.nv-bullet .nv-range:hover{fill-opacity:.7}.nvd3.nv-candlestickBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.positive rect{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.negative rect{stroke:#d62728;fill:#d62728}.with-transitions .nv-candlestickBar .nv-ticks .nv-tick{transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-moz-transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-webkit-transition:stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-candlestickBar .nv-ticks line{stroke:#333}.nv-force-node{stroke:#fff;stroke-width:1.5px}.nv-force-link{stroke:#999;stroke-opacity:.6}.nv-force-node text{stroke-width:0}.nvd3 .nv-check-box .nv-box{fill-opacity:0;stroke-width:2}.nvd3 .nv-check-box .nv-check{fill-opacity:0;stroke-width:4}.nvd3 .nv-series.nv-disabled .nv-check-box .nv-check{fill-opacity:0;stroke-opacity:0}.nvd3.nv-linePlusBar .nv-bar rect{fill-opacity:.75}.nvd3.nv-linePlusBar .nv-bar rect:hover{fill-opacity:1}.nvd3 .nv-groups path.nv-line{fill:none}.nvd3 .nv-groups path.nv-area{stroke:none}.nvd3.nv-line .nvd3.nv-scatter .nv-groups .nv-point{fill-opacity:0;stroke-opacity:0}.nvd3.nv-scatter.nv-single-point .nv-groups .nv-point{fill-opacity:.5!important;stroke-opacity:.5!important}.with-transitions .nvd3 .nv-groups .nv-point{transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-moz-transition:stroke-width 250ms linear,stroke-opacity 250ms linear;-webkit-transition:stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3 .nv-groups .nv-point.hover,.nvd3.nv-scatter .nv-groups .nv-point.hover{stroke-width:7px;fill-opacity:.95!important;stroke-opacity:.95!important}.nvd3 .nv-point-paths path{stroke:#aaa;stroke-opacity:0;fill:#eee;fill-opacity:0}.nvd3 .nv-indexLine{cursor:ew-resize}svg.nvd3-svg{-webkit-user-select:none;-ms-user-select:none;-moz-user-select:none;user-select:none;width:100%;height:100%}.nvtooltip.with-3d-shadow,.with-3d-shadow .nvtooltip{-moz-box-shadow:0 5px 10px rgba(0,0,0,.2);-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2);-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nvd3 text{font:400 12px Arial}.nvd3 .title{font:700 14px Arial}.nvd3 .nv-background{fill:#fff;fill-opacity:0}.nvd3.nv-noData{font-size:18px;font-weight:700}.nv-brush .extent{fill-opacity:.125}.nv-brush .resize path{fill:#eee;stroke:#666}.nvd3 .nv-legend .nv-series{cursor:pointer}.nvd3 .nv-legend .nv-disabled circle{fill-opacity:0}.nvd3 .nv-brush .extent{fill-opacity:0!important}.nvd3 .nv-brushBackground rect{stroke:#000;stroke-width:.4;fill:#fff;fill-opacity:.7}@media print{.nvd3 text{stroke-width:0;fill-opacity:1}}.nvd3.nv-ohlcBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.positive{stroke:#2ca02c}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.negative{stroke:#d62728}.nvd3 .background path{fill:none;stroke:#EEE;stroke-opacity:.4}.nvd3 .foreground path{fill:none;stroke-opacity:.7}.nvd3 .nv-parallelCoordinates-brush .extent{fill:#fff;fill-opacity:.6;stroke:gray;shape-rendering:crispEdges}.nvd3 .nv-parallelCoordinates .hover{fill-opacity:1;stroke-width:3px}.nvd3 .missingValuesline line{fill:none;stroke:#000;stroke-width:1;stroke-opacity:1;stroke-dasharray:5,5}.nvd3.nv-pie .nv-pie-title{font-size:24px;fill:rgba(19,196,249,.59)}.nvd3.nv-pie .nv-slice text{stroke:#000;stroke-width:0}.nvd3.nv-pie path{transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;stroke:#fff;stroke-width:1px;stroke-opacity:1;fill-opacity:.7}.nvd3.nv-pie .hover path{fill-opacity:1}.nvd3.nv-pie .nv-label rect{fill-opacity:0;stroke-opacity:0}.nvd3 .nv-groups .nv-point.hover{stroke-width:20px;stroke-opacity:.5}.nvd3 .nv-scatter .nv-point.hover{fill-opacity:1}.nvd3.nv-sparkline path{fill:none}.nvd3.nv-sparklineplus .nv-hoverValue line{stroke:#333;stroke-width:1.5px}.nvd3.nv-sparklineplus,.nvd3.nv-sparklineplus g{pointer-events:all}.nvd3 .nv-interactiveGuideLine,.nvtooltip{pointer-events:none}.nvd3 .nv-hoverArea{fill-opacity:0;stroke-opacity:0}.nvd3.nv-sparklineplus .nv-xValue,.nvd3.nv-sparklineplus .nv-yValue{stroke-width:0;font-size:.9em;font-weight:400}.nvd3.nv-sparklineplus .nv-yValue{stroke:#f66}.nvd3.nv-sparklineplus .nv-maxValue{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-sparklineplus .nv-minValue{stroke:#d62728;fill:#d62728}.nvd3.nv-sparklineplus .nv-currentValue{font-weight:700;font-size:1.1em}.nvtooltip h3,.nvtooltip table td.key{font-weight:400}.nvd3.nv-stackedarea path.nv-area{fill-opacity:.7;stroke-opacity:0;transition:fill-opacity 250ms linear,stroke-opacity 250ms linear;-moz-transition:fill-opacity 250ms linear,stroke-opacity 250ms linear;-webkit-transition:fill-opacity 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-stackedarea path.nv-area.hover{fill-opacity:.9}.nvd3.nv-stackedarea .nv-groups .nv-point{stroke-opacity:0;fill-opacity:0}.nvtooltip{position:absolute;color:rgba(0,0,0,1);padding:1px;z-index:10000;font-family:Arial;font-size:13px;text-align:left;white-space:nowrap;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background:rgba(255,255,255,.8);border:1px solid rgba(0,0,0,.5);border-radius:4px}.nvtooltip h3,.nvtooltip p{margin:0;text-align:center}.nvtooltip.with-transitions,.with-transitions .nvtooltip{transition:opacity 50ms linear;-moz-transition:opacity 50ms linear;-webkit-transition:opacity 50ms linear;transition-delay:200ms;-moz-transition-delay:200ms;-webkit-transition-delay:200ms}.nvtooltip.x-nvtooltip,.nvtooltip.y-nvtooltip{padding:8px}.nvtooltip h3{padding:4px 14px;line-height:18px;background-color:rgba(247,247,247,.75);color:rgba(0,0,0,1);border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0}.nvtooltip p{padding:5px 14px}.nvtooltip span{display:inline-block;margin:2px 0}.nvtooltip table{margin:6px;border-spacing:0}.nvtooltip table td{padding:2px 9px 2px 0;vertical-align:middle}.nvtooltip table td.key.total{font-weight:700}.nvtooltip table td.value{text-align:right;font-weight:700}.nvtooltip table td.percent{color:#a9a9a9}.nvtooltip table tr.highlight td{padding:1px 9px 1px 0;border-bottom-style:solid;border-bottom-width:1px;border-top-style:solid;border-top-width:1px}.nvtooltip table td.legend-color-guide div{vertical-align:middle;width:12px;height:12px;border:1px solid #999}.nvtooltip .footer{padding:3px;text-align:center}.nvtooltip-pending-removal{pointer-events:none;display:none}.nvd3 line.nv-guideline{stroke:#ccc}</style> + <style> + * { + box-sizing: border-box; + } + html, body { + position: relative; + height: 100%; + width: 100%; + border: 0; + margin: 0; + padding: 0; + font-size: 0; + } + svg g { + clip-path: none; + } + svg text { + fill: #333; + } + .nv-axislabel { + font-size: 16px !important; + } + .control { + cursor: pointer; + visibility: hidden; + pointer-events: visible; + } + @media (min-width: 768px) { + .control { + visibility: visible; + } + } + </style> + <script src="s1.js"></script> + <script src="s2.js"></script> + </head> + <body> + <svg id='chart'></svg> + <script type='text/javascript'> + \ No newline at end of file
diff --git a/bench/perf_monitoring/resources/footer.html b/bench/perf_monitoring/resources/footer.html new file mode 100644 index 0000000..81d8c88 --- /dev/null +++ b/bench/perf_monitoring/resources/footer.html
@@ -0,0 +1,3 @@ +</table> +</body> +</html>
diff --git a/bench/perf_monitoring/resources/header.html b/bench/perf_monitoring/resources/header.html new file mode 100644 index 0000000..1f22309 --- /dev/null +++ b/bench/perf_monitoring/resources/header.html
@@ -0,0 +1,42 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<title>Eigen performance monitoring</title> +<style type="text/css"> + +body +{ + background:#fff; +} +th { + +} +img +{ + width:auto; + box-shadow:0px 0px 20px #cecece; + margin: 20px 20px 20px 20px; + -moz-transform: scale(1); + -moz-transition-duration: 0.4s; + -webkit-transition-duration: 0.4s; + -webkit-transform: scale(1); + + -ms-transform: scale(1); + -ms-transition-duration: 0.4s; +} +img:hover +{ +box-shadow: 5px 5px 20px #dcdcdc; + -moz-transform: scale(1.1); + -moz-transition-duration: 0.4s; + -webkit-transition-duration: 0.4s; + -webkit-transform: scale(1.1); + + -ms-transform: scale(1.1); + -ms-transition-duration: 0.4s; + +} +</style> +</head> +<body>
diff --git a/bench/perf_monitoring/resources/s1.js b/bench/perf_monitoring/resources/s1.js new file mode 100644 index 0000000..cfff2d3 --- /dev/null +++ b/bench/perf_monitoring/resources/s1.js
@@ -0,0 +1 @@ +!function(){function n(n){return n&&(n.ownerDocument||n.document||n).documentElement}function t(n){return n&&(n.ownerDocument&&n.ownerDocument.defaultView||n.document&&n||n.defaultView)}function e(n,t){return t>n?-1:n>t?1:n>=t?0:NaN}function r(n){return null===n?NaN:+n}function i(n){return!isNaN(n)}function u(n){return{left:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)<0?r=u+1:i=u}return r},right:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)>0?i=u:r=u+1}return r}}}function o(n){return n.length}function a(n){for(var t=1;n*t%1;)t*=10;return t}function l(n,t){for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}function c(){this._=Object.create(null)}function f(n){return(n+="")===bo||n[0]===_o?_o+n:n}function s(n){return(n+="")[0]===_o?n.slice(1):n}function h(n){return f(n)in this._}function p(n){return(n=f(n))in this._&&delete this._[n]}function g(){var n=[];for(var t in this._)n.push(s(t));return n}function v(){var n=0;for(var t in this._)++n;return n}function d(){for(var n in this._)return!1;return!0}function y(){this._=Object.create(null)}function m(n){return n}function M(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function x(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.slice(1);for(var e=0,r=wo.length;r>e;++e){var i=wo[e]+t;if(i in n)return i}}function b(){}function _(){}function w(n){function t(){for(var t,r=e,i=-1,u=r.length;++i<u;)(t=r[i].on)&&t.apply(this,arguments);return n}var e=[],r=new c;return t.on=function(t,i){var u,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,u=e.indexOf(o)).concat(e.slice(u+1)),r.remove(t)),i&&e.push(r.set(t,{on:i})),n)},t}function S(){ao.event.preventDefault()}function k(){for(var n,t=ao.event;n=t.sourceEvent;)t=n;return t}function N(n){for(var t=new _,e=0,r=arguments.length;++e<r;)t[arguments[e]]=w(t);return t.of=function(e,r){return function(i){try{var u=i.sourceEvent=ao.event;i.target=n,ao.event=i,t[i.type].apply(e,r)}finally{ao.event=u}}},t}function E(n){return ko(n,Co),n}function A(n){return"function"==typeof n?n:function(){return No(n,this)}}function C(n){return"function"==typeof n?n:function(){return Eo(n,this)}}function z(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function i(){this.setAttribute(n,t)}function u(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=ao.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?u:i}function L(n){return n.trim().replace(/\s+/g," ")}function q(n){return new RegExp("(?:^|\\s+)"+ao.requote(n)+"(?:\\s+|$)","g")}function T(n){return(n+"").trim().split(/^|\s+/)}function R(n,t){function e(){for(var e=-1;++e<i;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<i;)n[e](this,r)}n=T(n).map(D);var i=n.length;return"function"==typeof t?r:e}function D(n){var t=q(n);return function(e,r){if(i=e.classList)return r?i.add(n):i.remove(n);var i=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(i)||e.setAttribute("class",L(i+" "+n))):e.setAttribute("class",L(i.replace(t," ")))}}function P(n,t,e){function r(){this.style.removeProperty(n)}function i(){this.style.setProperty(n,t,e)}function u(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?u:i}function U(n,t){function e(){delete this[n]}function r(){this[n]=t}function i(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?i:r}function j(n){function t(){var t=this.ownerDocument,e=this.namespaceURI;return e===zo&&t.documentElement.namespaceURI===zo?t.createElement(n):t.createElementNS(e,n)}function e(){return this.ownerDocument.createElementNS(n.space,n.local)}return"function"==typeof n?n:(n=ao.ns.qualify(n)).local?e:t}function F(){var n=this.parentNode;n&&n.removeChild(this)}function H(n){return{__data__:n}}function O(n){return function(){return Ao(this,n)}}function I(n){return arguments.length||(n=e),function(t,e){return t&&e?n(t.__data__,e.__data__):!t-!e}}function Y(n,t){for(var e=0,r=n.length;r>e;e++)for(var i,u=n[e],o=0,a=u.length;a>o;o++)(i=u[o])&&t(i,o,e);return n}function Z(n){return ko(n,qo),n}function V(n){var t,e;return function(r,i,u){var o,a=n[u].update,l=a.length;for(u!=e&&(e=u,t=0),i>=t&&(t=i+1);!(o=a[t])&&++t<l;);return o}}function X(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function i(){var i=l(t,co(arguments));r.call(this),this.addEventListener(n,this[o]=i,i.$=e),i._=t}function u(){var t,e=new RegExp("^__on([^.]+)"+ao.requote(n)+"$");for(var r in this)if(t=r.match(e)){var i=this[r];this.removeEventListener(t[1],i,i.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),l=$;a>0&&(n=n.slice(0,a));var c=To.get(n);return c&&(n=c,l=B),a?t?i:r:t?b:u}function $(n,t){return function(e){var r=ao.event;ao.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{ao.event=r}}}function B(n,t){var e=$(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function W(e){var r=".dragsuppress-"+ ++Do,i="click"+r,u=ao.select(t(e)).on("touchmove"+r,S).on("dragstart"+r,S).on("selectstart"+r,S);if(null==Ro&&(Ro="onselectstart"in e?!1:x(e.style,"userSelect")),Ro){var o=n(e).style,a=o[Ro];o[Ro]="none"}return function(n){if(u.on(r,null),Ro&&(o[Ro]=a),n){var t=function(){u.on(i,null)};u.on(i,function(){S(),t()},!0),setTimeout(t,0)}}}function J(n,e){e.changedTouches&&(e=e.changedTouches[0]);var r=n.ownerSVGElement||n;if(r.createSVGPoint){var i=r.createSVGPoint();if(0>Po){var u=t(n);if(u.scrollX||u.scrollY){r=ao.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var o=r[0][0].getScreenCTM();Po=!(o.f||o.e),r.remove()}}return Po?(i.x=e.pageX,i.y=e.pageY):(i.x=e.clientX,i.y=e.clientY),i=i.matrixTransform(n.getScreenCTM().inverse()),[i.x,i.y]}var a=n.getBoundingClientRect();return[e.clientX-a.left-n.clientLeft,e.clientY-a.top-n.clientTop]}function G(){return ao.event.changedTouches[0].identifier}function K(n){return n>0?1:0>n?-1:0}function Q(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function nn(n){return n>1?0:-1>n?Fo:Math.acos(n)}function tn(n){return n>1?Io:-1>n?-Io:Math.asin(n)}function en(n){return((n=Math.exp(n))-1/n)/2}function rn(n){return((n=Math.exp(n))+1/n)/2}function un(n){return((n=Math.exp(2*n))-1)/(n+1)}function on(n){return(n=Math.sin(n/2))*n}function an(){}function ln(n,t,e){return this instanceof ln?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof ln?new ln(n.h,n.s,n.l):_n(""+n,wn,ln):new ln(n,t,e)}function cn(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?u+(o-u)*n/60:180>n?o:240>n?u+(o-u)*(240-n)/60:u}function i(n){return Math.round(255*r(n))}var u,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,u=2*e-o,new mn(i(n+120),i(n),i(n-120))}function fn(n,t,e){return this instanceof fn?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof fn?new fn(n.h,n.c,n.l):n instanceof hn?gn(n.l,n.a,n.b):gn((n=Sn((n=ao.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new fn(n,t,e)}function sn(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new hn(e,Math.cos(n*=Yo)*t,Math.sin(n)*t)}function hn(n,t,e){return this instanceof hn?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof hn?new hn(n.l,n.a,n.b):n instanceof fn?sn(n.h,n.c,n.l):Sn((n=mn(n)).r,n.g,n.b):new hn(n,t,e)}function pn(n,t,e){var r=(n+16)/116,i=r+t/500,u=r-e/200;return i=vn(i)*na,r=vn(r)*ta,u=vn(u)*ea,new mn(yn(3.2404542*i-1.5371385*r-.4985314*u),yn(-.969266*i+1.8760108*r+.041556*u),yn(.0556434*i-.2040259*r+1.0572252*u))}function gn(n,t,e){return n>0?new fn(Math.atan2(e,t)*Zo,Math.sqrt(t*t+e*e),n):new fn(NaN,NaN,n)}function vn(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function dn(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function yn(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function mn(n,t,e){return this instanceof mn?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof mn?new mn(n.r,n.g,n.b):_n(""+n,mn,cn):new mn(n,t,e)}function Mn(n){return new mn(n>>16,n>>8&255,255&n)}function xn(n){return Mn(n)+""}function bn(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function _n(n,t,e){var r,i,u,o=0,a=0,l=0;if(r=/([a-z]+)\((.*)\)/.exec(n=n.toLowerCase()))switch(i=r[2].split(","),r[1]){case"hsl":return e(parseFloat(i[0]),parseFloat(i[1])/100,parseFloat(i[2])/100);case"rgb":return t(Nn(i[0]),Nn(i[1]),Nn(i[2]))}return(u=ua.get(n))?t(u.r,u.g,u.b):(null==n||"#"!==n.charAt(0)||isNaN(u=parseInt(n.slice(1),16))||(4===n.length?(o=(3840&u)>>4,o=o>>4|o,a=240&u,a=a>>4|a,l=15&u,l=l<<4|l):7===n.length&&(o=(16711680&u)>>16,a=(65280&u)>>8,l=255&u)),t(o,a,l))}function wn(n,t,e){var r,i,u=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-u,l=(o+u)/2;return a?(i=.5>l?a/(o+u):a/(2-o-u),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=NaN,i=l>0&&1>l?0:r),new ln(r,i,l)}function Sn(n,t,e){n=kn(n),t=kn(t),e=kn(e);var r=dn((.4124564*n+.3575761*t+.1804375*e)/na),i=dn((.2126729*n+.7151522*t+.072175*e)/ta),u=dn((.0193339*n+.119192*t+.9503041*e)/ea);return hn(116*i-16,500*(r-i),200*(i-u))}function kn(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function Nn(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function En(n){return"function"==typeof n?n:function(){return n}}function An(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),Cn(t,e,n,r)}}function Cn(n,t,e,r){function i(){var n,t=l.status;if(!t&&Ln(l)||t>=200&&300>t||304===t){try{n=e.call(u,l)}catch(r){return void o.error.call(u,r)}o.load.call(u,n)}else o.error.call(u,l)}var u={},o=ao.dispatch("beforesend","progress","load","error"),a={},l=new XMLHttpRequest,c=null;return!this.XDomainRequest||"withCredentials"in l||!/^(http(s)?:)?\/\//.test(n)||(l=new XDomainRequest),"onload"in l?l.onload=l.onerror=i:l.onreadystatechange=function(){l.readyState>3&&i()},l.onprogress=function(n){var t=ao.event;ao.event=n;try{o.progress.call(u,l)}finally{ao.event=t}},u.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",u)},u.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",u):t},u.responseType=function(n){return arguments.length?(c=n,u):c},u.response=function(n){return e=n,u},["get","post"].forEach(function(n){u[n]=function(){return u.send.apply(u,[n].concat(co(arguments)))}}),u.send=function(e,r,i){if(2===arguments.length&&"function"==typeof r&&(i=r,r=null),l.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),l.setRequestHeader)for(var f in a)l.setRequestHeader(f,a[f]);return null!=t&&l.overrideMimeType&&l.overrideMimeType(t),null!=c&&(l.responseType=c),null!=i&&u.on("error",i).on("load",function(n){i(null,n)}),o.beforesend.call(u,l),l.send(null==r?null:r),u},u.abort=function(){return l.abort(),u},ao.rebind(u,o,"on"),null==r?u:u.get(zn(r))}function zn(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function Ln(n){var t=n.responseType;return t&&"text"!==t?n.response:n.responseText}function qn(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var i=e+t,u={c:n,t:i,n:null};return aa?aa.n=u:oa=u,aa=u,la||(ca=clearTimeout(ca),la=1,fa(Tn)),u}function Tn(){var n=Rn(),t=Dn()-n;t>24?(isFinite(t)&&(clearTimeout(ca),ca=setTimeout(Tn,t)),la=0):(la=1,fa(Tn))}function Rn(){for(var n=Date.now(),t=oa;t;)n>=t.t&&t.c(n-t.t)&&(t.c=null),t=t.n;return n}function Dn(){for(var n,t=oa,e=1/0;t;)t.c?(t.t<e&&(e=t.t),t=(n=t).n):t=n?n.n=t.n:oa=t.n;return aa=n,e}function Pn(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function Un(n,t){var e=Math.pow(10,3*xo(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function jn(n){var t=n.decimal,e=n.thousands,r=n.grouping,i=n.currency,u=r&&e?function(n,t){for(var i=n.length,u=[],o=0,a=r[0],l=0;i>0&&a>0&&(l+a+1>t&&(a=Math.max(1,t-l)),u.push(n.substring(i-=a,i+a)),!((l+=a+1)>t));)a=r[o=(o+1)%r.length];return u.reverse().join(e)}:m;return function(n){var e=ha.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"-",l=e[4]||"",c=e[5],f=+e[6],s=e[7],h=e[8],p=e[9],g=1,v="",d="",y=!1,m=!0;switch(h&&(h=+h.substring(1)),(c||"0"===r&&"="===o)&&(c=r="0",o="="),p){case"n":s=!0,p="g";break;case"%":g=100,d="%",p="f";break;case"p":g=100,d="%",p="r";break;case"b":case"o":case"x":case"X":"#"===l&&(v="0"+p.toLowerCase());case"c":m=!1;case"d":y=!0,h=0;break;case"s":g=-1,p="r"}"$"===l&&(v=i[0],d=i[1]),"r"!=p||h||(p="g"),null!=h&&("g"==p?h=Math.max(1,Math.min(21,h)):"e"!=p&&"f"!=p||(h=Math.max(0,Math.min(20,h)))),p=pa.get(p)||Fn;var M=c&&s;return function(n){var e=d;if(y&&n%1)return"";var i=0>n||0===n&&0>1/n?(n=-n,"-"):"-"===a?"":a;if(0>g){var l=ao.formatPrefix(n,h);n=l.scale(n),e=l.symbol+d}else n*=g;n=p(n,h);var x,b,_=n.lastIndexOf(".");if(0>_){var w=m?n.lastIndexOf("e"):-1;0>w?(x=n,b=""):(x=n.substring(0,w),b=n.substring(w))}else x=n.substring(0,_),b=t+n.substring(_+1);!c&&s&&(x=u(x,1/0));var S=v.length+x.length+b.length+(M?0:i.length),k=f>S?new Array(S=f-S+1).join(r):"";return M&&(x=u(k+x,k.length?f-b.length:1/0)),i+=v,n=x+b,("<"===o?i+n+k:">"===o?k+i+n:"^"===o?k.substring(0,S>>=1)+i+n+k.substring(S):i+(M?n:k+n))+e}}}function Fn(n){return n+""}function Hn(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function On(n,t,e){function r(t){var e=n(t),r=u(e,1);return r-t>t-e?e:r}function i(e){return t(e=n(new va(e-1)),1),e}function u(n,e){return t(n=new va(+n),e),n}function o(n,r,u){var o=i(n),a=[];if(u>1)for(;r>o;)e(o)%u||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{va=Hn;var r=new Hn;return r._=n,o(r,t,e)}finally{va=Date}}n.floor=n,n.round=r,n.ceil=i,n.offset=u,n.range=o;var l=n.utc=In(n);return l.floor=l,l.round=In(r),l.ceil=In(i),l.offset=In(u),l.range=a,n}function In(n){return function(t,e){try{va=Hn;var r=new Hn;return r._=t,n(r,e)._}finally{va=Date}}}function Yn(n){function t(n){function t(t){for(var e,i,u,o=[],a=-1,l=0;++a<r;)37===n.charCodeAt(a)&&(o.push(n.slice(l,a)),null!=(i=ya[e=n.charAt(++a)])&&(e=n.charAt(++a)),(u=A[e])&&(e=u(t,null==i?"e"===e?" ":"0":i)),o.push(e),l=a+1);return o.push(n.slice(l,a)),o.join("")}var r=n.length;return t.parse=function(t){var r={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},i=e(r,n,t,0);if(i!=t.length)return null;"p"in r&&(r.H=r.H%12+12*r.p);var u=null!=r.Z&&va!==Hn,o=new(u?Hn:va);return"j"in r?o.setFullYear(r.y,0,r.j):"W"in r||"U"in r?("w"in r||(r.w="W"in r?1:0),o.setFullYear(r.y,0,1),o.setFullYear(r.y,0,"W"in r?(r.w+6)%7+7*r.W-(o.getDay()+5)%7:r.w+7*r.U-(o.getDay()+6)%7)):o.setFullYear(r.y,r.m,r.d),o.setHours(r.H+(r.Z/100|0),r.M+r.Z%100,r.S,r.L),u?o._:o},t.toString=function(){return n},t}function e(n,t,e,r){for(var i,u,o,a=0,l=t.length,c=e.length;l>a;){if(r>=c)return-1;if(i=t.charCodeAt(a++),37===i){if(o=t.charAt(a++),u=C[o in ya?t.charAt(a++):o],!u||(r=u(n,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){_.lastIndex=0;var r=_.exec(t.slice(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){x.lastIndex=0;var r=x.exec(t.slice(e));return r?(n.w=b.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){N.lastIndex=0;var r=N.exec(t.slice(e));return r?(n.m=E.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.slice(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,A.c.toString(),t,r)}function l(n,t,r){return e(n,A.x.toString(),t,r)}function c(n,t,r){return e(n,A.X.toString(),t,r)}function f(n,t,e){var r=M.get(t.slice(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var s=n.dateTime,h=n.date,p=n.time,g=n.periods,v=n.days,d=n.shortDays,y=n.months,m=n.shortMonths;t.utc=function(n){function e(n){try{va=Hn;var t=new va;return t._=n,r(t)}finally{va=Date}}var r=t(n);return e.parse=function(n){try{va=Hn;var t=r.parse(n);return t&&t._}finally{va=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=ct;var M=ao.map(),x=Vn(v),b=Xn(v),_=Vn(d),w=Xn(d),S=Vn(y),k=Xn(y),N=Vn(m),E=Xn(m);g.forEach(function(n,t){M.set(n.toLowerCase(),t)});var A={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return m[n.getMonth()]},B:function(n){return y[n.getMonth()]},c:t(s),d:function(n,t){return Zn(n.getDate(),t,2)},e:function(n,t){return Zn(n.getDate(),t,2)},H:function(n,t){return Zn(n.getHours(),t,2)},I:function(n,t){return Zn(n.getHours()%12||12,t,2)},j:function(n,t){return Zn(1+ga.dayOfYear(n),t,3)},L:function(n,t){return Zn(n.getMilliseconds(),t,3)},m:function(n,t){return Zn(n.getMonth()+1,t,2)},M:function(n,t){return Zn(n.getMinutes(),t,2)},p:function(n){return g[+(n.getHours()>=12)]},S:function(n,t){return Zn(n.getSeconds(),t,2)},U:function(n,t){return Zn(ga.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return Zn(ga.mondayOfYear(n),t,2)},x:t(h),X:t(p),y:function(n,t){return Zn(n.getFullYear()%100,t,2)},Y:function(n,t){return Zn(n.getFullYear()%1e4,t,4)},Z:at,"%":function(){return"%"}},C={a:r,A:i,b:u,B:o,c:a,d:tt,e:tt,H:rt,I:rt,j:et,L:ot,m:nt,M:it,p:f,S:ut,U:Bn,w:$n,W:Wn,x:l,X:c,y:Gn,Y:Jn,Z:Kn,"%":lt};return t}function Zn(n,t,e){var r=0>n?"-":"",i=(r?-n:n)+"",u=i.length;return r+(e>u?new Array(e-u+1).join(t)+i:i)}function Vn(n){return new RegExp("^(?:"+n.map(ao.requote).join("|")+")","i")}function Xn(n){for(var t=new c,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function $n(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function Bn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.U=+r[0],e+r[0].length):-1}function Wn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.W=+r[0],e+r[0].length):-1}function Jn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Gn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.y=Qn(+r[0]),e+r[0].length):-1}function Kn(n,t,e){return/^[+-]\d{4}$/.test(t=t.slice(e,e+5))?(n.Z=-t,e+5):-1}function Qn(n){return n+(n>68?1900:2e3)}function nt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function tt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function et(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function rt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function it(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function ut(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ot(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function at(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=xo(t)/60|0,i=xo(t)%60;return e+Zn(r,"0",2)+Zn(i,"0",2)}function lt(n,t,e){Ma.lastIndex=0;var r=Ma.exec(t.slice(e,e+1));return r?e+r[0].length:-1}function ct(n){for(var t=n.length,e=-1;++e<t;)n[e][0]=this(n[e][0]);return function(t){for(var e=0,r=n[e];!r[1](t);)r=n[++e];return r[0](t)}}function ft(){}function st(n,t,e){var r=e.s=n+t,i=r-n,u=r-i;e.t=n-u+(t-i)}function ht(n,t){n&&wa.hasOwnProperty(n.type)&&wa[n.type](n,t)}function pt(n,t,e){var r,i=-1,u=n.length-e;for(t.lineStart();++i<u;)r=n[i],t.point(r[0],r[1],r[2]);t.lineEnd()}function gt(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)pt(n[e],t,1);t.polygonEnd()}function vt(){function n(n,t){n*=Yo,t=t*Yo/2+Fo/4;var e=n-r,o=e>=0?1:-1,a=o*e,l=Math.cos(t),c=Math.sin(t),f=u*c,s=i*l+f*Math.cos(a),h=f*o*Math.sin(a);ka.add(Math.atan2(h,s)),r=n,i=l,u=c}var t,e,r,i,u;Na.point=function(o,a){Na.point=n,r=(t=o)*Yo,i=Math.cos(a=(e=a)*Yo/2+Fo/4),u=Math.sin(a)},Na.lineEnd=function(){n(t,e)}}function dt(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function yt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function mt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Mt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function xt(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function bt(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function _t(n){return[Math.atan2(n[1],n[0]),tn(n[2])]}function wt(n,t){return xo(n[0]-t[0])<Uo&&xo(n[1]-t[1])<Uo}function St(n,t){n*=Yo;var e=Math.cos(t*=Yo);kt(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function kt(n,t,e){++Ea,Ca+=(n-Ca)/Ea,za+=(t-za)/Ea,La+=(e-La)/Ea}function Nt(){function n(n,i){n*=Yo;var u=Math.cos(i*=Yo),o=u*Math.cos(n),a=u*Math.sin(n),l=Math.sin(i),c=Math.atan2(Math.sqrt((c=e*l-r*a)*c+(c=r*o-t*l)*c+(c=t*a-e*o)*c),t*o+e*a+r*l);Aa+=c,qa+=c*(t+(t=o)),Ta+=c*(e+(e=a)),Ra+=c*(r+(r=l)),kt(t,e,r)}var t,e,r;ja.point=function(i,u){i*=Yo;var o=Math.cos(u*=Yo);t=o*Math.cos(i),e=o*Math.sin(i),r=Math.sin(u),ja.point=n,kt(t,e,r)}}function Et(){ja.point=St}function At(){function n(n,t){n*=Yo;var e=Math.cos(t*=Yo),o=e*Math.cos(n),a=e*Math.sin(n),l=Math.sin(t),c=i*l-u*a,f=u*o-r*l,s=r*a-i*o,h=Math.sqrt(c*c+f*f+s*s),p=r*o+i*a+u*l,g=h&&-nn(p)/h,v=Math.atan2(h,p);Da+=g*c,Pa+=g*f,Ua+=g*s,Aa+=v,qa+=v*(r+(r=o)),Ta+=v*(i+(i=a)),Ra+=v*(u+(u=l)),kt(r,i,u)}var t,e,r,i,u;ja.point=function(o,a){t=o,e=a,ja.point=n,o*=Yo;var l=Math.cos(a*=Yo);r=l*Math.cos(o),i=l*Math.sin(o),u=Math.sin(a),kt(r,i,u)},ja.lineEnd=function(){n(t,e),ja.lineEnd=Et,ja.point=St}}function Ct(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.invert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function zt(){return!0}function Lt(n,t,e,r,i){var u=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(wt(e,r)){i.lineStart();for(var a=0;t>a;++a)i.point((e=n[a])[0],e[1]);return void i.lineEnd()}var l=new Tt(e,n,null,!0),c=new Tt(e,null,l,!1);l.o=c,u.push(l),o.push(c),l=new Tt(r,n,null,!1),c=new Tt(r,null,l,!0),l.o=c,u.push(l),o.push(c)}}),o.sort(t),qt(u),qt(o),u.length){for(var a=0,l=e,c=o.length;c>a;++a)o[a].e=l=!l;for(var f,s,h=u[0];;){for(var p=h,g=!0;p.v;)if((p=p.n)===h)return;f=p.z,i.lineStart();do{if(p.v=p.o.v=!0,p.e){if(g)for(var a=0,c=f.length;c>a;++a)i.point((s=f[a])[0],s[1]);else r(p.x,p.n.x,1,i);p=p.n}else{if(g){f=p.p.z;for(var a=f.length-1;a>=0;--a)i.point((s=f[a])[0],s[1])}else r(p.x,p.p.x,-1,i);p=p.p}p=p.o,f=p.z,g=!g}while(!p.v);i.lineEnd()}}}function qt(n){if(t=n.length){for(var t,e,r=0,i=n[0];++r<t;)i.n=e=n[r],e.p=i,i=e;i.n=e=n[0],e.p=i}}function Tt(n,t,e,r){this.x=n,this.z=t,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function Rt(n,t,e,r){return function(i,u){function o(t,e){var r=i(t,e);n(t=r[0],e=r[1])&&u.point(t,e)}function a(n,t){var e=i(n,t);d.point(e[0],e[1])}function l(){m.point=a,d.lineStart()}function c(){m.point=o,d.lineEnd()}function f(n,t){v.push([n,t]);var e=i(n,t);x.point(e[0],e[1])}function s(){x.lineStart(),v=[]}function h(){f(v[0][0],v[0][1]),x.lineEnd();var n,t=x.clean(),e=M.buffer(),r=e.length;if(v.pop(),g.push(v),v=null,r)if(1&t){n=e[0];var i,r=n.length-1,o=-1;if(r>0){for(b||(u.polygonStart(),b=!0),u.lineStart();++o<r;)u.point((i=n[o])[0],i[1]);u.lineEnd()}}else r>1&&2&t&&e.push(e.pop().concat(e.shift())),p.push(e.filter(Dt))}var p,g,v,d=t(u),y=i.invert(r[0],r[1]),m={point:o,lineStart:l,lineEnd:c,polygonStart:function(){m.point=f,m.lineStart=s,m.lineEnd=h,p=[],g=[]},polygonEnd:function(){m.point=o,m.lineStart=l,m.lineEnd=c,p=ao.merge(p);var n=Ot(y,g);p.length?(b||(u.polygonStart(),b=!0),Lt(p,Ut,n,e,u)):n&&(b||(u.polygonStart(),b=!0),u.lineStart(),e(null,null,1,u),u.lineEnd()),b&&(u.polygonEnd(),b=!1),p=g=null},sphere:function(){u.polygonStart(),u.lineStart(),e(null,null,1,u),u.lineEnd(),u.polygonEnd()}},M=Pt(),x=t(M),b=!1;return m}}function Dt(n){return n.length>1}function Pt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:b,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Ut(n,t){return((n=n.x)[0]<0?n[1]-Io-Uo:Io-n[1])-((t=t.x)[0]<0?t[1]-Io-Uo:Io-t[1])}function jt(n){var t,e=NaN,r=NaN,i=NaN;return{lineStart:function(){n.lineStart(),t=1},point:function(u,o){var a=u>0?Fo:-Fo,l=xo(u-e);xo(l-Fo)<Uo?(n.point(e,r=(r+o)/2>0?Io:-Io),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(u,r),t=0):i!==a&&l>=Fo&&(xo(e-i)<Uo&&(e-=i*Uo),xo(u-a)<Uo&&(u-=a*Uo),r=Ft(e,r,u,o),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=u,r=o),i=a},lineEnd:function(){n.lineEnd(),e=r=NaN},clean:function(){return 2-t}}}function Ft(n,t,e,r){var i,u,o=Math.sin(n-e);return xo(o)>Uo?Math.atan((Math.sin(t)*(u=Math.cos(r))*Math.sin(e)-Math.sin(r)*(i=Math.cos(t))*Math.sin(n))/(i*u*o)):(t+r)/2}function Ht(n,t,e,r){var i;if(null==n)i=e*Io,r.point(-Fo,i),r.point(0,i),r.point(Fo,i),r.point(Fo,0),r.point(Fo,-i),r.point(0,-i),r.point(-Fo,-i),r.point(-Fo,0),r.point(-Fo,i);else if(xo(n[0]-t[0])>Uo){var u=n[0]<t[0]?Fo:-Fo;i=e*u/2,r.point(-u,i),r.point(0,i),r.point(u,i)}else r.point(t[0],t[1])}function Ot(n,t){var e=n[0],r=n[1],i=[Math.sin(e),-Math.cos(e),0],u=0,o=0;ka.reset();for(var a=0,l=t.length;l>a;++a){var c=t[a],f=c.length;if(f)for(var s=c[0],h=s[0],p=s[1]/2+Fo/4,g=Math.sin(p),v=Math.cos(p),d=1;;){d===f&&(d=0),n=c[d];var y=n[0],m=n[1]/2+Fo/4,M=Math.sin(m),x=Math.cos(m),b=y-h,_=b>=0?1:-1,w=_*b,S=w>Fo,k=g*M;if(ka.add(Math.atan2(k*_*Math.sin(w),v*x+k*Math.cos(w))),u+=S?b+_*Ho:b,S^h>=e^y>=e){var N=mt(dt(s),dt(n));bt(N);var E=mt(i,N);bt(E);var A=(S^b>=0?-1:1)*tn(E[2]);(r>A||r===A&&(N[0]||N[1]))&&(o+=S^b>=0?1:-1)}if(!d++)break;h=y,g=M,v=x,s=n}}return(-Uo>u||Uo>u&&-Uo>ka)^1&o}function It(n){function t(n,t){return Math.cos(n)*Math.cos(t)>u}function e(n){var e,u,l,c,f;return{lineStart:function(){c=l=!1,f=1},point:function(s,h){var p,g=[s,h],v=t(s,h),d=o?v?0:i(s,h):v?i(s+(0>s?Fo:-Fo),h):0;if(!e&&(c=l=v)&&n.lineStart(),v!==l&&(p=r(e,g),(wt(e,p)||wt(g,p))&&(g[0]+=Uo,g[1]+=Uo,v=t(g[0],g[1]))),v!==l)f=0,v?(n.lineStart(),p=r(g,e),n.point(p[0],p[1])):(p=r(e,g),n.point(p[0],p[1]),n.lineEnd()),e=p;else if(a&&e&&o^v){var y;d&u||!(y=r(g,e,!0))||(f=0,o?(n.lineStart(),n.point(y[0][0],y[0][1]),n.point(y[1][0],y[1][1]),n.lineEnd()):(n.point(y[1][0],y[1][1]),n.lineEnd(),n.lineStart(),n.point(y[0][0],y[0][1])))}!v||e&&wt(e,g)||n.point(g[0],g[1]),e=g,l=v,u=d},lineEnd:function(){l&&n.lineEnd(),e=null},clean:function(){return f|(c&&l)<<1}}}function r(n,t,e){var r=dt(n),i=dt(t),o=[1,0,0],a=mt(r,i),l=yt(a,a),c=a[0],f=l-c*c;if(!f)return!e&&n;var s=u*l/f,h=-u*c/f,p=mt(o,a),g=xt(o,s),v=xt(a,h);Mt(g,v);var d=p,y=yt(g,d),m=yt(d,d),M=y*y-m*(yt(g,g)-1);if(!(0>M)){var x=Math.sqrt(M),b=xt(d,(-y-x)/m);if(Mt(b,g),b=_t(b),!e)return b;var _,w=n[0],S=t[0],k=n[1],N=t[1];w>S&&(_=w,w=S,S=_);var E=S-w,A=xo(E-Fo)<Uo,C=A||Uo>E;if(!A&&k>N&&(_=k,k=N,N=_),C?A?k+N>0^b[1]<(xo(b[0]-w)<Uo?k:N):k<=b[1]&&b[1]<=N:E>Fo^(w<=b[0]&&b[0]<=S)){var z=xt(d,(-y+x)/m);return Mt(z,g),[b,_t(z)]}}}function i(t,e){var r=o?n:Fo-n,i=0;return-r>t?i|=1:t>r&&(i|=2),-r>e?i|=4:e>r&&(i|=8),i}var u=Math.cos(n),o=u>0,a=xo(u)>Uo,l=ve(n,6*Yo);return Rt(t,e,l,o?[0,-n]:[-Fo,n-Fo])}function Yt(n,t,e,r){return function(i){var u,o=i.a,a=i.b,l=o.x,c=o.y,f=a.x,s=a.y,h=0,p=1,g=f-l,v=s-c;if(u=n-l,g||!(u>0)){if(u/=g,0>g){if(h>u)return;p>u&&(p=u)}else if(g>0){if(u>p)return;u>h&&(h=u)}if(u=e-l,g||!(0>u)){if(u/=g,0>g){if(u>p)return;u>h&&(h=u)}else if(g>0){if(h>u)return;p>u&&(p=u)}if(u=t-c,v||!(u>0)){if(u/=v,0>v){if(h>u)return;p>u&&(p=u)}else if(v>0){if(u>p)return;u>h&&(h=u)}if(u=r-c,v||!(0>u)){if(u/=v,0>v){if(u>p)return;u>h&&(h=u)}else if(v>0){if(h>u)return;p>u&&(p=u)}return h>0&&(i.a={x:l+h*g,y:c+h*v}),1>p&&(i.b={x:l+p*g,y:c+p*v}),i}}}}}}function Zt(n,t,e,r){function i(r,i){return xo(r[0]-n)<Uo?i>0?0:3:xo(r[0]-e)<Uo?i>0?2:1:xo(r[1]-t)<Uo?i>0?1:0:i>0?3:2}function u(n,t){return o(n.x,t.x)}function o(n,t){var e=i(n,1),r=i(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function l(n){for(var t=0,e=d.length,r=n[1],i=0;e>i;++i)for(var u,o=1,a=d[i],l=a.length,c=a[0];l>o;++o)u=a[o],c[1]<=r?u[1]>r&&Q(c,u,n)>0&&++t:u[1]<=r&&Q(c,u,n)<0&&--t,c=u;return 0!==t}function c(u,a,l,c){var f=0,s=0;if(null==u||(f=i(u,l))!==(s=i(a,l))||o(u,a)<0^l>0){do c.point(0===f||3===f?n:e,f>1?r:t);while((f=(f+l+4)%4)!==s)}else c.point(a[0],a[1])}function f(i,u){return i>=n&&e>=i&&u>=t&&r>=u}function s(n,t){f(n,t)&&a.point(n,t)}function h(){C.point=g,d&&d.push(y=[]),S=!0,w=!1,b=_=NaN}function p(){v&&(g(m,M),x&&w&&E.rejoin(),v.push(E.buffer())),C.point=s,w&&a.lineEnd()}function g(n,t){n=Math.max(-Ha,Math.min(Ha,n)),t=Math.max(-Ha,Math.min(Ha,t));var e=f(n,t);if(d&&y.push([n,t]),S)m=n,M=t,x=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:b,y:_},b:{x:n,y:t}};A(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}b=n,_=t,w=e}var v,d,y,m,M,x,b,_,w,S,k,N=a,E=Pt(),A=Yt(n,t,e,r),C={point:s,lineStart:h,lineEnd:p,polygonStart:function(){a=E,v=[],d=[],k=!0},polygonEnd:function(){a=N,v=ao.merge(v);var t=l([n,r]),e=k&&t,i=v.length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),c(null,null,1,a),a.lineEnd()),i&&Lt(v,u,t,c,a),a.polygonEnd()),v=d=y=null}};return C}}function Vt(n){var t=0,e=Fo/3,r=ae(n),i=r(t,e);return i.parallels=function(n){return arguments.length?r(t=n[0]*Fo/180,e=n[1]*Fo/180):[t/Fo*180,e/Fo*180]},i}function Xt(n,t){function e(n,t){var e=Math.sqrt(u-2*i*Math.sin(t))/i;return[e*Math.sin(n*=i),o-e*Math.cos(n)]}var r=Math.sin(n),i=(r+Math.sin(t))/2,u=1+r*(2*i-r),o=Math.sqrt(u)/i;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/i,tn((u-(n*n+e*e)*i*i)/(2*i))]},e}function $t(){function n(n,t){Ia+=i*n-r*t,r=n,i=t}var t,e,r,i;$a.point=function(u,o){$a.point=n,t=r=u,e=i=o},$a.lineEnd=function(){n(t,e)}}function Bt(n,t){Ya>n&&(Ya=n),n>Va&&(Va=n),Za>t&&(Za=t),t>Xa&&(Xa=t)}function Wt(){function n(n,t){o.push("M",n,",",t,u)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function i(){o.push("Z")}var u=Jt(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return u=Jt(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Jt(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function Gt(n,t){Ca+=n,za+=t,++La}function Kt(){function n(n,r){var i=n-t,u=r-e,o=Math.sqrt(i*i+u*u);qa+=o*(t+n)/2,Ta+=o*(e+r)/2,Ra+=o,Gt(t=n,e=r)}var t,e;Wa.point=function(r,i){Wa.point=n,Gt(t=r,e=i)}}function Qt(){Wa.point=Gt}function ne(){function n(n,t){var e=n-r,u=t-i,o=Math.sqrt(e*e+u*u);qa+=o*(r+n)/2,Ta+=o*(i+t)/2,Ra+=o,o=i*n-r*t,Da+=o*(r+n),Pa+=o*(i+t),Ua+=3*o,Gt(r=n,i=t)}var t,e,r,i;Wa.point=function(u,o){Wa.point=n,Gt(t=r=u,e=i=o)},Wa.lineEnd=function(){n(t,e)}}function te(n){function t(t,e){n.moveTo(t+o,e),n.arc(t,e,o,0,Ho)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function i(){a.point=t}function u(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:i,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=i,a.point=t},pointRadius:function(n){return o=n,a},result:b};return a}function ee(n){function t(n){return(a?r:e)(n)}function e(t){return ue(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){M=NaN,S.point=u,t.lineStart()}function u(e,r){var u=dt([e,r]),o=n(e,r);i(M,x,m,b,_,w,M=o[0],x=o[1],m=e,b=u[0],_=u[1],w=u[2],a,t),t.point(M,x)}function o(){S.point=e,t.lineEnd()}function l(){r(),S.point=c,S.lineEnd=f}function c(n,t){u(s=n,h=t),p=M,g=x,v=b,d=_,y=w,S.point=u}function f(){i(M,x,m,b,_,w,p,g,s,v,d,y,a,t),S.lineEnd=o,o()}var s,h,p,g,v,d,y,m,M,x,b,_,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=l},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function i(t,e,r,a,l,c,f,s,h,p,g,v,d,y){var m=f-t,M=s-e,x=m*m+M*M;if(x>4*u&&d--){var b=a+p,_=l+g,w=c+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),N=xo(xo(w)-1)<Uo||xo(r-h)<Uo?(r+h)/2:Math.atan2(_,b),E=n(N,k),A=E[0],C=E[1],z=A-t,L=C-e,q=M*z-m*L;(q*q/x>u||xo((m*z+M*L)/x-.5)>.3||o>a*p+l*g+c*v)&&(i(t,e,r,a,l,c,A,C,N,b/=S,_/=S,w,d,y),y.point(A,C),i(A,C,N,b,_,w,f,s,h,p,g,v,d,y))}}var u=.5,o=Math.cos(30*Yo),a=16;return t.precision=function(n){return arguments.length?(a=(u=n*n)>0&&16,t):Math.sqrt(u)},t}function re(n){var t=ee(function(t,e){return n([t*Zo,e*Zo])});return function(n){return le(t(n))}}function ie(n){this.stream=n}function ue(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function oe(n){return ae(function(){return n})()}function ae(n){function t(n){return n=a(n[0]*Yo,n[1]*Yo),[n[0]*h+l,c-n[1]*h]}function e(n){return n=a.invert((n[0]-l)/h,(c-n[1])/h),n&&[n[0]*Zo,n[1]*Zo]}function r(){a=Ct(o=se(y,M,x),u);var n=u(v,d);return l=p-n[0]*h,c=g+n[1]*h,i()}function i(){return f&&(f.valid=!1,f=null),t}var u,o,a,l,c,f,s=ee(function(n,t){return n=u(n,t),[n[0]*h+l,c-n[1]*h]}),h=150,p=480,g=250,v=0,d=0,y=0,M=0,x=0,b=Fa,_=m,w=null,S=null;return t.stream=function(n){return f&&(f.valid=!1),f=le(b(o,s(_(n)))),f.valid=!0,f},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Fa):It((w=+n)*Yo),i()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Zt(n[0][0],n[0][1],n[1][0],n[1][1]):m,i()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(p=+n[0],g=+n[1],r()):[p,g]},t.center=function(n){return arguments.length?(v=n[0]%360*Yo,d=n[1]%360*Yo,r()):[v*Zo,d*Zo]},t.rotate=function(n){return arguments.length?(y=n[0]%360*Yo,M=n[1]%360*Yo,x=n.length>2?n[2]%360*Yo:0,r()):[y*Zo,M*Zo,x*Zo]},ao.rebind(t,s,"precision"),function(){return u=n.apply(this,arguments),t.invert=u.invert&&e,r()}}function le(n){return ue(n,function(t,e){n.point(t*Yo,e*Yo)})}function ce(n,t){return[n,t]}function fe(n,t){return[n>Fo?n-Ho:-Fo>n?n+Ho:n,t]}function se(n,t,e){return n?t||e?Ct(pe(n),ge(t,e)):pe(n):t||e?ge(t,e):fe}function he(n){return function(t,e){return t+=n,[t>Fo?t-Ho:-Fo>t?t+Ho:t,e]}}function pe(n){var t=he(n);return t.invert=he(-n),t}function ge(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*r+a*i;return[Math.atan2(l*u-f*o,a*r-c*i),tn(f*u+l*o)]}var r=Math.cos(n),i=Math.sin(n),u=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*u-l*o;return[Math.atan2(l*u+c*o,a*r+f*i),tn(f*r-a*i)]},e}function ve(n,t){var e=Math.cos(n),r=Math.sin(n);return function(i,u,o,a){var l=o*t;null!=i?(i=de(e,i),u=de(e,u),(o>0?u>i:i>u)&&(i+=o*Ho)):(i=n+o*Ho,u=n-.5*l);for(var c,f=i;o>0?f>u:u>f;f-=l)a.point((c=_t([e,-r*Math.cos(f),-r*Math.sin(f)]))[0],c[1])}}function de(n,t){var e=dt(t);e[0]-=n,bt(e);var r=nn(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Uo)%(2*Math.PI)}function ye(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function me(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function Me(n){return n.source}function xe(n){return n.target}function be(n,t,e,r){var i=Math.cos(t),u=Math.sin(t),o=Math.cos(r),a=Math.sin(r),l=i*Math.cos(n),c=i*Math.sin(n),f=o*Math.cos(e),s=o*Math.sin(e),h=2*Math.asin(Math.sqrt(on(r-t)+i*o*on(e-n))),p=1/Math.sin(h),g=h?function(n){var t=Math.sin(n*=h)*p,e=Math.sin(h-n)*p,r=e*l+t*f,i=e*c+t*s,o=e*u+t*a;return[Math.atan2(i,r)*Zo,Math.atan2(o,Math.sqrt(r*r+i*i))*Zo]}:function(){return[n*Zo,t*Zo]};return g.distance=h,g}function _e(){function n(n,i){var u=Math.sin(i*=Yo),o=Math.cos(i),a=xo((n*=Yo)-t),l=Math.cos(a);Ja+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*u-e*o*l)*a),e*u+r*o*l),t=n,e=u,r=o}var t,e,r;Ga.point=function(i,u){t=i*Yo,e=Math.sin(u*=Yo),r=Math.cos(u),Ga.point=n},Ga.lineEnd=function(){Ga.point=Ga.lineEnd=b}}function we(n,t){function e(t,e){var r=Math.cos(t),i=Math.cos(e),u=n(r*i);return[u*i*Math.sin(t),u*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),i=t(r),u=Math.sin(i),o=Math.cos(i);return[Math.atan2(n*u,r*o),Math.asin(r&&e*u/r)]},e}function Se(n,t){function e(n,t){o>0?-Io+Uo>t&&(t=-Io+Uo):t>Io-Uo&&(t=Io-Uo);var e=o/Math.pow(i(t),u);return[e*Math.sin(u*n),o-e*Math.cos(u*n)]}var r=Math.cos(n),i=function(n){return Math.tan(Fo/4+n/2)},u=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(i(t)/i(n)),o=r*Math.pow(i(n),u)/u;return u?(e.invert=function(n,t){var e=o-t,r=K(u)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/u,2*Math.atan(Math.pow(o/r,1/u))-Io]},e):Ne}function ke(n,t){function e(n,t){var e=u-t;return[e*Math.sin(i*n),u-e*Math.cos(i*n)]}var r=Math.cos(n),i=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),u=r/i+n;return xo(i)<Uo?ce:(e.invert=function(n,t){var e=u-t;return[Math.atan2(n,e)/i,u-K(i)*Math.sqrt(n*n+e*e)]},e)}function Ne(n,t){return[n,Math.log(Math.tan(Fo/4+t/2))]}function Ee(n){var t,e=oe(n),r=e.scale,i=e.translate,u=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=i.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=u.apply(e,arguments);if(o===e){if(t=null==n){var a=Fo*r(),l=i();u([[l[0]-a,l[1]-a],[l[0]+a,l[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function Ae(n,t){return[Math.log(Math.tan(Fo/4+t/2)),-n]}function Ce(n){return n[0]}function ze(n){return n[1]}function Le(n){for(var t=n.length,e=[0,1],r=2,i=2;t>i;i++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[i])<=0;)--r;e[r++]=i}return e.slice(0,r)}function qe(n,t){return n[0]-t[0]||n[1]-t[1]}function Te(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Re(n,t,e,r){var i=n[0],u=e[0],o=t[0]-i,a=r[0]-u,l=n[1],c=e[1],f=t[1]-l,s=r[1]-c,h=(a*(l-c)-s*(i-u))/(s*o-a*f);return[i+h*o,l+h*f]}function De(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Pe(){rr(this),this.edge=this.site=this.circle=null}function Ue(n){var t=cl.pop()||new Pe;return t.site=n,t}function je(n){Be(n),ol.remove(n),cl.push(n),rr(n)}function Fe(n){var t=n.circle,e=t.x,r=t.cy,i={x:e,y:r},u=n.P,o=n.N,a=[n];je(n);for(var l=u;l.circle&&xo(e-l.circle.x)<Uo&&xo(r-l.circle.cy)<Uo;)u=l.P,a.unshift(l),je(l),l=u;a.unshift(l),Be(l);for(var c=o;c.circle&&xo(e-c.circle.x)<Uo&&xo(r-c.circle.cy)<Uo;)o=c.N,a.push(c),je(c),c=o;a.push(c),Be(c);var f,s=a.length;for(f=1;s>f;++f)c=a[f],l=a[f-1],nr(c.edge,l.site,c.site,i);l=a[0],c=a[s-1],c.edge=Ke(l.site,c.site,null,i),$e(l),$e(c)}function He(n){for(var t,e,r,i,u=n.x,o=n.y,a=ol._;a;)if(r=Oe(a,o)-u,r>Uo)a=a.L;else{if(i=u-Ie(a,o),!(i>Uo)){r>-Uo?(t=a.P,e=a):i>-Uo?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var l=Ue(n);if(ol.insert(t,l),t||e){if(t===e)return Be(t),e=Ue(t.site),ol.insert(l,e),l.edge=e.edge=Ke(t.site,l.site),$e(t),void $e(e);if(!e)return void(l.edge=Ke(t.site,l.site));Be(t),Be(e);var c=t.site,f=c.x,s=c.y,h=n.x-f,p=n.y-s,g=e.site,v=g.x-f,d=g.y-s,y=2*(h*d-p*v),m=h*h+p*p,M=v*v+d*d,x={x:(d*m-p*M)/y+f,y:(h*M-v*m)/y+s};nr(e.edge,c,g,x),l.edge=Ke(c,n,null,x),e.edge=Ke(n,g,null,x),$e(t),$e(e)}}function Oe(n,t){var e=n.site,r=e.x,i=e.y,u=i-t;if(!u)return r;var o=n.P;if(!o)return-(1/0);e=o.site;var a=e.x,l=e.y,c=l-t;if(!c)return a;var f=a-r,s=1/u-1/c,h=f/c;return s?(-h+Math.sqrt(h*h-2*s*(f*f/(-2*c)-l+c/2+i-u/2)))/s+r:(r+a)/2}function Ie(n,t){var e=n.N;if(e)return Oe(e,t);var r=n.site;return r.y===t?r.x:1/0}function Ye(n){this.site=n,this.edges=[]}function Ze(n){for(var t,e,r,i,u,o,a,l,c,f,s=n[0][0],h=n[1][0],p=n[0][1],g=n[1][1],v=ul,d=v.length;d--;)if(u=v[d],u&&u.prepare())for(a=u.edges,l=a.length,o=0;l>o;)f=a[o].end(),r=f.x,i=f.y,c=a[++o%l].start(),t=c.x,e=c.y,(xo(r-t)>Uo||xo(i-e)>Uo)&&(a.splice(o,0,new tr(Qe(u.site,f,xo(r-s)<Uo&&g-i>Uo?{x:s,y:xo(t-s)<Uo?e:g}:xo(i-g)<Uo&&h-r>Uo?{x:xo(e-g)<Uo?t:h,y:g}:xo(r-h)<Uo&&i-p>Uo?{x:h,y:xo(t-h)<Uo?e:p}:xo(i-p)<Uo&&r-s>Uo?{x:xo(e-p)<Uo?t:s,y:p}:null),u.site,null)),++l)}function Ve(n,t){return t.angle-n.angle}function Xe(){rr(this),this.x=this.y=this.arc=this.site=this.cy=null}function $e(n){var t=n.P,e=n.N;if(t&&e){var r=t.site,i=n.site,u=e.site;if(r!==u){var o=i.x,a=i.y,l=r.x-o,c=r.y-a,f=u.x-o,s=u.y-a,h=2*(l*s-c*f);if(!(h>=-jo)){var p=l*l+c*c,g=f*f+s*s,v=(s*p-c*g)/h,d=(l*g-f*p)/h,s=d+a,y=fl.pop()||new Xe;y.arc=n,y.site=i,y.x=v+o,y.y=s+Math.sqrt(v*v+d*d),y.cy=s,n.circle=y;for(var m=null,M=ll._;M;)if(y.y<M.y||y.y===M.y&&y.x<=M.x){if(!M.L){m=M.P;break}M=M.L}else{if(!M.R){m=M;break}M=M.R}ll.insert(m,y),m||(al=y)}}}}function Be(n){var t=n.circle;t&&(t.P||(al=t.N),ll.remove(t),fl.push(t),rr(t),n.circle=null)}function We(n){for(var t,e=il,r=Yt(n[0][0],n[0][1],n[1][0],n[1][1]),i=e.length;i--;)t=e[i],(!Je(t,n)||!r(t)||xo(t.a.x-t.b.x)<Uo&&xo(t.a.y-t.b.y)<Uo)&&(t.a=t.b=null,e.splice(i,1))}function Je(n,t){var e=n.b;if(e)return!0;var r,i,u=n.a,o=t[0][0],a=t[1][0],l=t[0][1],c=t[1][1],f=n.l,s=n.r,h=f.x,p=f.y,g=s.x,v=s.y,d=(h+g)/2,y=(p+v)/2;if(v===p){if(o>d||d>=a)return;if(h>g){if(u){if(u.y>=c)return}else u={x:d,y:l};e={x:d,y:c}}else{if(u){if(u.y<l)return}else u={x:d,y:c};e={x:d,y:l}}}else if(r=(h-g)/(v-p),i=y-r*d,-1>r||r>1)if(h>g){if(u){if(u.y>=c)return}else u={x:(l-i)/r,y:l};e={x:(c-i)/r,y:c}}else{if(u){if(u.y<l)return}else u={x:(c-i)/r,y:c};e={x:(l-i)/r,y:l}}else if(v>p){if(u){if(u.x>=a)return}else u={x:o,y:r*o+i};e={x:a,y:r*a+i}}else{if(u){if(u.x<o)return}else u={x:a,y:r*a+i};e={x:o,y:r*o+i}}return n.a=u,n.b=e,!0}function Ge(n,t){this.l=n,this.r=t,this.a=this.b=null}function Ke(n,t,e,r){var i=new Ge(n,t);return il.push(i),e&&nr(i,n,t,e),r&&nr(i,t,n,r),ul[n.i].edges.push(new tr(i,n,t)),ul[t.i].edges.push(new tr(i,t,n)),i}function Qe(n,t,e){var r=new Ge(n,null);return r.a=t,r.b=e,il.push(r),r}function nr(n,t,e,r){n.a||n.b?n.l===e?n.b=r:n.a=r:(n.a=r,n.l=t,n.r=e)}function tr(n,t,e){var r=n.a,i=n.b;this.edge=n,this.site=t,this.angle=e?Math.atan2(e.y-t.y,e.x-t.x):n.l===t?Math.atan2(i.x-r.x,r.y-i.y):Math.atan2(r.x-i.x,i.y-r.y)}function er(){this._=null}function rr(n){n.U=n.C=n.L=n.R=n.P=n.N=null}function ir(n,t){var e=t,r=t.R,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.R=r.L,e.R&&(e.R.U=e),r.L=e}function ur(n,t){var e=t,r=t.L,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.L=r.R,e.L&&(e.L.U=e),r.R=e}function or(n){for(;n.L;)n=n.L;return n}function ar(n,t){var e,r,i,u=n.sort(lr).pop();for(il=[],ul=new Array(n.length),ol=new er,ll=new er;;)if(i=al,u&&(!i||u.y<i.y||u.y===i.y&&u.x<i.x))u.x===e&&u.y===r||(ul[u.i]=new Ye(u),He(u),e=u.x,r=u.y),u=n.pop();else{if(!i)break;Fe(i.arc)}t&&(We(t),Ze(t));var o={cells:ul,edges:il};return ol=ll=il=ul=null,o}function lr(n,t){return t.y-n.y||t.x-n.x}function cr(n,t,e){return(n.x-e.x)*(t.y-n.y)-(n.x-t.x)*(e.y-n.y)}function fr(n){return n.x}function sr(n){return n.y}function hr(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function pr(n,t,e,r,i,u){if(!n(t,e,r,i,u)){var o=.5*(e+i),a=.5*(r+u),l=t.nodes;l[0]&&pr(n,l[0],e,r,o,a),l[1]&&pr(n,l[1],o,r,i,a),l[2]&&pr(n,l[2],e,a,o,u),l[3]&&pr(n,l[3],o,a,i,u)}}function gr(n,t,e,r,i,u,o){var a,l=1/0;return function c(n,f,s,h,p){if(!(f>u||s>o||r>h||i>p)){if(g=n.point){var g,v=t-n.x,d=e-n.y,y=v*v+d*d;if(l>y){var m=Math.sqrt(l=y);r=t-m,i=e-m,u=t+m,o=e+m,a=g}}for(var M=n.nodes,x=.5*(f+h),b=.5*(s+p),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:c(n,f,s,x,b);break;case 1:c(n,x,s,h,b);break;case 2:c(n,f,b,x,p);break;case 3:c(n,x,b,h,p)}}}(n,r,i,u,o),a}function vr(n,t){n=ao.rgb(n),t=ao.rgb(t);var e=n.r,r=n.g,i=n.b,u=t.r-e,o=t.g-r,a=t.b-i;return function(n){return"#"+bn(Math.round(e+u*n))+bn(Math.round(r+o*n))+bn(Math.round(i+a*n))}}function dr(n,t){var e,r={},i={};for(e in n)e in t?r[e]=Mr(n[e],t[e]):i[e]=n[e];for(e in t)e in n||(i[e]=t[e]);return function(n){for(e in r)i[e]=r[e](n);return i}}function yr(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function mr(n,t){var e,r,i,u=hl.lastIndex=pl.lastIndex=0,o=-1,a=[],l=[];for(n+="",t+="";(e=hl.exec(n))&&(r=pl.exec(t));)(i=r.index)>u&&(i=t.slice(u,i),a[o]?a[o]+=i:a[++o]=i),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,l.push({i:o,x:yr(e,r)})),u=pl.lastIndex;return u<t.length&&(i=t.slice(u),a[o]?a[o]+=i:a[++o]=i),a.length<2?l[0]?(t=l[0].x,function(n){return t(n)+""}):function(){return t}:(t=l.length,function(n){for(var e,r=0;t>r;++r)a[(e=l[r]).i]=e.x(n);return a.join("")})}function Mr(n,t){for(var e,r=ao.interpolators.length;--r>=0&&!(e=ao.interpolators[r](n,t)););return e}function xr(n,t){var e,r=[],i=[],u=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(Mr(n[e],t[e]));for(;u>e;++e)i[e]=n[e];for(;o>e;++e)i[e]=t[e];return function(n){for(e=0;a>e;++e)i[e]=r[e](n);return i}}function br(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function _r(n){return function(t){return 1-n(1-t)}}function wr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function Sr(n){return n*n}function kr(n){return n*n*n}function Nr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Er(n){return function(t){return Math.pow(t,n)}}function Ar(n){return 1-Math.cos(n*Io)}function Cr(n){return Math.pow(2,10*(n-1))}function zr(n){return 1-Math.sqrt(1-n*n)}function Lr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Ho*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Ho/t)}}function qr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Tr(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Rr(n,t){n=ao.hcl(n),t=ao.hcl(t);var e=n.h,r=n.c,i=n.l,u=t.h-e,o=t.c-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return sn(e+u*n,r+o*n,i+a*n)+""}}function Dr(n,t){n=ao.hsl(n),t=ao.hsl(t);var e=n.h,r=n.s,i=n.l,u=t.h-e,o=t.s-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return cn(e+u*n,r+o*n,i+a*n)+""}}function Pr(n,t){n=ao.lab(n),t=ao.lab(t);var e=n.l,r=n.a,i=n.b,u=t.l-e,o=t.a-r,a=t.b-i;return function(n){return pn(e+u*n,r+o*n,i+a*n)+""}}function Ur(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function jr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Hr(t),i=Fr(t,e),u=Hr(Or(e,t,-i))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,i*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*Zo,this.translate=[n.e,n.f],this.scale=[r,u],this.skew=u?Math.atan2(i,u)*Zo:0}function Fr(n,t){return n[0]*t[0]+n[1]*t[1]}function Hr(n){var t=Math.sqrt(Fr(n,n));return t&&(n[0]/=t,n[1]/=t),t}function Or(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function Ir(n){return n.length?n.pop()+",":""}function Yr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push("translate(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else(t[0]||t[1])&&e.push("translate("+t+")")}function Zr(n,t,e,r){n!==t?(n-t>180?t+=360:t-n>180&&(n+=360),r.push({i:e.push(Ir(e)+"rotate(",null,")")-2,x:yr(n,t)})):t&&e.push(Ir(e)+"rotate("+t+")")}function Vr(n,t,e,r){n!==t?r.push({i:e.push(Ir(e)+"skewX(",null,")")-2,x:yr(n,t)}):t&&e.push(Ir(e)+"skewX("+t+")")}function Xr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push(Ir(e)+"scale(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else 1===t[0]&&1===t[1]||e.push(Ir(e)+"scale("+t+")")}function $r(n,t){var e=[],r=[];return n=ao.transform(n),t=ao.transform(t),Yr(n.translate,t.translate,e,r),Zr(n.rotate,t.rotate,e,r),Vr(n.skew,t.skew,e,r),Xr(n.scale,t.scale,e,r),n=t=null,function(n){for(var t,i=-1,u=r.length;++i<u;)e[(t=r[i]).i]=t.x(n);return e.join("")}}function Br(n,t){return t=(t-=n=+n)||1/t,function(e){return(e-n)/t}}function Wr(n,t){return t=(t-=n=+n)||1/t,function(e){return Math.max(0,Math.min(1,(e-n)/t))}}function Jr(n){for(var t=n.source,e=n.target,r=Kr(t,e),i=[t];t!==r;)t=t.parent,i.push(t);for(var u=i.length;e!==r;)i.splice(u,0,e),e=e.parent;return i}function Gr(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function Kr(n,t){if(n===t)return n;for(var e=Gr(n),r=Gr(t),i=e.pop(),u=r.pop(),o=null;i===u;)o=i,i=e.pop(),u=r.pop();return o}function Qr(n){n.fixed|=2}function ni(n){n.fixed&=-7}function ti(n){n.fixed|=4,n.px=n.x,n.py=n.y}function ei(n){n.fixed&=-5}function ri(n,t,e){var r=0,i=0;if(n.charge=0,!n.leaf)for(var u,o=n.nodes,a=o.length,l=-1;++l<a;)u=o[l],null!=u&&(ri(u,t,e),n.charge+=u.charge,r+=u.charge*u.cx,i+=u.charge*u.cy);if(n.point){n.leaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var c=t*e[n.point.index];n.charge+=n.pointCharge=c,r+=c*n.point.x,i+=c*n.point.y}n.cx=r/n.charge,n.cy=i/n.charge}function ii(n,t){return ao.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=fi,n}function ui(n,t){for(var e=[n];null!=(n=e.pop());)if(t(n),(i=n.children)&&(r=i.length))for(var r,i;--r>=0;)e.push(i[r])}function oi(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(u=n.children)&&(i=u.length))for(var i,u,o=-1;++o<i;)e.push(u[o]);for(;null!=(n=r.pop());)t(n)}function ai(n){return n.children}function li(n){return n.value}function ci(n,t){return t.value-n.value}function fi(n){return ao.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function si(n){return n.x}function hi(n){return n.y}function pi(n,t,e){n.y0=t,n.y=e}function gi(n){return ao.range(n.length)}function vi(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function di(n){for(var t,e=1,r=0,i=n[0][1],u=n.length;u>e;++e)(t=n[e][1])>i&&(r=e,i=t);return r}function yi(n){return n.reduce(mi,0)}function mi(n,t){return n+t[1]}function Mi(n,t){return xi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function xi(n,t){for(var e=-1,r=+n[0],i=(n[1]-r)/t,u=[];++e<=t;)u[e]=i*e+r;return u}function bi(n){return[ao.min(n),ao.max(n)]}function _i(n,t){return n.value-t.value}function wi(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function Si(n,t){n._pack_next=t,t._pack_prev=n}function ki(n,t){var e=t.x-n.x,r=t.y-n.y,i=n.r+t.r;return.999*i*i>e*e+r*r}function Ni(n){function t(n){f=Math.min(n.x-n.r,f),s=Math.max(n.x+n.r,s),h=Math.min(n.y-n.r,h),p=Math.max(n.y+n.r,p)}if((e=n.children)&&(c=e.length)){var e,r,i,u,o,a,l,c,f=1/0,s=-(1/0),h=1/0,p=-(1/0);if(e.forEach(Ei),r=e[0],r.x=-r.r,r.y=0,t(r),c>1&&(i=e[1],i.x=i.r,i.y=0,t(i),c>2))for(u=e[2],zi(r,i,u),t(u),wi(r,u),r._pack_prev=u,wi(u,i),i=r._pack_next,o=3;c>o;o++){zi(r,i,u=e[o]);var g=0,v=1,d=1;for(a=i._pack_next;a!==i;a=a._pack_next,v++)if(ki(a,u)){g=1;break}if(1==g)for(l=r._pack_prev;l!==a._pack_prev&&!ki(l,u);l=l._pack_prev,d++);g?(d>v||v==d&&i.r<r.r?Si(r,i=a):Si(r=l,i),o--):(wi(r,u),i=u,t(u))}var y=(f+s)/2,m=(h+p)/2,M=0;for(o=0;c>o;o++)u=e[o],u.x-=y,u.y-=m,M=Math.max(M,u.r+Math.sqrt(u.x*u.x+u.y*u.y));n.r=M,e.forEach(Ai)}}function Ei(n){n._pack_next=n._pack_prev=n}function Ai(n){delete n._pack_next,delete n._pack_prev}function Ci(n,t,e,r){var i=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,i)for(var u=-1,o=i.length;++u<o;)Ci(i[u],t,e,r)}function zi(n,t,e){var r=n.r+e.r,i=t.x-n.x,u=t.y-n.y;if(r&&(i||u)){var o=t.r+e.r,a=i*i+u*u;o*=o,r*=r;var l=.5+(r-o)/(2*a),c=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+l*i+c*u,e.y=n.y+l*u-c*i}else e.x=n.x+r,e.y=n.y}function Li(n,t){return n.parent==t.parent?1:2}function qi(n){var t=n.children;return t.length?t[0]:n.t}function Ti(n){var t,e=n.children;return(t=e.length)?e[t-1]:n.t}function Ri(n,t,e){var r=e/(t.i-n.i);t.c-=r,t.s+=e,n.c+=r,t.z+=e,t.m+=e}function Di(n){for(var t,e=0,r=0,i=n.children,u=i.length;--u>=0;)t=i[u],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Pi(n,t,e){return n.a.parent===t.parent?n.a:e}function Ui(n){return 1+ao.max(n,function(n){return n.y})}function ji(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Fi(n){var t=n.children;return t&&t.length?Fi(t[0]):n}function Hi(n){var t,e=n.children;return e&&(t=e.length)?Hi(e[t-1]):n}function Oi(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Ii(n,t){var e=n.x+t[3],r=n.y+t[0],i=n.dx-t[1]-t[3],u=n.dy-t[0]-t[2];return 0>i&&(e+=i/2,i=0),0>u&&(r+=u/2,u=0),{x:e,y:r,dx:i,dy:u}}function Yi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Zi(n){return n.rangeExtent?n.rangeExtent():Yi(n.range())}function Vi(n,t,e,r){var i=e(n[0],n[1]),u=r(t[0],t[1]);return function(n){return u(i(n))}}function Xi(n,t){var e,r=0,i=n.length-1,u=n[r],o=n[i];return u>o&&(e=r,r=i,i=e,e=u,u=o,o=e),n[r]=t.floor(u),n[i]=t.ceil(o),n}function $i(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:Sl}function Bi(n,t,e,r){var i=[],u=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)i.push(e(n[o-1],n[o])),u.push(r(t[o-1],t[o]));return function(t){var e=ao.bisect(n,t,1,a)-1;return u[e](i[e](t))}}function Wi(n,t,e,r){function i(){var i=Math.min(n.length,t.length)>2?Bi:Vi,l=r?Wr:Br;return o=i(n,t,l,e),a=i(t,n,l,Mr),u}function u(n){return o(n)}var o,a;return u.invert=function(n){return a(n)},u.domain=function(t){return arguments.length?(n=t.map(Number),i()):n},u.range=function(n){return arguments.length?(t=n,i()):t},u.rangeRound=function(n){return u.range(n).interpolate(Ur)},u.clamp=function(n){return arguments.length?(r=n,i()):r},u.interpolate=function(n){return arguments.length?(e=n,i()):e},u.ticks=function(t){return Qi(n,t)},u.tickFormat=function(t,e){return nu(n,t,e)},u.nice=function(t){return Gi(n,t),i()},u.copy=function(){return Wi(n,t,e,r)},i()}function Ji(n,t){return ao.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Gi(n,t){return Xi(n,$i(Ki(n,t)[2])),Xi(n,$i(Ki(n,t)[2])),n}function Ki(n,t){null==t&&(t=10);var e=Yi(n),r=e[1]-e[0],i=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),u=t/r*i;return.15>=u?i*=10:.35>=u?i*=5:.75>=u&&(i*=2),e[0]=Math.ceil(e[0]/i)*i,e[1]=Math.floor(e[1]/i)*i+.5*i,e[2]=i,e}function Qi(n,t){return ao.range.apply(ao,Ki(n,t))}function nu(n,t,e){var r=Ki(n,t);if(e){var i=ha.exec(e);if(i.shift(),"s"===i[8]){var u=ao.formatPrefix(Math.max(xo(r[0]),xo(r[1])));return i[7]||(i[7]="."+tu(u.scale(r[2]))),i[8]="f",e=ao.format(i.join("")),function(n){return e(u.scale(n))+u.symbol}}i[7]||(i[7]="."+eu(i[8],r)),e=i.join("")}else e=",."+tu(r[2])+"f";return ao.format(e)}function tu(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function eu(n,t){var e=tu(t[2]);return n in kl?Math.abs(e-tu(Math.max(xo(t[0]),xo(t[1]))))+ +("e"!==n):e-2*("%"===n)}function ru(n,t,e,r){function i(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function u(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(i(t))}return o.invert=function(t){return u(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(i)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(i)),o):t},o.nice=function(){var t=Xi(r.map(i),e?Math:El);return n.domain(t),r=t.map(u),o},o.ticks=function(){var n=Yi(r),o=[],a=n[0],l=n[1],c=Math.floor(i(a)),f=Math.ceil(i(l)),s=t%1?2:t;if(isFinite(f-c)){if(e){for(;f>c;c++)for(var h=1;s>h;h++)o.push(u(c)*h);o.push(u(c))}else for(o.push(u(c));c++<f;)for(var h=s-1;h>0;h--)o.push(u(c)*h);for(c=0;o[c]<a;c++);for(f=o.length;o[f-1]>l;f--);o=o.slice(c,f)}return o},o.tickFormat=function(n,e){if(!arguments.length)return Nl;arguments.length<2?e=Nl:"function"!=typeof e&&(e=ao.format(e));var r=Math.max(1,t*n/o.ticks().length);return function(n){var o=n/u(Math.round(i(n)));return t-.5>o*t&&(o*=t),r>=o?e(n):""}},o.copy=function(){return ru(n.copy(),t,e,r)},Ji(o,n)}function iu(n,t,e){function r(t){return n(i(t))}var i=uu(t),u=uu(1/t);return r.invert=function(t){return u(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(i)),r):e},r.ticks=function(n){return Qi(e,n)},r.tickFormat=function(n,t){return nu(e,n,t)},r.nice=function(n){return r.domain(Gi(e,n))},r.exponent=function(o){return arguments.length?(i=uu(t=o),u=uu(1/t),n.domain(e.map(i)),r):t},r.copy=function(){return iu(n.copy(),t,e)},Ji(r,n)}function uu(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function ou(n,t){function e(e){return u[((i.get(e)||("range"===t.t?i.set(e,n.push(e)):NaN))-1)%u.length]}function r(t,e){return ao.range(n.length).map(function(n){return t+e*n})}var i,u,o;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new c;for(var u,o=-1,a=r.length;++o<a;)i.has(u=r[o])||i.set(u,n.push(u));return e[t.t].apply(e,t.a)},e.range=function(n){return arguments.length?(u=n,o=0,t={t:"range",a:arguments},e):u},e.rangePoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=(l+c)/2,0):(c-l)/(n.length-1+a);return u=r(l+f*a/2,f),o=0,t={t:"rangePoints",a:arguments},e},e.rangeRoundPoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=c=Math.round((l+c)/2),0):(c-l)/(n.length-1+a)|0;return u=r(l+Math.round(f*a/2+(c-l-(n.length-1+a)*f)/2),f),o=0,t={t:"rangeRoundPoints",a:arguments},e},e.rangeBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=(s-f)/(n.length-a+2*l);return u=r(f+h*l,h),c&&u.reverse(),o=h*(1-a),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=Math.floor((s-f)/(n.length-a+2*l));return u=r(f+Math.round((s-f-(n.length-a)*h)/2),h),c&&u.reverse(),o=Math.round(h*(1-a)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=function(){return o},e.rangeExtent=function(){return Yi(t.a[0])},e.copy=function(){return ou(n,t)},e.domain(n)}function au(n,t){function u(){var e=0,r=t.length;for(a=[];++e<r;)a[e-1]=ao.quantile(n,e/r);return o}function o(n){return isNaN(n=+n)?void 0:t[ao.bisect(a,n)]}var a;return o.domain=function(t){return arguments.length?(n=t.map(r).filter(i).sort(e),u()):n},o.range=function(n){return arguments.length?(t=n,u()):t},o.quantiles=function(){return a},o.invertExtent=function(e){return e=t.indexOf(e),0>e?[NaN,NaN]:[e>0?a[e-1]:n[0],e<a.length?a[e]:n[n.length-1]]},o.copy=function(){return au(n,t)},u()}function lu(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(u*(t-n))))]}function i(){return u=e.length/(t-n),o=e.length-1,r}var u,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],i()):[n,t]},r.range=function(n){return arguments.length?(e=n,i()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?NaN:t/u+n,[t,t+1/u]},r.copy=function(){return lu(n,t,e)},i()}function cu(n,t){function e(e){return e>=e?t[ao.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return cu(n,t)},e}function fu(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Qi(n,t)},t.tickFormat=function(t,e){return nu(n,t,e)},t.copy=function(){return fu(n)},t}function su(){return 0}function hu(n){return n.innerRadius}function pu(n){return n.outerRadius}function gu(n){return n.startAngle}function vu(n){return n.endAngle}function du(n){return n&&n.padAngle}function yu(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function mu(n,t,e,r,i){var u=n[0]-t[0],o=n[1]-t[1],a=(i?r:-r)/Math.sqrt(u*u+o*o),l=a*o,c=-a*u,f=n[0]+l,s=n[1]+c,h=t[0]+l,p=t[1]+c,g=(f+h)/2,v=(s+p)/2,d=h-f,y=p-s,m=d*d+y*y,M=e-r,x=f*p-h*s,b=(0>y?-1:1)*Math.sqrt(Math.max(0,M*M*m-x*x)),_=(x*y-d*b)/m,w=(-x*d-y*b)/m,S=(x*y+d*b)/m,k=(-x*d+y*b)/m,N=_-g,E=w-v,A=S-g,C=k-v;return N*N+E*E>A*A+C*C&&(_=S,w=k),[[_-l,w-c],[_*e/M,w*e/M]]}function Mu(n){function t(t){function o(){c.push("M",u(n(f),a))}for(var l,c=[],f=[],s=-1,h=t.length,p=En(e),g=En(r);++s<h;)i.call(this,l=t[s],s)?f.push([+p.call(this,l,s),+g.call(this,l,s)]):f.length&&(o(),f=[]);return f.length&&o(),c.length?c.join(""):null}var e=Ce,r=ze,i=zt,u=xu,o=u.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(i=n,t):i},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?u=n:(u=Tl.get(n)||xu).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function xu(n){return n.length>1?n.join("L"):n+"Z"}function bu(n){return n.join("L")+"Z"}function _u(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1]);return e>1&&i.push("H",r[0]),i.join("")}function wu(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("V",(r=n[t])[1],"H",r[0]);return i.join("")}function Su(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r=n[t])[0],"V",r[1]);return i.join("")}function ku(n,t){return n.length<4?xu(n):n[1]+Au(n.slice(1,-1),Cu(n,t))}function Nu(n,t){return n.length<3?bu(n):n[0]+Au((n.push(n[0]),n),Cu([n[n.length-2]].concat(n,[n[1]]),t))}function Eu(n,t){return n.length<3?xu(n):n[0]+Au(n,Cu(n,t))}function Au(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return xu(n);var e=n.length!=t.length,r="",i=n[0],u=n[1],o=t[0],a=o,l=1;if(e&&(r+="Q"+(u[0]-2*o[0]/3)+","+(u[1]-2*o[1]/3)+","+u[0]+","+u[1],i=n[1],l=2),t.length>1){a=t[1],u=n[l],l++,r+="C"+(i[0]+o[0])+","+(i[1]+o[1])+","+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1];for(var c=2;c<t.length;c++,l++)u=n[l],a=t[c],r+="S"+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1]}if(e){var f=n[l];r+="Q"+(u[0]+2*a[0]/3)+","+(u[1]+2*a[1]/3)+","+f[0]+","+f[1]}return r}function Cu(n,t){for(var e,r=[],i=(1-t)/2,u=n[0],o=n[1],a=1,l=n.length;++a<l;)e=u,u=o,o=n[a],r.push([i*(o[0]-e[0]),i*(o[1]-e[1])]);return r}function zu(n){if(n.length<3)return xu(n);var t=1,e=n.length,r=n[0],i=r[0],u=r[1],o=[i,i,i,(r=n[1])[0]],a=[u,u,u,r[1]],l=[i,",",u,"L",Ru(Pl,o),",",Ru(Pl,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),Du(l,o,a);return n.pop(),l.push("L",r),l.join("")}function Lu(n){if(n.length<4)return xu(n);for(var t,e=[],r=-1,i=n.length,u=[0],o=[0];++r<3;)t=n[r],u.push(t[0]),o.push(t[1]);for(e.push(Ru(Pl,u)+","+Ru(Pl,o)),--r;++r<i;)t=n[r],u.shift(),u.push(t[0]),o.shift(),o.push(t[1]),Du(e,u,o);return e.join("")}function qu(n){for(var t,e,r=-1,i=n.length,u=i+4,o=[],a=[];++r<4;)e=n[r%i],o.push(e[0]),a.push(e[1]);for(t=[Ru(Pl,o),",",Ru(Pl,a)],--r;++r<u;)e=n[r%i],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),Du(t,o,a);return t.join("")}function Tu(n,t){var e=n.length-1;if(e)for(var r,i,u=n[0][0],o=n[0][1],a=n[e][0]-u,l=n[e][1]-o,c=-1;++c<=e;)r=n[c],i=c/e,r[0]=t*r[0]+(1-t)*(u+i*a),r[1]=t*r[1]+(1-t)*(o+i*l);return zu(n)}function Ru(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function Du(n,t,e){n.push("C",Ru(Rl,t),",",Ru(Rl,e),",",Ru(Dl,t),",",Ru(Dl,e),",",Ru(Pl,t),",",Ru(Pl,e))}function Pu(n,t){return(t[1]-n[1])/(t[0]-n[0])}function Uu(n){for(var t=0,e=n.length-1,r=[],i=n[0],u=n[1],o=r[0]=Pu(i,u);++t<e;)r[t]=(o+(o=Pu(i=u,u=n[t+1])))/2;return r[t]=o,r}function ju(n){for(var t,e,r,i,u=[],o=Uu(n),a=-1,l=n.length-1;++a<l;)t=Pu(n[a],n[a+1]),xo(t)<Uo?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,i=e*e+r*r,i>9&&(i=3*t/Math.sqrt(i),o[a]=i*e,o[a+1]=i*r));for(a=-1;++a<=l;)i=(n[Math.min(l,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),u.push([i||0,o[a]*i||0]);return u}function Fu(n){return n.length<3?xu(n):n[0]+Au(n,ju(n))}function Hu(n){for(var t,e,r,i=-1,u=n.length;++i<u;)t=n[i],e=t[0],r=t[1]-Io,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function Ou(n){function t(t){function l(){v.push("M",a(n(y),s),f,c(n(d.reverse()),s),"Z")}for(var h,p,g,v=[],d=[],y=[],m=-1,M=t.length,x=En(e),b=En(i),_=e===r?function(){return p}:En(r),w=i===u?function(){return g}:En(u);++m<M;)o.call(this,h=t[m],m)?(d.push([p=+x.call(this,h,m),g=+b.call(this,h,m)]),y.push([+_.call(this,h,m),+w.call(this,h,m)])):d.length&&(l(),d=[],y=[]);return d.length&&l(),v.length?v.join(""):null}var e=Ce,r=Ce,i=0,u=ze,o=zt,a=xu,l=a.key,c=a,f="L",s=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(i=u=n,t):u},t.y0=function(n){return arguments.length?(i=n,t):i},t.y1=function(n){return arguments.length?(u=n,t):u},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(l="function"==typeof n?a=n:(a=Tl.get(n)||xu).key,c=a.reverse||a,f=a.closed?"M":"L",t):l},t.tension=function(n){return arguments.length?(s=n,t):s},t}function Iu(n){return n.radius}function Yu(n){return[n.x,n.y]}function Zu(n){return function(){var t=n.apply(this,arguments),e=t[0],r=t[1]-Io;return[e*Math.cos(r),e*Math.sin(r)]}}function Vu(){return 64}function Xu(){return"circle"}function $u(n){var t=Math.sqrt(n/Fo);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function Bu(n){return function(){var t,e,r;(t=this[n])&&(r=t[e=t.active])&&(r.timer.c=null,r.timer.t=NaN,--t.count?delete t[e]:delete this[n],t.active+=.5,r.event&&r.event.interrupt.call(this,this.__data__,r.index))}}function Wu(n,t,e){return ko(n,Yl),n.namespace=t,n.id=e,n}function Ju(n,t,e,r){var i=n.id,u=n.namespace;return Y(n,"function"==typeof e?function(n,o,a){n[u][i].tween.set(t,r(e.call(n,n.__data__,o,a)))}:(e=r(e),function(n){n[u][i].tween.set(t,e)}))}function Gu(n){return null==n&&(n=""),function(){this.textContent=n}}function Ku(n){return null==n?"__transition__":"__transition_"+n+"__"}function Qu(n,t,e,r,i){function u(n){var t=v.delay;return f.t=t+l,n>=t?o(n-t):void(f.c=o)}function o(e){var i=g.active,u=g[i];u&&(u.timer.c=null,u.timer.t=NaN,--g.count,delete g[i],u.event&&u.event.interrupt.call(n,n.__data__,u.index));for(var o in g)if(r>+o){var c=g[o];c.timer.c=null,c.timer.t=NaN,--g.count,delete g[o]}f.c=a,qn(function(){return f.c&&a(e||1)&&(f.c=null,f.t=NaN),1},0,l),g.active=r,v.event&&v.event.start.call(n,n.__data__,t),p=[],v.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&p.push(r)}),h=v.ease,s=v.duration}function a(i){for(var u=i/s,o=h(u),a=p.length;a>0;)p[--a].call(n,o);return u>=1?(v.event&&v.event.end.call(n,n.__data__,t),--g.count?delete g[r]:delete n[e],1):void 0}var l,f,s,h,p,g=n[e]||(n[e]={active:0,count:0}),v=g[r];v||(l=i.time,f=qn(u,0,l),v=g[r]={tween:new c,time:l,timer:f,delay:i.delay,duration:i.duration,ease:i.ease,index:t},i=null,++g.count)}function no(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function to(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function eo(n){return n.toISOString()}function ro(n,t,e){function r(t){return n(t)}function i(n,e){var r=n[1]-n[0],i=r/e,u=ao.bisect(Kl,i);return u==Kl.length?[t.year,Ki(n.map(function(n){return n/31536e6}),e)[2]]:u?t[i/Kl[u-1]<Kl[u]/i?u-1:u]:[tc,Ki(n,e)[2]]}return r.invert=function(t){return io(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(io)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,io(+e+1),t).length}var u=r.domain(),o=Yi(u),a=null==n?i(o,10):"number"==typeof n&&i(o,n);return a&&(n=a[0],t=a[1]),r.domain(Xi(u,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=io(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=io(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Yi(r.domain()),u=null==n?i(e,10):"number"==typeof n?i(e,n):!n.range&&[{range:n},t];return u&&(n=u[0],t=u[1]),n.range(e[0],io(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return ro(n.copy(),t,e)},Ji(r,n)}function io(n){return new Date(n)}function uo(n){return JSON.parse(n.responseText)}function oo(n){var t=fo.createRange();return t.selectNode(fo.body),t.createContextualFragment(n.responseText)}var ao={version:"3.5.17"},lo=[].slice,co=function(n){return lo.call(n)},fo=this.document;if(fo)try{co(fo.documentElement.childNodes)[0].nodeType}catch(so){co=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),fo)try{fo.createElement("DIV").style.setProperty("opacity",0,"")}catch(ho){var po=this.Element.prototype,go=po.setAttribute,vo=po.setAttributeNS,yo=this.CSSStyleDeclaration.prototype,mo=yo.setProperty;po.setAttribute=function(n,t){go.call(this,n,t+"")},po.setAttributeNS=function(n,t,e){vo.call(this,n,t,e+"")},yo.setProperty=function(n,t,e){mo.call(this,n,t+"",e)}}ao.ascending=e,ao.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:NaN},ao.min=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&e>r&&(e=r)}else{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&e>r&&(e=r)}return e},ao.max=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&r>e&&(e=r)}else{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&r>e&&(e=r)}return e},ao.extent=function(n,t){var e,r,i,u=-1,o=n.length;if(1===arguments.length){for(;++u<o;)if(null!=(r=n[u])&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=n[u])&&(e>r&&(e=r),r>i&&(i=r))}else{for(;++u<o;)if(null!=(r=t.call(n,n[u],u))&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=t.call(n,n[u],u))&&(e>r&&(e=r),r>i&&(i=r))}return[e,i]},ao.sum=function(n,t){var e,r=0,u=n.length,o=-1;if(1===arguments.length)for(;++o<u;)i(e=+n[o])&&(r+=e);else for(;++o<u;)i(e=+t.call(n,n[o],o))&&(r+=e);return r},ao.mean=function(n,t){var e,u=0,o=n.length,a=-1,l=o;if(1===arguments.length)for(;++a<o;)i(e=r(n[a]))?u+=e:--l;else for(;++a<o;)i(e=r(t.call(n,n[a],a)))?u+=e:--l;return l?u/l:void 0},ao.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),i=+n[r-1],u=e-r;return u?i+u*(n[r]-i):i},ao.median=function(n,t){var u,o=[],a=n.length,l=-1;if(1===arguments.length)for(;++l<a;)i(u=r(n[l]))&&o.push(u);else for(;++l<a;)i(u=r(t.call(n,n[l],l)))&&o.push(u);return o.length?ao.quantile(o.sort(e),.5):void 0},ao.variance=function(n,t){var e,u,o=n.length,a=0,l=0,c=-1,f=0;if(1===arguments.length)for(;++c<o;)i(e=r(n[c]))&&(u=e-a,a+=u/++f,l+=u*(e-a));else for(;++c<o;)i(e=r(t.call(n,n[c],c)))&&(u=e-a,a+=u/++f,l+=u*(e-a));return f>1?l/(f-1):void 0},ao.deviation=function(){var n=ao.variance.apply(this,arguments);return n?Math.sqrt(n):n};var Mo=u(e);ao.bisectLeft=Mo.left,ao.bisect=ao.bisectRight=Mo.right,ao.bisector=function(n){return u(1===n.length?function(t,r){return e(n(t),r)}:n)},ao.shuffle=function(n,t,e){(u=arguments.length)<3&&(e=n.length,2>u&&(t=0));for(var r,i,u=e-t;u;)i=Math.random()*u--|0,r=n[u+t],n[u+t]=n[i+t],n[i+t]=r;return n},ao.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ao.pairs=function(n){for(var t,e=0,r=n.length-1,i=n[0],u=new Array(0>r?0:r);r>e;)u[e]=[t=i,i=n[++e]];return u},ao.transpose=function(n){if(!(i=n.length))return[];for(var t=-1,e=ao.min(n,o),r=new Array(e);++t<e;)for(var i,u=-1,a=r[t]=new Array(i);++u<i;)a[u]=n[u][t];return r},ao.zip=function(){return ao.transpose(arguments)},ao.keys=function(n){var t=[];for(var e in n)t.push(e);return t},ao.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},ao.entries=function(n){var t=[];for(var e in n)t.push({key:e,value:n[e]});return t},ao.merge=function(n){for(var t,e,r,i=n.length,u=-1,o=0;++u<i;)o+=n[u].length;for(e=new Array(o);--i>=0;)for(r=n[i],t=r.length;--t>=0;)e[--o]=r[t];return e};var xo=Math.abs;ao.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,i=[],u=a(xo(e)),o=-1;if(n*=u,t*=u,e*=u,0>e)for(;(r=n+e*++o)>t;)i.push(r/u);else for(;(r=n+e*++o)<t;)i.push(r/u);return i},ao.map=function(n,t){var e=new c;if(n instanceof c)n.forEach(function(n,t){e.set(n,t)});else if(Array.isArray(n)){var r,i=-1,u=n.length;if(1===arguments.length)for(;++i<u;)e.set(i,n[i]);else for(;++i<u;)e.set(t.call(n,r=n[i],i),r)}else for(var o in n)e.set(o,n[o]);return e};var bo="__proto__",_o="\x00";l(c,{has:h,get:function(n){return this._[f(n)]},set:function(n,t){return this._[f(n)]=t},remove:p,keys:g,values:function(){var n=[];for(var t in this._)n.push(this._[t]);return n},entries:function(){var n=[];for(var t in this._)n.push({key:s(t),value:this._[t]});return n},size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t),this._[t])}}),ao.nest=function(){function n(t,o,a){if(a>=u.length)return r?r.call(i,o):e?o.sort(e):o;for(var l,f,s,h,p=-1,g=o.length,v=u[a++],d=new c;++p<g;)(h=d.get(l=v(f=o[p])))?h.push(f):d.set(l,[f]);return t?(f=t(),s=function(e,r){f.set(e,n(t,r,a))}):(f={},s=function(e,r){f[e]=n(t,r,a)}),d.forEach(s),f}function t(n,e){if(e>=u.length)return n;var r=[],i=o[e++];return n.forEach(function(n,i){r.push({key:n,values:t(i,e)})}),i?r.sort(function(n,t){return i(n.key,t.key)}):r}var e,r,i={},u=[],o=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n(ao.map,e,0),0)},i.key=function(n){return u.push(n),i},i.sortKeys=function(n){return o[u.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},ao.set=function(n){var t=new y;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},l(y,{has:h,add:function(n){return this._[f(n+="")]=!0,n},remove:p,values:g,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t))}}),ao.behavior={},ao.rebind=function(n,t){for(var e,r=1,i=arguments.length;++r<i;)n[e=arguments[r]]=M(n,t,t[e]);return n};var wo=["webkit","ms","moz","Moz","o","O"];ao.dispatch=function(){for(var n=new _,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=w(n);return n},_.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ao.event=null,ao.requote=function(n){return n.replace(So,"\\$&")};var So=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ko={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},No=function(n,t){return t.querySelector(n)},Eo=function(n,t){return t.querySelectorAll(n)},Ao=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(Ao=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(No=function(n,t){return Sizzle(n,t)[0]||null},Eo=Sizzle,Ao=Sizzle.matchesSelector),ao.selection=function(){return ao.select(fo.documentElement)};var Co=ao.selection.prototype=[];Co.select=function(n){var t,e,r,i,u=[];n=A(n);for(var o=-1,a=this.length;++o<a;){u.push(t=[]),t.parentNode=(r=this[o]).parentNode;for(var l=-1,c=r.length;++l<c;)(i=r[l])?(t.push(e=n.call(i,i.__data__,l,o)),e&&"__data__"in i&&(e.__data__=i.__data__)):t.push(null)}return E(u)},Co.selectAll=function(n){var t,e,r=[];n=C(n);for(var i=-1,u=this.length;++i<u;)for(var o=this[i],a=-1,l=o.length;++a<l;)(e=o[a])&&(r.push(t=co(n.call(e,e.__data__,a,i))),t.parentNode=e);return E(r)};var zo="http://www.w3.org/1999/xhtml",Lo={svg:"http://www.w3.org/2000/svg",xhtml:zo,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};ao.ns={prefix:Lo,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&"xmlns"!==(e=n.slice(0,t))&&(n=n.slice(t+1)),Lo.hasOwnProperty(e)?{space:Lo[e],local:n}:n}},Co.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ao.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},Co.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,i=-1;if(t=e.classList){for(;++i<r;)if(!t.contains(n[i]))return!1}else for(t=e.getAttribute("class");++i<r;)if(!q(n[i]).test(t))return!1;return!0}for(t in n)this.each(R(t,n[t]));return this}return this.each(R(n,t))},Co.style=function(n,e,r){var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>i){var u=this.node();return t(u).getComputedStyle(u,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},Co.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},Co.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},Co.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},Co.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},Co.insert=function(n,t){return n=j(n),t=A(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},Co.remove=function(){return this.each(F)},Co.data=function(n,t){function e(n,e){var r,i,u,o=n.length,s=e.length,h=Math.min(o,s),p=new Array(s),g=new Array(s),v=new Array(o);if(t){var d,y=new c,m=new Array(o);for(r=-1;++r<o;)(i=n[r])&&(y.has(d=t.call(i,i.__data__,r))?v[r]=i:y.set(d,i),m[r]=d);for(r=-1;++r<s;)(i=y.get(d=t.call(e,u=e[r],r)))?i!==!0&&(p[r]=i,i.__data__=u):g[r]=H(u),y.set(d,!0);for(r=-1;++r<o;)r in m&&y.get(m[r])!==!0&&(v[r]=n[r])}else{for(r=-1;++r<h;)i=n[r],u=e[r],i?(i.__data__=u,p[r]=i):g[r]=H(u);for(;s>r;++r)g[r]=H(e[r]);for(;o>r;++r)v[r]=n[r]}g.update=p,g.parentNode=p.parentNode=v.parentNode=n.parentNode,a.push(g),l.push(p),f.push(v)}var r,i,u=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++u<o;)(i=r[u])&&(n[u]=i.__data__);return n}var a=Z([]),l=E([]),f=E([]);if("function"==typeof n)for(;++u<o;)e(r=this[u],n.call(r,r.parentNode.__data__,u));else for(;++u<o;)e(r=this[u],n);return l.enter=function(){return a},l.exit=function(){return f},l},Co.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},Co.filter=function(n){var t,e,r,i=[];"function"!=typeof n&&(n=O(n));for(var u=0,o=this.length;o>u;u++){i.push(t=[]),t.parentNode=(e=this[u]).parentNode;for(var a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return E(i)},Co.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],i=r.length-1,u=r[i];--i>=0;)(e=r[i])&&(u&&u!==e.nextSibling&&u.parentNode.insertBefore(e,u),u=e);return this},Co.sort=function(n){n=I.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},Co.each=function(n){return Y(this,function(t,e,r){n.call(t,t.__data__,e,r)})},Co.call=function(n){var t=co(arguments);return n.apply(t[0]=this,t),this},Co.empty=function(){return!this.node()},Co.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,i=e.length;i>r;r++){var u=e[r];if(u)return u}return null},Co.size=function(){var n=0;return Y(this,function(){++n}),n};var qo=[];ao.selection.enter=Z,ao.selection.enter.prototype=qo,qo.append=Co.append,qo.empty=Co.empty,qo.node=Co.node,qo.call=Co.call,qo.size=Co.size,qo.select=function(n){for(var t,e,r,i,u,o=[],a=-1,l=this.length;++a<l;){r=(i=this[a]).update,o.push(t=[]),t.parentNode=i.parentNode;for(var c=-1,f=i.length;++c<f;)(u=i[c])?(t.push(r[c]=e=n.call(i.parentNode,u.__data__,c,a)),e.__data__=u.__data__):t.push(null)}return E(o)},qo.insert=function(n,t){return arguments.length<2&&(t=V(this)),Co.insert.call(this,n,t)},ao.select=function(t){var e;return"string"==typeof t?(e=[No(t,fo)],e.parentNode=fo.documentElement):(e=[t],e.parentNode=n(t)),E([e])},ao.selectAll=function(n){var t;return"string"==typeof n?(t=co(Eo(n,fo)),t.parentNode=fo.documentElement):(t=co(n),t.parentNode=null),E([t])},Co.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var To=ao.map({mouseenter:"mouseover",mouseleave:"mouseout"});fo&&To.forEach(function(n){"on"+n in fo&&To.remove(n)});var Ro,Do=0;ao.mouse=function(n){return J(n,k())};var Po=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ao.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,i=0,u=t.length;u>i;++i)if((r=t[i]).identifier===e)return J(n,r)},ao.behavior.drag=function(){function n(){this.on("mousedown.drag",u).on("touchstart.drag",o)}function e(n,t,e,u,o){return function(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],g|=n|e,M=r,p({type:"drag",x:r[0]+c[0],y:r[1]+c[1],dx:n,dy:e}))}function l(){t(h,v)&&(y.on(u+d,null).on(o+d,null),m(g),p({type:"dragend"}))}var c,f=this,s=ao.event.target.correspondingElement||ao.event.target,h=f.parentNode,p=r.of(f,arguments),g=0,v=n(),d=".drag"+(null==v?"":"-"+v),y=ao.select(e(s)).on(u+d,a).on(o+d,l),m=W(s),M=t(h,v);i?(c=i.apply(f,arguments),c=[c.x-M[0],c.y-M[1]]):c=[0,0],p({type:"dragstart"})}}var r=N(n,"drag","dragstart","dragend"),i=null,u=e(b,ao.mouse,t,"mousemove","mouseup"),o=e(G,ao.touch,m,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},ao.rebind(n,r,"on")},ao.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?co(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Uo=1e-6,jo=Uo*Uo,Fo=Math.PI,Ho=2*Fo,Oo=Ho-Uo,Io=Fo/2,Yo=Fo/180,Zo=180/Fo,Vo=Math.SQRT2,Xo=2,$o=4;ao.interpolateZoom=function(n,t){var e,r,i=n[0],u=n[1],o=n[2],a=t[0],l=t[1],c=t[2],f=a-i,s=l-u,h=f*f+s*s;if(jo>h)r=Math.log(c/o)/Vo,e=function(n){return[i+n*f,u+n*s,o*Math.exp(Vo*n*r)]};else{var p=Math.sqrt(h),g=(c*c-o*o+$o*h)/(2*o*Xo*p),v=(c*c-o*o-$o*h)/(2*c*Xo*p),d=Math.log(Math.sqrt(g*g+1)-g),y=Math.log(Math.sqrt(v*v+1)-v);r=(y-d)/Vo,e=function(n){var t=n*r,e=rn(d),a=o/(Xo*p)*(e*un(Vo*t+d)-en(d));return[i+a*f,u+a*s,o*e/rn(Vo*t+d)]}}return e.duration=1e3*r,e},ao.behavior.zoom=function(){function n(n){n.on(L,s).on(Wo+".zoom",p).on("dblclick.zoom",g).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function i(n){k.k=Math.max(A[0],Math.min(A[1],n))}function u(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},i(Math.pow(2,o)),u(d=e,r),t=ao.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k}).map(_.invert))}function l(n){z++||n({type:"zoomstart"})}function c(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function f(n){--z||(n({type:"zoomend"}),d=null)}function s(){function n(){a=1,u(ao.mouse(i),h),c(o)}function r(){s.on(q,null).on(T,null),p(a),f(o)}var i=this,o=D.of(i,arguments),a=0,s=ao.select(t(i)).on(q,n).on(T,r),h=e(ao.mouse(i)),p=W(i);Il.call(i),l(o)}function h(){function n(){var n=ao.touches(g);return p=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ao.event.target;ao.select(t).on(x,r).on(b,a),_.push(t);for(var e=ao.event.changedTouches,i=0,u=e.length;u>i;++i)d[e[i].identifier]=null;var l=n(),c=Date.now();if(1===l.length){if(500>c-M){var f=l[0];o(g,f,d[f.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=c}else if(l.length>1){var f=l[0],s=l[1],h=f[0]-s[0],p=f[1]-s[1];y=h*h+p*p}}function r(){var n,t,e,r,o=ao.touches(g);Il.call(g);for(var a=0,l=o.length;l>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e,t=r}if(r){var f=(f=e[0]-n[0])*f+(f=e[1]-n[1])*f,s=y&&Math.sqrt(f/y);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],i(s*p)}M=null,u(n,t),c(v)}function a(){if(ao.event.touches.length){for(var t=ao.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var i in d)return void n()}ao.selectAll(_).on(m,null),w.on(L,s).on(R,h),N(),f(v)}var p,g=this,v=D.of(g,arguments),d={},y=0,m=".zoom-"+ao.event.changedTouches[0].identifier,x="touchmove"+m,b="touchend"+m,_=[],w=ao.select(g),N=W(g);t(),l(v),w.on(L,null).on(R,t)}function p(){var n=D.of(this,arguments);m?clearTimeout(m):(Il.call(this),v=e(d=y||ao.mouse(this)),l(n)),m=setTimeout(function(){m=null,f(n)},50),S(),i(Math.pow(2,.002*Bo())*k.k),u(d,v),c(n)}function g(){var n=ao.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ao.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,y,m,M,x,b,_,w,k={x:0,y:0,k:1},E=[960,500],A=Jo,C=250,z=0,L="mousedown.zoom",q="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoom",D=N(n,"zoomstart","zoom","zoomend");return Wo||(Wo="onwheel"in fo?(Bo=function(){return-ao.event.deltaY*(ao.event.deltaMode?120:1)},"wheel"):"onmousewheel"in fo?(Bo=function(){return ao.event.wheelDelta},"mousewheel"):(Bo=function(){return-ao.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Hl?ao.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},l(n)}).tween("zoom:zoom",function(){var e=E[0],r=E[1],i=d?d[0]:e/2,u=d?d[1]:r/2,o=ao.interpolateZoom([(i-k.x)/k.k,(u-k.y)/k.k,e/k.k],[(i-t.x)/t.k,(u-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:i-r[0]*a,y:u-r[1]*a,k:a},c(n)}}).each("interrupt.zoom",function(){f(n)}).each("end.zoom",function(){f(n)}):(this.__chart__=k,l(n),c(n),f(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:null},i(+t),a(),n):k.k},n.scaleExtent=function(t){return arguments.length?(A=null==t?Jo:[+t[0],+t[1]],n):A},n.center=function(t){return arguments.length?(y=t&&[+t[0],+t[1]],n):y},n.size=function(t){return arguments.length?(E=t&&[+t[0],+t[1]],n):E},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ao.rebind(n,D,"on")};var Bo,Wo,Jo=[0,1/0];ao.color=an,an.prototype.toString=function(){return this.rgb()+""},ao.hsl=ln;var Go=ln.prototype=new an;Go.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,this.l/n)},Go.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,n*this.l)},Go.rgb=function(){return cn(this.h,this.s,this.l)},ao.hcl=fn;var Ko=fn.prototype=new an;Ko.brighter=function(n){return new fn(this.h,this.c,Math.min(100,this.l+Qo*(arguments.length?n:1)))},Ko.darker=function(n){return new fn(this.h,this.c,Math.max(0,this.l-Qo*(arguments.length?n:1)))},Ko.rgb=function(){return sn(this.h,this.c,this.l).rgb()},ao.lab=hn;var Qo=18,na=.95047,ta=1,ea=1.08883,ra=hn.prototype=new an;ra.brighter=function(n){return new hn(Math.min(100,this.l+Qo*(arguments.length?n:1)),this.a,this.b)},ra.darker=function(n){return new hn(Math.max(0,this.l-Qo*(arguments.length?n:1)),this.a,this.b)},ra.rgb=function(){return pn(this.l,this.a,this.b)},ao.rgb=mn;var ia=mn.prototype=new an;ia.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,i=30;return t||e||r?(t&&i>t&&(t=i),e&&i>e&&(e=i),r&&i>r&&(r=i),new mn(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mn(i,i,i)},ia.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mn(n*this.r,n*this.g,n*this.b)},ia.hsl=function(){return wn(this.r,this.g,this.b)},ia.toString=function(){return"#"+bn(this.r)+bn(this.g)+bn(this.b)};var ua=ao.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});ua.forEach(function(n,t){ua.set(n,Mn(t))}),ao.functor=En,ao.xhr=An(m),ao.dsv=function(n,t){function e(n,e,u){arguments.length<3&&(u=e,e=null);var o=Cn(n,t,null==e?r:i(e),u);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:i(n)):e},o}function r(n){return e.parse(n.responseText)}function i(n){return function(t){return e.parse(t.responseText,n)}}function u(t){return t.map(o).join(n)}function o(n){return a.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var a=new RegExp('["'+n+"\n]"),l=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var i=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(i(n),e)}:i})},e.parseRows=function(n,t){function e(){if(f>=c)return o;if(i)return i=!1,u;var t=f;if(34===n.charCodeAt(t)){for(var e=t;e++<c;)if(34===n.charCodeAt(e)){if(34!==n.charCodeAt(e+1))break;++e}f=e+2;var r=n.charCodeAt(e+1);return 13===r?(i=!0,10===n.charCodeAt(e+2)&&++f):10===r&&(i=!0),n.slice(t+1,e).replace(/""/g,'"')}for(;c>f;){var r=n.charCodeAt(f++),a=1;if(10===r)i=!0;else if(13===r)i=!0,10===n.charCodeAt(f)&&(++f,++a);else if(r!==l)continue;return n.slice(t,f-a)}return n.slice(t)}for(var r,i,u={},o={},a=[],c=n.length,f=0,s=0;(r=e())!==o;){for(var h=[];r!==u&&r!==o;)h.push(r),r=e();t&&null==(h=t(h,s++))||a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new y,i=[];return t.forEach(function(n){for(var t in n)r.has(t)||i.push(r.add(t))}),[i.map(o).join(n)].concat(t.map(function(t){return i.map(function(n){return o(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(u).join("\n")},e},ao.csv=ao.dsv(",","text/csv"),ao.tsv=ao.dsv(" ","text/tab-separated-values");var oa,aa,la,ca,fa=this[x(this,"requestAnimationFrame")]||function(n){setTimeout(n,17)};ao.timer=function(){qn.apply(this,arguments)},ao.timer.flush=function(){Rn(),Dn()},ao.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)};var sa=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(Un);ao.formatPrefix=function(n,t){var e=0;return(n=+n)&&(0>n&&(n*=-1),t&&(n=ao.round(n,Pn(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((e-1)/3)))),sa[8+e/3]};var ha=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,pa=ao.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=ao.round(n,Pn(n,t))).toFixed(Math.max(0,Math.min(20,Pn(n*(1+1e-15),t))))}}),ga=ao.time={},va=Date;Hn.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){da.setUTCDate.apply(this._,arguments)},setDay:function(){da.setUTCDay.apply(this._,arguments)},setFullYear:function(){da.setUTCFullYear.apply(this._,arguments)},setHours:function(){da.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){da.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){da.setUTCMinutes.apply(this._,arguments)},setMonth:function(){da.setUTCMonth.apply(this._,arguments)},setSeconds:function(){da.setUTCSeconds.apply(this._,arguments)},setTime:function(){da.setTime.apply(this._,arguments)}};var da=Date.prototype;ga.year=On(function(n){return n=ga.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),ga.years=ga.year.range,ga.years.utc=ga.year.utc.range,ga.day=On(function(n){var t=new va(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),ga.days=ga.day.range,ga.days.utc=ga.day.utc.range,ga.dayOfYear=function(n){var t=ga.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},["sunday","monday","tuesday","wednesday","thursday","friday","saturday"].forEach(function(n,t){t=7-t;var e=ga[n]=On(function(n){return(n=ga.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=ga.year(n).getDay();return Math.floor((ga.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});ga[n+"s"]=e.range,ga[n+"s"].utc=e.utc.range,ga[n+"OfYear"]=function(n){var e=ga.year(n).getDay();return Math.floor((ga.dayOfYear(n)+(e+t)%7)/7)}}),ga.week=ga.sunday,ga.weeks=ga.sunday.range,ga.weeks.utc=ga.sunday.utc.range,ga.weekOfYear=ga.sundayOfYear;var ya={"-":"",_:" ",0:"0"},ma=/^\s*\d+/,Ma=/^%/;ao.locale=function(n){return{numberFormat:jn(n),timeFormat:Yn(n)}};var xa=ao.locale({decimal:".",thousands:",",grouping:[3],currency:["$",""],dateTime:"%a %b %e %X %Y",date:"%m/%d/%Y",time:"%H:%M:%S",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});ao.format=xa.numberFormat,ao.geo={},ft.prototype={s:0,t:0,add:function(n){st(n,this.t,ba),st(ba.s,this.s,this),this.s?this.t+=ba.t:this.s=ba.t},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var ba=new ft;ao.geo.stream=function(n,t){n&&_a.hasOwnProperty(n.type)?_a[n.type](n,t):ht(n,t)};var _a={Feature:function(n,t){ht(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,i=e.length;++r<i;)ht(e[r].geometry,t)}},wa={Sphere:function(n,t){t.sphere()},Point:function(n,t){n=n.coordinates,t.point(n[0],n[1],n[2])},MultiPoint:function(n,t){for(var e=n.coordinates,r=-1,i=e.length;++r<i;)n=e[r],t.point(n[0],n[1],n[2])},LineString:function(n,t){pt(n.coordinates,t,0)},MultiLineString:function(n,t){for(var e=n.coordinates,r=-1,i=e.length;++r<i;)pt(e[r],t,0)},Polygon:function(n,t){gt(n.coordinates,t)},MultiPolygon:function(n,t){for(var e=n.coordinates,r=-1,i=e.length;++r<i;)gt(e[r],t)},GeometryCollection:function(n,t){for(var e=n.geometries,r=-1,i=e.length;++r<i;)ht(e[r],t)}};ao.geo.area=function(n){return Sa=0,ao.geo.stream(n,Na),Sa};var Sa,ka=new ft,Na={sphere:function(){Sa+=4*Fo},point:b,lineStart:b,lineEnd:b,polygonStart:function(){ka.reset(),Na.lineStart=vt},polygonEnd:function(){var n=2*ka;Sa+=0>n?4*Fo+n:n,Na.lineStart=Na.lineEnd=Na.point=b}};ao.geo.bounds=function(){function n(n,t){M.push(x=[f=n,h=n]),s>t&&(s=t),t>p&&(p=t)}function t(t,e){var r=dt([t*Yo,e*Yo]);if(y){var i=mt(y,r),u=[i[1],-i[0],0],o=mt(u,i);bt(o),o=_t(o);var l=t-g,c=l>0?1:-1,v=o[0]*Zo*c,d=xo(l)>180;if(d^(v>c*g&&c*t>v)){var m=o[1]*Zo;m>p&&(p=m)}else if(v=(v+360)%360-180,d^(v>c*g&&c*t>v)){var m=-o[1]*Zo;s>m&&(s=m)}else s>e&&(s=e),e>p&&(p=e);d?g>t?a(f,t)>a(f,h)&&(h=t):a(t,h)>a(f,h)&&(f=t):h>=f?(f>t&&(f=t),t>h&&(h=t)):t>g?a(f,t)>a(f,h)&&(h=t):a(t,h)>a(f,h)&&(f=t)}else n(t,e);y=r,g=t}function e(){b.point=t}function r(){x[0]=f,x[1]=h,b.point=n,y=null}function i(n,e){if(y){var r=n-g;m+=xo(r)>180?r+(r>0?360:-360):r}else v=n,d=e;Na.point(n,e),t(n,e)}function u(){Na.lineStart()}function o(){i(v,d),Na.lineEnd(),xo(m)>Uo&&(f=-(h=180)),x[0]=f,x[1]=h,y=null}function a(n,t){return(t-=n)<0?t+360:t}function l(n,t){return n[0]-t[0]}function c(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:n<t[0]||t[1]<n}var f,s,h,p,g,v,d,y,m,M,x,b={point:n,lineStart:e,lineEnd:r,polygonStart:function(){b.point=i,b.lineStart=u,b.lineEnd=o,m=0,Na.polygonStart()},polygonEnd:function(){Na.polygonEnd(),b.point=n,b.lineStart=e,b.lineEnd=r,0>ka?(f=-(h=180),s=-(p=90)):m>Uo?p=90:-Uo>m&&(s=-90),x[0]=f,x[1]=h}};return function(n){p=h=-(f=s=1/0),M=[],ao.geo.stream(n,b);var t=M.length;if(t){M.sort(l);for(var e,r=1,i=M[0],u=[i];t>r;++r)e=M[r],c(e[0],i)||c(e[1],i)?(a(i[0],e[1])>a(i[0],i[1])&&(i[1]=e[1]),a(e[0],i[1])>a(i[0],i[1])&&(i[0]=e[0])):u.push(i=e);for(var o,e,g=-(1/0),t=u.length-1,r=0,i=u[t];t>=r;i=e,++r)e=u[r],(o=a(i[1],e[0]))>g&&(g=o,f=e[0],h=i[1])}return M=x=null,f===1/0||s===1/0?[[NaN,NaN],[NaN,NaN]]:[[f,s],[h,p]]}}(),ao.geo.centroid=function(n){Ea=Aa=Ca=za=La=qa=Ta=Ra=Da=Pa=Ua=0,ao.geo.stream(n,ja);var t=Da,e=Pa,r=Ua,i=t*t+e*e+r*r;return jo>i&&(t=qa,e=Ta,r=Ra,Uo>Aa&&(t=Ca,e=za,r=La),i=t*t+e*e+r*r,jo>i)?[NaN,NaN]:[Math.atan2(e,t)*Zo,tn(r/Math.sqrt(i))*Zo]};var Ea,Aa,Ca,za,La,qa,Ta,Ra,Da,Pa,Ua,ja={sphere:b,point:St,lineStart:Nt,lineEnd:Et,polygonStart:function(){ja.lineStart=At},polygonEnd:function(){ja.lineStart=Nt}},Fa=Rt(zt,jt,Ht,[-Fo,-Fo/2]),Ha=1e9;ao.geo.clipExtent=function(){var n,t,e,r,i,u,o={stream:function(n){return i&&(i.valid=!1),i=u(n),i.valid=!0,i},extent:function(a){return arguments.length?(u=Zt(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),i&&(i.valid=!1,i=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(ao.geo.conicEqualArea=function(){return Vt(Xt)}).raw=Xt,ao.geo.albers=function(){return ao.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},ao.geo.albersUsa=function(){function n(n){var u=n[0],o=n[1];return t=null,e(u,o),t||(r(u,o),t)||i(u,o),t}var t,e,r,i,u=ao.geo.albers(),o=ao.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=ao.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),l={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=u.scale(),e=u.translate(),r=(n[0]-e[0])/t,i=(n[1]-e[1])/t;return(i>=.12&&.234>i&&r>=-.425&&-.214>r?o:i>=.166&&.234>i&&r>=-.214&&-.115>r?a:u).invert(n)},n.stream=function(n){var t=u.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,i){t.point(n,i),e.point(n,i),r.point(n,i)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(u.precision(t),o.precision(t),a.precision(t),n):u.precision()},n.scale=function(t){return arguments.length?(u.scale(t),o.scale(.35*t),a.scale(t),n.translate(u.translate())):u.scale()},n.translate=function(t){if(!arguments.length)return u.translate();var c=u.scale(),f=+t[0],s=+t[1];return e=u.translate(t).clipExtent([[f-.455*c,s-.238*c],[f+.455*c,s+.238*c]]).stream(l).point,r=o.translate([f-.307*c,s+.201*c]).clipExtent([[f-.425*c+Uo,s+.12*c+Uo],[f-.214*c-Uo,s+.234*c-Uo]]).stream(l).point,i=a.translate([f-.205*c,s+.212*c]).clipExtent([[f-.214*c+Uo,s+.166*c+Uo],[f-.115*c-Uo,s+.234*c-Uo]]).stream(l).point,n},n.scale(1070)};var Oa,Ia,Ya,Za,Va,Xa,$a={point:b,lineStart:b,lineEnd:b,polygonStart:function(){Ia=0,$a.lineStart=$t},polygonEnd:function(){$a.lineStart=$a.lineEnd=$a.point=b,Oa+=xo(Ia/2)}},Ba={point:Bt,lineStart:b,lineEnd:b,polygonStart:b,polygonEnd:b},Wa={point:Gt,lineStart:Kt,lineEnd:Qt,polygonStart:function(){Wa.lineStart=ne},polygonEnd:function(){Wa.point=Gt,Wa.lineStart=Kt,Wa.lineEnd=Qt}};ao.geo.path=function(){function n(n){return n&&("function"==typeof a&&u.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=i(u)),ao.geo.stream(n,o)),u.result()}function t(){return o=null,n}var e,r,i,u,o,a=4.5;return n.area=function(n){return Oa=0,ao.geo.stream(n,i($a)),Oa},n.centroid=function(n){return Ca=za=La=qa=Ta=Ra=Da=Pa=Ua=0,ao.geo.stream(n,i(Wa)),Ua?[Da/Ua,Pa/Ua]:Ra?[qa/Ra,Ta/Ra]:La?[Ca/La,za/La]:[NaN,NaN]},n.bounds=function(n){return Va=Xa=-(Ya=Za=1/0),ao.geo.stream(n,i(Ba)),[[Ya,Za],[Va,Xa]]},n.projection=function(n){return arguments.length?(i=(e=n)?n.stream||re(n):m,t()):e},n.context=function(n){return arguments.length?(u=null==(r=n)?new Wt:new te(n),"function"!=typeof a&&u.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(u.pointRadius(+t),+t),n):a},n.projection(ao.geo.albersUsa()).context(null)},ao.geo.transform=function(n){return{stream:function(t){var e=new ie(t);for(var r in n)e[r]=n[r];return e}}},ie.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},ao.geo.projection=oe,ao.geo.projectionMutator=ae,(ao.geo.equirectangular=function(){return oe(ce)}).raw=ce.invert=ce,ao.geo.rotation=function(n){function t(t){return t=n(t[0]*Yo,t[1]*Yo),t[0]*=Zo,t[1]*=Zo,t}return n=se(n[0]%360*Yo,n[1]*Yo,n.length>2?n[2]*Yo:0),t.invert=function(t){return t=n.invert(t[0]*Yo,t[1]*Yo),t[0]*=Zo,t[1]*=Zo,t},t},fe.invert=ce,ao.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=se(-n[0]*Yo,-n[1]*Yo,0).invert,i=[];return e(null,null,1,{point:function(n,e){i.push(n=t(n,e)),n[0]*=Zo,n[1]*=Zo}}),{type:"Polygon",coordinates:[i]}}var t,e,r=[0,0],i=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=ve((t=+r)*Yo,i*Yo),n):t},n.precision=function(r){return arguments.length?(e=ve(t*Yo,(i=+r)*Yo),n):i},n.angle(90)},ao.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Yo,i=n[1]*Yo,u=t[1]*Yo,o=Math.sin(r),a=Math.cos(r),l=Math.sin(i),c=Math.cos(i),f=Math.sin(u),s=Math.cos(u);return Math.atan2(Math.sqrt((e=s*o)*e+(e=c*f-l*s*a)*e),l*f+c*s*a)},ao.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return ao.range(Math.ceil(u/d)*d,i,d).map(h).concat(ao.range(Math.ceil(c/y)*y,l,y).map(p)).concat(ao.range(Math.ceil(r/g)*g,e,g).filter(function(n){return xo(n%d)>Uo}).map(f)).concat(ao.range(Math.ceil(a/v)*v,o,v).filter(function(n){return xo(n%y)>Uo}).map(s))}var e,r,i,u,o,a,l,c,f,s,h,p,g=10,v=g,d=90,y=360,m=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(u).concat(p(l).slice(1),h(i).reverse().slice(1),p(c).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(u=+t[0][0],i=+t[1][0],c=+t[0][1],l=+t[1][1],u>i&&(t=u,u=i,i=t),c>l&&(t=c,c=l,l=t),n.precision(m)):[[u,c],[i,l]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(m)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],y=+t[1],n):[d,y]},n.minorStep=function(t){return arguments.length?(g=+t[0],v=+t[1],n):[g,v]},n.precision=function(t){return arguments.length?(m=+t,f=ye(a,o,90),s=me(r,e,m),h=ye(c,l,90),p=me(u,i,m),n):m},n.majorExtent([[-180,-90+Uo],[180,90-Uo]]).minorExtent([[-180,-80-Uo],[180,80+Uo]])},ao.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||i.apply(this,arguments)]}}var t,e,r=Me,i=xe;return n.distance=function(){return ao.geo.distance(t||r.apply(this,arguments),e||i.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(i=t,e="function"==typeof t?null:t,n):i},n.precision=function(){return arguments.length?n:0},n},ao.geo.interpolate=function(n,t){return be(n[0]*Yo,n[1]*Yo,t[0]*Yo,t[1]*Yo)},ao.geo.length=function(n){return Ja=0,ao.geo.stream(n,Ga),Ja};var Ja,Ga={sphere:b,point:b,lineStart:_e,lineEnd:b,polygonStart:b,polygonEnd:b},Ka=we(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(ao.geo.azimuthalEqualArea=function(){return oe(Ka)}).raw=Ka;var Qa=we(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},m);(ao.geo.azimuthalEquidistant=function(){return oe(Qa)}).raw=Qa,(ao.geo.conicConformal=function(){return Vt(Se)}).raw=Se,(ao.geo.conicEquidistant=function(){return Vt(ke)}).raw=ke;var nl=we(function(n){return 1/n},Math.atan);(ao.geo.gnomonic=function(){return oe(nl)}).raw=nl,Ne.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Io]},(ao.geo.mercator=function(){return Ee(Ne)}).raw=Ne;var tl=we(function(){return 1},Math.asin);(ao.geo.orthographic=function(){return oe(tl)}).raw=tl;var el=we(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(ao.geo.stereographic=function(){return oe(el)}).raw=el,Ae.invert=function(n,t){return[-t,2*Math.atan(Math.exp(n))-Io]},(ao.geo.transverseMercator=function(){var n=Ee(Ae),t=n.center,e=n.rotate;return n.center=function(n){return n?t([-n[1],n[0]]):(n=t(),[n[1],-n[0]])},n.rotate=function(n){return n?e([n[0],n[1],n.length>2?n[2]+90:90]):(n=e(),[n[0],n[1],n[2]-90])},e([0,0,90])}).raw=Ae,ao.geom={},ao.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,i=En(e),u=En(r),o=n.length,a=[],l=[];for(t=0;o>t;t++)a.push([+i.call(this,n[t],t),+u.call(this,n[t],t),t]);for(a.sort(qe),t=0;o>t;t++)l.push([a[t][0],-a[t][1]]);var c=Le(a),f=Le(l),s=f[0]===c[0],h=f[f.length-1]===c[c.length-1],p=[];for(t=c.length-1;t>=0;--t)p.push(n[a[c[t]][2]]);for(t=+s;t<f.length-h;++t)p.push(n[a[f[t]][2]]);return p}var e=Ce,r=ze;return arguments.length?t(n):(t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t)},ao.geom.polygon=function(n){return ko(n,rl),n};var rl=ao.geom.polygon.prototype=[];rl.area=function(){for(var n,t=-1,e=this.length,r=this[e-1],i=0;++t<e;)n=r,r=this[t],i+=n[1]*r[0]-n[0]*r[1];return.5*i},rl.centroid=function(n){var t,e,r=-1,i=this.length,u=0,o=0,a=this[i-1];for(arguments.length||(n=-1/(6*this.area()));++r<i;)t=a,a=this[r],e=t[0]*a[1]-a[0]*t[1],u+=(t[0]+a[0])*e,o+=(t[1]+a[1])*e;return[u*n,o*n]},rl.clip=function(n){for(var t,e,r,i,u,o,a=De(n),l=-1,c=this.length-De(this),f=this[c-1];++l<c;){for(t=n.slice(),n.length=0,i=this[l],u=t[(r=t.length-a)-1],e=-1;++e<r;)o=t[e],Te(o,f,i)?(Te(u,f,i)||n.push(Re(u,o,f,i)),n.push(o)):Te(u,f,i)&&n.push(Re(u,o,f,i)),u=o;a&&n.push(n[0]),f=i}return n};var il,ul,ol,al,ll,cl=[],fl=[];Ye.prototype.prepare=function(){for(var n,t=this.edges,e=t.length;e--;)n=t[e].edge,n.b&&n.a||t.splice(e,1);return t.sort(Ve),t.length},tr.prototype={start:function(){return this.edge.l===this.site?this.edge.a:this.edge.b},end:function(){return this.edge.l===this.site?this.edge.b:this.edge.a}},er.prototype={insert:function(n,t){var e,r,i;if(n){if(t.P=n,t.N=n.N,n.N&&(n.N.P=t),n.N=t,n.R){for(n=n.R;n.L;)n=n.L;n.L=t}else n.R=t;e=n}else this._?(n=or(this._),t.P=null,t.N=n,n.P=n.L=t,e=n):(t.P=t.N=null,this._=t,e=null);for(t.L=t.R=null,t.U=e,t.C=!0,n=t;e&&e.C;)r=e.U,e===r.L?(i=r.R,i&&i.C?(e.C=i.C=!1,r.C=!0,n=r):(n===e.R&&(ir(this,e),n=e,e=n.U),e.C=!1,r.C=!0,ur(this,r))):(i=r.L,i&&i.C?(e.C=i.C=!1,r.C=!0,n=r):(n===e.L&&(ur(this,e),n=e,e=n.U),e.C=!1,r.C=!0,ir(this,r))),e=n.U;this._.C=!1},remove:function(n){n.N&&(n.N.P=n.P),n.P&&(n.P.N=n.N),n.N=n.P=null;var t,e,r,i=n.U,u=n.L,o=n.R;if(e=u?o?or(o):u:o,i?i.L===n?i.L=e:i.R=e:this._=e,u&&o?(r=e.C,e.C=n.C,e.L=u,u.U=e,e!==o?(i=e.U,e.U=n.U,n=e.R,i.L=n,e.R=o,o.U=e):(e.U=i,i=e,n=e.R)):(r=n.C,n=e),n&&(n.U=i),!r){if(n&&n.C)return void(n.C=!1);do{if(n===this._)break;if(n===i.L){if(t=i.R,t.C&&(t.C=!1,i.C=!0,ir(this,i),t=i.R),t.L&&t.L.C||t.R&&t.R.C){t.R&&t.R.C||(t.L.C=!1,t.C=!0,ur(this,t),t=i.R),t.C=i.C,i.C=t.R.C=!1,ir(this,i),n=this._;break}}else if(t=i.L,t.C&&(t.C=!1,i.C=!0,ur(this,i),t=i.L),t.L&&t.L.C||t.R&&t.R.C){t.L&&t.L.C||(t.R.C=!1,t.C=!0,ir(this,t),t=i.L),t.C=i.C,i.C=t.L.C=!1,ur(this,i),n=this._;break}t.C=!0,n=i,i=i.U}while(!n.C);n&&(n.C=!1)}}},ao.geom.voronoi=function(n){function t(n){var t=new Array(n.length),r=a[0][0],i=a[0][1],u=a[1][0],o=a[1][1];return ar(e(n),a).cells.forEach(function(e,a){var l=e.edges,c=e.site,f=t[a]=l.length?l.map(function(n){var t=n.start();return[t.x,t.y]}):c.x>=r&&c.x<=u&&c.y>=i&&c.y<=o?[[r,o],[u,o],[u,i],[r,i]]:[];f.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(u(n,t)/Uo)*Uo,y:Math.round(o(n,t)/Uo)*Uo,i:t}})}var r=Ce,i=ze,u=r,o=i,a=sl;return n?t(n):(t.links=function(n){return ar(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return ar(e(n)).cells.forEach(function(e,r){for(var i,u,o=e.site,a=e.edges.sort(Ve),l=-1,c=a.length,f=a[c-1].edge,s=f.l===o?f.r:f.l;++l<c;)i=f,u=s,f=a[l].edge,s=f.l===o?f.r:f.l,r<u.i&&r<s.i&&cr(o,u,s)<0&&t.push([n[r],n[u.i],n[s.i]])}),t},t.x=function(n){return arguments.length?(u=En(r=n),t):r},t.y=function(n){return arguments.length?(o=En(i=n),t):i},t.clipExtent=function(n){return arguments.length?(a=null==n?sl:n,t):a===sl?null:a},t.size=function(n){return arguments.length?t.clipExtent(n&&[[0,0],n]):a===sl?null:a&&a[1]},t)};var sl=[[-1e6,-1e6],[1e6,1e6]];ao.geom.delaunay=function(n){return ao.geom.voronoi().triangles(n)},ao.geom.quadtree=function(n,t,e,r,i){function u(n){function u(n,t,e,r,i,u,o,a){if(!isNaN(e)&&!isNaN(r))if(n.leaf){var l=n.x,f=n.y;if(null!=l)if(xo(l-e)+xo(f-r)<.01)c(n,t,e,r,i,u,o,a);else{var s=n.point;n.x=n.y=n.point=null,c(n,s,l,f,i,u,o,a),c(n,t,e,r,i,u,o,a)}else n.x=e,n.y=r,n.point=t}else c(n,t,e,r,i,u,o,a)}function c(n,t,e,r,i,o,a,l){var c=.5*(i+a),f=.5*(o+l),s=e>=c,h=r>=f,p=h<<1|s;n.leaf=!1,n=n.nodes[p]||(n.nodes[p]=hr()),s?i=c:a=c,h?o=f:l=f,u(n,t,e,r,i,o,a,l)}var f,s,h,p,g,v,d,y,m,M=En(a),x=En(l);if(null!=t)v=t,d=e,y=r,m=i;else if(y=m=-(v=d=1/0),s=[],h=[],g=n.length,o)for(p=0;g>p;++p)f=n[p],f.x<v&&(v=f.x),f.y<d&&(d=f.y),f.x>y&&(y=f.x),f.y>m&&(m=f.y),s.push(f.x),h.push(f.y);else for(p=0;g>p;++p){var b=+M(f=n[p],p),_=+x(f,p);v>b&&(v=b),d>_&&(d=_),b>y&&(y=b),_>m&&(m=_),s.push(b),h.push(_)}var w=y-v,S=m-d;w>S?m=d+w:y=v+S;var k=hr();if(k.add=function(n){u(k,n,+M(n,++p),+x(n,p),v,d,y,m)},k.visit=function(n){pr(n,k,v,d,y,m)},k.find=function(n){return gr(k,n[0],n[1],v,d,y,m)},p=-1,null==t){for(;++p<g;)u(k,n[p],s[p],h[p],v,d,y,m);--p}else n.forEach(k.add);return s=h=n=f=null,k}var o,a=Ce,l=ze;return(o=arguments.length)?(a=fr,l=sr,3===o&&(i=e,r=t,e=t=0),u(n)):(u.x=function(n){return arguments.length?(a=n,u):a},u.y=function(n){return arguments.length?(l=n,u):l},u.extent=function(n){return arguments.length?(null==n?t=e=r=i=null:(t=+n[0][0],e=+n[0][1],r=+n[1][0],i=+n[1][1]),u):null==t?null:[[t,e],[r,i]]},u.size=function(n){return arguments.length?(null==n?t=e=r=i=null:(t=e=0,r=+n[0],i=+n[1]),u):null==t?null:[r-t,i-e]},u)},ao.interpolateRgb=vr,ao.interpolateObject=dr,ao.interpolateNumber=yr,ao.interpolateString=mr;var hl=/[-+]?(?:\d+\.?\d*|\.?\d+)(?:[eE][-+]?\d+)?/g,pl=new RegExp(hl.source,"g");ao.interpolate=Mr,ao.interpolators=[function(n,t){var e=typeof t;return("string"===e?ua.has(t.toLowerCase())||/^(#|rgb\(|hsl\()/i.test(t)?vr:mr:t instanceof an?vr:Array.isArray(t)?xr:"object"===e&&isNaN(t)?dr:yr)(n,t)}],ao.interpolateArray=xr;var gl=function(){return m},vl=ao.map({linear:gl,poly:Er,quad:function(){return Sr},cubic:function(){return kr},sin:function(){return Ar},exp:function(){return Cr},circle:function(){return zr},elastic:Lr,back:qr,bounce:function(){return Tr}}),dl=ao.map({"in":m,out:_r,"in-out":wr,"out-in":function(n){return wr(_r(n))}});ao.ease=function(n){var t=n.indexOf("-"),e=t>=0?n.slice(0,t):n,r=t>=0?n.slice(t+1):"in";return e=vl.get(e)||gl,r=dl.get(r)||m,br(r(e.apply(null,lo.call(arguments,1))))},ao.interpolateHcl=Rr,ao.interpolateHsl=Dr,ao.interpolateLab=Pr,ao.interpolateRound=Ur,ao.transform=function(n){var t=fo.createElementNS(ao.ns.prefix.svg,"g");return(ao.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new jr(e?e.matrix:yl)})(n)},jr.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var yl={a:1,b:0,c:0,d:1,e:0,f:0};ao.interpolateTransform=$r,ao.layout={},ao.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++e<r;)t.push(Jr(n[e]));return t}},ao.layout.chord=function(){function n(){var n,c,s,h,p,g={},v=[],d=ao.range(u),y=[];for(e=[],r=[],n=0,h=-1;++h<u;){for(c=0,p=-1;++p<u;)c+=i[h][p];v.push(c),y.push(ao.range(u)),n+=c}for(o&&d.sort(function(n,t){return o(v[n],v[t])}),a&&y.forEach(function(n,t){n.sort(function(n,e){return a(i[t][n],i[t][e])})}),n=(Ho-f*u)/n,c=0,h=-1;++h<u;){for(s=c,p=-1;++p<u;){var m=d[h],M=y[m][p],x=i[m][M],b=c,_=c+=x*n;g[m+"-"+M]={index:m,subindex:M,startAngle:b,endAngle:_,value:x}}r[m]={index:m,startAngle:s,endAngle:c,value:v[m]},c+=f}for(h=-1;++h<u;)for(p=h-1;++p<u;){var w=g[h+"-"+p],S=g[p+"-"+h];(w.value||S.value)&&e.push(w.value<S.value?{source:S,target:w}:{source:w,target:S})}l&&t()}function t(){e.sort(function(n,t){return l((n.source.value+n.target.value)/2,(t.source.value+t.target.value)/2)})}var e,r,i,u,o,a,l,c={},f=0;return c.matrix=function(n){return arguments.length?(u=(i=n)&&i.length,e=r=null,c):i},c.padding=function(n){return arguments.length?(f=n,e=r=null,c):f},c.sortGroups=function(n){return arguments.length?(o=n,e=r=null,c):o},c.sortSubgroups=function(n){return arguments.length?(a=n,e=null,c):a},c.sortChords=function(n){return arguments.length?(l=n,e&&t(),c):l},c.chords=function(){return e||n(),e},c.groups=function(){return r||n(),r},c},ao.layout.force=function(){function n(n){return function(t,e,r,i){if(t.point!==n){var u=t.cx-n.x,o=t.cy-n.y,a=i-e,l=u*u+o*o;if(l>a*a/y){if(v>l){var c=t.charge/l;n.px-=u*c,n.py-=o*c}return!0}if(t.point&&l&&v>l){var c=t.pointCharge/l;n.px-=u*c,n.py-=o*c}}return!t.charge}}function t(n){n.px=ao.event.x,n.py=ao.event.y,l.resume()}var e,r,i,u,o,a,l={},c=ao.dispatch("start","tick","end"),f=[1,1],s=.9,h=ml,p=Ml,g=-30,v=xl,d=.1,y=.64,M=[],x=[];return l.tick=function(){if((i*=.99)<.005)return e=null,c.end({type:"end",alpha:i=0}),!0;var t,r,l,h,p,v,y,m,b,_=M.length,w=x.length;for(r=0;w>r;++r)l=x[r],h=l.source,p=l.target,m=p.x-h.x,b=p.y-h.y,(v=m*m+b*b)&&(v=i*o[r]*((v=Math.sqrt(v))-u[r])/v,m*=v,b*=v,p.x-=m*(y=h.weight+p.weight?h.weight/(h.weight+p.weight):.5),p.y-=b*y,h.x+=m*(y=1-y),h.y+=b*y);if((y=i*d)&&(m=f[0]/2,b=f[1]/2,r=-1,y))for(;++r<_;)l=M[r],l.x+=(m-l.x)*y,l.y+=(b-l.y)*y;if(g)for(ri(t=ao.geom.quadtree(M),i,a),r=-1;++r<_;)(l=M[r]).fixed||t.visit(n(l));for(r=-1;++r<_;)l=M[r],l.fixed?(l.x=l.px,l.y=l.py):(l.x-=(l.px-(l.px=l.x))*s,l.y-=(l.py-(l.py=l.y))*s);c.tick({type:"tick",alpha:i})},l.nodes=function(n){return arguments.length?(M=n,l):M},l.links=function(n){return arguments.length?(x=n,l):x},l.size=function(n){return arguments.length?(f=n,l):f},l.linkDistance=function(n){return arguments.length?(h="function"==typeof n?n:+n,l):h},l.distance=l.linkDistance,l.linkStrength=function(n){return arguments.length?(p="function"==typeof n?n:+n,l):p},l.friction=function(n){return arguments.length?(s=+n,l):s},l.charge=function(n){return arguments.length?(g="function"==typeof n?n:+n,l):g},l.chargeDistance=function(n){return arguments.length?(v=n*n,l):Math.sqrt(v)},l.gravity=function(n){return arguments.length?(d=+n,l):d},l.theta=function(n){return arguments.length?(y=n*n,l):Math.sqrt(y)},l.alpha=function(n){return arguments.length?(n=+n,i?n>0?i=n:(e.c=null,e.t=NaN,e=null,c.end({type:"end",alpha:i=0})):n>0&&(c.start({type:"start",alpha:i=n}),e=qn(l.tick)),l):i},l.start=function(){function n(n,r){if(!e){for(e=new Array(i),l=0;i>l;++l)e[l]=[];for(l=0;c>l;++l){var u=x[l];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var o,a=e[t],l=-1,f=a.length;++l<f;)if(!isNaN(o=a[l][n]))return o;return Math.random()*r}var t,e,r,i=M.length,c=x.length,s=f[0],v=f[1];for(t=0;i>t;++t)(r=M[t]).index=t,r.weight=0;for(t=0;c>t;++t)r=x[t],"number"==typeof r.source&&(r.source=M[r.source]),"number"==typeof r.target&&(r.target=M[r.target]),++r.source.weight,++r.target.weight;for(t=0;i>t;++t)r=M[t],isNaN(r.x)&&(r.x=n("x",s)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof h)for(t=0;c>t;++t)u[t]=+h.call(this,x[t],t);else for(t=0;c>t;++t)u[t]=h;if(o=[],"function"==typeof p)for(t=0;c>t;++t)o[t]=+p.call(this,x[t],t);else for(t=0;c>t;++t)o[t]=p;if(a=[],"function"==typeof g)for(t=0;i>t;++t)a[t]=+g.call(this,M[t],t);else for(t=0;i>t;++t)a[t]=g;return l.resume()},l.resume=function(){return l.alpha(.1)},l.stop=function(){return l.alpha(0)},l.drag=function(){return r||(r=ao.behavior.drag().origin(m).on("dragstart.force",Qr).on("drag.force",t).on("dragend.force",ni)),arguments.length?void this.on("mouseover.force",ti).on("mouseout.force",ei).call(r):r},ao.rebind(l,c,"on")};var ml=20,Ml=1,xl=1/0;ao.layout.hierarchy=function(){function n(i){var u,o=[i],a=[];for(i.depth=0;null!=(u=o.pop());)if(a.push(u),(c=e.call(n,u,u.depth))&&(l=c.length)){for(var l,c,f;--l>=0;)o.push(f=c[l]),f.parent=u,f.depth=u.depth+1;r&&(u.value=0),u.children=c}else r&&(u.value=+r.call(n,u,u.depth)||0),delete u.children;return oi(i,function(n){var e,i;t&&(e=n.children)&&e.sort(t),r&&(i=n.parent)&&(i.value+=n.value)}),a}var t=ci,e=ai,r=li;return n.sort=function(e){return arguments.length?(t=e,n):t},n.children=function(t){return arguments.length?(e=t,n):e},n.value=function(t){return arguments.length?(r=t,n):r},n.revalue=function(t){return r&&(ui(t,function(n){n.children&&(n.value=0)}),oi(t,function(t){var e;t.children||(t.value=+r.call(n,t,t.depth)||0),(e=t.parent)&&(e.value+=t.value)})),t},n},ao.layout.partition=function(){function n(t,e,r,i){var u=t.children;if(t.x=e,t.y=t.depth*i,t.dx=r,t.dy=i,u&&(o=u.length)){var o,a,l,c=-1;for(r=t.value?r/t.value:0;++c<o;)n(a=u[c],e,l=a.value*r,i),e+=l}}function t(n){var e=n.children,r=0;if(e&&(i=e.length))for(var i,u=-1;++u<i;)r=Math.max(r,t(e[u]));return 1+r}function e(e,u){var o=r.call(this,e,u);return n(o[0],0,i[0],i[1]/t(o[0])),o}var r=ao.layout.hierarchy(),i=[1,1];return e.size=function(n){return arguments.length?(i=n,e):i},ii(e,r)},ao.layout.pie=function(){function n(o){var a,l=o.length,c=o.map(function(e,r){return+t.call(n,e,r)}),f=+("function"==typeof r?r.apply(this,arguments):r),s=("function"==typeof i?i.apply(this,arguments):i)-f,h=Math.min(Math.abs(s)/l,+("function"==typeof u?u.apply(this,arguments):u)),p=h*(0>s?-1:1),g=ao.sum(c),v=g?(s-l*p)/g:0,d=ao.range(l),y=[];return null!=e&&d.sort(e===bl?function(n,t){return c[t]-c[n]}:function(n,t){return e(o[n],o[t])}),d.forEach(function(n){y[n]={data:o[n],value:a=c[n],startAngle:f,endAngle:f+=a*v+p,padAngle:h}}),y}var t=Number,e=bl,r=0,i=Ho,u=0;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(i=t,n):i},n.padAngle=function(t){return arguments.length?(u=t,n):u},n};var bl={};ao.layout.stack=function(){function n(a,l){if(!(h=a.length))return a;var c=a.map(function(e,r){return t.call(n,e,r)}),f=c.map(function(t){return t.map(function(t,e){return[u.call(n,t,e),o.call(n,t,e)]})}),s=e.call(n,f,l);c=ao.permute(c,s),f=ao.permute(f,s);var h,p,g,v,d=r.call(n,f,l),y=c[0].length;for(g=0;y>g;++g)for(i.call(n,c[0][g],v=d[g],f[0][g][1]),p=1;h>p;++p)i.call(n,c[p][g],v+=f[p-1][g][1],f[p][g][1]);return a}var t=m,e=gi,r=vi,i=pi,u=si,o=hi;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:_l.get(t)||gi,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:wl.get(t)||vi,n):r},n.x=function(t){return arguments.length?(u=t,n):u},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(i=t,n):i},n};var _l=ao.map({"inside-out":function(n){var t,e,r=n.length,i=n.map(di),u=n.map(yi),o=ao.range(r).sort(function(n,t){return i[n]-i[t]}),a=0,l=0,c=[],f=[];for(t=0;r>t;++t)e=o[t],l>a?(a+=u[e],c.push(e)):(l+=u[e],f.push(e));return f.reverse().concat(c)},reverse:function(n){return ao.range(n.length).reverse()},"default":gi}),wl=ao.map({silhouette:function(n){var t,e,r,i=n.length,u=n[0].length,o=[],a=0,l=[];for(e=0;u>e;++e){for(t=0,r=0;i>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;u>e;++e)l[e]=(a-o[e])/2;return l},wiggle:function(n){var t,e,r,i,u,o,a,l,c,f=n.length,s=n[0],h=s.length,p=[];for(p[0]=l=c=0,e=1;h>e;++e){for(t=0,i=0;f>t;++t)i+=n[t][e][1];for(t=0,u=0,a=s[e][0]-s[e-1][0];f>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;u+=o*n[t][e][1]}p[e]=l-=i?u/i*a:0,c>l&&(c=l)}for(e=0;h>e;++e)p[e]-=c;return p},expand:function(n){var t,e,r,i=n.length,u=n[0].length,o=1/i,a=[];for(e=0;u>e;++e){for(t=0,r=0;i>t;t++)r+=n[t][e][1];if(r)for(t=0;i>t;t++)n[t][e][1]/=r;else for(t=0;i>t;t++)n[t][e][1]=o}for(e=0;u>e;++e)a[e]=0;return a},zero:vi});ao.layout.histogram=function(){function n(n,u){for(var o,a,l=[],c=n.map(e,this),f=r.call(this,c,u),s=i.call(this,f,c,u),u=-1,h=c.length,p=s.length-1,g=t?1:1/h;++u<p;)o=l[u]=[],o.dx=s[u+1]-(o.x=s[u]),o.y=0;if(p>0)for(u=-1;++u<h;)a=c[u],a>=f[0]&&a<=f[1]&&(o=l[ao.bisect(s,a,1,p)-1],o.y+=g,o.push(n[u]));return l}var t=!0,e=Number,r=bi,i=Mi;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=En(t),n):r},n.bins=function(t){return arguments.length?(i="number"==typeof t?function(n){return xi(n,t)}:En(t),n):i},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},ao.layout.pack=function(){function n(n,u){var o=e.call(this,n,u),a=o[0],l=i[0],c=i[1],f=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,oi(a,function(n){n.r=+f(n.value)}),oi(a,Ni),r){var s=r*(t?1:Math.max(2*a.r/l,2*a.r/c))/2;oi(a,function(n){n.r+=s}),oi(a,Ni),oi(a,function(n){n.r-=s})}return Ci(a,l/2,c/2,t?1:1/Math.max(2*a.r/l,2*a.r/c)),o}var t,e=ao.layout.hierarchy().sort(_i),r=0,i=[1,1];return n.size=function(t){return arguments.length?(i=t,n):i},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},ii(n,e)},ao.layout.tree=function(){function n(n,i){var f=o.call(this,n,i),s=f[0],h=t(s);if(oi(h,e),h.parent.m=-h.z,ui(h,r),c)ui(s,u);else{var p=s,g=s,v=s;ui(s,function(n){n.x<p.x&&(p=n),n.x>g.x&&(g=n),n.depth>v.depth&&(v=n)});var d=a(p,g)/2-p.x,y=l[0]/(g.x+a(g,p)/2+d),m=l[1]/(v.depth||1);ui(s,function(n){n.x=(n.x+d)*y,n.y=n.depth*m})}return f}function t(n){for(var t,e={A:null,children:[n]},r=[e];null!=(t=r.pop());)for(var i,u=t.children,o=0,a=u.length;a>o;++o)r.push((u[o]=i={_:u[o],parent:t,children:(i=u[o].children)&&i.slice()||[],A:null,a:null,z:0,m:0,c:0,s:0,t:null,i:o}).a=i);return e.children[0]}function e(n){var t=n.children,e=n.parent.children,r=n.i?e[n.i-1]:null;if(t.length){Di(n);var u=(t[0].z+t[t.length-1].z)/2;r?(n.z=r.z+a(n._,r._),n.m=n.z-u):n.z=u}else r&&(n.z=r.z+a(n._,r._));n.parent.A=i(n,r,n.parent.A||e[0])}function r(n){n._.x=n.z+n.parent.m,n.m+=n.parent.m}function i(n,t,e){if(t){for(var r,i=n,u=n,o=t,l=i.parent.children[0],c=i.m,f=u.m,s=o.m,h=l.m;o=Ti(o),i=qi(i),o&&i;)l=qi(l),u=Ti(u),u.a=n,r=o.z+s-i.z-c+a(o._,i._),r>0&&(Ri(Pi(o,n,e),n,r),c+=r,f+=r),s+=o.m,c+=i.m,h+=l.m,f+=u.m;o&&!Ti(u)&&(u.t=o,u.m+=s-f),i&&!qi(l)&&(l.t=i,l.m+=c-h,e=n)}return e}function u(n){n.x*=l[0],n.y=n.depth*l[1]}var o=ao.layout.hierarchy().sort(null).value(null),a=Li,l=[1,1],c=null;return n.separation=function(t){return arguments.length?(a=t,n):a},n.size=function(t){return arguments.length?(c=null==(l=t)?u:null,n):c?null:l},n.nodeSize=function(t){return arguments.length?(c=null==(l=t)?null:u,n):c?l:null},ii(n,o)},ao.layout.cluster=function(){function n(n,u){var o,a=t.call(this,n,u),l=a[0],c=0;oi(l,function(n){var t=n.children;t&&t.length?(n.x=ji(t),n.y=Ui(t)):(n.x=o?c+=e(n,o):0,n.y=0,o=n)});var f=Fi(l),s=Hi(l),h=f.x-e(f,s)/2,p=s.x+e(s,f)/2;return oi(l,i?function(n){n.x=(n.x-l.x)*r[0],n.y=(l.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(p-h)*r[0],n.y=(1-(l.y?n.y/l.y:1))*r[1]}),a}var t=ao.layout.hierarchy().sort(null).value(null),e=Li,r=[1,1],i=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(i=null==(r=t),n):i?null:r},n.nodeSize=function(t){return arguments.length?(i=null!=(r=t),n):i?r:null},ii(n,t)},ao.layout.treemap=function(){function n(n,t){for(var e,r,i=-1,u=n.length;++i<u;)r=(e=n[i]).value*(0>t?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var u=e.children;if(u&&u.length){var o,a,l,c=s(e),f=[],h=u.slice(),g=1/0,v="slice"===p?c.dx:"dice"===p?c.dy:"slice-dice"===p?1&e.depth?c.dy:c.dx:Math.min(c.dx,c.dy);for(n(h,c.dx*c.dy/e.value),f.area=0;(l=h.length)>0;)f.push(o=h[l-1]),f.area+=o.area,"squarify"!==p||(a=r(f,v))<=g?(h.pop(),g=a):(f.area-=f.pop().area,i(f,v,c,!1),v=Math.min(c.dx,c.dy),f.length=f.area=0,g=1/0);f.length&&(i(f,v,c,!0),f.length=f.area=0),u.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var u,o=s(t),a=r.slice(),l=[];for(n(a,o.dx*o.dy/t.value),l.area=0;u=a.pop();)l.push(u),l.area+=u.area,null!=u.z&&(i(l,u.z?o.dx:o.dy,o,!a.length),l.length=l.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,i=0,u=1/0,o=-1,a=n.length;++o<a;)(e=n[o].area)&&(u>e&&(u=e),e>i&&(i=e));return r*=r,t*=t,r?Math.max(t*i*g/r,r/(t*u*g)):1/0}function i(n,t,e,r){var i,u=-1,o=n.length,a=e.x,c=e.y,f=t?l(n.area/t):0;if(t==e.dx){for((r||f>e.dy)&&(f=e.dy);++u<o;)i=n[u],i.x=a,i.y=c,i.dy=f,a+=i.dx=Math.min(e.x+e.dx-a,f?l(i.area/f):0);i.z=!0,i.dx+=e.x+e.dx-a,e.y+=f,e.dy-=f}else{for((r||f>e.dx)&&(f=e.dx);++u<o;)i=n[u],i.x=a,i.y=c,i.dx=f,c+=i.dy=Math.min(e.y+e.dy-c,f?l(i.area/f):0);i.z=!1,i.dy+=e.y+e.dy-c,e.x+=f,e.dx-=f}}function u(r){var i=o||a(r),u=i[0];return u.x=u.y=0,u.value?(u.dx=c[0],u.dy=c[1]):u.dx=u.dy=0,o&&a.revalue(u),n([u],u.dx*u.dy/u.value),(o?e:t)(u),h&&(o=i),i}var o,a=ao.layout.hierarchy(),l=Math.round,c=[1,1],f=null,s=Oi,h=!1,p="squarify",g=.5*(1+Math.sqrt(5));return u.size=function(n){return arguments.length?(c=n,u):c},u.padding=function(n){function t(t){var e=n.call(u,t,t.depth);return null==e?Oi(t):Ii(t,"number"==typeof e?[e,e,e,e]:e)}function e(t){return Ii(t,n)}if(!arguments.length)return f;var r;return s=null==(f=n)?Oi:"function"==(r=typeof n)?t:"number"===r?(n=[n,n,n,n],e):e,u},u.round=function(n){return arguments.length?(l=n?Math.round:Number,u):l!=Number},u.sticky=function(n){return arguments.length?(h=n,o=null,u):h},u.ratio=function(n){return arguments.length?(g=n,u):g},u.mode=function(n){return arguments.length?(p=n+"",u):p},ii(u,a)},ao.random={normal:function(n,t){var e=arguments.length;return 2>e&&(t=1),1>e&&(n=0),function(){var e,r,i;do e=2*Math.random()-1,r=2*Math.random()-1,i=e*e+r*r;while(!i||i>1);return n+t*e*Math.sqrt(-2*Math.log(i)/i)}},logNormal:function(){var n=ao.random.normal.apply(ao,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=ao.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},ao.scale={};var Sl={floor:m,ceil:m};ao.scale.linear=function(){return Wi([0,1],[0,1],Mr,!1)};var kl={s:1,g:1,p:1,r:1,e:1};ao.scale.log=function(){return ru(ao.scale.linear().domain([0,1]),10,!0,[1,10])};var Nl=ao.format(".0e"),El={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};ao.scale.pow=function(){return iu(ao.scale.linear(),1,[0,1])},ao.scale.sqrt=function(){return ao.scale.pow().exponent(.5)},ao.scale.ordinal=function(){return ou([],{t:"range",a:[[]]})},ao.scale.category10=function(){return ao.scale.ordinal().range(Al)},ao.scale.category20=function(){return ao.scale.ordinal().range(Cl)},ao.scale.category20b=function(){return ao.scale.ordinal().range(zl)},ao.scale.category20c=function(){return ao.scale.ordinal().range(Ll)};var Al=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(xn),Cl=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(xn),zl=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(xn),Ll=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(xn);ao.scale.quantile=function(){return au([],[])},ao.scale.quantize=function(){return lu(0,1,[0,1])},ao.scale.threshold=function(){return cu([.5],[0,1])},ao.scale.identity=function(){return fu([0,1])},ao.svg={},ao.svg.arc=function(){function n(){var n=Math.max(0,+e.apply(this,arguments)),c=Math.max(0,+r.apply(this,arguments)),f=o.apply(this,arguments)-Io,s=a.apply(this,arguments)-Io,h=Math.abs(s-f),p=f>s?0:1;if(n>c&&(g=c,c=n,n=g),h>=Oo)return t(c,p)+(n?t(n,1-p):"")+"Z";var g,v,d,y,m,M,x,b,_,w,S,k,N=0,E=0,A=[];if((y=(+l.apply(this,arguments)||0)/2)&&(d=u===ql?Math.sqrt(n*n+c*c):+u.apply(this,arguments),p||(E*=-1),c&&(E=tn(d/c*Math.sin(y))),n&&(N=tn(d/n*Math.sin(y)))),c){m=c*Math.cos(f+E),M=c*Math.sin(f+E),x=c*Math.cos(s-E),b=c*Math.sin(s-E);var C=Math.abs(s-f-2*E)<=Fo?0:1;if(E&&yu(m,M,x,b)===p^C){var z=(f+s)/2;m=c*Math.cos(z),M=c*Math.sin(z),x=b=null}}else m=M=0;if(n){_=n*Math.cos(s-N),w=n*Math.sin(s-N),S=n*Math.cos(f+N),k=n*Math.sin(f+N);var L=Math.abs(f-s+2*N)<=Fo?0:1;if(N&&yu(_,w,S,k)===1-p^L){var q=(f+s)/2;_=n*Math.cos(q),w=n*Math.sin(q),S=k=null}}else _=w=0;if(h>Uo&&(g=Math.min(Math.abs(c-n)/2,+i.apply(this,arguments)))>.001){v=c>n^p?0:1;var T=g,R=g;if(Fo>h){var D=null==S?[_,w]:null==x?[m,M]:Re([m,M],[S,k],[x,b],[_,w]),P=m-D[0],U=M-D[1],j=x-D[0],F=b-D[1],H=1/Math.sin(Math.acos((P*j+U*F)/(Math.sqrt(P*P+U*U)*Math.sqrt(j*j+F*F)))/2),O=Math.sqrt(D[0]*D[0]+D[1]*D[1]);R=Math.min(g,(n-O)/(H-1)),T=Math.min(g,(c-O)/(H+1))}if(null!=x){var I=mu(null==S?[_,w]:[S,k],[m,M],c,T,p),Y=mu([x,b],[_,w],c,T,p);g===T?A.push("M",I[0],"A",T,",",T," 0 0,",v," ",I[1],"A",c,",",c," 0 ",1-p^yu(I[1][0],I[1][1],Y[1][0],Y[1][1]),",",p," ",Y[1],"A",T,",",T," 0 0,",v," ",Y[0]):A.push("M",I[0],"A",T,",",T," 0 1,",v," ",Y[0])}else A.push("M",m,",",M);if(null!=S){var Z=mu([m,M],[S,k],n,-R,p),V=mu([_,w],null==x?[m,M]:[x,b],n,-R,p);g===R?A.push("L",V[0],"A",R,",",R," 0 0,",v," ",V[1],"A",n,",",n," 0 ",p^yu(V[1][0],V[1][1],Z[1][0],Z[1][1]),",",1-p," ",Z[1],"A",R,",",R," 0 0,",v," ",Z[0]):A.push("L",V[0],"A",R,",",R," 0 0,",v," ",Z[0])}else A.push("L",_,",",w)}else A.push("M",m,",",M),null!=x&&A.push("A",c,",",c," 0 ",C,",",p," ",x,",",b),A.push("L",_,",",w),null!=S&&A.push("A",n,",",n," 0 ",L,",",1-p," ",S,",",k);return A.push("Z"),A.join("")}function t(n,t){return"M0,"+n+"A"+n+","+n+" 0 1,"+t+" 0,"+-n+"A"+n+","+n+" 0 1,"+t+" 0,"+n}var e=hu,r=pu,i=su,u=ql,o=gu,a=vu,l=du;return n.innerRadius=function(t){return arguments.length?(e=En(t),n):e},n.outerRadius=function(t){return arguments.length?(r=En(t),n):r},n.cornerRadius=function(t){return arguments.length?(i=En(t),n):i},n.padRadius=function(t){return arguments.length?(u=t==ql?ql:En(t),n):u},n.startAngle=function(t){return arguments.length?(o=En(t),n):o},n.endAngle=function(t){return arguments.length?(a=En(t),n):a},n.padAngle=function(t){return arguments.length?(l=En(t),n):l},n.centroid=function(){var n=(+e.apply(this,arguments)+ +r.apply(this,arguments))/2,t=(+o.apply(this,arguments)+ +a.apply(this,arguments))/2-Io;return[Math.cos(t)*n,Math.sin(t)*n]},n};var ql="auto";ao.svg.line=function(){return Mu(m)};var Tl=ao.map({linear:xu,"linear-closed":bu,step:_u,"step-before":wu,"step-after":Su,basis:zu,"basis-open":Lu,"basis-closed":qu,bundle:Tu,cardinal:Eu,"cardinal-open":ku,"cardinal-closed":Nu,monotone:Fu});Tl.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Rl=[0,2/3,1/3,0],Dl=[0,1/3,2/3,0],Pl=[0,1/6,2/3,1/6];ao.svg.line.radial=function(){var n=Mu(Hu);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},wu.reverse=Su,Su.reverse=wu,ao.svg.area=function(){return Ou(m)},ao.svg.area.radial=function(){var n=Ou(Hu);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},ao.svg.chord=function(){function n(n,a){var l=t(this,u,n,a),c=t(this,o,n,a);return"M"+l.p0+r(l.r,l.p1,l.a1-l.a0)+(e(l,c)?i(l.r,l.p1,l.r,l.p0):i(l.r,l.p1,c.r,c.p0)+r(c.r,c.p1,c.a1-c.a0)+i(c.r,c.p1,l.r,l.p0))+"Z"}function t(n,t,e,r){var i=t.call(n,e,r),u=a.call(n,i,r),o=l.call(n,i,r)-Io,f=c.call(n,i,r)-Io;return{r:u,a0:o,a1:f,p0:[u*Math.cos(o),u*Math.sin(o)],p1:[u*Math.cos(f),u*Math.sin(f)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>Fo)+",1 "+t}function i(n,t,e,r){return"Q 0,0 "+r}var u=Me,o=xe,a=Iu,l=gu,c=vu;return n.radius=function(t){return arguments.length?(a=En(t),n):a},n.source=function(t){return arguments.length?(u=En(t),n):u},n.target=function(t){return arguments.length?(o=En(t),n):o},n.startAngle=function(t){return arguments.length?(l=En(t),n):l},n.endAngle=function(t){return arguments.length?(c=En(t),n):c},n},ao.svg.diagonal=function(){function n(n,i){var u=t.call(this,n,i),o=e.call(this,n,i),a=(u.y+o.y)/2,l=[u,{x:u.x,y:a},{x:o.x,y:a},o];return l=l.map(r),"M"+l[0]+"C"+l[1]+" "+l[2]+" "+l[3]}var t=Me,e=xe,r=Yu;return n.source=function(e){return arguments.length?(t=En(e),n):t},n.target=function(t){return arguments.length?(e=En(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},ao.svg.diagonal.radial=function(){var n=ao.svg.diagonal(),t=Yu,e=n.projection;return n.projection=function(n){return arguments.length?e(Zu(t=n)):t},n},ao.svg.symbol=function(){function n(n,r){return(Ul.get(t.call(this,n,r))||$u)(e.call(this,n,r))}var t=Xu,e=Vu;return n.type=function(e){return arguments.length?(t=En(e),n):t},n.size=function(t){return arguments.length?(e=En(t),n):e},n};var Ul=ao.map({circle:$u,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Fl)),e=t*Fl;return"M0,"+-t+"L"+e+",0 0,"+t+" "+-e+",0Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/jl),e=t*jl/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/jl),e=t*jl/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});ao.svg.symbolTypes=Ul.keys();var jl=Math.sqrt(3),Fl=Math.tan(30*Yo);Co.transition=function(n){for(var t,e,r=Hl||++Zl,i=Ku(n),u=[],o=Ol||{time:Date.now(),ease:Nr,delay:0,duration:250},a=-1,l=this.length;++a<l;){u.push(t=[]);for(var c=this[a],f=-1,s=c.length;++f<s;)(e=c[f])&&Qu(e,f,i,r,o),t.push(e)}return Wu(u,i,r)},Co.interrupt=function(n){return this.each(null==n?Il:Bu(Ku(n)))};var Hl,Ol,Il=Bu(Ku()),Yl=[],Zl=0;Yl.call=Co.call,Yl.empty=Co.empty,Yl.node=Co.node,Yl.size=Co.size,ao.transition=function(n,t){return n&&n.transition?Hl?n.transition(t):n:ao.selection().transition(n)},ao.transition.prototype=Yl,Yl.select=function(n){var t,e,r,i=this.id,u=this.namespace,o=[];n=A(n);for(var a=-1,l=this.length;++a<l;){o.push(t=[]);for(var c=this[a],f=-1,s=c.length;++f<s;)(r=c[f])&&(e=n.call(r,r.__data__,f,a))?("__data__"in r&&(e.__data__=r.__data__),Qu(e,f,u,i,r[u][i]),t.push(e)):t.push(null)}return Wu(o,u,i)},Yl.selectAll=function(n){var t,e,r,i,u,o=this.id,a=this.namespace,l=[];n=C(n);for(var c=-1,f=this.length;++c<f;)for(var s=this[c],h=-1,p=s.length;++h<p;)if(r=s[h]){u=r[a][o],e=n.call(r,r.__data__,h,c),l.push(t=[]);for(var g=-1,v=e.length;++g<v;)(i=e[g])&&Qu(i,g,a,o,u),t.push(i)}return Wu(l,a,o)},Yl.filter=function(n){var t,e,r,i=[];"function"!=typeof n&&(n=O(n));for(var u=0,o=this.length;o>u;u++){i.push(t=[]);for(var e=this[u],a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return Wu(i,this.namespace,this.id)},Yl.tween=function(n,t){var e=this.id,r=this.namespace;return arguments.length<2?this.node()[r][e].tween.get(n):Y(this,null==t?function(t){t[r][e].tween.remove(n)}:function(i){i[r][e].tween.set(n,t)})},Yl.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function i(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function u(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?$r:Mr,a=ao.ns.qualify(n);return Ju(this,"attr."+n,t,a.local?u:i)},Yl.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(i));return r&&function(n){this.setAttribute(i,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(i.space,i.local));return r&&function(n){this.setAttributeNS(i.space,i.local,r(n))}}var i=ao.ns.qualify(n);return this.tween("attr."+n,i.local?r:e)},Yl.style=function(n,e,r){function i(){this.style.removeProperty(n)}function u(e){return null==e?i:(e+="",function(){var i,u=t(this).getComputedStyle(this,null).getPropertyValue(n);return u!==e&&(i=Mr(u,e),function(t){this.style.setProperty(n,i(t),r)})})}var o=arguments.length;if(3>o){if("string"!=typeof n){2>o&&(e="");for(r in n)this.style(r,n[r],e);return this}r=""}return Ju(this,"style."+n,e,u)},Yl.styleTween=function(n,e,r){function i(i,u){var o=e.call(this,i,u,t(this).getComputedStyle(this,null).getPropertyValue(n));return o&&function(t){this.style.setProperty(n,o(t),r)}}return arguments.length<3&&(r=""),this.tween("style."+n,i)},Yl.text=function(n){return Ju(this,"text",n,Gu)},Yl.remove=function(){var n=this.namespace;return this.each("end.transition",function(){var t;this[n].count<2&&(t=this.parentNode)&&t.removeChild(this)})},Yl.ease=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].ease:("function"!=typeof n&&(n=ao.ease.apply(ao,arguments)),Y(this,function(r){r[e][t].ease=n}))},Yl.delay=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].delay:Y(this,"function"==typeof n?function(r,i,u){r[e][t].delay=+n.call(r,r.__data__,i,u)}:(n=+n,function(r){r[e][t].delay=n}))},Yl.duration=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].duration:Y(this,"function"==typeof n?function(r,i,u){r[e][t].duration=Math.max(1,n.call(r,r.__data__,i,u))}:(n=Math.max(1,n),function(r){r[e][t].duration=n}))},Yl.each=function(n,t){var e=this.id,r=this.namespace;if(arguments.length<2){var i=Ol,u=Hl;try{Hl=e,Y(this,function(t,i,u){Ol=t[r][e],n.call(t,t.__data__,i,u)})}finally{Ol=i,Hl=u}}else Y(this,function(i){var u=i[r][e];(u.event||(u.event=ao.dispatch("start","end","interrupt"))).on(n,t)});return this},Yl.transition=function(){for(var n,t,e,r,i=this.id,u=++Zl,o=this.namespace,a=[],l=0,c=this.length;c>l;l++){a.push(n=[]);for(var t=this[l],f=0,s=t.length;s>f;f++)(e=t[f])&&(r=e[o][i],Qu(e,f,o,u,{time:r.time,ease:r.ease,delay:r.delay+r.duration,duration:r.duration})),n.push(e)}return Wu(a,o,u)},ao.svg.axis=function(){function n(n){n.each(function(){var n,c=ao.select(this),f=this.__chart__||e,s=this.__chart__=e.copy(),h=null==l?s.ticks?s.ticks.apply(s,a):s.domain():l,p=null==t?s.tickFormat?s.tickFormat.apply(s,a):m:t,g=c.selectAll(".tick").data(h,s),v=g.enter().insert("g",".domain").attr("class","tick").style("opacity",Uo),d=ao.transition(g.exit()).style("opacity",Uo).remove(),y=ao.transition(g.order()).style("opacity",1),M=Math.max(i,0)+o,x=Zi(s),b=c.selectAll(".domain").data([0]),_=(b.enter().append("path").attr("class","domain"),ao.transition(b));v.append("line"),v.append("text");var w,S,k,N,E=v.select("line"),A=y.select("line"),C=g.select("text").text(p),z=v.select("text"),L=y.select("text"),q="top"===r||"left"===r?-1:1;if("bottom"===r||"top"===r?(n=no,w="x",k="y",S="x2",N="y2",C.attr("dy",0>q?"0em":".71em").style("text-anchor","middle"),_.attr("d","M"+x[0]+","+q*u+"V0H"+x[1]+"V"+q*u)):(n=to,w="y",k="x",S="y2",N="x2",C.attr("dy",".32em").style("text-anchor",0>q?"end":"start"),_.attr("d","M"+q*u+","+x[0]+"H0V"+x[1]+"H"+q*u)),E.attr(N,q*i),z.attr(k,q*M),A.attr(S,0).attr(N,q*i),L.attr(w,0).attr(k,q*M),s.rangeBand){var T=s,R=T.rangeBand()/2;f=s=function(n){return T(n)+R}}else f.rangeBand?f=s:d.call(n,s,f);v.call(n,f,s),y.call(n,s,s)})}var t,e=ao.scale.linear(),r=Vl,i=6,u=6,o=3,a=[10],l=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Xl?t+"":Vl,n):r},n.ticks=function(){return arguments.length?(a=co(arguments),n):a},n.tickValues=function(t){return arguments.length?(l=t,n):l},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(i=+t,u=+arguments[e-1],n):i},n.innerTickSize=function(t){return arguments.length?(i=+t,n):i},n.outerTickSize=function(t){return arguments.length?(u=+t,n):u},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var Vl="bottom",Xl={top:1,right:1,bottom:1,left:1};ao.svg.brush=function(){function n(t){t.each(function(){var t=ao.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",u).on("touchstart.brush",u),o=t.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),t.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=t.selectAll(".resize").data(v,m);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return $l[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var l,s=ao.transition(t),h=ao.transition(o);c&&(l=Zi(c),h.attr("x",l[0]).attr("width",l[1]-l[0]),r(s)),f&&(l=Zi(f),h.attr("y",l[0]).attr("height",l[1]-l[0]),i(s)),e(s)})}function e(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+s[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function r(n){n.select(".extent").attr("x",s[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",s[1]-s[0])}function i(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function u(){function u(){32==ao.event.keyCode&&(C||(M=null,L[0]-=s[1],L[1]-=h[1],C=2),S())}function v(){32==ao.event.keyCode&&2==C&&(L[0]+=s[1],L[1]+=h[1],C=0,S())}function d(){var n=ao.mouse(b),t=!1;x&&(n[0]+=x[0],n[1]+=x[1]),C||(ao.event.altKey?(M||(M=[(s[0]+s[1])/2,(h[0]+h[1])/2]),L[0]=s[+(n[0]<M[0])],L[1]=h[+(n[1]<M[1])]):M=null),E&&y(n,c,0)&&(r(k),t=!0),A&&y(n,f,1)&&(i(k),t=!0),t&&(e(k),w({type:"brush",mode:C?"move":"resize"}))}function y(n,t,e){var r,i,u=Zi(t),l=u[0],c=u[1],f=L[e],v=e?h:s,d=v[1]-v[0];return C&&(l-=f,c-=d+f),r=(e?g:p)?Math.max(l,Math.min(c,n[e])):n[e],C?i=(r+=f)+d:(M&&(f=Math.max(l,Math.min(c,2*M[e]-r))),r>f?(i=r,r=f):i=f),v[0]!=r||v[1]!=i?(e?a=null:o=null,v[0]=r,v[1]=i,!0):void 0}function m(){d(),k.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),ao.select("body").style("cursor",null),q.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),z(),w({type:"brushend"})}var M,x,b=this,_=ao.select(ao.event.target),w=l.of(b,arguments),k=ao.select(b),N=_.datum(),E=!/^(n|s)$/.test(N)&&c,A=!/^(e|w)$/.test(N)&&f,C=_.classed("extent"),z=W(b),L=ao.mouse(b),q=ao.select(t(b)).on("keydown.brush",u).on("keyup.brush",v);if(ao.event.changedTouches?q.on("touchmove.brush",d).on("touchend.brush",m):q.on("mousemove.brush",d).on("mouseup.brush",m),k.interrupt().selectAll("*").interrupt(),C)L[0]=s[0]-L[0],L[1]=h[0]-L[1];else if(N){var T=+/w$/.test(N),R=+/^n/.test(N);x=[s[1-T]-L[0],h[1-R]-L[1]],L[0]=s[T],L[1]=h[R]}else ao.event.altKey&&(M=L.slice());k.style("pointer-events","none").selectAll(".resize").style("display",null),ao.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),d()}var o,a,l=N(n,"brushstart","brush","brushend"),c=null,f=null,s=[0,0],h=[0,0],p=!0,g=!0,v=Bl[0];return n.event=function(n){n.each(function(){var n=l.of(this,arguments),t={x:s,y:h,i:o,j:a},e=this.__chart__||t;this.__chart__=t,Hl?ao.select(this).transition().each("start.brush",function(){o=e.i,a=e.j,s=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=xr(s,t.x),r=xr(h,t.y);return o=a=null,function(i){s=t.x=e(i),h=t.y=r(i),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){o=t.i,a=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(c=t,v=Bl[!c<<1|!f],n):c},n.y=function(t){return arguments.length?(f=t,v=Bl[!c<<1|!f],n):f},n.clamp=function(t){return arguments.length?(c&&f?(p=!!t[0],g=!!t[1]):c?p=!!t:f&&(g=!!t),n):c&&f?[p,g]:c?p:f?g:null},n.extent=function(t){var e,r,i,u,l;return arguments.length?(c&&(e=t[0],r=t[1],f&&(e=e[0],r=r[0]),o=[e,r],c.invert&&(e=c(e),r=c(r)),e>r&&(l=e,e=r,r=l),e==s[0]&&r==s[1]||(s=[e,r])),f&&(i=t[0],u=t[1],c&&(i=i[1],u=u[1]),a=[i,u],f.invert&&(i=f(i),u=f(u)),i>u&&(l=i,i=u,u=l),i==h[0]&&u==h[1]||(h=[i,u])),n):(c&&(o?(e=o[0],r=o[1]):(e=s[0],r=s[1],c.invert&&(e=c.invert(e),r=c.invert(r)),e>r&&(l=e,e=r,r=l))),f&&(a?(i=a[0],u=a[1]):(i=h[0],u=h[1],f.invert&&(i=f.invert(i),u=f.invert(u)),i>u&&(l=i,i=u,u=l))),c&&f?[[e,i],[r,u]]:c?[e,r]:f&&[i,u])},n.clear=function(){return n.empty()||(s=[0,0],h=[0,0],o=a=null),n},n.empty=function(){return!!c&&s[0]==s[1]||!!f&&h[0]==h[1]},ao.rebind(n,l,"on")};var $l={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Bl=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Wl=ga.format=xa.timeFormat,Jl=Wl.utc,Gl=Jl("%Y-%m-%dT%H:%M:%S.%LZ");Wl.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?eo:Gl,eo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},eo.toString=Gl.toString,ga.second=On(function(n){return new va(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),ga.seconds=ga.second.range,ga.seconds.utc=ga.second.utc.range,ga.minute=On(function(n){return new va(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),ga.minutes=ga.minute.range,ga.minutes.utc=ga.minute.utc.range,ga.hour=On(function(n){var t=n.getTimezoneOffset()/60;return new va(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),ga.hours=ga.hour.range,ga.hours.utc=ga.hour.utc.range,ga.month=On(function(n){return n=ga.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),ga.months=ga.month.range,ga.months.utc=ga.month.utc.range;var Kl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],Ql=[[ga.second,1],[ga.second,5],[ga.second,15],[ga.second,30],[ga.minute,1],[ga.minute,5],[ga.minute,15],[ga.minute,30],[ga.hour,1],[ga.hour,3],[ga.hour,6],[ga.hour,12],[ga.day,1],[ga.day,2],[ga.week,1],[ga.month,1],[ga.month,3],[ga.year,1]],nc=Wl.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",zt]]),tc={range:function(n,t,e){return ao.range(Math.ceil(n/e)*e,+t,e).map(io)},floor:m,ceil:m};Ql.year=ga.year,ga.scale=function(){return ro(ao.scale.linear(),Ql,nc)};var ec=Ql.map(function(n){return[n[0].utc,n[1]]}),rc=Jl.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",zt]]);ec.year=ga.year.utc,ga.scale.utc=function(){return ro(ao.scale.linear(),ec,rc)},ao.text=An(function(n){return n.responseText}),ao.json=function(n,t){return Cn(n,"application/json",uo,t)},ao.html=function(n,t){return Cn(n,"text/html",oo,t)},ao.xml=An(function(n){return n.responseXML}),"function"==typeof define&&define.amd?(this.d3=ao,define(ao)):"object"==typeof module&&module.exports?module.exports=ao:this.d3=ao}();
diff --git a/bench/perf_monitoring/resources/s2.js b/bench/perf_monitoring/resources/s2.js new file mode 100644 index 0000000..54eda2c --- /dev/null +++ b/bench/perf_monitoring/resources/s2.js
@@ -0,0 +1 @@ +!function(){var a={};a.dev=!1,a.tooltip=a.tooltip||{},a.utils=a.utils||{},a.models=a.models||{},a.charts={},a.logs={},a.dom={},"undefined"!=typeof module&&"undefined"!=typeof exports&&"undefined"==typeof d3&&(d3=require("d3")),a.dispatch=d3.dispatch("render_start","render_end"),Function.prototype.bind||(Function.prototype.bind=function(a){if("function"!=typeof this)throw new TypeError("Function.prototype.bind - what is trying to be bound is not callable");var b=Array.prototype.slice.call(arguments,1),c=this,d=function(){},e=function(){return c.apply(this instanceof d&&a?this:a,b.concat(Array.prototype.slice.call(arguments)))};return d.prototype=this.prototype,e.prototype=new d,e}),a.dev&&(a.dispatch.on("render_start",function(b){a.logs.startTime=+new Date}),a.dispatch.on("render_end",function(b){a.logs.endTime=+new Date,a.logs.totalTime=a.logs.endTime-a.logs.startTime,a.log("total",a.logs.totalTime)})),a.log=function(){if(a.dev&&window.console&&console.log&&console.log.apply)console.log.apply(console,arguments);else if(a.dev&&window.console&&"function"==typeof console.log&&Function.prototype.bind){var b=Function.prototype.bind.call(console.log,console);b.apply(console,arguments)}return arguments[arguments.length-1]},a.deprecated=function(a,b){console&&console.warn&&console.warn("nvd3 warning: `"+a+"` has been deprecated. ",b||"")},a.render=function(b){b=b||1,a.render.active=!0,a.dispatch.render_start();var c=function(){for(var d,e,f=0;b>f&&(e=a.render.queue[f]);f++)d=e.generate(),typeof e.callback==typeof Function&&e.callback(d);a.render.queue.splice(0,f),a.render.queue.length?setTimeout(c):(a.dispatch.render_end(),a.render.active=!1)};setTimeout(c)},a.render.active=!1,a.render.queue=[],a.addGraph=function(b){typeof arguments[0]==typeof Function&&(b={generate:arguments[0],callback:arguments[1]}),a.render.queue.push(b),a.render.active||a.render()},"undefined"!=typeof module&&"undefined"!=typeof exports&&(module.exports=a),"undefined"!=typeof window&&(window.nv=a),a.dom.write=function(a){return void 0!==window.fastdom?fastdom.mutate(a):a()},a.dom.read=function(a){return void 0!==window.fastdom?fastdom.measure(a):a()},a.interactiveGuideline=function(){"use strict";function b(l){l.each(function(l){function m(){var a=d3.mouse(this),d=a[0],e=a[1],h=!0,i=!1;if(k&&(d=d3.event.offsetX,e=d3.event.offsetY,"svg"!==d3.event.target.tagName&&(h=!1),d3.event.target.className.baseVal.match("nv-legend")&&(i=!0)),h&&(d-=c.left,e-=c.top),"mouseout"===d3.event.type||0>d||0>e||d>o||e>p||d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement||i){if(k&&d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement&&(void 0===d3.event.relatedTarget.className||d3.event.relatedTarget.className.match(j.nvPointerEventsClass)))return;return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0)}j.hidden(!1);var l="function"==typeof f.rangeBands,m=void 0;if(l){var n=d3.bisect(f.range(),d)-1;if(!(f.range()[n]+f.rangeBand()>=d))return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0);m=f.domain()[d3.bisect(f.range(),d)-1]}else m=f.invert(d);g.elementMousemove({mouseX:d,mouseY:e,pointXValue:m}),"dblclick"===d3.event.type&&g.elementDblclick({mouseX:d,mouseY:e,pointXValue:m}),"click"===d3.event.type&&g.elementClick({mouseX:d,mouseY:e,pointXValue:m}),"mousedown"===d3.event.type&&g.elementMouseDown({mouseX:d,mouseY:e,pointXValue:m}),"mouseup"===d3.event.type&&g.elementMouseUp({mouseX:d,mouseY:e,pointXValue:m})}var n=d3.select(this),o=d||960,p=e||400,q=n.selectAll("g.nv-wrap.nv-interactiveLineLayer").data([l]),r=q.enter().append("g").attr("class"," nv-wrap nv-interactiveLineLayer");r.append("g").attr("class","nv-interactiveGuideLine"),i&&(i.on("touchmove",m).on("mousemove",m,!0).on("mouseout",m,!0).on("mousedown",m,!0).on("mouseup",m,!0).on("dblclick",m).on("click",m),b.guideLine=null,b.renderGuideLine=function(c){h&&(b.guideLine&&b.guideLine.attr("x1")===c||a.dom.write(function(){var b=q.select(".nv-interactiveGuideLine").selectAll("line").data(null!=c?[a.utils.NaNtoZero(c)]:[],String);b.enter().append("line").attr("class","nv-guideline").attr("x1",function(a){return a}).attr("x2",function(a){return a}).attr("y1",p).attr("y2",0),b.exit().remove()}))})})}var c={left:0,top:0},d=null,e=null,f=d3.scale.linear(),g=d3.dispatch("elementMousemove","elementMouseout","elementClick","elementDblclick","elementMouseDown","elementMouseUp"),h=!0,i=null,j=a.models.tooltip(),k=window.ActiveXObject;return j.duration(0).hideDelay(0).hidden(!1),b.dispatch=g,b.tooltip=j,b.margin=function(a){return arguments.length?(c.top="undefined"!=typeof a.top?a.top:c.top,c.left="undefined"!=typeof a.left?a.left:c.left,b):c},b.width=function(a){return arguments.length?(d=a,b):d},b.height=function(a){return arguments.length?(e=a,b):e},b.xScale=function(a){return arguments.length?(f=a,b):f},b.showGuideLine=function(a){return arguments.length?(h=a,b):h},b.svgContainer=function(a){return arguments.length?(i=a,b):i},b},a.interactiveBisect=function(a,b,c){"use strict";if(!(a instanceof Array))return null;var d;d="function"!=typeof c?function(a){return a.x}:c;var e=function(a,b){return d(a)-b},f=d3.bisector(e).left,g=d3.max([0,f(a,b)-1]),h=d(a[g]);if("undefined"==typeof h&&(h=g),h===b)return g;var i=d3.min([g+1,a.length-1]),j=d(a[i]);return"undefined"==typeof j&&(j=i),Math.abs(j-b)>=Math.abs(h-b)?g:i},a.nearestValueIndex=function(a,b,c){"use strict";var d=1/0,e=null;return a.forEach(function(a,f){var g=Math.abs(b-a);null!=a&&d>=g&&c>g&&(d=g,e=f)}),e},a.models.tooltip=function(){"use strict";function b(){if(!l||!l.node()){var a=[1];l=d3.select(document.body).selectAll(".nvtooltip").data(a),l.enter().append("div").attr("class","nvtooltip "+(i?i:"xy-tooltip")).attr("id",d).style("top",0).style("left",0).style("opacity",0).style("position","fixed").selectAll("div, table, td, tr").classed(q,!0).classed(q,!0),l.exit().remove()}}function c(){return n&&w(e)?(a.dom.write(function(){b();var a=u(e);a&&(l.node().innerHTML=a),y()}),c):void 0}var d="nvtooltip-"+Math.floor(1e5*Math.random()),e=null,f="w",g=25,h=0,i=null,j=!0,k=200,l=null,m={left:null,top:null},n=!0,o=100,p=!0,q="nv-pointer-events-none",r=function(a,b){return a},s=function(a){return a},t=function(a,b){return a},u=function(a){if(null===a)return"";var b=d3.select(document.createElement("table"));if(p){var c=b.selectAll("thead").data([a]).enter().append("thead");c.append("tr").append("td").attr("colspan",3).append("strong").classed("x-value",!0).html(s(a.value))}var d=b.selectAll("tbody").data([a]).enter().append("tbody"),e=d.selectAll("tr").data(function(a){return a.series}).enter().append("tr").classed("highlight",function(a){return a.highlight});e.append("td").classed("legend-color-guide",!0).append("div").style("background-color",function(a){return a.color}),e.append("td").classed("key",!0).classed("total",function(a){return!!a.total}).html(function(a,b){return t(a.key,b)}),e.append("td").classed("value",!0).html(function(a,b){return r(a.value,b)}),e.filter(function(a,b){return void 0!==a.percent}).append("td").classed("percent",!0).html(function(a,b){return"("+d3.format("%")(a.percent)+")"}),e.selectAll("td").each(function(a){if(a.highlight){var b=d3.scale.linear().domain([0,1]).range(["#fff",a.color]),c=.6;d3.select(this).style("border-bottom-color",b(c)).style("border-top-color",b(c))}});var f=b.node().outerHTML;return void 0!==a.footer&&(f+="<div class='footer'>"+a.footer+"</div>"),f},v=function(){var a={left:null!==d3.event?d3.event.clientX:0,top:null!==d3.event?d3.event.clientY:0};if("none"!=getComputedStyle(document.body).transform){var b=document.body.getBoundingClientRect();a.left-=b.left,a.top-=b.top}return a},w=function(b){if(b&&b.series){if(a.utils.isArray(b.series))return!0;if(a.utils.isObject(b.series))return b.series=[b.series],!0}return!1},x=function(a){var b,c,d,e=l.node().offsetHeight,h=l.node().offsetWidth,i=document.documentElement.clientWidth,j=document.documentElement.clientHeight;switch(f){case"e":b=-h-g,c=-(e/2),a.left+b<0&&(b=g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"w":b=g,c=-(e/2),a.left+b+h>i&&(b=-h-g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"n":b=-(h/2)-5,c=g,a.top+c+e>j&&(c=-e-g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"s":b=-(h/2),c=-e-g,a.top+c<0&&(c=g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"center":b=-(h/2),c=-(e/2);break;default:b=0,c=0}return{left:b,top:c}},y=function(){a.dom.read(function(){var a=v(),b=x(a),c=a.left+b.left,d=a.top+b.top;if(j)l.interrupt().transition().delay(k).duration(0).style("opacity",0);else{var e="translate("+m.left+"px, "+m.top+"px)",f="translate("+Math.round(c)+"px, "+Math.round(d)+"px)",g=d3.interpolateString(e,f),h=l.style("opacity")<.1;l.interrupt().transition().duration(h?0:o).styleTween("transform",function(a){return g},"important").styleTween("-webkit-transform",function(a){return g}).style("-ms-transform",f).style("opacity",1)}m.left=c,m.top=d})};return c.nvPointerEventsClass=q,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{duration:{get:function(){return o},set:function(a){o=a}},gravity:{get:function(){return f},set:function(a){f=a}},distance:{get:function(){return g},set:function(a){g=a}},snapDistance:{get:function(){return h},set:function(a){h=a}},classes:{get:function(){return i},set:function(a){i=a}},enabled:{get:function(){return n},set:function(a){n=a}},hideDelay:{get:function(){return k},set:function(a){k=a}},contentGenerator:{get:function(){return u},set:function(a){u=a}},valueFormatter:{get:function(){return r},set:function(a){r=a}},headerFormatter:{get:function(){return s},set:function(a){s=a}},keyFormatter:{get:function(){return t},set:function(a){t=a}},headerEnabled:{get:function(){return p},set:function(a){p=a}},position:{get:function(){return v},set:function(a){v=a}},chartContainer:{get:function(){return document.body},set:function(b){a.deprecated("chartContainer","feature removed after 1.8.3")}},fixedTop:{get:function(){return null},set:function(b){a.deprecated("fixedTop","feature removed after 1.8.1")}},offset:{get:function(){return{left:0,top:0}},set:function(b){a.deprecated("offset","use chart.tooltip.distance() instead")}},hidden:{get:function(){return j},set:function(a){j!=a&&(j=!!a,c())}},data:{get:function(){return e},set:function(a){a.point&&(a.value=a.point.x,a.series=a.series||{},a.series.value=a.point.y,a.series.color=a.point.color||a.series.color),e=a}},node:{get:function(){return l.node()},set:function(a){}},id:{get:function(){return d},set:function(a){}}}),a.utils.initOptions(c),c},a.utils.windowSize=function(){var a={width:640,height:480};return window.innerWidth&&window.innerHeight?(a.width=window.innerWidth,a.height=window.innerHeight,a):"CSS1Compat"==document.compatMode&&document.documentElement&&document.documentElement.offsetWidth?(a.width=document.documentElement.offsetWidth,a.height=document.documentElement.offsetHeight,a):document.body&&document.body.offsetWidth?(a.width=document.body.offsetWidth,a.height=document.body.offsetHeight,a):a},a.utils.isArray=Array.isArray,a.utils.isObject=function(a){return null!==a&&"object"==typeof a},a.utils.isFunction=function(a){return"function"==typeof a},a.utils.isDate=function(a){return"[object Date]"===toString.call(a)},a.utils.isNumber=function(a){return!isNaN(a)&&"number"==typeof a},a.utils.windowResize=function(b){return window.addEventListener?window.addEventListener("resize",b):a.log("ERROR: Failed to bind to window.resize with: ",b),{callback:b,clear:function(){window.removeEventListener("resize",b)}}},a.utils.getColor=function(b){if(void 0===b)return a.utils.defaultColor();if(a.utils.isArray(b)){var c=d3.scale.ordinal().range(b);return function(a,b){var d=void 0===b?a:b;return a.color||c(d)}}return b},a.utils.defaultColor=function(){return a.utils.getColor(d3.scale.category20().range())},a.utils.customTheme=function(b,c,d){c=c||function(a){return a.key},d=d||d3.scale.category20().range();var e=d.length;return function(f,g){var h=c(f);return a.utils.isFunction(b[h])?b[h]():void 0!==b[h]?b[h]:(e||(e=d.length),e-=1,d[e])}},a.utils.pjax=function(b,c){var d=function(d){d3.html(d,function(d){var e=d3.select(c).node();e.parentNode.replaceChild(d3.select(d).select(c).node(),e),a.utils.pjax(b,c)})};d3.selectAll(b).on("click",function(){history.pushState(this.href,this.textContent,this.href),d(this.href),d3.event.preventDefault()}),d3.select(window).on("popstate",function(){d3.event.state&&d(d3.event.state)})},a.utils.calcApproxTextWidth=function(b){if(a.utils.isFunction(b.style)&&a.utils.isFunction(b.text)){var c=parseInt(b.style("font-size").replace("px",""),10),d=b.text().length;return a.utils.NaNtoZero(d*c*.5)}return 0},a.utils.NaNtoZero=function(b){return!a.utils.isNumber(b)||isNaN(b)||null===b||b===1/0||b===-(1/0)?0:b},d3.selection.prototype.watchTransition=function(a){var b=[this].concat([].slice.call(arguments,1));return a.transition.apply(a,b)},a.utils.renderWatch=function(b,c){if(!(this instanceof a.utils.renderWatch))return new a.utils.renderWatch(b,c);var d=void 0!==c?c:250,e=[],f=this;this.models=function(a){return a=[].slice.call(arguments,0),a.forEach(function(a){a.__rendered=!1,function(a){a.dispatch.on("renderEnd",function(b){a.__rendered=!0,f.renderEnd("model")})}(a),e.indexOf(a)<0&&e.push(a)}),this},this.reset=function(a){void 0!==a&&(d=a),e=[]},this.transition=function(a,b,c){if(b=arguments.length>1?[].slice.call(arguments,1):[],c=b.length>1?b.pop():void 0!==d?d:250,a.__rendered=!1,e.indexOf(a)<0&&e.push(a),0===c)return a.__rendered=!0,a.delay=function(){return this},a.duration=function(){return this},a;0===a.length?a.__rendered=!0:a.every(function(a){return!a.length})?a.__rendered=!0:a.__rendered=!1;var g=0;return a.transition().duration(c).each(function(){++g}).each("end",function(c,d){0===--g&&(a.__rendered=!0,f.renderEnd.apply(this,b))})},this.renderEnd=function(){e.every(function(a){return a.__rendered})&&(e.forEach(function(a){a.__rendered=!1}),b.renderEnd.apply(this,arguments))}},a.utils.deepExtend=function(b){var c=arguments.length>1?[].slice.call(arguments,1):[];c.forEach(function(c){for(var d in c){var e=a.utils.isArray(b[d]),f=a.utils.isObject(b[d]),g=a.utils.isObject(c[d]);f&&!e&&g?a.utils.deepExtend(b[d],c[d]):b[d]=c[d]}})},a.utils.state=function(){if(!(this instanceof a.utils.state))return new a.utils.state;var b={},c=function(){},d=function(){return{}},e=null,f=null;this.dispatch=d3.dispatch("change","set"),this.dispatch.on("set",function(a){c(a,!0)}),this.getter=function(a){return d=a,this},this.setter=function(a,b){return b||(b=function(){}),c=function(c,d){a(c),d&&b()},this},this.init=function(b){e=e||{},a.utils.deepExtend(e,b)};var g=function(){var a=d();if(JSON.stringify(a)===JSON.stringify(b))return!1;for(var c in a)void 0===b[c]&&(b[c]={}),b[c]=a[c],f=!0;return!0};this.update=function(){e&&(c(e,!1),e=null),g.call(this)&&this.dispatch.change(b)}},a.utils.optionsFunc=function(b){return b&&d3.map(b).forEach(function(b,c){a.utils.isFunction(this[b])&&this[b](c)}.bind(this)),this},a.utils.calcTicksX=function(b,c){var d=1,e=0;for(e;e<c.length;e+=1){var f=c[e]&&c[e].values?c[e].values.length:0;d=f>d?f:d}return a.log("Requested number of ticks: ",b),a.log("Calculated max values to be: ",d),b=b>d?b=d-1:b,b=1>b?1:b,b=Math.floor(b),a.log("Calculating tick count as: ",b),b},a.utils.calcTicksY=function(b,c){return a.utils.calcTicksX(b,c)},a.utils.initOption=function(a,b){a._calls&&a._calls[b]?a[b]=a._calls[b]:(a[b]=function(c){return arguments.length?(a._overrides[b]=!0,a._options[b]=c,a):a._options[b]},a["_"+b]=function(c){return arguments.length?(a._overrides[b]||(a._options[b]=c),a):a._options[b]})},a.utils.initOptions=function(b){b._overrides=b._overrides||{};var c=Object.getOwnPropertyNames(b._options||{}),d=Object.getOwnPropertyNames(b._calls||{});c=c.concat(d);for(var e in c)a.utils.initOption(b,c[e])},a.utils.inheritOptionsD3=function(a,b,c){a._d3options=c.concat(a._d3options||[]),c.unshift(b),c.unshift(a),d3.rebind.apply(this,c)},a.utils.arrayUnique=function(a){return a.sort().filter(function(b,c){return!c||b!=a[c-1]})},a.utils.symbolMap=d3.map(),a.utils.symbol=function(){function b(b,e){var f=c.call(this,b,e),g=d.call(this,b,e);return-1!==d3.svg.symbolTypes.indexOf(f)?d3.svg.symbol().type(f).size(g)():a.utils.symbolMap.get(f)(g)}var c,d=64;return b.type=function(a){return arguments.length?(c=d3.functor(a),b):c},b.size=function(a){return arguments.length?(d=d3.functor(a),b):d},b},a.utils.inheritOptions=function(b,c){var d=Object.getOwnPropertyNames(c._options||{}),e=Object.getOwnPropertyNames(c._calls||{}),f=c._inherited||[],g=c._d3options||[],h=d.concat(e).concat(f).concat(g);h.unshift(c),h.unshift(b),d3.rebind.apply(this,h),b._inherited=a.utils.arrayUnique(d.concat(e).concat(f).concat(d).concat(b._inherited||[])),b._d3options=a.utils.arrayUnique(g.concat(b._d3options||[]))},a.utils.initSVG=function(a){a.classed({"nvd3-svg":!0})},a.utils.sanitizeHeight=function(a,b){return a||parseInt(b.style("height"),10)||400},a.utils.sanitizeWidth=function(a,b){return a||parseInt(b.style("width"),10)||960},a.utils.availableHeight=function(b,c,d){return Math.max(0,a.utils.sanitizeHeight(b,c)-d.top-d.bottom)},a.utils.availableWidth=function(b,c,d){return Math.max(0,a.utils.sanitizeWidth(b,c)-d.left-d.right)},a.utils.noData=function(b,c){var d=b.options(),e=d.margin(),f=d.noData(),g=null==f?["No Data Available."]:[f],h=a.utils.availableHeight(null,c,e),i=a.utils.availableWidth(null,c,e),j=e.left+i/2,k=e.top+h/2;c.selectAll("g").remove();var l=c.selectAll(".nv-noData").data(g);l.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),l.attr("x",j).attr("y",k).text(function(a){return a})},a.utils.wrapTicks=function(a,b){a.each(function(){for(var a,c=d3.select(this),d=c.text().split(/\s+/).reverse(),e=[],f=0,g=1.1,h=c.attr("y"),i=parseFloat(c.attr("dy")),j=c.text(null).append("tspan").attr("x",0).attr("y",h).attr("dy",i+"em");a=d.pop();)e.push(a),j.text(e.join(" ")),j.node().getComputedTextLength()>b&&(e.pop(),j.text(e.join(" ")),e=[a],j=c.append("tspan").attr("x",0).attr("y",h).attr("dy",++f*g+i+"em").text(a))})},a.utils.arrayEquals=function(b,c){if(b===c)return!0;if(!b||!c)return!1;if(b.length!=c.length)return!1;for(var d=0,e=b.length;e>d;d++)if(b[d]instanceof Array&&c[d]instanceof Array){if(!a.arrayEquals(b[d],c[d]))return!1}else if(b[d]!=c[d])return!1;return!0},a.models.axis=function(){"use strict";function b(g){return t.reset(),g.each(function(b){var g=d3.select(this);a.utils.initSVG(g);var q=g.selectAll("g.nv-wrap.nv-axis").data([b]),r=q.enter().append("g").attr("class","nvd3 nv-wrap nv-axis"),u=(r.append("g"),q.select("g"));null!==n?c.ticks(n):("top"==c.orient()||"bottom"==c.orient())&&c.ticks(Math.abs(d.range()[1]-d.range()[0])/100),u.watchTransition(t,"axis").call(c),s=s||c.scale();var v=c.tickFormat();null==v&&(v=s.tickFormat());var w=u.selectAll("text.nv-axislabel").data([h||null]);w.exit().remove(),void 0!==p&&u.selectAll("g").select("text").style("font-size",p);var x,y,z;switch(c.orient()){case"top":w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",0).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b))+",0)"}).select("text").attr("dy","-0.5em").attr("y",-c.tickPadding()).attr("text-anchor","middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max top").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d.range()[c])+",0)"}));break;case"bottom":x=o+36;var A=30,B=0,C=u.selectAll("g").select("text"),D="";if(j%360){C.attr("transform",""),C.each(function(a,b){var c=this.getBoundingClientRect(),d=c.width;B=c.height,d>A&&(A=d)}),D="rotate("+j+" 0,"+(B/2+c.tickPadding())+")";var E=Math.abs(Math.sin(j*Math.PI/180));x=(E?E*A:A)+30,C.attr("transform",D).style("text-anchor",j%360>0?"start":"end")}else l?C.attr("transform",function(a,b){return"translate(0,"+(b%2==0?"0":"12")+")"}):C.attr("transform","translate(0,0)");w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",x).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data([d.domain()[0],d.domain()[d.domain().length-1]]),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}).select("text").attr("dy",".71em").attr("y",c.tickPadding()).attr("transform",D).style("text-anchor",j?j%360>0?"start":"end":"middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max bottom").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}));break;case"right":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"begin").attr("transform",k?"rotate(90)":"").attr("y",k?-Math.max(e.right,f)+12-(o||0):-10).attr("x",k?d3.max(d.range())/2:c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").style("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",c.tickPadding()).style("text-anchor","start").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1));break;case"left":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"end").attr("transform",k?"rotate(-90)":"").attr("y",k?-Math.max(e.left,f)+25-(o||0):-10).attr("x",k?-d3.max(d.range())/2:-c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").style("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(s(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",-c.tickPadding()).attr("text-anchor","end").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1))}if(w.text(function(a){return a}),!i||"left"!==c.orient()&&"right"!==c.orient()||(u.selectAll("g").each(function(a,b){d3.select(this).select("text").attr("opacity",1),(d(a)<d.range()[1]+10||d(a)>d.range()[0]-10)&&((a>1e-10||-1e-10>a)&&d3.select(this).attr("opacity",0),d3.select(this).select("text").attr("opacity",0))}),d.domain()[0]==d.domain()[1]&&0==d.domain()[0]&&q.selectAll("g.nv-axisMaxMin").style("opacity",function(a,b){return b?0:1})),i&&("top"===c.orient()||"bottom"===c.orient())){var F=[];q.selectAll("g.nv-axisMaxMin").each(function(a,b){try{b?F.push(d(a)-this.getBoundingClientRect().width-4):F.push(d(a)+this.getBoundingClientRect().width+4)}catch(c){b?F.push(d(a)-4):F.push(d(a)+4)}}),u.selectAll("g").each(function(a,b){(d(a)<F[0]||d(a)>F[1])&&(a>1e-10||-1e-10>a?d3.select(this).remove():d3.select(this).select("text").remove())})}u.selectAll(".tick").filter(function(a){return!parseFloat(Math.round(1e5*a)/1e6)&&void 0!==a}).classed("zero",!0),s=d.copy()}),t.renderEnd("axis immediate"),b}var c=d3.svg.axis(),d=d3.scale.linear(),e={top:0,right:0,bottom:0,left:0},f=75,g=60,h=null,i=!0,j=0,k=!0,l=!1,m=!1,n=null,o=0,p=void 0,q=250,r=d3.dispatch("renderEnd");c.scale(d).orient("bottom").tickFormat(function(a){return a});var s,t=a.utils.renderWatch(r,q);return b.axis=c,b.dispatch=r,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{axisLabelDistance:{get:function(){return o},set:function(a){o=a}},staggerLabels:{get:function(){return l},set:function(a){l=a}},rotateLabels:{get:function(){return j},set:function(a){j=a}},rotateYLabel:{get:function(){return k},set:function(a){k=a}},showMaxMin:{get:function(){return i},set:function(a){i=a}},axisLabel:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return g},set:function(a){g=a}},ticks:{get:function(){return n},set:function(a){n=a}},width:{get:function(){return f},set:function(a){f=a}},fontSize:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left}},duration:{get:function(){return q},set:function(a){q=a,t.reset(q)}},scale:{get:function(){return d},set:function(e){d=e,c.scale(d),m="function"==typeof d.rangeBands,a.utils.inheritOptionsD3(b,d,["domain","range","rangeBand","rangeBands"])}}}),a.utils.initOptions(b),a.utils.inheritOptionsD3(b,c,["orient","tickValues","tickSubdivide","tickSize","tickPadding","tickFormat"]),a.utils.inheritOptionsD3(b,d,["domain","range","rangeBand","rangeBands"]),b},a.models.boxPlot=function(){"use strict";function b(l){return E.reset(),l.each(function(b){var l=j-i.left-i.right,F=k-i.top-i.bottom;A=d3.select(this),a.utils.initSVG(A),m.domain(c||b.map(function(a,b){return o(a,b)})).rangeBands(d||[0,l],.1);var G=[];if(!e){var H,I,J=[];b.forEach(function(a,b){var c=p(a),d=r(a),e=s(a),f=t(a),g=v(a);g&&g.forEach(function(a,b){J.push(w(a,b,void 0))}),e&&J.push(e),c&&J.push(c),d&&J.push(d),f&&J.push(f)}),H=d3.min(J),I=d3.max(J),G=[H,I]}n.domain(e||G),n.range(f||[F,0]),g=g||m,h=h||n.copy().range([n(0),n(0)]);var K=A.selectAll("g.nv-wrap").data([b]);K.enter().append("g").attr("class","nvd3 nv-wrap");K.attr("transform","translate("+i.left+","+i.top+")");var L=K.selectAll(".nv-boxplot").data(function(a){return a}),M=L.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6);L.attr("class","nv-boxplot").attr("transform",function(a,b,c){return"translate("+(m(o(a,b))+.05*m.rangeBand())+", 0)"}).classed("hover",function(a){return a.hover}),L.watchTransition(E,"nv-boxplot: boxplots").style("stroke-opacity",1).style("fill-opacity",.75).delay(function(a,c){return c*C/b.length}).attr("transform",function(a,b){return"translate("+(m(o(a,b))+.05*m.rangeBand())+", 0)"}),L.exit().remove(),M.each(function(a,b){var c=d3.select(this);[s,t].forEach(function(d){if(d(a)){var e=d===s?"low":"high";c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-whisker nv-boxplot-"+e),c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-tick nv-boxplot-"+e)}})});var N=function(){return null===D?.9*m.rangeBand():Math.min(75,.9*m.rangeBand())},O=function(){return.45*m.rangeBand()-N()/2},P=function(){return.45*m.rangeBand()+N()/2};[s,t].forEach(function(a){var b=a===s?"low":"high",c=a===s?p:r;L.select("line.nv-boxplot-whisker.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",.45*m.rangeBand()).attr("y1",function(b,c){return n(a(b))}).attr("x2",.45*m.rangeBand()).attr("y2",function(a,b){return n(c(a))}),L.select("line.nv-boxplot-tick.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",O).attr("y1",function(b,c){return n(a(b))}).attr("x2",P).attr("y2",function(b,c){return n(a(b))})}),[s,t].forEach(function(a){var b=a===s?"low":"high";M.selectAll(".nv-boxplot-"+b).on("mouseover",function(b,c,d){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mouseout",function(b,c,d){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})})}),M.append("rect").attr("class","nv-boxplot-box").on("mouseover",function(a,b){d3.select(this).classed("hover",!0),B.elementMouseover({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}],data:a,index:b,e:d3.event})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),B.elementMouseout({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}],data:a,index:b,e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),L.select("rect.nv-boxplot-box").watchTransition(E,"nv-boxplot: boxes").attr("y",function(a,b){return n(r(a))}).attr("width",N).attr("x",O).attr("height",function(a,b){return Math.abs(n(r(a))-n(p(a)))||1}).style("fill",function(a,b){return u(a)||z(a,b)}).style("stroke",function(a,b){return u(a)||z(a,b)}),M.append("line").attr("class","nv-boxplot-median"),L.select("line.nv-boxplot-median").watchTransition(E,"nv-boxplot: boxplots line").attr("x1",O).attr("y1",function(a,b){return n(q(a))}).attr("x2",P).attr("y2",function(a,b){return n(q(a))});var Q=L.selectAll(".nv-boxplot-outlier").data(function(a){return v(a)||[]});Q.enter().append("circle").style("fill",function(a,b,c){return y(a,b,c)||z(a,c)}).style("stroke",function(a,b,c){return y(a,b,c)||z(a,c)}).style("z-index",9e3).on("mouseover",function(a,b,c){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mouseout",function(a,b,c){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),Q.attr("class","nv-boxplot-outlier"),Q.watchTransition(E,"nv-boxplot: nv-boxplot-outlier").attr("cx",.45*m.rangeBand()).attr("cy",function(a,b,c){return n(w(a,b,c))}).attr("r","3"),Q.exit().remove(),g=m.copy(),h=n.copy()}),E.renderEnd("nv-boxplot immediate"),b}var c,d,e,f,g,h,i={top:0,right:0,bottom:0,left:0},j=960,k=500,l=Math.floor(1e4*Math.random()),m=d3.scale.ordinal(),n=d3.scale.linear(),o=function(a){return a.label},p=function(a){return a.values.Q1},q=function(a){return a.values.Q2},r=function(a){return a.values.Q3},s=function(a){return a.values.whisker_low},t=function(a){return a.values.whisker_high},u=function(a){return a.color},v=function(a){return a.values.outliers},w=function(a,b,c){return a},x=function(a,b,c){return a},y=function(a,b,c){return void 0},z=a.utils.defaultColor(),A=null,B=d3.dispatch("elementMouseover","elementMouseout","elementMousemove","renderEnd"),C=250,D=null,E=a.utils.renderWatch(B,C);return b.dispatch=B,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return j},set:function(a){j=a}},height:{get:function(){return k},set:function(a){k=a}},maxBoxWidth:{get:function(){return D},set:function(a){D=a}},x:{get:function(){return o},set:function(a){o=a}},q1:{get:function(){return p},set:function(a){p=a}},q2:{get:function(){return q},set:function(a){q=a}},q3:{get:function(){return r},set:function(a){r=a}},wl:{get:function(){return s},set:function(a){s=a}},wh:{get:function(){return t},set:function(a){t=a}},itemColor:{get:function(){return u},set:function(a){u=a}},outliers:{get:function(){return v},set:function(a){v=a;}},outlierValue:{get:function(){return w},set:function(a){w=a}},outlierLabel:{get:function(){return x},set:function(a){x=a}},outlierColor:{get:function(){return y},set:function(a){y=a}},xScale:{get:function(){return m},set:function(a){m=a}},yScale:{get:function(){return n},set:function(a){n=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return d},set:function(a){d=a}},yRange:{get:function(){return f},set:function(a){f=a}},id:{get:function(){return l},set:function(a){l=a}},y:{get:function(){return console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead."),{}},set:function(a){console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead.")}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.left:i.left}},color:{get:function(){return z},set:function(b){z=a.utils.getColor(b)}},duration:{get:function(){return C},set:function(a){C=a,E.reset(C)}}}),a.utils.initOptions(b),b},a.models.boxPlotChart=function(){"use strict";function b(k){return t.reset(),t.models(e),l&&t.models(f),m&&t.models(g),k.each(function(k){var p=d3.select(this);a.utils.initSVG(p);var t=(i||parseInt(p.style("width"))||960)-h.left-h.right,u=(j||parseInt(p.style("height"))||400)-h.top-h.bottom;if(b.update=function(){r.beforeUpdate(),p.transition().duration(s).call(b)},b.container=this,!k||!k.length){var v=p.selectAll(".nv-noData").data([q]);return v.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),v.attr("x",h.left+t/2).attr("y",h.top+u/2).text(function(a){return a}),b}p.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var w=p.selectAll("g.nv-wrap.nv-boxPlotWithAxes").data([k]),x=w.enter().append("g").attr("class","nvd3 nv-wrap nv-boxPlotWithAxes").append("g"),y=x.append("defs"),z=w.select("g");x.append("g").attr("class","nv-x nv-axis"),x.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),x.append("g").attr("class","nv-barsWrap"),z.attr("transform","translate("+h.left+","+h.top+")"),n&&z.select(".nv-y.nv-axis").attr("transform","translate("+t+",0)"),e.width(t).height(u);var A=z.select(".nv-barsWrap").datum(k.filter(function(a){return!a.disabled}));if(A.transition().call(e),y.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),z.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(o?2:1)).attr("height",16).attr("x",-c.rangeBand()/(o?1:2)),l){f.scale(c).ticks(a.utils.calcTicksX(t/100,k)).tickSize(-u,0),z.select(".nv-x.nv-axis").attr("transform","translate(0,"+d.range()[0]+")"),z.select(".nv-x.nv-axis").call(f);var B=z.select(".nv-x.nv-axis").selectAll("g");o&&B.selectAll("text").attr("transform",function(a,b,c){return"translate(0,"+(c%2===0?"5":"17")+")"})}m&&(g.scale(d).ticks(Math.floor(u/36)).tickSize(-t,0),z.select(".nv-y.nv-axis").call(g)),z.select(".nv-zeroLine line").attr("x1",0).attr("x2",t).attr("y1",d(0)).attr("y2",d(0))}),t.renderEnd("nv-boxplot chart immediate"),b}var c,d,e=a.models.boxPlot(),f=a.models.axis(),g=a.models.axis(),h={top:15,right:10,bottom:50,left:60},i=null,j=null,k=a.utils.getColor(),l=!0,m=!0,n=!1,o=!1,p=a.models.tooltip(),q="No Data Available.",r=d3.dispatch("beforeUpdate","renderEnd"),s=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(n?"right":"left").tickFormat(d3.format(",.1f")),p.duration(0);var t=a.utils.renderWatch(r,s);return e.dispatch.on("elementMouseover.tooltip",function(a){p.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){p.data(a).hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){p()}),b.dispatch=r,b.boxplot=e,b.xAxis=f,b.yAxis=g,b.tooltip=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},staggerLabels:{get:function(){return o},set:function(a){o=a}},showXAxis:{get:function(){return l},set:function(a){l=a}},showYAxis:{get:function(){return m},set:function(a){m=a}},tooltipContent:{get:function(){return p},set:function(a){p=a}},noData:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!==a.top?a.top:h.top,h.right=void 0!==a.right?a.right:h.right,h.bottom=void 0!==a.bottom?a.bottom:h.bottom,h.left=void 0!==a.left?a.left:h.left}},duration:{get:function(){return s},set:function(a){s=a,t.reset(s),e.duration(s),f.duration(s),g.duration(s)}},color:{get:function(){return k},set:function(b){k=a.utils.getColor(b),e.color(k)}},rightAlignYAxis:{get:function(){return n},set:function(a){n=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.bullet=function(){"use strict";function b(a,b){var c=a.slice();a.sort(function(a,d){var e=c.indexOf(a),f=c.indexOf(d);return d3.descending(b[e],b[f])})}function c(e){return e.each(function(c,e){var s=p-d.left-d.right,x=q-d.top-d.bottom;r=d3.select(this),a.utils.initSVG(r);var y=g.call(this,c,e).slice(),z=h.call(this,c,e).slice(),A=i.call(this,c,e).slice().sort(d3.descending),B=j.call(this,c,e).slice(),C=k.call(this,c,e).slice(),D=l.call(this,c,e).slice(),E=m.call(this,c,e).slice(),F=n.call(this,c,e).slice();b(C,y),b(D,z),b(E,A),b(F,B),y.sort(d3.descending),z.sort(d3.descending),B.sort(d3.descending);var G=d3.scale.linear().domain(d3.extent(d3.merge([o,y]))).range(f?[s,0]:[0,s]);this.__chart__||d3.scale.linear().domain([0,1/0]).range(G.range());this.__chart__=G;for(var H=(d3.min(y),d3.max(y),y[1],r.selectAll("g.nv-wrap.nv-bullet").data([c])),I=H.enter().append("g").attr("class","nvd3 nv-wrap nv-bullet"),J=I.append("g"),K=H.select("g"),e=0,L=y.length;L>e;e++){var M="nv-range nv-range"+e;2>=e&&(M=M+" nv-range"+w[e]),J.append("rect").attr("class",M)}J.append("rect").attr("class","nv-measure"),H.attr("transform","translate("+d.left+","+d.top+")");for(var N=function(a){return Math.abs(G(a)-G(0))},O=function(a){return G(0>a?a:0)},e=0,L=y.length;L>e;e++){var P=y[e];K.select("rect.nv-range"+e).attr("height",x).attr("width",N(P)).attr("x",O(P)).datum(P)}K.select("rect.nv-measure").style("fill",t).attr("height",x/3).attr("y",x/3).attr("width",0>B?G(0)-G(B[0]):G(B[0])-G(0)).attr("x",O(B)).on("mouseover",function(){u.elementMouseover({value:B[0],label:F[0]||"Current",color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:B[0],label:F[0]||"Current",color:d3.select(this).style("fill")})}).on("mouseout",function(){u.elementMouseout({value:B[0],label:F[0]||"Current",color:d3.select(this).style("fill")})});var Q=x/6,R=z.map(function(a,b){return{value:a,label:D[b]}});J.selectAll("path.nv-markerTriangle").data(R).enter().append("path").attr("class","nv-markerTriangle").attr("d","M0,"+Q+"L"+Q+","+-Q+" "+-Q+","+-Q+"Z").on("mouseover",function(a){u.elementMouseover({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill"),pos:[G(a.value),x/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),K.selectAll("path.nv-markerTriangle").data(R).attr("transform",function(a){return"translate("+G(a.value)+","+x/2+")"});var S=A.map(function(a,b){return{value:a,label:E[b]}});J.selectAll("path.nv-markerLine").data(S).enter().append("line").attr("cursor","").attr("class","nv-markerLine").attr("x1",function(a){return G(a.value)}).attr("y1","2").attr("x2",function(a){return G(a.value)}).attr("y2",x-2).on("mouseover",function(a){u.elementMouseover({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill"),pos:[G(a.value),x/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),K.selectAll("path.nv-markerLines").data(S).attr("transform",function(a){return"translate("+G(a.value)+","+x/2+")"}),H.selectAll(".nv-range").on("mouseover",function(a,b){var c=C[b]||v[b];u.elementMouseover({value:a,label:c,color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:B[0],label:F[0]||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){var c=C[b]||v[b];u.elementMouseout({value:a,label:c,color:d3.select(this).style("fill")})})}),c}var d={top:0,right:0,bottom:0,left:0},e="left",f=!1,g=function(a){return a.ranges},h=function(a){return a.markers?a.markers:[]},i=function(a){return a.markerLines?a.markerLines:[0]},j=function(a){return a.measures},k=function(a){return a.rangeLabels?a.rangeLabels:[]},l=function(a){return a.markerLabels?a.markerLabels:[]},m=function(a){return a.markerLineLabels?a.markerLineLabels:[]},n=function(a){return a.measureLabels?a.measureLabels:[]},o=[0],p=380,q=30,r=null,s=null,t=a.utils.getColor(["#1f77b4"]),u=d3.dispatch("elementMouseover","elementMouseout","elementMousemove"),v=["Maximum","Mean","Minimum"],w=["Max","Avg","Min"];return c.dispatch=u,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{ranges:{get:function(){return g},set:function(a){g=a}},markers:{get:function(){return h},set:function(a){h=a}},measures:{get:function(){return j},set:function(a){j=a}},forceX:{get:function(){return o},set:function(a){o=a}},width:{get:function(){return p},set:function(a){p=a}},height:{get:function(){return q},set:function(a){q=a}},tickFormat:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return d},set:function(a){d.top=void 0!==a.top?a.top:d.top,d.right=void 0!==a.right?a.right:d.right,d.bottom=void 0!==a.bottom?a.bottom:d.bottom,d.left=void 0!==a.left?a.left:d.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}},color:{get:function(){return t},set:function(b){t=a.utils.getColor(b)}}}),a.utils.initOptions(c),c},a.models.bulletChart=function(){"use strict";function b(d){return d.each(function(e,o){var p=d3.select(this);a.utils.initSVG(p);var q=a.utils.availableWidth(k,p,g),r=l-g.top-g.bottom;if(b.update=function(){b(d)},b.container=this,!e||!h.call(this,e,o))return a.utils.noData(b,p),b;p.selectAll(".nv-noData").remove();var s=h.call(this,e,o).slice().sort(d3.descending),t=i.call(this,e,o).slice().sort(d3.descending),u=j.call(this,e,o).slice().sort(d3.descending),v=p.selectAll("g.nv-wrap.nv-bulletChart").data([e]),w=v.enter().append("g").attr("class","nvd3 nv-wrap nv-bulletChart"),x=w.append("g"),y=v.select("g");x.append("g").attr("class","nv-bulletWrap"),x.append("g").attr("class","nv-titles"),v.attr("transform","translate("+g.left+","+g.top+")");var z=d3.scale.linear().domain([0,Math.max(s[0],t[0]||0,u[0])]).range(f?[q,0]:[0,q]),A=this.__chart__||d3.scale.linear().domain([0,1/0]).range(z.range());this.__chart__=z;var B=x.select(".nv-titles").append("g").attr("text-anchor","end").attr("transform","translate(-6,"+(l-g.top-g.bottom)/2+")");B.append("text").attr("class","nv-title").text(function(a){return a.title}),B.append("text").attr("class","nv-subtitle").attr("dy","1em").text(function(a){return a.subtitle}),c.width(q).height(r);var C=y.select(".nv-bulletWrap");d3.transition(C).call(c);var D=m||z.tickFormat(q/100),E=y.selectAll("g.nv-tick").data(z.ticks(n?n:q/50),function(a){return this.textContent||D(a)}),F=E.enter().append("g").attr("class","nv-tick").attr("transform",function(a){return"translate("+A(a)+",0)"}).style("opacity",1e-6);F.append("line").attr("y1",r).attr("y2",7*r/6),F.append("text").attr("text-anchor","middle").attr("dy","1em").attr("y",7*r/6).text(D);var G=d3.transition(E).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1);G.select("line").attr("y1",r).attr("y2",7*r/6),G.select("text").attr("y",7*r/6),d3.transition(E.exit()).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1e-6).remove()}),d3.timer.flush(),b}var c=a.models.bullet(),d=a.models.tooltip(),e="left",f=!1,g={top:5,right:40,bottom:20,left:120},h=function(a){return a.ranges},i=function(a){return a.markers?a.markers:[]},j=function(a){return a.measures},k=null,l=55,m=null,n=null,o=null,p=d3.dispatch();return d.duration(0).headerEnabled(!1),c.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:a.label,value:a.value,color:a.color},d.data(a).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",function(a){d.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(a){d()}),b.bullet=c,b.dispatch=p,b.tooltip=d,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{ranges:{get:function(){return h},set:function(a){h=a}},markers:{get:function(){return i},set:function(a){i=a}},measures:{get:function(){return j},set:function(a){j=a}},width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},tickFormat:{get:function(){return m},set:function(a){m=a}},ticks:{get:function(){return n},set:function(a){n=a}},noData:{get:function(){return o},set:function(a){o=a}},margin:{get:function(){return g},set:function(a){g.top=void 0!==a.top?a.top:g.top,g.right=void 0!==a.right?a.right:g.right,g.bottom=void 0!==a.bottom?a.bottom:g.bottom,g.left=void 0!==a.left?a.left:g.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.candlestickBar=function(){"use strict";function b(x){return x.each(function(b){c=d3.select(this);var x=a.utils.availableWidth(i,c,h),y=a.utils.availableHeight(j,c,h);a.utils.initSVG(c);var A=x/b[0].values.length*.45;l.domain(d||d3.extent(b[0].values.map(n).concat(t))),v?l.range(f||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(f||[5+A/2,x-A/2-5]),m.domain(e||[d3.min(b[0].values.map(s).concat(u)),d3.max(b[0].values.map(r).concat(u))]).range(g||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var B=d3.select(this).selectAll("g.nv-wrap.nv-candlestickBar").data([b[0].values]),C=B.enter().append("g").attr("class","nvd3 nv-wrap nv-candlestickBar"),D=C.append("defs"),E=C.append("g"),F=B.select("g");E.append("g").attr("class","nv-ticks"),B.attr("transform","translate("+h.left+","+h.top+")"),c.on("click",function(a,b){z.chartClick({data:a,index:b,pos:d3.event,id:k})}),D.append("clipPath").attr("id","nv-chart-clip-path-"+k).append("rect"),B.select("#nv-chart-clip-path-"+k+" rect").attr("width",x).attr("height",y),F.attr("clip-path",w?"url(#nv-chart-clip-path-"+k+")":"");var G=B.select(".nv-ticks").selectAll(".nv-tick").data(function(a){return a});G.exit().remove();var H=G.enter().append("g");G.attr("class",function(a,b,c){return(p(a,b)>q(a,b)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+c+"-"+b});H.append("line").attr("class","nv-candlestick-lines").attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),H.append("rect").attr("class","nv-candlestick-rects nv-bars").attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)});G.select(".nv-candlestick-lines").transition().attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),G.select(".nv-candlestick-rects").transition().attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)})}),b}var c,d,e,f,g,h={top:0,right:0,bottom:0,left:0},i=null,j=null,k=Math.floor(1e4*Math.random()),l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=function(a){return a.open},q=function(a){return a.close},r=function(a){return a.high},s=function(a){return a.low},t=[],u=[],v=!1,w=!0,x=a.utils.defaultColor(),y=!1,z=d3.dispatch("stateChange","changeState","renderEnd","chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove");return b.highlightPoint=function(a,d){b.clearHighlights(),c.select(".nv-candlestickBar .nv-tick-0-"+a).classed("hover",d)},b.clearHighlights=function(){c.select(".nv-candlestickBar .nv-tick.hover").classed("hover",!1)},b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},forceX:{get:function(){return t},set:function(a){t=a}},forceY:{get:function(){return u},set:function(a){u=a}},padData:{get:function(){return v},set:function(a){v=a}},clipEdge:{get:function(){return w},set:function(a){w=a}},id:{get:function(){return k},set:function(a){k=a}},interactive:{get:function(){return y},set:function(a){y=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},open:{get:function(){return p()},set:function(a){p=a}},close:{get:function(){return q()},set:function(a){q=a}},high:{get:function(){return r},set:function(a){r=a}},low:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!=a.top?a.top:h.top,h.right=void 0!=a.right?a.right:h.right,h.bottom=void 0!=a.bottom?a.bottom:h.bottom,h.left=void 0!=a.left?a.left:h.left}},color:{get:function(){return x},set:function(b){x=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.cumulativeLineChart=function(){"use strict";function b(l){return H.reset(),H.models(f),r&&H.models(g),s&&H.models(h),l.each(function(l){function A(a,c){d3.select(b.container).style("cursor","ew-resize")}function E(a,b){G.x=d3.event.x,G.i=Math.round(F.invert(G.x)),K()}function H(a,c){d3.select(b.container).style("cursor","auto"),y.index=G.i,C.stateChange(y)}function K(){aa.data([G]);var a=b.duration();b.duration(0),b.update(),b.duration(a)}var L=d3.select(this);a.utils.initSVG(L),L.classed("nv-chart-"+x,!0);var M=a.utils.availableWidth(o,L,m),N=a.utils.availableHeight(p,L,m);if(b.update=function(){0===D?L.call(b):L.transition().duration(D).call(b)},b.container=this,y.setter(J(l),b.update).getter(I(l)).update(),y.disabled=l.map(function(a){return!!a.disabled}),!z){var O;z={};for(O in y)y[O]instanceof Array?z[O]=y[O].slice(0):z[O]=y[O]}var P=d3.behavior.drag().on("dragstart",A).on("drag",E).on("dragend",H);if(!(l&&l.length&&l.filter(function(a){return a.values.length}).length))return a.utils.noData(b,L),b;if(L.selectAll(".nv-noData").remove(),d=f.xScale(),e=f.yScale(),w)f.yDomain(null);else{var Q=l.filter(function(a){return!a.disabled}).map(function(a,b){var c=d3.extent(a.values,f.y());return c[0]<-.95&&(c[0]=-.95),[(c[0]-c[1])/(1+c[1]),(c[1]-c[0])/(1+c[0])]}),R=[d3.min(Q,function(a){return a[0]}),d3.max(Q,function(a){return a[1]})];f.yDomain(R)}F.domain([0,l[0].values.length-1]).range([0,M]).clamp(!0);var l=c(G.i,l),S=v?"none":"all",T=L.selectAll("g.nv-wrap.nv-cumulativeLine").data([l]),U=T.enter().append("g").attr("class","nvd3 nv-wrap nv-cumulativeLine").append("g"),V=T.select("g");if(U.append("g").attr("class","nv-interactive"),U.append("g").attr("class","nv-x nv-axis").style("pointer-events","none"),U.append("g").attr("class","nv-y nv-axis"),U.append("g").attr("class","nv-background"),U.append("g").attr("class","nv-linesWrap").style("pointer-events",S),U.append("g").attr("class","nv-avgLinesWrap").style("pointer-events","none"),U.append("g").attr("class","nv-legendWrap"),U.append("g").attr("class","nv-controlsWrap"),q?(i.width(M),V.select(".nv-legendWrap").datum(l).call(i),i.height()>m.top&&(m.top=i.height(),N=a.utils.availableHeight(p,L,m)),V.select(".nv-legendWrap").attr("transform","translate(0,"+-m.top+")")):V.select(".nv-legendWrap").selectAll("*").remove(),u){var W=[{key:"Re-scale y-axis",disabled:!w}];j.width(140).color(["#444","#444","#444"]).rightAlign(!1).margin({top:5,right:0,bottom:5,left:20}),V.select(".nv-controlsWrap").datum(W).attr("transform","translate(0,"+-m.top+")").call(j)}else V.select(".nv-controlsWrap").selectAll("*").remove();T.attr("transform","translate("+m.left+","+m.top+")"),t&&V.select(".nv-y.nv-axis").attr("transform","translate("+M+",0)");var X=l.filter(function(a){return a.tempDisabled});T.select(".tempDisabled").remove(),X.length&&T.append("text").attr("class","tempDisabled").attr("x",M/2).attr("y","-.71em").style("text-anchor","end").text(X.map(function(a){return a.key}).join(", ")+" values cannot be calculated for this time period."),v&&(k.width(M).height(N).margin({left:m.left,top:m.top}).svgContainer(L).xScale(d),T.select(".nv-interactive").call(k)),U.select(".nv-background").append("rect"),V.select(".nv-background rect").attr("width",M).attr("height",N),f.y(function(a){return a.display.y}).width(M).height(N).color(l.map(function(a,b){return a.color||n(a,b)}).filter(function(a,b){return!l[b].disabled&&!l[b].tempDisabled}));var Y=V.select(".nv-linesWrap").datum(l.filter(function(a){return!a.disabled&&!a.tempDisabled}));Y.call(f),l.forEach(function(a,b){a.seriesIndex=b});var Z=l.filter(function(a){return!a.disabled&&!!B(a)}),$=V.select(".nv-avgLinesWrap").selectAll("line").data(Z,function(a){return a.key}),_=function(a){var b=e(B(a));return 0>b?0:b>N?N:b};$.enter().append("line").style("stroke-width",2).style("stroke-dasharray","10,10").style("stroke",function(a,b){return f.color()(a,a.seriesIndex)}).attr("x1",0).attr("x2",M).attr("y1",_).attr("y2",_),$.style("stroke-opacity",function(a){var b=e(B(a));return 0>b||b>N?0:1}).attr("x1",0).attr("x2",M).attr("y1",_).attr("y2",_),$.exit().remove();var aa=Y.selectAll(".nv-indexLine").data([G]);aa.enter().append("rect").attr("class","nv-indexLine").attr("width",3).attr("x",-2).attr("fill","red").attr("fill-opacity",.5).style("pointer-events","all").call(P),aa.attr("transform",function(a){return"translate("+F(a.i)+",0)"}).attr("height",N),r&&(g.scale(d)._ticks(a.utils.calcTicksX(M/70,l)).tickSize(-N,0),V.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),V.select(".nv-x.nv-axis").call(g)),s&&(h.scale(e)._ticks(a.utils.calcTicksY(N/36,l)).tickSize(-M,0),V.select(".nv-y.nv-axis").call(h)),V.select(".nv-background rect").on("click",function(){G.x=d3.mouse(this)[0],G.i=Math.round(F.invert(G.x)),y.index=G.i,C.stateChange(y),K()}),f.dispatch.on("elementClick",function(a){G.i=a.pointIndex,G.x=F(G.i),y.index=G.i,C.stateChange(y),K()}),j.dispatch.on("legendClick",function(a,c){a.disabled=!a.disabled,w=!a.disabled,y.rescaleY=w,C.stateChange(y),b.update()}),i.dispatch.on("stateChange",function(a){for(var c in a)y[c]=a[c];C.stateChange(y),b.update()}),k.dispatch.on("elementMousemove",function(c){f.clearHighlights();var d,e,i,j=[];if(l.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(g,h){e=a.interactiveBisect(g.values,c.pointXValue,b.x()),f.highlightPoint(h,e,!0);var k=g.values[e];"undefined"!=typeof k&&("undefined"==typeof d&&(d=k),"undefined"==typeof i&&(i=b.xScale()(b.x()(k,e))),j.push({key:g.key,value:b.y()(k,e),color:n(g,g.seriesIndex)}))}),j.length>2){var m=b.yScale().invert(c.mouseY),o=Math.abs(b.yScale().domain()[0]-b.yScale().domain()[1]),p=.03*o,q=a.nearestValueIndex(j.map(function(a){return a.value}),m,p);null!==q&&(j[q].highlight=!0)}var r=g.tickFormat()(b.x()(d,e),e);k.tooltip.valueFormatter(function(a,b){return h.tickFormat()(a)}).data({value:r,series:j})(),k.renderGuideLine(i)}),k.dispatch.on("elementMouseout",function(a){f.clearHighlights()}),C.on("changeState",function(a){"undefined"!=typeof a.disabled&&(l.forEach(function(b,c){b.disabled=a.disabled[c]}),y.disabled=a.disabled),"undefined"!=typeof a.index&&(G.i=a.index,G.x=F(G.i),y.index=a.index,aa.data([G])),"undefined"!=typeof a.rescaleY&&(w=a.rescaleY),b.update()})}),H.renderEnd("cumulativeLineChart immediate"),b}function c(a,b){return K||(K=f.y()),b.map(function(b,c){if(!b.values)return b;var d=b.values[a];if(null==d)return b;var e=K(d,a);return-.95>e&&!E?(b.tempDisabled=!0,b):(b.tempDisabled=!1,b.values=b.values.map(function(a,b){return a.display={y:(K(a,b)-e)/(1+e)},a}),b)})}var d,e,f=a.models.line(),g=a.models.axis(),h=a.models.axis(),i=a.models.legend(),j=a.models.legend(),k=a.interactiveGuideline(),l=a.models.tooltip(),m={top:30,right:30,bottom:50,left:60},n=a.utils.defaultColor(),o=null,p=null,q=!0,r=!0,s=!0,t=!1,u=!0,v=!1,w=!0,x=f.id(),y=a.utils.state(),z=null,A=null,B=function(a){return a.average},C=d3.dispatch("stateChange","changeState","renderEnd"),D=250,E=!1;y.index=0,y.rescaleY=w,g.orient("bottom").tickPadding(7),h.orient(t?"right":"left"),l.valueFormatter(function(a,b){return h.tickFormat()(a,b)}).headerFormatter(function(a,b){return g.tickFormat()(a,b)}),j.updateState(!1);var F=d3.scale.linear(),G={i:0,x:0},H=a.utils.renderWatch(C,D),I=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),index:G.i,rescaleY:w}}},J=function(a){return function(b){void 0!==b.index&&(G.i=b.index),void 0!==b.rescaleY&&(w=b.rescaleY),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};f.dispatch.on("elementMouseover.tooltip",function(a){var c={x:b.x()(a.point),y:b.y()(a.point),color:a.point.color};a.point=c,l.data(a).hidden(!1)}),f.dispatch.on("elementMouseout.tooltip",function(a){l.hidden(!0)});var K=null;return b.dispatch=C,b.lines=f,b.legend=i,b.controls=j,b.xAxis=g,b.yAxis=h,b.interactiveLayer=k,b.state=y,b.tooltip=l,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return o},set:function(a){o=a}},height:{get:function(){return p},set:function(a){p=a}},rescaleY:{get:function(){return w},set:function(a){w=a}},showControls:{get:function(){return u},set:function(a){u=a}},showLegend:{get:function(){return q},set:function(a){q=a}},average:{get:function(){return B},set:function(a){B=a}},defaultState:{get:function(){return z},set:function(a){z=a}},noData:{get:function(){return A},set:function(a){A=a}},showXAxis:{get:function(){return r},set:function(a){r=a}},showYAxis:{get:function(){return s},set:function(a){s=a}},noErrorCheck:{get:function(){return E},set:function(a){E=a}},margin:{get:function(){return m},set:function(a){m.top=void 0!==a.top?a.top:m.top,m.right=void 0!==a.right?a.right:m.right,m.bottom=void 0!==a.bottom?a.bottom:m.bottom,m.left=void 0!==a.left?a.left:m.left}},color:{get:function(){return n},set:function(b){n=a.utils.getColor(b),i.color(n)}},useInteractiveGuideline:{get:function(){return v},set:function(a){v=a,a===!0&&(b.interactive(!1),b.useVoronoi(!1))}},rightAlignYAxis:{get:function(){return t},set:function(a){t=a,h.orient(a?"right":"left")}},duration:{get:function(){return D},set:function(a){D=a,f.duration(D),g.duration(D),h.duration(D),H.reset(D)}}}),a.utils.inheritOptions(b,f),a.utils.initOptions(b),b},a.models.discreteBar=function(){"use strict";function b(m){return y.reset(),m.each(function(b){var m=k-j.left-j.right,x=l-j.top-j.bottom;c=d3.select(this),a.utils.initSVG(c),b.forEach(function(a,b){a.values.forEach(function(a){a.series=b})});var z=d&&e?[]:b.map(function(a){return a.values.map(function(a,b){return{x:p(a,b),y:q(a,b),y0:a.y0}})});n.domain(d||d3.merge(z).map(function(a){return a.x})).rangeBands(f||[0,m],.1),o.domain(e||d3.extent(d3.merge(z).map(function(a){return a.y}).concat(r))),t?o.range(g||[x-(o.domain()[0]<0?12:0),o.domain()[1]>0?12:0]):o.range(g||[x,0]),h=h||n,i=i||o.copy().range([o(0),o(0)]);var A=c.selectAll("g.nv-wrap.nv-discretebar").data([b]),B=A.enter().append("g").attr("class","nvd3 nv-wrap nv-discretebar"),C=B.append("g");A.select("g");C.append("g").attr("class","nv-groups"),A.attr("transform","translate("+j.left+","+j.top+")");var D=A.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a){return a.key});D.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),D.exit().watchTransition(y,"discreteBar: exit groups").style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),D.attr("class",function(a,b){return"nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}),D.watchTransition(y,"discreteBar: groups").style("stroke-opacity",1).style("fill-opacity",.75);var E=D.selectAll("g.nv-bar").data(function(a){return a.values});E.exit().remove();var F=E.enter().append("g").attr("transform",function(a,b,c){return"translate("+(n(p(a,b))+.05*n.rangeBand())+", "+o(0)+")"}).on("mouseover",function(a,b){d3.select(this).classed("hover",!0),v.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),v.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")})}).on("mousemove",function(a,b){v.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){var c=this;v.elementClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}).on("dblclick",function(a,b){v.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation()});F.append("rect").attr("height",0).attr("width",.9*n.rangeBand()/b.length),t?(F.append("text").attr("text-anchor","middle"),E.select("text").text(function(a,b){return u(q(a,b))}).watchTransition(y,"discreteBar: bars text").attr("x",.9*n.rangeBand()/2).attr("y",function(a,b){return q(a,b)<0?o(q(a,b))-o(0)+12:-4})):E.selectAll("text").remove(),E.attr("class",function(a,b){return q(a,b)<0?"nv-bar negative":"nv-bar positive"}).style("fill",function(a,b){return a.color||s(a,b)}).style("stroke",function(a,b){return a.color||s(a,b)}).select("rect").attr("class",w).watchTransition(y,"discreteBar: bars rect").attr("width",.9*n.rangeBand()/b.length),E.watchTransition(y,"discreteBar: bars").attr("transform",function(a,b){var c=n(p(a,b))+.05*n.rangeBand(),d=q(a,b)<0?o(0):o(0)-o(q(a,b))<1?o(0)-1:o(q(a,b));return"translate("+c+", "+d+")"}).select("rect").attr("height",function(a,b){return Math.max(Math.abs(o(q(a,b))-o(0)),1)}),h=n.copy(),i=o.copy()}),y.renderEnd("discreteBar immediate"),b}var c,d,e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=960,l=500,m=Math.floor(1e4*Math.random()),n=d3.scale.ordinal(),o=d3.scale.linear(),p=function(a){return a.x},q=function(a){return a.y},r=[0],s=a.utils.defaultColor(),t=!1,u=d3.format(",.2f"),v=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),w="discreteBar",x=250,y=a.utils.renderWatch(v,x);return b.dispatch=v,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},forceY:{get:function(){return r},set:function(a){r=a}},showValues:{get:function(){return t},set:function(a){t=a}},x:{get:function(){return p},set:function(a){p=a}},y:{get:function(){return q},set:function(a){q=a}},xScale:{get:function(){return n},set:function(a){n=a}},yScale:{get:function(){return o},set:function(a){o=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},valueFormat:{get:function(){return u},set:function(a){u=a}},id:{get:function(){return m},set:function(a){m=a}},rectClass:{get:function(){return w},set:function(a){w=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},color:{get:function(){return s},set:function(b){s=a.utils.getColor(b)}},duration:{get:function(){return x},set:function(a){x=a,y.reset(x)}}}),a.utils.initOptions(b),b},a.models.discreteBarChart=function(){"use strict";function b(i){return x.reset(),x.models(e),o&&x.models(f),p&&x.models(g),i.each(function(i){var m=d3.select(this);a.utils.initSVG(m);var u=a.utils.availableWidth(k,m,j),x=a.utils.availableHeight(l,m,j);if(b.update=function(){v.beforeUpdate(),m.transition().duration(w).call(b)},b.container=this,!(i&&i.length&&i.filter(function(a){return a.values.length}).length))return a.utils.noData(b,m),b;m.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var y=m.selectAll("g.nv-wrap.nv-discreteBarWithAxes").data([i]),z=y.enter().append("g").attr("class","nvd3 nv-wrap nv-discreteBarWithAxes").append("g"),A=z.append("defs"),B=y.select("g");z.append("g").attr("class","nv-x nv-axis"),z.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),z.append("g").attr("class","nv-barsWrap"),z.append("g").attr("class","nv-legendWrap"),B.attr("transform","translate("+j.left+","+j.top+")"),n?(h.width(u),B.select(".nv-legendWrap").datum(i).call(h),h.height()>j.top&&(j.top=h.height(),x=a.utils.availableHeight(l,m,j)),y.select(".nv-legendWrap").attr("transform","translate(0,"+-j.top+")")):B.select(".nv-legendWrap").selectAll("*").remove(),q&&B.select(".nv-y.nv-axis").attr("transform","translate("+u+",0)"),e.width(u).height(x);var C=B.select(".nv-barsWrap").datum(i.filter(function(a){return!a.disabled}));if(C.transition().call(e),A.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),B.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(r?2:1)).attr("height",16).attr("x",-c.rangeBand()/(r?1:2)),o){f.scale(c)._ticks(a.utils.calcTicksX(u/100,i)).tickSize(-x,0),B.select(".nv-x.nv-axis").attr("transform","translate(0,"+(d.range()[0]+(e.showValues()&&d.domain()[0]<0?16:0))+")"),B.select(".nv-x.nv-axis").call(f);var D=B.select(".nv-x.nv-axis").selectAll("g");r&&D.selectAll("text").attr("transform",function(a,b,c){return"translate(0,"+(c%2==0?"5":"17")+")"}),t&&D.selectAll(".tick text").attr("transform","rotate("+t+" 0,0)").style("text-anchor",t>0?"start":"end"),s&&B.selectAll(".tick text").call(a.utils.wrapTicks,b.xAxis.rangeBand())}p&&(g.scale(d)._ticks(a.utils.calcTicksY(x/36,i)).tickSize(-u,0),B.select(".nv-y.nv-axis").call(g)),B.select(".nv-zeroLine line").attr("x1",0).attr("x2",q?-u:u).attr("y1",d(0)).attr("y2",d(0))}),x.renderEnd("discreteBar chart immediate"),b}var c,d,e=a.models.discreteBar(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend(),i=a.models.tooltip(),j={top:15,right:10,bottom:50,left:60},k=null,l=null,m=a.utils.getColor(),n=!1,o=!0,p=!0,q=!1,r=!1,s=!1,t=0,u=null,v=d3.dispatch("beforeUpdate","renderEnd"),w=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(q?"right":"left").tickFormat(d3.format(",.1f")),i.duration(0).headerEnabled(!1).valueFormatter(function(a,b){return g.tickFormat()(a,b)}).keyFormatter(function(a,b){return f.tickFormat()(a,b)});var x=a.utils.renderWatch(v,w);return e.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:b.x()(a.data),value:b.y()(a.data),color:a.color},i.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){i.hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){i()}),b.dispatch=v,b.discretebar=e,b.legend=h,b.xAxis=f,b.yAxis=g,b.tooltip=i,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},showLegend:{get:function(){return n},set:function(a){n=a}},staggerLabels:{get:function(){return r},set:function(a){r=a}},rotateLabels:{get:function(){return t},set:function(a){t=a}},wrapLabels:{get:function(){return s},set:function(a){s=!!a}},showXAxis:{get:function(){return o},set:function(a){o=a}},showYAxis:{get:function(){return p},set:function(a){p=a}},noData:{get:function(){return u},set:function(a){u=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return w},set:function(a){w=a,x.reset(w),e.duration(w),f.duration(w),g.duration(w)}},color:{get:function(){return m},set:function(b){m=a.utils.getColor(b),e.color(m),h.color(m)}},rightAlignYAxis:{get:function(){return q},set:function(a){q=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.distribution=function(){"use strict";function b(k){return m.reset(),k.each(function(b){var k=(e-("x"===g?d.left+d.right:d.top+d.bottom),"x"==g?"y":"x"),l=d3.select(this);a.utils.initSVG(l),c=c||j;var n=l.selectAll("g.nv-distribution").data([b]),o=n.enter().append("g").attr("class","nvd3 nv-distribution"),p=(o.append("g"),n.select("g"));n.attr("transform","translate("+d.left+","+d.top+")");var q=p.selectAll("g.nv-dist").data(function(a){return a},function(a){return a.key});q.enter().append("g"),q.attr("class",function(a,b){return"nv-dist nv-series-"+b}).style("stroke",function(a,b){return i(a,b)});var r=q.selectAll("line.nv-dist"+g).data(function(a){return a.values});r.enter().append("line").attr(g+"1",function(a,b){return c(h(a,b))}).attr(g+"2",function(a,b){return c(h(a,b))}),m.transition(q.exit().selectAll("line.nv-dist"+g),"dist exit").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}).style("stroke-opacity",0).remove(),r.attr("class",function(a,b){return"nv-dist"+g+" nv-dist"+g+"-"+b}).attr(k+"1",0).attr(k+"2",f),m.transition(r,"dist").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}),c=j.copy()}),m.renderEnd("distribution immediate"),b}var c,d={top:0,right:0,bottom:0,left:0},e=400,f=8,g="x",h=function(a){return a[g]},i=a.utils.defaultColor(),j=d3.scale.linear(),k=250,l=d3.dispatch("renderEnd"),m=a.utils.renderWatch(l,k);return b.options=a.utils.optionsFunc.bind(b),b.dispatch=l,b.margin=function(a){return arguments.length?(d.top="undefined"!=typeof a.top?a.top:d.top,d.right="undefined"!=typeof a.right?a.right:d.right,d.bottom="undefined"!=typeof a.bottom?a.bottom:d.bottom,d.left="undefined"!=typeof a.left?a.left:d.left,b):d},b.width=function(a){return arguments.length?(e=a,b):e},b.axis=function(a){return arguments.length?(g=a,b):g},b.size=function(a){return arguments.length?(f=a,b):f},b.getData=function(a){return arguments.length?(h=d3.functor(a),b):h},b.scale=function(a){return arguments.length?(j=a,b):j},b.color=function(c){return arguments.length?(i=a.utils.getColor(c),b):i},b.duration=function(a){return arguments.length?(k=a,m.reset(k),b):k},b},a.models.focus=function(b){"use strict";function c(t){return s.reset(),s.models(b),m&&s.models(f),n&&s.models(g),t.each(function(s){function t(a){var b=+("e"==a),c=b?1:-1,d=y/3;return"M"+.5*c+","+d+"A6,6 0 0 "+b+" "+6.5*c+","+(d+6)+"V"+(2*d-6)+"A6,6 0 0 "+b+" "+.5*c+","+2*d+"ZM"+2.5*c+","+(d+8)+"V"+(2*d-8)+"M"+4.5*c+","+(d+8)+"V"+(2*d-8)}function u(){h.empty()||h.extent(p),D.data([h.empty()?d.domain():p]).each(function(a,b){var c=d(a[0])-d.range()[0],e=x-d(a[1]);d3.select(this).select(".left").attr("width",0>c?0:c),d3.select(this).select(".right").attr("x",d(a[1])).attr("width",0>e?0:e)})}function v(){p=h.empty()?null:h.extent();var a=h.empty()?d.domain():h.extent();Math.abs(a[0]-a[1])<=1||(r.brush({extent:a,brush:h}),u(),r.onBrush(a))}var w=d3.select(this);a.utils.initSVG(w);var x=a.utils.availableWidth(k,w,i),y=l-i.top-i.bottom;c.update=function(){0===q?w.call(c):w.transition().duration(q).call(c)},c.container=this,d=b.xScale(),e=b.yScale();var z=w.selectAll("g.nv-focus").data([s]),A=z.enter().append("g").attr("class","nvd3 nv-focus").append("g"),B=z.select("g");z.attr("transform","translate("+i.left+","+i.top+")"),A.append("g").attr("class","nv-background").append("rect"),A.append("g").attr("class","nv-x nv-axis"),A.append("g").attr("class","nv-y nv-axis"),A.append("g").attr("class","nv-contentWrap"),A.append("g").attr("class","nv-brushBackground"),A.append("g").attr("class","nv-x nv-brush"),o&&B.select(".nv-y.nv-axis").attr("transform","translate("+x+",0)"),B.select(".nv-background rect").attr("width",x).attr("height",y),b.width(x).height(y).color(s.map(function(a,b){return a.color||j(a,b)}).filter(function(a,b){return!s[b].disabled}));var C=B.select(".nv-contentWrap").datum(s.filter(function(a){return!a.disabled}));d3.transition(C).call(b),h.x(d).on("brush",function(){v()}),p&&h.extent(p);var D=B.select(".nv-brushBackground").selectAll("g").data([p||h.extent()]),E=D.enter().append("g");E.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",y),E.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",y);var F=B.select(".nv-x.nv-brush").call(h);F.selectAll("rect").attr("height",y),F.selectAll(".resize").append("path").attr("d",t),v(),B.select(".nv-background rect").attr("width",x).attr("height",y),m&&(f.scale(d)._ticks(a.utils.calcTicksX(x/100,s)).tickSize(-y,0),B.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),d3.transition(B.select(".nv-x.nv-axis")).call(f)),n&&(g.scale(e)._ticks(a.utils.calcTicksY(y/36,s)).tickSize(-x,0),d3.transition(B.select(".nv-y.nv-axis")).call(g)),B.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")")}),s.renderEnd("focus immediate"),c}var d,e,b=b||a.models.line(),f=a.models.axis(),g=a.models.axis(),h=d3.svg.brush(),i={top:10,right:0,bottom:30,left:0},j=a.utils.defaultColor(),k=null,l=70,m=!0,n=!1,o=!1,p=null,q=250,r=d3.dispatch("brush","onBrush","renderEnd");b.interactive(!1),b.pointActive(function(a){return!1});var s=a.utils.renderWatch(r,q);return c.dispatch=r,c.content=b,c.brush=h,c.xAxis=f,c.yAxis=g,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},showXAxis:{get:function(){return m},set:function(a){m=a}},showYAxis:{get:function(){return n},set:function(a){n=a}},brushExtent:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.left:i.left}},duration:{get:function(){return q},set:function(a){q=a,s.reset(q),b.duration(q),f.duration(q),g.duration(q)}},color:{get:function(){return j},set:function(c){j=a.utils.getColor(c),b.color(j)}},interpolate:{get:function(){return b.interpolate()},set:function(a){b.interpolate(a)}},xTickFormat:{get:function(){return f.tickFormat()},set:function(a){f.tickFormat(a)}},yTickFormat:{get:function(){return g.tickFormat()},set:function(a){g.tickFormat(a)}},x:{get:function(){return b.x()},set:function(a){b.x(a)}},y:{get:function(){return b.y()},set:function(a){b.y(a)}},rightAlignYAxis:{get:function(){return o},set:function(a){o=a,g.orient(o?"right":"left")}}}),a.utils.inheritOptions(c,b),a.utils.initOptions(c),c},a.models.forceDirectedGraph=function(){"use strict";function b(g){return u.reset(),g.each(function(g){f=d3.select(this),a.utils.initSVG(f);var j=a.utils.availableWidth(d,f,c),u=a.utils.availableHeight(e,f,c);if(f.attr("width",j).attr("height",u),!(g&&g.links&&g.nodes))return a.utils.noData(b,f),b;f.selectAll(".nv-noData").remove(),f.selectAll("*").remove();var v=new Set;g.nodes.forEach(function(a){var b=Object.keys(a);b.forEach(function(a){v.add(a)})});var w=d3.layout.force().nodes(g.nodes).links(g.links).size([j,u]).linkStrength(k).friction(l).linkDistance(m).charge(n).gravity(o).theta(p).alpha(q).start(),x=f.selectAll(".link").data(g.links).enter().append("line").attr("class","nv-force-link").style("stroke-width",function(a){return Math.sqrt(a.value)}),y=f.selectAll(".node").data(g.nodes).enter().append("g").attr("class","nv-force-node").call(w.drag);y.append("circle").attr("r",r).style("fill",function(a){return h(a)}).on("mouseover",function(a){f.select(".nv-series-"+a.seriesIndex+" .nv-distx-"+a.pointIndex).attr("y1",a.py),f.select(".nv-series-"+a.seriesIndex+" .nv-disty-"+a.pointIndex).attr("x2",a.px);var b=h(a);a.series=[],v.forEach(function(c){a.series.push({color:b,key:c,value:a[c]})}),i.data(a).hidden(!1)}).on("mouseout",function(a){i.hidden(!0)}),i.headerFormatter(function(a){return"Node"}),t(x),s(y),w.on("tick",function(){x.attr("x1",function(a){return a.source.x}).attr("y1",function(a){return a.source.y}).attr("x2",function(a){return a.target.x}).attr("y2",function(a){return a.target.y}),y.attr("transform",function(a){return"translate("+a.x+", "+a.y+")"})})}),b}var c={top:2,right:0,bottom:2,left:0},d=400,e=32,f=null,g=d3.dispatch("renderEnd"),h=a.utils.getColor(["#000"]),i=a.models.tooltip(),j=null,k=.1,l=.9,m=30,n=-120,o=.1,p=.8,q=.1,r=5,s=function(a){},t=function(a){},u=a.utils.renderWatch(g);return b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},linkStrength:{get:function(){return k},set:function(a){k=a}},friction:{get:function(){return l},set:function(a){l=a}},linkDist:{get:function(){return m},set:function(a){m=a}},charge:{get:function(){return n},set:function(a){n=a}},gravity:{get:function(){return o},set:function(a){o=a}},theta:{get:function(){return p},set:function(a){p=a}},alpha:{get:function(){return q},set:function(a){q=a}},radius:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return getX},set:function(a){getX=d3.functor(a)}},y:{get:function(){return getY},set:function(a){getY=d3.functor(a)}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}},noData:{get:function(){return j},set:function(a){j=a}},nodeExtras:{get:function(){return s},set:function(a){s=a}},linkExtras:{get:function(){return t},set:function(a){t=a}}}),b.dispatch=g,b.tooltip=i,a.utils.initOptions(b),b},a.models.furiousLegend=function(){"use strict";function b(r){function s(a,b){return"furious"!=q?"#000":o?a.disengaged?h(a,b):"#fff":o?void 0:a.disabled?h(a,b):"#fff"}function t(a,b){return o&&"furious"==q?a.disengaged?"#fff":h(a,b):a.disabled?"#fff":h(a,b)}return r.each(function(b){var r=d-c.left-c.right,u=d3.select(this);a.utils.initSVG(u);var v=u.selectAll("g.nv-legend").data([b]),w=(v.enter().append("g").attr("class","nvd3 nv-legend").append("g"),v.select("g"));v.attr("transform","translate("+c.left+","+c.top+")");var x,y=w.selectAll(".nv-series").data(function(a){return"furious"!=q?a:a.filter(function(a){return o?!0:!a.disengaged})}),z=y.enter().append("g").attr("class","nv-series");if("classic"==q)z.append("circle").style("stroke-width",2).attr("class","nv-legend-symbol").attr("r",5),x=y.select("circle");else if("furious"==q){z.append("rect").style("stroke-width",2).attr("class","nv-legend-symbol").attr("rx",3).attr("ry",3),x=y.select("rect"),z.append("g").attr("class","nv-check-box").property("innerHTML",'<path d="M0.5,5 L22.5,5 L22.5,26.5 L0.5,26.5 L0.5,5 Z" class="nv-box"></path><path d="M5.5,12.8618467 L11.9185089,19.2803556 L31,0.198864511" class="nv-check"></path>').attr("transform","translate(-10,-8)scale(0.5)");var A=y.select(".nv-check-box");A.each(function(a,b){d3.select(this).selectAll("path").attr("stroke",s(a,b))})}z.append("text").attr("text-anchor","start").attr("class","nv-legend-text").attr("dy",".32em").attr("dx","8");var B=y.select("text.nv-legend-text");y.on("mouseover",function(a,b){p.legendMouseover(a,b)}).on("mouseout",function(a,b){p.legendMouseout(a,b)}).on("click",function(a,b){p.legendClick(a,b);var c=y.data();if(m){if("classic"==q)n?(c.forEach(function(a){a.disabled=!0}),a.disabled=!1):(a.disabled=!a.disabled,c.every(function(a){return a.disabled})&&c.forEach(function(a){a.disabled=!1}));else if("furious"==q)if(o)a.disengaged=!a.disengaged,a.userDisabled=void 0==a.userDisabled?!!a.disabled:a.userDisabled,a.disabled=a.disengaged||a.userDisabled;else if(!o){a.disabled=!a.disabled,a.userDisabled=a.disabled;var d=c.filter(function(a){return!a.disengaged});d.every(function(a){return a.userDisabled})&&c.forEach(function(a){a.disabled=a.userDisabled=!1})}p.stateChange({disabled:c.map(function(a){return!!a.disabled}),disengaged:c.map(function(a){return!!a.disengaged})})}}).on("dblclick",function(a,b){if(("furious"!=q||!o)&&(p.legendDblclick(a,b),m)){var c=y.data();c.forEach(function(a){a.disabled=!0,"furious"==q&&(a.userDisabled=a.disabled)}),a.disabled=!1,"furious"==q&&(a.userDisabled=a.disabled),p.stateChange({disabled:c.map(function(a){return!!a.disabled})})}}),y.classed("nv-disabled",function(a){return a.userDisabled}),y.exit().remove(),B.attr("fill",s).text(function(a){return g(f(a))});var C;switch(q){case"furious":C=23;break;case"classic":C=20}if(j){var D=[];y.each(function(b,c){var d;if(g(f(b))&&g(f(b)).length>i){var e=g(f(b)).substring(0,i);d=d3.select(this).select("text").text(e+"..."),d3.select(this).append("svg:title").text(g(f(b)))}else d=d3.select(this).select("text");var h;try{if(h=d.node().getComputedTextLength(),0>=h)throw Error()}catch(j){h=a.utils.calcApproxTextWidth(d)}D.push(h+k)});for(var E=0,F=0,G=[];r>F&&E<D.length;)G[E]=D[E],F+=D[E++];for(0===E&&(E=1);F>r&&E>1;){G=[],E--;for(var H=0;H<D.length;H++)D[H]>(G[H%E]||0)&&(G[H%E]=D[H]);F=G.reduce(function(a,b,c,d){return a+b})}for(var I=[],J=0,K=0;E>J;J++)I[J]=K,K+=G[J];y.attr("transform",function(a,b){return"translate("+I[b%E]+","+(5+Math.floor(b/E)*C)+")"}),l?w.attr("transform","translate("+(d-c.right-F)+","+c.top+")"):w.attr("transform","translate(0,"+c.top+")"),e=c.top+c.bottom+Math.ceil(D.length/E)*C}else{var L,M=5,N=5,O=0;y.attr("transform",function(a,b){var e=d3.select(this).select("text").node().getComputedTextLength()+k;return L=N,d<c.left+c.right+L+e&&(N=L=5,M+=C),N+=e,N>O&&(O=N),"translate("+L+","+M+")"}),w.attr("transform","translate("+(d-c.right-O)+","+c.top+")"),e=c.top+c.bottom+M+15}"furious"==q&&x.attr("width",function(a,b){return B[0][b].getComputedTextLength()+27}).attr("height",18).attr("y",-9).attr("x",-15),x.style("fill",t).style("stroke",function(a,b){return a.color||h(a,b)})}),b}var c={top:5,right:0,bottom:5,left:0},d=400,e=20,f=function(a){return a.key},g=function(a){return a},h=a.utils.getColor(),i=20,j=!0,k=28,l=!0,m=!0,n=!1,o=!1,p=d3.dispatch("legendClick","legendDblclick","legendMouseover","legendMouseout","stateChange"),q="classic";return b.dispatch=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},key:{get:function(){return f},set:function(a){f=a}},keyFormatter:{get:function(){return g},set:function(a){g=a}},align:{get:function(){return j},set:function(a){j=a}},rightAlign:{get:function(){return l},set:function(a){l=a}},maxKeyLength:{get:function(){return i},set:function(a){i=a}},padding:{get:function(){return k},set:function(a){k=a}},updateState:{get:function(){return m},set:function(a){m=a}},radioButtonMode:{get:function(){return n},set:function(a){n=a}},expanded:{get:function(){return o},set:function(a){o=a}},vers:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.historicalBar=function(){"use strict";function b(x){return x.each(function(b){w.reset(),k=d3.select(this);var x=a.utils.availableWidth(h,k,g),y=a.utils.availableHeight(i,k,g);a.utils.initSVG(k),l.domain(c||d3.extent(b[0].values.map(n).concat(p))),r?l.range(e||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(e||[0,x]),m.domain(d||d3.extent(b[0].values.map(o).concat(q))).range(f||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var z=k.selectAll("g.nv-wrap.nv-historicalBar-"+j).data([b[0].values]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBar-"+j),B=A.append("defs"),C=A.append("g"),D=z.select("g");C.append("g").attr("class","nv-bars"),z.attr("transform","translate("+g.left+","+g.top+")"),k.on("click",function(a,b){u.chartClick({data:a,index:b,pos:d3.event,id:j})}),B.append("clipPath").attr("id","nv-chart-clip-path-"+j).append("rect"),z.select("#nv-chart-clip-path-"+j+" rect").attr("width",x).attr("height",y),D.attr("clip-path",s?"url(#nv-chart-clip-path-"+j+")":"");var E=z.select(".nv-bars").selectAll(".nv-bar").data(function(a){return a},function(a,b){return n(a,b)});E.exit().remove(),E.enter().append("rect").attr("x",0).attr("y",function(b,c){return a.utils.NaNtoZero(m(Math.max(0,o(b,c))))}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.abs(m(o(b,c))-m(0)))}).attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).on("mouseover",function(a,b){v&&(d3.select(this).classed("hover",!0),u.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mouseout",function(a,b){v&&(d3.select(this).classed("hover",!1),u.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mousemove",function(a,b){v&&u.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){v&&(u.elementClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation())}).on("dblclick",function(a,b){v&&(u.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation())}),E.attr("fill",function(a,b){return t(a,b)}).attr("class",function(a,b,c){return(o(a,b)<0?"nv-bar negative":"nv-bar positive")+" nv-bar-"+c+"-"+b}).watchTransition(w,"bars").attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).attr("width",x/b[0].values.length*.9),E.watchTransition(w,"bars").attr("y",function(b,c){var d=o(b,c)<0?m(0):m(0)-m(o(b,c))<1?m(0)-1:m(o(b,c));return a.utils.NaNtoZero(d)}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.max(Math.abs(m(o(b,c))-m(0)),1))})}),w.renderEnd("historicalBar immediate"),b}var c,d,e,f,g={top:0,right:0,bottom:0,left:0},h=null,i=null,j=Math.floor(1e4*Math.random()),k=null,l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=[],q=[0],r=!1,s=!0,t=a.utils.defaultColor(),u=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),v=!0,w=a.utils.renderWatch(u,0);return b.highlightPoint=function(a,b){k.select(".nv-bars .nv-bar-0-"+a).classed("hover",b)},b.clearHighlights=function(){k.select(".nv-bars .nv-bar.hover").classed("hover",!1)},b.dispatch=u,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return i},set:function(a){i=a}},forceX:{get:function(){return p},set:function(a){p=a}},forceY:{get:function(){return q},set:function(a){q=a}},padData:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return d},set:function(a){d=a}},xRange:{get:function(){return e},set:function(a){e=a}},yRange:{get:function(){return f},set:function(a){f=a}},clipEdge:{get:function(){return s},set:function(a){s=a}},id:{get:function(){return j},set:function(a){j=a}},interactive:{get:function(){return v},set:function(a){v=a}},margin:{get:function(){return g},set:function(a){g.top=void 0!==a.top?a.top:g.top,g.right=void 0!==a.right?a.right:g.right,g.bottom=void 0!==a.bottom?a.bottom:g.bottom,g.left=void 0!==a.left?a.left:g.left}},color:{get:function(){return t},set:function(b){t=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.historicalBarChart=function(b){"use strict";function c(b){return b.each(function(k){z.reset(),z.models(f),q&&z.models(g),r&&z.models(h);var w=d3.select(this);a.utils.initSVG(w);var A=a.utils.availableWidth(n,w,l),B=a.utils.availableHeight(o,w,l);if(c.update=function(){w.transition().duration(y).call(c)},c.container=this,u.disabled=k.map(function(a){return!!a.disabled}),!v){var C;v={};for(C in u)u[C]instanceof Array?v[C]=u[C].slice(0):v[C]=u[C]}if(!(k&&k.length&&k.filter(function(a){return a.values.length}).length))return a.utils.noData(c,w),c;w.selectAll(".nv-noData").remove(),d=f.xScale(),e=f.yScale();var D=w.selectAll("g.nv-wrap.nv-historicalBarChart").data([k]),E=D.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBarChart").append("g"),F=D.select("g");E.append("g").attr("class","nv-x nv-axis"),E.append("g").attr("class","nv-y nv-axis"),E.append("g").attr("class","nv-barsWrap"),E.append("g").attr("class","nv-legendWrap"),E.append("g").attr("class","nv-interactive"),p?(i.width(A),F.select(".nv-legendWrap").datum(k).call(i),i.height()>l.top&&(l.top=i.height(),B=a.utils.availableHeight(o,w,l)),D.select(".nv-legendWrap").attr("transform","translate(0,"+-l.top+")")):F.select(".nv-legendWrap").selectAll("*").remove(),D.attr("transform","translate("+l.left+","+l.top+")"),s&&F.select(".nv-y.nv-axis").attr("transform","translate("+A+",0)"),t&&(j.width(A).height(B).margin({left:l.left,top:l.top}).svgContainer(w).xScale(d),D.select(".nv-interactive").call(j)),f.width(A).height(B).color(k.map(function(a,b){return a.color||m(a,b)}).filter(function(a,b){return!k[b].disabled}));var G=F.select(".nv-barsWrap").datum(k.filter(function(a){return!a.disabled}));G.transition().call(f),q&&(g.scale(d)._ticks(a.utils.calcTicksX(A/100,k)).tickSize(-B,0),F.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),F.select(".nv-x.nv-axis").transition().call(g)),r&&(h.scale(e)._ticks(a.utils.calcTicksY(B/36,k)).tickSize(-A,0),F.select(".nv-y.nv-axis").transition().call(h)),j.dispatch.on("elementMousemove",function(b){f.clearHighlights();var d,e,i,l=[];k.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(g,h){e=a.interactiveBisect(g.values,b.pointXValue,c.x()),f.highlightPoint(e,!0);var j=g.values[e];void 0!==j&&(void 0===d&&(d=j),void 0===i&&(i=c.xScale()(c.x()(j,e))),l.push({key:g.key,value:c.y()(j,e),color:m(g,g.seriesIndex),data:g.values[e]}))});var n=g.tickFormat()(c.x()(d,e));j.tooltip.valueFormatter(function(a,b){return h.tickFormat()(a)}).data({value:n,index:e,series:l})(),j.renderGuideLine(i)}),j.dispatch.on("elementMouseout",function(a){x.tooltipHide(),f.clearHighlights()}),i.dispatch.on("legendClick",function(a,d){a.disabled=!a.disabled,k.filter(function(a){return!a.disabled}).length||k.map(function(a){return a.disabled=!1,D.selectAll(".nv-series").classed("disabled",!1),a}),u.disabled=k.map(function(a){return!!a.disabled}),x.stateChange(u),b.transition().call(c)}),i.dispatch.on("legendDblclick",function(a){k.forEach(function(a){a.disabled=!0}),a.disabled=!1,u.disabled=k.map(function(a){return!!a.disabled}),x.stateChange(u),c.update()}),x.on("changeState",function(a){"undefined"!=typeof a.disabled&&(k.forEach(function(b,c){b.disabled=a.disabled[c]}),u.disabled=a.disabled),c.update()})}),z.renderEnd("historicalBarChart immediate"),c}var d,e,f=b||a.models.historicalBar(),g=a.models.axis(),h=a.models.axis(),i=a.models.legend(),j=a.interactiveGuideline(),k=a.models.tooltip(),l={top:30,right:90,bottom:50,left:90},m=a.utils.defaultColor(),n=null,o=null,p=!1,q=!0,r=!0,s=!1,t=!1,u={},v=null,w=null,x=d3.dispatch("tooltipHide","stateChange","changeState","renderEnd"),y=250;g.orient("bottom").tickPadding(7),h.orient(s?"right":"left"),k.duration(0).headerEnabled(!1).valueFormatter(function(a,b){return h.tickFormat()(a,b)}).headerFormatter(function(a,b){return g.tickFormat()(a,b)});var z=a.utils.renderWatch(x,0);return f.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:c.x()(a.data),value:c.y()(a.data),color:a.color},k.data(a).hidden(!1)}),f.dispatch.on("elementMouseout.tooltip",function(a){k.hidden(!0)}),f.dispatch.on("elementMousemove.tooltip",function(a){k()}),c.dispatch=x,c.bars=f,c.legend=i,c.xAxis=g,c.yAxis=h,c.interactiveLayer=j,c.tooltip=k,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return n},set:function(a){n=a}},height:{get:function(){return o},set:function(a){o=a}},showLegend:{get:function(){return p},set:function(a){p=a}},showXAxis:{get:function(){return q},set:function(a){q=a}},showYAxis:{get:function(){return r},set:function(a){r=a}},defaultState:{get:function(){return v},set:function(a){v=a}},noData:{get:function(){return w},set:function(a){w=a}},margin:{get:function(){return l},set:function(a){l.top=void 0!==a.top?a.top:l.top,l.right=void 0!==a.right?a.right:l.right,l.bottom=void 0!==a.bottom?a.bottom:l.bottom,l.left=void 0!==a.left?a.left:l.left}},color:{get:function(){return m},set:function(b){m=a.utils.getColor(b),i.color(m),f.color(m)}},duration:{get:function(){return y},set:function(a){y=a,z.reset(y),h.duration(y),g.duration(y)}},rightAlignYAxis:{get:function(){return s},set:function(a){s=a,h.orient(a?"right":"left")}},useInteractiveGuideline:{get:function(){return t},set:function(a){t=a,a===!0&&c.interactive(!1)}}}),a.utils.inheritOptions(c,f),a.utils.initOptions(c),c},a.models.ohlcBarChart=function(){var b=a.models.historicalBarChart(a.models.ohlcBar());return b.useInteractiveGuideline(!0),b.interactiveLayer.tooltip.contentGenerator(function(a){var c=a.series[0].data,d=c.open<c.close?"2ca02c":"d62728";return'<h3 style="color: #'+d+'">'+a.value+"</h3><table><tr><td>open:</td><td>"+b.yAxis.tickFormat()(c.open)+"</td></tr><tr><td>close:</td><td>"+b.yAxis.tickFormat()(c.close)+"</td></tr><tr><td>high</td><td>"+b.yAxis.tickFormat()(c.high)+"</td></tr><tr><td>low:</td><td>"+b.yAxis.tickFormat()(c.low)+"</td></tr></table>"}),b},a.models.candlestickBarChart=function(){var b=a.models.historicalBarChart(a.models.candlestickBar());return b.useInteractiveGuideline(!0),b.interactiveLayer.tooltip.contentGenerator(function(a){var c=a.series[0].data,d=c.open<c.close?"2ca02c":"d62728";return'<h3 style="color: #'+d+'">'+a.value+"</h3><table><tr><td>open:</td><td>"+b.yAxis.tickFormat()(c.open)+"</td></tr><tr><td>close:</td><td>"+b.yAxis.tickFormat()(c.close)+"</td></tr><tr><td>high</td><td>"+b.yAxis.tickFormat()(c.high)+"</td></tr><tr><td>low:</td><td>"+b.yAxis.tickFormat()(c.low)+"</td></tr></table>"}),b},a.models.legend=function(){"use strict";function b(r){function s(a,b){return"furious"!=q?"#000":o?a.disengaged?"#000":"#fff":o?void 0:(a.color||(a.color=h(a,b)),a.disabled?a.color:"#fff")}function t(a,b){return o&&"furious"==q&&a.disengaged?"#eee":a.color||h(a,b)}function u(a,b){return o&&"furious"==q?1:a.disabled?0:1}return r.each(function(b){var h=d-c.left-c.right,r=d3.select(this);a.utils.initSVG(r);var v=r.selectAll("g.nv-legend").data([b]),w=v.enter().append("g").attr("class","nvd3 nv-legend").append("g"),x=v.select("g");v.attr("transform","translate("+c.left+","+c.top+")");var y,z,A=x.selectAll(".nv-series").data(function(a){return"furious"!=q?a:a.filter(function(a){return o?!0:!a.disengaged})}),B=A.enter().append("g").attr("class","nv-series");switch(q){case"furious":z=23;break;case"classic":z=20}if("classic"==q)B.append("circle").style("stroke-width",2).attr("class","nv-legend-symbol").attr("r",5),y=A.select(".nv-legend-symbol");else if("furious"==q){B.append("rect").style("stroke-width",2).attr("class","nv-legend-symbol").attr("rx",3).attr("ry",3),y=A.select(".nv-legend-symbol"),B.append("g").attr("class","nv-check-box").property("innerHTML",'<path d="M0.5,5 L22.5,5 L22.5,26.5 L0.5,26.5 L0.5,5 Z" class="nv-box"></path><path d="M5.5,12.8618467 L11.9185089,19.2803556 L31,0.198864511" class="nv-check"></path>').attr("transform","translate(-10,-8)scale(0.5)");var C=A.select(".nv-check-box");C.each(function(a,b){d3.select(this).selectAll("path").attr("stroke",s(a,b))})}B.append("text").attr("text-anchor","start").attr("class","nv-legend-text").attr("dy",".32em").attr("dx","8");var D=A.select("text.nv-legend-text");A.on("mouseover",function(a,b){p.legendMouseover(a,b)}).on("mouseout",function(a,b){p.legendMouseout(a,b)}).on("click",function(a,b){p.legendClick(a,b);var c=A.data();if(m){if("classic"==q)n?(c.forEach(function(a){a.disabled=!0}),a.disabled=!1):(a.disabled=!a.disabled,c.every(function(a){return a.disabled})&&c.forEach(function(a){a.disabled=!1}));else if("furious"==q)if(o)a.disengaged=!a.disengaged,a.userDisabled=void 0==a.userDisabled?!!a.disabled:a.userDisabled,a.disabled=a.disengaged||a.userDisabled;else if(!o){a.disabled=!a.disabled,a.userDisabled=a.disabled;var d=c.filter(function(a){return!a.disengaged});d.every(function(a){return a.userDisabled})&&c.forEach(function(a){a.disabled=a.userDisabled=!1})}p.stateChange({disabled:c.map(function(a){return!!a.disabled}),disengaged:c.map(function(a){return!!a.disengaged})})}}).on("dblclick",function(a,b){if(("furious"!=q||!o)&&(p.legendDblclick(a,b),m)){var c=A.data();c.forEach(function(a){a.disabled=!0,"furious"==q&&(a.userDisabled=a.disabled)}),a.disabled=!1,"furious"==q&&(a.userDisabled=a.disabled),p.stateChange({disabled:c.map(function(a){return!!a.disabled})})}}),A.classed("nv-disabled",function(a){return a.userDisabled}),A.exit().remove(),D.attr("fill",s).text(function(a){return g(f(a))});var E=0;if(j){var F=[];A.each(function(b,c){var d;if(g(f(b))&&g(f(b)).length>i){var e=g(f(b)).substring(0,i);d=d3.select(this).select("text").text(e+"..."),d3.select(this).append("svg:title").text(g(f(b)))}else d=d3.select(this).select("text");var h;try{if(h=d.node().getComputedTextLength(),0>=h)throw Error()}catch(j){h=a.utils.calcApproxTextWidth(d)}F.push(h+k)});var G=0,H=[];for(E=0;h>E&&G<F.length;)H[G]=F[G],E+=F[G++];for(0===G&&(G=1);E>h&&G>1;){H=[],G--;for(var I=0;I<F.length;I++)F[I]>(H[I%G]||0)&&(H[I%G]=F[I]);E=H.reduce(function(a,b,c,d){return a+b})}for(var J=[],K=0,L=0;G>K;K++)J[K]=L,L+=H[K];A.attr("transform",function(a,b){return"translate("+J[b%G]+","+(5+Math.floor(b/G)*z)+")"}),l?x.attr("transform","translate("+(d-c.right-E)+","+c.top+")"):x.attr("transform","translate(0,"+c.top+")"),e=c.top+c.bottom+Math.ceil(F.length/G)*z}else{var M,N=5,O=5,P=0;A.attr("transform",function(a,b){var e=d3.select(this).select("text").node().getComputedTextLength()+k;return M=O,d<c.left+c.right+M+e&&(O=M=5,N+=z),O+=e,O>P&&(P=O),M+P>E&&(E=M+P),"translate("+M+","+N+")"}),x.attr("transform","translate("+(d-c.right-P)+","+c.top+")"),e=c.top+c.bottom+N+15}if("furious"==q){y.attr("width",function(a,b){return D[0][b].getComputedTextLength()+27}).attr("height",18).attr("y",-9).attr("x",-15),w.insert("rect",":first-child").attr("class","nv-legend-bg").attr("fill","#eee").attr("opacity",0);var Q=x.select(".nv-legend-bg");Q.transition().duration(300).attr("x",-z).attr("width",E+z-12).attr("height",e+10).attr("y",-c.top-10).attr("opacity",o?1:0)}y.style("fill",t).style("fill-opacity",u).style("stroke",t)}),b}var c={top:5,right:0,bottom:5,left:0},d=400,e=20,f=function(a){return a.key},g=function(a){return a},h=a.utils.getColor(),i=20,j=!0,k=32,l=!0,m=!0,n=!1,o=!1,p=d3.dispatch("legendClick","legendDblclick","legendMouseover","legendMouseout","stateChange"),q="classic";return b.dispatch=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},key:{get:function(){return f},set:function(a){f=a}},keyFormatter:{get:function(){return g},set:function(a){g=a}},align:{get:function(){return j},set:function(a){j=a}},maxKeyLength:{get:function(){return i},set:function(a){i=a}},rightAlign:{get:function(){return l},set:function(a){l=a}},padding:{get:function(){return k},set:function(a){k=a}},updateState:{get:function(){return m},set:function(a){m=a}},radioButtonMode:{get:function(){return n},set:function(a){n=a}},expanded:{get:function(){return o},set:function(a){o=a}},vers:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.line=function(){"use strict";function b(r){return v.reset(),v.models(e),r.each(function(b){i=d3.select(this);var r=a.utils.availableWidth(g,i,f),s=a.utils.availableHeight(h,i,f);a.utils.initSVG(i),c=e.xScale(),d=e.yScale(),t=t||c,u=u||d;var w=i.selectAll("g.nv-wrap.nv-line").data([b]),x=w.enter().append("g").attr("class","nvd3 nv-wrap nv-line"),y=x.append("defs"),z=x.append("g"),A=w.select("g");z.append("g").attr("class","nv-groups"),z.append("g").attr("class","nv-scatterWrap"),w.attr("transform","translate("+f.left+","+f.top+")"),e.width(r).height(s);var B=w.select(".nv-scatterWrap");B.call(e),y.append("clipPath").attr("id","nv-edge-clip-"+e.id()).append("rect"),w.select("#nv-edge-clip-"+e.id()+" rect").attr("width",r).attr("height",s>0?s:0),A.attr("clip-path",p?"url(#nv-edge-clip-"+e.id()+")":""),B.attr("clip-path",p?"url(#nv-edge-clip-"+e.id()+")":"");var C=w.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a){return a.key});C.enter().append("g").style("stroke-opacity",1e-6).style("stroke-width",function(a){return a.strokeWidth||j}).style("fill-opacity",1e-6),C.exit().remove(),C.attr("class",function(a,b){return(a.classed||"")+" nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}).style("fill",function(a,b){return k(a,b)}).style("stroke",function(a,b){return k(a,b)}),C.watchTransition(v,"line: groups").style("stroke-opacity",1).style("fill-opacity",function(a){return a.fillOpacity||.5});var D=C.selectAll("path.nv-area").data(function(a){return o(a)?[a]:[]});D.enter().append("path").attr("class","nv-area").attr("d",function(b){return d3.svg.area().interpolate(q).defined(n).x(function(b,c){return a.utils.NaNtoZero(t(l(b,c)))}).y0(function(b,c){return a.utils.NaNtoZero(u(m(b,c)))}).y1(function(a,b){return u(d.domain()[0]<=0?d.domain()[1]>=0?0:d.domain()[1]:d.domain()[0])}).apply(this,[b.values])}),C.exit().selectAll("path.nv-area").remove(),D.watchTransition(v,"line: areaPaths").attr("d",function(b){return d3.svg.area().interpolate(q).defined(n).x(function(b,d){return a.utils.NaNtoZero(c(l(b,d)))}).y0(function(b,c){return a.utils.NaNtoZero(d(m(b,c)))}).y1(function(a,b){return d(d.domain()[0]<=0?d.domain()[1]>=0?0:d.domain()[1]:d.domain()[0])}).apply(this,[b.values])});var E=C.selectAll("path.nv-line").data(function(a){return[a.values]});E.enter().append("path").attr("class","nv-line").attr("d",d3.svg.line().interpolate(q).defined(n).x(function(b,c){return a.utils.NaNtoZero(t(l(b,c)))}).y(function(b,c){return a.utils.NaNtoZero(u(m(b,c)))})),E.watchTransition(v,"line: linePaths").attr("d",d3.svg.line().interpolate(q).defined(n).x(function(b,d){return a.utils.NaNtoZero(c(l(b,d)))}).y(function(b,c){return a.utils.NaNtoZero(d(m(b,c)))})),t=c.copy(),u=d.copy()}),v.renderEnd("line immediate"),b}var c,d,e=a.models.scatter(),f={top:0,right:0,bottom:0,left:0},g=960,h=500,i=null,j=1.5,k=a.utils.defaultColor(),l=function(a){return a.x},m=function(a){return a.y},n=function(a,b){return!isNaN(m(a,b))&&null!==m(a,b)},o=function(a){return a.area},p=!1,q="linear",r=250,s=d3.dispatch("elementClick","elementMouseover","elementMouseout","renderEnd");e.pointSize(16).pointDomain([16,256]);var t,u,v=a.utils.renderWatch(s,r);return b.dispatch=s,b.scatter=e,e.dispatch.on("elementClick",function(){s.elementClick.apply(this,arguments)}),e.dispatch.on("elementMouseover",function(){s.elementMouseover.apply(this,arguments)}),e.dispatch.on("elementMouseout",function(){s.elementMouseout.apply(this,arguments)}),b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},defined:{get:function(){return n},set:function(a){n=a}},interpolate:{get:function(){return q},set:function(a){q=a}},clipEdge:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return f},set:function(a){f.top=void 0!==a.top?a.top:f.top,f.right=void 0!==a.right?a.right:f.right,f.bottom=void 0!==a.bottom?a.bottom:f.bottom,f.left=void 0!==a.left?a.left:f.left}},duration:{get:function(){return r},set:function(a){r=a,v.reset(r),e.duration(r)}},isArea:{get:function(){return o},set:function(a){o=d3.functor(a)}},x:{get:function(){return l},set:function(a){l=a,e.x(a)}},y:{get:function(){return m},set:function(a){m=a,e.y(a)}},color:{get:function(){return k},set:function(b){k=a.utils.getColor(b),e.color(k)}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.lineChart=function(){"use strict";function b(j){return B.reset(),B.models(e),r&&B.models(f),s&&B.models(g),j.each(function(j){function y(){r&&L.select(".nv-focus .nv-x.nv-axis").transition().duration(A).call(f)}function B(){s&&L.select(".nv-focus .nv-y.nv-axis").transition().duration(A).call(g)}function E(a){var b=L.select(".nv-focus .nv-linesWrap").datum(j.filter(function(a){return!a.disabled}).map(function(b,c){return{key:b.key,area:b.area,classed:b.classed,values:b.values.filter(function(b,c){return e.x()(b,c)>=a[0]&&e.x()(b,c)<=a[1]}),disableTooltip:b.disableTooltip}}));b.transition().duration(A).call(e),y(),B()}var F=d3.select(this);a.utils.initSVG(F);var G=a.utils.availableWidth(n,F,l),H=a.utils.availableHeight(o,F,l)-(v?k.height():0);if(b.update=function(){0===A?F.call(b):F.transition().duration(A).call(b)},b.container=this,w.setter(D(j),b.update).getter(C(j)).update(),w.disabled=j.map(function(a){return!!a.disabled}),!x){var I;x={};for(I in w)w[I]instanceof Array?x[I]=w[I].slice(0):x[I]=w[I]}if(!(j&&j.length&&j.filter(function(a){return a.values.length}).length))return a.utils.noData(b,F),b;F.selectAll(".nv-noData").remove(),k.dispatch.on("onBrush",function(a){E(a)}),c=e.xScale(),d=e.yScale();var J=F.selectAll("g.nv-wrap.nv-lineChart").data([j]),K=J.enter().append("g").attr("class","nvd3 nv-wrap nv-lineChart").append("g"),L=J.select("g");K.append("g").attr("class","nv-legendWrap");var M=K.append("g").attr("class","nv-focus");M.append("g").attr("class","nv-background").append("rect"),M.append("g").attr("class","nv-x nv-axis"),M.append("g").attr("class","nv-y nv-axis"),M.append("g").attr("class","nv-linesWrap"),M.append("g").attr("class","nv-interactive");K.append("g").attr("class","nv-focusWrap");p?(h.width(G),L.select(".nv-legendWrap").datum(j).call(h),"bottom"===q?J.select(".nv-legendWrap").attr("transform","translate(0,"+H+")"):"top"===q&&(h.height()>l.top&&(l.top=h.height(),H=a.utils.availableHeight(o,F,l)-(v?k.height():0)),J.select(".nv-legendWrap").attr("transform","translate(0,"+-l.top+")"))):L.select(".nv-legendWrap").selectAll("*").remove(),J.attr("transform","translate("+l.left+","+l.top+")"),t&&L.select(".nv-y.nv-axis").attr("transform","translate("+G+",0)"),u&&(i.width(G).height(H).margin({left:l.left,top:l.top}).svgContainer(F).xScale(c),J.select(".nv-interactive").call(i)),L.select(".nv-focus .nv-background rect").attr("width",G).attr("height",H),e.width(G).height(H).color(j.map(function(a,b){return a.color||m(a,b)}).filter(function(a,b){return!j[b].disabled}));var N=L.select(".nv-linesWrap").datum(j.filter(function(a){return!a.disabled}));if(r&&f.scale(c)._ticks(a.utils.calcTicksX(G/100,j)).tickSize(-H,0),s&&g.scale(d)._ticks(a.utils.calcTicksY(H/36,j)).tickSize(-G,0),L.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+H+")"),v){k.width(G),L.select(".nv-focusWrap").attr("transform","translate(0,"+(H+l.bottom+k.margin().top)+")").datum(j.filter(function(a){return!a.disabled})).call(k);var O=k.brush.empty()?k.xDomain():k.brush.extent();null!==O&&E(O)}else N.call(e),y(),B();h.dispatch.on("stateChange",function(a){for(var c in a)w[c]=a[c];z.stateChange(w),b.update()}),i.dispatch.on("elementMousemove",function(d){e.clearHighlights();var f,h,l,n=[];if(j.filter(function(a,b){return a.seriesIndex=b,!a.disabled&&!a.disableTooltip}).forEach(function(g,i){var j=v?k.brush.empty()?k.xScale().domain():k.brush.extent():c.domain(),o=g.values.filter(function(a,b){return e.x()(a,b)>=j[0]&&e.x()(a,b)<=j[1]});h=a.interactiveBisect(o,d.pointXValue,e.x());var p=o[h],q=b.y()(p,h);null!==q&&e.highlightPoint(g.seriesIndex,h,!0),void 0!==p&&(void 0===f&&(f=p),void 0===l&&(l=b.xScale()(b.x()(p,h))),n.push({key:g.key,value:q,color:m(g,g.seriesIndex),data:p}))}),n.length>2){var o=b.yScale().invert(d.mouseY),p=Math.abs(b.yScale().domain()[0]-b.yScale().domain()[1]),q=.03*p,r=a.nearestValueIndex(n.map(function(a){return a.value}),o,q);null!==r&&(n[r].highlight=!0)}var s=function(a,b){return null==a?"N/A":g.tickFormat()(a)};i.tooltip.valueFormatter(i.tooltip.valueFormatter()||s).data({value:b.x()(f,h),index:h,series:n})(),i.renderGuideLine(l)}),i.dispatch.on("elementClick",function(c){var d,f=[];j.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(e){var g=a.interactiveBisect(e.values,c.pointXValue,b.x()),h=e.values[g];if("undefined"!=typeof h){"undefined"==typeof d&&(d=b.xScale()(b.x()(h,g)));var i=b.yScale()(b.y()(h,g));f.push({point:h,pointIndex:g,pos:[d,i],seriesIndex:e.seriesIndex,series:e})}}),e.dispatch.elementClick(f)}),i.dispatch.on("elementMouseout",function(a){e.clearHighlights()}),z.on("changeState",function(a){"undefined"!=typeof a.disabled&&j.length===a.disabled.length&&(j.forEach(function(b,c){b.disabled=a.disabled[c]}),w.disabled=a.disabled),b.update()})}),B.renderEnd("lineChart immediate"),b}var c,d,e=a.models.line(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend(),i=a.interactiveGuideline(),j=a.models.tooltip(),k=a.models.focus(a.models.line()),l={top:30,right:20,bottom:50,left:60},m=a.utils.defaultColor(),n=null,o=null,p=!0,q="top",r=!0,s=!0,t=!1,u=!1,v=!1,w=a.utils.state(),x=null,y=null,z=d3.dispatch("tooltipShow","tooltipHide","stateChange","changeState","renderEnd"),A=250;f.orient("bottom").tickPadding(7),g.orient(t?"right":"left"),e.clipEdge(!0).duration(0),j.valueFormatter(function(a,b){return g.tickFormat()(a,b)}).headerFormatter(function(a,b){return f.tickFormat()(a,b)}),i.tooltip.valueFormatter(function(a,b){return g.tickFormat()(a,b)}).headerFormatter(function(a,b){return f.tickFormat()(a,b)});var B=a.utils.renderWatch(z,A),C=function(a){return function(){return{active:a.map(function(a){return!a.disabled})}}},D=function(a){return function(b){void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};return e.dispatch.on("elementMouseover.tooltip",function(a){a.series.disableTooltip||j.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){j.hidden(!0)}),b.dispatch=z,b.lines=e,b.legend=h,b.focus=k,b.xAxis=f,b.x2Axis=k.xAxis,b.yAxis=g,b.y2Axis=k.yAxis,b.interactiveLayer=i,b.tooltip=j,b.state=w,b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return n},set:function(a){n=a}},height:{get:function(){return o},set:function(a){o=a}},showLegend:{get:function(){return p},set:function(a){p=a}},legendPosition:{get:function(){return q},set:function(a){q=a}},showXAxis:{get:function(){return r},set:function(a){r=a}},showYAxis:{get:function(){return s},set:function(a){s=a}},defaultState:{get:function(){return x},set:function(a){x=a}},noData:{get:function(){return y},set:function(a){y=a}},focusEnable:{get:function(){return v},set:function(a){v=a}},focusHeight:{get:function(){return k.height()},set:function(a){k.height(a)}},focusShowAxisX:{get:function(){return k.showXAxis()},set:function(a){k.showXAxis(a)}},focusShowAxisY:{get:function(){return k.showYAxis()},set:function(a){k.showYAxis(a)}},brushExtent:{get:function(){return k.brushExtent()},set:function(a){k.brushExtent(a)}},focusMargin:{get:function(){return k.margin},set:function(a){k.margin.top=void 0!==a.top?a.top:k.margin.top,k.margin.right=void 0!==a.right?a.right:k.margin.right,k.margin.bottom=void 0!==a.bottom?a.bottom:k.margin.bottom,k.margin.left=void 0!==a.left?a.left:k.margin.left}},margin:{get:function(){return l},set:function(a){l.top=void 0!==a.top?a.top:l.top,l.right=void 0!==a.right?a.right:l.right,l.bottom=void 0!==a.bottom?a.bottom:l.bottom,l.left=void 0!==a.left?a.left:l.left}},duration:{get:function(){return A},set:function(a){A=a,B.reset(A),e.duration(A),k.duration(A),f.duration(A),g.duration(A)}},color:{get:function(){return m},set:function(b){m=a.utils.getColor(b),h.color(m),e.color(m),k.color(m)}},interpolate:{get:function(){return e.interpolate()},set:function(a){e.interpolate(a),k.interpolate(a)}},xTickFormat:{get:function(){return f.tickFormat()},set:function(a){f.tickFormat(a),k.xTickFormat(a)}},yTickFormat:{get:function(){return g.tickFormat()},set:function(a){g.tickFormat(a),k.yTickFormat(a)}},x:{get:function(){return e.x()},set:function(a){e.x(a),k.x(a)}},y:{get:function(){return e.y()},set:function(a){e.y(a),k.y(a)}},rightAlignYAxis:{get:function(){return t},set:function(a){t=a,g.orient(t?"right":"left")}},useInteractiveGuideline:{get:function(){return u},set:function(a){u=a,u&&(e.interactive(!1),e.useVoronoi(!1))}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.lineWithFocusChart=function(){return a.models.lineChart().margin({bottom:30}).focusEnable(!0)},a.models.linePlusBarChart=function(){"use strict";function b(v){return v.each(function(v){function J(a){var b=+("e"==a),c=b?1:-1,d=Z/3;return"M"+.5*c+","+d+"A6,6 0 0 "+b+" "+6.5*c+","+(d+6)+"V"+(2*d-6)+"A6,6 0 0 "+b+" "+.5*c+","+2*d+"ZM"+2.5*c+","+(d+8)+"V"+(2*d-8)+"M"+4.5*c+","+(d+8)+"V"+(2*d-8)}function R(){u.empty()||u.extent(I),ma.data([u.empty()?e.domain():I]).each(function(a,b){var c=e(a[0])-e.range()[0],d=e.range()[1]-e(a[1]);d3.select(this).select(".left").attr("width",0>c?0:c),d3.select(this).select(".right").attr("x",e(a[1])).attr("width",0>d?0:d)})}function S(){I=u.empty()?null:u.extent(),c=u.empty()?e.domain():u.extent(),K.brush({extent:c,brush:u}),R(),l.width(X).height(Y).color(v.map(function(a,b){return a.color||C(a,b)}).filter(function(a,b){return!v[b].disabled&&v[b].bar})),j.width(X).height(Y).color(v.map(function(a,b){return a.color||C(a,b)}).filter(function(a,b){return!v[b].disabled&&!v[b].bar}));var b=fa.select(".nv-focus .nv-barsWrap").datum(_.length?_.map(function(a,b){return{key:a.key,values:a.values.filter(function(a,b){return l.x()(a,b)>=c[0]&&l.x()(a,b)<=c[1]})}}):[{values:[]}]),h=fa.select(".nv-focus .nv-linesWrap").datum(V(aa)?[{values:[]}]:aa.filter(function(a){return!a.disabled}).map(function(a,b){return{area:a.area,fillOpacity:a.fillOpacity,strokeWidth:a.strokeWidth,key:a.key,values:a.values.filter(function(a,b){return j.x()(a,b)>=c[0]&&j.x()(a,b)<=c[1]})}}));d=_.length&&!Q?l.xScale():j.xScale(),n.scale(d)._ticks(a.utils.calcTicksX(X/100,v)).tickSize(-Y,0),n.domain([Math.ceil(c[0]),Math.floor(c[1])]),fa.select(".nv-x.nv-axis").transition().duration(L).call(n),b.transition().duration(L).call(l),h.transition().duration(L).call(j),fa.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+f.range()[0]+")"),p.scale(f)._ticks(a.utils.calcTicksY(Y/36,v)).tickSize(-X,0),q.scale(g)._ticks(a.utils.calcTicksY(Y/36,v)),Q?q.tickSize(aa.length?0:-X,0):q.tickSize(_.length?0:-X,0);var i=_.length?1:0,k=aa.length&&!V(aa)?1:0,m=Q?k:i,o=Q?i:k;fa.select(".nv-focus .nv-y1.nv-axis").style("opacity",m),fa.select(".nv-focus .nv-y2.nv-axis").style("opacity",o).attr("transform","translate("+d.range()[1]+",0)"),fa.select(".nv-focus .nv-y1.nv-axis").transition().duration(L).call(p),fa.select(".nv-focus .nv-y2.nv-axis").transition().duration(L).call(q)}var W=d3.select(this);a.utils.initSVG(W);var X=a.utils.availableWidth(y,W,w),Y=a.utils.availableHeight(z,W,w)-(E?H:0),Z=H-x.top-x.bottom;if(b.update=function(){W.transition().duration(L).call(b)},b.container=this,M.setter(U(v),b.update).getter(T(v)).update(),M.disabled=v.map(function(a){return!!a.disabled}),!N){var $;N={};for($ in M)M[$]instanceof Array?N[$]=M[$].slice(0):N[$]=M[$]}if(!(v&&v.length&&v.filter(function(a){return a.values.length}).length))return a.utils.noData(b,W),b;W.selectAll(".nv-noData").remove();var _=v.filter(function(a){return!a.disabled&&a.bar}),aa=v.filter(function(a){return!a.bar});d=_.length&&!Q?l.xScale():j.xScale(),e=o.scale(),f=Q?j.yScale():l.yScale(),g=Q?l.yScale():j.yScale(),h=Q?k.yScale():m.yScale(),i=Q?m.yScale():k.yScale();var ba=v.filter(function(a){return!a.disabled&&(Q?!a.bar:a.bar)}).map(function(a){return a.values.map(function(a,b){return{x:A(a,b),y:B(a,b)}})}),ca=v.filter(function(a){return!a.disabled&&(Q?a.bar:!a.bar)}).map(function(a){return a.values.map(function(a,b){return{x:A(a,b),y:B(a,b)}})});d.range([0,X]),e.domain(d3.extent(d3.merge(ba.concat(ca)),function(a){return a.x})).range([0,X]);var da=W.selectAll("g.nv-wrap.nv-linePlusBar").data([v]),ea=da.enter().append("g").attr("class","nvd3 nv-wrap nv-linePlusBar").append("g"),fa=da.select("g");ea.append("g").attr("class","nv-legendWrap");var ga=ea.append("g").attr("class","nv-focus");ga.append("g").attr("class","nv-x nv-axis"),ga.append("g").attr("class","nv-y1 nv-axis"),ga.append("g").attr("class","nv-y2 nv-axis"),ga.append("g").attr("class","nv-barsWrap"),ga.append("g").attr("class","nv-linesWrap");var ha=ea.append("g").attr("class","nv-context");if(ha.append("g").attr("class","nv-x nv-axis"),ha.append("g").attr("class","nv-y1 nv-axis"),ha.append("g").attr("class","nv-y2 nv-axis"),ha.append("g").attr("class","nv-barsWrap"),ha.append("g").attr("class","nv-linesWrap"),ha.append("g").attr("class","nv-brushBackground"),ha.append("g").attr("class","nv-x nv-brush"),D){var ia=t.align()?X/2:X,ja=t.align()?ia:0;t.width(ia),fa.select(".nv-legendWrap").datum(v.map(function(a){return a.originalKey=void 0===a.originalKey?a.key:a.originalKey,Q?a.key=a.originalKey+(a.bar?P:O):a.key=a.originalKey+(a.bar?O:P),a})).call(t),t.height()>w.top&&(w.top=t.height(),Y=a.utils.availableHeight(z,W,w)-H),fa.select(".nv-legendWrap").attr("transform","translate("+ja+","+-w.top+")")}else fa.select(".nv-legendWrap").selectAll("*").remove();da.attr("transform","translate("+w.left+","+w.top+")"),fa.select(".nv-context").style("display",E?"initial":"none"),m.width(X).height(Z).color(v.map(function(a,b){return a.color||C(a,b)}).filter(function(a,b){return!v[b].disabled&&v[b].bar})),k.width(X).height(Z).color(v.map(function(a,b){return a.color||C(a,b)}).filter(function(a,b){return!v[b].disabled&&!v[b].bar}));var ka=fa.select(".nv-context .nv-barsWrap").datum(_.length?_:[{values:[]}]),la=fa.select(".nv-context .nv-linesWrap").datum(V(aa)?[{values:[]}]:aa.filter(function(a){return!a.disabled}));fa.select(".nv-context").attr("transform","translate(0,"+(Y+w.bottom+x.top)+")"),ka.transition().call(m),la.transition().call(k),G&&(o._ticks(a.utils.calcTicksX(X/100,v)).tickSize(-Z,0),fa.select(".nv-context .nv-x.nv-axis").attr("transform","translate(0,"+h.range()[0]+")"),fa.select(".nv-context .nv-x.nv-axis").transition().call(o)),F&&(r.scale(h)._ticks(Z/36).tickSize(-X,0),s.scale(i)._ticks(Z/36).tickSize(_.length?0:-X,0),fa.select(".nv-context .nv-y3.nv-axis").style("opacity",_.length?1:0).attr("transform","translate(0,"+e.range()[0]+")"),fa.select(".nv-context .nv-y2.nv-axis").style("opacity",aa.length?1:0).attr("transform","translate("+e.range()[1]+",0)"),fa.select(".nv-context .nv-y1.nv-axis").transition().call(r),fa.select(".nv-context .nv-y2.nv-axis").transition().call(s)),u.x(e).on("brush",S),I&&u.extent(I);var ma=fa.select(".nv-brushBackground").selectAll("g").data([I||u.extent()]),na=ma.enter().append("g");na.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",Z),na.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",Z);var oa=fa.select(".nv-x.nv-brush").call(u);oa.selectAll("rect").attr("height",Z),oa.selectAll(".resize").append("path").attr("d",J),t.dispatch.on("stateChange",function(a){for(var c in a)M[c]=a[c];K.stateChange(M),b.update()}),K.on("changeState",function(a){"undefined"!=typeof a.disabled&&(v.forEach(function(b,c){b.disabled=a.disabled[c]}),M.disabled=a.disabled),b.update()}),S()}),b}var c,d,e,f,g,h,i,j=a.models.line(),k=a.models.line(),l=a.models.historicalBar(),m=a.models.historicalBar(),n=a.models.axis(),o=a.models.axis(),p=a.models.axis(),q=a.models.axis(),r=a.models.axis(),s=a.models.axis(),t=a.models.legend(),u=d3.svg.brush(),v=a.models.tooltip(),w={top:30,right:30,bottom:30,left:60},x={top:0,right:30,bottom:20,left:60},y=null,z=null,A=function(a){return a.x},B=function(a){return a.y},C=a.utils.defaultColor(),D=!0,E=!0,F=!1,G=!0,H=50,I=null,J=null,K=d3.dispatch("brush","stateChange","changeState"),L=0,M=a.utils.state(),N=null,O=" (left axis)",P=" (right axis)",Q=!1;j.clipEdge(!0),k.interactive(!1),k.pointActive(function(a){return!1}),n.orient("bottom").tickPadding(5),p.orient("left"),q.orient("right"),o.orient("bottom").tickPadding(5),r.orient("left"),s.orient("right"),v.headerEnabled(!0).headerFormatter(function(a,b){return n.tickFormat()(a,b)});var R=function(){return Q?{main:q,focus:s}:{main:p,focus:r}},S=function(){return Q?{main:p,focus:r}:{main:q,focus:s}},T=function(a){return function(){return{active:a.map(function(a){return!a.disabled})}}},U=function(a){return function(b){void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}},V=function(a){return a.every(function(a){return a.disabled})};return j.dispatch.on("elementMouseover.tooltip",function(a){v.duration(100).valueFormatter(function(a,b){return S().main.tickFormat()(a,b)}).data(a).hidden(!1)}),j.dispatch.on("elementMouseout.tooltip",function(a){v.hidden(!0)}),l.dispatch.on("elementMouseover.tooltip",function(a){a.value=b.x()(a.data),a.series={value:b.y()(a.data),color:a.color},v.duration(0).valueFormatter(function(a,b){return R().main.tickFormat()(a,b)}).data(a).hidden(!1)}),l.dispatch.on("elementMouseout.tooltip",function(a){v.hidden(!0)}),l.dispatch.on("elementMousemove.tooltip",function(a){v()}),b.dispatch=K,b.legend=t,b.lines=j,b.lines2=k,b.bars=l,b.bars2=m,b.xAxis=n,b.x2Axis=o,b.y1Axis=p,b.y2Axis=q,b.y3Axis=r,b.y4Axis=s,b.tooltip=v,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return y},set:function(a){y=a}},height:{get:function(){return z},set:function(a){z=a}},showLegend:{get:function(){return D},set:function(a){D=a}},brushExtent:{get:function(){return I},set:function(a){I=a}},noData:{get:function(){return J},set:function(a){J=a}},focusEnable:{get:function(){return E},set:function(a){E=a}},focusHeight:{get:function(){return H},set:function(a){H=a}},focusShowAxisX:{get:function(){return G},set:function(a){G=a}},focusShowAxisY:{get:function(){return F},set:function(a){F=a}},legendLeftAxisHint:{get:function(){return O},set:function(a){O=a}},legendRightAxisHint:{get:function(){return P},set:function(a){P=a}},margin:{get:function(){return w},set:function(a){w.top=void 0!==a.top?a.top:w.top,w.right=void 0!==a.right?a.right:w.right,w.bottom=void 0!==a.bottom?a.bottom:w.bottom,w.left=void 0!==a.left?a.left:w.left}},focusMargin:{get:function(){return x},set:function(a){x.top=void 0!==a.top?a.top:x.top,x.right=void 0!==a.right?a.right:x.right,x.bottom=void 0!==a.bottom?a.bottom:x.bottom,x.left=void 0!==a.left?a.left:x.left}},duration:{get:function(){return L},set:function(a){L=a}},color:{get:function(){return C},set:function(b){C=a.utils.getColor(b),t.color(C)}},x:{get:function(){return A},set:function(a){A=a,j.x(a),k.x(a),l.x(a),m.x(a)}},y:{get:function(){return B},set:function(a){B=a,j.y(a),k.y(a),l.y(a),m.y(a)}},switchYAxisOrder:{get:function(){return Q},set:function(a){if(Q!==a){var b=p;p=q,q=b;var c=r;r=s,s=c}Q=a,p.orient("left"),q.orient("right"),r.orient("left"),s.orient("right")}}}),a.utils.inheritOptions(b,j),a.utils.initOptions(b),b},a.models.multiBar=function(){"use strict";function b(F){return D.reset(),F.each(function(b){var F=k-j.left-j.right,G=l-j.top-j.bottom;p=d3.select(this),a.utils.initSVG(p);var H=0;if(x&&b.length&&(x=[{values:b[0].values.map(function(a){return{x:a.x,y:0,series:a.series,size:.01}})}]),u){var I=d3.layout.stack().offset(v).values(function(a){return a.values}).y(r)(!b.length&&x?x:b);I.forEach(function(a,c){a.nonStackable?(b[c].nonStackableSeries=H++,I[c]=b[c]):c>0&&I[c-1].nonStackable&&I[c].values.map(function(a,b){a.y0-=I[c-1].values[b].y,a.y1=a.y0+a.y})}),b=I}b.forEach(function(a,b){a.values.forEach(function(c){c.series=b,c.key=a.key})}),u&&b.length>0&&b[0].values.map(function(a,c){var d=0,e=0;b.map(function(a,f){if(!b[f].nonStackable){var g=a.values[c];g.size=Math.abs(g.y),g.y<0?(g.y1=e,e-=g.size):(g.y1=g.size+d,d+=g.size)}})});var J=d&&e?[]:b.map(function(a,b){return a.values.map(function(a,c){return{x:q(a,c),y:r(a,c),y0:a.y0,y1:a.y1,idx:b}})});m.domain(d||d3.merge(J).map(function(a){return a.x})).rangeBands(f||[0,F],A),n.domain(e||d3.extent(d3.merge(J).map(function(a){var c=a.y;return u&&!b[a.idx].nonStackable&&(c=a.y>0?a.y1:a.y1+a.y),c}).concat(s))).range(g||[G,0]),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]-.01*m.domain()[0],m.domain()[1]+.01*m.domain()[1]]):m.domain([-1,1])),n.domain()[0]===n.domain()[1]&&(n.domain()[0]?n.domain([n.domain()[0]+.01*n.domain()[0],n.domain()[1]-.01*n.domain()[1]]):n.domain([-1,1])),h=h||m,i=i||n;var K=p.selectAll("g.nv-wrap.nv-multibar").data([b]),L=K.enter().append("g").attr("class","nvd3 nv-wrap nv-multibar"),M=L.append("defs"),N=L.append("g"),O=K.select("g");N.append("g").attr("class","nv-groups"),K.attr("transform","translate("+j.left+","+j.top+")"),M.append("clipPath").attr("id","nv-edge-clip-"+o).append("rect"),K.select("#nv-edge-clip-"+o+" rect").attr("width",F).attr("height",G),O.attr("clip-path",t?"url(#nv-edge-clip-"+o+")":"");var P=K.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a,b){return b});P.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6);var Q=D.transition(P.exit().selectAll("rect.nv-bar"),"multibarExit",Math.min(100,z)).attr("y",function(a,c,d){var e=i(0)||0;return u&&b[a.series]&&!b[a.series].nonStackable&&(e=i(a.y0)),e}).attr("height",0).remove();Q.delay&&Q.delay(function(a,b){var c=b*(z/(E+1))-b;return c}),P.attr("class",function(a,b){return"nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}).style("fill",function(a,b){return w(a,b)}).style("stroke",function(a,b){return w(a,b)}),P.style("stroke-opacity",1).style("fill-opacity",B);var R=P.selectAll("rect.nv-bar").data(function(a){return x&&!b.length?x.values:a.values});R.exit().remove();R.enter().append("rect").attr("class",function(a,b){return r(a,b)<0?"nv-bar negative":"nv-bar positive"}).attr("x",function(a,c,d){return u&&!b[d].nonStackable?0:d*m.rangeBand()/b.length}).attr("y",function(a,c,d){return i(u&&!b[d].nonStackable?a.y0:0)||0}).attr("height",0).attr("width",function(a,c,d){return m.rangeBand()/(u&&!b[d].nonStackable?1:b.length)}).attr("transform",function(a,b){return"translate("+m(q(a,b))+",0)"});R.style("fill",function(a,b,c){return w(a,c,b)}).style("stroke",function(a,b,c){return w(a,c,b)}).on("mouseover",function(a,b){d3.select(this).classed("hover",!0),C.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),C.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")})}).on("mousemove",function(a,b){C.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){var c=this;C.elementClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}).on("dblclick",function(a,b){C.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation()}),R.attr("class",function(a,b){return r(a,b)<0?"nv-bar negative":"nv-bar positive"}).attr("transform",function(a,b){return"translate("+m(q(a,b))+",0)"}),y&&(c||(c=b.map(function(){return!0})),R.style("fill",function(a,b,d){return d3.rgb(y(a,b)).darker(c.map(function(a,b){return b}).filter(function(a,b){return!c[b]})[d]).toString()}).style("stroke",function(a,b,d){return d3.rgb(y(a,b)).darker(c.map(function(a,b){return b}).filter(function(a,b){return!c[b]})[d]).toString()}));var S=R.watchTransition(D,"multibar",Math.min(250,z)).delay(function(a,c){return c*z/b[0].values.length});u?S.attr("y",function(a,c,d){var e=0;return e=b[d].nonStackable?r(a,c)<0?n(0):n(0)-n(r(a,c))<-1?n(0)-1:n(r(a,c))||0:n(a.y1)}).attr("height",function(a,c,d){return b[d].nonStackable?Math.max(Math.abs(n(r(a,c))-n(0)),0)||0:Math.max(Math.abs(n(a.y+a.y0)-n(a.y0)),0)}).attr("x",function(a,c,d){var e=0;return b[d].nonStackable&&(e=a.series*m.rangeBand()/b.length,b.length!==H&&(e=b[d].nonStackableSeries*m.rangeBand()/(2*H))),e}).attr("width",function(a,c,d){if(b[d].nonStackable){var e=m.rangeBand()/H;return b.length!==H&&(e=m.rangeBand()/(2*H)),e}return m.rangeBand()}):S.attr("x",function(a,c){return a.series*m.rangeBand()/b.length}).attr("width",m.rangeBand()/b.length).attr("y",function(a,b){return r(a,b)<0?n(0):n(0)-n(r(a,b))<1?n(0)-1:n(r(a,b))||0}).attr("height",function(a,b){return Math.max(Math.abs(n(r(a,b))-n(0)),1)||0}),h=m.copy(),i=n.copy(),b[0]&&b[0].values&&(E=b[0].values.length)}),D.renderEnd("multibar immediate"),b}var c,d,e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=960,l=500,m=d3.scale.ordinal(),n=d3.scale.linear(),o=Math.floor(1e4*Math.random()),p=null,q=function(a){return a.x},r=function(a){return a.y},s=[0],t=!0,u=!1,v="zero",w=a.utils.defaultColor(),x=!1,y=null,z=500,A=.1,B=.75,C=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),D=a.utils.renderWatch(C,z),E=0;return b.dispatch=C,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},x:{get:function(){return q},set:function(a){q=a}},y:{get:function(){return r},set:function(a){r=a}},xScale:{get:function(){return m},set:function(a){m=a}},yScale:{get:function(){return n},set:function(a){n=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},forceY:{get:function(){return s},set:function(a){s=a}},stacked:{get:function(){return u},set:function(a){u=a}},stackOffset:{get:function(){return v},set:function(a){v=a}},clipEdge:{get:function(){return t},set:function(a){t=a}},disabled:{get:function(){return c},set:function(a){c=a}},id:{get:function(){return o},set:function(a){o=a}},hideable:{get:function(){return x},set:function(a){x=a}},groupSpacing:{get:function(){return A},set:function(a){A=a}},fillOpacity:{get:function(){return B},set:function(a){B=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return z},set:function(a){z=a,D.reset(z)}},color:{get:function(){return w},set:function(b){w=a.utils.getColor(b)}},barColor:{get:function(){return y},set:function(b){y=b?a.utils.getColor(b):null}}}),a.utils.initOptions(b),b},a.models.multiBarChart=function(){"use strict";function b(B){return G.reset(),G.models(e),s&&G.models(f),t&&G.models(g),B.each(function(B){var G=d3.select(this);a.utils.initSVG(G);var K=a.utils.availableWidth(m,G,l),L=a.utils.availableHeight(n,G,l);if(b.update=function(){0===E?G.call(b):G.transition().duration(E).call(b)},b.container=this,z.setter(J(B),b.update).getter(I(B)).update(),z.disabled=B.map(function(a){return!!a.disabled}),!A){var M;A={};for(M in z)z[M]instanceof Array?A[M]=z[M].slice(0):A[M]=z[M]}if(!(B&&B.length&&B.filter(function(a){return a.values.length}).length))return a.utils.noData(b,G),b;G.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale();var N=G.selectAll("g.nv-wrap.nv-multiBarWithLegend").data([B]),O=N.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarWithLegend").append("g"),P=N.select("g");if(O.append("g").attr("class","nv-x nv-axis"),O.append("g").attr("class","nv-y nv-axis"),O.append("g").attr("class","nv-barsWrap"),O.append("g").attr("class","nv-legendWrap"),O.append("g").attr("class","nv-controlsWrap"),O.append("g").attr("class","nv-interactive"),r?(i.width(K-D()),P.select(".nv-legendWrap").datum(B).call(i),i.height()>l.top&&(l.top=i.height(),L=a.utils.availableHeight(n,G,l)),P.select(".nv-legendWrap").attr("transform","translate("+D()+","+-l.top+")")):P.select(".nv-legendWrap").selectAll("*").remove(),p){var Q=[{key:q.grouped||"Grouped",disabled:e.stacked()},{key:q.stacked||"Stacked",disabled:!e.stacked()}];j.width(D()).color(["#444","#444","#444"]),P.select(".nv-controlsWrap").datum(Q).attr("transform","translate(0,"+-l.top+")").call(j)}else P.select(".nv-controlsWrap").selectAll("*").remove();N.attr("transform","translate("+l.left+","+l.top+")"),u&&P.select(".nv-y.nv-axis").attr("transform","translate("+K+",0)"),e.disabled(B.map(function(a){return a.disabled})).width(K).height(L).color(B.map(function(a,b){return a.color||o(a,b)}).filter(function(a,b){return!B[b].disabled}));var R=P.select(".nv-barsWrap").datum(B.filter(function(a){return!a.disabled}));if(R.call(e),s){f.scale(c)._ticks(a.utils.calcTicksX(K/100,B)).tickSize(-L,0),P.select(".nv-x.nv-axis").attr("transform","translate(0,"+d.range()[0]+")"),P.select(".nv-x.nv-axis").call(f);var S=P.select(".nv-x.nv-axis > g").selectAll("g");if(S.selectAll("line, text").style("opacity",1),w){var T=function(a,b){return"translate("+a+","+b+")"},U=5,V=17;S.selectAll("text").attr("transform",function(a,b,c){return T(0,c%2==0?U:V)});var W=d3.selectAll(".nv-x.nv-axis .nv-wrap g g text")[0].length;P.selectAll(".nv-x.nv-axis .nv-axisMaxMin text").attr("transform",function(a,b){return T(0,0===b||W%2!==0?V:U)})}x&&P.selectAll(".tick text").call(a.utils.wrapTicks,b.xAxis.rangeBand()),v&&S.filter(function(a,b){return b%Math.ceil(B[0].values.length/(K/100))!==0}).selectAll("text, line").style("opacity",0),y&&S.selectAll(".tick text").attr("transform","rotate("+y+" 0,0)").style("text-anchor",y>0?"start":"end"),P.select(".nv-x.nv-axis").selectAll("g.nv-axisMaxMin text").style("opacity",1)}t&&(g.scale(d)._ticks(a.utils.calcTicksY(L/36,B)).tickSize(-K,0),P.select(".nv-y.nv-axis").call(g)),F&&(h.width(K).height(L).margin({left:l.left,top:l.top}).svgContainer(G).xScale(c),N.select(".nv-interactive").call(h)),i.dispatch.on("stateChange",function(a){for(var c in a)z[c]=a[c];C.stateChange(z),b.update()}),j.dispatch.on("legendClick",function(a,c){if(a.disabled){switch(Q=Q.map(function(a){return a.disabled=!0,a}),a.disabled=!1,a.key){case"Grouped":case q.grouped:e.stacked(!1);break;case"Stacked":case q.stacked:e.stacked(!0)}z.stacked=e.stacked(),C.stateChange(z),b.update()}}),C.on("changeState",function(a){"undefined"!=typeof a.disabled&&(B.forEach(function(b,c){b.disabled=a.disabled[c]}),z.disabled=a.disabled),"undefined"!=typeof a.stacked&&(e.stacked(a.stacked),z.stacked=a.stacked,H=a.stacked),b.update()}),F?(h.dispatch.on("elementMousemove",function(a){if(void 0!=a.pointXValue){var d,e,f,g,i=[];B.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(h,j){e=c.domain().indexOf(a.pointXValue);var k=h.values[e];void 0!==k&&(g=k.x,void 0===d&&(d=k),void 0===f&&(f=a.mouseX),i.push({key:h.key,value:b.y()(k,e),color:o(h,h.seriesIndex),data:h.values[e]}))}),h.tooltip.data({value:g,index:e,series:i})(),h.renderGuideLine(f)}}),h.dispatch.on("elementMouseout",function(a){h.tooltip.hidden(!0)})):(e.dispatch.on("elementMouseover.tooltip",function(a){a.value=b.x()(a.data),a.series={key:a.data.key,value:b.y()(a.data),color:a.color},k.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){k.hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){k()}))}),G.renderEnd("multibarchart immediate"),b}var c,d,e=a.models.multiBar(),f=a.models.axis(),g=a.models.axis(),h=a.interactiveGuideline(),i=a.models.legend(),j=a.models.legend(),k=a.models.tooltip(),l={top:30,right:20,bottom:50,left:60},m=null,n=null,o=a.utils.defaultColor(),p=!0,q={},r=!0,s=!0,t=!0,u=!1,v=!0,w=!1,x=!1,y=0,z=a.utils.state(),A=null,B=null,C=d3.dispatch("stateChange","changeState","renderEnd"),D=function(){return p?180:0},E=250,F=!1;z.stacked=!1,e.stacked(!1),f.orient("bottom").tickPadding(7).showMaxMin(!1).tickFormat(function(a){return a}),g.orient(u?"right":"left").tickFormat(d3.format(",.1f")),k.duration(0).valueFormatter(function(a,b){return g.tickFormat()(a,b)}).headerFormatter(function(a,b){return f.tickFormat()(a,b)}),j.updateState(!1);var G=a.utils.renderWatch(C),H=!1,I=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),stacked:H}}},J=function(a){return function(b){void 0!==b.stacked&&(H=b.stacked),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};return b.dispatch=C,b.multibar=e,b.legend=i,b.controls=j,b.xAxis=f,b.yAxis=g,b.state=z,b.tooltip=k,b.interactiveLayer=h,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return m},set:function(a){m=a}},height:{get:function(){return n},set:function(a){n=a}},showLegend:{get:function(){return r},set:function(a){r=a}},showControls:{get:function(){return p},set:function(a){p=a}},controlLabels:{get:function(){return q},set:function(a){q=a}},showXAxis:{get:function(){return s},set:function(a){s=a}},showYAxis:{get:function(){return t},set:function(a){t=a}},defaultState:{get:function(){return A},set:function(a){A=a}},noData:{get:function(){return B},set:function(a){B=a}},reduceXTicks:{get:function(){return v},set:function(a){v=a}},rotateLabels:{get:function(){return y},set:function(a){y=a}},staggerLabels:{get:function(){return w},set:function(a){w=a}},wrapLabels:{get:function(){return x},set:function(a){x=!!a}},margin:{get:function(){return l},set:function(a){l.top=void 0!==a.top?a.top:l.top,l.right=void 0!==a.right?a.right:l.right,l.bottom=void 0!==a.bottom?a.bottom:l.bottom,l.left=void 0!==a.left?a.left:l.left}},duration:{get:function(){return E},set:function(a){E=a,e.duration(E),f.duration(E),g.duration(E),G.reset(E)}},color:{get:function(){return o},set:function(b){o=a.utils.getColor(b),i.color(o)}},rightAlignYAxis:{get:function(){return u},set:function(a){u=a,g.orient(u?"right":"left")}},useInteractiveGuideline:{get:function(){return F},set:function(a){F=a}},barColor:{get:function(){return e.barColor},set:function(a){e.barColor(a),i.color(function(a,b){return d3.rgb("#ccc").darker(1.5*b).toString()})}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.multiBarHorizontal=function(){"use strict";function b(m){return F.reset(),m.each(function(b){var m=k-j.left-j.right,D=l-j.top-j.bottom;n=d3.select(this),a.utils.initSVG(n),w&&(b=d3.layout.stack().offset("zero").values(function(a){return a.values}).y(r)(b)),b.forEach(function(a,b){a.values.forEach(function(c){c.series=b,c.key=a.key})}),w&&b[0].values.map(function(a,c){var d=0,e=0;b.map(function(a){var b=a.values[c];b.size=Math.abs(b.y),b.y<0?(b.y1=e-b.size,e-=b.size):(b.y1=d,d+=b.size)})});var G=d&&e?[]:b.map(function(a){return a.values.map(function(a,b){return{x:q(a,b),y:r(a,b),y0:a.y0,y1:a.y1}})});o.domain(d||d3.merge(G).map(function(a){return a.x})).rangeBands(f||[0,D],A),p.domain(e||d3.extent(d3.merge(G).map(function(a){return w?a.y>0?a.y1+a.y:a.y1:a.y}).concat(t))),x&&!w?p.range(g||[p.domain()[0]<0?z:0,m-(p.domain()[1]>0?z:0)]):p.range(g||[0,m]),h=h||o,i=i||d3.scale.linear().domain(p.domain()).range([p(0),p(0)]);var H=d3.select(this).selectAll("g.nv-wrap.nv-multibarHorizontal").data([b]),I=H.enter().append("g").attr("class","nvd3 nv-wrap nv-multibarHorizontal"),J=(I.append("defs"),I.append("g"));H.select("g");J.append("g").attr("class","nv-groups"),H.attr("transform","translate("+j.left+","+j.top+")");var K=H.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a,b){return b});K.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),K.exit().watchTransition(F,"multibarhorizontal: exit groups").style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),K.attr("class",function(a,b){return"nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}).style("fill",function(a,b){return u(a,b)}).style("stroke",function(a,b){return u(a,b)}),K.watchTransition(F,"multibarhorizontal: groups").style("stroke-opacity",1).style("fill-opacity",B);var L=K.selectAll("g.nv-bar").data(function(a){return a.values});L.exit().remove();var M=L.enter().append("g").attr("transform",function(a,c,d){return"translate("+i(w?a.y0:0)+","+(w?0:d*o.rangeBand()/b.length+o(q(a,c)))+")"});M.append("rect").attr("width",0).attr("height",o.rangeBand()/(w?1:b.length)),L.on("mouseover",function(a,b){d3.select(this).classed("hover",!0),E.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),E.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){E.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")})}).on("mousemove",function(a,b){E.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){var c=this;E.elementClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}).on("dblclick",function(a,b){E.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation()}),s(b[0],0)&&(M.append("polyline"),L.select("polyline").attr("fill","none").attr("points",function(a,c){var d=s(a,c),e=.8*o.rangeBand()/(2*(w?1:b.length));d=d.length?d:[-Math.abs(d),Math.abs(d)],d=d.map(function(a){return p(a)-p(0)});var f=[[d[0],-e],[d[0],e],[d[0],0],[d[1],0],[d[1],-e],[d[1],e]];return f.map(function(a){return a.join(",")}).join(" ")}).attr("transform",function(a,c){var d=o.rangeBand()/(2*(w?1:b.length));return"translate("+(r(a,c)<0?0:p(r(a,c))-p(0))+", "+d+")"})),M.append("text"),x&&!w?(L.select("text").attr("text-anchor",function(a,b){return r(a,b)<0?"end":"start"}).attr("y",o.rangeBand()/(2*b.length)).attr("dy",".32em").text(function(a,b){var c=C(r(a,b)),d=s(a,b);return void 0===d?c:d.length?c+"+"+C(Math.abs(d[1]))+"-"+C(Math.abs(d[0])):c+"±"+C(Math.abs(d))}),L.watchTransition(F,"multibarhorizontal: bars").select("text").attr("x",function(a,b){return r(a,b)<0?-4:p(r(a,b))-p(0)+4})):L.selectAll("text").text(""),y&&!w?(M.append("text").classed("nv-bar-label",!0),L.select("text.nv-bar-label").attr("text-anchor",function(a,b){return r(a,b)<0?"start":"end"}).attr("y",o.rangeBand()/(2*b.length)).attr("dy",".32em").text(function(a,b){return q(a,b)}),L.watchTransition(F,"multibarhorizontal: bars").select("text.nv-bar-label").attr("x",function(a,b){return r(a,b)<0?p(0)-p(r(a,b))+4:-4})):L.selectAll("text.nv-bar-label").text(""),L.attr("class",function(a,b){return r(a,b)<0?"nv-bar negative":"nv-bar positive"}),v&&(c||(c=b.map(function(){return!0})),L.style("fill",function(a,b,d){return d3.rgb(v(a,b)).darker(c.map(function(a,b){return b}).filter(function(a,b){return!c[b]})[d]).toString()}).style("stroke",function(a,b,d){return d3.rgb(v(a,b)).darker(c.map(function(a,b){return b}).filter(function(a,b){return!c[b]})[d]).toString()})),w?L.watchTransition(F,"multibarhorizontal: bars").attr("transform",function(a,b){return"translate("+p(a.y1)+","+o(q(a,b))+")"}).select("rect").attr("width",function(a,b){return Math.abs(p(r(a,b)+a.y0)-p(a.y0))||0}).attr("height",o.rangeBand()):L.watchTransition(F,"multibarhorizontal: bars").attr("transform",function(a,c){return"translate("+p(r(a,c)<0?r(a,c):0)+","+(a.series*o.rangeBand()/b.length+o(q(a,c)))+")"}).select("rect").attr("height",o.rangeBand()/b.length).attr("width",function(a,b){return Math.max(Math.abs(p(r(a,b))-p(0)),1)||0}),h=o.copy(),i=p.copy()}),F.renderEnd("multibarHorizontal immediate"),b}var c,d,e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=960,l=500,m=Math.floor(1e4*Math.random()),n=null,o=d3.scale.ordinal(),p=d3.scale.linear(),q=function(a){return a.x},r=function(a){return a.y},s=function(a){return a.yErr},t=[0],u=a.utils.defaultColor(),v=null,w=!1,x=!1,y=!1,z=60,A=.1,B=.75,C=d3.format(",.2f"),D=250,E=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),F=a.utils.renderWatch(E,D);return b.dispatch=E,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},x:{get:function(){return q},set:function(a){q=a}},y:{get:function(){return r},set:function(a){r=a}},yErr:{get:function(){return s},set:function(a){s=a}},xScale:{get:function(){return o},set:function(a){o=a}},yScale:{get:function(){return p},set:function(a){p=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},forceY:{get:function(){return t},set:function(a){t=a}},stacked:{get:function(){return w},set:function(a){w=a}},showValues:{get:function(){return x},set:function(a){x=a}},disabled:{get:function(){return c},set:function(a){c=a}},id:{get:function(){return m},set:function(a){m=a}},valueFormat:{get:function(){return C},set:function(a){C=a}},valuePadding:{get:function(){return z},set:function(a){z=a}},groupSpacing:{get:function(){return A},set:function(a){A=a}},fillOpacity:{get:function(){return B},set:function(a){B=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return D},set:function(a){D=a,F.reset(D)}},color:{get:function(){return u},set:function(b){u=a.utils.getColor(b)}},barColor:{get:function(){return v},set:function(b){v=b?a.utils.getColor(b):null}}}),a.utils.initOptions(b),b},a.models.multiBarHorizontalChart=function(){"use strict";function b(j){return C.reset(),C.models(e),r&&C.models(f),s&&C.models(g),j.each(function(j){var w=d3.select(this);a.utils.initSVG(w);var C=a.utils.availableWidth(l,w,k),D=a.utils.availableHeight(m,w,k);if(b.update=function(){w.transition().duration(z).call(b)},b.container=this,t=e.stacked(),u.setter(B(j),b.update).getter(A(j)).update(),u.disabled=j.map(function(a){return!!a.disabled}),!v){var E;v={};for(E in u)u[E]instanceof Array?v[E]=u[E].slice(0):v[E]=u[E]}if(!(j&&j.length&&j.filter(function(a){return a.values.length}).length))return a.utils.noData(b,w),b;w.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var F=w.selectAll("g.nv-wrap.nv-multiBarHorizontalChart").data([j]),G=F.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarHorizontalChart").append("g"),H=F.select("g");if(G.append("g").attr("class","nv-x nv-axis"),G.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),G.append("g").attr("class","nv-barsWrap"),G.append("g").attr("class","nv-legendWrap"),G.append("g").attr("class","nv-controlsWrap"),q?(h.width(C-y()),H.select(".nv-legendWrap").datum(j).call(h),h.height()>k.top&&(k.top=h.height(),D=a.utils.availableHeight(m,w,k)),H.select(".nv-legendWrap").attr("transform","translate("+y()+","+-k.top+")")):H.select(".nv-legendWrap").selectAll("*").remove(),o){var I=[{key:p.grouped||"Grouped",disabled:e.stacked()},{key:p.stacked||"Stacked",disabled:!e.stacked()}];i.width(y()).color(["#444","#444","#444"]),H.select(".nv-controlsWrap").datum(I).attr("transform","translate(0,"+-k.top+")").call(i)}else H.select(".nv-controlsWrap").selectAll("*").remove();F.attr("transform","translate("+k.left+","+k.top+")"),e.disabled(j.map(function(a){return a.disabled})).width(C).height(D).color(j.map(function(a,b){return a.color||n(a,b)}).filter(function(a,b){return!j[b].disabled}));var J=H.select(".nv-barsWrap").datum(j.filter(function(a){return!a.disabled}));if(J.transition().call(e),r){f.scale(c)._ticks(a.utils.calcTicksY(D/24,j)).tickSize(-C,0),H.select(".nv-x.nv-axis").call(f);var K=H.select(".nv-x.nv-axis").selectAll("g");K.selectAll("line, text")}s&&(g.scale(d)._ticks(a.utils.calcTicksX(C/100,j)).tickSize(-D,0),H.select(".nv-y.nv-axis").attr("transform","translate(0,"+D+")"),H.select(".nv-y.nv-axis").call(g)),H.select(".nv-zeroLine line").attr("x1",d(0)).attr("x2",d(0)).attr("y1",0).attr("y2",-D),h.dispatch.on("stateChange",function(a){for(var c in a)u[c]=a[c];x.stateChange(u),b.update()}),i.dispatch.on("legendClick",function(a,c){if(a.disabled){switch(I=I.map(function(a){return a.disabled=!0,a}),a.disabled=!1,a.key){case"Grouped":case p.grouped:e.stacked(!1);break;case"Stacked":case p.stacked:e.stacked(!0)}u.stacked=e.stacked(),x.stateChange(u),t=e.stacked(),b.update()}}),x.on("changeState",function(a){"undefined"!=typeof a.disabled&&(j.forEach(function(b,c){b.disabled=a.disabled[c]}),u.disabled=a.disabled),"undefined"!=typeof a.stacked&&(e.stacked(a.stacked),u.stacked=a.stacked,t=a.stacked),b.update()})}),C.renderEnd("multibar horizontal chart immediate"),b}var c,d,e=a.models.multiBarHorizontal(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend().height(30),i=a.models.legend().height(30),j=a.models.tooltip(),k={top:30,right:20,bottom:50,left:60},l=null,m=null,n=a.utils.defaultColor(),o=!0,p={},q=!0,r=!0,s=!0,t=!1,u=a.utils.state(),v=null,w=null,x=d3.dispatch("stateChange","changeState","renderEnd"),y=function(){return o?180:0},z=250;u.stacked=!1,e.stacked(t),f.orient("left").tickPadding(5).showMaxMin(!1).tickFormat(function(a){return a}),g.orient("bottom").tickFormat(d3.format(",.1f")),j.duration(0).valueFormatter(function(a,b){return g.tickFormat()(a,b)}).headerFormatter(function(a,b){return f.tickFormat()(a,b)}),i.updateState(!1);var A=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),stacked:t}}},B=function(a){return function(b){void 0!==b.stacked&&(t=b.stacked),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}},C=a.utils.renderWatch(x,z);return e.dispatch.on("elementMouseover.tooltip",function(a){a.value=b.x()(a.data),a.series={key:a.data.key,value:b.y()(a.data),color:a.color},j.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){j.hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){j()}),b.dispatch=x,b.multibar=e,b.legend=h,b.controls=i,b.xAxis=f,b.yAxis=g,b.state=u,b.tooltip=j,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return l},set:function(a){l=a}},height:{get:function(){return m},set:function(a){m=a}},showLegend:{get:function(){return q},set:function(a){q=a}},showControls:{get:function(){return o},set:function(a){o=a}},controlLabels:{get:function(){return p},set:function(a){p=a}},showXAxis:{get:function(){return r},set:function(a){r=a}},showYAxis:{get:function(){return s},set:function(a){s=a}},defaultState:{get:function(){return v},set:function(a){v=a}},noData:{get:function(){return w},set:function(a){w=a}},margin:{get:function(){return k},set:function(a){k.top=void 0!==a.top?a.top:k.top,k.right=void 0!==a.right?a.right:k.right,k.bottom=void 0!==a.bottom?a.bottom:k.bottom,k.left=void 0!==a.left?a.left:k.left}},duration:{get:function(){return z},set:function(a){z=a,C.reset(z),e.duration(z),f.duration(z),g.duration(z)}},color:{get:function(){return n},set:function(b){n=a.utils.getColor(b),h.color(n)}},barColor:{get:function(){return e.barColor},set:function(a){e.barColor(a),h.color(function(a,b){return d3.rgb("#ccc").darker(1.5*b).toString()})}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.multiChart=function(){"use strict";function b(j){return j.each(function(j){function n(a){var b=2===j[a.seriesIndex].yAxis?E:D;a.value=a.point.x,a.series={value:a.point.y,color:a.point.color,key:a.series.key},G.duration(0).headerFormatter(function(a,b){return C.tickFormat()(a,b)}).valueFormatter(function(a,c){return b.tickFormat()(a,c)}).data(a).hidden(!1)}function H(a){var b=2===j[a.seriesIndex].yAxis?E:D;a.value=a.point.x,a.series={value:a.point.y,color:a.point.color,key:a.series.key},G.duration(100).headerFormatter(function(a,b){return C.tickFormat()(a,b)}).valueFormatter(function(a,c){return b.tickFormat()(a,c)}).data(a).hidden(!1)}function J(a){var b=2===j[a.seriesIndex].yAxis?E:D;a.point.x=A.x()(a.point),a.point.y=A.y()(a.point),G.duration(0).headerFormatter(function(a,b){return C.tickFormat()(a,b)}).valueFormatter(function(a,c){return b.tickFormat()(a,c)}).data(a).hidden(!1)}function K(a){var b=2===j[a.data.series].yAxis?E:D;a.value=y.x()(a.data),a.series={value:y.y()(a.data),color:a.color,key:a.data.key},G.duration(0).headerFormatter(function(a,b){return C.tickFormat()(a,b)}).valueFormatter(function(a,c){return b.tickFormat()(a,c)}).data(a).hidden(!1)}function L(){for(var a=0,b=I.length;b>a;a++){var c=I[a];try{c.clearHighlights()}catch(d){}}}function M(a,b,c){for(var d=0,e=I.length;e>d;d++){var f=I[d];try{f.highlightPoint(a,b,c)}catch(g){}}}var N=d3.select(this);a.utils.initSVG(N),b.update=function(){N.transition().call(b)},b.container=this;var O=a.utils.availableWidth(g,N,e),P=a.utils.availableHeight(h,N,e),Q=j.filter(function(a){return"line"==a.type&&1==a.yAxis}),R=j.filter(function(a){return"line"==a.type&&2==a.yAxis}),S=j.filter(function(a){return"scatter"==a.type&&1==a.yAxis}),T=j.filter(function(a){return"scatter"==a.type&&2==a.yAxis}),U=j.filter(function(a){return"bar"==a.type&&1==a.yAxis}),V=j.filter(function(a){return"bar"==a.type&&2==a.yAxis}),W=j.filter(function(a){return"area"==a.type&&1==a.yAxis}),X=j.filter(function(a){return"area"==a.type&&2==a.yAxis});if(!(j&&j.length&&j.filter(function(a){return a.values.length}).length))return a.utils.noData(b,N),b;N.selectAll(".nv-noData").remove();var Y=j.filter(function(a){return!a.disabled&&1==a.yAxis}).map(function(a){return a.values.map(function(a,b){return{x:k(a),y:l(a)}})}),Z=j.filter(function(a){return!a.disabled&&2==a.yAxis}).map(function(a){return a.values.map(function(a,b){return{x:k(a),y:l(a)}})});r.domain(d3.extent(d3.merge(Y.concat(Z)),function(a){return a.x})).range([0,O]);var $=N.selectAll("g.wrap.multiChart").data([j]),_=$.enter().append("g").attr("class","wrap nvd3 multiChart").append("g");_.append("g").attr("class","nv-x nv-axis"),_.append("g").attr("class","nv-y1 nv-axis"),_.append("g").attr("class","nv-y2 nv-axis"),_.append("g").attr("class","stack1Wrap"),_.append("g").attr("class","stack2Wrap"),_.append("g").attr("class","bars1Wrap"),_.append("g").attr("class","bars2Wrap"),_.append("g").attr("class","scatters1Wrap"),_.append("g").attr("class","scatters2Wrap"),_.append("g").attr("class","lines1Wrap"),_.append("g").attr("class","lines2Wrap"),_.append("g").attr("class","legendWrap"),_.append("g").attr("class","nv-interactive");var aa=$.select("g"),ba=j.map(function(a,b){return j[b].color||f(a,b)});if(i){var ca=F.align()?O/2:O,da=F.align()?ca:0;F.width(ca),F.color(ba),aa.select(".legendWrap").datum(j.map(function(a){return a.originalKey=void 0===a.originalKey?a.key:a.originalKey,a.key=a.originalKey+(1==a.yAxis?"":q),a})).call(F),F.height()>e.top&&(e.top=F.height(),P=a.utils.availableHeight(h,N,e)),aa.select(".legendWrap").attr("transform","translate("+da+","+-e.top+")")}else aa.select(".legendWrap").selectAll("*").remove();u.width(O).height(P).interpolate(m).color(ba.filter(function(a,b){return!j[b].disabled&&1==j[b].yAxis&&"line"==j[b].type})),v.width(O).height(P).interpolate(m).color(ba.filter(function(a,b){return!j[b].disabled&&2==j[b].yAxis&&"line"==j[b].type})),w.width(O).height(P).color(ba.filter(function(a,b){return!j[b].disabled&&1==j[b].yAxis&&"scatter"==j[b].type})),x.width(O).height(P).color(ba.filter(function(a,b){return!j[b].disabled&&2==j[b].yAxis&&"scatter"==j[b].type})),y.width(O).height(P).color(ba.filter(function(a,b){return!j[b].disabled&&1==j[b].yAxis&&"bar"==j[b].type})),z.width(O).height(P).color(ba.filter(function(a,b){return!j[b].disabled&&2==j[b].yAxis&&"bar"==j[b].type})),A.width(O).height(P).interpolate(m).color(ba.filter(function(a,b){return!j[b].disabled&&1==j[b].yAxis&&"area"==j[b].type})),B.width(O).height(P).interpolate(m).color(ba.filter(function(a,b){return!j[b].disabled&&2==j[b].yAxis&&"area"==j[b].type})),aa.attr("transform","translate("+e.left+","+e.top+")");var ea=aa.select(".lines1Wrap").datum(Q.filter(function(a){return!a.disabled})),fa=aa.select(".scatters1Wrap").datum(S.filter(function(a){return!a.disabled})),ga=aa.select(".bars1Wrap").datum(U.filter(function(a){return!a.disabled})),ha=aa.select(".stack1Wrap").datum(W.filter(function(a){return!a.disabled})),ia=aa.select(".lines2Wrap").datum(R.filter(function(a){return!a.disabled})),ja=aa.select(".scatters2Wrap").datum(T.filter(function(a){return!a.disabled})),ka=aa.select(".bars2Wrap").datum(V.filter(function(a){return!a.disabled})),la=aa.select(".stack2Wrap").datum(X.filter(function(a){return!a.disabled})),ma=W.length?W.map(function(a){return a.values}).reduce(function(a,b){return a.map(function(a,c){return{x:a.x,y:a.y+b[c].y}})}).concat([{x:0,y:0}]):[],na=X.length?X.map(function(a){return a.values}).reduce(function(a,b){return a.map(function(a,c){return{x:a.x,y:a.y+b[c].y}})}).concat([{x:0,y:0}]):[];s.domain(c||d3.extent(d3.merge(Y).concat(ma),function(a){return a.y})).range([0,P]),t.domain(d||d3.extent(d3.merge(Z).concat(na),function(a){return a.y})).range([0,P]),u.yDomain(s.domain()),w.yDomain(s.domain()),y.yDomain(s.domain()),A.yDomain(s.domain()),v.yDomain(t.domain()),x.yDomain(t.domain()),z.yDomain(t.domain()),B.yDomain(t.domain()),W.length&&d3.transition(ha).call(A),X.length&&d3.transition(la).call(B),U.length&&d3.transition(ga).call(y),V.length&&d3.transition(ka).call(z),Q.length&&d3.transition(ea).call(u),R.length&&d3.transition(ia).call(v),S.length&&d3.transition(fa).call(w),T.length&&d3.transition(ja).call(x),C._ticks(a.utils.calcTicksX(O/100,j)).tickSize(-P,0),aa.select(".nv-x.nv-axis").attr("transform","translate(0,"+P+")"),d3.transition(aa.select(".nv-x.nv-axis")).call(C),D._ticks(a.utils.calcTicksY(P/36,j)).tickSize(-O,0),d3.transition(aa.select(".nv-y1.nv-axis")).call(D),E._ticks(a.utils.calcTicksY(P/36,j)).tickSize(-O,0),d3.transition(aa.select(".nv-y2.nv-axis")).call(E),aa.select(".nv-y1.nv-axis").classed("nv-disabled",Y.length?!1:!0).attr("transform","translate("+r.range()[0]+",0)"),aa.select(".nv-y2.nv-axis").classed("nv-disabled",Z.length?!1:!0).attr("transform","translate("+r.range()[1]+",0)"),F.dispatch.on("stateChange",function(a){b.update()}),p&&(o.width(O).height(P).margin({left:e.left,top:e.top}).svgContainer(N).xScale(r),$.select(".nv-interactive").call(o)),p?(o.dispatch.on("elementMousemove",function(c){L();var d,e,g,h=[];j.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(i,j){var k=r.domain(),l=i.values.filter(function(a,c){return b.x()(a,c)>=k[0]&&b.x()(a,c)<=k[1]});e=a.interactiveBisect(l,c.pointXValue,b.x());var m=l[e],n=b.y()(m,e);null!==n&&M(j,e,!0),void 0!==m&&(void 0===d&&(d=m),void 0===g&&(g=r(b.x()(m,e))),h.push({key:i.key,value:n,color:f(i,i.seriesIndex),data:m,yAxis:2==i.yAxis?E:D}))});var i=function(a,b){var c=h[b].yAxis;return null==a?"N/A":c.tickFormat()(a)};o.tooltip.headerFormatter(function(a,b){return C.tickFormat()(a,b)}).valueFormatter(o.tooltip.valueFormatter()||i).data({value:b.x()(d,e),index:e,series:h})(),o.renderGuideLine(g)}),o.dispatch.on("elementMouseout",function(a){L()})):(u.dispatch.on("elementMouseover.tooltip",n),v.dispatch.on("elementMouseover.tooltip",n),u.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),v.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),w.dispatch.on("elementMouseover.tooltip",H),x.dispatch.on("elementMouseover.tooltip",H),w.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),x.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),A.dispatch.on("elementMouseover.tooltip",J),B.dispatch.on("elementMouseover.tooltip",J),A.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),B.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),y.dispatch.on("elementMouseover.tooltip",K),z.dispatch.on("elementMouseover.tooltip",K),y.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),z.dispatch.on("elementMouseout.tooltip",function(a){G.hidden(!0)}),y.dispatch.on("elementMousemove.tooltip",function(a){G()}),z.dispatch.on("elementMousemove.tooltip",function(a){G()}))}),b}var c,d,e={top:30,right:20,bottom:50,left:60},f=a.utils.defaultColor(),g=null,h=null,i=!0,j=null,k=function(a){return a.x},l=function(a){return a.y},m="linear",n=!0,o=a.interactiveGuideline(),p=!1,q=" (right axis)",r=d3.scale.linear(),s=d3.scale.linear(),t=d3.scale.linear(),u=a.models.line().yScale(s),v=a.models.line().yScale(t),w=a.models.scatter().yScale(s),x=a.models.scatter().yScale(t),y=a.models.multiBar().stacked(!1).yScale(s),z=a.models.multiBar().stacked(!1).yScale(t),A=a.models.stackedArea().yScale(s),B=a.models.stackedArea().yScale(t),C=a.models.axis().scale(r).orient("bottom").tickPadding(5),D=a.models.axis().scale(s).orient("left"),E=a.models.axis().scale(t).orient("right"),F=a.models.legend().height(30),G=a.models.tooltip(),H=d3.dispatch(),I=[u,v,w,x,y,z,A,B];return b.dispatch=H,b.legend=F,b.lines1=u,b.lines2=v,b.scatters1=w,b.scatters2=x,b.bars1=y,b.bars2=z,b.stack1=A,b.stack2=B,b.xAxis=C,b.yAxis1=D,b.yAxis2=E,b.tooltip=G,b.interactiveLayer=o,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},showLegend:{get:function(){return i},set:function(a){i=a}},yDomain1:{get:function(){return c},set:function(a){c=a}},yDomain2:{get:function(){return d},set:function(a){d=a}},noData:{get:function(){return j},set:function(a){j=a}},interpolate:{get:function(){return m},set:function(a){m=a}},legendRightAxisHint:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left}},color:{get:function(){return f},set:function(b){f=a.utils.getColor(b)}},x:{get:function(){return k},set:function(a){k=a,u.x(a),v.x(a),w.x(a),x.x(a),y.x(a),z.x(a),A.x(a),B.x(a)}},y:{get:function(){return l},set:function(a){l=a,u.y(a),v.y(a),w.y(a),x.y(a),A.y(a),B.y(a),y.y(a),z.y(a)}},useVoronoi:{get:function(){return n},set:function(a){n=a,u.useVoronoi(a),v.useVoronoi(a),A.useVoronoi(a),B.useVoronoi(a)}},useInteractiveGuideline:{get:function(){return p},set:function(a){p=a,p&&(u.interactive(!1),u.useVoronoi(!1),v.interactive(!1),v.useVoronoi(!1),A.interactive(!1),A.useVoronoi(!1),B.interactive(!1),B.useVoronoi(!1),w.interactive(!1),x.interactive(!1))}}}),a.utils.initOptions(b),b},a.models.ohlcBar=function(){"use strict";function b(y){return y.each(function(b){k=d3.select(this);var y=a.utils.availableWidth(h,k,g),A=a.utils.availableHeight(i,k,g);a.utils.initSVG(k);var B=y/b[0].values.length*.9;l.domain(c||d3.extent(b[0].values.map(n).concat(t))),v?l.range(e||[.5*y/b[0].values.length,y*(b[0].values.length-.5)/b[0].values.length]):l.range(e||[5+B/2,y-B/2-5]),m.domain(d||[d3.min(b[0].values.map(s).concat(u)),d3.max(b[0].values.map(r).concat(u))]).range(f||[A,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var C=d3.select(this).selectAll("g.nv-wrap.nv-ohlcBar").data([b[0].values]),D=C.enter().append("g").attr("class","nvd3 nv-wrap nv-ohlcBar"),E=D.append("defs"),F=D.append("g"),G=C.select("g");F.append("g").attr("class","nv-ticks"),C.attr("transform","translate("+g.left+","+g.top+")"),k.on("click",function(a,b){z.chartClick({data:a,index:b,pos:d3.event,id:j})}),E.append("clipPath").attr("id","nv-chart-clip-path-"+j).append("rect"),C.select("#nv-chart-clip-path-"+j+" rect").attr("width",y).attr("height",A),G.attr("clip-path",w?"url(#nv-chart-clip-path-"+j+")":"");var H=C.select(".nv-ticks").selectAll(".nv-tick").data(function(a){return a});H.exit().remove(),H.enter().append("path").attr("class",function(a,b,c){return(p(a,b)>q(a,b)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+c+"-"+b}).attr("d",function(a,b){return"m0,0l0,"+(m(p(a,b))-m(r(a,b)))+"l"+-B/2+",0l"+B/2+",0l0,"+(m(s(a,b))-m(p(a,b)))+"l0,"+(m(q(a,b))-m(s(a,b)))+"l"+B/2+",0l"+-B/2+",0z"}).attr("transform",function(a,b){return"translate("+l(n(a,b))+","+m(r(a,b))+")"}).attr("fill",function(a,b){return x[0]}).attr("stroke",function(a,b){return x[0]}).attr("x",0).attr("y",function(a,b){return m(Math.max(0,o(a,b)))}).attr("height",function(a,b){return Math.abs(m(o(a,b))-m(0))}),H.attr("class",function(a,b,c){return(p(a,b)>q(a,b)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+c+"-"+b}),d3.transition(H).attr("transform",function(a,b){return"translate("+l(n(a,b))+","+m(r(a,b))+")"}).attr("d",function(a,c){var d=y/b[0].values.length*.9;return"m0,0l0,"+(m(p(a,c))-m(r(a,c)))+"l"+-d/2+",0l"+d/2+",0l0,"+(m(s(a,c))-m(p(a,c)))+"l0,"+(m(q(a,c))-m(s(a,c)))+"l"+d/2+",0l"+-d/2+",0z"})}),b}var c,d,e,f,g={top:0,right:0,bottom:0,left:0},h=null,i=null,j=Math.floor(1e4*Math.random()),k=null,l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=function(a){return a.open},q=function(a){return a.close},r=function(a){return a.high},s=function(a){return a.low},t=[],u=[],v=!1,w=!0,x=a.utils.defaultColor(),y=!1,z=d3.dispatch("stateChange","changeState","renderEnd","chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove");return b.highlightPoint=function(a,c){b.clearHighlights(),k.select(".nv-ohlcBar .nv-tick-0-"+a).classed("hover",c)},b.clearHighlights=function(){k.select(".nv-ohlcBar .nv-tick.hover").classed("hover",!1)},b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return i},set:function(a){i=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return d},set:function(a){d=a}},xRange:{get:function(){return e},set:function(a){e=a}},yRange:{get:function(){return f},set:function(a){f=a}},forceX:{get:function(){return t},set:function(a){t=a}},forceY:{get:function(){return u},set:function(a){u=a}},padData:{get:function(){return v},set:function(a){v=a}},clipEdge:{get:function(){return w},set:function(a){w=a}},id:{get:function(){return j},set:function(a){j=a}},interactive:{get:function(){return y},set:function(a){y=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},open:{get:function(){return p()},set:function(a){p=a}},close:{get:function(){return q()},set:function(a){q=a}},high:{get:function(){return r},set:function(a){r=a}},low:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return g},set:function(a){g.top=void 0!=a.top?a.top:g.top,g.right=void 0!=a.right?a.right:g.right,g.bottom=void 0!=a.bottom?a.bottom:g.bottom,g.left=void 0!=a.left?a.left:g.left}},color:{get:function(){return x},set:function(b){x=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.parallelCoordinates=function(){"use strict";function b(B){return A.reset(),B.each(function(b){function A(a){return x(o.map(function(b){if(isNaN(a.values[b.key])||isNaN(parseFloat(a.values[b.key]))||O){var c=l[b.key].domain(),d=l[b.key].range(),e=c[0]-(c[1]-c[0])/9;if(v.indexOf(b.key)<0){var f=d3.scale.linear().domain([e,c[1]]).range([j-12,d[1]]);l[b.key].brush.y(f),v.push(b.key)}if(isNaN(a.values[b.key])||isNaN(parseFloat(a.values[b.key])))return[k(b.key),l[b.key](e)]}return void 0!==U&&(v.length>0||O?(U.style("display","inline"),V.style("display","inline")):(U.style("display","none"),V.style("display","none"))),[k(b.key),l[b.key](a.values[b.key])]}))}function B(a){s.forEach(function(b){var c=l[b.dimension].brush.y().domain();b.hasOnlyNaN&&(b.extent[1]=(l[b.dimension].domain()[1]-c[0])*(b.extent[1]-b.extent[0])/(N[b.dimension]-b.extent[0])+c[0]),b.hasNaN&&(b.extent[0]=c[0]),a&&l[b.dimension].brush.extent(b.extent)}),e.select(".nv-brushBackground").each(function(a){d3.select(this).call(l[a.key].brush)}).selectAll("rect").attr("x",-8).attr("width",16),F()}function C(){q===!1&&(q=!0,B(!0))}function D(){$=p.filter(function(a){return!l[a].brush.empty()}),_=$.map(function(a){return l[a].brush.extent()}),s=[],$.forEach(function(a,b){s[b]={dimension:a,extent:_[b],hasNaN:!1,hasOnlyNaN:!1}}),t=[],c.style("display",function(a){var b=$.every(function(b,c){return(isNaN(a.values[b])||isNaN(parseFloat(a.values[b])))&&_[c][0]==l[b].brush.y().domain()[0]?!0:_[c][0]<=a.values[b]&&a.values[b]<=_[c][1]&&!isNaN(parseFloat(a.values[b]))});return b&&t.push(a),b?null:"none"}),F(),z.brush({filters:s,active:t})}function E(){var a=$.length>0?!0:!1;s.forEach(function(a){a.extent[0]===l[a.dimension].brush.y().domain()[0]&&v.indexOf(a.dimension)>=0&&(a.hasNaN=!0),a.extent[1]<l[a.dimension].domain()[0]&&(a.hasOnlyNaN=!0)}),z.brushEnd(t,a)}function F(){e.select(".nv-axis").each(function(a,b){var c=s.filter(function(b){return b.dimension==a.key});P[a.key]=l[a.key].domain(),0!=c.length&&q&&(P[a.key]=[],c[0].extent[1]>l[a.key].domain()[0]&&(P[a.key]=[c[0].extent[1]]),c[0].extent[0]>=l[a.key].domain()[0]&&P[a.key].push(c[0].extent[0])),d3.select(this).call(y.scale(l[a.key]).tickFormat(a.format).tickValues(P[a.key]))})}function G(a){u[a.key]=this.parentNode.__origin__=k(a.key),d.attr("visibility","hidden")}function H(a){u[a.key]=Math.min(i,Math.max(0,this.parentNode.__origin__+=d3.event.x)),c.attr("d",A),o.sort(function(a,b){return J(a.key)-J(b.key)}),o.forEach(function(a,b){return a.currentPosition=b}),k.domain(o.map(function(a){return a.key})),e.attr("transform",function(a){return"translate("+J(a.key)+")"})}function I(a,b){delete this.parentNode.__origin__,delete u[a.key],d3.select(this.parentNode).attr("transform","translate("+k(a.key)+")"),c.attr("d",A),d.attr("d",A).attr("visibility",null),z.dimensionsOrder(o)}function J(a){var b=u[a];return null==b?k(a):b}var K=d3.select(this);if(i=a.utils.availableWidth(g,K,f),j=a.utils.availableHeight(h,K,f),a.utils.initSVG(K),void 0===b[0].values){var L=[];b.forEach(function(a){var b={},c=Object.keys(a);c.forEach(function(c){"name"!==c&&(b[c]=a[c])}),L.push({key:a.name,values:b})}),b=L}var M=b.map(function(a){return a.values});0===t.length&&(t=b),p=n.sort(function(a,b){return a.currentPosition-b.currentPosition}).map(function(a){return a.key}),o=n.filter(function(a){return!a.disabled}),k.rangePoints([0,i],1).domain(o.map(function(a){return a.key}));var N={},O=!1,P=[];p.forEach(function(a){var b=d3.extent(M,function(b){return+b[a]}),c=b[0],d=b[1],e=!1;(isNaN(c)||isNaN(d))&&(e=!0,c=0,d=0),c===d&&(c-=1,d+=1);var f=s.filter(function(b){return b.dimension==a});0!==f.length&&(e?(c=l[a].domain()[0],d=l[a].domain()[1]):!f[0].hasOnlyNaN&&q?(c=c>f[0].extent[0]?f[0].extent[0]:c,d=d<f[0].extent[1]?f[0].extent[1]:d):f[0].hasNaN&&(d=d<f[0].extent[1]?f[0].extent[1]:d,N[a]=l[a].domain()[1],O=!0)),l[a]=d3.scale.linear().domain([c,d]).range([.9*(j-12),0]),v=[],l[a].brush=d3.svg.brush().y(l[a]).on("brushstart",C).on("brush",D).on("brushend",E)});var Q=K.selectAll("g.nv-wrap.nv-parallelCoordinates").data([b]),R=Q.enter().append("g").attr("class","nvd3 nv-wrap nv-parallelCoordinates"),S=R.append("g"),T=Q.select("g");S.append("g").attr("class","nv-parallelCoordinates background"),S.append("g").attr("class","nv-parallelCoordinates foreground"),S.append("g").attr("class","nv-parallelCoordinates missingValuesline"),Q.attr("transform","translate("+f.left+","+f.top+")"),x.interpolate("cardinal").tension(w),y.orient("left");var U,V,W=d3.behavior.drag().on("dragstart",G).on("drag",H).on("dragend",I),X=k.range()[1]-k.range()[0];if(!isNaN(X)){var Y=[0+X/2,j-12,i-X/2,j-12];U=Q.select(".missingValuesline").selectAll("line").data([Y]),U.enter().append("line"),U.exit().remove(),U.attr("x1",function(a){return a[0]}).attr("y1",function(a){return a[1]}).attr("x2",function(a){return a[2]}).attr("y2",function(a){return a[3]}),V=Q.select(".missingValuesline").selectAll("text").data([m]),V.append("text").data([m]),V.enter().append("text"),V.exit().remove(),V.attr("y",j).attr("x",i-92-X/2).text(function(a){return a})}d=Q.select(".background").selectAll("path").data(b),d.enter().append("path"),d.exit().remove(),d.attr("d",A),c=Q.select(".foreground").selectAll("path").data(b),c.enter().append("path"),c.exit().remove(),c.attr("d",A).style("stroke-width",function(a,b){return isNaN(a.strokeWidth)&&(a.strokeWidth=1),a.strokeWidth}).attr("stroke",function(a,b){return a.color||r(a,b)}),c.on("mouseover",function(a,b){d3.select(this).classed("hover",!0).style("stroke-width",a.strokeWidth+2+"px").style("stroke-opacity",1),z.elementMouseover({label:a.name,color:a.color||r(a,b),values:a.values,dimensions:o})}),c.on("mouseout",function(a,b){d3.select(this).classed("hover",!1).style("stroke-width",a.strokeWidth+"px").style("stroke-opacity",.7),z.elementMouseout({label:a.name,index:b})}),c.on("mousemove",function(a,b){z.elementMousemove()}),c.on("click",function(a){z.elementClick({id:a.id})}),e=T.selectAll(".dimension").data(o);var Z=e.enter().append("g").attr("class","nv-parallelCoordinates dimension");e.attr("transform",function(a){return"translate("+k(a.key)+",0)"}),Z.append("g").attr("class","nv-axis"),Z.append("text").attr("class","nv-label").style("cursor","move").attr("dy","-1em").attr("text-anchor","middle").on("mouseover",function(a,b){z.elementMouseover({label:a.tooltip||a.key,color:a.color})}).on("mouseout",function(a,b){z.elementMouseout({label:a.tooltip})}).on("mousemove",function(a,b){z.elementMousemove()}).call(W),Z.append("g").attr("class","nv-brushBackground"),e.exit().remove(),e.select(".nv-label").text(function(a){return a.key}),B(q);var $=p.filter(function(a){return!l[a].brush.empty()}),_=$.map(function(a){return l[a].brush.extent()}),aa=t.slice(0);t=[],c.style("display",function(a){var b=$.every(function(b,c){return(isNaN(a.values[b])||isNaN(parseFloat(a.values[b])))&&_[c][0]==l[b].brush.y().domain()[0]?!0:_[c][0]<=a.values[b]&&a.values[b]<=_[c][1]&&!isNaN(parseFloat(a.values[b]))});return b&&t.push(a),b?null:"none"}),(s.length>0||!a.utils.arrayEquals(t,aa))&&z.activeChanged(t)}),b}var c,d,e,f={top:30,right:0,bottom:10,left:0},g=null,h=null,i=null,j=null,k=d3.scale.ordinal(),l={},m="undefined values",n=[],o=[],p=[],q=!0,r=a.utils.defaultColor(),s=[],t=[],u=[],v=[],w=1,x=d3.svg.line(),y=d3.svg.axis(),z=d3.dispatch("brushstart","brush","brushEnd","dimensionsOrder","stateChange","elementClick","elementMouseover","elementMouseout","elementMousemove","renderEnd","activeChanged"),A=a.utils.renderWatch(z);return b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},dimensionData:{get:function(){return n},set:function(a){n=a}},displayBrush:{get:function(){return q},set:function(a){q=a}},filters:{get:function(){return s},set:function(a){s=a}},active:{get:function(){return t},set:function(a){t=a}},lineTension:{get:function(){return w},set:function(a){w=a}},undefinedValuesLabel:{get:function(){return m},set:function(a){m=a}},dimensions:{get:function(){return n.map(function(a){return a.key})},set:function(b){a.deprecated("dimensions","use dimensionData instead"),0===n.length?b.forEach(function(a){n.push({key:a})}):b.forEach(function(a,b){n[b].key=a})}},dimensionNames:{get:function(){return n.map(function(a){return a.key})},set:function(b){a.deprecated("dimensionNames","use dimensionData instead"),p=[],0===n.length?b.forEach(function(a){n.push({key:a})}):b.forEach(function(a,b){n[b].key=a})}},dimensionFormats:{get:function(){return n.map(function(a){return a.format})},set:function(b){a.deprecated("dimensionFormats","use dimensionData instead"),0===n.length?b.forEach(function(a){n.push({format:a})}):b.forEach(function(a,b){n[b].format=a})}},margin:{get:function(){return f},set:function(a){f.top=void 0!==a.top?a.top:f.top,f.right=void 0!==a.right?a.right:f.right,f.bottom=void 0!==a.bottom?a.bottom:f.bottom,f.left=void 0!==a.left?a.left:f.left}},color:{get:function(){return r},set:function(b){r=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.parallelCoordinatesChart=function(){"use strict";function b(e){return r.reset(),r.models(c),e.each(function(e){var j=d3.select(this);a.utils.initSVG(j);var o=a.utils.availableWidth(g,j,f),p=a.utils.availableHeight(h,j,f);if(b.update=function(){j.call(b)},b.container=this,k.setter(t(l),b.update).getter(s(l)).update(),k.disabled=l.map(function(a){return!!a.disabled}),l=l.map(function(a){return a.disabled=!!a.disabled,a}),l.forEach(function(a,b){a.originalPosition=isNaN(a.originalPosition)?b:a.originalPosition,a.currentPosition=isNaN(a.currentPosition)?b:a.currentPosition}),!n){var r;n={};for(r in k)k[r]instanceof Array?n[r]=k[r].slice(0):n[r]=k[r]}if(!e||!e.length)return a.utils.noData(b,j),b;j.selectAll(".nv-noData").remove();var u=j.selectAll("g.nv-wrap.nv-parallelCoordinatesChart").data([e]),v=u.enter().append("g").attr("class","nvd3 nv-wrap nv-parallelCoordinatesChart").append("g"),w=u.select("g");v.append("g").attr("class","nv-parallelCoordinatesWrap"),v.append("g").attr("class","nv-legendWrap"),w.select("rect").attr("width",o).attr("height",p>0?p:0),i?(d.width(o).color(function(a){return"rgb(188,190,192)"}),w.select(".nv-legendWrap").datum(l.sort(function(a,b){return a.originalPosition-b.originalPosition})).call(d),d.height()>f.top&&(f.top=d.height(),p=a.utils.availableHeight(h,j,f)),u.select(".nv-legendWrap").attr("transform","translate( 0 ,"+-f.top+")")):w.select(".nv-legendWrap").selectAll("*").remove(),u.attr("transform","translate("+f.left+","+f.top+")"),c.width(o).height(p).dimensionData(l).displayBrush(m);var x=w.select(".nv-parallelCoordinatesWrap ").datum(e);x.transition().call(c),c.dispatch.on("brushEnd",function(a,b){b?(m=!0,q.brushEnd(a)):m=!1}),d.dispatch.on("stateChange",function(a){for(var c in a)k[c]=a[c];q.stateChange(k),b.update()}),c.dispatch.on("dimensionsOrder",function(a){l.sort(function(a,b){return a.currentPosition-b.currentPosition});var b=!1;l.forEach(function(a,c){a.currentPosition=c,a.currentPosition!==a.originalPosition&&(b=!0)}),q.dimensionsOrder(l,b)}),q.on("changeState",function(a){"undefined"!=typeof a.disabled&&(l.forEach(function(b,c){b.disabled=a.disabled[c]}),k.disabled=a.disabled),b.update()})}),r.renderEnd("parraleleCoordinateChart immediate"),b}var c=a.models.parallelCoordinates(),d=a.models.legend(),e=a.models.tooltip(),f=(a.models.tooltip(),{top:0,right:0,bottom:0,left:0}),g=null,h=null,i=!0,j=a.utils.defaultColor(),k=a.utils.state(),l=[],m=!0,n=null,o=null,p="undefined",q=d3.dispatch("dimensionsOrder","brushEnd","stateChange","changeState","renderEnd"),r=a.utils.renderWatch(q),s=function(a){return function(){return{active:a.map(function(a){return!a.disabled})}}},t=function(a){return function(b){void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};return e.contentGenerator(function(a){var b='<table><thead><tr><td class="legend-color-guide"><div style="background-color:'+a.color+'"></div></td><td><strong>'+a.key+"</strong></td></tr></thead>";return 0!==a.series.length&&(b+='<tbody><tr><td height ="10px"></td></tr>',a.series.forEach(function(a){b=b+'<tr><td class="legend-color-guide"><div style="background-color:'+a.color+'"></div></td><td class="key">'+a.key+'</td><td class="value">'+a.value+"</td></tr>"}),b+="</tbody>"),b+="</table>"}),c.dispatch.on("elementMouseover.tooltip",function(a){var b={key:a.label,color:a.color,series:[]};a.values&&(Object.keys(a.values).forEach(function(c){var d=a.dimensions.filter(function(a){return a.key===c})[0];if(d){var e;e=isNaN(a.values[c])||isNaN(parseFloat(a.values[c]))?p:d.format(a.values[c]),b.series.push({idx:d.currentPosition,key:c,value:e,color:d.color})}}),b.series.sort(function(a,b){return a.idx-b.idx})),e.data(b).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",function(a){e.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(){e()}),b.dispatch=q,b.parallelCoordinates=c,b.legend=d,b.tooltip=e,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},showLegend:{get:function(){return i},set:function(a){i=a}},defaultState:{get:function(){return n},set:function(a){n=a}},dimensionData:{get:function(){return l},set:function(a){l=a}},displayBrush:{get:function(){return m},set:function(a){m=a}},noData:{get:function(){return o},set:function(a){o=a}},nanValue:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return f},set:function(a){f.top=void 0!==a.top?a.top:f.top,f.right=void 0!==a.right?a.right:f.right,f.bottom=void 0!==a.bottom?a.bottom:f.bottom,f.left=void 0!==a.left?a.left:f.left}},color:{get:function(){return j},set:function(b){j=a.utils.getColor(b),d.color(j),c.color(j)}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.pie=function(){"use strict";function b(F){return E.reset(),F.each(function(b){function F(a,b){a.endAngle=isNaN(a.endAngle)?0:a.endAngle,a.startAngle=isNaN(a.startAngle)?0:a.startAngle,p||(a.innerRadius=0);var c=d3.interpolate(this._current,a);return this._current=c(0),function(a){return C[b](c(a))}}var G=d-c.left-c.right,H=e-c.top-c.bottom,I=Math.min(G,H)/2,J=[],K=[];if(i=d3.select(this),0===A.length)for(var L=I-I/5,M=y*I,N=0;N<b[0].length;N++)J.push(L),K.push(M);else r?(J=A.map(function(a){return(a.outer-a.outer/5)*I}),K=A.map(function(a){return(a.inner-a.inner/5)*I}),y=d3.min(A.map(function(a){return a.inner-a.inner/5}))):(J=A.map(function(a){return a.outer*I}),K=A.map(function(a){return a.inner*I}),y=d3.min(A.map(function(a){return a.inner})));a.utils.initSVG(i);var O=i.selectAll(".nv-wrap.nv-pie").data(b),P=O.enter().append("g").attr("class","nvd3 nv-wrap nv-pie nv-chart-"+h),Q=P.append("g"),R=O.select("g"),S=Q.append("g").attr("class","nv-pie");Q.append("g").attr("class","nv-pieLabels"),O.attr("transform","translate("+c.left+","+c.top+")"),R.select(".nv-pie").attr("transform","translate("+G/2+","+H/2+")"),R.select(".nv-pieLabels").attr("transform","translate("+G/2+","+H/2+")"),i.on("click",function(a,b){B.chartClick({data:a,index:b,pos:d3.event,id:h})}),C=[],D=[];for(var N=0;N<b[0].length;N++){var T=d3.svg.arc().outerRadius(J[N]),U=d3.svg.arc().outerRadius(J[N]+5);u!==!1&&(T.startAngle(u),U.startAngle(u)),w!==!1&&(T.endAngle(w),U.endAngle(w)),p&&(T.innerRadius(K[N]),U.innerRadius(K[N])),T.cornerRadius&&x&&(T.cornerRadius(x),U.cornerRadius(x)),C.push(T),D.push(U)}var V=d3.layout.pie().sort(null).value(function(a){return a.disabled?0:g(a)});V.padAngle&&v&&V.padAngle(v),p&&q&&(S.append("text").attr("class","nv-pie-title"),O.select(".nv-pie-title").style("text-anchor","middle").text(function(a){return q}).style("font-size",Math.min(G,H)*y*2/(q.length+2)+"px").attr("dy","0.35em").attr("transform",function(a,b){return"translate(0, "+s+")"}));var W=O.select(".nv-pie").selectAll(".nv-slice").data(V),X=O.select(".nv-pieLabels").selectAll(".nv-label").data(V);W.exit().remove(),X.exit().remove();var Y=W.enter().append("g");Y.attr("class","nv-slice"),Y.on("mouseover",function(a,b){d3.select(this).classed("hover",!0),r&&d3.select(this).select("path").transition().duration(70).attr("d",D[b]),B.elementMouseover({data:a.data,index:b,color:d3.select(this).style("fill"),percent:(a.endAngle-a.startAngle)/(2*Math.PI)})}),Y.on("mouseout",function(a,b){d3.select(this).classed("hover",!1),r&&d3.select(this).select("path").transition().duration(50).attr("d",C[b]),B.elementMouseout({data:a.data,index:b})}),Y.on("mousemove",function(a,b){B.elementMousemove({data:a.data,index:b})}),Y.on("click",function(a,b){var c=this;B.elementClick({data:a.data,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c})}),Y.on("dblclick",function(a,b){B.elementDblClick({data:a.data,index:b,color:d3.select(this).style("fill")})}),W.attr("fill",function(a,b){return j(a.data,b)}),W.attr("stroke",function(a,b){return j(a.data,b)});Y.append("path").each(function(a){this._current=a});if(W.select("path").transition().duration(z).attr("d",function(a,b){return C[b](a)}).attrTween("d",F),l){for(var Z=[],N=0;N<b[0].length;N++)Z.push(C[N]),m?p&&(Z[N]=d3.svg.arc().outerRadius(C[N].outerRadius()),u!==!1&&Z[N].startAngle(u),w!==!1&&Z[N].endAngle(w)):p||Z[N].innerRadius(0);X.enter().append("g").classed("nv-label",!0).each(function(a,b){var c=d3.select(this);c.attr("transform",function(a,b){if(t){a.outerRadius=J[b]+10,a.innerRadius=J[b]+15;var c=(a.startAngle+a.endAngle)/2*(180/Math.PI);return(a.startAngle+a.endAngle)/2<Math.PI?c-=90:c+=90,"translate("+Z[b].centroid(a)+") rotate("+c+")"}return a.outerRadius=I+10,a.innerRadius=I+15,"translate("+Z[b].centroid(a)+")"}),c.append("rect").style("stroke","#fff").style("fill","#fff").attr("rx",3).attr("ry",3),c.append("text").style("text-anchor",t?(a.startAngle+a.endAngle)/2<Math.PI?"start":"end":"middle").style("fill","#000")});var $={},_=14,aa=140,ba=function(a){return Math.floor(a[0]/aa)*aa+","+Math.floor(a[1]/_)*_},ca=function(a){return(a.endAngle-a.startAngle)/(2*Math.PI)};X.watchTransition(E,"pie labels").attr("transform",function(a,b){if(t){a.outerRadius=J[b]+10,a.innerRadius=J[b]+15;var c=(a.startAngle+a.endAngle)/2*(180/Math.PI);return(a.startAngle+a.endAngle)/2<Math.PI?c-=90:c+=90,"translate("+Z[b].centroid(a)+") rotate("+c+")"}a.outerRadius=I+10,a.innerRadius=I+15;var d=Z[b].centroid(a),e=ca(a);if(a.value&&e>=o){var f=ba(d);$[f]&&(d[1]-=_),$[ba(d)]=!0}return"translate("+d+")"}),X.select(".nv-label text").style("text-anchor",function(a,b){return t?(a.startAngle+a.endAngle)/2<Math.PI?"start":"end":"middle"}).text(function(a,b){var c=ca(a),d="";if(!a.value||o>c)return"";if("function"==typeof n)d=n(a,b,{key:f(a.data),value:g(a.data),percent:k(c)});else switch(n){case"key":d=f(a.data);break;case"value":d=k(g(a.data));break;case"percent":d=d3.format("%")(c)}return d})}}),E.renderEnd("pie immediate"),b}var c={top:0,right:0,bottom:0,left:0},d=500,e=500,f=function(a){return a.x},g=function(a){return a.y},h=Math.floor(1e4*Math.random()),i=null,j=a.utils.defaultColor(),k=d3.format(",.2f"),l=!0,m=!1,n="key",o=.02,p=!1,q=!1,r=!0,s=0,t=!1,u=!1,v=!1,w=!1,x=0,y=.5,z=250,A=[],B=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),C=[],D=[],E=a.utils.renderWatch(B);return b.dispatch=B,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{arcsRadius:{get:function(){return A},set:function(a){A=a}},width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},showLabels:{get:function(){return l},set:function(a){l=a}},title:{get:function(){return q},set:function(a){q=a}},titleOffset:{get:function(){return s},set:function(a){s=a}},labelThreshold:{get:function(){return o},set:function(a){o=a}},valueFormat:{get:function(){return k},set:function(a){k=a}},x:{get:function(){return f},set:function(a){f=a}},id:{get:function(){return h},set:function(a){h=a}},endAngle:{get:function(){return w},set:function(a){w=a}},startAngle:{get:function(){return u},set:function(a){u=a}},padAngle:{get:function(){return v},set:function(a){v=a}},cornerRadius:{get:function(){return x},set:function(a){x=a}},donutRatio:{get:function(){return y},set:function(a){y=a}},labelsOutside:{get:function(){return m},set:function(a){m=a}},labelSunbeamLayout:{get:function(){return t},set:function(a){t=a}},donut:{get:function(){return p},set:function(a){p=a}},growOnHover:{get:function(){return r},set:function(a){r=a}},pieLabelsOutside:{get:function(){return m},set:function(b){m=b,a.deprecated("pieLabelsOutside","use labelsOutside instead")}},donutLabelsOutside:{get:function(){return m},set:function(b){m=b,a.deprecated("donutLabelsOutside","use labelsOutside instead")}},labelFormat:{get:function(){return k},set:function(b){k=b,a.deprecated("labelFormat","use valueFormat instead")}},margin:{get:function(){return c},set:function(a){c.top="undefined"!=typeof a.top?a.top:c.top,c.right="undefined"!=typeof a.right?a.right:c.right,c.bottom="undefined"!=typeof a.bottom?a.bottom:c.bottom,c.left="undefined"!=typeof a.left?a.left:c.left}},duration:{get:function(){return z},set:function(a){z=a,E.reset(z)}},y:{get:function(){return g},set:function(a){g=d3.functor(a)}},color:{get:function(){return j},set:function(b){j=a.utils.getColor(b)}},labelType:{get:function(){return n},set:function(a){n=a||"key"}}}),a.utils.initOptions(b),b},a.models.pieChart=function(){"use strict";function b(e){return r.reset(),r.models(c),e.each(function(e){var i=d3.select(this);a.utils.initSVG(i);var l=a.utils.availableWidth(g,i,f),o=a.utils.availableHeight(h,i,f);if(b.update=function(){i.transition().call(b)},b.container=this,m.setter(t(e),b.update).getter(s(e)).update(),m.disabled=e.map(function(a){return!!a.disabled}),!n){var p;n={};for(p in m)m[p]instanceof Array?n[p]=m[p].slice(0):n[p]=m[p]}if(!e||!e.length)return a.utils.noData(b,i),b;i.selectAll(".nv-noData").remove();var r=i.selectAll("g.nv-wrap.nv-pieChart").data([e]),u=r.enter().append("g").attr("class","nvd3 nv-wrap nv-pieChart").append("g"),v=r.select("g");if(u.append("g").attr("class","nv-pieWrap"),u.append("g").attr("class","nv-legendWrap"),j){if("top"===k)d.width(l).key(c.x()),r.select(".nv-legendWrap").datum(e).call(d),d.height()>f.top&&(f.top=d.height(),o=a.utils.availableHeight(h,i,f)),r.select(".nv-legendWrap").attr("transform","translate(0,"+-f.top+")");else if("right"===k){var w=a.models.legend().width();w>l/2&&(w=l/2),d.height(o).key(c.x()),d.width(w),l-=d.width(),r.select(".nv-legendWrap").datum(e).call(d).attr("transform","translate("+l+",0)")}}else v.select(".nv-legendWrap").selectAll("*").remove();r.attr("transform","translate("+f.left+","+f.top+")"),c.width(l).height(o);var x=v.select(".nv-pieWrap").datum([e]);d3.transition(x).call(c),d.dispatch.on("stateChange",function(a){for(var c in a)m[c]=a[c];q.stateChange(m),b.update()}),q.on("changeState",function(a){"undefined"!=typeof a.disabled&&(e.forEach(function(b,c){b.disabled=a.disabled[c]}),m.disabled=a.disabled),b.update()})}),r.renderEnd("pieChart immediate"),b}var c=a.models.pie(),d=a.models.legend(),e=a.models.tooltip(),f={top:30,right:20,bottom:20,left:20},g=null,h=null,i=!1,j=!0,k="top",l=a.utils.defaultColor(),m=a.utils.state(),n=null,o=null,p=250,q=d3.dispatch("stateChange","changeState","renderEnd");e.duration(0).headerEnabled(!1).valueFormatter(function(a,b){return c.valueFormat()(a,b)});var r=a.utils.renderWatch(q),s=function(a){return function(){return{active:a.map(function(a){return!a.disabled})}}},t=function(a){return function(b){void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};return c.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:b.x()(a.data),value:b.y()(a.data),color:a.color,percent:a.percent},i||(delete a.percent,delete a.series.percent),e.data(a).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",function(a){e.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(a){e()}),b.legend=d,b.dispatch=q,b.pie=c,b.tooltip=e,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},noData:{get:function(){return o},set:function(a){o=a}},showTooltipPercent:{get:function(){return i},set:function(a){i=a}},showLegend:{get:function(){return j},set:function(a){j=a}},legendPosition:{get:function(){return k},set:function(a){k=a}},defaultState:{get:function(){return n},set:function(a){n=a}},color:{get:function(){return l},set:function(a){l=a,d.color(l),c.color(l)}},duration:{get:function(){return p},set:function(a){p=a,r.reset(p),c.duration(p)}},margin:{get:function(){return f},set:function(a){f.top=void 0!==a.top?a.top:f.top,f.right=void 0!==a.right?a.right:f.right,f.bottom=void 0!==a.bottom?a.bottom:f.bottom,f.left=void 0!==a.left?a.left:f.left}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.scatter=function(){"use strict";function b(a){var b,c;return b=i=i||{},c=a[0].series,b=b[c]=b[c]||{},c=a[1],b=b[c]=b[c]||{}}function c(a){var c,d,e=a[0],f=b(a),g=!1;for(c=1;c<arguments.length;c++)d=arguments[c],f[d]===e[d]&&f.hasOwnProperty(d)||(f[d]=e[d],g=!0);return g}function d(b){return U.reset(),b.each(function(b){function i(){if(T=!1,!z)return!1;if(P===!0){var c=d3.merge(b.map(function(b,c){return b.values.map(function(b,d){var e=s(b,d),f=t(b,d);return[a.utils.NaNtoZero(p(e))+1e-4*Math.random(),a.utils.NaNtoZero(q(f))+1e-4*Math.random(),c,d,b]}).filter(function(a,b){return A(a[4],b)})}));if(0==c.length)return!1;c.length<3&&(c.push([p.range()[0]-20,q.range()[0]-20,null,null]),c.push([p.range()[1]+20,q.range()[1]+20,null,null]),c.push([p.range()[0]-20,q.range()[0]+20,null,null]),c.push([p.range()[1]+20,q.range()[1]-20,null,null]));var d=d3.geom.polygon([[-10,-10],[-10,l+10],[k+10,l+10],[k+10,-10]]),e=d3.geom.voronoi(c).map(function(a,b){return{data:d.clip(a),series:c[b][2],point:c[b][3]}});_.select(".nv-point-paths").selectAll("path").remove();var f=_.select(".nv-point-paths").selectAll("path").data(e),g=f.enter().append("svg:path").attr("d",function(a){return a&&a.data&&0!==a.data.length?"M"+a.data.join(",")+"Z":"M 0 0"}).attr("id",function(a,b){return"nv-path-"+b}).attr("clip-path",function(a,b){return"url(#nv-clip-"+n+"-"+b+")"});if(F&&g.style("fill",d3.rgb(230,230,230)).style("fill-opacity",.4).style("stroke-opacity",1).style("stroke",d3.rgb(200,200,200)),E){_.select(".nv-point-clips").selectAll("*").remove();var h=_.select(".nv-point-clips").selectAll("clipPath").data(c);h.enter().append("svg:clipPath").attr("id",function(a,b){return"nv-clip-"+n+"-"+b}).append("svg:circle").attr("cx",function(a){return a[0]}).attr("cy",function(a){return a[1]}).attr("r",G)}var i=function(a,c){if(T)return 0;var d=b[a.series];if(void 0!==d){var e=d.values[a.point];e.color=m(d,a.series),e.x=s(e),e.y=t(e);var f=o.node().getBoundingClientRect(),g=window.pageYOffset||document.documentElement.scrollTop,h=window.pageXOffset||document.documentElement.scrollLeft,i={left:p(s(e,a.point))+f.left+h+j.left+10,top:q(t(e,a.point))+f.top+g+j.top+10};c({point:e,series:d,pos:i,relativePos:[p(s(e,a.point))+j.left,q(t(e,a.point))+j.top],seriesIndex:a.series,pointIndex:a.point})}};f.on("click",function(a){i(a,O.elementClick)}).on("dblclick",function(a){i(a,O.elementDblClick)}).on("mouseover",function(a){i(a,O.elementMouseover)}).on("mouseout",function(a,b){i(a,O.elementMouseout)})}else _.select(".nv-groups").selectAll(".nv-group").selectAll(".nv-point").on("click",function(a,c){if(T||!b[a.series])return 0;var d=b[a.series],e=d.values[c],f=this;O.elementClick({point:e,series:d,pos:[p(s(e,c))+j.left,q(t(e,c))+j.top],relativePos:[p(s(e,c))+j.left,q(t(e,c))+j.top],seriesIndex:a.series,pointIndex:c,event:d3.event,element:f})}).on("dblclick",function(a,c){if(T||!b[a.series])return 0;var d=b[a.series],e=d.values[c];O.elementDblClick({point:e,series:d,pos:[p(s(e,c))+j.left,q(t(e,c))+j.top],relativePos:[p(s(e,c))+j.left,q(t(e,c))+j.top],seriesIndex:a.series,pointIndex:c})}).on("mouseover",function(a,c){if(T||!b[a.series])return 0;var d=b[a.series],e=d.values[c];O.elementMouseover({point:e,series:d,pos:[p(s(e,c))+j.left,q(t(e,c))+j.top],relativePos:[p(s(e,c))+j.left,q(t(e,c))+j.top],seriesIndex:a.series,pointIndex:c,color:m(a,c)})}).on("mouseout",function(a,c){if(T||!b[a.series])return 0;var d=b[a.series],e=d.values[c];O.elementMouseout({point:e,series:d,pos:[p(s(e,c))+j.left,q(t(e,c))+j.top],relativePos:[p(s(e,c))+j.left,q(t(e,c))+j.top],seriesIndex:a.series,pointIndex:c,color:m(a,c)})})}o=d3.select(this);var Q=a.utils.availableWidth(k,o,j),W=a.utils.availableHeight(l,o,j);a.utils.initSVG(o),b.forEach(function(a,b){a.values.forEach(function(a){a.series=b})});var X=d.yScale().name===d3.scale.log().name?!0:!1,Y=H&&I&&L?[]:d3.merge(b.map(function(a){return a.values.map(function(a,b){return{x:s(a,b),y:t(a,b),size:u(a,b)}})}));if(p.domain(H||d3.extent(Y.map(function(a){return a.x}).concat(w))),B&&b[0]?p.range(J||[(Q*C+Q)/(2*b[0].values.length),Q-Q*(1+C)/(2*b[0].values.length)]):p.range(J||[0,Q]),X){var Z=d3.min(Y.map(function(a){return 0!==a.y?a.y:void 0}));q.clamp(!0).domain(I||d3.extent(Y.map(function(a){return 0!==a.y?a.y:.1*Z}).concat(x))).range(K||[W,0])}else q.domain(I||d3.extent(Y.map(function(a){return a.y}).concat(x))).range(K||[W,0]);r.domain(L||d3.extent(Y.map(function(a){return a.size}).concat(y))).range(M||V),N=p.domain()[0]===p.domain()[1]||q.domain()[0]===q.domain()[1],p.domain()[0]===p.domain()[1]&&(p.domain()[0]?p.domain([p.domain()[0]-.01*p.domain()[0],p.domain()[1]+.01*p.domain()[1]]):p.domain([-1,1])),q.domain()[0]===q.domain()[1]&&(q.domain()[0]?q.domain([q.domain()[0]-.01*q.domain()[0],q.domain()[1]+.01*q.domain()[1]]):q.domain([-1,1])),isNaN(p.domain()[0])&&p.domain([-1,1]),isNaN(q.domain()[0])&&q.domain([-1,1]),e=e||p,f=f||q,g=g||r;var $=p(1)!==e(1)||q(1)!==f(1)||r(1)!==g(1),_=o.selectAll("g.nv-wrap.nv-scatter").data([b]),aa=_.enter().append("g").attr("class","nvd3 nv-wrap nv-scatter nv-chart-"+n),ba=aa.append("defs"),ca=aa.append("g"),da=_.select("g");_.classed("nv-single-point",N),ca.append("g").attr("class","nv-groups"),ca.append("g").attr("class","nv-point-paths"),aa.append("g").attr("class","nv-point-clips"),_.attr("transform","translate("+j.left+","+j.top+")"),ba.append("clipPath").attr("id","nv-edge-clip-"+n).append("rect"),_.select("#nv-edge-clip-"+n+" rect").attr("width",Q).attr("height",W>0?W:0),da.attr("clip-path",D?"url(#nv-edge-clip-"+n+")":""),T=!0;var ea=_.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a){return a.key});ea.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),ea.exit().remove(),ea.attr("class",function(a,b){return(a.classed||"")+" nv-group nv-series-"+b}).classed("nv-noninteractive",!z).classed("hover",function(a){return a.hover}),ea.watchTransition(U,"scatter: groups").style("fill",function(a,b){return m(a,b)}).style("stroke",function(a,b){return m(a,b)}).style("stroke-opacity",1).style("fill-opacity",.5);var fa=ea.selectAll("path.nv-point").data(function(a){return a.values.map(function(a,b){return[a,b]}).filter(function(a,b){return A(a[0],b)})});if(fa.enter().append("path").attr("class",function(a){return"nv-point nv-point-"+a[1]}).style("fill",function(a){return a.color}).style("stroke",function(a){return a.color}).attr("transform",function(b){return"translate("+a.utils.NaNtoZero(e(s(b[0],b[1])))+","+a.utils.NaNtoZero(f(t(b[0],b[1])))+")"}).attr("d",a.utils.symbol().type(function(a){return v(a[0])}).size(function(a){return r(u(a[0],a[1]))})),fa.exit().remove(),ea.exit().selectAll("path.nv-point").watchTransition(U,"scatter exit").attr("transform",function(b){return"translate("+a.utils.NaNtoZero(p(s(b[0],b[1])))+","+a.utils.NaNtoZero(q(t(b[0],b[1])))+")"}).remove(),fa.filter(function(a){return $||c(a,"x","y")}).watchTransition(U,"scatter points").attr("transform",function(b){return"translate("+a.utils.NaNtoZero(p(s(b[0],b[1])))+","+a.utils.NaNtoZero(q(t(b[0],b[1])))+")"}),fa.filter(function(a){return $||c(a,"shape","size")}).watchTransition(U,"scatter points").attr("d",a.utils.symbol().type(function(a){return v(a[0])}).size(function(a){return r(u(a[0],a[1]))})),S){var ga=ea.selectAll(".nv-label").data(function(a){return a.values.map(function(a,b){return[a,b]}).filter(function(a,b){return A(a[0],b)})});ga.enter().append("text").style("fill",function(a,b){return a.color}).style("stroke-opacity",0).style("fill-opacity",1).attr("transform",function(b){var c=a.utils.NaNtoZero(e(s(b[0],b[1])))+Math.sqrt(r(u(b[0],b[1]))/Math.PI)+2;return"translate("+c+","+a.utils.NaNtoZero(f(t(b[0],b[1])))+")"}).text(function(a,b){return a[0].label}),ga.exit().remove(),ea.exit().selectAll("path.nv-label").watchTransition(U,"scatter exit").attr("transform",function(b){var c=a.utils.NaNtoZero(p(s(b[0],b[1])))+Math.sqrt(r(u(b[0],b[1]))/Math.PI)+2;return"translate("+c+","+a.utils.NaNtoZero(q(t(b[0],b[1])))+")"}).remove(),ga.each(function(a){d3.select(this).classed("nv-label",!0).classed("nv-label-"+a[1],!1).classed("hover",!1)}),ga.watchTransition(U,"scatter labels").attr("transform",function(b){var c=a.utils.NaNtoZero(p(s(b[0],b[1])))+Math.sqrt(r(u(b[0],b[1]))/Math.PI)+2;return"translate("+c+","+a.utils.NaNtoZero(q(t(b[0],b[1])))+")"})}R?(clearTimeout(h),h=setTimeout(i,R)):i(),e=p.copy(),f=q.copy(),g=r.copy()}),U.renderEnd("scatter immediate"),d}var e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=null,l=null,m=a.utils.defaultColor(),n=Math.floor(1e5*Math.random()),o=null,p=d3.scale.linear(),q=d3.scale.linear(),r=d3.scale.linear(),s=function(a){return a.x},t=function(a){return a.y},u=function(a){return a.size||1},v=function(a){return a.shape||"circle"},w=[],x=[],y=[],z=!0,A=function(a){return!a.notActive},B=!1,C=.1,D=!1,E=!0,F=!1,G=function(){return 25},H=null,I=null,J=null,K=null,L=null,M=null,N=!1,O=d3.dispatch("elementClick","elementDblClick","elementMouseover","elementMouseout","renderEnd"),P=!0,Q=250,R=300,S=!1,T=!1,U=a.utils.renderWatch(O,Q),V=[16,256];return d.dispatch=O,d.options=a.utils.optionsFunc.bind(d),d._calls=new function(){this.clearHighlights=function(){return a.dom.write(function(){o.selectAll(".nv-point.hover").classed("hover",!1)}),null},this.highlightPoint=function(b,c,d){a.dom.write(function(){o.select(".nv-groups").selectAll(".nv-series-"+b).selectAll(".nv-point-"+c).classed("hover",d)})}},O.on("elementMouseover.point",function(a){z&&d._calls.highlightPoint(a.seriesIndex,a.pointIndex,!0)}),O.on("elementMouseout.point",function(a){z&&d._calls.highlightPoint(a.seriesIndex,a.pointIndex,!1)}),d._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},xScale:{get:function(){return p},set:function(a){p=a}},yScale:{get:function(){return q},set:function(a){q=a}},pointScale:{get:function(){return r},set:function(a){r=a}},xDomain:{get:function(){return H},set:function(a){H=a}},yDomain:{get:function(){return I},set:function(a){I=a}},pointDomain:{get:function(){return L},set:function(a){L=a}},xRange:{get:function(){return J},set:function(a){J=a}},yRange:{get:function(){return K},set:function(a){K=a}},pointRange:{get:function(){return M},set:function(a){M=a}},forceX:{get:function(){return w},set:function(a){w=a}},forceY:{get:function(){return x},set:function(a){x=a}},forcePoint:{get:function(){return y},set:function(a){y=a}},interactive:{get:function(){return z},set:function(a){z=a}},pointActive:{get:function(){return A},set:function(a){A=a}},padDataOuter:{get:function(){return C},set:function(a){C=a}},padData:{get:function(){return B},set:function(a){B=a}},clipEdge:{get:function(){return D},set:function(a){D=a}},clipVoronoi:{get:function(){return E},set:function(a){E=a}},clipRadius:{get:function(){return G},set:function(a){G=a}},showVoronoi:{get:function(){return F},set:function(a){F=a}},id:{get:function(){return n},set:function(a){n=a}},interactiveUpdateDelay:{get:function(){return R},set:function(a){R=a}},showLabels:{get:function(){return S},set:function(a){S=a}},x:{get:function(){return s},set:function(a){s=d3.functor(a)}},y:{get:function(){return t},set:function(a){t=d3.functor(a)}},pointSize:{get:function(){return u},set:function(a){u=d3.functor(a)}},pointShape:{get:function(){return v},set:function(a){v=d3.functor(a)}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return Q},set:function(a){Q=a,U.reset(Q)}},color:{get:function(){return m},set:function(b){m=a.utils.getColor(b)}},useVoronoi:{get:function(){return P},set:function(a){P=a,P===!1&&(E=!1)}}}),a.utils.initOptions(d),d},a.models.scatterChart=function(){"use strict";function b(z){return E.reset(),E.models(c),t&&E.models(d),u&&E.models(e),q&&E.models(g),r&&E.models(h),z.each(function(z){m=d3.select(this),a.utils.initSVG(m);var H=a.utils.availableWidth(k,m,j),I=a.utils.availableHeight(l,m,j);if(b.update=function(){0===A?m.call(b):m.transition().duration(A).call(b)},b.container=this,w.setter(G(z),b.update).getter(F(z)).update(),w.disabled=z.map(function(a){return!!a.disabled}),!x){var J;x={};for(J in w)w[J]instanceof Array?x[J]=w[J].slice(0):x[J]=w[J]}if(!(z&&z.length&&z.filter(function(a){return a.values.length}).length))return a.utils.noData(b,m),E.renderEnd("scatter immediate"),b;m.selectAll(".nv-noData").remove(),o=c.xScale(),p=c.yScale();var K=m.selectAll("g.nv-wrap.nv-scatterChart").data([z]),L=K.enter().append("g").attr("class","nvd3 nv-wrap nv-scatterChart nv-chart-"+c.id()),M=L.append("g"),N=K.select("g");if(M.append("rect").attr("class","nvd3 nv-background").style("pointer-events","none"),M.append("g").attr("class","nv-x nv-axis"),M.append("g").attr("class","nv-y nv-axis"),M.append("g").attr("class","nv-scatterWrap"),M.append("g").attr("class","nv-regressionLinesWrap"),M.append("g").attr("class","nv-distWrap"),M.append("g").attr("class","nv-legendWrap"),v&&N.select(".nv-y.nv-axis").attr("transform","translate("+H+",0)"),s){var O=H;f.width(O),K.select(".nv-legendWrap").datum(z).call(f),f.height()>j.top&&(j.top=f.height(),I=a.utils.availableHeight(l,m,j)),K.select(".nv-legendWrap").attr("transform","translate(0,"+-j.top+")")}else N.select(".nv-legendWrap").selectAll("*").remove();K.attr("transform","translate("+j.left+","+j.top+")"),c.width(H).height(I).color(z.map(function(a,b){return a.color=a.color||n(a,b),a.color}).filter(function(a,b){return!z[b].disabled})).showLabels(B),K.select(".nv-scatterWrap").datum(z.filter(function(a){return!a.disabled})).call(c),K.select(".nv-regressionLinesWrap").attr("clip-path","url(#nv-edge-clip-"+c.id()+")");var P=K.select(".nv-regressionLinesWrap").selectAll(".nv-regLines").data(function(a){return a});P.enter().append("g").attr("class","nv-regLines");var Q=P.selectAll(".nv-regLine").data(function(a){return[a]});Q.enter().append("line").attr("class","nv-regLine").style("stroke-opacity",0),Q.filter(function(a){return a.intercept&&a.slope}).watchTransition(E,"scatterPlusLineChart: regline").attr("x1",o.range()[0]).attr("x2",o.range()[1]).attr("y1",function(a,b){return p(o.domain()[0]*a.slope+a.intercept)}).attr("y2",function(a,b){return p(o.domain()[1]*a.slope+a.intercept)}).style("stroke",function(a,b,c){return n(a,c)}).style("stroke-opacity",function(a,b){return a.disabled||"undefined"==typeof a.slope||"undefined"==typeof a.intercept?0:1}),t&&(d.scale(o)._ticks(a.utils.calcTicksX(H/100,z)).tickSize(-I,0),N.select(".nv-x.nv-axis").attr("transform","translate(0,"+p.range()[0]+")").call(d)),u&&(e.scale(p)._ticks(a.utils.calcTicksY(I/36,z)).tickSize(-H,0),N.select(".nv-y.nv-axis").call(e)),q&&(g.getData(c.x()).scale(o).width(H).color(z.map(function(a,b){return a.color||n(a,b)}).filter(function(a,b){return!z[b].disabled})),M.select(".nv-distWrap").append("g").attr("class","nv-distributionX"),N.select(".nv-distributionX").attr("transform","translate(0,"+p.range()[0]+")").datum(z.filter(function(a){return!a.disabled})).call(g)),r&&(h.getData(c.y()).scale(p).width(I).color(z.map(function(a,b){return a.color||n(a,b)}).filter(function(a,b){return!z[b].disabled})),M.select(".nv-distWrap").append("g").attr("class","nv-distributionY"),N.select(".nv-distributionY").attr("transform","translate("+(v?H:-h.size())+",0)").datum(z.filter(function(a){return!a.disabled})).call(h)),f.dispatch.on("stateChange",function(a){for(var c in a)w[c]=a[c];y.stateChange(w),b.update()}),y.on("changeState",function(a){"undefined"!=typeof a.disabled&&(z.forEach(function(b,c){b.disabled=a.disabled[c]}),w.disabled=a.disabled),b.update()}),c.dispatch.on("elementMouseout.tooltip",function(a){i.hidden(!0),m.select(".nv-chart-"+c.id()+" .nv-series-"+a.seriesIndex+" .nv-distx-"+a.pointIndex).attr("y1",0),m.select(".nv-chart-"+c.id()+" .nv-series-"+a.seriesIndex+" .nv-disty-"+a.pointIndex).attr("x2",h.size())}),c.dispatch.on("elementMouseover.tooltip",function(a){m.select(".nv-series-"+a.seriesIndex+" .nv-distx-"+a.pointIndex).attr("y1",a.relativePos[1]-I),m.select(".nv-series-"+a.seriesIndex+" .nv-disty-"+a.pointIndex).attr("x2",a.relativePos[0]+g.size()),i.data(a).hidden(!1)}),C=o.copy(),D=p.copy()}),E.renderEnd("scatter with line immediate"),b}var c=a.models.scatter(),d=a.models.axis(),e=a.models.axis(),f=a.models.legend(),g=a.models.distribution(),h=a.models.distribution(),i=a.models.tooltip(),j={top:30,right:20,bottom:50,left:75},k=null,l=null,m=null,n=a.utils.defaultColor(),o=c.xScale(),p=c.yScale(),q=!1,r=!1,s=!0,t=!0,u=!0,v=!1,w=a.utils.state(),x=null,y=d3.dispatch("stateChange","changeState","renderEnd"),z=null,A=250,B=!1;c.xScale(o).yScale(p),d.orient("bottom").tickPadding(10),e.orient(v?"right":"left").tickPadding(10),g.axis("x"),h.axis("y"),i.headerFormatter(function(a,b){return d.tickFormat()(a,b)}).valueFormatter(function(a,b){return e.tickFormat()(a,b)});var C,D,E=a.utils.renderWatch(y,A),F=function(a){return function(){return{active:a.map(function(a){return!a.disabled})}}},G=function(a){return function(b){void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};return b.dispatch=y,b.scatter=c,b.legend=f,b.xAxis=d,b.yAxis=e,b.distX=g,b.distY=h,b.tooltip=i,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},container:{get:function(){return m},set:function(a){m=a}},showDistX:{get:function(){return q},set:function(a){q=a}},showDistY:{get:function(){return r},set:function(a){r=a}},showLegend:{get:function(){return s},set:function(a){s=a}},showXAxis:{get:function(){return t},set:function(a){t=a}},showYAxis:{get:function(){return u},set:function(a){u=a}},defaultState:{get:function(){return x},set:function(a){x=a}},noData:{get:function(){return z},set:function(a){z=a}},duration:{get:function(){return A},set:function(a){A=a}},showLabels:{get:function(){return B},set:function(a){B=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},rightAlignYAxis:{get:function(){return v},set:function(a){v=a,e.orient(a?"right":"left")}},color:{get:function(){return n},set:function(b){n=a.utils.getColor(b),f.color(n),g.color(n),h.color(n)}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.sparkline=function(){"use strict";function b(k){return t.reset(),k.each(function(b){var k=h-g.left-g.right,s=i-g.top-g.bottom;j=d3.select(this),a.utils.initSVG(j),l.domain(c||d3.extent(b,n)).range(e||[0,k]),m.domain(d||d3.extent(b,o)).range(f||[s,0]);var t=j.selectAll("g.nv-wrap.nv-sparkline").data([b]),u=t.enter().append("g").attr("class","nvd3 nv-wrap nv-sparkline");u.append("g"),t.select("g");t.attr("transform","translate("+g.left+","+g.top+")");var v=t.selectAll("path").data(function(a){return[a]});v.enter().append("path"),v.exit().remove(),v.style("stroke",function(a,b){return a.color||p(a,b)}).attr("d",d3.svg.line().x(function(a,b){return l(n(a,b))}).y(function(a,b){return m(o(a,b))}));var w=t.selectAll("circle.nv-point").data(function(a){function b(b){if(-1!=b){var c=a[b];return c.pointIndex=b,c}return null}var c=a.map(function(a,b){return o(a,b)}),d=b(c.lastIndexOf(m.domain()[1])),e=b(c.indexOf(m.domain()[0])),f=b(c.length-1);return[q?e:null,q?d:null,r?f:null].filter(function(a){return null!=a})});w.enter().append("circle"),w.exit().remove(),w.attr("cx",function(a,b){return l(n(a,a.pointIndex))}).attr("cy",function(a,b){return m(o(a,a.pointIndex))}).attr("r",2).attr("class",function(a,b){return n(a,a.pointIndex)==l.domain()[1]?"nv-point nv-currentValue":o(a,a.pointIndex)==m.domain()[0]?"nv-point nv-minValue":"nv-point nv-maxValue"})}),t.renderEnd("sparkline immediate"),b}var c,d,e,f,g={top:2,right:0,bottom:2,left:0},h=400,i=32,j=null,k=!0,l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=a.utils.getColor(["#000"]),q=!0,r=!0,s=d3.dispatch("renderEnd"),t=a.utils.renderWatch(s);return b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return i},set:function(a){i=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return d},set:function(a){d=a}},xRange:{get:function(){return e},set:function(a){e=a}},yRange:{get:function(){return f},set:function(a){f=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},animate:{get:function(){return k},set:function(a){k=a}},showMinMaxPoints:{get:function(){return q},set:function(a){q=a}},showCurrentPoint:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return n},set:function(a){n=d3.functor(a)}},y:{get:function(){return o},set:function(a){o=d3.functor(a)}},margin:{get:function(){return g},set:function(a){g.top=void 0!==a.top?a.top:g.top,g.right=void 0!==a.right?a.right:g.right,g.bottom=void 0!==a.bottom?a.bottom:g.bottom,g.left=void 0!==a.left?a.left:g.left}},color:{get:function(){return p},set:function(b){p=a.utils.getColor(b)}}}),b.dispatch=s,a.utils.initOptions(b),b},a.models.sparklinePlus=function(){"use strict";function b(p){return r.reset(),r.models(e),p.each(function(p){function q(){if(!j){var a=z.selectAll(".nv-hoverValue").data(i),b=a.enter().append("g").attr("class","nv-hoverValue").style("stroke-opacity",0).style("fill-opacity",0);a.exit().transition().duration(250).style("stroke-opacity",0).style("fill-opacity",0).remove(),a.attr("transform",function(a){return"translate("+c(e.x()(p[a],a))+",0)"}).transition().duration(250).style("stroke-opacity",1).style("fill-opacity",1),i.length&&(b.append("line").attr("x1",0).attr("y1",-f.top).attr("x2",0).attr("y2",u),b.append("text").attr("class","nv-xValue").attr("x",-6).attr("y",-f.top).attr("text-anchor","end").attr("dy",".9em"),z.select(".nv-hoverValue .nv-xValue").text(k(e.x()(p[i[0]],i[0]))),b.append("text").attr("class","nv-yValue").attr("x",6).attr("y",-f.top).attr("text-anchor","start").attr("dy",".9em"),z.select(".nv-hoverValue .nv-yValue").text(l(e.y()(p[i[0]],i[0]))))}}function r(){function a(a,b){for(var c=Math.abs(e.x()(a[0],0)-b),d=0,f=0;f<a.length;f++)Math.abs(e.x()(a[f],f)-b)<c&&(c=Math.abs(e.x()(a[f],f)-b),d=f);return d}if(!j){var b=d3.mouse(this)[0]-f.left;i=[a(p,Math.round(c.invert(b)))],q()}}var s=d3.select(this);a.utils.initSVG(s);var t=a.utils.availableWidth(g,s,f),u=a.utils.availableHeight(h,s,f);if(b.update=function(){s.call(b)},b.container=this,!p||!p.length)return a.utils.noData(b,s),b;s.selectAll(".nv-noData").remove();var v=e.y()(p[p.length-1],p.length-1);c=e.xScale(),d=e.yScale();var w=s.selectAll("g.nv-wrap.nv-sparklineplus").data([p]),x=w.enter().append("g").attr("class","nvd3 nv-wrap nv-sparklineplus"),y=x.append("g"),z=w.select("g");y.append("g").attr("class","nv-sparklineWrap"),y.append("g").attr("class","nv-valueWrap"),y.append("g").attr("class","nv-hoverArea"),w.attr("transform","translate("+f.left+","+f.top+")");var A=z.select(".nv-sparklineWrap");if(e.width(t).height(u),A.call(e),m){var B=z.select(".nv-valueWrap"),C=B.selectAll(".nv-currentValue").data([v]);C.enter().append("text").attr("class","nv-currentValue").attr("dx",o?-8:8).attr("dy",".9em").style("text-anchor",o?"end":"start"),C.attr("x",t+(o?f.right:0)).attr("y",n?function(a){return d(a)}:0).style("fill",e.color()(p[p.length-1],p.length-1)).text(l(v))}y.select(".nv-hoverArea").append("rect").on("mousemove",r).on("click",function(){j=!j}).on("mouseout",function(){i=[],q()}),z.select(".nv-hoverArea rect").attr("transform",function(a){return"translate("+-f.left+","+-f.top+")"}).attr("width",t+f.left+f.right).attr("height",u+f.top)}),r.renderEnd("sparklinePlus immediate"),b}var c,d,e=a.models.sparkline(),f={top:15,right:100,bottom:10,left:50},g=null,h=null,i=[],j=!1,k=d3.format(",r"),l=d3.format(",.2f"),m=!0,n=!0,o=!1,p=null,q=d3.dispatch("renderEnd"),r=a.utils.renderWatch(q);return b.dispatch=q,b.sparkline=e,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return g},set:function(a){g=a}},height:{get:function(){return h},set:function(a){h=a}},xTickFormat:{get:function(){return k},set:function(a){k=a}},yTickFormat:{get:function(){return l},set:function(a){l=a}},showLastValue:{get:function(){return m},set:function(a){m=a}},alignValue:{get:function(){return n},set:function(a){n=a}},rightAlignValue:{get:function(){return o},set:function(a){o=a}},noData:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return f},set:function(a){f.top=void 0!==a.top?a.top:f.top,f.right=void 0!==a.right?a.right:f.right,f.bottom=void 0!==a.bottom?a.bottom:f.bottom,f.left=void 0!==a.left?a.left:f.left}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.stackedArea=function(){"use strict";function b(n){return v.reset(),v.models(s),n.each(function(n){var t=f-e.left-e.right,w=g-e.top-e.bottom;j=d3.select(this),a.utils.initSVG(j),c=s.xScale(),d=s.yScale();var x=n;n.forEach(function(a,b){a.seriesIndex=b,a.values=a.values.map(function(a,c){return a.index=c,a.seriesIndex=b,a})});var y=n.filter(function(a){return!a.disabled});n=d3.layout.stack().order(p).offset(o).values(function(a){return a.values}).x(k).y(l).out(function(a,b,c){a.display={y:c,y0:b}})(y);var z=j.selectAll("g.nv-wrap.nv-stackedarea").data([n]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedarea"),B=A.append("defs"),C=A.append("g"),D=z.select("g");C.append("g").attr("class","nv-areaWrap"),C.append("g").attr("class","nv-scatterWrap"),z.attr("transform","translate("+e.left+","+e.top+")"),0==s.forceY().length&&s.forceY().push(0),s.width(t).height(w).x(k).y(function(a){return void 0!==a.display?a.display.y+a.display.y0:void 0}).color(n.map(function(a,b){return a.color=a.color||h(a,a.seriesIndex),a.color}));var E=D.select(".nv-scatterWrap").datum(n);E.call(s),B.append("clipPath").attr("id","nv-edge-clip-"+i).append("rect"),z.select("#nv-edge-clip-"+i+" rect").attr("width",t).attr("height",w),D.attr("clip-path",r?"url(#nv-edge-clip-"+i+")":"");var F=d3.svg.area().defined(m).x(function(a,b){return c(k(a,b))}).y0(function(a){return d(a.display.y0)}).y1(function(a){return d(a.display.y+a.display.y0)}).interpolate(q),G=d3.svg.area().defined(m).x(function(a,b){return c(k(a,b))}).y0(function(a){return d(a.display.y0)}).y1(function(a){return d(a.display.y0)}),H=D.select(".nv-areaWrap").selectAll("path.nv-area").data(function(a){return a});H.enter().append("path").attr("class",function(a,b){return"nv-area nv-area-"+b}).attr("d",function(a,b){return G(a.values,a.seriesIndex)}).on("mouseover",function(a,b){d3.select(this).classed("hover",!0),u.areaMouseover({point:a,series:a.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:a.seriesIndex})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),u.areaMouseout({point:a,series:a.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:a.seriesIndex})}).on("click",function(a,b){d3.select(this).classed("hover",!1),u.areaClick({point:a,series:a.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:a.seriesIndex})}),H.exit().remove(),H.style("fill",function(a,b){return a.color||h(a,a.seriesIndex)}).style("stroke",function(a,b){return a.color||h(a,a.seriesIndex)}),H.watchTransition(v,"stackedArea path").attr("d",function(a,b){return F(a.values,b)}),s.dispatch.on("elementMouseover.area",function(a){D.select(".nv-chart-"+i+" .nv-area-"+a.seriesIndex).classed("hover",!0)}),s.dispatch.on("elementMouseout.area",function(a){D.select(".nv-chart-"+i+" .nv-area-"+a.seriesIndex).classed("hover",!1)}),b.d3_stackedOffset_stackPercent=function(a){var b,c,d,e=a.length,f=a[0].length,g=[];for(c=0;f>c;++c){for(b=0,d=0;b<x.length;b++)d+=l(x[b].values[c]);if(d)for(b=0;e>b;b++)a[b][c][1]/=d;else for(b=0;e>b;b++)a[b][c][1]=0}for(c=0;f>c;++c)g[c]=0;return g}}),v.renderEnd("stackedArea immediate"),b}var c,d,e={top:0,right:0,bottom:0,left:0},f=960,g=500,h=a.utils.defaultColor(),i=Math.floor(1e5*Math.random()),j=null,k=function(a){return a.x},l=function(a){return a.y},m=function(a,b){return!isNaN(l(a,b))&&null!==l(a,b)},n="stack",o="zero",p="default",q="linear",r=!1,s=a.models.scatter(),t=250,u=d3.dispatch("areaClick","areaMouseover","areaMouseout","renderEnd","elementClick","elementMouseover","elementMouseout");s.pointSize(2.2).pointDomain([2.2,2.2]);var v=a.utils.renderWatch(u,t);return b.dispatch=u,b.scatter=s,s.dispatch.on("elementClick",function(){u.elementClick.apply(this,arguments)}),s.dispatch.on("elementMouseover",function(){u.elementMouseover.apply(this,arguments)}),s.dispatch.on("elementMouseout",function(){u.elementMouseout.apply(this,arguments)}),b.interpolate=function(a){return arguments.length?(q=a,b):q},b.duration=function(a){return arguments.length?(t=a,v.reset(t),s.duration(t),b):t},b.dispatch=u,b.scatter=s,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return f},set:function(a){f=a}},height:{get:function(){return g},set:function(a){g=a}},defined:{get:function(){return m},set:function(a){m=a}},clipEdge:{get:function(){return r},set:function(a){r=a}},offset:{get:function(){return o},set:function(a){o=a}},order:{get:function(){return p},set:function(a){p=a}},interpolate:{get:function(){return q},set:function(a){q=a}},x:{get:function(){return k},set:function(a){k=d3.functor(a)}},y:{get:function(){return l},set:function(a){l=d3.functor(a)}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}},style:{get:function(){return n},set:function(a){switch(n=a){case"stack":b.offset("zero"),b.order("default");break;case"stream":b.offset("wiggle"),b.order("inside-out");break;case"stream-center":b.offset("silhouette"),b.order("inside-out");break;case"expand":b.offset("expand"),b.order("default");break;case"stack_percent":b.offset(b.d3_stackedOffset_stackPercent),b.order("default")}}},duration:{get:function(){return t},set:function(a){t=a,v.reset(t),s.duration(t)}}}),a.utils.inheritOptions(b,s),a.utils.initOptions(b),b},a.models.stackedAreaChart=function(){"use strict";function b(k){return J.reset(),J.models(e),s&&J.models(f),t&&J.models(g),k.each(function(k){function B(){s&&V.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+R+")").transition().duration(G).call(f)}function J(){if(t){if("expand"===e.style()||"stack_percent"===e.style()){var a=g.tickFormat();H&&a===N||(H=a),g.tickFormat(N)}else H&&(g.tickFormat(H),H=null);V.select(".nv-focus .nv-y.nv-axis").transition().duration(0).call(g)}}function O(a){var b=V.select(".nv-focus .nv-stackedWrap").datum(k.filter(function(a){return!a.disabled}).map(function(b,c){return{key:b.key,area:b.area,classed:b.classed,values:b.values.filter(function(b,c){return e.x()(b,c)>=a[0]&&e.x()(b,c)<=a[1]}),disableTooltip:b.disableTooltip}}));b.transition().duration(G).call(e),B(),J()}var P=d3.select(this);a.utils.initSVG(P);var Q=a.utils.availableWidth(n,P,m),R=a.utils.availableHeight(o,P,m)-(v?l.height():0);if(b.update=function(){P.transition().duration(G).call(b)},b.container=this,z.setter(M(k),b.update).getter(L(k)).update(),z.disabled=k.map(function(a){return!!a.disabled}),!A){var S;A={};for(S in z)z[S]instanceof Array?A[S]=z[S].slice(0):A[S]=z[S]}if(!(k&&k.length&&k.filter(function(a){return a.values.length}).length))return a.utils.noData(b,P),b;P.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale();var T=P.selectAll("g.nv-wrap.nv-stackedAreaChart").data([k]),U=T.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedAreaChart").append("g"),V=T.select("g");U.append("g").attr("class","nv-legendWrap"),U.append("g").attr("class","nv-controlsWrap");var W=U.append("g").attr("class","nv-focus");W.append("g").attr("class","nv-background").append("rect"),W.append("g").attr("class","nv-x nv-axis"),W.append("g").attr("class","nv-y nv-axis"),W.append("g").attr("class","nv-stackedWrap"),W.append("g").attr("class","nv-interactive");U.append("g").attr("class","nv-focusWrap");if(r){var X=q?Q-D:Q;h.width(X),V.select(".nv-legendWrap").datum(k).call(h),h.height()>m.top&&(m.top=h.height(),R=a.utils.availableHeight(o,P,m)-(v?l.height():0)),V.select(".nv-legendWrap").attr("transform","translate("+(Q-X)+","+-m.top+")")}else V.select(".nv-legendWrap").selectAll("*").remove();if(q){var Y=[{key:F.stacked||"Stacked",metaKey:"Stacked",disabled:"stack"!=e.style(),style:"stack"},{key:F.stream||"Stream",metaKey:"Stream",disabled:"stream"!=e.style(),style:"stream"},{key:F.expanded||"Expanded",metaKey:"Expanded",disabled:"expand"!=e.style(),style:"expand"},{key:F.stack_percent||"Stack %",metaKey:"Stack_Percent",disabled:"stack_percent"!=e.style(),style:"stack_percent"}];D=E.length/3*260,Y=Y.filter(function(a){return-1!==E.indexOf(a.metaKey)}),i.width(D).color(["#444","#444","#444"]),V.select(".nv-controlsWrap").datum(Y).call(i),Math.max(i.height(),h.height())>m.top&&(m.top=Math.max(i.height(),h.height()),R=a.utils.availableHeight(o,P,m)),V.select(".nv-controlsWrap").attr("transform","translate(0,"+-m.top+")")}else V.select(".nv-controlsWrap").selectAll("*").remove();T.attr("transform","translate("+m.left+","+m.top+")"),u&&V.select(".nv-y.nv-axis").attr("transform","translate("+Q+",0)"),w&&(j.width(Q).height(R).margin({left:m.left,top:m.top}).svgContainer(P).xScale(c),T.select(".nv-interactive").call(j)),V.select(".nv-focus .nv-background rect").attr("width",Q).attr("height",R),e.width(Q).height(R).color(k.map(function(a,b){return a.color||p(a,b)}).filter(function(a,b){return!k[b].disabled}));var Z=V.select(".nv-focus .nv-stackedWrap").datum(k.filter(function(a){return!a.disabled}));if(s&&f.scale(c)._ticks(a.utils.calcTicksX(Q/100,k)).tickSize(-R,0),t){var $;$="wiggle"===e.offset()?0:a.utils.calcTicksY(R/36,k),g.scale(d)._ticks($).tickSize(-Q,0)}if(v){l.width(Q),V.select(".nv-focusWrap").attr("transform","translate(0,"+(R+m.bottom+l.margin().top)+")").datum(k.filter(function(a){return!a.disabled})).call(l);var _=l.brush.empty()?l.xDomain():l.brush.extent();null!==_&&O(_)}else Z.transition().call(e),B(),J();e.dispatch.on("areaClick.toggle",function(a){1===k.filter(function(a){return!a.disabled}).length?k.forEach(function(a){a.disabled=!1}):k.forEach(function(b,c){b.disabled=c!=a.seriesIndex}),z.disabled=k.map(function(a){return!!a.disabled}),C.stateChange(z),b.update()}),h.dispatch.on("stateChange",function(a){for(var c in a)z[c]=a[c];C.stateChange(z),b.update()}),i.dispatch.on("legendClick",function(a,c){a.disabled&&(Y=Y.map(function(a){return a.disabled=!0,a}),a.disabled=!1,e.style(a.style),z.style=e.style(),C.stateChange(z),b.update())}),j.dispatch.on("elementMousemove",function(c){e.clearHighlights();var d,f,g,h=[],i=0,l=!0;if(k.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(j,k){f=a.interactiveBisect(j.values,c.pointXValue,b.x());var m=j.values[f],n=b.y()(m,f);if(null!=n&&e.highlightPoint(k,f,!0),"undefined"!=typeof m){"undefined"==typeof d&&(d=m),"undefined"==typeof g&&(g=b.xScale()(b.x()(m,f)));var o="expand"==e.style()?m.display.y:b.y()(m,f);h.push({key:j.key,value:o,color:p(j,j.seriesIndex),point:m}),x&&"expand"!=e.style()&&null!=o&&(i+=o,l=!1)}}),h.reverse(),h.length>2){var m=b.yScale().invert(c.mouseY),n=null;h.forEach(function(a,b){m=Math.abs(m);var c=Math.abs(a.point.display.y0),d=Math.abs(a.point.display.y);return m>=c&&d+c>=m?void(n=b):void 0}),null!=n&&(h[n].highlight=!0)}x&&"expand"!=e.style()&&h.length>=2&&!l&&h.push({key:y,value:i,total:!0});var o=b.x()(d,f),q=j.tooltip.valueFormatter();"expand"===e.style()||"stack_percent"===e.style()?(I||(I=q),q=d3.format(".1%")):I&&(q=I,I=null),j.tooltip.valueFormatter(q).data({value:o,series:h})(),j.renderGuideLine(g)}),j.dispatch.on("elementMouseout",function(a){e.clearHighlights()}),l.dispatch.on("onBrush",function(a){O(a)}),C.on("changeState",function(a){"undefined"!=typeof a.disabled&&k.length===a.disabled.length&&(k.forEach(function(b,c){b.disabled=a.disabled[c]}),z.disabled=a.disabled),"undefined"!=typeof a.style&&(e.style(a.style),K=a.style),b.update()})}),J.renderEnd("stacked Area chart immediate"),b}var c,d,e=a.models.stackedArea(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend(),i=a.models.legend(),j=a.interactiveGuideline(),k=a.models.tooltip(),l=a.models.focus(a.models.stackedArea()),m={top:30,right:25,bottom:50,left:60},n=null,o=null,p=a.utils.defaultColor(),q=!0,r=!0,s=!0,t=!0,u=!1,v=!1,w=!1,x=!0,y="TOTAL",z=a.utils.state(),A=null,B=null,C=d3.dispatch("stateChange","changeState","renderEnd"),D=250,E=["Stacked","Stream","Expanded"],F={},G=250;z.style=e.style(),f.orient("bottom").tickPadding(7),g.orient(u?"right":"left"),k.headerFormatter(function(a,b){return f.tickFormat()(a,b)}).valueFormatter(function(a,b){return g.tickFormat()(a,b)}),j.tooltip.headerFormatter(function(a,b){return f.tickFormat()(a,b)}).valueFormatter(function(a,b){return null==a?"N/A":g.tickFormat()(a,b)});var H=null,I=null;i.updateState(!1);var J=a.utils.renderWatch(C),K=e.style(),L=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),style:e.style()}}},M=function(a){return function(b){void 0!==b.style&&(K=b.style),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}},N=d3.format("%");return e.dispatch.on("elementMouseover.tooltip",function(a){a.point.x=e.x()(a.point),a.point.y=e.y()(a.point),k.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){k.hidden(!0)}),b.dispatch=C,b.stacked=e,b.legend=h,b.controls=i,b.xAxis=f,b.x2Axis=l.xAxis,b.yAxis=g,b.y2Axis=l.yAxis,b.interactiveLayer=j,b.tooltip=k,b.focus=l,b.dispatch=C,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return n},set:function(a){n=a}},height:{get:function(){return o},set:function(a){o=a}},showLegend:{get:function(){return r},set:function(a){r=a}},showXAxis:{get:function(){return s},set:function(a){s=a}},showYAxis:{get:function(){return t},set:function(a){t=a}},defaultState:{get:function(){return A},set:function(a){A=a}},noData:{get:function(){return B},set:function(a){B=a}},showControls:{get:function(){return q},set:function(a){q=a}},controlLabels:{get:function(){return F},set:function(a){F=a}},controlOptions:{get:function(){return E},set:function(a){E=a}},showTotalInTooltip:{get:function(){return x},set:function(a){x=a}},totalLabel:{get:function(){return y},set:function(a){y=a}},focusEnable:{get:function(){return v},set:function(a){v=a}},focusHeight:{get:function(){return l.height()},set:function(a){l.height(a)}},brushExtent:{get:function(){return l.brushExtent()},set:function(a){l.brushExtent(a)}},margin:{get:function(){return m},set:function(a){m.top=void 0!==a.top?a.top:m.top,m.right=void 0!==a.right?a.right:m.right,m.bottom=void 0!==a.bottom?a.bottom:m.bottom,m.left=void 0!==a.left?a.left:m.left}},focusMargin:{get:function(){return l.margin},set:function(a){l.margin.top=void 0!==a.top?a.top:l.margin.top,l.margin.right=void 0!==a.right?a.right:l.margin.right,l.margin.bottom=void 0!==a.bottom?a.bottom:l.margin.bottom,l.margin.left=void 0!==a.left?a.left:l.margin.left}},duration:{get:function(){return G},set:function(a){G=a,J.reset(G),e.duration(G),f.duration(G),g.duration(G)}},color:{get:function(){return p},set:function(b){p=a.utils.getColor(b),h.color(p),e.color(p),l.color(p)}},x:{get:function(){return e.x()},set:function(a){e.x(a),l.x(a)}},y:{get:function(){return e.y()},set:function(a){e.y(a),l.y(a)}},rightAlignYAxis:{get:function(){return u},set:function(a){u=a,g.orient(u?"right":"left")}},useInteractiveGuideline:{get:function(){return w},set:function(a){w=!!a,b.interactive(!a),b.useVoronoi(!a),e.scatter.interactive(!a)}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.stackedAreaWithFocusChart=function(){return a.models.stackedAreaChart().margin({bottom:30}).focusEnable(!0)},a.models.sunburst=function(){"use strict";function b(a){var b=c(a);return b>90?180:0}function c(a){var b=Math.max(0,Math.min(2*Math.PI,F(a.x))),c=Math.max(0,Math.min(2*Math.PI,F(a.x+a.dx))),d=(b+c)/2*(180/Math.PI)-90;return d}function d(a){var b=Math.max(0,Math.min(2*Math.PI,F(a.x))),c=Math.max(0,Math.min(2*Math.PI,F(a.x+a.dx)));return(c-b)/(2*Math.PI)}function e(a){var b=Math.max(0,Math.min(2*Math.PI,F(a.x))),c=Math.max(0,Math.min(2*Math.PI,F(a.x+a.dx))),d=c-b;return d>z}function f(a,b){var c=d3.interpolate(F.domain(),[l.x,l.x+l.dx]),d=d3.interpolate(G.domain(),[l.y,1]),e=d3.interpolate(G.range(),[l.y?20:0,o]);return 0===b?function(){return J(a)}:function(b){return F.domain(c(b)),G.domain(d(b)).range(e(b)),J(a)}}function g(a){var b=d3.interpolate({x:a.x0,dx:a.dx0,y:a.y0,dy:a.dy0},a);return function(c){var d=b(c);return a.x0=d.x,a.dx0=d.dx,a.y0=d.y,a.dy0=d.dy,J(d)}}function h(a){var b=B(a);I[b]||(I[b]={});var c=I[b];c.dx=a.dx,c.x=a.x,c.dy=a.dy,c.y=a.y}function i(a){a.forEach(function(a){var b=B(a),c=I[b];c?(a.dx0=c.dx,a.x0=c.x,a.dy0=c.dy,a.y0=c.y):(a.dx0=a.dx,a.x0=a.x,a.dy0=a.dy,a.y0=a.y),h(a)})}function j(a){var d=v.selectAll("text"),g=v.selectAll("path");d.transition().attr("opacity",0),l=a,g.transition().duration(D).attrTween("d",f).each("end",function(d){if(d.x>=a.x&&d.x<a.x+a.dx&&d.depth>=a.depth){var f=d3.select(this.parentNode),g=f.select("text");g.transition().duration(D).text(function(a){return y(a)}).attr("opacity",function(a){return e(a)?1:0}).attr("transform",function(){var e=this.getBBox().width;if(0===d.depth)return"translate("+e/2*-1+",0)";if(d.depth===a.depth)return"translate("+(G(d.y)+5)+",0)";var f=c(d),g=b(d);return 0===g?"rotate("+f+")translate("+(G(d.y)+5)+",0)":"rotate("+f+")translate("+(G(d.y)+e+5)+",0)rotate("+g+")"})}})}function k(f){return K.reset(),f.each(function(f){v=d3.select(this),m=a.utils.availableWidth(q,v,p),n=a.utils.availableHeight(r,v,p),o=Math.min(m,n)/2,G.range([0,o]);var h=v.select("g.nvd3.nv-wrap.nv-sunburst");h[0][0]?h.attr("transform","translate("+(m/2+p.left+p.right)+","+(n/2+p.top+p.bottom)+")"):h=v.append("g").attr("class","nvd3 nv-wrap nv-sunburst nv-chart-"+u).attr("transform","translate("+(m/2+p.left+p.right)+","+(n/2+p.top+p.bottom)+")"),v.on("click",function(a,b){E.chartClick({data:a,index:b,pos:d3.event,id:u})}),H.value(t[s]||t.count);var k=H.nodes(f[0]).reverse();i(k);var l=h.selectAll(".arc-container").data(k,B),z=l.enter().append("g").attr("class","arc-container");z.append("path").attr("d",J).style("fill",function(a){return a.color?a.color:w(C?(a.children?a:a.parent).name:a.name)}).style("stroke","#FFF").on("click",j).on("mouseover",function(a,b){d3.select(this).classed("hover",!0).style("opacity",.8),E.elementMouseover({data:a,color:d3.select(this).style("fill"),percent:d(a)})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1).style("opacity",1),E.elementMouseout({data:a})}).on("mousemove",function(a,b){E.elementMousemove({data:a})}),l.each(function(a){d3.select(this).select("path").transition().duration(D).attrTween("d",g)}),x&&(l.selectAll("text").remove(),l.append("text").text(function(a){return y(a)}).transition().duration(D).attr("opacity",function(a){return e(a)?1:0}).attr("transform",function(a){var d=this.getBBox().width;if(0===a.depth)return"rotate(0)translate("+d/2*-1+",0)";var e=c(a),f=b(a);return 0===f?"rotate("+e+")translate("+(G(a.y)+5)+",0)":"rotate("+e+")translate("+(G(a.y)+d+5)+",0)rotate("+f+")"})),j(k[k.length-1]),l.exit().transition().duration(D).attr("opacity",0).each("end",function(a){var b=B(a);I[b]=void 0}).remove()}),K.renderEnd("sunburst immediate"),k}var l,m,n,o,p={top:0,right:0,bottom:0,left:0},q=600,r=600,s="count",t={count:function(a){return 1},value:function(a){return a.value||a.size},size:function(a){return a.value||a.size}},u=Math.floor(1e4*Math.random()),v=null,w=a.utils.defaultColor(),x=!1,y=function(a){return"count"===s?a.name+" #"+a.value:a.name+" "+(a.value||a.size)},z=.02,A=function(a,b){return a.name>b.name},B=function(a,b){return a.name},C=!0,D=500,E=d3.dispatch("chartClick","elementClick","elementDblClick","elementMousemove","elementMouseover","elementMouseout","renderEnd"),F=d3.scale.linear().range([0,2*Math.PI]),G=d3.scale.sqrt(),H=d3.layout.partition().sort(A),I={},J=d3.svg.arc().startAngle(function(a){return Math.max(0,Math.min(2*Math.PI,F(a.x)))}).endAngle(function(a){return Math.max(0,Math.min(2*Math.PI,F(a.x+a.dx)))}).innerRadius(function(a){return Math.max(0,G(a.y))}).outerRadius(function(a){return Math.max(0,G(a.y+a.dy))}),K=a.utils.renderWatch(E);return k.dispatch=E,k.options=a.utils.optionsFunc.bind(k),k._options=Object.create({},{width:{get:function(){return q},set:function(a){q=a}},height:{get:function(){return r},set:function(a){r=a}},mode:{get:function(){return s},set:function(a){s=a}},id:{get:function(){return u},set:function(a){u=a}},duration:{get:function(){return D},set:function(a){D=a}},groupColorByParent:{get:function(){return C},set:function(a){C=!!a}},showLabels:{get:function(){return x},set:function(a){x=!!a}},labelFormat:{get:function(){return y},set:function(a){y=a}},labelThreshold:{get:function(){return z},set:function(a){z=a}},sort:{get:function(){return A},set:function(a){A=a}},key:{get:function(){return B},set:function(a){B=a}},margin:{get:function(){return p},set:function(a){p.top=void 0!=a.top?a.top:p.top,p.right=void 0!=a.right?a.right:p.right,p.bottom=void 0!=a.bottom?a.bottom:p.bottom,p.left=void 0!=a.left?a.left:p.left}},color:{get:function(){return w},set:function(b){w=a.utils.getColor(b)}}}),a.utils.initOptions(k),k},a.models.sunburstChart=function(){"use strict";function b(d){return n.reset(),n.models(c),d.each(function(d){var h=d3.select(this);a.utils.initSVG(h);var i=a.utils.availableWidth(f,h,e),j=a.utils.availableHeight(g,h,e);return b.update=function(){0===l?h.call(b):h.transition().duration(l).call(b)},b.container=h,d&&d.length?(h.selectAll(".nv-noData").remove(),c.width(i).height(j).margin(e),void h.call(c)):(a.utils.noData(b,h),b)}),n.renderEnd("sunburstChart immediate"),b}var c=a.models.sunburst(),d=a.models.tooltip(),e={top:30,right:20,bottom:20,left:20},f=null,g=null,h=a.utils.defaultColor(),i=!1,j=(Math.round(1e5*Math.random()),null),k=null,l=250,m=d3.dispatch("stateChange","changeState","renderEnd"),n=a.utils.renderWatch(m);return d.duration(0).headerEnabled(!1).valueFormatter(function(a){return a}),c.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:a.data.name,value:a.data.value||a.data.size,color:a.color,percent:a.percent},i||(delete a.percent,delete a.series.percent),d.data(a).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",function(a){d.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(a){d()}),b.dispatch=m,b.sunburst=c,b.tooltip=d,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{noData:{get:function(){return k},set:function(a){k=a}},defaultState:{get:function(){return j},set:function(a){j=a}},showTooltipPercent:{get:function(){return i},set:function(a){i=a}},color:{get:function(){return h},set:function(a){h=a,c.color(h)}},duration:{get:function(){return l},set:function(a){l=a,n.reset(l),c.duration(l)}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left,c.margin(e)}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.version="1.8.4-dev"}(); \ No newline at end of file
diff --git a/bench/perf_monitoring/run.sh b/bench/perf_monitoring/run.sh new file mode 100755 index 0000000..3022adf --- /dev/null +++ b/bench/perf_monitoring/run.sh
@@ -0,0 +1,183 @@ +#!/bin/bash + +# ./run.sh gemm gemm_settings.txt +# ./run.sh lazy_gemm lazy_gemm_settings.txt +# ./run.sh gemv gemv_settings.txt +# ./run.sh trmv_up gemv_square_settings.txt +# ... + +# Examples of environment variables to be set: +# PREFIX="haswell-fma-" +# CXX_FLAGS="-mfma" +# CXX=clang++ + +# Options: +# -up : enforce the recomputation of existing data, and keep best results as a merging strategy +# -s : recompute selected changesets only and keep bests +# -np : no plotting of results, just generate the data + +bench=$1 +settings_file=$2 + +if [[ "$*" =~ '-up' ]]; then + update=true +else + update=false +fi + +if [[ "$*" =~ '-s' ]]; then + selected=true +else + selected=false +fi + +if [[ "$*" =~ '-np' ]]; then + do_plot=false +else + do_plot=true +fi + + +WORKING_DIR=${PREFIX:?"default"} + +if [ -z "$PREFIX" ]; then + WORKING_DIR_PREFIX="$WORKING_DIR/" +else + WORKING_DIR_PREFIX="$WORKING_DIR/$PREFIX-" +fi +echo "WORKING_DIR_PREFIX=$WORKING_DIR_PREFIX" +mkdir -p $WORKING_DIR + +global_args="$*" + +if $selected ; then + echo "Recompute selected changesets only and keep bests" +elif $update ; then + echo "(Re-)Compute all changesets and keep bests" +else + echo "Skip previously computed changesets" +fi + + + +if [ ! -d "eigen_src" ]; then + hg clone https://bitbucket.org/eigen/eigen eigen_src +else + cd eigen_src + hg pull -u + cd .. +fi + +if [ -z "$CXX" ]; then + CXX=g++ +fi + +function make_backup +{ + if [ -f "$1.out" ]; then + mv "$1.out" "$1.backup" + fi +} + +function merge +{ + count1=`echo $1 | wc -w` + count2=`echo $2 | wc -w` + + if [ $count1 == $count2 ]; then + a=( $1 ); b=( $2 ) + res="" + for (( i=0 ; i<$count1 ; i++ )); do + ai=${a[$i]}; bi=${b[$i]} + tmp=`echo "if ($ai > $bi) $ai else $bi " | bc -l` + res="$res $tmp" + done + echo $res + + else + echo $1 + fi +} + +function test_current +{ + rev=$1 + scalar=$2 + name=$3 + + prev="" + if [ -e "$name.backup" ]; then + prev=`grep $rev "$name.backup" | cut -c 14-` + fi + res=$prev + count_rev=`echo $prev | wc -w` + count_ref=`cat $settings_file | wc -l` + if echo "$global_args" | grep "$rev" > /dev/null; then + rev_found=true + else + rev_found=false + fi +# echo $update et $selected et $rev_found because $rev et "$global_args" +# echo $count_rev et $count_ref + if $update || [ $count_rev != $count_ref ] || ( $selected && $rev_found ); then + echo "RUN: $CXX -O3 -DNDEBUG -march=native $CXX_FLAGS -I eigen_src $bench.cpp -DSCALAR=$scalar -o $name" + if $CXX -O3 -DNDEBUG -march=native $CXX_FLAGS -I eigen_src $bench.cpp -DSCALAR=$scalar -o $name; then + curr=`./$name $settings_file` + if [ $count_rev == $count_ref ]; then + echo "merge previous $prev" + echo "with new $curr" + else + echo "got $curr" + fi + res=`merge "$curr" "$prev"` +# echo $res + echo "$rev $res" >> $name.out + else + echo "Compilation failed, skip rev $rev" + fi + else + echo "Skip existing results for $rev / $name" + echo "$rev $res" >> $name.out + fi +} + +make_backup $WORKING_DIR_PREFIX"s"$bench +make_backup $WORKING_DIR_PREFIX"d"$bench +make_backup $WORKING_DIR_PREFIX"c"$bench + +cut -f1 -d"#" < changesets.txt | grep -E '[[:alnum:]]' | while read rev +do + if [ ! -z '$rev' ]; then + rev2=`echo $rev | cut -f 2 -d':'` + echo "Testing rev $rev, $rev2" + cd eigen_src + hg up -C $rev2 > /dev/null + actual_rev=`hg identify | cut -f1 -d' '` + cd .. + + test_current $actual_rev float $WORKING_DIR_PREFIX"s"$bench + test_current $actual_rev double $WORKING_DIR_PREFIX"d"$bench + test_current $actual_rev "std::complex<double>" $WORKING_DIR_PREFIX"c"$bench + fi + +done + +echo "Float:" +cat $WORKING_DIR_PREFIX"s""$bench.out" +echo " " + +echo "Double:" +cat $WORKING_DIR_PREFIX"d""$bench.out" +echo "" + +echo "Complex:" +cat $WORKING_DIR_PREFIX"c""$bench.out" +echo "" + +if $do_plot ; then + +./make_plot.sh $WORKING_DIR_PREFIX"s"$bench $bench $settings_file +./make_plot.sh $WORKING_DIR_PREFIX"d"$bench $bench $settings_file +./make_plot.sh $WORKING_DIR_PREFIX"c"$bench $bench $settings_file + +fi
diff --git a/bench/perf_monitoring/runall.sh b/bench/perf_monitoring/runall.sh new file mode 100755 index 0000000..cdbe48e --- /dev/null +++ b/bench/perf_monitoring/runall.sh
@@ -0,0 +1,72 @@ +#!/bin/bash + +# ./runall.sh "Title" + +# Examples of environment variables to be set: +# PREFIX="haswell-fma-" +# CXX_FLAGS="-mfma" +# CXX=clang++ + +# Options: +# -up : enforce the recomputation of existing data, and keep best results as a merging strategy +# -s : recompute selected changesets only and keep bests +# -np : no plotting of results, just generate the data + +if [[ "$*" =~ '-np' ]]; then + do_plot=false +else + do_plot=true +fi + +./run.sh gemm gemm_settings.txt $* +./run.sh lazy_gemm lazy_gemm_settings.txt $* +./run.sh gemv gemv_settings.txt $* +./run.sh gemvt gemv_settings.txt $* +./run.sh trmv_up gemv_square_settings.txt $* +./run.sh trmv_lo gemv_square_settings.txt $* +./run.sh trmv_upt gemv_square_settings.txt $* +./run.sh trmv_lot gemv_square_settings.txt $* +./run.sh llt gemm_square_settings.txt $* + +if $do_plot ; then + +# generate html file + +function print_td { + echo '<td><a href="'$PREFIX'-'$1"$2"'.html"><img src="'$PREFIX'-'$1"$2"'.png" title="'$3'"></a></td>' >> $htmlfile +} + +function print_tr { + echo '<tr><th colspan="3">'"$2"'</th></tr>' >> $htmlfile + echo '<tr>' >> $htmlfile + print_td s $1 float + print_td d $1 double + print_td c $1 complex + echo '</tr>' >> $htmlfile +} + +if [ -n "$PREFIX" ]; then + + +cp resources/s1.js $PREFIX/ +cp resources/s2.js $PREFIX/ + +htmlfile="$PREFIX/index.html" +cat resources/header.html > $htmlfile + +echo '<h1>'$1'</h1>' >> $htmlfile +echo '<table>' >> $htmlfile +print_tr gemm 'C += A · B (gemm)' +print_tr lazy_gemm 'C += A · B (gemm lazy)' +print_tr gemv 'y += A · x (gemv)' +print_tr gemvt 'y += A<sup>T</sup> · x (gemv)' +print_tr trmv_up 'y += U · x (trmv)' +print_tr trmv_upt 'y += U<sup>T</sup> · x (trmv)' +print_tr trmv_lo 'y += L · x (trmv)' +print_tr trmv_lot 'y += L<sup>T</sup> · x (trmv)' +print_tr trmv_lot 'L · L<sup>T<sup> = A (Cholesky,potrf)' + +cat resources/footer.html >> $htmlfile + +fi +fi
diff --git a/bench/perf_monitoring/trmv_lo.cpp b/bench/perf_monitoring/trmv_lo.cpp new file mode 100644 index 0000000..3fabb6e --- /dev/null +++ b/bench/perf_monitoring/trmv_lo.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void trmv(const Mat &A, const Vec &B, Vec &C) +{ + C.noalias() += A.triangularView<Lower>() * B; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, trmv); +}
diff --git a/bench/perf_monitoring/trmv_lot.cpp b/bench/perf_monitoring/trmv_lot.cpp new file mode 100644 index 0000000..32e085a --- /dev/null +++ b/bench/perf_monitoring/trmv_lot.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void trmv(const Mat &A, Vec &B, const Vec &C) +{ + B.noalias() += A.transpose().triangularView<Lower>() * C; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, trmv); +}
diff --git a/bench/perf_monitoring/trmv_up.cpp b/bench/perf_monitoring/trmv_up.cpp new file mode 100644 index 0000000..c58e471 --- /dev/null +++ b/bench/perf_monitoring/trmv_up.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void trmv(const Mat &A, const Vec &B, Vec &C) +{ + C.noalias() += A.triangularView<Upper>() * B; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, trmv); +}
diff --git a/bench/perf_monitoring/trmv_upt.cpp b/bench/perf_monitoring/trmv_upt.cpp new file mode 100644 index 0000000..511e008 --- /dev/null +++ b/bench/perf_monitoring/trmv_upt.cpp
@@ -0,0 +1,12 @@ +#include "gemv_common.h" + +EIGEN_DONT_INLINE +void trmv(const Mat &A, Vec &B, const Vec &C) +{ + B.noalias() += A.transpose().triangularView<Upper>() * C; +} + +int main(int argc, char **argv) +{ + return main_gemv(argc, argv, trmv); +}
diff --git a/bench/spbench/CMakeLists.txt b/bench/spbench/CMakeLists.txt index 8d53f4a..029ba6d 100644 --- a/bench/spbench/CMakeLists.txt +++ b/bench/spbench/CMakeLists.txt
@@ -38,25 +38,32 @@ endif() -find_package(Pastix) -find_package(Scotch) -find_package(Metis) -if(PASTIX_FOUND AND BLAS_FOUND) +find_package(PASTIX QUIET COMPONENTS METIS SCOTCH) +# check that the PASTIX found is a version without MPI +find_path(PASTIX_pastix_nompi.h_INCLUDE_DIRS + NAMES pastix_nompi.h + HINTS ${PASTIX_INCLUDE_DIRS} +) +if (NOT PASTIX_pastix_nompi.h_INCLUDE_DIRS) + message(STATUS "A version of Pastix has been found but pastix_nompi.h does not exist in the include directory." + " Because Eigen tests require a version without MPI, we disable the Pastix backend.") +endif() +if(PASTIX_FOUND AND PASTIX_pastix_nompi.h_INCLUDE_DIRS AND BLAS_FOUND) add_definitions("-DEIGEN_PASTIX_SUPPORT") - include_directories(${PASTIX_INCLUDES}) + include_directories(${PASTIX_INCLUDE_DIRS_DEP}) if(SCOTCH_FOUND) - include_directories(${SCOTCH_INCLUDES}) + include_directories(${SCOTCH_INCLUDE_DIRS}) set(PASTIX_LIBRARIES ${PASTIX_LIBRARIES} ${SCOTCH_LIBRARIES}) elseif(METIS_FOUND) - include_directories(${METIS_INCLUDES}) + include_directories(${METIS_INCLUDE_DIRS}) set(PASTIX_LIBRARIES ${PASTIX_LIBRARIES} ${METIS_LIBRARIES}) endif(SCOTCH_FOUND) - set(SPARSE_LIBS ${SPARSE_LIBS} ${PASTIX_LIBRARIES} ${ORDERING_LIBRARIES} ${BLAS_LIBRARIES}) - set(PASTIX_ALL_LIBS ${PASTIX_LIBRARIES} ${BLAS_LIBRARIES}) -endif(PASTIX_FOUND AND BLAS_FOUND) + set(SPARSE_LIBS ${SPARSE_LIBS} ${PASTIX_LIBRARIES_DEP} ${ORDERING_LIBRARIES}) + set(PASTIX_ALL_LIBS ${PASTIX_LIBRARIES_DEP}) +endif() if(METIS_FOUND) - include_directories(${METIS_INCLUDES}) + include_directories(${METIS_INCLUDE_DIRS}) set (SPARSE_LIBS ${SPARSE_LIBS} ${METIS_LIBRARIES}) add_definitions("-DEIGEN_METIS_SUPPORT") endif(METIS_FOUND)
diff --git a/bench/spbench/spbenchsolver.cpp b/bench/spbench/spbenchsolver.cpp index 4acd003..2a73511 100644 --- a/bench/spbench/spbenchsolver.cpp +++ b/bench/spbench/spbenchsolver.cpp
@@ -54,7 +54,7 @@ statbuf.close(); } else - std::cerr << "Unable to open the provided file for writting... \n"; + std::cerr << "Unable to open the provided file for writing... \n"; } // Get the maximum number of iterations and the tolerance
diff --git a/bench/tensors/README b/bench/tensors/README index 4398aa8..69342cc 100644 --- a/bench/tensors/README +++ b/bench/tensors/README
@@ -1,12 +1,25 @@ -Each benchmark comes in 2 flavors: one that runs on CPU, and one that runs on GPU. +The tensor benchmark suite is made of several parts. + +The first part is a generic suite, in which each benchmark comes in 2 flavors: one that runs on CPU, and one that runs on GPU. To compile the floating point CPU benchmarks, simply call: g++ tensor_benchmarks_cpu.cc benchmark_main.cc -I ../../ -std=c++11 -O3 -DNDEBUG -pthread -mavx -o benchmarks_cpu To compile the floating point GPU benchmarks, simply call: -nvcc tensor_benchmarks_gpu.cu benchmark_main.cc -I ../../ -std=c++11 -O2 -DNDEBUG -arch compute_35 -o benchmarks_gpu +nvcc tensor_benchmarks_gpu.cu benchmark_main.cc -I ../../ -std=c++11 -O2 -DNDEBUG -use_fast_math -ftz=true -arch compute_35 -o benchmarks_gpu +We also provide a version of the generic GPU tensor benchmarks that uses half floats (aka fp16) instead of regular floats. To compile these benchmarks, simply call the command line below. You'll need a recent GPU that supports compute capability 5.3 or higher to run them and nvcc 7.5 or higher to compile the code. +nvcc tensor_benchmarks_fp16_gpu.cu benchmark_main.cc -I ../../ -std=c++11 -O2 -DNDEBUG -use_fast_math -ftz=true -arch compute_53 -o benchmarks_fp16_gpu -To compile the half float GPU benchmarks, simply call the command line below. You'll need a recent GPU that supports compute capability 5.3 or higher to run them and nvcc 7.5 or higher to compile the code. -nvcc tensor_benchmarks_fp16_gpu.cu benchmark_main.cc -I ../../ -std=c++11 -O2 -DNDEBUG -arch compute_53 -o benchmarks_fp16_gpu +last but not least, we also provide a suite of benchmarks to measure the scalability of the contraction code on CPU. To compile these benchmarks, call +g++ contraction_benchmarks_cpu.cc benchmark_main.cc -I ../../ -std=c++11 -O3 -DNDEBUG -pthread -mavx -o benchmarks_cpu +To compile and run the benchmark for SYCL, using ComputeCpp you currently need following passes (only for translation units containing device code): +1. The device compilation pass that generates the device code (SYCL kernels and referenced device functions) and glue code needed by the host compiler to reference the device code from host code. +{ComputeCpp_ROOT}/bin/compute++ -I ../../ -I {ComputeCpp_ROOT}/include/ -std=c++11 -mllvm -inline-threshold=1000 -Wno-ignored-attributes -sycl -intelspirmetadata -emit-llvm -no-serial-memop -sycl-compress-name -DBUILD_PLATFORM_SPIR -DNDBUG -O3 -c tensor_benchmarks_sycl.cc -DEIGEN_USE_SYCL=1 +2. The host compilation pass that generates the final host binary. +clang++ -O3 -c benchmark_main.cc -pthread -I ../../ -D_GLIBCXX_USE_CXX11_ABI=0 -DEIGEN_USE_SYCL=1 -std=c++11 -o benchmark_main.o +clang++ -O3 tensor_benchmarks_sycl_include_headers.cc -pthread -I ../../ -I {ComputeCpp_ROOT}/include/ -L {ComputeCpp_ROOT}/lib/ -lComputeCpp -lOpenCL -D_GLIBCXX_USE_CXX11_ABI=0 -DEIGEN_USE_SYCL=1 -std=c++11 benchmark_main.o -o tensor_benchmark_sycl +export LD_LIBRARY_PATH={ComputeCpp_ROOT}/lib +3. Run the benchmark +./tensor_benchmark_sycl
diff --git a/bench/tensors/contraction_benchmarks_cpu.cc b/bench/tensors/contraction_benchmarks_cpu.cc new file mode 100644 index 0000000..f9e57ad --- /dev/null +++ b/bench/tensors/contraction_benchmarks_cpu.cc
@@ -0,0 +1,39 @@ +#define EIGEN_USE_THREADS + +#include <string> + +#include "tensor_benchmarks.h" + +#define CREATE_THREAD_POOL(threads) \ +Eigen::ThreadPool pool(threads); \ +Eigen::ThreadPoolDevice device(&pool, threads); + + +// Contractions for number of threads ranging from 1 to 32 +// Dimensions are Rows, Cols, Depth +#define BM_ContractionCPU(D1, D2, D3) \ + static void BM_##Contraction##_##D1##x##D2##x##D3(int iters, int Threads) { \ + StopBenchmarkTiming(); \ + CREATE_THREAD_POOL(Threads); \ + BenchmarkSuite<Eigen::ThreadPoolDevice, float> suite(device, D1, D2, D3); \ + suite.contraction(iters); \ + } \ + BENCHMARK_RANGE(BM_##Contraction##_##D1##x##D2##x##D3, 1, 32); + + +// Vector Matrix and Matrix Vector products +BM_ContractionCPU(1, 2000, 500); +BM_ContractionCPU(2000, 1, 500); + +// Various skinny matrices +BM_ContractionCPU(250, 3, 512); +BM_ContractionCPU(1500, 3, 512); + +BM_ContractionCPU(512, 800, 4); +BM_ContractionCPU(512, 80, 800); +BM_ContractionCPU(512, 80, 13522); +BM_ContractionCPU(1, 80, 13522); + +BM_ContractionCPU(3200, 512, 4); +BM_ContractionCPU(3200, 512, 80); +BM_ContractionCPU(3200, 80, 512);
diff --git a/bench/tensors/tensor_benchmarks.h b/bench/tensors/tensor_benchmarks.h index 62533a6..3a640ed 100644 --- a/bench/tensors/tensor_benchmarks.h +++ b/bench/tensors/tensor_benchmarks.h
@@ -35,6 +35,11 @@ void memcpy(int num_iters) { eigen_assert(m_ == k_ && k_ == n_); +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + device_.memcpy(c_, a_, m_ * m_ * sizeof(T)); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { device_.memcpy(c_, a_, m_ * m_ * sizeof(T)); @@ -55,7 +60,11 @@ } const TensorMap<Tensor<int, 2, 0, TensorIndex>, Eigen::Aligned> A((int*)a_, sizes); TensorMap<Tensor<T, 2, 0, TensorIndex>, Eigen::Aligned> B(b_, sizes); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + B.device(device_) = A.template cast<T>(); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { B.device(device_) = A.template cast<T>(); @@ -70,7 +79,6 @@ sizes[0] = m_; sizes[1] = m_; TensorMap<Tensor<T, 2>, Eigen::Aligned> C(c_, sizes); - StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = C.random(); @@ -93,7 +101,18 @@ const Eigen::DSizes<TensorIndex, 2> second_quadrant(0, m_/2); const Eigen::DSizes<TensorIndex, 2> third_quadrant(m_/2, 0); const Eigen::DSizes<TensorIndex, 2> fourth_quadrant(m_/2, m_/2); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.slice(first_quadrant, quarter_sizes).device(device_) = + A.slice(first_quadrant, quarter_sizes); + C.slice(second_quadrant, quarter_sizes).device(device_) = + B.slice(second_quadrant, quarter_sizes); + C.slice(third_quadrant, quarter_sizes).device(device_) = + A.slice(third_quadrant, quarter_sizes); + C.slice(fourth_quadrant, quarter_sizes).device(device_) = + B.slice(fourth_quadrant, quarter_sizes); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.slice(first_quadrant, quarter_sizes).device(device_) = @@ -118,7 +137,11 @@ Eigen::array<TensorIndex, 1> output_size; output_size[0] = n_; TensorMap<Tensor<T, 1, 0, TensorIndex>, Eigen::Aligned> C(c_, output_size); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = B.chip(iter % k_, 0); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = B.chip(iter % k_, 0); @@ -135,7 +158,11 @@ Eigen::array<TensorIndex, 1> output_size; output_size[0] = n_; TensorMap<Tensor<T, 1, 0, TensorIndex>, Eigen::Aligned> C(c_, output_size); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = B.chip(iter % n_, 1); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = B.chip(iter % n_, 1); @@ -158,7 +185,11 @@ Eigen::array<int, 2> shuffle; shuffle[0] = 1; shuffle[1] = 0; - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + B.device(device_) = A.shuffle(shuffle); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { B.device(device_) = A.shuffle(shuffle); @@ -178,10 +209,19 @@ size_b[1] = m_; TensorMap<Tensor<T, 2>, Eigen::Aligned> B(b_, size_b); +#if defined(EIGEN_HAS_INDEX_LIST) + Eigen::IndexPairList<Eigen::type2indexpair<0, 0>, + Eigen::type2indexpair<2, 1> > paddings; +#else Eigen::array<Eigen::IndexPair<TensorIndex>, 2> paddings; paddings[0] = Eigen::IndexPair<TensorIndex>(0, 0); paddings[1] = Eigen::IndexPair<TensorIndex>(2, 1); - +#endif +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + B.device(device_) = A.pad(paddings); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { B.device(device_) = A.pad(paddings); @@ -211,6 +251,11 @@ Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> > strides; #endif +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + B.device(device_) = A.stride(strides); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { B.device(device_) = A.stride(strides); @@ -240,6 +285,11 @@ broadcast.set(1, n_); #endif +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A.broadcast(broadcast); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A.broadcast(broadcast); @@ -256,7 +306,11 @@ const TensorMap<Tensor<T, 2>, Eigen::Aligned> A(a_, sizes); const TensorMap<Tensor<T, 2>, Eigen::Aligned> B(b_, sizes); TensorMap<Tensor<T, 2>, Eigen::Aligned> C(c_, sizes); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A * A.constant(static_cast<T>(3.14)) + B * B.constant(static_cast<T>(2.7)); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A * A.constant(static_cast<T>(3.14)) + B * B.constant(static_cast<T>(2.7)); @@ -275,6 +329,11 @@ const TensorMap<Tensor<T, 2>, Eigen::Aligned> B(b_, sizes); TensorMap<Tensor<T, 2>, Eigen::Aligned> C(c_, sizes); +#ifdef EIGEN_USE_SYCL // warmup for sycl +for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A.rsqrt() + B.sqrt() * B.square(); +} +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A.rsqrt() + B.sqrt() * B.square(); @@ -292,7 +351,11 @@ const TensorMap<Tensor<T, 2>, Eigen::Aligned> A(a_, sizes); const TensorMap<Tensor<T, 2>, Eigen::Aligned> B(b_, sizes); TensorMap<Tensor<T, 2>, Eigen::Aligned> C(c_, sizes); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A.exp() + B.log(); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A.exp() + B.log(); @@ -320,7 +383,11 @@ // optimize the code. Eigen::IndexList<Eigen::type2index<0>> sum_along_dim; #endif - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = B.sum(sum_along_dim); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = B.sum(sum_along_dim); @@ -350,7 +417,11 @@ // optimize the code. Eigen::IndexList<Eigen::type2index<1>> sum_along_dim; #endif - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = B.sum(sum_along_dim); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = B.sum(sum_along_dim); @@ -368,9 +439,13 @@ const TensorMap<Tensor<T, 2, 0, TensorIndex>, Eigen::Aligned> B( b_, input_size); Eigen::array<TensorIndex, 0> output_size; - TensorMap<Tensor<float, 0, 0, TensorIndex>, Eigen::Aligned> C( + TensorMap<Tensor<T, 0, 0, TensorIndex>, Eigen::Aligned> C( c_, output_size); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = B.sum(); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = B.sum(); @@ -399,7 +474,11 @@ typedef typename Tensor<T, 2>::DimensionPair DimPair; Eigen::array<DimPair, 1> dims; dims[0] = DimPair(1, 0); - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A.contract(B, dims); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A.contract(B, dims); @@ -425,7 +504,11 @@ Eigen::array<TensorIndex, 2> dims; dims[0] = 0; dims[1] = 1; - +#ifdef EIGEN_USE_SYCL // warmup for sycl + for (int iter = 0; iter < 10; ++iter) { + C.device(device_) = A.convolve(B, dims); + } +#endif StartBenchmarkTiming(); for (int iter = 0; iter < num_iters; ++iter) { C.device(device_) = A.convolve(B, dims); @@ -456,6 +539,11 @@ if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) { device_.synchronize(); } +#elif defined(EIGEN_USE_SYCL) + if (Eigen::internal::is_same<Device, Eigen::SyclDevice>::value) { + device_.synchronize(); + } + #endif StopBenchmarkTiming(); SetBenchmarkFlopsProcessed(num_items);
diff --git a/bench/tensors/tensor_benchmarks_fp16_gpu.cu b/bench/tensors/tensor_benchmarks_fp16_gpu.cu index 1487655..65784d0 100644 --- a/bench/tensors/tensor_benchmarks_fp16_gpu.cu +++ b/bench/tensors/tensor_benchmarks_fp16_gpu.cu
@@ -33,6 +33,7 @@ BM_FuncGPU(transcendentalFunc); BM_FuncGPU(rowReduction); BM_FuncGPU(colReduction); +BM_FuncGPU(fullReduction); // Contractions
diff --git a/bench/tensors/tensor_benchmarks_sycl.cc b/bench/tensors/tensor_benchmarks_sycl.cc new file mode 100644 index 0000000..cb6daac --- /dev/null +++ b/bench/tensors/tensor_benchmarks_sycl.cc
@@ -0,0 +1,73 @@ +#ifdef EIGEN_USE_SYCL + +#include <SYCL/sycl.hpp> +#include <iostream> + +#include "tensor_benchmarks.h" + +#define BM_FuncGPU(FUNC) \ + static void BM_##FUNC(int iters, int N) { \ + StopBenchmarkTiming(); \ + cl::sycl::gpu_selector selector; \ + Eigen::QueueInterface queue(selector); \ + Eigen::SyclDevice device(&queue); \ + BenchmarkSuite<Eigen::SyclDevice, float> suite(device, N); \ + suite.FUNC(iters); \ + } \ + BENCHMARK_RANGE(BM_##FUNC, 10, 5000); + +BM_FuncGPU(memcpy); +BM_FuncGPU(typeCasting); +BM_FuncGPU(slicing); +BM_FuncGPU(rowChip); +BM_FuncGPU(colChip); +BM_FuncGPU(shuffling); +BM_FuncGPU(padding); +BM_FuncGPU(striding); +BM_FuncGPU(broadcasting); +BM_FuncGPU(coeffWiseOp); +BM_FuncGPU(algebraicFunc); +BM_FuncGPU(transcendentalFunc); +BM_FuncGPU(rowReduction); +BM_FuncGPU(colReduction); +BM_FuncGPU(fullReduction); + + +// Contractions +#define BM_FuncWithInputDimsGPU(FUNC, D1, D2, D3) \ + static void BM_##FUNC##_##D1##x##D2##x##D3(int iters, int N) { \ + StopBenchmarkTiming(); \ + cl::sycl::gpu_selector selector; \ + Eigen::QueueInterface queue(selector); \ + Eigen::SyclDevice device(&queue); \ + BenchmarkSuite<Eigen::SyclDevice, float> suite(device, D1, D2, D3); \ + suite.FUNC(iters); \ + } \ + BENCHMARK_RANGE(BM_##FUNC##_##D1##x##D2##x##D3, 10, 5000); + + +BM_FuncWithInputDimsGPU(contraction, N, N, N); +BM_FuncWithInputDimsGPU(contraction, 64, N, N); +BM_FuncWithInputDimsGPU(contraction, N, 64, N); +BM_FuncWithInputDimsGPU(contraction, N, N, 64); + + +// Convolutions +#define BM_FuncWithKernelDimsGPU(FUNC, DIM1, DIM2) \ + static void BM_##FUNC##_##DIM1##x##DIM2(int iters, int N) { \ + StopBenchmarkTiming(); \ + cl::sycl::gpu_selector selector; \ + Eigen::QueueInterface queue(selector); \ + Eigen::SyclDevice device(&queue); \ + BenchmarkSuite<Eigen::SyclDevice, float> suite(device, N); \ + suite.FUNC(iters, DIM1, DIM2); \ + } \ + BENCHMARK_RANGE(BM_##FUNC##_##DIM1##x##DIM2, 128, 5000); + +BM_FuncWithKernelDimsGPU(convolution, 7, 1); +BM_FuncWithKernelDimsGPU(convolution, 1, 7); +BM_FuncWithKernelDimsGPU(convolution, 7, 4); +BM_FuncWithKernelDimsGPU(convolution, 4, 7); +BM_FuncWithKernelDimsGPU(convolution, 7, 64); +BM_FuncWithKernelDimsGPU(convolution, 64, 7); +#endif
diff --git a/bench/tensors/tensor_benchmarks_sycl_include_headers.cc b/bench/tensors/tensor_benchmarks_sycl_include_headers.cc new file mode 100644 index 0000000..bcc3c4c --- /dev/null +++ b/bench/tensors/tensor_benchmarks_sycl_include_headers.cc
@@ -0,0 +1,2 @@ +#include "tensor_benchmarks_sycl.cc" +#include "tensor_benchmarks_sycl.sycl"
diff --git a/blas/CMakeLists.txt b/blas/CMakeLists.txt index d0efb41..9887d58 100644 --- a/blas/CMakeLists.txt +++ b/blas/CMakeLists.txt
@@ -45,10 +45,12 @@ if(EIGEN_Fortran_COMPILER_WORKS) -if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) - add_subdirectory(testing) # can't do EXCLUDE_FROM_ALL here, breaks CTest -else() - add_subdirectory(testing EXCLUDE_FROM_ALL) +if(BUILD_TESTING) + if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) + add_subdirectory(testing) # can't do EXCLUDE_FROM_ALL here, breaks CTest + else() + add_subdirectory(testing EXCLUDE_FROM_ALL) + endif() endif() endif()
diff --git a/blas/PackedTriangularMatrixVector.h b/blas/PackedTriangularMatrixVector.h index e9886d5..0039536 100644 --- a/blas/PackedTriangularMatrixVector.h +++ b/blas/PackedTriangularMatrixVector.h
@@ -18,7 +18,7 @@ template<typename Index, int Mode, typename LhsScalar, bool ConjLhs, typename RhsScalar, bool ConjRhs> struct packed_triangular_matrix_vector_product<Index,Mode,LhsScalar,ConjLhs,RhsScalar,ConjRhs,ColMajor> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { IsLower = (Mode & Lower) ==Lower, HasUnitDiag = (Mode & UnitDiag)==UnitDiag, @@ -47,7 +47,7 @@ template<typename Index, int Mode, typename LhsScalar, bool ConjLhs, typename RhsScalar, bool ConjRhs> struct packed_triangular_matrix_vector_product<Index,Mode,LhsScalar,ConjLhs,RhsScalar,ConjRhs,RowMajor> { - typedef typename scalar_product_traits<LhsScalar, RhsScalar>::ReturnType ResScalar; + typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResScalar; enum { IsLower = (Mode & Lower) ==Lower, HasUnitDiag = (Mode & UnitDiag)==UnitDiag,
diff --git a/blas/common.h b/blas/common.h index 61d8344..960c09c 100644 --- a/blas/common.h +++ b/blas/common.h
@@ -10,6 +10,14 @@ #ifndef EIGEN_BLAS_COMMON_H #define EIGEN_BLAS_COMMON_H +#ifdef __GNUC__ +# if __GNUC__<5 +// GCC < 5.0 does not like the global Scalar typedef +// we just keep shadow-warnings disabled permanently +# define EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS +# endif +#endif + #include "../Eigen/Core" #include "../Eigen/Jacobi"
diff --git a/blas/f2c/ctbmv.c b/blas/f2c/ctbmv.c index 790fd58..a6e0dae 100644 --- a/blas/f2c/ctbmv.c +++ b/blas/f2c/ctbmv.c
@@ -147,7 +147,7 @@ /* ( 1 + ( n - 1 )*abs( INCX ) ). */ /* Before entry, the incremented array X must contain the n */ /* element vector x. On exit, X is overwritten with the */ -/* tranformed vector x. */ +/* transformed vector x. */ /* INCX - INTEGER. */ /* On entry, INCX specifies the increment for the elements of */
diff --git a/blas/f2c/dtbmv.c b/blas/f2c/dtbmv.c index fdf73eb..aa67d19 100644 --- a/blas/f2c/dtbmv.c +++ b/blas/f2c/dtbmv.c
@@ -143,7 +143,7 @@ /* ( 1 + ( n - 1 )*abs( INCX ) ). */ /* Before entry, the incremented array X must contain the n */ /* element vector x. On exit, X is overwritten with the */ -/* tranformed vector x. */ +/* transformed vector x. */ /* INCX - INTEGER. */ /* On entry, INCX specifies the increment for the elements of */
diff --git a/blas/f2c/stbmv.c b/blas/f2c/stbmv.c index fcf9ce3..b5a68b5 100644 --- a/blas/f2c/stbmv.c +++ b/blas/f2c/stbmv.c
@@ -143,7 +143,7 @@ /* ( 1 + ( n - 1 )*abs( INCX ) ). */ /* Before entry, the incremented array X must contain the n */ /* element vector x. On exit, X is overwritten with the */ -/* tranformed vector x. */ +/* transformed vector x. */ /* INCX - INTEGER. */ /* On entry, INCX specifies the increment for the elements of */
diff --git a/blas/f2c/ztbmv.c b/blas/f2c/ztbmv.c index 4cdcd7f..3bf0beb 100644 --- a/blas/f2c/ztbmv.c +++ b/blas/f2c/ztbmv.c
@@ -147,7 +147,7 @@ /* ( 1 + ( n - 1 )*abs( INCX ) ). */ /* Before entry, the incremented array X must contain the n */ /* element vector x. On exit, X is overwritten with the */ -/* tranformed vector x. */ +/* transformed vector x. */ /* INCX - INTEGER. */ /* On entry, INCX specifies the increment for the elements of */
diff --git a/blas/level1_impl.h b/blas/level1_impl.h index f857bfa..6e7f8c9 100644 --- a/blas/level1_impl.h +++ b/blas/level1_impl.h
@@ -33,7 +33,7 @@ Scalar* x = reinterpret_cast<Scalar*>(px); Scalar* y = reinterpret_cast<Scalar*>(py); - // be carefull, *incx==0 is allowed !! + // be careful, *incx==0 is allowed !! if(*incx==1 && *incy==1) make_vector(y,*n) = make_vector(x,*n); else
diff --git a/blas/level3_impl.h b/blas/level3_impl.h index beb36c4..6c802cd 100644 --- a/blas/level3_impl.h +++ b/blas/level3_impl.h
@@ -159,7 +159,7 @@ return 0; int code = OP(*opa) | (SIDE(*side) << 2) | (UPLO(*uplo) << 3) | (DIAG(*diag) << 4); - + if(SIDE(*side)==LEFT) { internal::gemm_blocking_space<ColMajor,Scalar,Scalar,Dynamic,Dynamic,Dynamic,4> blocking(*m,*n,*m,1,false); @@ -385,7 +385,7 @@ int info = 0; if(UPLO(*uplo)==INVALID) info = 1; - else if(OP(*op)==INVALID) info = 2; + else if(OP(*op)==INVALID || (ISCOMPLEX && OP(*op)==ADJ) ) info = 2; else if(*n<0) info = 3; else if(*k<0) info = 4; else if(*lda<std::max(1,(OP(*op)==NOTR)?*n:*k)) info = 7; @@ -447,7 +447,7 @@ int info = 0; if(UPLO(*uplo)==INVALID) info = 1; - else if(OP(*op)==INVALID) info = 2; + else if(OP(*op)==INVALID || (ISCOMPLEX && OP(*op)==ADJ) ) info = 2; else if(*n<0) info = 3; else if(*k<0) info = 4; else if(*lda<std::max(1,(OP(*op)==NOTR)?*n:*k)) info = 7; @@ -609,7 +609,7 @@ else if(beta==Scalar(0)) matrix(c, *n, *n, *ldc).triangularView<Lower>().setZero(); else matrix(c, *n, *n, *ldc).triangularView<StrictlyLower>() *= beta; - + if(beta!=Scalar(0)) { matrix(c, *n, *n, *ldc).diagonal().real() *= beta;
diff --git a/blas/single.cpp b/blas/single.cpp index 836e3ee..20ea57d 100644 --- a/blas/single.cpp +++ b/blas/single.cpp
@@ -19,4 +19,4 @@ #include "level3_impl.h" float BLASFUNC(sdsdot)(int* n, float* alpha, float* x, int* incx, float* y, int* incy) -{ return *alpha + BLASFUNC(dsdot)(n, x, incx, y, incy); } +{ return double(*alpha) + BLASFUNC(dsdot)(n, x, incx, y, incy); }
diff --git a/blas/testing/cblat1.f b/blas/testing/cblat1.f index 8ca67fb..73015f5 100644 --- a/blas/testing/cblat1.f +++ b/blas/testing/cblat1.f
@@ -619,7 +619,7 @@ SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC) * ************************* STEST1 ***************************** * -* THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN +* THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN * REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE * ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT. *
diff --git a/blas/testing/dblat1.f b/blas/testing/dblat1.f index 30691f9..03d9f13 100644 --- a/blas/testing/dblat1.f +++ b/blas/testing/dblat1.f
@@ -990,7 +990,7 @@ SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC) * ************************* STEST1 ***************************** * -* THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN +* THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN * REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE * ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT. *
diff --git a/blas/testing/sblat1.f b/blas/testing/sblat1.f index 6657c26..4d43d9b 100644 --- a/blas/testing/sblat1.f +++ b/blas/testing/sblat1.f
@@ -946,7 +946,7 @@ SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC) * ************************* STEST1 ***************************** * -* THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN +* THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN * REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE * ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT. *
diff --git a/blas/testing/zblat1.f b/blas/testing/zblat1.f index d30112c..c00b67d 100644 --- a/blas/testing/zblat1.f +++ b/blas/testing/zblat1.f
@@ -619,7 +619,7 @@ SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC) * ************************* STEST1 ***************************** * -* THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN +* THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN * REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE * ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT. *
diff --git a/cmake/Eigen3Config.cmake.in b/cmake/Eigen3Config.cmake.in index 04e7886..0a1ac61 100644 --- a/cmake/Eigen3Config.cmake.in +++ b/cmake/Eigen3Config.cmake.in
@@ -1,28 +1,23 @@ -# -*- cmake -*- -# -# Eigen3Config.cmake(.in) +# This file exports the Eigen3::Eigen CMake target which should be passed to the +# target_link_libraries command. -# Use the following variables to compile and link against Eigen: -# EIGEN3_FOUND - True if Eigen was found on your system -# EIGEN3_USE_FILE - The file making Eigen usable -# EIGEN3_DEFINITIONS - Definitions needed to build with Eigen -# EIGEN3_INCLUDE_DIR - Directory where signature_of_eigen3_matrix_library can be found -# EIGEN3_INCLUDE_DIRS - List of directories of Eigen and it's dependencies -# EIGEN3_ROOT_DIR - The base directory of Eigen -# EIGEN3_VERSION_STRING - A human-readable string containing the version -# EIGEN3_VERSION_MAJOR - The major version of Eigen -# EIGEN3_VERSION_MINOR - The minor version of Eigen -# EIGEN3_VERSION_PATCH - The patch version of Eigen +@PACKAGE_INIT@ -set ( EIGEN3_FOUND 1 ) -set ( EIGEN3_USE_FILE "${CMAKE_CURRENT_LIST_DIR}/UseEigen3.cmake" ) +if (NOT TARGET eigen) + include ("${CMAKE_CURRENT_LIST_DIR}/Eigen3Targets.cmake") +endif () -set ( EIGEN3_DEFINITIONS "@EIGEN_DEFINITIONS@" ) -set ( EIGEN3_INCLUDE_DIR "@EIGEN_INCLUDE_DIR@" ) -set ( EIGEN3_INCLUDE_DIRS "@EIGEN_INCLUDE_DIRS@" ) -set ( EIGEN3_ROOT_DIR "@EIGEN_ROOT_DIR@" ) +# Legacy variables, do *not* use. May be removed in the future. -set ( EIGEN3_VERSION_STRING "@EIGEN_VERSION_STRING@" ) -set ( EIGEN3_VERSION_MAJOR "@EIGEN_VERSION_MAJOR@" ) -set ( EIGEN3_VERSION_MINOR "@EIGEN_VERSION_MINOR@" ) -set ( EIGEN3_VERSION_PATCH "@EIGEN_VERSION_PATCH@" ) +set (EIGEN3_FOUND 1) +set (EIGEN3_USE_FILE "${CMAKE_CURRENT_LIST_DIR}/UseEigen3.cmake") + +set (EIGEN3_DEFINITIONS "@EIGEN_DEFINITIONS@") +set (EIGEN3_INCLUDE_DIR "@PACKAGE_EIGEN_INCLUDE_DIR@") +set (EIGEN3_INCLUDE_DIRS "@PACKAGE_EIGEN_INCLUDE_DIR@") +set (EIGEN3_ROOT_DIR "@PACKAGE_EIGEN_ROOT_DIR@") + +set (EIGEN3_VERSION_STRING "@EIGEN_VERSION_STRING@") +set (EIGEN3_VERSION_MAJOR "@EIGEN_VERSION_MAJOR@") +set (EIGEN3_VERSION_MINOR "@EIGEN_VERSION_MINOR@") +set (EIGEN3_VERSION_PATCH "@EIGEN_VERSION_PATCH@")
diff --git a/cmake/Eigen3ConfigLegacy.cmake.in b/cmake/Eigen3ConfigLegacy.cmake.in new file mode 100644 index 0000000..62d7224 --- /dev/null +++ b/cmake/Eigen3ConfigLegacy.cmake.in
@@ -0,0 +1,30 @@ +# -*- cmake -*- +# +# Eigen3Config.cmake(.in) + +# Use the following variables to compile and link against Eigen: +# EIGEN3_FOUND - True if Eigen was found on your system +# EIGEN3_USE_FILE - The file making Eigen usable +# EIGEN3_DEFINITIONS - Definitions needed to build with Eigen +# EIGEN3_INCLUDE_DIR - Directory where signature_of_eigen3_matrix_library can be found +# EIGEN3_INCLUDE_DIRS - List of directories of Eigen and it's dependencies +# EIGEN3_ROOT_DIR - The base directory of Eigen +# EIGEN3_VERSION_STRING - A human-readable string containing the version +# EIGEN3_VERSION_MAJOR - The major version of Eigen +# EIGEN3_VERSION_MINOR - The minor version of Eigen +# EIGEN3_VERSION_PATCH - The patch version of Eigen + +@PACKAGE_INIT@ + +set ( EIGEN3_FOUND 1 ) +set ( EIGEN3_USE_FILE "${CMAKE_CURRENT_LIST_DIR}/UseEigen3.cmake" ) + +set ( EIGEN3_DEFINITIONS "@EIGEN_DEFINITIONS@" ) +set ( EIGEN3_INCLUDE_DIR "@PACKAGE_EIGEN_INCLUDE_DIR@" ) +set ( EIGEN3_INCLUDE_DIRS "@PACKAGE_EIGEN_INCLUDE_DIR@" ) +set ( EIGEN3_ROOT_DIR "@PACKAGE_EIGEN_ROOT_DIR@" ) + +set ( EIGEN3_VERSION_STRING "@EIGEN_VERSION_STRING@" ) +set ( EIGEN3_VERSION_MAJOR "@EIGEN_VERSION_MAJOR@" ) +set ( EIGEN3_VERSION_MINOR "@EIGEN_VERSION_MINOR@" ) +set ( EIGEN3_VERSION_PATCH "@EIGEN_VERSION_PATCH@" )
diff --git a/cmake/EigenConfigureTesting.cmake b/cmake/EigenConfigureTesting.cmake index afc24b5..1e72d92 100644 --- a/cmake/EigenConfigureTesting.cmake +++ b/cmake/EigenConfigureTesting.cmake
@@ -11,16 +11,18 @@ add_custom_target(check COMMAND "ctest") add_dependencies(check buildtests) -# check whether /bin/bash exists -find_file(EIGEN_BIN_BASH_EXISTS "/bin/bash" PATHS "/" NO_DEFAULT_PATH) +# check whether /bin/bash exists (disabled as not used anymore) +# find_file(EIGEN_BIN_BASH_EXISTS "/bin/bash" PATHS "/" NO_DEFAULT_PATH) # This call activates testing and generates the DartConfiguration.tcl include(CTest) set(EIGEN_TEST_BUILD_FLAGS "" CACHE STRING "Options passed to the build command of unit tests") +set(EIGEN_DASHBOARD_BUILD_TARGET "buildtests" CACHE STRING "Target to be built in dashboard mode, default is buildtests") +set(EIGEN_CTEST_ERROR_EXCEPTION "" CACHE STRING "Regular expression for build error messages to be filtered out") # Overwrite default DartConfiguration.tcl such that ctest can build our unit tests. -# Recall that our unit tests are not in the "all" target, so we have to explicitely ask ctest to build our custom 'buildtests' target. +# Recall that our unit tests are not in the "all" target, so we have to explicitly ask ctest to build our custom 'buildtests' target. # At this stage, we can also add custom flags to the build tool through the user defined EIGEN_TEST_BUILD_FLAGS variable. file(READ "${CMAKE_CURRENT_BINARY_DIR}/DartConfiguration.tcl" EIGEN_DART_CONFIG_FILE) # try to grab the default flags @@ -28,7 +30,7 @@ if(NOT CMAKE_MATCH_1) string(REGEX MATCH "MakeCommand:.*[^c]make (.*)\nDefaultCTestConfigurationType" EIGEN_DUMMY ${EIGEN_DART_CONFIG_FILE}) endif() -string(REGEX REPLACE "MakeCommand:.*DefaultCTestConfigurationType" "MakeCommand: ${CMAKE_COMMAND} --build . --target buildtests --config \"\${CTEST_CONFIGURATION_TYPE}\" -- ${CMAKE_MATCH_1} ${EIGEN_TEST_BUILD_FLAGS}\nDefaultCTestConfigurationType" +string(REGEX REPLACE "MakeCommand:.*DefaultCTestConfigurationType" "MakeCommand: ${CMAKE_COMMAND} --build . --target ${EIGEN_DASHBOARD_BUILD_TARGET} --config \"\${CTEST_CONFIGURATION_TYPE}\" -- ${CMAKE_MATCH_1} ${EIGEN_TEST_BUILD_FLAGS}\nDefaultCTestConfigurationType" EIGEN_DART_CONFIG_FILE2 ${EIGEN_DART_CONFIG_FILE}) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/DartConfiguration.tcl" ${EIGEN_DART_CONFIG_FILE2}) @@ -39,7 +41,7 @@ # configure Eigen related testing options option(EIGEN_NO_ASSERTION_CHECKING "Disable checking of assertions using exceptions" OFF) -option(EIGEN_DEBUG_ASSERTS "Enable advanced debuging of assertions" OFF) +option(EIGEN_DEBUG_ASSERTS "Enable advanced debugging of assertions" OFF) if(CMAKE_COMPILER_IS_GNUCXX) option(EIGEN_COVERAGE_TESTING "Enable/disable gcov" OFF) @@ -54,8 +56,3 @@ endif(CMAKE_COMPILER_IS_GNUCXX) -check_cxx_compiler_flag("-std=c++11" EIGEN_COMPILER_SUPPORT_CXX11) - -if(EIGEN_TEST_CXX11 AND EIGEN_COMPILER_SUPPORT_CXX11) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") -endif()
diff --git a/cmake/EigenTesting.cmake b/cmake/EigenTesting.cmake index 6f36619..8cb2d54 100644 --- a/cmake/EigenTesting.cmake +++ b/cmake/EigenTesting.cmake
@@ -1,37 +1,46 @@ macro(ei_add_property prop value) - get_property(previous GLOBAL PROPERTY ${prop}) + get_property(previous GLOBAL PROPERTY ${prop}) if ((NOT previous) OR (previous STREQUAL "")) set_property(GLOBAL PROPERTY ${prop} "${value}") else() set_property(GLOBAL PROPERTY ${prop} "${previous} ${value}") - endif() + endif() endmacro(ei_add_property) #internal. See documentation of ei_add_test for details. macro(ei_add_test_internal testname testname_with_suffix) set(targetname ${testname_with_suffix}) - + if(EIGEN_ADD_TEST_FILENAME_EXTENSION) set(filename ${testname}.${EIGEN_ADD_TEST_FILENAME_EXTENSION}) else() set(filename ${testname}.cpp) endif() - + if(EIGEN_ADD_TEST_FILENAME_EXTENSION STREQUAL cu) - if(EIGEN_TEST_CUDA_CLANG) + if(EIGEN_TEST_HIP) + hip_reset_flags() + hip_add_executable(${targetname} ${filename} HIPCC_OPTIONS "-DEIGEN_USE_HIP ${ARGV2}") + elseif(EIGEN_TEST_CUDA_CLANG) set_source_files_properties(${filename} PROPERTIES LANGUAGE CXX) - if(CUDA_64_BIT_DEVICE_CODE) + + if(CUDA_64_BIT_DEVICE_CODE AND (EXISTS "${CUDA_TOOLKIT_ROOT_DIR}/lib64")) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") else() link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib") endif() + if (${ARGC} GREATER 2) add_executable(${targetname} ${filename}) else() add_executable(${targetname} ${filename} OPTIONS ${ARGV2}) endif() - target_link_libraries(${targetname} "cudart_static" "cuda" "dl" "rt" "pthread") + set(CUDA_CLANG_LINK_LIBRARIES "cudart_static" "cuda" "dl" "pthread") + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(CUDA_CLANG_LINK_LIBRARIES ${CUDA_CLANG_LINK_LIBRARIES} "rt") + endif() + target_link_libraries(${targetname} ${CUDA_CLANG_LINK_LIBRARIES}) else() if (${ARGC} GREATER 2) cuda_add_executable(${targetname} ${filename} OPTIONS ${ARGV2}) @@ -42,7 +51,7 @@ else() add_executable(${targetname} ${filename}) endif() - + if (targetname MATCHES "^eigen2_") add_dependencies(eigen2_buildtests ${targetname}) else() @@ -56,20 +65,18 @@ ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_DEBUG_ASSERTS=1") endif(EIGEN_DEBUG_ASSERTS) endif(EIGEN_NO_ASSERTION_CHECKING) - + ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_TEST_MAX_SIZE=${EIGEN_TEST_MAX_SIZE}") - ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_TEST_FUNC=${testname}") - - if(MSVC AND NOT EIGEN_SPLIT_LARGE_TESTS) + if(MSVC) ei_add_target_property(${targetname} COMPILE_FLAGS "/bigobj") - endif() + endif() # let the user pass flags. if(${ARGC} GREATER 2) ei_add_target_property(${targetname} COMPILE_FLAGS "${ARGV2}") endif(${ARGC} GREATER 2) - + if(EIGEN_TEST_CUSTOM_CXX_FLAGS) ei_add_target_property(${targetname} COMPILE_FLAGS "${EIGEN_TEST_CUSTOM_CXX_FLAGS}") endif() @@ -95,12 +102,12 @@ # notice: no double quotes around ${libs_to_link} here. It may be a list. target_link_libraries(${targetname} ${libs_to_link}) endif() - endif() + endif() add_test(${testname_with_suffix} "${targetname}") - - # Specify target and test labels accoirding to EIGEN_CURRENT_SUBPROJECT - get_property(current_subproject GLOBAL PROPERTY EIGEN_CURRENT_SUBPROJECT) + + # Specify target and test labels according to EIGEN_CURRENT_SUBPROJECT + get_property(current_subproject GLOBAL PROPERTY EIGEN_CURRENT_SUBPROJECT) if ((current_subproject) AND (NOT (current_subproject STREQUAL ""))) set_property(TARGET ${targetname} PROPERTY LABELS "Build${current_subproject}") add_dependencies("Build${current_subproject}" ${targetname}) @@ -109,6 +116,108 @@ endmacro(ei_add_test_internal) +# SYCL +macro(ei_add_test_internal_sycl testname testname_with_suffix) + set(targetname ${testname_with_suffix}) + + if(EIGEN_ADD_TEST_FILENAME_EXTENSION) + set(filename ${testname}.${EIGEN_ADD_TEST_FILENAME_EXTENSION}) + else() + set(filename ${testname}.cpp) + endif() + + set( include_file "${CMAKE_CURRENT_BINARY_DIR}/inc_${filename}") + set( bc_file "${CMAKE_CURRENT_BINARY_DIR}/${filename}.sycl") + set( host_file "${CMAKE_CURRENT_SOURCE_DIR}/${filename}") + + if(NOT EIGEN_SYCL_TRISYCL) + include_directories( SYSTEM ${COMPUTECPP_PACKAGE_ROOT_DIR}/include) + + ADD_CUSTOM_COMMAND( + OUTPUT ${include_file} + COMMAND ${CMAKE_COMMAND} -E echo "\\#include \\\"${host_file}\\\"" > ${include_file} + COMMAND ${CMAKE_COMMAND} -E echo "\\#include \\\"${bc_file}\\\"" >> ${include_file} + DEPENDS ${filename} ${bc_file} + COMMENT "Building ComputeCpp integration header file ${include_file}" + ) + + # Add a custom target for the generated integration header + add_custom_target("${testname}_integration_header_sycl" DEPENDS ${include_file}) + + add_executable(${targetname} ${include_file}) + add_dependencies(${targetname} "${testname}_integration_header_sycl") + else() + add_executable(${targetname} ${host_file}) + endif() + + add_sycl_to_target(${targetname} ${CMAKE_CURRENT_BINARY_DIR} ${filename}) + + if (targetname MATCHES "^eigen2_") + add_dependencies(eigen2_buildtests ${targetname}) + else() + add_dependencies(buildtests ${targetname}) + endif() + + if(EIGEN_NO_ASSERTION_CHECKING) + ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_NO_ASSERTION_CHECKING=1") + else(EIGEN_NO_ASSERTION_CHECKING) + if(EIGEN_DEBUG_ASSERTS) + ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_DEBUG_ASSERTS=1") + endif(EIGEN_DEBUG_ASSERTS) + endif(EIGEN_NO_ASSERTION_CHECKING) + + ei_add_target_property(${targetname} COMPILE_FLAGS "-DEIGEN_TEST_MAX_SIZE=${EIGEN_TEST_MAX_SIZE}") + + if(MSVC AND NOT EIGEN_SPLIT_LARGE_TESTS) + ei_add_target_property(${targetname} COMPILE_FLAGS "/bigobj") + endif() + + # let the user pass flags. + if(${ARGC} GREATER 2) + ei_add_target_property(${targetname} COMPILE_FLAGS "${ARGV2}") + endif(${ARGC} GREATER 2) + + if(EIGEN_TEST_CUSTOM_CXX_FLAGS) + ei_add_target_property(${targetname} COMPILE_FLAGS "${EIGEN_TEST_CUSTOM_CXX_FLAGS}") + endif() + + if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) + target_link_libraries(${targetname} ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO}) + endif() + if(EXTERNAL_LIBS) + target_link_libraries(${targetname} ${EXTERNAL_LIBS}) + endif() + if(EIGEN_TEST_CUSTOM_LINKER_FLAGS) + target_link_libraries(${targetname} ${EIGEN_TEST_CUSTOM_LINKER_FLAGS}) + endif() + + if(${ARGC} GREATER 3) + set(libs_to_link ${ARGV3}) + # it could be that some cmake module provides a bad library string " " (just spaces), + # and that severely breaks target_link_libraries ("can't link to -l-lstdc++" errors). + # so we check for strings containing only spaces. + string(STRIP "${libs_to_link}" libs_to_link_stripped) + string(LENGTH "${libs_to_link_stripped}" libs_to_link_stripped_length) + if(${libs_to_link_stripped_length} GREATER 0) + # notice: no double quotes around ${libs_to_link} here. It may be a list. + target_link_libraries(${targetname} ${libs_to_link}) + endif() + endif() + + add_test(${testname_with_suffix} "${targetname}") + + # Specify target and test labels according to EIGEN_CURRENT_SUBPROJECT + get_property(current_subproject GLOBAL PROPERTY EIGEN_CURRENT_SUBPROJECT) + if ((current_subproject) AND (NOT (current_subproject STREQUAL ""))) + set_property(TARGET ${targetname} PROPERTY LABELS "Build${current_subproject}") + add_dependencies("Build${current_subproject}" ${targetname}) + set_property(TEST ${testname_with_suffix} PROPERTY LABELS "${current_subproject}") + endif() + + +endmacro(ei_add_test_internal_sycl) + + # Macro to add a test # # the unique mandatory parameter testname must correspond to a file @@ -143,7 +252,7 @@ # # If EIGEN_SPLIT_LARGE_TESTS is ON, the test is split into multiple executables # test_<testname>_<N> -# where N runs from 1 to the greatest occurence found in the source file. Each of these +# where N runs from 1 to the greatest occurrence found in the source file. Each of these # executables is built passing -DEIGEN_TEST_PART_N. This allows to split large tests # into smaller executables. # @@ -161,17 +270,53 @@ else() set(filename ${testname}.cpp) endif() - + + file(READ "${filename}" test_source) + string(REGEX MATCHALL "CALL_SUBTEST_[0-9]+|EIGEN_TEST_PART_[0-9]+|EIGEN_SUFFIXES(;[0-9]+)+" + occurrences "${test_source}") + string(REGEX REPLACE "CALL_SUBTEST_|EIGEN_TEST_PART_|EIGEN_SUFFIXES" "" suffixes "${occurrences}") + list(REMOVE_DUPLICATES suffixes) + set(explicit_suffixes "") + if( (NOT EIGEN_SPLIT_LARGE_TESTS) AND suffixes) + # Check whether we have EIGEN_TEST_PART_* statements, in which case we likely must enforce splitting. + # For instance, indexed_view activate a different c++ version for each part. + string(REGEX MATCHALL "EIGEN_TEST_PART_[0-9]+" occurrences "${test_source}") + string(REGEX REPLACE "EIGEN_TEST_PART_" "" explicit_suffixes "${occurrences}") + list(REMOVE_DUPLICATES explicit_suffixes) + endif() + if( (EIGEN_SPLIT_LARGE_TESTS AND suffixes) OR explicit_suffixes) + add_custom_target(${testname}) + foreach(suffix ${suffixes}) + ei_add_test_internal(${testname} ${testname}_${suffix} + "${ARGV1} -DEIGEN_TEST_PART_${suffix}=1" "${ARGV2}") + add_dependencies(${testname} ${testname}_${suffix}) + endforeach(suffix) + else() + ei_add_test_internal(${testname} ${testname} "${ARGV1} -DEIGEN_TEST_PART_ALL=1" "${ARGV2}") + endif() +endmacro(ei_add_test) + +macro(ei_add_test_sycl testname) + get_property(EIGEN_TESTS_LIST GLOBAL PROPERTY EIGEN_TESTS_LIST) + set(EIGEN_TESTS_LIST "${EIGEN_TESTS_LIST}${testname}\n") + set_property(GLOBAL PROPERTY EIGEN_TESTS_LIST "${EIGEN_TESTS_LIST}") + + if(EIGEN_ADD_TEST_FILENAME_EXTENSION) + set(filename ${testname}.${EIGEN_ADD_TEST_FILENAME_EXTENSION}) + else() + set(filename ${testname}.cpp) + endif() + file(READ "${filename}" test_source) set(parts 0) string(REGEX MATCHALL "CALL_SUBTEST_[0-9]+|EIGEN_TEST_PART_[0-9]+|EIGEN_SUFFIXES(;[0-9]+)+" - occurences "${test_source}") - string(REGEX REPLACE "CALL_SUBTEST_|EIGEN_TEST_PART_|EIGEN_SUFFIXES" "" suffixes "${occurences}") + occurrences "${test_source}") + string(REGEX REPLACE "CALL_SUBTEST_|EIGEN_TEST_PART_|EIGEN_SUFFIXES" "" suffixes "${occurrences}") list(REMOVE_DUPLICATES suffixes) if(EIGEN_SPLIT_LARGE_TESTS AND suffixes) add_custom_target(${testname}) foreach(suffix ${suffixes}) - ei_add_test_internal(${testname} ${testname}_${suffix} + ei_add_test_internal_sycl(${testname} ${testname}_${suffix} "${ARGV1} -DEIGEN_TEST_PART_${suffix}=1" "${ARGV2}") add_dependencies(${testname} ${testname}_${suffix}) endforeach(suffix) @@ -181,46 +326,40 @@ set(symbols_to_enable_all_parts "${symbols_to_enable_all_parts} -DEIGEN_TEST_PART_${suffix}=1") endforeach(suffix) - ei_add_test_internal(${testname} ${testname} "${ARGV1} ${symbols_to_enable_all_parts}" "${ARGV2}") + ei_add_test_internal_sycl(${testname} ${testname} "${ARGV1} ${symbols_to_enable_all_parts}" "${ARGV2}") endif(EIGEN_SPLIT_LARGE_TESTS AND suffixes) -endmacro(ei_add_test) - +endmacro(ei_add_test_sycl) # adds a failtest, i.e. a test that succeed if the program fails to compile # note that the test runner for these is CMake itself, when passed -DEIGEN_FAILTEST=ON # so here we're just running CMake commands immediately, we're not adding any targets. macro(ei_add_failtest testname) - get_property(EIGEN_FAILTEST_FAILURE_COUNT GLOBAL PROPERTY EIGEN_FAILTEST_FAILURE_COUNT) - get_property(EIGEN_FAILTEST_COUNT GLOBAL PROPERTY EIGEN_FAILTEST_COUNT) - message(STATUS "Checking failtest: ${testname}") - set(filename "${testname}.cpp") - file(READ "${filename}" test_source) + set(test_target_ok ${testname}_ok) + set(test_target_ko ${testname}_ko) - try_compile(succeeds_when_it_should_fail - "${CMAKE_CURRENT_BINARY_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/${filename}" - COMPILE_DEFINITIONS "-DEIGEN_SHOULD_FAIL_TO_BUILD") - if (succeeds_when_it_should_fail) - message(STATUS "FAILED: ${testname} build succeeded when it should have failed") - endif() + # Add executables + add_executable(${test_target_ok} ${testname}.cpp) + add_executable(${test_target_ko} ${testname}.cpp) - try_compile(succeeds_when_it_should_succeed - "${CMAKE_CURRENT_BINARY_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/${filename}" - COMPILE_DEFINITIONS) - if (NOT succeeds_when_it_should_succeed) - message(STATUS "FAILED: ${testname} build failed when it should have succeeded") - endif() + # Remove them from the normal build process + set_target_properties(${test_target_ok} ${test_target_ko} PROPERTIES + EXCLUDE_FROM_ALL TRUE + EXCLUDE_FROM_DEFAULT_BUILD TRUE) - if (succeeds_when_it_should_fail OR NOT succeeds_when_it_should_succeed) - math(EXPR EIGEN_FAILTEST_FAILURE_COUNT ${EIGEN_FAILTEST_FAILURE_COUNT}+1) - endif() + # Configure the failing test + target_compile_definitions(${test_target_ko} PRIVATE EIGEN_SHOULD_FAIL_TO_BUILD) - math(EXPR EIGEN_FAILTEST_COUNT ${EIGEN_FAILTEST_COUNT}+1) + # Add the tests to ctest. + add_test(NAME ${test_target_ok} + COMMAND ${CMAKE_COMMAND} --build . --target ${test_target_ok} --config $<CONFIGURATION> + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + add_test(NAME ${test_target_ko} + COMMAND ${CMAKE_COMMAND} --build . --target ${test_target_ko} --config $<CONFIGURATION> + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_property(GLOBAL PROPERTY EIGEN_FAILTEST_FAILURE_COUNT ${EIGEN_FAILTEST_FAILURE_COUNT}) - set_property(GLOBAL PROPERTY EIGEN_FAILTEST_COUNT ${EIGEN_FAILTEST_COUNT}) + # Expect the second test to fail + set_tests_properties(${test_target_ko} PROPERTIES WILL_FAIL TRUE) endmacro(ei_add_failtest) # print a summary of the different options @@ -249,7 +388,7 @@ elseif(EIGEN_TEST_NO_EXPLICIT_VECTORIZATION) message(STATUS "Explicit vectorization disabled (alignment kept enabled)") else() - + message(STATUS "Maximal matrix/vector size: ${EIGEN_TEST_MAX_SIZE}") if(EIGEN_TEST_SSE2) @@ -288,12 +427,18 @@ message(STATUS "AVX: Using architecture defaults") endif() - if(EIGEN_TEST_FMA) + if(EIGEN_TEST_FMA) message(STATUS "FMA: ON") else() message(STATUS "FMA: Using architecture defaults") endif() + if(EIGEN_TEST_AVX512) + message(STATUS "AVX512: ON") + else() + message(STATUS "AVX512: Using architecture defaults") + endif() + if(EIGEN_TEST_ALTIVEC) message(STATUS "Altivec: ON") else() @@ -306,6 +451,12 @@ message(STATUS "VSX: Using architecture defaults") endif() + if(EIGEN_TEST_MSA) + message(STATUS "MIPS MSA: ON") + else() + message(STATUS "MIPS MSA: Using architecture defaults") + endif() + if(EIGEN_TEST_NEON) message(STATUS "ARM NEON: ON") else() @@ -317,19 +468,28 @@ else() message(STATUS "ARMv8 NEON: Using architecture defaults") endif() - + if(EIGEN_TEST_ZVECTOR) message(STATUS "S390X ZVECTOR: ON") else() message(STATUS "S390X ZVECTOR: Using architecture defaults") endif() - + if(EIGEN_TEST_CXX11) message(STATUS "C++11: ON") else() message(STATUS "C++11: OFF") endif() + if(EIGEN_TEST_SYCL) + if(EIGEN_SYCL_TRISYCL) + message(STATUS "SYCL: ON (using triSYCL)") + else() + message(STATUS "SYCL: ON (using computeCPP)") + endif() + else() + message(STATUS "SYCL: OFF") + endif() if(EIGEN_TEST_CUDA) if(EIGEN_TEST_CUDA_CLANG) message(STATUS "CUDA: ON (using clang)") @@ -339,6 +499,11 @@ else() message(STATUS "CUDA: OFF") endif() + if(EIGEN_TEST_HIP) + message(STATUS "HIP: ON (using hipcc)") + else() + message(STATUS "HIP: OFF") + endif() endif() # vectorization / alignment options @@ -364,7 +529,7 @@ set_property(GLOBAL PROPERTY EIGEN_FAILTEST_FAILURE_COUNT "0") set_property(GLOBAL PROPERTY EIGEN_FAILTEST_COUNT "0") - + # uncomment anytime you change the ei_get_compilerver_from_cxx_version_string macro # ei_test_get_compilerver_from_cxx_version_string() endmacro(ei_init_testing) @@ -373,22 +538,22 @@ # if the sitename is not yet set, try to set it if(NOT ${SITE} OR ${SITE} STREQUAL "") set(eigen_computername $ENV{COMPUTERNAME}) - set(eigen_hostname $ENV{HOSTNAME}) + set(eigen_hostname $ENV{HOSTNAME}) if(eigen_hostname) set(SITE ${eigen_hostname}) - elseif(eigen_computername) - set(SITE ${eigen_computername}) + elseif(eigen_computername) + set(SITE ${eigen_computername}) endif() endif() # in case it is already set, enforce lower case if(SITE) string(TOLOWER ${SITE} SITE) - endif() + endif() endmacro(ei_set_sitename) macro(ei_get_compilerver VAR) if(MSVC) - # on windows system, we use a modified CMake script + # on windows system, we use a modified CMake script include(EigenDetermineVSServicePack) EigenDetermineVSServicePack( my_service_pack ) @@ -397,23 +562,25 @@ else() set(${VAR} "na") endif() + elseif(${CMAKE_CXX_COMPILER_ID} MATCHES "PGI") + set(${VAR} "${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}") else() # on all other system we rely on ${CMAKE_CXX_COMPILER} # supporting a "--version" or "/version" flag - + if(WIN32 AND ${CMAKE_CXX_COMPILER_ID} EQUAL "Intel") set(EIGEN_CXX_FLAG_VERSION "/version") else() set(EIGEN_CXX_FLAG_VERSION "--version") endif() - + execute_process(COMMAND ${CMAKE_CXX_COMPILER} ${EIGEN_CXX_FLAG_VERSION} OUTPUT_VARIABLE eigen_cxx_compiler_version_string OUTPUT_STRIP_TRAILING_WHITESPACE) string(REGEX REPLACE "[\n\r].*" "" eigen_cxx_compiler_version_string ${eigen_cxx_compiler_version_string}) - + ei_get_compilerver_from_cxx_version_string("${eigen_cxx_compiler_version_string}" CNAME CVER) set(${VAR} "${CNAME}-${CVER}") - + endif() endmacro(ei_get_compilerver) @@ -422,18 +589,20 @@ # the testing macro call in ei_init_testing() of the EigenTesting.cmake file. # See also the ei_test_get_compilerver_from_cxx_version_string macro at the end of the file macro(ei_get_compilerver_from_cxx_version_string VERSTRING CNAME CVER) - # extract possible compiler names + # extract possible compiler names string(REGEX MATCH "g\\+\\+" ei_has_gpp ${VERSTRING}) string(REGEX MATCH "llvm|LLVM" ei_has_llvm ${VERSTRING}) string(REGEX MATCH "gcc|GCC" ei_has_gcc ${VERSTRING}) string(REGEX MATCH "icpc|ICC" ei_has_icpc ${VERSTRING}) string(REGEX MATCH "clang|CLANG" ei_has_clang ${VERSTRING}) - + # combine them if((ei_has_llvm) AND (ei_has_gpp OR ei_has_gcc)) set(${CNAME} "llvm-g++") elseif((ei_has_llvm) AND (ei_has_clang)) set(${CNAME} "llvm-clang++") + elseif(ei_has_clang) + set(${CNAME} "clang++") elseif(ei_has_icpc) set(${CNAME} "icpc") elseif(ei_has_gpp OR ei_has_gcc) @@ -441,7 +610,7 @@ else() set(${CNAME} "_") endif() - + # extract possible version numbers # first try to extract 3 isolated numbers: string(REGEX MATCH " [0-9]+\\.[0-9]+\\.[0-9]+" eicver ${VERSTRING}) @@ -459,9 +628,9 @@ endif() endif() endif() - + string(REGEX REPLACE ".(.*)" "\\1" ${CVER} ${eicver}) - + endmacro(ei_get_compilerver_from_cxx_version_string) macro(ei_get_cxxflags VAR) @@ -490,30 +659,32 @@ elseif(EIGEN_TEST_SSE3) set(${VAR} SSE3) elseif(EIGEN_TEST_SSE2 OR IS_64BIT_ENV) - set(${VAR} SSE2) + set(${VAR} SSE2) + elseif(EIGEN_TEST_MSA) + set(${VAR} MSA) endif() if(EIGEN_TEST_OPENMP) if (${VAR} STREQUAL "") - set(${VAR} OMP) - else() - set(${VAR} ${${VAR}}-OMP) - endif() + set(${VAR} OMP) + else() + set(${VAR} ${${VAR}}-OMP) + endif() endif() - + if(EIGEN_DEFAULT_TO_ROW_MAJOR) if (${VAR} STREQUAL "") - set(${VAR} ROW) - else() - set(${VAR} ${${VAR}}-ROWMAJ) - endif() + set(${VAR} ROW) + else() + set(${VAR} ${${VAR}}-ROWMAJ) + endif() endif() endmacro(ei_get_cxxflags) macro(ei_set_build_string) ei_get_compilerver(LOCAL_COMPILER_VERSION) ei_get_cxxflags(LOCAL_COMPILER_FLAGS) - + include(EigenDetermineOSVersion) DetermineOSVersion(OS_VERSION) @@ -523,17 +694,21 @@ set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-${LOCAL_COMPILER_FLAGS}) endif() + if(EIGEN_TEST_EXTERNAL_BLAS) + set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-external_blas) + endif() + ei_is_64bit_env(IS_64BIT_ENV) if(NOT IS_64BIT_ENV) set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-32bit) else() set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-64bit) endif() - + if(EIGEN_TEST_CXX11) set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-cxx11) endif() - + if(EIGEN_BUILD_STRING_SUFFIX) set(TMP_BUILD_STRING ${TMP_BUILD_STRING}-${EIGEN_BUILD_STRING_SUFFIX}) endif()
diff --git a/cmake/FindBLAS.cmake b/cmake/FindBLAS.cmake index 68c4e07..e3395bc 100644 --- a/cmake/FindBLAS.cmake +++ b/cmake/FindBLAS.cmake
@@ -1,385 +1,1363 @@ -# Find BLAS library +### # -# This module finds an installed library that implements the BLAS +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2016 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find BLAS library +# This module finds an installed fortran library that implements the BLAS # linear-algebra interface (see http://www.netlib.org/blas/). -# The list of libraries searched for is mainly taken +# The list of libraries searched for is taken # from the autoconf macro file, acx_blas.m4 (distributed at # http://ac-archive.sourceforge.net/ac-archive/acx_blas.html). # # This module sets the following variables: # BLAS_FOUND - set to true if a library implementing the BLAS interface # is found -# BLAS_INCLUDE_DIR - Directories containing the BLAS header files -# BLAS_DEFINITIONS - Compilation options to use BLAS -# BLAS_LINKER_FLAGS - Linker flags to use BLAS (excluding -l +# BLAS_LINKER_FLAGS - uncached list of required linker flags (excluding -l # and -L). -# BLAS_LIBRARIES_DIR - Directories containing the BLAS libraries. -# May be null if BLAS_LIBRARIES contains libraries name using full path. -# BLAS_LIBRARIES - List of libraries to link against BLAS interface. -# May be null if the compiler supports auto-link (e.g. VC++). -# BLAS_USE_FILE - The name of the cmake module to include to compile -# applications or libraries using BLAS. +# BLAS_COMPILER_FLAGS - uncached list of required compiler flags (including -I for mkl headers). +# BLAS_LIBRARIES - uncached list of libraries (using full path name) to +# link against to use BLAS +# BLAS95_LIBRARIES - uncached list of libraries (using full path name) +# to link against to use BLAS95 interface +# BLAS95_FOUND - set to true if a library implementing the BLAS f95 interface +# is found +# BLA_STATIC if set on this determines what kind of linkage we do (static) +# BLA_VENDOR if set checks only the specified vendor, if not set checks +# all the possibilities +# BLAS_VENDOR_FOUND stores the BLAS vendor found +# BLA_F95 if set on tries to find the f95 interfaces for BLAS/LAPACK +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DBLAS_DIR=path/to/blas): +# BLAS_DIR - Where to find the base directory of blas +# BLAS_INCDIR - Where to find the header files +# BLAS_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: BLAS_DIR, BLAS_INCDIR, BLAS_LIBDIR +# For MKL case and if no paths are given as hints, we will try to use the MKLROOT +# environment variable +# BLAS_VERBOSE Print some additional information during BLAS libraries detection +########## +### List of vendors (BLA_VENDOR) valid in this module +########## List of vendors (BLA_VENDOR) valid in this module +## Open (for OpenBlas), Eigen (for EigenBlas), Goto, ATLAS PhiPACK, +## CXML, DXML, SunPerf, SCSL, SGIMATH, IBMESSL, IBMESSLMT +## Intel10_32 (intel mkl v10 32 bit), Intel10_64lp (intel mkl v10 64 bit,lp thread model, lp64 model), +## Intel10_64lp_seq (intel mkl v10 64 bit,sequential code, lp64 model), +## Intel( older versions of mkl 32 and 64 bit), +## ACML, ACML_MP, ACML_GPU, Apple, NAS, Generic +# C/CXX should be enabled to use Intel mkl +### +# We handle different modes to find the dependency # -# This module was modified by CGAL team: -# - find libraries for a C++ compiler, instead of Fortran -# - added BLAS_INCLUDE_DIR, BLAS_DEFINITIONS and BLAS_LIBRARIES_DIR -# - removed BLAS95_LIBRARIES +# - Detection if already installed on the system +# - BLAS libraries can be detected from different ways +# Here is the order of precedence: +# 1) we look in cmake variable BLAS_LIBDIR or BLAS_DIR (we guess the libdirs) if defined +# 2) we look in environment variable BLAS_LIBDIR or BLAS_DIR (we guess the libdirs) if defined +# 3) we look in common environnment variables depending on the system (INCLUDE, C_INCLUDE_PATH, CPATH - LIB, DYLD_LIBRARY_PATH, LD_LIBRARY_PATH) +# 4) we look in common system paths depending on the system, see for example paths contained in the following cmake variables: +# - CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES, CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES +# - CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES, CMAKE_C_IMPLICIT_LINK_DIRECTORIES +# + +#============================================================================= +# Copyright 2007-2009 Kitware, Inc. +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +## Some macros to print status when search for headers and libs +# This macro informs why the _lib_to_find file has not been found +macro(Print_Find_Library_Blas_Status _libname _lib_to_find) + + # save _libname upper/lower case + string(TOUPPER ${_libname} LIBNAME) + string(TOLOWER ${_libname} libname) + + # print status + #message(" ") + if(${LIBNAME}_LIBDIR) + message("${Yellow}${LIBNAME}_LIBDIR is defined but ${_lib_to_find}" + "has not been found in ${ARGN}${ColourReset}") + else() + if(${LIBNAME}_DIR) + message("${Yellow}${LIBNAME}_DIR is defined but ${_lib_to_find}" + "has not been found in ${ARGN}${ColourReset}") + else() + message("${Yellow}${_lib_to_find} not found." + "Nor ${LIBNAME}_DIR neither ${LIBNAME}_LIBDIR" + "are defined so that we look for ${_lib_to_find} in" + "system paths (Linux: LD_LIBRARY_PATH, Windows: LIB," + "Mac: DYLD_LIBRARY_PATH," + "CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES," + "CMAKE_C_IMPLICIT_LINK_DIRECTORIES)${ColourReset}") + if(_lib_env) + message("${Yellow}${_lib_to_find} has not been found in" + "${_lib_env}${ColourReset}") + endif() + endif() + endif() + message("${BoldYellow}Please indicate where to find ${_lib_to_find}. You have three options:\n" + "- Option 1: Provide the Installation directory of BLAS library with cmake option: -D${LIBNAME}_DIR=your/path/to/${libname}/\n" + "- Option 2: Provide the directory where to find the library with cmake option: -D${LIBNAME}_LIBDIR=your/path/to/${libname}/lib/\n" + "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n" + "- Option 4: If your library provides a PkgConfig file, make sure pkg-config finds your library${ColourReset}") + +endmacro() + +# This macro informs why the _lib_to_find file has not been found +macro(Print_Find_Library_Blas_CheckFunc_Status _name) + + # save _libname upper/lower case + string(TOUPPER ${_name} FUNCNAME) + string(TOLOWER ${_name} funcname) + + # print status + #message(" ") + message("${Red}Libs have been found but check of symbol ${_name} failed " + "with following libraries ${ARGN}${ColourReset}") + message("${BoldRed}Please open your error file CMakeFiles/CMakeError.log" + "to figure out why it fails${ColourReset}") + #message(" ") + +endmacro() + +if (NOT BLAS_FOUND) + set(BLAS_DIR "" CACHE PATH "Installation directory of BLAS library") + if (NOT BLAS_FIND_QUIETLY) + message(STATUS "A cache variable, namely BLAS_DIR, has been set to specify the install directory of BLAS") + endif() +endif() + +option(BLAS_VERBOSE "Print some additional information during BLAS libraries detection" OFF) +mark_as_advanced(BLAS_VERBOSE) include(CheckFunctionExists) +include(CheckFortranFunctionExists) +set(_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) -# This macro checks for the existence of the combination of fortran libraries -# given by _list. If the combination is found, this macro checks (using the -# check_function_exists macro) whether can link against that library -# combination using the name of a routine given by _name using the linker -# flags given by _flags. If the combination of libraries is found and passes -# the link test, LIBRARIES is set to the list of complete library paths that -# have been found and DEFINITIONS to the required definitions. -# Otherwise, LIBRARIES is set to FALSE. -# N.B. _prefix is the prefix applied to the names of all cached variables that -# are generated internally and marked advanced by this macro. -macro(check_fortran_libraries DEFINITIONS LIBRARIES _prefix _name _flags _list _path) - #message("DEBUG: check_fortran_libraries(${_list} in ${_path})") +# Check the language being used +get_property( _LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES ) +if( _LANGUAGES_ MATCHES Fortran AND CMAKE_Fortran_COMPILER) + set( _CHECK_FORTRAN TRUE ) +elseif( (_LANGUAGES_ MATCHES C) OR (_LANGUAGES_ MATCHES CXX) ) + set( _CHECK_FORTRAN FALSE ) +else() + if(BLAS_FIND_REQUIRED) + message(FATAL_ERROR "FindBLAS requires Fortran, C, or C++ to be enabled.") + else() + message(STATUS "Looking for BLAS... - NOT found (Unsupported languages)") + return() + endif() +endif() - # Check for the existence of the libraries given by _list - set(_libraries_found TRUE) - set(_libraries_work FALSE) - set(${DEFINITIONS} "") - set(${LIBRARIES} "") +macro(Check_Fortran_Libraries LIBRARIES _prefix _name _flags _list _thread) + # This macro checks for the existence of the combination of fortran libraries + # given by _list. If the combination is found, this macro checks (using the + # Check_Fortran_Function_Exists macro) whether can link against that library + # combination using the name of a routine given by _name using the linker + # flags given by _flags. If the combination of libraries is found and passes + # the link test, LIBRARIES is set to the list of complete library paths that + # have been found. Otherwise, LIBRARIES is set to FALSE. + + # N.B. _prefix is the prefix applied to the names of all cached variables that + # are generated internally and marked advanced by this macro. + + set(_libdir ${ARGN}) + + set(_libraries_work TRUE) + set(${LIBRARIES}) set(_combined_name) + set(ENV_MKLROOT "$ENV{MKLROOT}") + set(ENV_BLAS_DIR "$ENV{BLAS_DIR}") + set(ENV_BLAS_LIBDIR "$ENV{BLAS_LIBDIR}") + if (NOT _libdir) + if (BLAS_LIBDIR) + list(APPEND _libdir "${BLAS_LIBDIR}") + elseif (BLAS_DIR) + list(APPEND _libdir "${BLAS_DIR}") + list(APPEND _libdir "${BLAS_DIR}/lib") + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + list(APPEND _libdir "${BLAS_DIR}/lib64") + list(APPEND _libdir "${BLAS_DIR}/lib/intel64") + else() + list(APPEND _libdir "${BLAS_DIR}/lib32") + list(APPEND _libdir "${BLAS_DIR}/lib/ia32") + endif() + elseif(ENV_BLAS_LIBDIR) + list(APPEND _libdir "${ENV_BLAS_LIBDIR}") + elseif(ENV_BLAS_DIR) + list(APPEND _libdir "${ENV_BLAS_DIR}") + list(APPEND _libdir "${ENV_BLAS_DIR}/lib") + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + list(APPEND _libdir "${ENV_BLAS_DIR}/lib64") + list(APPEND _libdir "${ENV_BLAS_DIR}/lib/intel64") + else() + list(APPEND _libdir "${ENV_BLAS_DIR}/lib32") + list(APPEND _libdir "${ENV_BLAS_DIR}/lib/ia32") + endif() + else() + if (ENV_MKLROOT) + list(APPEND _libdir "${ENV_MKLROOT}/lib") + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + list(APPEND _libdir "${ENV_MKLROOT}/lib64") + list(APPEND _libdir "${ENV_MKLROOT}/lib/intel64") + else() + list(APPEND _libdir "${ENV_MKLROOT}/lib32") + list(APPEND _libdir "${ENV_MKLROOT}/lib/ia32") + endif() + endif() + if (WIN32) + string(REPLACE ":" ";" _libdir2 "$ENV{LIB}") + elseif (APPLE) + string(REPLACE ":" ";" _libdir2 "$ENV{DYLD_LIBRARY_PATH}") + else () + string(REPLACE ":" ";" _libdir2 "$ENV{LD_LIBRARY_PATH}") + endif () + list(APPEND _libdir "${_libdir2}") + list(APPEND _libdir "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _libdir "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() + endif () + + if (BLAS_VERBOSE) + message("${Cyan}Try to find BLAS libraries: ${_list}") + endif () + foreach(_library ${_list}) set(_combined_name ${_combined_name}_${_library}) - if(_libraries_found) - # search first in ${_path} - find_library(${_prefix}_${_library}_LIBRARY - NAMES ${_library} - PATHS ${_path} NO_DEFAULT_PATH - ) - # if not found, search in environment variables and system - if ( WIN32 ) - find_library(${_prefix}_${_library}_LIBRARY - NAMES ${_library} - PATHS ENV LIB - ) - elseif ( APPLE ) - find_library(${_prefix}_${_library}_LIBRARY - NAMES ${_library} - PATHS /usr/local/lib /usr/lib /usr/local/lib64 /usr/lib64 ENV DYLD_LIBRARY_PATH - ) + if(_libraries_work) + if (BLA_STATIC) + if (WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib ${CMAKE_FIND_LIBRARY_SUFFIXES}) + endif () + if (APPLE) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else () + set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) + endif () else () - find_library(${_prefix}_${_library}_LIBRARY - NAMES ${_library} - PATHS /usr/local/lib /usr/lib /usr/local/lib64 /usr/lib64 ENV LD_LIBRARY_PATH - ) - endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + # for ubuntu's libblas3gf and liblapack3gf packages + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES} .so.3gf) + endif () + endif () + find_library(${_prefix}_${_library}_LIBRARY + NAMES ${_library} + HINTS ${_libdir} + NO_DEFAULT_PATH + ) mark_as_advanced(${_prefix}_${_library}_LIBRARY) + # Print status if not found + # ------------------------- + if (NOT ${_prefix}_${_library}_LIBRARY AND NOT BLAS_FIND_QUIETLY AND BLAS_VERBOSE) + Print_Find_Library_Blas_Status(blas ${_library} ${_libdir}) + endif () set(${LIBRARIES} ${${LIBRARIES}} ${${_prefix}_${_library}_LIBRARY}) - set(_libraries_found ${${_prefix}_${_library}_LIBRARY}) - endif(_libraries_found) + set(_libraries_work ${${_prefix}_${_library}_LIBRARY}) + endif(_libraries_work) endforeach(_library ${_list}) - if(_libraries_found) - set(_libraries_found ${${LIBRARIES}}) - endif() - # Test this combination of libraries with the Fortran/f2c interface. - # We test the Fortran interface first as it is well standardized. - if(_libraries_found AND NOT _libraries_work) - set(${DEFINITIONS} "-D${_prefix}_USE_F2C") - set(${LIBRARIES} ${_libraries_found}) - # Some C++ linkers require the f2c library to link with Fortran libraries. - # I do not know which ones, thus I just add the f2c library if it is available. - find_package( F2C QUIET ) - if ( F2C_FOUND ) - set(${DEFINITIONS} ${${DEFINITIONS}} ${F2C_DEFINITIONS}) - set(${LIBRARIES} ${${LIBRARIES}} ${F2C_LIBRARIES}) + if(_libraries_work) + # Test this combination of libraries. + if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND BLA_STATIC) + list(INSERT ${LIBRARIES} 0 "-Wl,--start-group") + list(APPEND ${LIBRARIES} "-Wl,--end-group") endif() - set(CMAKE_REQUIRED_DEFINITIONS ${${DEFINITIONS}}) - set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}}) - #message("DEBUG: CMAKE_REQUIRED_DEFINITIONS = ${CMAKE_REQUIRED_DEFINITIONS}") - #message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}") - # Check if function exists with f2c calling convention (ie a trailing underscore) - check_function_exists(${_name}_ ${_prefix}_${_name}_${_combined_name}_f2c_WORKS) - set(CMAKE_REQUIRED_DEFINITIONS} "") - set(CMAKE_REQUIRED_LIBRARIES "") - mark_as_advanced(${_prefix}_${_name}_${_combined_name}_f2c_WORKS) - set(_libraries_work ${${_prefix}_${_name}_${_combined_name}_f2c_WORKS}) - endif(_libraries_found AND NOT _libraries_work) - - # If not found, test this combination of libraries with a C interface. - # A few implementations (ie ACML) provide a C interface. Unfortunately, there is no standard. - if(_libraries_found AND NOT _libraries_work) - set(${DEFINITIONS} "") - set(${LIBRARIES} ${_libraries_found}) - set(CMAKE_REQUIRED_DEFINITIONS "") - set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}}) - #message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}") - check_function_exists(${_name} ${_prefix}_${_name}${_combined_name}_WORKS) - set(CMAKE_REQUIRED_LIBRARIES "") - mark_as_advanced(${_prefix}_${_name}${_combined_name}_WORKS) - set(_libraries_work ${${_prefix}_${_name}${_combined_name}_WORKS}) - endif(_libraries_found AND NOT _libraries_work) - - # on failure - if(NOT _libraries_work) - set(${DEFINITIONS} "") - set(${LIBRARIES} FALSE) + set(CMAKE_REQUIRED_LIBRARIES "${_flags};${${LIBRARIES}};${_thread}") + set(CMAKE_REQUIRED_FLAGS "${BLAS_COMPILER_FLAGS}") + if (BLAS_VERBOSE) + message("${Cyan}BLAS libs found for BLA_VENDOR ${BLA_VENDOR}." + "Try to compile symbol ${_name} with following libraries:" + "${CMAKE_REQUIRED_LIBRARIES}") + endif () + if(NOT BLAS_FOUND) + unset(${_prefix}${_combined_name}_WORKS CACHE) + endif() + if (_CHECK_FORTRAN) + if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + string(REPLACE "mkl_intel_lp64" "mkl_gf_lp64" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + string(REPLACE "mkl_intel_ilp64" "mkl_gf_ilp64" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + endif() + check_fortran_function_exists("${_name}" ${_prefix}${_combined_name}_WORKS) + else() + check_function_exists("${_name}_" ${_prefix}${_combined_name}_WORKS) + endif() + mark_as_advanced(${_prefix}${_combined_name}_WORKS) + set(_libraries_work ${${_prefix}${_combined_name}_WORKS}) + # Print status if not found + # ------------------------- + if (NOT _libraries_work AND NOT BLAS_FIND_QUIETLY AND BLAS_VERBOSE) + Print_Find_Library_Blas_CheckFunc_Status(${_name} ${CMAKE_REQUIRED_LIBRARIES}) + endif () + set(CMAKE_REQUIRED_LIBRARIES) endif() - #message("DEBUG: ${DEFINITIONS} = ${${DEFINITIONS}}") - #message("DEBUG: ${LIBRARIES} = ${${LIBRARIES}}") -endmacro(check_fortran_libraries) + + if(_libraries_work) + set(${LIBRARIES} ${${LIBRARIES}} ${_thread}) + else(_libraries_work) + set(${LIBRARIES} FALSE) + endif(_libraries_work) + +endmacro(Check_Fortran_Libraries) -# -# main -# +set(BLAS_LINKER_FLAGS) +set(BLAS_LIBRARIES) +set(BLAS95_LIBRARIES) +if ($ENV{BLA_VENDOR} MATCHES ".+") + set(BLA_VENDOR $ENV{BLA_VENDOR}) +else () + if(NOT BLA_VENDOR) + set(BLA_VENDOR "All") + endif() +endif () -# Is it already configured? -if (BLAS_LIBRARIES_DIR OR BLAS_LIBRARIES) +#BLAS in intel mkl 10 library? (em64t 64bit) +if (BLA_VENDOR MATCHES "Intel*" OR BLA_VENDOR STREQUAL "All") - set(BLAS_FOUND TRUE) + if(NOT BLAS_LIBRARIES OR BLA_VENDOR MATCHES "Intel*") + # Looking for include + # ------------------- -else() + # Add system include paths to search include + # ------------------------------------------ + unset(_inc_env) + set(ENV_MKLROOT "$ENV{MKLROOT}") + set(ENV_BLAS_DIR "$ENV{BLAS_DIR}") + set(ENV_BLAS_INCDIR "$ENV{BLAS_INCDIR}") + if(ENV_BLAS_INCDIR) + list(APPEND _inc_env "${ENV_BLAS_INCDIR}") + elseif(ENV_BLAS_DIR) + list(APPEND _inc_env "${ENV_BLAS_DIR}") + list(APPEND _inc_env "${ENV_BLAS_DIR}/include") + else() + if (ENV_MKLROOT) + list(APPEND _inc_env "${ENV_MKLROOT}/include") + endif() + # system variables + if(WIN32) + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() + endif() + list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") + list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") + list(REMOVE_DUPLICATES _inc_env) - # reset variables - set( BLAS_INCLUDE_DIR "" ) - set( BLAS_DEFINITIONS "" ) - set( BLAS_LINKER_FLAGS "" ) - set( BLAS_LIBRARIES "" ) - set( BLAS_LIBRARIES_DIR "" ) + # set paths where to look for + set(PATH_TO_LOOK_FOR "${_inc_env}") - # - # If Unix, search for BLAS function in possible libraries - # + # Try to find the fftw header in the given paths + # ------------------------------------------------- + # call cmake macro to find the header path + if(BLAS_INCDIR) + set(BLAS_mkl.h_DIRS "BLAS_mkl.h_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_DIRS + NAMES mkl.h + HINTS ${BLAS_INCDIR}) + else() + if(BLAS_DIR) + set(BLAS_mkl.h_DIRS "BLAS_mkl.h_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_DIRS + NAMES mkl.h + HINTS ${BLAS_DIR} + PATH_SUFFIXES "include") + else() + set(BLAS_mkl.h_DIRS "BLAS_mkl.h_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_DIRS + NAMES mkl.h + HINTS ${PATH_TO_LOOK_FOR}) + endif() + endif() + mark_as_advanced(BLAS_mkl.h_DIRS) - # BLAS in ATLAS library? (http://math-atlas.sourceforge.net/) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + # If found, add path to cmake variable + # ------------------------------------ + if (BLAS_mkl.h_DIRS) + set(BLAS_INCLUDE_DIRS "${BLAS_mkl.h_DIRS}") + else () + set(BLAS_INCLUDE_DIRS "BLAS_INCLUDE_DIRS-NOTFOUND") + if(NOT BLAS_FIND_QUIETLY) + message(STATUS "Looking for BLAS -- mkl.h not found") + endif() + endif() + + if (WIN32) + string(REPLACE ":" ";" _libdir "$ENV{LIB}") + elseif (APPLE) + string(REPLACE ":" ";" _libdir "$ENV{DYLD_LIBRARY_PATH}") + else () + string(REPLACE ":" ";" _libdir "$ENV{LD_LIBRARY_PATH}") + endif () + list(APPEND _libdir "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _libdir "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + # libiomp5 + # -------- + set(OMP_iomp5_LIBRARY "OMP_iomp5_LIBRARY-NOTFOUND") + find_library(OMP_iomp5_LIBRARY + NAMES iomp5 + HINTS ${_libdir} + ) + mark_as_advanced(OMP_iomp5_LIBRARY) + set(OMP_LIB "") + # libgomp + # ------- + set(OMP_gomp_LIBRARY "OMP_gomp_LIBRARY-NOTFOUND") + find_library(OMP_gomp_LIBRARY + NAMES gomp + HINTS ${_libdir} + ) + mark_as_advanced(OMP_gomp_LIBRARY) + # choose one or another depending on the compilo + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + if (OMP_gomp_LIBRARY) + set(OMP_LIB "${OMP_gomp_LIBRARY}") + endif() + else(CMAKE_C_COMPILER_ID STREQUAL "Intel") + if (OMP_iomp5_LIBRARY) + set(OMP_LIB "${OMP_iomp5_LIBRARY}") + endif() + endif() + + if (UNIX AND NOT WIN32) + # m + find_library(M_LIBRARY + NAMES m + HINTS ${_libdir}) + mark_as_advanced(M_LIBRARY) + if(M_LIBRARY) + set(LM "-lm") + else() + set(LM "") + endif() + # Fortran + set(LGFORTRAN "") + if (CMAKE_C_COMPILER_ID MATCHES "GNU") + find_library( + FORTRAN_gfortran_LIBRARY + NAMES gfortran + HINTS ${_libdir} + ) + mark_as_advanced(FORTRAN_gfortran_LIBRARY) + if (FORTRAN_gfortran_LIBRARY) + set(LGFORTRAN "${FORTRAN_gfortran_LIBRARY}") + endif() + elseif (CMAKE_C_COMPILER_ID MATCHES "Intel") + find_library( + FORTRAN_ifcore_LIBRARY + NAMES ifcore + HINTS ${_libdir} + ) + mark_as_advanced(FORTRAN_ifcore_LIBRARY) + if (FORTRAN_ifcore_LIBRARY) + set(LGFORTRAN "{FORTRAN_ifcore_LIBRARY}") + endif() + endif() + set(BLAS_COMPILER_FLAGS "") + if (NOT BLA_VENDOR STREQUAL "Intel10_64lp_seq") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + list(APPEND BLAS_COMPILER_FLAGS "-openmp") + endif() + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + list(APPEND BLAS_COMPILER_FLAGS "-fopenmp") + endif() + endif() + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + if (BLA_VENDOR STREQUAL "Intel10_32") + list(APPEND BLAS_COMPILER_FLAGS "-m32") + else() + list(APPEND BLAS_COMPILER_FLAGS "-m64") + endif() + if (NOT BLA_VENDOR STREQUAL "Intel10_64lp_seq") + list(APPEND OMP_LIB "-ldl") + endif() + if (ENV_MKLROOT) + list(APPEND BLAS_COMPILER_FLAGS "-I${ENV_MKLROOT}/include") + endif() + endif() + + set(additional_flags "") + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(additional_flags "-Wl,--no-as-needed") + endif() + endif () + + if (_LANGUAGES_ MATCHES C OR _LANGUAGES_ MATCHES CXX) + if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED) + find_package(Threads) + else() + find_package(Threads REQUIRED) + endif() + + set(BLAS_SEARCH_LIBS "") + + if(BLA_F95) + + set(BLAS_mkl_SEARCH_SYMBOL SGEMM) + set(_LIBRARIES BLAS95_LIBRARIES) + if (WIN32) + if (BLA_STATIC) + set(BLAS_mkl_DLL_SUFFIX "") + else() + set(BLAS_mkl_DLL_SUFFIX "_dll") + endif() + + # Find the main file (32-bit or 64-bit) + set(BLAS_SEARCH_LIBS_WIN_MAIN "") + if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN + "mkl_blas95${BLAS_mkl_DLL_SUFFIX} mkl_intel_c${BLAS_mkl_DLL_SUFFIX}") + endif() + if (BLA_VENDOR STREQUAL "Intel10_64lp*" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN + "mkl_blas95_lp64${BLAS_mkl_DLL_SUFFIX} mkl_intel_lp64${BLAS_mkl_DLL_SUFFIX}") + endif () + + # Add threading/sequential libs + set(BLAS_SEARCH_LIBS_WIN_THREAD "") + if (BLA_VENDOR STREQUAL "*_seq" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "mkl_sequential${BLAS_mkl_DLL_SUFFIX}") + endif() + if (NOT BLA_VENDOR STREQUAL "*_seq" OR BLA_VENDOR STREQUAL "All") + # old version + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}") + # mkl >= 10.3 + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}") + endif() + + # Cartesian product of the above + foreach (MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN}) + foreach (THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD}) + list(APPEND BLAS_SEARCH_LIBS + "${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}") + endforeach() + endforeach() + else (WIN32) + if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS + "mkl_blas95 mkl_intel mkl_intel_thread mkl_core guide") + endif () + if (BLA_VENDOR STREQUAL "Intel10_64lp" OR BLA_VENDOR STREQUAL "All") + # old version + list(APPEND BLAS_SEARCH_LIBS + "mkl_blas95 mkl_intel_lp64 mkl_intel_thread mkl_core guide") + # mkl >= 10.3 + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + list(APPEND BLAS_SEARCH_LIBS + "mkl_blas95_lp64 mkl_intel_lp64 mkl_intel_thread mkl_core") + endif() + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + list(APPEND BLAS_SEARCH_LIBS + "mkl_blas95_lp64 mkl_intel_lp64 mkl_gnu_thread mkl_core") + endif() + endif () + if (BLA_VENDOR STREQUAL "Intel10_64lp_seq" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel_lp64 mkl_sequential mkl_core") + if (BLA_VENDOR STREQUAL "Intel10_64lp_seq") + set(OMP_LIB "") + endif() + endif () + endif (WIN32) + + else (BLA_F95) + + set(BLAS_mkl_SEARCH_SYMBOL sgemm) + set(_LIBRARIES BLAS_LIBRARIES) + if (WIN32) + if (BLA_STATIC) + set(BLAS_mkl_DLL_SUFFIX "") + else() + set(BLAS_mkl_DLL_SUFFIX "_dll") + endif() + + # Find the main file (32-bit or 64-bit) + set(BLAS_SEARCH_LIBS_WIN_MAIN "") + if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN + "mkl_intel_c${BLAS_mkl_DLL_SUFFIX}") + endif() + if (BLA_VENDOR STREQUAL "Intel10_64lp*" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN + "mkl_intel_lp64${BLAS_mkl_DLL_SUFFIX}") + endif () + + # Add threading/sequential libs + set(BLAS_SEARCH_LIBS_WIN_THREAD "") + if (NOT BLA_VENDOR STREQUAL "*_seq" OR BLA_VENDOR STREQUAL "All") + # old version + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}") + # mkl >= 10.3 + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}") + endif() + if (BLA_VENDOR STREQUAL "*_seq" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD + "mkl_sequential${BLAS_mkl_DLL_SUFFIX}") + endif() + + # Cartesian product of the above + foreach (MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN}) + foreach (THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD}) + list(APPEND BLAS_SEARCH_LIBS + "${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}") + endforeach() + endforeach() + else (WIN32) + if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel mkl_intel_thread mkl_core guide") + endif () + if (BLA_VENDOR STREQUAL "Intel10_64lp" OR BLA_VENDOR STREQUAL "All") + # old version + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel_lp64 mkl_intel_thread mkl_core guide") + # mkl >= 10.3 + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel_lp64 mkl_intel_thread mkl_core") + endif() + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel_lp64 mkl_gnu_thread mkl_core") + endif() + endif () + if (BLA_VENDOR STREQUAL "Intel10_64lp_seq" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS + "mkl_intel_lp64 mkl_sequential mkl_core") + if (BLA_VENDOR STREQUAL "Intel10_64lp_seq") + set(OMP_LIB "") + endif() + endif () + #older vesions of intel mkl libs + if (BLA_VENDOR STREQUAL "Intel" OR BLA_VENDOR STREQUAL "All") + list(APPEND BLAS_SEARCH_LIBS + "mkl") + list(APPEND BLAS_SEARCH_LIBS + "mkl_ia32") + list(APPEND BLAS_SEARCH_LIBS + "mkl_em64t") + endif () + endif (WIN32) + + endif (BLA_F95) + + foreach (IT ${BLAS_SEARCH_LIBS}) + string(REPLACE " " ";" SEARCH_LIBS ${IT}) + if (${_LIBRARIES}) + else () + check_fortran_libraries( + ${_LIBRARIES} + BLAS + ${BLAS_mkl_SEARCH_SYMBOL} + "${additional_flags}" + "${SEARCH_LIBS}" + "${OMP_LIB};${CMAKE_THREAD_LIBS_INIT};${LM}" + ) + if(_LIBRARIES) + set(BLAS_LINKER_FLAGS "${additional_flags}") + endif() + endif() + endforeach () + if(NOT BLAS_FIND_QUIETLY) + if(${_LIBRARIES}) + message(STATUS "Looking for MKL BLAS: found") + else() + message(STATUS "Looking for MKL BLAS: not found") + endif() + endif() + if (${_LIBRARIES} AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Intel MKL") + endif() + endif (_LANGUAGES_ MATCHES C OR _LANGUAGES_ MATCHES CXX) + endif(NOT BLAS_LIBRARIES OR BLA_VENDOR MATCHES "Intel*") +endif (BLA_VENDOR MATCHES "Intel*" OR BLA_VENDOR STREQUAL "All") + + +if (BLA_VENDOR STREQUAL "Goto" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + # gotoblas (http://www.tacc.utexas.edu/tacc-projects/gotoblas2) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "cblas;f77blas;atlas" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "goto2" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Goto BLAS: found") + else() + message(STATUS "Looking for Goto BLAS: not found") + endif() endif() + endif() + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Goto") + endif() - # BLAS in PhiPACK libraries? (requires generic BLAS lib, too) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS +endif (BLA_VENDOR STREQUAL "Goto" OR BLA_VENDOR STREQUAL "All") + + +# OpenBlas +if (BLA_VENDOR STREQUAL "Open" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + # openblas (http://www.openblas.net/) + check_fortran_libraries( + BLAS_LIBRARIES + BLAS + sgemm + "" + "openblas" + "" + ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Open BLAS: found") + else() + message(STATUS "Looking for Open BLAS: not found") + endif() + endif() + endif() + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Openblas") + endif() + +endif (BLA_VENDOR STREQUAL "Open" OR BLA_VENDOR STREQUAL "All") + + +# EigenBlas +if (BLA_VENDOR STREQUAL "Eigen" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + # eigenblas (http://eigen.tuxfamily.org/index.php?title=Main_Page) + check_fortran_libraries( + BLAS_LIBRARIES + BLAS + sgemm + "" + "eigen_blas" + "" + ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + message(STATUS "Looking for Eigen BLAS: found") + else() + message(STATUS "Looking for Eigen BLAS: not found") + endif() + endif() + endif() + + if(NOT BLAS_LIBRARIES) + # eigenblas (http://eigen.tuxfamily.org/index.php?title=Main_Page) + check_fortran_libraries( + BLAS_LIBRARIES + BLAS + sgemm + "" + "eigen_blas_static" + "" + ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Eigen BLAS: found") + else() + message(STATUS "Looking for Eigen BLAS: not found") + endif() + endif() + endif() + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Eigen") + endif() + +endif (BLA_VENDOR STREQUAL "Eigen" OR BLA_VENDOR STREQUAL "All") + + +if (BLA_VENDOR STREQUAL "ATLAS" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + # BLAS in ATLAS library? (http://math-atlas.sourceforge.net/) + check_fortran_libraries( + BLAS_LIBRARIES + BLAS + dgemm + "" + "f77blas;atlas" + "" + ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Atlas BLAS: found") + else() + message(STATUS "Looking for Atlas BLAS: not found") + endif() + endif() + endif() + + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Atlas") + endif() + +endif (BLA_VENDOR STREQUAL "ATLAS" OR BLA_VENDOR STREQUAL "All") + + +# BLAS in PhiPACK libraries? (requires generic BLAS lib, too) +if (BLA_VENDOR STREQUAL "PhiPACK" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" "sgemm;dgemm;blas" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for PhiPACK BLAS: found") + else() + message(STATUS "Looking for PhiPACK BLAS: not found") + endif() endif() + endif() - # BLAS in Alpha CXML library? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "PhiPACK") + endif() + +endif (BLA_VENDOR STREQUAL "PhiPACK" OR BLA_VENDOR STREQUAL "All") + + +# BLAS in Alpha CXML library? +if (BLA_VENDOR STREQUAL "CXML" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" "cxml" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for CXML BLAS: found") + else() + message(STATUS "Looking for CXML BLAS: not found") + endif() endif() + endif() - # BLAS in Alpha DXML library? (now called CXML, see above) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "CXML") + endif() + +endif (BLA_VENDOR STREQUAL "CXML" OR BLA_VENDOR STREQUAL "All") + + +# BLAS in Alpha DXML library? (now called CXML, see above) +if (BLA_VENDOR STREQUAL "DXML" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" "dxml" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for DXML BLAS: found") + else() + message(STATUS "Looking for DXML BLAS: not found") + endif() endif() + endif() - # BLAS in Sun Performance library? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "DXML") + endif() + +endif (BLA_VENDOR STREQUAL "DXML" OR BLA_VENDOR STREQUAL "All") + + +# BLAS in Sun Performance library? +if (BLA_VENDOR STREQUAL "SunPerf" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "-xlic_lib=sunperf" "sunperf;sunmath" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(BLAS_LIBRARIES) + set(BLAS_LINKER_FLAGS "-xlic_lib=sunperf") + endif() + if(NOT BLAS_FIND_QUIETLY) if(BLAS_LIBRARIES) - # Extra linker flag - set(BLAS_LINKER_FLAGS "-xlic_lib=sunperf") + message(STATUS "Looking for SunPerf BLAS: found") + else() + message(STATUS "Looking for SunPerf BLAS: not found") endif() endif() + endif() - # BLAS in SCSL library? (SGI/Cray Scientific Library) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "SunPerf") + endif() + +endif () + + +# BLAS in SCSL library? (SGI/Cray Scientific Library) +if (BLA_VENDOR STREQUAL "SCSL" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" "scsl" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for SCSL BLAS: found") + else() + message(STATUS "Looking for SCSL BLAS: not found") + endif() endif() + endif() - # BLAS in SGIMATH library? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "SunPerf") + endif() + +endif () + + +# BLAS in SGIMATH library? +if (BLA_VENDOR STREQUAL "SGIMATH" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" "complib.sgimath" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for SGIMATH BLAS: found") + else() + message(STATUS "Looking for SGIMATH BLAS: not found") + endif() endif() + endif() - # BLAS in IBM ESSL library? (requires generic BLAS lib, too) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "SGIMATH") + endif() + +endif () + + +# BLAS in IBM ESSL library (requires generic BLAS lib, too) +if (BLA_VENDOR STREQUAL "IBMESSL" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "essl;blas" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "essl;xlfmath;xlf90_r;blas" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for IBM ESSL BLAS: found") + else() + message(STATUS "Looking for IBM ESSL BLAS: not found") + endif() endif() + endif() - #BLAS in intel mkl 10 library? (em64t 64bit) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "IBM ESSL") + endif() + +endif () + +# BLAS in IBM ESSL_MT library (requires generic BLAS lib, too) +if (BLA_VENDOR STREQUAL "IBMESSLMT" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "mkl_intel_lp64;mkl_intel_thread;mkl_core;guide;pthread" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" - ) - endif() - - ### windows version of intel mkl 10? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS - BLAS_LIBRARIES - BLAS - SGEMM + "esslsmp;xlsmp;xlfmath;xlf90_r;blas" "" - "mkl_c_dll;mkl_intel_thread_dll;mkl_core_dll;libguide40" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for IBM ESSL MT BLAS: found") + else() + message(STATUS "Looking for IBM ESSL MT BLAS: not found") + endif() endif() + endif() - #older versions of intel mkl libs + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "IBM ESSL MT") + endif() - # BLAS in intel mkl library? (shared) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS +endif () + + +#BLAS in acml library? +if (BLA_VENDOR MATCHES "ACML.*" OR BLA_VENDOR STREQUAL "All") + + if( ((BLA_VENDOR STREQUAL "ACML") AND (NOT BLAS_ACML_LIB_DIRS)) OR + ((BLA_VENDOR STREQUAL "ACML_MP") AND (NOT BLAS_ACML_MP_LIB_DIRS)) OR + ((BLA_VENDOR STREQUAL "ACML_GPU") AND (NOT BLAS_ACML_GPU_LIB_DIRS))) + + # try to find acml in "standard" paths + if( WIN32 ) + file( GLOB _ACML_ROOT "C:/AMD/acml*/ACML-EULA.txt" ) + else() + file( GLOB _ACML_ROOT "/opt/acml*/ACML-EULA.txt" ) + endif() + if( WIN32 ) + file( GLOB _ACML_GPU_ROOT "C:/AMD/acml*/GPGPUexamples" ) + else() + file( GLOB _ACML_GPU_ROOT "/opt/acml*/GPGPUexamples" ) + endif() + list(GET _ACML_ROOT 0 _ACML_ROOT) + list(GET _ACML_GPU_ROOT 0 _ACML_GPU_ROOT) + + if( _ACML_ROOT ) + + get_filename_component( _ACML_ROOT ${_ACML_ROOT} PATH ) + if( SIZEOF_INTEGER EQUAL 8 ) + set( _ACML_PATH_SUFFIX "_int64" ) + else() + set( _ACML_PATH_SUFFIX "" ) + endif() + if( CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" ) + set( _ACML_COMPILER32 "ifort32" ) + set( _ACML_COMPILER64 "ifort64" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "SunPro" ) + set( _ACML_COMPILER32 "sun32" ) + set( _ACML_COMPILER64 "sun64" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "PGI" ) + set( _ACML_COMPILER32 "pgi32" ) + if( WIN32 ) + set( _ACML_COMPILER64 "win64" ) + else() + set( _ACML_COMPILER64 "pgi64" ) + endif() + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "Open64" ) + # 32 bit builds not supported on Open64 but for code simplicity + # We'll just use the same directory twice + set( _ACML_COMPILER32 "open64_64" ) + set( _ACML_COMPILER64 "open64_64" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "NAG" ) + set( _ACML_COMPILER32 "nag32" ) + set( _ACML_COMPILER64 "nag64" ) + else() + set( _ACML_COMPILER32 "gfortran32" ) + set( _ACML_COMPILER64 "gfortran64" ) + endif() + + if( BLA_VENDOR STREQUAL "ACML_MP" ) + set(_ACML_MP_LIB_DIRS + "${_ACML_ROOT}/${_ACML_COMPILER32}_mp${_ACML_PATH_SUFFIX}/lib" + "${_ACML_ROOT}/${_ACML_COMPILER64}_mp${_ACML_PATH_SUFFIX}/lib" ) + else() + set(_ACML_LIB_DIRS + "${_ACML_ROOT}/${_ACML_COMPILER32}${_ACML_PATH_SUFFIX}/lib" + "${_ACML_ROOT}/${_ACML_COMPILER64}${_ACML_PATH_SUFFIX}/lib" ) + endif() + + endif(_ACML_ROOT) + + elseif(BLAS_${BLA_VENDOR}_LIB_DIRS) + + set(_${BLA_VENDOR}_LIB_DIRS ${BLAS_${BLA_VENDOR}_LIB_DIRS}) + + endif() + + if( BLA_VENDOR STREQUAL "ACML_MP" ) + foreach( BLAS_ACML_MP_LIB_DIRS ${_ACML_MP_LIB_DIRS}) + check_fortran_libraries ( + BLAS_LIBRARIES + BLAS + sgemm + "" "acml_mp;acml_mv" "" ${BLAS_ACML_MP_LIB_DIRS} + ) + if( BLAS_LIBRARIES ) + break() + endif() + endforeach() + elseif( BLA_VENDOR STREQUAL "ACML_GPU" ) + foreach( BLAS_ACML_GPU_LIB_DIRS ${_ACML_GPU_LIB_DIRS}) + check_fortran_libraries ( + BLAS_LIBRARIES + BLAS + sgemm + "" "acml;acml_mv;CALBLAS" "" ${BLAS_ACML_GPU_LIB_DIRS} + ) + if( BLAS_LIBRARIES ) + break() + endif() + endforeach() + else() + foreach( BLAS_ACML_LIB_DIRS ${_ACML_LIB_DIRS} ) + check_fortran_libraries ( + BLAS_LIBRARIES + BLAS + sgemm + "" "acml;acml_mv" "" ${BLAS_ACML_LIB_DIRS} + ) + if( BLAS_LIBRARIES ) + break() + endif() + endforeach() + endif() + + # Either acml or acml_mp should be in LD_LIBRARY_PATH but not both + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "mkl;guide;pthread" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "acml;acml_mv" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for ACML BLAS: found") + else() + message(STATUS "Looking for ACML BLAS: not found") + endif() endif() + endif() - #BLAS in intel mkl library? (static, 32bit) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "mkl_ia32;guide;pthread" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "acml_mp;acml_mv" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for ACML BLAS: found") + else() + message(STATUS "Looking for ACML BLAS: not found") + endif() endif() + endif() - #BLAS in intel mkl library? (static, em64t 64bit) - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS sgemm "" - "mkl_em64t;guide;pthread" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" - ) - endif() - - #BLAS in acml library? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS - BLAS_LIBRARIES - BLAS - sgemm + "acml;acml_mv;CALBLAS" "" - "acml" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for ACML BLAS: found") + else() + message(STATUS "Looking for ACML BLAS: not found") + endif() endif() + endif() - # Apple BLAS library? - if(NOT BLAS_LIBRARIES) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "ACML") + endif() + +endif (BLA_VENDOR MATCHES "ACML.*" OR BLA_VENDOR STREQUAL "All") # ACML + + +# Apple BLAS library? +if (BLA_VENDOR STREQUAL "Apple" OR BLA_VENDOR STREQUAL "All") + + if(NOT BLAS_LIBRARIES) + check_fortran_libraries( BLAS_LIBRARIES BLAS - sgemm + dgemm "" "Accelerate" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" + "" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Apple BLAS: found") + else() + message(STATUS "Looking for Apple BLAS: not found") + endif() endif() + endif() - if ( NOT BLAS_LIBRARIES ) - check_fortran_libraries( - BLAS_DEFINITIONS + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Apple Accelerate") + endif() + +endif (BLA_VENDOR STREQUAL "Apple" OR BLA_VENDOR STREQUAL "All") + + +if (BLA_VENDOR STREQUAL "NAS" OR BLA_VENDOR STREQUAL "All") + + if ( NOT BLAS_LIBRARIES ) + check_fortran_libraries( BLAS_LIBRARIES BLAS - sgemm + dgemm "" "vecLib" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" - ) - endif ( NOT BLAS_LIBRARIES ) - - # Generic BLAS library? - # This configuration *must* be the last try as this library is notably slow. - if ( NOT BLAS_LIBRARIES ) - check_fortran_libraries( - BLAS_DEFINITIONS - BLAS_LIBRARIES - BLAS - sgemm "" - "blas" - "${CGAL_TAUCS_LIBRARIES_DIR} ENV BLAS_LIB_DIR" ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for NAS BLAS: found") + else() + message(STATUS "Looking for NAS BLAS: not found") + endif() endif() + endif () - if(BLAS_LIBRARIES_DIR OR BLAS_LIBRARIES) + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "NAS") + endif() + +endif (BLA_VENDOR STREQUAL "NAS" OR BLA_VENDOR STREQUAL "All") + + +# Generic BLAS library? +if (BLA_VENDOR STREQUAL "Generic" OR BLA_VENDOR STREQUAL "All") + + set(BLAS_SEARCH_LIBS "blas;blas_LINUX;blas_MAC;blas_WINDOWS;refblas") + foreach (SEARCH_LIB ${BLAS_SEARCH_LIBS}) + if (BLAS_LIBRARIES) + else () + check_fortran_libraries( + BLAS_LIBRARIES + BLAS + sgemm + "" + "${SEARCH_LIB}" + "${LGFORTRAN}" + ) + if(NOT BLAS_FIND_QUIETLY) + if(BLAS_LIBRARIES) + message(STATUS "Looking for Generic BLAS: found") + else() + message(STATUS "Looking for Generic BLAS: not found") + endif() + endif() + endif() + endforeach () + + if (BLAS_LIBRARIES AND NOT BLAS_VENDOR_FOUND) + set (BLAS_VENDOR_FOUND "Netlib or other Generic libblas") + endif() + +endif (BLA_VENDOR STREQUAL "Generic" OR BLA_VENDOR STREQUAL "All") + + +if(BLA_F95) + + if(BLAS95_LIBRARIES) + set(BLAS95_FOUND TRUE) + else() + set(BLAS95_FOUND FALSE) + endif() + + if(NOT BLAS_FIND_QUIETLY) + if(BLAS95_FOUND) + message(STATUS "A library with BLAS95 API found.") + message(STATUS "BLAS_LIBRARIES ${BLAS_LIBRARIES}") + else(BLAS95_FOUND) + message(WARNING "BLA_VENDOR has been set to ${BLA_VENDOR} but blas 95 libraries could not be found or check of symbols failed." + "\nPlease indicate where to find blas libraries. You have three options:\n" + "- Option 1: Provide the installation directory of BLAS library with cmake option: -DBLAS_DIR=your/path/to/blas\n" + "- Option 2: Provide the directory where to find BLAS libraries with cmake option: -DBLAS_LIBDIR=your/path/to/blas/libs\n" + "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n" + "\nTo follow libraries detection more precisely you can activate a verbose mode with -DBLAS_VERBOSE=ON at cmake configure." + "\nYou could also specify a BLAS vendor to look for by setting -DBLA_VENDOR=blas_vendor_name." + "\nList of possible BLAS vendor: Goto, ATLAS PhiPACK, CXML, DXML, SunPerf, SCSL, SGIMATH, IBMESSL, Intel10_32 (intel mkl v10 32 bit)," + "Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model), Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model)," + "Intel( older versions of mkl 32 and 64 bit), ACML, ACML_MP, ACML_GPU, Apple, NAS, Generic") + if(BLAS_FIND_REQUIRED) + message(FATAL_ERROR + "A required library with BLAS95 API not found. Please specify library location.") + else() + message(STATUS + "A library with BLAS95 API not found. Please specify library location.") + endif() + endif(BLAS95_FOUND) + endif(NOT BLAS_FIND_QUIETLY) + + set(BLAS_FOUND TRUE) + set(BLAS_LIBRARIES "${BLAS95_LIBRARIES}") + +else(BLA_F95) + + if(BLAS_LIBRARIES) set(BLAS_FOUND TRUE) else() set(BLAS_FOUND FALSE) @@ -388,32 +1366,41 @@ if(NOT BLAS_FIND_QUIETLY) if(BLAS_FOUND) message(STATUS "A library with BLAS API found.") + message(STATUS "BLAS_LIBRARIES ${BLAS_LIBRARIES}") else(BLAS_FOUND) + message(WARNING "BLA_VENDOR has been set to ${BLA_VENDOR} but blas libraries could not be found or check of symbols failed." + "\nPlease indicate where to find blas libraries. You have three options:\n" + "- Option 1: Provide the installation directory of BLAS library with cmake option: -DBLAS_DIR=your/path/to/blas\n" + "- Option 2: Provide the directory where to find BLAS libraries with cmake option: -DBLAS_LIBDIR=your/path/to/blas/libs\n" + "- Option 3: Update your environment variable (Linux: LD_LIBRARY_PATH, Windows: LIB, Mac: DYLD_LIBRARY_PATH)\n" + "\nTo follow libraries detection more precisely you can activate a verbose mode with -DBLAS_VERBOSE=ON at cmake configure." + "\nYou could also specify a BLAS vendor to look for by setting -DBLA_VENDOR=blas_vendor_name." + "\nList of possible BLAS vendor: Goto, ATLAS PhiPACK, CXML, DXML, SunPerf, SCSL, SGIMATH, IBMESSL, Intel10_32 (intel mkl v10 32 bit)," + "Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model), Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model)," + "Intel( older versions of mkl 32 and 64 bit), ACML, ACML_MP, ACML_GPU, Apple, NAS, Generic") if(BLAS_FIND_REQUIRED) - message(FATAL_ERROR "A required library with BLAS API not found. Please specify library location.") + message(FATAL_ERROR + "A required library with BLAS API not found. Please specify library location.") else() - message(STATUS "A library with BLAS API not found. Please specify library location.") + message(STATUS + "A library with BLAS API not found. Please specify library location.") endif() endif(BLAS_FOUND) endif(NOT BLAS_FIND_QUIETLY) - # Add variables to cache - set( BLAS_INCLUDE_DIR "${BLAS_INCLUDE_DIR}" - CACHE PATH "Directories containing the BLAS header files" FORCE ) - set( BLAS_DEFINITIONS "${BLAS_DEFINITIONS}" - CACHE STRING "Compilation options to use BLAS" FORCE ) - set( BLAS_LINKER_FLAGS "${BLAS_LINKER_FLAGS}" - CACHE STRING "Linker flags to use BLAS" FORCE ) - set( BLAS_LIBRARIES "${BLAS_LIBRARIES}" - CACHE FILEPATH "BLAS libraries name" FORCE ) - set( BLAS_LIBRARIES_DIR "${BLAS_LIBRARIES_DIR}" - CACHE PATH "Directories containing the BLAS libraries" FORCE ) +endif(BLA_F95) - #message("DEBUG: BLAS_INCLUDE_DIR = ${BLAS_INCLUDE_DIR}") - #message("DEBUG: BLAS_DEFINITIONS = ${BLAS_DEFINITIONS}") - #message("DEBUG: BLAS_LINKER_FLAGS = ${BLAS_LINKER_FLAGS}") - #message("DEBUG: BLAS_LIBRARIES = ${BLAS_LIBRARIES}") - #message("DEBUG: BLAS_LIBRARIES_DIR = ${BLAS_LIBRARIES_DIR}") - #message("DEBUG: BLAS_FOUND = ${BLAS_FOUND}") +set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -endif(BLAS_LIBRARIES_DIR OR BLAS_LIBRARIES) +if (BLAS_FOUND) + list(GET BLAS_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + if (${first_lib_path} MATCHES "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)") + string(REGEX REPLACE "(/lib(32|64)?$)|(/lib/intel64$|/lib/ia32$)" "" not_cached_dir "${first_lib_path}") + set(BLAS_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of BLAS library" FORCE) + else() + set(BLAS_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of BLAS library" FORCE) + endif() +endif() +mark_as_advanced(BLAS_DIR) +mark_as_advanced(BLAS_DIR_FOUND)
diff --git a/cmake/FindBLASEXT.cmake b/cmake/FindBLASEXT.cmake new file mode 100644 index 0000000..0fe7fb8 --- /dev/null +++ b/cmake/FindBLASEXT.cmake
@@ -0,0 +1,380 @@ +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2016 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find BLAS EXTENDED for MORSE projects: find include dirs and libraries +# +# This module allows to find BLAS libraries by calling the official FindBLAS module +# and handles the creation of different library lists whether the user wishes to link +# with a sequential BLAS or a multihreaded (BLAS_SEQ_LIBRARIES and BLAS_PAR_LIBRARIES). +# BLAS is detected with a FindBLAS call then if the BLAS vendor is Intel10_64lp, ACML +# or IBMESSLMT then the module attempts to find the corresponding multithreaded libraries. +# +# The following variables have been added to manage links with sequential or multithreaded +# versions: +# BLAS_INCLUDE_DIRS - BLAS include directories +# BLAS_LIBRARY_DIRS - Link directories for BLAS libraries +# BLAS_SEQ_LIBRARIES - BLAS component libraries to be linked (sequential) +# BLAS_PAR_LIBRARIES - BLAS component libraries to be linked (multithreaded) + +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013-2016 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) + +# macro to factorize this call +macro(find_package_blas) + if(BLASEXT_FIND_REQUIRED) + if(BLASEXT_FIND_QUIETLY) + find_package(BLAS REQUIRED QUIET) + else() + find_package(BLAS REQUIRED) + endif() + else() + if(BLASEXT_FIND_QUIETLY) + find_package(BLAS QUIET) + else() + find_package(BLAS) + endif() + endif() +endmacro() + +# add a cache variable to let the user specify the BLAS vendor +set(BLA_VENDOR "" CACHE STRING "list of possible BLAS vendor: + Open, Eigen, Goto, ATLAS PhiPACK, CXML, DXML, SunPerf, SCSL, SGIMATH, IBMESSL, IBMESSLMT, + Intel10_32 (intel mkl v10 32 bit), + Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model), + Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model), + Intel( older versions of mkl 32 and 64 bit), + ACML, ACML_MP, ACML_GPU, Apple, NAS, Generic") + +if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "In FindBLASEXT") + message(STATUS "If you want to force the use of one specific library, " + "\n please specify the BLAS vendor by setting -DBLA_VENDOR=blas_vendor_name" + "\n at cmake configure.") + message(STATUS "List of possible BLAS vendor: Goto, ATLAS PhiPACK, CXML, " + "\n DXML, SunPerf, SCSL, SGIMATH, IBMESSL, IBMESSLMT, Intel10_32 (intel mkl v10 32 bit)," + "\n Intel10_64lp (intel mkl v10 64 bit, lp thread model, lp64 model)," + "\n Intel10_64lp_seq (intel mkl v10 64 bit, sequential code, lp64 model)," + "\n Intel( older versions of mkl 32 and 64 bit)," + "\n ACML, ACML_MP, ACML_GPU, Apple, NAS, Generic") +endif() + +if (NOT BLAS_FOUND) + # First try to detect two cases: + # 1: only SEQ libs are handled + # 2: both SEQ and PAR libs are handled + find_package_blas() +endif () + +# detect the cases where SEQ and PAR libs are handled +if(BLA_VENDOR STREQUAL "All" AND + (BLAS_mkl_core_LIBRARY OR BLAS_mkl_core_dll_LIBRARY) + ) + set(BLA_VENDOR "Intel") + if(BLAS_mkl_intel_LIBRARY) + set(BLA_VENDOR "Intel10_32") + endif() + if(BLAS_mkl_intel_lp64_LIBRARY) + set(BLA_VENDOR "Intel10_64lp") + endif() + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "A BLAS library has been found (${BLAS_LIBRARIES}) but we" + "\n have also potentially detected some multithreaded BLAS libraries from the MKL." + "\n We try to find both libraries lists (Sequential/Multithreaded).") + endif() + set(BLAS_FOUND "") +elseif(BLA_VENDOR STREQUAL "All" AND BLAS_acml_LIBRARY) + set(BLA_VENDOR "ACML") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "A BLAS library has been found (${BLAS_LIBRARIES}) but we" + "\n have also potentially detected some multithreaded BLAS libraries from the ACML." + "\n We try to find both libraries lists (Sequential/Multithreaded).") + endif() + set(BLAS_FOUND "") +elseif(BLA_VENDOR STREQUAL "All" AND BLAS_essl_LIBRARY) + set(BLA_VENDOR "IBMESSL") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "A BLAS library has been found (${BLAS_LIBRARIES}) but we" + "\n have also potentially detected some multithreaded BLAS libraries from the ESSL." + "\n We try to find both libraries lists (Sequential/Multithreaded).") + endif() + set(BLAS_FOUND "") +endif() + +# Intel case +if(BLA_VENDOR MATCHES "Intel*") + + ### + # look for include path if the BLAS vendor is Intel + ### + + # gather system include paths + unset(_inc_env) + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() + list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") + list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") + set(ENV_MKLROOT "$ENV{MKLROOT}") + if (ENV_MKLROOT) + list(APPEND _inc_env "${ENV_MKLROOT}/include") + endif() + list(REMOVE_DUPLICATES _inc_env) + + # find mkl.h inside known include paths + set(BLAS_mkl.h_INCLUDE_DIRS "BLAS_mkl.h_INCLUDE_DIRS-NOTFOUND") + if(BLAS_INCDIR) + set(BLAS_mkl.h_INCLUDE_DIRS "BLAS_mkl.h_INCLUDE_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_INCLUDE_DIRS + NAMES mkl.h + HINTS ${BLAS_INCDIR}) + else() + if(BLAS_DIR) + set(BLAS_mkl.h_INCLUDE_DIRS "BLAS_mkl.h_INCLUDE_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_INCLUDE_DIRS + NAMES mkl.h + HINTS ${BLAS_DIR} + PATH_SUFFIXES include) + else() + set(BLAS_mkl.h_INCLUDE_DIRS "BLAS_mkl.h_INCLUDE_DIRS-NOTFOUND") + find_path(BLAS_mkl.h_INCLUDE_DIRS + NAMES mkl.h + HINTS ${_inc_env}) + endif() + endif() + mark_as_advanced(BLAS_mkl.h_INCLUDE_DIRS) + ## Print status if not found + ## ------------------------- + #if (NOT BLAS_mkl.h_INCLUDE_DIRS AND MORSE_VERBOSE) + # Print_Find_Header_Status(blas mkl.h) + #endif () + set(BLAS_INCLUDE_DIRS "") + if(BLAS_mkl.h_INCLUDE_DIRS) + list(APPEND BLAS_INCLUDE_DIRS "${BLAS_mkl.h_INCLUDE_DIRS}" ) + endif() + + ### + # look for libs + ### + # if Intel 10 64 bit -> look for sequential and multithreaded versions + if(BLA_VENDOR MATCHES "Intel10_64lp*") + + ## look for the sequential version + set(BLA_VENDOR "Intel10_64lp_seq") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "Look for the sequential version Intel10_64lp_seq") + endif() + find_package_blas() + if(BLAS_FOUND) + set(BLAS_SEQ_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_SEQ_LIBRARIES "${BLAS_SEQ_LIBRARIES-NOTFOUND}") + endif() + + ## look for the multithreaded version + set(BLA_VENDOR "Intel10_64lp") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "Look for the multithreaded version Intel10_64lp") + endif() + find_package_blas() + if(BLAS_FOUND) + set(BLAS_PAR_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_PAR_LIBRARIES "${BLAS_PAR_LIBRARIES-NOTFOUND}") + endif() + + else() + + if(BLAS_FOUND) + set(BLAS_SEQ_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_SEQ_LIBRARIES "${BLAS_SEQ_LIBRARIES-NOTFOUND}") + endif() + + endif() + + # ACML case +elseif(BLA_VENDOR MATCHES "ACML*") + + ## look for the sequential version + set(BLA_VENDOR "ACML") + find_package_blas() + if(BLAS_FOUND) + set(BLAS_SEQ_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_SEQ_LIBRARIES "${BLAS_SEQ_LIBRARIES-NOTFOUND}") + endif() + + ## look for the multithreaded version + set(BLA_VENDOR "ACML_MP") + find_package_blas() + if(BLAS_FOUND) + set(BLAS_PAR_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_PAR_LIBRARIES "${BLAS_PAR_LIBRARIES-NOTFOUND}") + endif() + + # IBMESSL case +elseif(BLA_VENDOR MATCHES "IBMESSL*") + + ## look for the sequential version + set(BLA_VENDOR "IBMESSL") + find_package_blas() + if(BLAS_FOUND) + set(BLAS_SEQ_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_SEQ_LIBRARIES "${BLAS_SEQ_LIBRARIES-NOTFOUND}") + endif() + + ## look for the multithreaded version + set(BLA_VENDOR "IBMESSLMT") + find_package_blas() + if(BLAS_FOUND) + set(BLAS_PAR_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_PAR_LIBRARIES "${BLAS_PAR_LIBRARIES-NOTFOUND}") + endif() + +else() + + if(BLAS_FOUND) + # define the SEQ libs as the BLAS_LIBRARIES + set(BLAS_SEQ_LIBRARIES "${BLAS_LIBRARIES}") + else() + set(BLAS_SEQ_LIBRARIES "${BLAS_SEQ_LIBRARIES-NOTFOUND}") + endif() + set(BLAS_PAR_LIBRARIES "${BLAS_PAR_LIBRARIES-NOTFOUND}") + +endif() + + +if(BLAS_SEQ_LIBRARIES) + set(BLAS_LIBRARIES "${BLAS_SEQ_LIBRARIES}") +endif() + +# extract libs paths +# remark: because it is not given by find_package(BLAS) +set(BLAS_LIBRARY_DIRS "") +string(REPLACE " " ";" BLAS_LIBRARIES "${BLAS_LIBRARIES}") +foreach(blas_lib ${BLAS_LIBRARIES}) + if (EXISTS "${blas_lib}") + get_filename_component(a_blas_lib_dir "${blas_lib}" PATH) + list(APPEND BLAS_LIBRARY_DIRS "${a_blas_lib_dir}" ) + else() + string(REPLACE "-L" "" blas_lib "${blas_lib}") + if (EXISTS "${blas_lib}") + list(APPEND BLAS_LIBRARY_DIRS "${blas_lib}" ) + else() + get_filename_component(a_blas_lib_dir "${blas_lib}" PATH) + if (EXISTS "${a_blas_lib_dir}") + list(APPEND BLAS_LIBRARY_DIRS "${a_blas_lib_dir}" ) + endif() + endif() + endif() +endforeach() +if (BLAS_LIBRARY_DIRS) + list(REMOVE_DUPLICATES BLAS_LIBRARY_DIRS) +endif () + +# check that BLAS has been found +# --------------------------------- +include(FindPackageHandleStandardArgs) +if(BLA_VENDOR MATCHES "Intel*") + if(BLA_VENDOR MATCHES "Intel10_64lp*") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS found is Intel MKL:" + "\n we manage two lists of libs, one sequential and one parallel if found" + "\n (see BLAS_SEQ_LIBRARIES and BLAS_PAR_LIBRARIES)") + message(STATUS "BLAS sequential libraries stored in BLAS_SEQ_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_SEQ_LIBRARIES + BLAS_LIBRARY_DIRS + BLAS_INCLUDE_DIRS) + if(BLAS_PAR_LIBRARIES) + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS parallel libraries stored in BLAS_PAR_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_PAR_LIBRARIES) + endif() + else() + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS sequential libraries stored in BLAS_SEQ_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_SEQ_LIBRARIES + BLAS_LIBRARY_DIRS + BLAS_INCLUDE_DIRS) + endif() +elseif(BLA_VENDOR MATCHES "ACML*") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS found is ACML:" + "\n we manage two lists of libs, one sequential and one parallel if found" + "\n (see BLAS_SEQ_LIBRARIES and BLAS_PAR_LIBRARIES)") + message(STATUS "BLAS sequential libraries stored in BLAS_SEQ_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_SEQ_LIBRARIES + BLAS_LIBRARY_DIRS) + if(BLAS_PAR_LIBRARIES) + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS parallel libraries stored in BLAS_PAR_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_PAR_LIBRARIES) + endif() +elseif(BLA_VENDOR MATCHES "IBMESSL*") + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS found is ESSL:" + "\n we manage two lists of libs, one sequential and one parallel if found" + "\n (see BLAS_SEQ_LIBRARIES and BLAS_PAR_LIBRARIES)") + message(STATUS "BLAS sequential libraries stored in BLAS_SEQ_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_SEQ_LIBRARIES + BLAS_LIBRARY_DIRS) + if(BLAS_PAR_LIBRARIES) + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS parallel libraries stored in BLAS_PAR_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_PAR_LIBRARIES) + endif() +else() + if(NOT BLASEXT_FIND_QUIETLY) + message(STATUS "BLAS sequential libraries stored in BLAS_SEQ_LIBRARIES") + endif() + find_package_handle_standard_args(BLAS DEFAULT_MSG + BLAS_SEQ_LIBRARIES + BLAS_LIBRARY_DIRS) +endif()
diff --git a/cmake/FindComputeCpp.cmake b/cmake/FindComputeCpp.cmake new file mode 100644 index 0000000..29f2a50 --- /dev/null +++ b/cmake/FindComputeCpp.cmake
@@ -0,0 +1,266 @@ +#.rst: +# FindComputeCpp +#--------------- +# +# Copyright 2016 Codeplay Software Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use these files except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################### +# FindComputeCpp.cmake +######################### +# +# Tools for finding and building with ComputeCpp. +# +# User must define COMPUTECPP_PACKAGE_ROOT_DIR pointing to the ComputeCpp +# installation. +# +# Latest version of this file can be found at: +# https://github.com/codeplaysoftware/computecpp-sdk + +# Require CMake version 3.2.2 or higher +cmake_minimum_required(VERSION 3.2.2) + +# Check that a supported host compiler can be found +if(CMAKE_COMPILER_IS_GNUCXX) + # Require at least gcc 4.8 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) + message(FATAL_ERROR + "host compiler - Not found! (gcc version must be at least 4.8)") + else() + message(STATUS "host compiler - gcc ${CMAKE_CXX_COMPILER_VERSION}") + endif() +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # Require at least clang 3.6 + if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.6) + message(FATAL_ERROR + "host compiler - Not found! (clang version must be at least 3.6)") + else() + message(STATUS "host compiler - clang ${CMAKE_CXX_COMPILER_VERSION}") + endif() +else() + message(WARNING + "host compiler - Not found! (ComputeCpp supports GCC and Clang, see readme)") +endif() + +set(COMPUTECPP_64_BIT_DEFAULT ON) +option(COMPUTECPP_64_BIT_CODE "Compile device code in 64 bit mode" + ${COMPUTECPP_64_BIT_DEFAULT}) +mark_as_advanced(COMPUTECPP_64_BIT_CODE) + +option(COMPUTECPP_DISABLE_GCC_DUAL_ABI "Compile with pre-5.1 ABI" OFF) +mark_as_advanced(COMPUTECPP_DISABLE_GCC_DUAL_ABI) + +set(COMPUTECPP_USER_FLAGS "" CACHE STRING "User flags for compute++") +mark_as_advanced(COMPUTECPP_USER_FLAGS) + +# Find OpenCL package +find_package(OpenCL REQUIRED) + +# Find ComputeCpp packagee +if(NOT COMPUTECPP_PACKAGE_ROOT_DIR) + message(FATAL_ERROR + "ComputeCpp package - Not found! (please set COMPUTECPP_PACKAGE_ROOT_DIR") +else() + message(STATUS "ComputeCpp package - Found") +endif() + +# Obtain the path to compute++ +find_program(COMPUTECPP_DEVICE_COMPILER compute++ PATHS + ${COMPUTECPP_PACKAGE_ROOT_DIR} PATH_SUFFIXES bin) +if (EXISTS ${COMPUTECPP_DEVICE_COMPILER}) + mark_as_advanced(COMPUTECPP_DEVICE_COMPILER) + message(STATUS "compute++ - Found") +else() + message(FATAL_ERROR "compute++ - Not found! (${COMPUTECPP_DEVICE_COMPILER})") +endif() + +# Obtain the path to computecpp_info +find_program(COMPUTECPP_INFO_TOOL computecpp_info PATHS + ${COMPUTECPP_PACKAGE_ROOT_DIR} PATH_SUFFIXES bin) +if (EXISTS ${COMPUTECPP_INFO_TOOL}) + mark_as_advanced(${COMPUTECPP_INFO_TOOL}) + message(STATUS "computecpp_info - Found") +else() + message(FATAL_ERROR "computecpp_info - Not found! (${COMPUTECPP_INFO_TOOL})") +endif() + +# Obtain the path to the ComputeCpp runtime library +find_library(COMPUTECPP_RUNTIME_LIBRARY ComputeCpp PATHS ${COMPUTECPP_PACKAGE_ROOT_DIR} + HINTS ${COMPUTECPP_PACKAGE_ROOT_DIR}/lib PATH_SUFFIXES lib + DOC "ComputeCpp Runtime Library" NO_DEFAULT_PATH) + +if (EXISTS ${COMPUTECPP_RUNTIME_LIBRARY}) + mark_as_advanced(COMPUTECPP_RUNTIME_LIBRARY) + message(STATUS "libComputeCpp.so - Found") +else() + message(FATAL_ERROR "libComputeCpp.so - Not found!") +endif() + +# Obtain the ComputeCpp include directory +set(COMPUTECPP_INCLUDE_DIRECTORY ${COMPUTECPP_PACKAGE_ROOT_DIR}/include/) +if (NOT EXISTS ${COMPUTECPP_INCLUDE_DIRECTORY}) + message(FATAL_ERROR "ComputeCpp includes - Not found!") +else() + message(STATUS "ComputeCpp includes - Found") +endif() + +# Obtain the package version +execute_process(COMMAND ${COMPUTECPP_INFO_TOOL} "--dump-version" + OUTPUT_VARIABLE COMPUTECPP_PACKAGE_VERSION + RESULT_VARIABLE COMPUTECPP_INFO_TOOL_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT COMPUTECPP_INFO_TOOL_RESULT EQUAL "0") + message(FATAL_ERROR "Package version - Error obtaining version!") +else() + mark_as_advanced(COMPUTECPP_PACKAGE_VERSION) + message(STATUS "Package version - ${COMPUTECPP_PACKAGE_VERSION}") +endif() + +# Obtain the device compiler flags +execute_process(COMMAND ${COMPUTECPP_INFO_TOOL} "--dump-device-compiler-flags" + OUTPUT_VARIABLE COMPUTECPP_DEVICE_COMPILER_FLAGS + RESULT_VARIABLE COMPUTECPP_INFO_TOOL_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT COMPUTECPP_INFO_TOOL_RESULT EQUAL "0") + message(FATAL_ERROR "compute++ flags - Error obtaining compute++ flags!") +else() + mark_as_advanced(COMPUTECPP_COMPILER_FLAGS) + message(STATUS "compute++ flags - ${COMPUTECPP_DEVICE_COMPILER_FLAGS}") +endif() + +# Check if the platform is supported +execute_process(COMMAND ${COMPUTECPP_INFO_TOOL} "--dump-is-supported" + OUTPUT_VARIABLE COMPUTECPP_PLATFORM_IS_SUPPORTED + RESULT_VARIABLE COMPUTECPP_INFO_TOOL_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT COMPUTECPP_INFO_TOOL_RESULT EQUAL "0") + message(FATAL_ERROR "platform - Error checking platform support!") +else() + mark_as_advanced(COMPUTECPP_PLATFORM_IS_SUPPORTED) + if (COMPUTECPP_PLATFORM_IS_SUPPORTED) + message(STATUS "platform - your system can support ComputeCpp") + else() + message(STATUS "platform - your system CANNOT support ComputeCpp") + endif() +endif() + +set(COMPUTECPP_USER_FLAGS + -sycl-compress-name + -Wall + -no-serial-memop + -DEIGEN_NO_ASSERTION_CHECKING=1 + ) + +#################### +# __build_sycl +#################### +# +# Adds a custom target for running compute++ and adding a dependency for the +# resulting integration header. +# +# targetName : Name of the target. +# sourceFile : Source file to be compiled. +# binaryDir : Intermediate directory to output the integration header. +# fileCounter : Counter included in name of custom target. Different counter +# values prevent duplicated names of custom target when source files with the same name, +# but located in different directories, are used for the same target. +# +function(__build_spir targetName sourceFile binaryDir fileCounter) + + # Retrieve source file name. + get_filename_component(sourceFileName ${sourceFile} NAME) + + # Set the path to the Sycl file. + set(outputSyclFile ${binaryDir}/${sourceFileName}.sycl) + + # Add any user-defined include to the device compiler + set(device_compiler_includes "") + get_property(includeDirectories DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY + INCLUDE_DIRECTORIES) + foreach(directory ${includeDirectories}) + set(device_compiler_includes "-I${directory}" ${device_compiler_includes}) + endforeach() + get_target_property(targetIncludeDirectories ${targetName} INCLUDE_DIRECTORIES) + foreach(directory ${targetIncludeDirectories}) + set(device_compiler_includes "-I${directory}" ${device_compiler_includes}) + endforeach() + if (CMAKE_INCLUDE_PATH) + foreach(directory ${CMAKE_INCLUDE_PATH}) + set(device_compiler_includes "-I${directory}" + ${device_compiler_includes}) + endforeach() + endif() + + set(COMPUTECPP_DEVICE_COMPILER_FLAGS + ${COMPUTECPP_DEVICE_COMPILER_FLAGS} + ${COMPUTECPP_USER_FLAGS}) + # Convert argument list format + separate_arguments(COMPUTECPP_DEVICE_COMPILER_FLAGS) + + # Add custom command for running compute++ + add_custom_command( + OUTPUT ${outputSyclFile} + COMMAND ${COMPUTECPP_DEVICE_COMPILER} + ${COMPUTECPP_DEVICE_COMPILER_FLAGS} + -isystem ${COMPUTECPP_INCLUDE_DIRECTORY} + ${COMPUTECPP_PLATFORM_SPECIFIC_ARGS} + ${device_compiler_includes} + -o ${outputSyclFile} + -c ${CMAKE_CURRENT_SOURCE_DIR}/${sourceFile} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${sourceFile} + IMPLICIT_DEPENDS CXX "${CMAKE_CURRENT_SOURCE_DIR}/${sourceFile}" + WORKING_DIRECTORY ${binaryDir} + COMMENT "Building ComputeCpp integration header file ${outputSyclFile}") + + # Add a custom target for the generated integration header + add_custom_target(${targetName}_integration_header DEPENDS ${outputSyclFile}) + + # Add a dependency on the integration header + add_dependencies(${targetName} ${targetName}_integration_header) + + # Set the host compiler C++ standard to C++11 + set_property(TARGET ${targetName} PROPERTY CXX_STANDARD 11) + + # Disable GCC dual ABI on GCC 5.1 and higher + if(COMPUTECPP_DISABLE_GCC_DUAL_ABI) + set_property(TARGET ${targetName} APPEND PROPERTY COMPILE_DEFINITIONS + "_GLIBCXX_USE_CXX11_ABI=0") + endif() + +endfunction() + +####################### +# add_sycl_to_target +####################### +# +# Adds a SYCL compilation custom command associated with an existing +# target and sets a dependency on that new command. +# +# targetName : Name of the target to add a SYCL to. +# binaryDir : Intermediate directory to output the integration header. +# sourceFiles : Source files to be compiled for SYCL. +# +function(add_sycl_to_target targetName binaryDir sourceFiles) + + set(sourceFiles ${sourceFiles} ${ARGN}) + set(fileCounter 0) + # Add custom target to run compute++ and generate the integration header + foreach(sourceFile ${sourceFiles}) + __build_spir(${targetName} ${sourceFile} ${binaryDir} ${fileCounter}) + MATH(EXPR fileCounter "${fileCounter} + 1") + endforeach() + + # Link with the ComputeCpp runtime library + target_link_libraries(${targetName} PUBLIC ${COMPUTECPP_RUNTIME_LIBRARY} + PUBLIC ${OpenCL_LIBRARIES}) + +endfunction(add_sycl_to_target)
diff --git a/cmake/FindEigen3.cmake b/cmake/FindEigen3.cmake index cea1afe..4d9bc1a 100644 --- a/cmake/FindEigen3.cmake +++ b/cmake/FindEigen3.cmake
@@ -10,8 +10,12 @@ # EIGEN3_INCLUDE_DIR - the eigen include directory # EIGEN3_VERSION - eigen version # +# and the following imported target: +# +# Eigen3::Eigen - The header-only Eigen library +# # This module reads hints about search locations from -# the following enviroment variables: +# the following environment variables: # # EIGEN3_ROOT # EIGEN3_ROOT_DIR @@ -64,18 +68,26 @@ # in cache already _eigen3_check_version() set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) + set(Eigen3_FOUND ${EIGEN3_VERSION_OK}) else (EIGEN3_INCLUDE_DIR) + + # search first if an Eigen3Config.cmake is available in the system, + # if successful this would set EIGEN3_INCLUDE_DIR and the rest of + # the script will work as usual + find_package(Eigen3 ${Eigen3_FIND_VERSION} NO_MODULE QUIET) - find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library - HINTS - ENV EIGEN3_ROOT - ENV EIGEN3_ROOT_DIR - PATHS - ${CMAKE_INSTALL_PREFIX}/include - ${KDE4_INCLUDE_DIR} - PATH_SUFFIXES eigen3 eigen - ) + if(NOT EIGEN3_INCLUDE_DIR) + find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library + HINTS + ENV EIGEN3_ROOT + ENV EIGEN3_ROOT_DIR + PATHS + ${CMAKE_INSTALL_PREFIX}/include + ${KDE4_INCLUDE_DIR} + PATH_SUFFIXES eigen3 eigen + ) + endif(NOT EIGEN3_INCLUDE_DIR) if(EIGEN3_INCLUDE_DIR) _eigen3_check_version() @@ -88,3 +100,8 @@ endif(EIGEN3_INCLUDE_DIR) +if(EIGEN3_FOUND AND NOT TARGET Eigen3::Eigen) + add_library(Eigen3::Eigen INTERFACE IMPORTED) + set_target_properties(Eigen3::Eigen PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${EIGEN3_INCLUDE_DIR}") +endif()
diff --git a/cmake/FindHWLOC.cmake b/cmake/FindHWLOC.cmake new file mode 100644 index 0000000..a831b5c --- /dev/null +++ b/cmake/FindHWLOC.cmake
@@ -0,0 +1,331 @@ +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2014 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find HWLOC include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(HWLOC +# [REQUIRED]) # Fail with error if hwloc is not found +# +# This module finds headers and hwloc library. +# Results are reported in variables: +# HWLOC_FOUND - True if headers and requested libraries were found +# HWLOC_INCLUDE_DIRS - hwloc include directories +# HWLOC_LIBRARY_DIRS - Link directories for hwloc libraries +# HWLOC_LIBRARIES - hwloc component libraries to be linked +# +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DHWLOC_DIR=path/to/hwloc): +# HWLOC_DIR - Where to find the base directory of hwloc +# HWLOC_INCDIR - Where to find the header files +# HWLOC_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: HWLOC_DIR, HWLOC_INCDIR, HWLOC_LIBDIR + +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) + +include(CheckStructHasMember) +include(CheckCSourceCompiles) + +if (NOT HWLOC_FOUND) + set(HWLOC_DIR "" CACHE PATH "Installation directory of HWLOC library") + if (NOT HWLOC_FIND_QUIETLY) + message(STATUS "A cache variable, namely HWLOC_DIR, has been set to specify the install directory of HWLOC") + endif() +endif() + +set(ENV_HWLOC_DIR "$ENV{HWLOC_DIR}") +set(ENV_HWLOC_INCDIR "$ENV{HWLOC_INCDIR}") +set(ENV_HWLOC_LIBDIR "$ENV{HWLOC_LIBDIR}") +set(HWLOC_GIVEN_BY_USER "FALSE") +if ( HWLOC_DIR OR ( HWLOC_INCDIR AND HWLOC_LIBDIR) OR ENV_HWLOC_DIR OR (ENV_HWLOC_INCDIR AND ENV_HWLOC_LIBDIR) ) + set(HWLOC_GIVEN_BY_USER "TRUE") +endif() + +# Optionally use pkg-config to detect include/library dirs (if pkg-config is available) +# ------------------------------------------------------------------------------------- +include(FindPkgConfig) +find_package(PkgConfig QUIET) +if( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER ) + + pkg_search_module(HWLOC hwloc) + if (NOT HWLOC_FIND_QUIETLY) + if (HWLOC_FOUND AND HWLOC_LIBRARIES) + message(STATUS "Looking for HWLOC - found using PkgConfig") + #if(NOT HWLOC_INCLUDE_DIRS) + # message("${Magenta}HWLOC_INCLUDE_DIRS is empty using PkgConfig." + # "Perhaps the path to hwloc headers is already present in your" + # "C(PLUS)_INCLUDE_PATH environment variable.${ColourReset}") + #endif() + else() + message(STATUS "${Magenta}Looking for HWLOC - not found using PkgConfig." + "\n Perhaps you should add the directory containing hwloc.pc to" + "\n the PKG_CONFIG_PATH environment variable.${ColourReset}") + endif() + endif() + +endif( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER ) + +if( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) ) + + if (NOT HWLOC_FIND_QUIETLY) + message(STATUS "Looking for HWLOC - PkgConfig not used") + endif() + + # Looking for include + # ------------------- + + # Add system include paths to search include + # ------------------------------------------ + unset(_inc_env) + if(ENV_HWLOC_INCDIR) + list(APPEND _inc_env "${ENV_HWLOC_INCDIR}") + elseif(ENV_HWLOC_DIR) + list(APPEND _inc_env "${ENV_HWLOC_DIR}") + list(APPEND _inc_env "${ENV_HWLOC_DIR}/include") + list(APPEND _inc_env "${ENV_HWLOC_DIR}/include/hwloc") + else() + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() + endif() + list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") + list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") + list(REMOVE_DUPLICATES _inc_env) + + # set paths where to look for + set(PATH_TO_LOOK_FOR "${_inc_env}") + + # Try to find the hwloc header in the given paths + # ------------------------------------------------- + # call cmake macro to find the header path + if(HWLOC_INCDIR) + set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND") + find_path(HWLOC_hwloc.h_DIRS + NAMES hwloc.h + HINTS ${HWLOC_INCDIR}) + else() + if(HWLOC_DIR) + set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND") + find_path(HWLOC_hwloc.h_DIRS + NAMES hwloc.h + HINTS ${HWLOC_DIR} + PATH_SUFFIXES "include" "include/hwloc") + else() + set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND") + find_path(HWLOC_hwloc.h_DIRS + NAMES hwloc.h + HINTS ${PATH_TO_LOOK_FOR} + PATH_SUFFIXES "hwloc") + endif() + endif() + mark_as_advanced(HWLOC_hwloc.h_DIRS) + + # Add path to cmake variable + # ------------------------------------ + if (HWLOC_hwloc.h_DIRS) + set(HWLOC_INCLUDE_DIRS "${HWLOC_hwloc.h_DIRS}") + else () + set(HWLOC_INCLUDE_DIRS "HWLOC_INCLUDE_DIRS-NOTFOUND") + if(NOT HWLOC_FIND_QUIETLY) + message(STATUS "Looking for hwloc -- hwloc.h not found") + endif() + endif () + + if (HWLOC_INCLUDE_DIRS) + list(REMOVE_DUPLICATES HWLOC_INCLUDE_DIRS) + endif () + + + # Looking for lib + # --------------- + + # Add system library paths to search lib + # -------------------------------------- + unset(_lib_env) + if(ENV_HWLOC_LIBDIR) + list(APPEND _lib_env "${ENV_HWLOC_LIBDIR}") + elseif(ENV_HWLOC_DIR) + list(APPEND _lib_env "${ENV_HWLOC_DIR}") + list(APPEND _lib_env "${ENV_HWLOC_DIR}/lib") + else() + if(WIN32) + string(REPLACE ":" ";" _lib_env "$ENV{LIB}") + else() + if(APPLE) + string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}") + else() + string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}") + endif() + list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() + endif() + list(REMOVE_DUPLICATES _lib_env) + + # set paths where to look for + set(PATH_TO_LOOK_FOR "${_lib_env}") + + # Try to find the hwloc lib in the given paths + # ---------------------------------------------- + + # call cmake macro to find the lib path + if(HWLOC_LIBDIR) + set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND") + find_library(HWLOC_hwloc_LIBRARY + NAMES hwloc + HINTS ${HWLOC_LIBDIR}) + else() + if(HWLOC_DIR) + set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND") + find_library(HWLOC_hwloc_LIBRARY + NAMES hwloc + HINTS ${HWLOC_DIR} + PATH_SUFFIXES lib lib32 lib64) + else() + set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND") + find_library(HWLOC_hwloc_LIBRARY + NAMES hwloc + HINTS ${PATH_TO_LOOK_FOR}) + endif() + endif() + mark_as_advanced(HWLOC_hwloc_LIBRARY) + + # If found, add path to cmake variable + # ------------------------------------ + if (HWLOC_hwloc_LIBRARY) + get_filename_component(hwloc_lib_path ${HWLOC_hwloc_LIBRARY} PATH) + # set cmake variables (respects naming convention) + set(HWLOC_LIBRARIES "${HWLOC_hwloc_LIBRARY}") + set(HWLOC_LIBRARY_DIRS "${hwloc_lib_path}") + else () + set(HWLOC_LIBRARIES "HWLOC_LIBRARIES-NOTFOUND") + set(HWLOC_LIBRARY_DIRS "HWLOC_LIBRARY_DIRS-NOTFOUND") + if(NOT HWLOC_FIND_QUIETLY) + message(STATUS "Looking for hwloc -- lib hwloc not found") + endif() + endif () + + if (HWLOC_LIBRARY_DIRS) + list(REMOVE_DUPLICATES HWLOC_LIBRARY_DIRS) + endif () + + # check a function to validate the find + if(HWLOC_LIBRARIES) + + set(REQUIRED_INCDIRS) + set(REQUIRED_LIBDIRS) + set(REQUIRED_LIBS) + + # HWLOC + if (HWLOC_INCLUDE_DIRS) + set(REQUIRED_INCDIRS "${HWLOC_INCLUDE_DIRS}") + endif() + if (HWLOC_LIBRARY_DIRS) + set(REQUIRED_LIBDIRS "${HWLOC_LIBRARY_DIRS}") + endif() + set(REQUIRED_LIBS "${HWLOC_LIBRARIES}") + + # set required libraries for link + set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}") + set(CMAKE_REQUIRED_LIBRARIES) + foreach(lib_dir ${REQUIRED_LIBDIRS}) + list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}") + endforeach() + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}") + string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + + # test link + unset(HWLOC_WORKS CACHE) + include(CheckFunctionExists) + check_function_exists(hwloc_topology_init HWLOC_WORKS) + mark_as_advanced(HWLOC_WORKS) + + if(NOT HWLOC_WORKS) + if(NOT HWLOC_FIND_QUIETLY) + message(STATUS "Looking for hwloc : test of hwloc_topology_init with hwloc library fails") + message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}") + message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}") + message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails") + endif() + endif() + set(CMAKE_REQUIRED_INCLUDES) + set(CMAKE_REQUIRED_FLAGS) + set(CMAKE_REQUIRED_LIBRARIES) + endif(HWLOC_LIBRARIES) + +endif( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) ) + +if (HWLOC_LIBRARIES) + if (HWLOC_LIBRARY_DIRS) + list(GET HWLOC_LIBRARY_DIRS 0 first_lib_path) + else() + list(GET HWLOC_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + endif() + if (${first_lib_path} MATCHES "/lib(32|64)?$") + string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}") + set(HWLOC_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of HWLOC library" FORCE) + else() + set(HWLOC_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of HWLOC library" FORCE) + endif() +endif() +mark_as_advanced(HWLOC_DIR) +mark_as_advanced(HWLOC_DIR_FOUND) + +# check that HWLOC has been found +# ------------------------------- +include(FindPackageHandleStandardArgs) +if (PKG_CONFIG_EXECUTABLE AND HWLOC_FOUND) + find_package_handle_standard_args(HWLOC DEFAULT_MSG + HWLOC_LIBRARIES) +else() + find_package_handle_standard_args(HWLOC DEFAULT_MSG + HWLOC_LIBRARIES + HWLOC_WORKS) +endif() + +if (HWLOC_FOUND) + set(HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) + list(APPEND CMAKE_REQUIRED_INCLUDES ${HWLOC_INCLUDE_DIRS}) + + # test headers to guess the version + check_struct_has_member( "struct hwloc_obj" parent hwloc.h HAVE_HWLOC_PARENT_MEMBER ) + check_struct_has_member( "struct hwloc_cache_attr_s" size hwloc.h HAVE_HWLOC_CACHE_ATTR ) + check_c_source_compiles( "#include <hwloc.h> + int main(void) { hwloc_obj_t o; o->type = HWLOC_OBJ_PU; return 0;}" HAVE_HWLOC_OBJ_PU) + include(CheckLibraryExists) + check_library_exists(${HWLOC_LIBRARIES} hwloc_bitmap_free "" HAVE_HWLOC_BITMAP) + + set(CMAKE_REQUIRED_INCLUDES ${HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES}) +endif()
diff --git a/cmake/FindKLU.cmake b/cmake/FindKLU.cmake new file mode 100644 index 0000000..4a8f8e0 --- /dev/null +++ b/cmake/FindKLU.cmake
@@ -0,0 +1,48 @@ +# KLU lib usually requires linking to a blas library. +# It is up to the user of this module to find a BLAS and link to it. + +if (KLU_INCLUDES AND KLU_LIBRARIES) + set(KLU_FIND_QUIETLY TRUE) +endif (KLU_INCLUDES AND KLU_LIBRARIES) + +find_path(KLU_INCLUDES + NAMES + klu.h + PATHS + $ENV{KLUDIR} + ${INCLUDE_INSTALL_DIR} + PATH_SUFFIXES + suitesparse + ufsparse +) + +find_library(KLU_LIBRARIES klu PATHS $ENV{KLUDIR} ${LIB_INSTALL_DIR}) + +if(KLU_LIBRARIES) + + if(NOT KLU_LIBDIR) + get_filename_component(KLU_LIBDIR ${KLU_LIBRARIES} PATH) + endif(NOT KLU_LIBDIR) + + find_library(COLAMD_LIBRARY colamd PATHS ${KLU_LIBDIR} $ENV{KLUDIR} ${LIB_INSTALL_DIR}) + if(COLAMD_LIBRARY) + set(KLU_LIBRARIES ${KLU_LIBRARIES} ${COLAMD_LIBRARY}) + endif () + + find_library(AMD_LIBRARY amd PATHS ${KLU_LIBDIR} $ENV{KLUDIR} ${LIB_INSTALL_DIR}) + if(AMD_LIBRARY) + set(KLU_LIBRARIES ${KLU_LIBRARIES} ${AMD_LIBRARY}) + endif () + + find_library(BTF_LIBRARY btf PATHS $ENV{KLU_LIBDIR} $ENV{KLUDIR} ${LIB_INSTALL_DIR}) + if(BTF_LIBRARY) + set(KLU_LIBRARIES ${KLU_LIBRARIES} ${BTF_LIBRARY}) + endif() + +endif(KLU_LIBRARIES) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(KLU DEFAULT_MSG + KLU_INCLUDES KLU_LIBRARIES) + +mark_as_advanced(KLU_INCLUDES KLU_LIBRARIES AMD_LIBRARY COLAMD_LIBRARY BTF_LIBRARY)
diff --git a/cmake/FindMetis.cmake b/cmake/FindMetis.cmake index 6a0ce79..da2f1f1 100644 --- a/cmake/FindMetis.cmake +++ b/cmake/FindMetis.cmake
@@ -1,59 +1,264 @@ -# Pastix requires METIS or METIS (partitioning and reordering tools) +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2014 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find METIS include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(METIS +# [REQUIRED] # Fail with error if metis is not found +# ) +# +# This module finds headers and metis library. +# Results are reported in variables: +# METIS_FOUND - True if headers and requested libraries were found +# METIS_INCLUDE_DIRS - metis include directories +# METIS_LIBRARY_DIRS - Link directories for metis libraries +# METIS_LIBRARIES - metis component libraries to be linked +# +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DMETIS_DIR=path/to/metis): +# METIS_DIR - Where to find the base directory of metis +# METIS_INCDIR - Where to find the header files +# METIS_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: METIS_DIR, METIS_INCDIR, METIS_LIBDIR -if (METIS_INCLUDES AND METIS_LIBRARIES) - set(METIS_FIND_QUIETLY TRUE) -endif (METIS_INCLUDES AND METIS_LIBRARIES) +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) -find_path(METIS_INCLUDES - NAMES - metis.h - PATHS - $ENV{METISDIR} - ${INCLUDE_INSTALL_DIR} - PATH_SUFFIXES - . - metis - include -) - -macro(_metis_check_version) - file(READ "${METIS_INCLUDES}/metis.h" _metis_version_header) - - string(REGEX MATCH "define[ \t]+METIS_VER_MAJOR[ \t]+([0-9]+)" _metis_major_version_match "${_metis_version_header}") - set(METIS_MAJOR_VERSION "${CMAKE_MATCH_1}") - string(REGEX MATCH "define[ \t]+METIS_VER_MINOR[ \t]+([0-9]+)" _metis_minor_version_match "${_metis_version_header}") - set(METIS_MINOR_VERSION "${CMAKE_MATCH_1}") - string(REGEX MATCH "define[ \t]+METIS_VER_SUBMINOR[ \t]+([0-9]+)" _metis_subminor_version_match "${_metis_version_header}") - set(METIS_SUBMINOR_VERSION "${CMAKE_MATCH_1}") - if(NOT METIS_MAJOR_VERSION) - message(STATUS "Could not determine Metis version. Assuming version 4.0.0") - set(METIS_VERSION 4.0.0) - else() - set(METIS_VERSION ${METIS_MAJOR_VERSION}.${METIS_MINOR_VERSION}.${METIS_SUBMINOR_VERSION}) +if (NOT METIS_FOUND) + set(METIS_DIR "" CACHE PATH "Installation directory of METIS library") + if (NOT METIS_FIND_QUIETLY) + message(STATUS "A cache variable, namely METIS_DIR, has been set to specify the install directory of METIS") endif() - if(${METIS_VERSION} VERSION_LESS ${Metis_FIND_VERSION}) - set(METIS_VERSION_OK FALSE) +endif() + +# Looking for include +# ------------------- + +# Add system include paths to search include +# ------------------------------------------ +unset(_inc_env) +set(ENV_METIS_DIR "$ENV{METIS_DIR}") +set(ENV_METIS_INCDIR "$ENV{METIS_INCDIR}") +if(ENV_METIS_INCDIR) + list(APPEND _inc_env "${ENV_METIS_INCDIR}") +elseif(ENV_METIS_DIR) + list(APPEND _inc_env "${ENV_METIS_DIR}") + list(APPEND _inc_env "${ENV_METIS_DIR}/include") + list(APPEND _inc_env "${ENV_METIS_DIR}/include/metis") +else() + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") else() - set(METIS_VERSION_OK TRUE) + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() +endif() +list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") +list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") +list(REMOVE_DUPLICATES _inc_env) + + +# Try to find the metis header in the given paths +# ------------------------------------------------- +# call cmake macro to find the header path +if(METIS_INCDIR) + set(METIS_metis.h_DIRS "METIS_metis.h_DIRS-NOTFOUND") + find_path(METIS_metis.h_DIRS + NAMES metis.h + HINTS ${METIS_INCDIR}) +else() + if(METIS_DIR) + set(METIS_metis.h_DIRS "METIS_metis.h_DIRS-NOTFOUND") + find_path(METIS_metis.h_DIRS + NAMES metis.h + HINTS ${METIS_DIR} + PATH_SUFFIXES "include" "include/metis") + else() + set(METIS_metis.h_DIRS "METIS_metis.h_DIRS-NOTFOUND") + find_path(METIS_metis.h_DIRS + NAMES metis.h + HINTS ${_inc_env}) + endif() +endif() +mark_as_advanced(METIS_metis.h_DIRS) + + +# If found, add path to cmake variable +# ------------------------------------ +if (METIS_metis.h_DIRS) + set(METIS_INCLUDE_DIRS "${METIS_metis.h_DIRS}") +else () + set(METIS_INCLUDE_DIRS "METIS_INCLUDE_DIRS-NOTFOUND") + if(NOT METIS_FIND_QUIETLY) + message(STATUS "Looking for metis -- metis.h not found") + endif() +endif() + + +# Looking for lib +# --------------- + +# Add system library paths to search lib +# -------------------------------------- +unset(_lib_env) +set(ENV_METIS_LIBDIR "$ENV{METIS_LIBDIR}") +if(ENV_METIS_LIBDIR) + list(APPEND _lib_env "${ENV_METIS_LIBDIR}") +elseif(ENV_METIS_DIR) + list(APPEND _lib_env "${ENV_METIS_DIR}") + list(APPEND _lib_env "${ENV_METIS_DIR}/lib") +else() + if(WIN32) + string(REPLACE ":" ";" _lib_env "$ENV{LIB}") + else() + if(APPLE) + string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}") + else() + string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}") + endif() + list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() +endif() +list(REMOVE_DUPLICATES _lib_env) + +# Try to find the metis lib in the given paths +# ---------------------------------------------- +# call cmake macro to find the lib path +if(METIS_LIBDIR) + set(METIS_metis_LIBRARY "METIS_metis_LIBRARY-NOTFOUND") + find_library(METIS_metis_LIBRARY + NAMES metis + HINTS ${METIS_LIBDIR}) +else() + if(METIS_DIR) + set(METIS_metis_LIBRARY "METIS_metis_LIBRARY-NOTFOUND") + find_library(METIS_metis_LIBRARY + NAMES metis + HINTS ${METIS_DIR} + PATH_SUFFIXES lib lib32 lib64) + else() + set(METIS_metis_LIBRARY "METIS_metis_LIBRARY-NOTFOUND") + find_library(METIS_metis_LIBRARY + NAMES metis + HINTS ${_lib_env}) + endif() +endif() +mark_as_advanced(METIS_metis_LIBRARY) + + +# If found, add path to cmake variable +# ------------------------------------ +if (METIS_metis_LIBRARY) + get_filename_component(metis_lib_path "${METIS_metis_LIBRARY}" PATH) + # set cmake variables + set(METIS_LIBRARIES "${METIS_metis_LIBRARY}") + set(METIS_LIBRARY_DIRS "${metis_lib_path}") +else () + set(METIS_LIBRARIES "METIS_LIBRARIES-NOTFOUND") + set(METIS_LIBRARY_DIRS "METIS_LIBRARY_DIRS-NOTFOUND") + if(NOT METIS_FIND_QUIETLY) + message(STATUS "Looking for metis -- lib metis not found") + endif() +endif () + +# check a function to validate the find +if(METIS_LIBRARIES) + + set(REQUIRED_INCDIRS) + set(REQUIRED_LIBDIRS) + set(REQUIRED_LIBS) + + # METIS + if (METIS_INCLUDE_DIRS) + set(REQUIRED_INCDIRS "${METIS_INCLUDE_DIRS}") + endif() + if (METIS_LIBRARY_DIRS) + set(REQUIRED_LIBDIRS "${METIS_LIBRARY_DIRS}") + endif() + set(REQUIRED_LIBS "${METIS_LIBRARIES}") + # m + find_library(M_LIBRARY NAMES m) + mark_as_advanced(M_LIBRARY) + if(M_LIBRARY) + list(APPEND REQUIRED_LIBS "-lm") endif() - if(NOT METIS_VERSION_OK) - message(STATUS "Metis version ${METIS_VERSION} found in ${METIS_INCLUDES}, " - "but at least version ${Metis_FIND_VERSION} is required") - endif(NOT METIS_VERSION_OK) -endmacro(_metis_check_version) + # set required libraries for link + set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}") + set(CMAKE_REQUIRED_LIBRARIES) + foreach(lib_dir ${REQUIRED_LIBDIRS}) + list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}") + endforeach() + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}") + string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") - if(METIS_INCLUDES AND Metis_FIND_VERSION) - _metis_check_version() - else() - set(METIS_VERSION_OK TRUE) + # test link + unset(METIS_WORKS CACHE) + include(CheckFunctionExists) + check_function_exists(METIS_NodeND METIS_WORKS) + mark_as_advanced(METIS_WORKS) + + if(NOT METIS_WORKS) + if(NOT METIS_FIND_QUIETLY) + message(STATUS "Looking for METIS : test of METIS_NodeND with METIS library fails") + message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}") + message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}") + message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails") + endif() endif() + set(CMAKE_REQUIRED_INCLUDES) + set(CMAKE_REQUIRED_FLAGS) + set(CMAKE_REQUIRED_LIBRARIES) +endif(METIS_LIBRARIES) +if (METIS_LIBRARIES) + list(GET METIS_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + if (${first_lib_path} MATCHES "/lib(32|64)?$") + string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}") + set(METIS_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of METIS library" FORCE) + else() + set(METIS_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of METIS library" FORCE) + endif() +endif() +mark_as_advanced(METIS_DIR) +mark_as_advanced(METIS_DIR_FOUND) -find_library(METIS_LIBRARIES metis PATHS $ENV{METISDIR} ${LIB_INSTALL_DIR} PATH_SUFFIXES lib) - +# check that METIS has been found +# --------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(METIS DEFAULT_MSG - METIS_INCLUDES METIS_LIBRARIES METIS_VERSION_OK) - -mark_as_advanced(METIS_INCLUDES METIS_LIBRARIES) + METIS_LIBRARIES + METIS_WORKS) +# +# TODO: Add possibility to check for specific functions in the library +#
diff --git a/cmake/FindPTSCOTCH.cmake b/cmake/FindPTSCOTCH.cmake new file mode 100644 index 0000000..1396d05 --- /dev/null +++ b/cmake/FindPTSCOTCH.cmake
@@ -0,0 +1,423 @@ +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2016 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find PTSCOTCH include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(PTSCOTCH +# [REQUIRED] # Fail with error if ptscotch is not found +# [COMPONENTS <comp1> <comp2> ...] # dependencies +# ) +# +# PTSCOTCH depends on the following libraries: +# - Threads +# - MPI +# +# COMPONENTS can be some of the following: +# - ESMUMPS: to activate detection of PT-Scotch with the esmumps interface +# +# This module finds headers and ptscotch library. +# Results are reported in variables: +# PTSCOTCH_FOUND - True if headers and requested libraries were found +# PTSCOTCH_LINKER_FLAGS - list of required linker flags (excluding -l and -L) +# PTSCOTCH_INCLUDE_DIRS - ptscotch include directories +# PTSCOTCH_LIBRARY_DIRS - Link directories for ptscotch libraries +# PTSCOTCH_LIBRARIES - ptscotch component libraries to be linked +# PTSCOTCH_INCLUDE_DIRS_DEP - ptscotch + dependencies include directories +# PTSCOTCH_LIBRARY_DIRS_DEP - ptscotch + dependencies link directories +# PTSCOTCH_LIBRARIES_DEP - ptscotch libraries + dependencies +# PTSCOTCH_INTSIZE - Number of octets occupied by a SCOTCH_Num +# +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DPTSCOTCH=path/to/ptscotch): +# PTSCOTCH_DIR - Where to find the base directory of ptscotch +# PTSCOTCH_INCDIR - Where to find the header files +# PTSCOTCH_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: PTSCOTCH_DIR, PTSCOTCH_INCDIR, PTSCOTCH_LIBDIR + +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013-2016 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) + +if (NOT PTSCOTCH_FOUND) + set(PTSCOTCH_DIR "" CACHE PATH "Installation directory of PTSCOTCH library") + if (NOT PTSCOTCH_FIND_QUIETLY) + message(STATUS "A cache variable, namely PTSCOTCH_DIR, has been set to specify the install directory of PTSCOTCH") + endif() +endif() + +# Set the version to find +set(PTSCOTCH_LOOK_FOR_ESMUMPS OFF) + +if( PTSCOTCH_FIND_COMPONENTS ) + foreach( component ${PTSCOTCH_FIND_COMPONENTS} ) + if (${component} STREQUAL "ESMUMPS") + # means we look for esmumps library + set(PTSCOTCH_LOOK_FOR_ESMUMPS ON) + endif() + endforeach() +endif() + +# PTSCOTCH depends on Threads, try to find it +if (NOT THREADS_FOUND) + if (PTSCOTCH_FIND_REQUIRED) + find_package(Threads REQUIRED) + else() + find_package(Threads) + endif() +endif() + +# PTSCOTCH depends on MPI, try to find it +if (NOT MPI_FOUND) + if (PTSCOTCH_FIND_REQUIRED) + find_package(MPI REQUIRED) + else() + find_package(MPI) + endif() +endif() + +# Looking for include +# ------------------- + +# Add system include paths to search include +# ------------------------------------------ +unset(_inc_env) +set(ENV_PTSCOTCH_DIR "$ENV{PTSCOTCH_DIR}") +set(ENV_PTSCOTCH_INCDIR "$ENV{PTSCOTCH_INCDIR}") +if(ENV_PTSCOTCH_INCDIR) + list(APPEND _inc_env "${ENV_PTSCOTCH_INCDIR}") +elseif(ENV_PTSCOTCH_DIR) + list(APPEND _inc_env "${ENV_PTSCOTCH_DIR}") + list(APPEND _inc_env "${ENV_PTSCOTCH_DIR}/include") + list(APPEND _inc_env "${ENV_PTSCOTCH_DIR}/include/ptscotch") +else() + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() +endif() +list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") +list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") +list(REMOVE_DUPLICATES _inc_env) + + +# Try to find the ptscotch header in the given paths +# ------------------------------------------------- + +set(PTSCOTCH_hdrs_to_find "ptscotch.h;scotch.h") + +# call cmake macro to find the header path +if(PTSCOTCH_INCDIR) + foreach(ptscotch_hdr ${PTSCOTCH_hdrs_to_find}) + set(PTSCOTCH_${ptscotch_hdr}_DIRS "PTSCOTCH_${ptscotch_hdr}_DIRS-NOTFOUND") + find_path(PTSCOTCH_${ptscotch_hdr}_DIRS + NAMES ${ptscotch_hdr} + HINTS ${PTSCOTCH_INCDIR}) + mark_as_advanced(PTSCOTCH_${ptscotch_hdr}_DIRS) + endforeach() +else() + if(PTSCOTCH_DIR) + foreach(ptscotch_hdr ${PTSCOTCH_hdrs_to_find}) + set(PTSCOTCH_${ptscotch_hdr}_DIRS "PTSCOTCH_${ptscotch_hdr}_DIRS-NOTFOUND") + find_path(PTSCOTCH_${ptscotch_hdr}_DIRS + NAMES ${ptscotch_hdr} + HINTS ${PTSCOTCH_DIR} + PATH_SUFFIXES "include" "include/scotch") + mark_as_advanced(PTSCOTCH_${ptscotch_hdr}_DIRS) + endforeach() + else() + foreach(ptscotch_hdr ${PTSCOTCH_hdrs_to_find}) + set(PTSCOTCH_${ptscotch_hdr}_DIRS "PTSCOTCH_${ptscotch_hdr}_DIRS-NOTFOUND") + find_path(PTSCOTCH_${ptscotch_hdr}_DIRS + NAMES ${ptscotch_hdr} + HINTS ${_inc_env} + PATH_SUFFIXES "scotch") + mark_as_advanced(PTSCOTCH_${ptscotch_hdr}_DIRS) + endforeach() + endif() +endif() + +# If found, add path to cmake variable +# ------------------------------------ +foreach(ptscotch_hdr ${PTSCOTCH_hdrs_to_find}) + if (PTSCOTCH_${ptscotch_hdr}_DIRS) + list(APPEND PTSCOTCH_INCLUDE_DIRS "${PTSCOTCH_${ptscotch_hdr}_DIRS}") + else () + set(PTSCOTCH_INCLUDE_DIRS "PTSCOTCH_INCLUDE_DIRS-NOTFOUND") + if (NOT PTSCOTCH_FIND_QUIETLY) + message(STATUS "Looking for ptscotch -- ${ptscotch_hdr} not found") + endif() + endif() +endforeach() +list(REMOVE_DUPLICATES PTSCOTCH_INCLUDE_DIRS) + +# Looking for lib +# --------------- + +# Add system library paths to search lib +# -------------------------------------- +unset(_lib_env) +set(ENV_PTSCOTCH_LIBDIR "$ENV{PTSCOTCH_LIBDIR}") +if(ENV_PTSCOTCH_LIBDIR) + list(APPEND _lib_env "${ENV_PTSCOTCH_LIBDIR}") +elseif(ENV_PTSCOTCH_DIR) + list(APPEND _lib_env "${ENV_PTSCOTCH_DIR}") + list(APPEND _lib_env "${ENV_PTSCOTCH_DIR}/lib") +else() + if(WIN32) + string(REPLACE ":" ";" _lib_env "$ENV{LIB}") + else() + if(APPLE) + string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}") + else() + string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}") + endif() + list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() +endif() +list(REMOVE_DUPLICATES _lib_env) + +# Try to find the ptscotch lib in the given paths +# ---------------------------------------------- + +set(PTSCOTCH_libs_to_find "ptscotch;ptscotcherr") +if (PTSCOTCH_LOOK_FOR_ESMUMPS) + list(INSERT PTSCOTCH_libs_to_find 0 "ptesmumps") + list(APPEND PTSCOTCH_libs_to_find "esmumps" ) +endif() +list(APPEND PTSCOTCH_libs_to_find "scotch;scotcherr") + +# call cmake macro to find the lib path +if(PTSCOTCH_LIBDIR) + foreach(ptscotch_lib ${PTSCOTCH_libs_to_find}) + set(PTSCOTCH_${ptscotch_lib}_LIBRARY "PTSCOTCH_${ptscotch_lib}_LIBRARY-NOTFOUND") + find_library(PTSCOTCH_${ptscotch_lib}_LIBRARY + NAMES ${ptscotch_lib} + HINTS ${PTSCOTCH_LIBDIR}) + endforeach() +else() + if(PTSCOTCH_DIR) + foreach(ptscotch_lib ${PTSCOTCH_libs_to_find}) + set(PTSCOTCH_${ptscotch_lib}_LIBRARY "PTSCOTCH_${ptscotch_lib}_LIBRARY-NOTFOUND") + find_library(PTSCOTCH_${ptscotch_lib}_LIBRARY + NAMES ${ptscotch_lib} + HINTS ${PTSCOTCH_DIR} + PATH_SUFFIXES lib lib32 lib64) + endforeach() + else() + foreach(ptscotch_lib ${PTSCOTCH_libs_to_find}) + set(PTSCOTCH_${ptscotch_lib}_LIBRARY "PTSCOTCH_${ptscotch_lib}_LIBRARY-NOTFOUND") + find_library(PTSCOTCH_${ptscotch_lib}_LIBRARY + NAMES ${ptscotch_lib} + HINTS ${_lib_env}) + endforeach() + endif() +endif() + +set(PTSCOTCH_LIBRARIES "") +set(PTSCOTCH_LIBRARY_DIRS "") +# If found, add path to cmake variable +# ------------------------------------ +foreach(ptscotch_lib ${PTSCOTCH_libs_to_find}) + + if (PTSCOTCH_${ptscotch_lib}_LIBRARY) + get_filename_component(${ptscotch_lib}_lib_path "${PTSCOTCH_${ptscotch_lib}_LIBRARY}" PATH) + # set cmake variables + list(APPEND PTSCOTCH_LIBRARIES "${PTSCOTCH_${ptscotch_lib}_LIBRARY}") + list(APPEND PTSCOTCH_LIBRARY_DIRS "${${ptscotch_lib}_lib_path}") + else () + list(APPEND PTSCOTCH_LIBRARIES "${PTSCOTCH_${ptscotch_lib}_LIBRARY}") + if (NOT PTSCOTCH_FIND_QUIETLY) + message(STATUS "Looking for ptscotch -- lib ${ptscotch_lib} not found") + endif() + endif () + + mark_as_advanced(PTSCOTCH_${ptscotch_lib}_LIBRARY) + +endforeach() +list(REMOVE_DUPLICATES PTSCOTCH_LIBRARY_DIRS) + +# check a function to validate the find +if(PTSCOTCH_LIBRARIES) + + set(REQUIRED_LDFLAGS) + set(REQUIRED_INCDIRS) + set(REQUIRED_LIBDIRS) + set(REQUIRED_LIBS) + + # PTSCOTCH + if (PTSCOTCH_INCLUDE_DIRS) + set(REQUIRED_INCDIRS "${PTSCOTCH_INCLUDE_DIRS}") + endif() + if (PTSCOTCH_LIBRARY_DIRS) + set(REQUIRED_LIBDIRS "${PTSCOTCH_LIBRARY_DIRS}") + endif() + set(REQUIRED_LIBS "${PTSCOTCH_LIBRARIES}") + # MPI + if (MPI_FOUND) + if (MPI_C_INCLUDE_PATH) + list(APPEND CMAKE_REQUIRED_INCLUDES "${MPI_C_INCLUDE_PATH}") + endif() + if (MPI_C_LINK_FLAGS) + if (${MPI_C_LINK_FLAGS} MATCHES " -") + string(REGEX REPLACE " -" "-" MPI_C_LINK_FLAGS ${MPI_C_LINK_FLAGS}) + endif() + list(APPEND REQUIRED_LDFLAGS "${MPI_C_LINK_FLAGS}") + endif() + list(APPEND REQUIRED_LIBS "${MPI_C_LIBRARIES}") + endif() + # THREADS + if(CMAKE_THREAD_LIBS_INIT) + list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}") + endif() + set(Z_LIBRARY "Z_LIBRARY-NOTFOUND") + find_library(Z_LIBRARY NAMES z) + mark_as_advanced(Z_LIBRARY) + if(Z_LIBRARY) + list(APPEND REQUIRED_LIBS "-lz") + endif() + set(M_LIBRARY "M_LIBRARY-NOTFOUND") + find_library(M_LIBRARY NAMES m) + mark_as_advanced(M_LIBRARY) + if(M_LIBRARY) + list(APPEND REQUIRED_LIBS "-lm") + endif() + set(RT_LIBRARY "RT_LIBRARY-NOTFOUND") + find_library(RT_LIBRARY NAMES rt) + mark_as_advanced(RT_LIBRARY) + if(RT_LIBRARY) + list(APPEND REQUIRED_LIBS "-lrt") + endif() + + # set required libraries for link + set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}") + set(CMAKE_REQUIRED_LIBRARIES) + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LDFLAGS}") + foreach(lib_dir ${REQUIRED_LIBDIRS}) + list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}") + endforeach() + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}") + list(APPEND CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}") + string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + + # test link + unset(PTSCOTCH_WORKS CACHE) + include(CheckFunctionExists) + check_function_exists(SCOTCH_dgraphInit PTSCOTCH_WORKS) + mark_as_advanced(PTSCOTCH_WORKS) + + if(PTSCOTCH_WORKS) + # save link with dependencies + set(PTSCOTCH_LIBRARIES_DEP "${REQUIRED_LIBS}") + set(PTSCOTCH_LIBRARY_DIRS_DEP "${REQUIRED_LIBDIRS}") + set(PTSCOTCH_INCLUDE_DIRS_DEP "${REQUIRED_INCDIRS}") + set(PTSCOTCH_LINKER_FLAGS "${REQUIRED_LDFLAGS}") + list(REMOVE_DUPLICATES PTSCOTCH_LIBRARY_DIRS_DEP) + list(REMOVE_DUPLICATES PTSCOTCH_INCLUDE_DIRS_DEP) + list(REMOVE_DUPLICATES PTSCOTCH_LINKER_FLAGS) + else() + if(NOT PTSCOTCH_FIND_QUIETLY) + message(STATUS "Looking for PTSCOTCH : test of SCOTCH_dgraphInit with PTSCOTCH library fails") + message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}") + message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}") + message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails") + endif() + endif() + set(CMAKE_REQUIRED_INCLUDES) + set(CMAKE_REQUIRED_FLAGS) + set(CMAKE_REQUIRED_LIBRARIES) +endif(PTSCOTCH_LIBRARIES) + +if (PTSCOTCH_LIBRARIES) + list(GET PTSCOTCH_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + if (${first_lib_path} MATCHES "/lib(32|64)?$") + string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}") + set(PTSCOTCH_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of PTSCOTCH library" FORCE) + else() + set(PTSCOTCH_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of PTSCOTCH library" FORCE) + endif() +endif() +mark_as_advanced(PTSCOTCH_DIR) +mark_as_advanced(PTSCOTCH_DIR_FOUND) + +# Check the size of SCOTCH_Num +# --------------------------------- +set(CMAKE_REQUIRED_INCLUDES ${PTSCOTCH_INCLUDE_DIRS}) + +include(CheckCSourceRuns) +#stdio.h and stdint.h should be included by scotch.h directly +set(PTSCOTCH_C_TEST_SCOTCH_Num_4 " +#include <stdio.h> +#include <stdint.h> +#include <ptscotch.h> +int main(int argc, char **argv) { + if (sizeof(SCOTCH_Num) == 4) + return 0; + else + return 1; +} +") + +set(PTSCOTCH_C_TEST_SCOTCH_Num_8 " +#include <stdio.h> +#include <stdint.h> +#include <ptscotch.h> +int main(int argc, char **argv) { + if (sizeof(SCOTCH_Num) == 8) + return 0; + else + return 1; +} +") +check_c_source_runs("${PTSCOTCH_C_TEST_SCOTCH_Num_4}" PTSCOTCH_Num_4) +if(NOT PTSCOTCH_Num_4) + check_c_source_runs("${PTSCOTCH_C_TEST_SCOTCH_Num_8}" PTSCOTCH_Num_8) + if(NOT PTSCOTCH_Num_8) + set(PTSCOTCH_INTSIZE -1) + else() + set(PTSCOTCH_INTSIZE 8) + endif() +else() + set(PTSCOTCH_INTSIZE 4) +endif() +set(CMAKE_REQUIRED_INCLUDES "") + +# check that PTSCOTCH has been found +# --------------------------------- +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PTSCOTCH DEFAULT_MSG + PTSCOTCH_LIBRARIES + PTSCOTCH_WORKS) +# +# TODO: Add possibility to check for specific functions in the library +#
diff --git a/cmake/FindPastix.cmake b/cmake/FindPastix.cmake index e2e6c81..470477f 100644 --- a/cmake/FindPastix.cmake +++ b/cmake/FindPastix.cmake
@@ -1,25 +1,704 @@ -# Pastix lib requires linking to a blas library. -# It is up to the user of this module to find a BLAS and link to it. -# Pastix requires SCOTCH or METIS (partitioning and reordering tools) as well +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2014 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find PASTIX include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(PASTIX +# [REQUIRED] # Fail with error if pastix is not found +# [COMPONENTS <comp1> <comp2> ...] # dependencies +# ) +# +# PASTIX depends on the following libraries: +# - Threads, m, rt +# - MPI +# - HWLOC +# - BLAS +# +# COMPONENTS are optional libraries PASTIX could be linked with, +# Use it to drive detection of a specific compilation chain +# COMPONENTS can be some of the following: +# - MPI: to activate detection of the parallel MPI version (default) +# it looks for Threads, HWLOC, BLAS, MPI and ScaLAPACK libraries +# - SEQ: to activate detection of the sequential version (exclude MPI version) +# - STARPU: to activate detection of StarPU version +# it looks for MPI version of StarPU (default behaviour) +# if SEQ and STARPU are given, it looks for a StarPU without MPI +# - STARPU_CUDA: to activate detection of StarPU with CUDA +# - STARPU_FXT: to activate detection of StarPU with FxT +# - SCOTCH: to activate detection of PASTIX linked with SCOTCH +# - PTSCOTCH: to activate detection of PASTIX linked with SCOTCH +# - METIS: to activate detection of PASTIX linked with SCOTCH +# +# This module finds headers and pastix library. +# Results are reported in variables: +# PASTIX_FOUND - True if headers and requested libraries were found +# PASTIX_LINKER_FLAGS - list of required linker flags (excluding -l and -L) +# PASTIX_INCLUDE_DIRS - pastix include directories +# PASTIX_LIBRARY_DIRS - Link directories for pastix libraries +# PASTIX_LIBRARIES - pastix libraries +# PASTIX_INCLUDE_DIRS_DEP - pastix + dependencies include directories +# PASTIX_LIBRARY_DIRS_DEP - pastix + dependencies link directories +# PASTIX_LIBRARIES_DEP - pastix libraries + dependencies +# +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DPASTIX_DIR=path/to/pastix): +# PASTIX_DIR - Where to find the base directory of pastix +# PASTIX_INCDIR - Where to find the header files +# PASTIX_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: PASTIX_DIR, PASTIX_INCDIR, PASTIX_LIBDIR -if (PASTIX_INCLUDES AND PASTIX_LIBRARIES) - set(PASTIX_FIND_QUIETLY TRUE) -endif (PASTIX_INCLUDES AND PASTIX_LIBRARIES) - -find_path(PASTIX_INCLUDES - NAMES - pastix_nompi.h - PATHS - $ENV{PASTIXDIR} - ${INCLUDE_INSTALL_DIR} -) - -find_library(PASTIX_LIBRARIES pastix PATHS $ENV{PASTIXDIR} ${LIB_INSTALL_DIR}) +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) +if (NOT PASTIX_FOUND) + set(PASTIX_DIR "" CACHE PATH "Installation directory of PASTIX library") + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "A cache variable, namely PASTIX_DIR, has been set to specify the install directory of PASTIX") + endif() +endif() +# Set the version to find +set(PASTIX_LOOK_FOR_MPI ON) +set(PASTIX_LOOK_FOR_SEQ OFF) +set(PASTIX_LOOK_FOR_STARPU OFF) +set(PASTIX_LOOK_FOR_STARPU_CUDA OFF) +set(PASTIX_LOOK_FOR_STARPU_FXT OFF) +set(PASTIX_LOOK_FOR_SCOTCH ON) +set(PASTIX_LOOK_FOR_PTSCOTCH OFF) +set(PASTIX_LOOK_FOR_METIS OFF) + +if( PASTIX_FIND_COMPONENTS ) + foreach( component ${PASTIX_FIND_COMPONENTS} ) + if (${component} STREQUAL "SEQ") + # means we look for the sequential version of PaStiX (without MPI) + set(PASTIX_LOOK_FOR_SEQ ON) + set(PASTIX_LOOK_FOR_MPI OFF) + endif() + if (${component} STREQUAL "MPI") + # means we look for the MPI version of PaStiX (default) + set(PASTIX_LOOK_FOR_SEQ OFF) + set(PASTIX_LOOK_FOR_MPI ON) + endif() + if (${component} STREQUAL "STARPU") + # means we look for PaStiX with StarPU + set(PASTIX_LOOK_FOR_STARPU ON) + endif() + if (${component} STREQUAL "STARPU_CUDA") + # means we look for PaStiX with StarPU + CUDA + set(PASTIX_LOOK_FOR_STARPU ON) + set(PASTIX_LOOK_FOR_STARPU_CUDA ON) + endif() + if (${component} STREQUAL "STARPU_FXT") + # means we look for PaStiX with StarPU + FxT + set(PASTIX_LOOK_FOR_STARPU_FXT ON) + endif() + if (${component} STREQUAL "SCOTCH") + set(PASTIX_LOOK_FOR_SCOTCH ON) + endif() + if (${component} STREQUAL "SCOTCH") + set(PASTIX_LOOK_FOR_PTSCOTCH ON) + endif() + if (${component} STREQUAL "METIS") + set(PASTIX_LOOK_FOR_METIS ON) + endif() + endforeach() +endif() + +# Dependencies detection +# ---------------------- + + +# Required dependencies +# --------------------- + +if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect pthread") +endif() +if (PASTIX_FIND_REQUIRED) + find_package(Threads REQUIRED QUIET) +else() + find_package(Threads QUIET) +endif() +set(PASTIX_EXTRA_LIBRARIES "") +if( THREADS_FOUND ) + list(APPEND PASTIX_EXTRA_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +endif () + +# Add math library to the list of extra +# it normally exists on all common systems provided with a C compiler +if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect libm") +endif() +set(PASTIX_M_LIBRARIES "") +if(UNIX OR WIN32) + find_library( + PASTIX_M_m_LIBRARY + NAMES m + ) + mark_as_advanced(PASTIX_M_m_LIBRARY) + if (PASTIX_M_m_LIBRARY) + list(APPEND PASTIX_M_LIBRARIES "${PASTIX_M_m_LIBRARY}") + list(APPEND PASTIX_EXTRA_LIBRARIES "${PASTIX_M_m_LIBRARY}") + else() + if (PASTIX_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find libm on your system." + "Are you sure to a have a C compiler installed?") + endif() + endif() +endif() + +# Try to find librt (libposix4 - POSIX.1b Realtime Extensions library) +# on Unix systems except Apple ones because it does not exist on it +if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect librt") +endif() +set(PASTIX_RT_LIBRARIES "") +if(UNIX AND NOT APPLE) + find_library( + PASTIX_RT_rt_LIBRARY + NAMES rt + ) + mark_as_advanced(PASTIX_RT_rt_LIBRARY) + if (PASTIX_RT_rt_LIBRARY) + list(APPEND PASTIX_RT_LIBRARIES "${PASTIX_RT_rt_LIBRARY}") + list(APPEND PASTIX_EXTRA_LIBRARIES "${PASTIX_RT_rt_LIBRARY}") + else() + if (PASTIX_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find librt on your system") + endif() + endif() +endif() + +# PASTIX depends on HWLOC +#------------------------ +if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect HWLOC") +endif() +if (PASTIX_FIND_REQUIRED) + find_package(HWLOC REQUIRED QUIET) +else() + find_package(HWLOC QUIET) +endif() + +# PASTIX depends on BLAS +#----------------------- +if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect BLAS") +endif() +if (PASTIX_FIND_REQUIRED) + find_package(BLASEXT REQUIRED QUIET) +else() + find_package(BLASEXT QUIET) +endif() + +# Optional dependencies +# --------------------- + +# PASTIX may depend on MPI +#------------------------- +if (NOT MPI_FOUND AND PASTIX_LOOK_FOR_MPI) + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect MPI") + endif() + # allows to use an external mpi compilation by setting compilers with + # -DMPI_C_COMPILER=path/to/mpicc -DMPI_Fortran_COMPILER=path/to/mpif90 + # at cmake configure + if(NOT MPI_C_COMPILER) + set(MPI_C_COMPILER mpicc) + endif() + if (PASTIX_FIND_REQUIRED AND PASTIX_FIND_REQUIRED_MPI) + find_package(MPI REQUIRED QUIET) + else() + find_package(MPI QUIET) + endif() + if (MPI_FOUND) + mark_as_advanced(MPI_LIBRARY) + mark_as_advanced(MPI_EXTRA_LIBRARY) + endif() +endif (NOT MPI_FOUND AND PASTIX_LOOK_FOR_MPI) + +# PASTIX may depend on STARPU +#---------------------------- +if( NOT STARPU_FOUND AND PASTIX_LOOK_FOR_STARPU) + + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect StarPU") + endif() + + set(PASTIX_STARPU_VERSION "1.1" CACHE STRING "oldest STARPU version desired") + + # create list of components in order to make a single call to find_package(starpu...) + # we explicitly need a StarPU version built with hwloc + set(STARPU_COMPONENT_LIST "HWLOC") + + # StarPU may depend on MPI + # allows to use an external mpi compilation by setting compilers with + # -DMPI_C_COMPILER=path/to/mpicc -DMPI_Fortran_COMPILER=path/to/mpif90 + # at cmake configure + if (PASTIX_LOOK_FOR_MPI) + if(NOT MPI_C_COMPILER) + set(MPI_C_COMPILER mpicc) + endif() + list(APPEND STARPU_COMPONENT_LIST "MPI") + endif() + if (PASTIX_LOOK_FOR_STARPU_CUDA) + list(APPEND STARPU_COMPONENT_LIST "CUDA") + endif() + if (PASTIX_LOOK_FOR_STARPU_FXT) + list(APPEND STARPU_COMPONENT_LIST "FXT") + endif() + # set the list of optional dependencies we may discover + if (PASTIX_FIND_REQUIRED AND PASTIX_FIND_REQUIRED_STARPU) + find_package(STARPU ${PASTIX_STARPU_VERSION} REQUIRED + COMPONENTS ${STARPU_COMPONENT_LIST}) + else() + find_package(STARPU ${PASTIX_STARPU_VERSION} + COMPONENTS ${STARPU_COMPONENT_LIST}) + endif() + +endif( NOT STARPU_FOUND AND PASTIX_LOOK_FOR_STARPU) + +# PASTIX may depends on SCOTCH +#----------------------------- +if (NOT SCOTCH_FOUND AND PASTIX_LOOK_FOR_SCOTCH) + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect SCOTCH") + endif() + if (PASTIX_FIND_REQUIRED AND PASTIX_FIND_REQUIRED_SCOTCH) + find_package(SCOTCH REQUIRED QUIET) + else() + find_package(SCOTCH QUIET) + endif() +endif() + +# PASTIX may depends on PTSCOTCH +#------------------------------- +if (NOT PTSCOTCH_FOUND AND PASTIX_LOOK_FOR_PTSCOTCH) + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect PTSCOTCH") + endif() + if (PASTIX_FIND_REQUIRED AND PASTIX_FIND_REQUIRED_PTSCOTCH) + find_package(PTSCOTCH REQUIRED QUIET) + else() + find_package(PTSCOTCH QUIET) + endif() +endif() + +# PASTIX may depends on METIS +#---------------------------- +if (NOT METIS_FOUND AND PASTIX_LOOK_FOR_METIS) + if (NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX - Try to detect METIS") + endif() + if (PASTIX_FIND_REQUIRED AND PASTIX_FIND_REQUIRED_METIS) + find_package(METIS REQUIRED QUIET) + else() + find_package(METIS QUIET) + endif() +endif() + +# Error if pastix required and no partitioning lib found +if (PASTIX_FIND_REQUIRED AND NOT SCOTCH_FOUND AND NOT PTSCOTCH_FOUND AND NOT METIS_FOUND) + message(FATAL_ERROR "Could NOT find any partitioning library on your system" + " (install scotch, ptscotch or metis)") +endif() + + +# Looking for PaStiX +# ------------------ + +# Looking for include +# ------------------- + +# Add system include paths to search include +# ------------------------------------------ +unset(_inc_env) +set(ENV_PASTIX_DIR "$ENV{PASTIX_DIR}") +set(ENV_PASTIX_INCDIR "$ENV{PASTIX_INCDIR}") +if(ENV_PASTIX_INCDIR) + list(APPEND _inc_env "${ENV_PASTIX_INCDIR}") +elseif(ENV_PASTIX_DIR) + list(APPEND _inc_env "${ENV_PASTIX_DIR}") + list(APPEND _inc_env "${ENV_PASTIX_DIR}/include") + list(APPEND _inc_env "${ENV_PASTIX_DIR}/include/pastix") +else() + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() +endif() +list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") +list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") +list(REMOVE_DUPLICATES _inc_env) + + +# Try to find the pastix header in the given paths +# --------------------------------------------------- +# call cmake macro to find the header path +if(PASTIX_INCDIR) + set(PASTIX_pastix.h_DIRS "PASTIX_pastix.h_DIRS-NOTFOUND") + find_path(PASTIX_pastix.h_DIRS + NAMES pastix.h + HINTS ${PASTIX_INCDIR}) +else() + if(PASTIX_DIR) + set(PASTIX_pastix.h_DIRS "PASTIX_pastix.h_DIRS-NOTFOUND") + find_path(PASTIX_pastix.h_DIRS + NAMES pastix.h + HINTS ${PASTIX_DIR} + PATH_SUFFIXES "include" "include/pastix") + else() + set(PASTIX_pastix.h_DIRS "PASTIX_pastix.h_DIRS-NOTFOUND") + find_path(PASTIX_pastix.h_DIRS + NAMES pastix.h + HINTS ${_inc_env} + PATH_SUFFIXES "pastix") + endif() +endif() +mark_as_advanced(PASTIX_pastix.h_DIRS) + +# If found, add path to cmake variable +# ------------------------------------ +if (PASTIX_pastix.h_DIRS) + set(PASTIX_INCLUDE_DIRS "${PASTIX_pastix.h_DIRS}") +else () + set(PASTIX_INCLUDE_DIRS "PASTIX_INCLUDE_DIRS-NOTFOUND") + if(NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for pastix -- pastix.h not found") + endif() +endif() + + +# Looking for lib +# --------------- + +# Add system library paths to search lib +# -------------------------------------- +unset(_lib_env) +set(ENV_PASTIX_LIBDIR "$ENV{PASTIX_LIBDIR}") +if(ENV_PASTIX_LIBDIR) + list(APPEND _lib_env "${ENV_PASTIX_LIBDIR}") +elseif(ENV_PASTIX_DIR) + list(APPEND _lib_env "${ENV_PASTIX_DIR}") + list(APPEND _lib_env "${ENV_PASTIX_DIR}/lib") +else() + if(WIN32) + string(REPLACE ":" ";" _lib_env "$ENV{LIB}") + else() + if(APPLE) + string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}") + else() + string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}") + endif() + list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() +endif() +list(REMOVE_DUPLICATES _lib_env) + +# Try to find the pastix lib in the given paths +# ------------------------------------------------ + +# create list of libs to find +set(PASTIX_libs_to_find "pastix_murge;pastix") + +# call cmake macro to find the lib path +if(PASTIX_LIBDIR) + foreach(pastix_lib ${PASTIX_libs_to_find}) + set(PASTIX_${pastix_lib}_LIBRARY "PASTIX_${pastix_lib}_LIBRARY-NOTFOUND") + find_library(PASTIX_${pastix_lib}_LIBRARY + NAMES ${pastix_lib} + HINTS ${PASTIX_LIBDIR}) + endforeach() +else() + if(PASTIX_DIR) + foreach(pastix_lib ${PASTIX_libs_to_find}) + set(PASTIX_${pastix_lib}_LIBRARY "PASTIX_${pastix_lib}_LIBRARY-NOTFOUND") + find_library(PASTIX_${pastix_lib}_LIBRARY + NAMES ${pastix_lib} + HINTS ${PASTIX_DIR} + PATH_SUFFIXES lib lib32 lib64) + endforeach() + else() + foreach(pastix_lib ${PASTIX_libs_to_find}) + set(PASTIX_${pastix_lib}_LIBRARY "PASTIX_${pastix_lib}_LIBRARY-NOTFOUND") + find_library(PASTIX_${pastix_lib}_LIBRARY + NAMES ${pastix_lib} + HINTS ${_lib_env}) + endforeach() + endif() +endif() + +# If found, add path to cmake variable +# ------------------------------------ +foreach(pastix_lib ${PASTIX_libs_to_find}) + + get_filename_component(${pastix_lib}_lib_path ${PASTIX_${pastix_lib}_LIBRARY} PATH) + # set cmake variables (respects naming convention) + if (PASTIX_LIBRARIES) + list(APPEND PASTIX_LIBRARIES "${PASTIX_${pastix_lib}_LIBRARY}") + else() + set(PASTIX_LIBRARIES "${PASTIX_${pastix_lib}_LIBRARY}") + endif() + if (PASTIX_LIBRARY_DIRS) + list(APPEND PASTIX_LIBRARY_DIRS "${${pastix_lib}_lib_path}") + else() + set(PASTIX_LIBRARY_DIRS "${${pastix_lib}_lib_path}") + endif() + mark_as_advanced(PASTIX_${pastix_lib}_LIBRARY) + +endforeach(pastix_lib ${PASTIX_libs_to_find}) + +# check a function to validate the find +if(PASTIX_LIBRARIES) + + set(REQUIRED_LDFLAGS) + set(REQUIRED_INCDIRS) + set(REQUIRED_LIBDIRS) + set(REQUIRED_LIBS) + + # PASTIX + if (PASTIX_INCLUDE_DIRS) + set(REQUIRED_INCDIRS "${PASTIX_INCLUDE_DIRS}") + endif() + foreach(libdir ${PASTIX_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + set(REQUIRED_LIBS "${PASTIX_LIBRARIES}") + # STARPU + if (PASTIX_LOOK_FOR_STARPU AND STARPU_FOUND) + if (STARPU_INCLUDE_DIRS_DEP) + list(APPEND REQUIRED_INCDIRS "${STARPU_INCLUDE_DIRS_DEP}") + elseif (STARPU_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${STARPU_INCLUDE_DIRS}") + endif() + if(STARPU_LIBRARY_DIRS_DEP) + list(APPEND REQUIRED_LIBDIRS "${STARPU_LIBRARY_DIRS_DEP}") + elseif(STARPU_LIBRARY_DIRS) + list(APPEND REQUIRED_LIBDIRS "${STARPU_LIBRARY_DIRS}") + endif() + if (STARPU_LIBRARIES_DEP) + list(APPEND REQUIRED_LIBS "${STARPU_LIBRARIES_DEP}") + elseif (STARPU_LIBRARIES) + foreach(lib ${STARPU_LIBRARIES}) + if (EXISTS ${lib} OR ${lib} MATCHES "^-") + list(APPEND REQUIRED_LIBS "${lib}") + else() + list(APPEND REQUIRED_LIBS "-l${lib}") + endif() + endforeach() + endif() + endif() + # CUDA + if (PASTIX_LOOK_FOR_STARPU_CUDA AND CUDA_FOUND) + if (CUDA_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${CUDA_INCLUDE_DIRS}") + endif() + foreach(libdir ${CUDA_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + list(APPEND REQUIRED_LIBS "${CUDA_CUBLAS_LIBRARIES};${CUDA_LIBRARIES}") + endif() + # MPI + if (PASTIX_LOOK_FOR_MPI AND MPI_FOUND) + if (MPI_C_INCLUDE_PATH) + list(APPEND REQUIRED_INCDIRS "${MPI_C_INCLUDE_PATH}") + endif() + if (MPI_C_LINK_FLAGS) + if (${MPI_C_LINK_FLAGS} MATCHES " -") + string(REGEX REPLACE " -" "-" MPI_C_LINK_FLAGS ${MPI_C_LINK_FLAGS}) + endif() + list(APPEND REQUIRED_LDFLAGS "${MPI_C_LINK_FLAGS}") + endif() + list(APPEND REQUIRED_LIBS "${MPI_C_LIBRARIES}") + endif() + # HWLOC + if (HWLOC_FOUND) + if (HWLOC_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${HWLOC_INCLUDE_DIRS}") + endif() + foreach(libdir ${HWLOC_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + foreach(lib ${HWLOC_LIBRARIES}) + if (EXISTS ${lib} OR ${lib} MATCHES "^-") + list(APPEND REQUIRED_LIBS "${lib}") + else() + list(APPEND REQUIRED_LIBS "-l${lib}") + endif() + endforeach() + endif() + # BLAS + if (BLAS_FOUND) + if (BLAS_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${BLAS_INCLUDE_DIRS}") + endif() + foreach(libdir ${BLAS_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + list(APPEND REQUIRED_LIBS "${BLAS_LIBRARIES}") + if (BLAS_LINKER_FLAGS) + list(APPEND REQUIRED_LDFLAGS "${BLAS_LINKER_FLAGS}") + endif() + endif() + # SCOTCH + if (PASTIX_LOOK_FOR_SCOTCH AND SCOTCH_FOUND) + if (SCOTCH_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${SCOTCH_INCLUDE_DIRS}") + endif() + foreach(libdir ${SCOTCH_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + list(APPEND REQUIRED_LIBS "${SCOTCH_LIBRARIES}") + endif() + # PTSCOTCH + if (PASTIX_LOOK_FOR_PTSCOTCH AND PTSCOTCH_FOUND) + if (PTSCOTCH_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${PTSCOTCH_INCLUDE_DIRS}") + endif() + foreach(libdir ${PTSCOTCH_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + list(APPEND REQUIRED_LIBS "${PTSCOTCH_LIBRARIES}") + endif() + # METIS + if (PASTIX_LOOK_FOR_METIS AND METIS_FOUND) + if (METIS_INCLUDE_DIRS) + list(APPEND REQUIRED_INCDIRS "${METIS_INCLUDE_DIRS}") + endif() + foreach(libdir ${METIS_LIBRARY_DIRS}) + if (libdir) + list(APPEND REQUIRED_LIBDIRS "${libdir}") + endif() + endforeach() + list(APPEND REQUIRED_LIBS "${METIS_LIBRARIES}") + endif() + # Fortran + if (CMAKE_C_COMPILER_ID MATCHES "GNU") + find_library( + FORTRAN_gfortran_LIBRARY + NAMES gfortran + HINTS ${_lib_env} + ) + mark_as_advanced(FORTRAN_gfortran_LIBRARY) + if (FORTRAN_gfortran_LIBRARY) + list(APPEND REQUIRED_LIBS "${FORTRAN_gfortran_LIBRARY}") + endif() + elseif (CMAKE_C_COMPILER_ID MATCHES "Intel") + find_library( + FORTRAN_ifcore_LIBRARY + NAMES ifcore + HINTS ${_lib_env} + ) + mark_as_advanced(FORTRAN_ifcore_LIBRARY) + if (FORTRAN_ifcore_LIBRARY) + list(APPEND REQUIRED_LIBS "${FORTRAN_ifcore_LIBRARY}") + endif() + endif() + # EXTRA LIBS such that pthread, m, rt + list(APPEND REQUIRED_LIBS ${PASTIX_EXTRA_LIBRARIES}) + + # set required libraries for link + set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}") + set(CMAKE_REQUIRED_LIBRARIES) + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LDFLAGS}") + foreach(lib_dir ${REQUIRED_LIBDIRS}) + list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}") + endforeach() + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}") + list(APPEND CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}") + string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + + # test link + unset(PASTIX_WORKS CACHE) + include(CheckFunctionExists) + check_function_exists(pastix PASTIX_WORKS) + mark_as_advanced(PASTIX_WORKS) + + if(PASTIX_WORKS) + # save link with dependencies + set(PASTIX_LIBRARIES_DEP "${REQUIRED_LIBS}") + set(PASTIX_LIBRARY_DIRS_DEP "${REQUIRED_LIBDIRS}") + set(PASTIX_INCLUDE_DIRS_DEP "${REQUIRED_INCDIRS}") + set(PASTIX_LINKER_FLAGS "${REQUIRED_LDFLAGS}") + list(REMOVE_DUPLICATES PASTIX_LIBRARY_DIRS_DEP) + list(REMOVE_DUPLICATES PASTIX_INCLUDE_DIRS_DEP) + list(REMOVE_DUPLICATES PASTIX_LINKER_FLAGS) + else() + if(NOT PASTIX_FIND_QUIETLY) + message(STATUS "Looking for PASTIX : test of pastix() fails") + message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}") + message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}") + message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails") + message(STATUS "Maybe PASTIX is linked with specific libraries. " + "Have you tried with COMPONENTS (MPI/SEQ, STARPU, STARPU_CUDA, SCOTCH, PTSCOTCH, METIS)? " + "See the explanation in FindPASTIX.cmake.") + endif() + endif() + set(CMAKE_REQUIRED_INCLUDES) + set(CMAKE_REQUIRED_FLAGS) + set(CMAKE_REQUIRED_LIBRARIES) +endif(PASTIX_LIBRARIES) + +if (PASTIX_LIBRARIES) + list(GET PASTIX_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + if (${first_lib_path} MATCHES "/lib(32|64)?$") + string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}") + set(PASTIX_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of PASTIX library" FORCE) + else() + set(PASTIX_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of PASTIX library" FORCE) + endif() +endif() +mark_as_advanced(PASTIX_DIR) +mark_as_advanced(PASTIX_DIR_FOUND) + +# check that PASTIX has been found +# --------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(PASTIX DEFAULT_MSG - PASTIX_INCLUDES PASTIX_LIBRARIES) - -mark_as_advanced(PASTIX_INCLUDES PASTIX_LIBRARIES) + PASTIX_LIBRARIES + PASTIX_WORKS)
diff --git a/cmake/FindScotch.cmake b/cmake/FindScotch.cmake index 530340b..89d295a 100644 --- a/cmake/FindScotch.cmake +++ b/cmake/FindScotch.cmake
@@ -1,24 +1,369 @@ -# Pastix requires SCOTCH or METIS (partitioning and reordering tools) +### +# +# @copyright (c) 2009-2014 The University of Tennessee and The University +# of Tennessee Research Foundation. +# All rights reserved. +# @copyright (c) 2012-2014 Inria. All rights reserved. +# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +# +### +# +# - Find SCOTCH include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(SCOTCH +# [REQUIRED] # Fail with error if scotch is not found +# [COMPONENTS <comp1> <comp2> ...] # dependencies +# ) +# +# COMPONENTS can be some of the following: +# - ESMUMPS: to activate detection of Scotch with the esmumps interface +# +# This module finds headers and scotch library. +# Results are reported in variables: +# SCOTCH_FOUND - True if headers and requested libraries were found +# SCOTCH_INCLUDE_DIRS - scotch include directories +# SCOTCH_LIBRARY_DIRS - Link directories for scotch libraries +# SCOTCH_LIBRARIES - scotch component libraries to be linked +# SCOTCH_INTSIZE - Number of octets occupied by a SCOTCH_Num +# +# The user can give specific paths where to find the libraries adding cmake +# options at configure (ex: cmake path/to/project -DSCOTCH=path/to/scotch): +# SCOTCH_DIR - Where to find the base directory of scotch +# SCOTCH_INCDIR - Where to find the header files +# SCOTCH_LIBDIR - Where to find the library files +# The module can also look for the following environment variables if paths +# are not given as cmake variable: SCOTCH_DIR, SCOTCH_INCDIR, SCOTCH_LIBDIR -if (SCOTCH_INCLUDES AND SCOTCH_LIBRARIES) - set(SCOTCH_FIND_QUIETLY TRUE) -endif (SCOTCH_INCLUDES AND SCOTCH_LIBRARIES) +#============================================================================= +# Copyright 2012-2013 Inria +# Copyright 2012-2013 Emmanuel Agullo +# Copyright 2012-2013 Mathieu Faverge +# Copyright 2012 Cedric Castagnede +# Copyright 2013 Florent Pruvost +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file MORSE-Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of Morse, substitute the full +# License text for the above reference.) -find_path(SCOTCH_INCLUDES - NAMES - scotch.h - PATHS - $ENV{SCOTCHDIR} - ${INCLUDE_INSTALL_DIR} - PATH_SUFFIXES - scotch -) +if (NOT SCOTCH_FOUND) + set(SCOTCH_DIR "" CACHE PATH "Installation directory of SCOTCH library") + if (NOT SCOTCH_FIND_QUIETLY) + message(STATUS "A cache variable, namely SCOTCH_DIR, has been set to specify the install directory of SCOTCH") + endif() +endif() + +# Set the version to find +set(SCOTCH_LOOK_FOR_ESMUMPS OFF) + +if( SCOTCH_FIND_COMPONENTS ) + foreach( component ${SCOTCH_FIND_COMPONENTS} ) + if (${component} STREQUAL "ESMUMPS") + # means we look for esmumps library + set(SCOTCH_LOOK_FOR_ESMUMPS ON) + endif() + endforeach() +endif() + +# SCOTCH may depend on Threads, try to find it +if (NOT THREADS_FOUND) + if (SCOTCH_FIND_REQUIRED) + find_package(Threads REQUIRED) + else() + find_package(Threads) + endif() +endif() + +# Looking for include +# ------------------- + +# Add system include paths to search include +# ------------------------------------------ +unset(_inc_env) +set(ENV_SCOTCH_DIR "$ENV{SCOTCH_DIR}") +set(ENV_SCOTCH_INCDIR "$ENV{SCOTCH_INCDIR}") +if(ENV_SCOTCH_INCDIR) + list(APPEND _inc_env "${ENV_SCOTCH_INCDIR}") +elseif(ENV_SCOTCH_DIR) + list(APPEND _inc_env "${ENV_SCOTCH_DIR}") + list(APPEND _inc_env "${ENV_SCOTCH_DIR}/include") + list(APPEND _inc_env "${ENV_SCOTCH_DIR}/include/scotch") +else() + if(WIN32) + string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}") + else() + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{CPATH}") + list(APPEND _inc_env "${_path_env}") + string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}") + list(APPEND _inc_env "${_path_env}") + endif() +endif() +list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}") +list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}") +list(REMOVE_DUPLICATES _inc_env) -find_library(SCOTCH_LIBRARIES scotch PATHS $ENV{SCOTCHDIR} ${LIB_INSTALL_DIR}) +# Try to find the scotch header in the given paths +# ------------------------------------------------- +# call cmake macro to find the header path +if(SCOTCH_INCDIR) + set(SCOTCH_scotch.h_DIRS "SCOTCH_scotch.h_DIRS-NOTFOUND") + find_path(SCOTCH_scotch.h_DIRS + NAMES scotch.h + HINTS ${SCOTCH_INCDIR}) +else() + if(SCOTCH_DIR) + set(SCOTCH_scotch.h_DIRS "SCOTCH_scotch.h_DIRS-NOTFOUND") + find_path(SCOTCH_scotch.h_DIRS + NAMES scotch.h + HINTS ${SCOTCH_DIR} + PATH_SUFFIXES "include" "include/scotch") + else() + set(SCOTCH_scotch.h_DIRS "SCOTCH_scotch.h_DIRS-NOTFOUND") + find_path(SCOTCH_scotch.h_DIRS + NAMES scotch.h + HINTS ${_inc_env} + PATH_SUFFIXES "scotch") + endif() +endif() +mark_as_advanced(SCOTCH_scotch.h_DIRS) +# If found, add path to cmake variable +# ------------------------------------ +if (SCOTCH_scotch.h_DIRS) + set(SCOTCH_INCLUDE_DIRS "${SCOTCH_scotch.h_DIRS}") +else () + set(SCOTCH_INCLUDE_DIRS "SCOTCH_INCLUDE_DIRS-NOTFOUND") + if (NOT SCOTCH_FIND_QUIETLY) + message(STATUS "Looking for scotch -- scotch.h not found") + endif() +endif() +list(REMOVE_DUPLICATES SCOTCH_INCLUDE_DIRS) + +# Looking for lib +# --------------- + +# Add system library paths to search lib +# -------------------------------------- +unset(_lib_env) +set(ENV_SCOTCH_LIBDIR "$ENV{SCOTCH_LIBDIR}") +if(ENV_SCOTCH_LIBDIR) + list(APPEND _lib_env "${ENV_SCOTCH_LIBDIR}") +elseif(ENV_SCOTCH_DIR) + list(APPEND _lib_env "${ENV_SCOTCH_DIR}") + list(APPEND _lib_env "${ENV_SCOTCH_DIR}/lib") +else() + if(WIN32) + string(REPLACE ":" ";" _lib_env "$ENV{LIB}") + else() + if(APPLE) + string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}") + else() + string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}") + endif() + list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}") + list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}") + endif() +endif() +list(REMOVE_DUPLICATES _lib_env) + +# Try to find the scotch lib in the given paths +# ---------------------------------------------- + +set(SCOTCH_libs_to_find "scotch;scotcherrexit") +if (SCOTCH_LOOK_FOR_ESMUMPS) + list(INSERT SCOTCH_libs_to_find 0 "esmumps") +endif() + +# call cmake macro to find the lib path +if(SCOTCH_LIBDIR) + foreach(scotch_lib ${SCOTCH_libs_to_find}) + set(SCOTCH_${scotch_lib}_LIBRARY "SCOTCH_${scotch_lib}_LIBRARY-NOTFOUND") + find_library(SCOTCH_${scotch_lib}_LIBRARY + NAMES ${scotch_lib} + HINTS ${SCOTCH_LIBDIR}) + endforeach() +else() + if(SCOTCH_DIR) + foreach(scotch_lib ${SCOTCH_libs_to_find}) + set(SCOTCH_${scotch_lib}_LIBRARY "SCOTCH_${scotch_lib}_LIBRARY-NOTFOUND") + find_library(SCOTCH_${scotch_lib}_LIBRARY + NAMES ${scotch_lib} + HINTS ${SCOTCH_DIR} + PATH_SUFFIXES lib lib32 lib64) + endforeach() + else() + foreach(scotch_lib ${SCOTCH_libs_to_find}) + set(SCOTCH_${scotch_lib}_LIBRARY "SCOTCH_${scotch_lib}_LIBRARY-NOTFOUND") + find_library(SCOTCH_${scotch_lib}_LIBRARY + NAMES ${scotch_lib} + HINTS ${_lib_env}) + endforeach() + endif() +endif() + +set(SCOTCH_LIBRARIES "") +set(SCOTCH_LIBRARY_DIRS "") +# If found, add path to cmake variable +# ------------------------------------ +foreach(scotch_lib ${SCOTCH_libs_to_find}) + + if (SCOTCH_${scotch_lib}_LIBRARY) + get_filename_component(${scotch_lib}_lib_path "${SCOTCH_${scotch_lib}_LIBRARY}" PATH) + # set cmake variables + list(APPEND SCOTCH_LIBRARIES "${SCOTCH_${scotch_lib}_LIBRARY}") + list(APPEND SCOTCH_LIBRARY_DIRS "${${scotch_lib}_lib_path}") + else () + list(APPEND SCOTCH_LIBRARIES "${SCOTCH_${scotch_lib}_LIBRARY}") + if (NOT SCOTCH_FIND_QUIETLY) + message(STATUS "Looking for scotch -- lib ${scotch_lib} not found") + endif() + endif () + + mark_as_advanced(SCOTCH_${scotch_lib}_LIBRARY) + +endforeach() +list(REMOVE_DUPLICATES SCOTCH_LIBRARY_DIRS) + +# check a function to validate the find +if(SCOTCH_LIBRARIES) + + set(REQUIRED_INCDIRS) + set(REQUIRED_LIBDIRS) + set(REQUIRED_LIBS) + + # SCOTCH + if (SCOTCH_INCLUDE_DIRS) + set(REQUIRED_INCDIRS "${SCOTCH_INCLUDE_DIRS}") + endif() + if (SCOTCH_LIBRARY_DIRS) + set(REQUIRED_LIBDIRS "${SCOTCH_LIBRARY_DIRS}") + endif() + set(REQUIRED_LIBS "${SCOTCH_LIBRARIES}") + # THREADS + if(CMAKE_THREAD_LIBS_INIT) + list(APPEND REQUIRED_LIBS "${CMAKE_THREAD_LIBS_INIT}") + endif() + set(Z_LIBRARY "Z_LIBRARY-NOTFOUND") + find_library(Z_LIBRARY NAMES z) + mark_as_advanced(Z_LIBRARY) + if(Z_LIBRARY) + list(APPEND REQUIRED_LIBS "-lz") + endif() + set(M_LIBRARY "M_LIBRARY-NOTFOUND") + find_library(M_LIBRARY NAMES m) + mark_as_advanced(M_LIBRARY) + if(M_LIBRARY) + list(APPEND REQUIRED_LIBS "-lm") + endif() + set(RT_LIBRARY "RT_LIBRARY-NOTFOUND") + find_library(RT_LIBRARY NAMES rt) + mark_as_advanced(RT_LIBRARY) + if(RT_LIBRARY) + list(APPEND REQUIRED_LIBS "-lrt") + endif() + + # set required libraries for link + set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}") + set(CMAKE_REQUIRED_LIBRARIES) + foreach(lib_dir ${REQUIRED_LIBDIRS}) + list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}") + endforeach() + list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}") + string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + + # test link + unset(SCOTCH_WORKS CACHE) + include(CheckFunctionExists) + check_function_exists(SCOTCH_graphInit SCOTCH_WORKS) + mark_as_advanced(SCOTCH_WORKS) + + if(SCOTCH_WORKS) + # save link with dependencies + set(SCOTCH_LIBRARIES "${REQUIRED_LIBS}") + else() + if(NOT SCOTCH_FIND_QUIETLY) + message(STATUS "Looking for SCOTCH : test of SCOTCH_graphInit with SCOTCH library fails") + message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}") + message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}") + message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails") + endif() + endif() + set(CMAKE_REQUIRED_INCLUDES) + set(CMAKE_REQUIRED_FLAGS) + set(CMAKE_REQUIRED_LIBRARIES) +endif(SCOTCH_LIBRARIES) + +if (SCOTCH_LIBRARIES) + list(GET SCOTCH_LIBRARIES 0 first_lib) + get_filename_component(first_lib_path "${first_lib}" PATH) + if (${first_lib_path} MATCHES "/lib(32|64)?$") + string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}") + set(SCOTCH_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of SCOTCH library" FORCE) + else() + set(SCOTCH_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of SCOTCH library" FORCE) + endif() +endif() +mark_as_advanced(SCOTCH_DIR) +mark_as_advanced(SCOTCH_DIR_FOUND) + +# Check the size of SCOTCH_Num +# --------------------------------- +set(CMAKE_REQUIRED_INCLUDES ${SCOTCH_INCLUDE_DIRS}) + +include(CheckCSourceRuns) +#stdio.h and stdint.h should be included by scotch.h directly +set(SCOTCH_C_TEST_SCOTCH_Num_4 " +#include <stdio.h> +#include <stdint.h> +#include <scotch.h> +int main(int argc, char **argv) { + if (sizeof(SCOTCH_Num) == 4) + return 0; + else + return 1; +} +") + +set(SCOTCH_C_TEST_SCOTCH_Num_8 " +#include <stdio.h> +#include <stdint.h> +#include <scotch.h> +int main(int argc, char **argv) { + if (sizeof(SCOTCH_Num) == 8) + return 0; + else + return 1; +} +") +check_c_source_runs("${SCOTCH_C_TEST_SCOTCH_Num_4}" SCOTCH_Num_4) +if(NOT SCOTCH_Num_4) + check_c_source_runs("${SCOTCH_C_TEST_SCOTCH_Num_8}" SCOTCH_Num_8) + if(NOT SCOTCH_Num_8) + set(SCOTCH_INTSIZE -1) + else() + set(SCOTCH_INTSIZE 8) + endif() +else() + set(SCOTCH_INTSIZE 4) +endif() +set(CMAKE_REQUIRED_INCLUDES "") + +# check that SCOTCH has been found +# --------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(SCOTCH DEFAULT_MSG - SCOTCH_INCLUDES SCOTCH_LIBRARIES) - -mark_as_advanced(SCOTCH_INCLUDES SCOTCH_LIBRARIES) + SCOTCH_LIBRARIES + SCOTCH_WORKS) +# +# TODO: Add possibility to check for specific functions in the library +#
diff --git a/cmake/FindSuperLU.cmake b/cmake/FindSuperLU.cmake index e4142fe..f38146e 100644 --- a/cmake/FindSuperLU.cmake +++ b/cmake/FindSuperLU.cmake
@@ -17,7 +17,10 @@ SRC ) -find_library(SUPERLU_LIBRARIES NAMES "superlu_4.3" "superlu_4.2" "superlu_4.1" "superlu_4.0" "superlu_3.1" "superlu_3.0" "superlu" PATHS $ENV{SUPERLUDIR} ${LIB_INSTALL_DIR} PATH_SUFFIXES lib) +find_library(SUPERLU_LIBRARIES + NAMES "superlu_5.2.1" "superlu_5.2" "superlu_5.1.1" "superlu_5.1" "superlu_5.0" "superlu_4.3" "superlu_4.2" "superlu_4.1" "superlu_4.0" "superlu_3.1" "superlu_3.0" "superlu" + PATHS $ENV{SUPERLUDIR} ${LIB_INSTALL_DIR} + PATH_SUFFIXES lib) if(SUPERLU_INCLUDES AND SUPERLU_LIBRARIES) @@ -48,11 +51,25 @@ }" SUPERLU_HAS_CLEAN_ENUMS) -if(SUPERLU_HAS_CLEAN_ENUMS) +check_cxx_source_compiles(" +typedef int int_t; +#include <supermatrix.h> +#include <slu_util.h> +int main(void) +{ + GlobalLU_t glu; + return 0; +}" +SUPERLU_HAS_GLOBALLU_T) + +if(SUPERLU_HAS_GLOBALLU_T) + # at least 5.0 + set(SUPERLU_VERSION_VAR "5.0") +elseif(SUPERLU_HAS_CLEAN_ENUMS) # at least 4.3 set(SUPERLU_VERSION_VAR "4.3") elseif(SUPERLU_HAS_GLOBAL_MEM_USAGE_T) - # at least 4.3 + # at least 4.0 set(SUPERLU_VERSION_VAR "4.0") else() set(SUPERLU_VERSION_VAR "3.0")
diff --git a/cmake/FindTriSYCL.cmake b/cmake/FindTriSYCL.cmake new file mode 100644 index 0000000..cb21541 --- /dev/null +++ b/cmake/FindTriSYCL.cmake
@@ -0,0 +1,152 @@ +#.rst: +# FindTriSYCL +#--------------- +# +# TODO : insert Copyright and licence + +######################### +# FindTriSYCL.cmake +######################### +# +# Tools for finding and building with TriSYCL. +# +# User must define TRISYCL_INCLUDE_DIR pointing to the triSYCL +# include directory. +# +# Latest version of this file can be found at: +# https://github.com/triSYCL/triSYCL + +# Requite CMake version 3.5 or higher +cmake_minimum_required (VERSION 3.5) + +# Check that a supported host compiler can be found +if(CMAKE_COMPILER_IS_GNUCXX) + # Require at least gcc 5.4 + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.4) + message(FATAL_ERROR + "host compiler - Not found! (gcc version must be at least 5.4)") + else() + message(STATUS "host compiler - gcc ${CMAKE_CXX_COMPILER_VERSION}") + endif() +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # Require at least clang 3.9 + if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.9) + message(FATAL_ERROR + "host compiler - Not found! (clang version must be at least 3.9)") + else() + message(STATUS "host compiler - clang ${CMAKE_CXX_COMPILER_VERSION}") + endif() +else() + message(WARNING + "host compiler - Not found! (triSYCL supports GCC and Clang)") +endif() + +#triSYCL options +option(TRISYCL_OPENMP "triSYCL multi-threading with OpenMP" ON) +option(TRISYCL_OPENCL "triSYCL OpenCL interoperability mode" OFF) +option(TRISYCL_NO_ASYNC "triSYCL use synchronous kernel execution" OFF) +option(TRISYCL_DEBUG "triSCYL use debug mode" OFF) +option(TRISYCL_DEBUG_STRUCTORS "triSYCL trace of object lifetimes" OFF) +option(TRISYCL_TRACE_KERNEL "triSYCL trace of kernel execution" OFF) + +mark_as_advanced(TRISYCL_OPENMP) +mark_as_advanced(TRISYCL_OPENCL) +mark_as_advanced(TRISYCL_NO_ASYNC) +mark_as_advanced(TRISYCL_DEBUG) +mark_as_advanced(TRISYCL_DEBUG_STRUCTORS) +mark_as_advanced(TRISYCL_TRACE_KERNEL) + +#triSYCL definitions +set(CL_SYCL_LANGUAGE_VERSION 220 CACHE VERSION + "Host language version to be used by trisYCL (default is: 220)") +set(TRISYCL_CL_LANGUAGE_VERSION 220 CACHE VERSION + "Device language version to be used by trisYCL (default is: 220)") +#set(TRISYCL_COMPILE_OPTIONS "-std=c++1z -Wall -Wextra") +set(CMAKE_CXX_STANDARD 14) +set(CXX_STANDARD_REQUIRED ON) + + +# Find OpenCL package +if(TRISYCL_OPENCL) + find_package(OpenCL REQUIRED) + if(UNIX) + set(BOOST_COMPUTE_INCPATH /usr/include/compute CACHE PATH + "Path to Boost.Compute headers (default is: /usr/include/compute)") + endif(UNIX) +endif() + +# Find OpenMP package +if(TRISYCL_OPENMP) + find_package(OpenMP REQUIRED) +endif() + +# Find Boost +find_package(Boost 1.58 REQUIRED COMPONENTS chrono log) + +# If debug or trace we need boost log +if(TRISYCL_DEBUG OR TRISYCL_DEBUG_STRUCTORS OR TRISYCL_TRACE_KERNEL) + set(LOG_NEEDED ON) +else() + set(LOG_NEEDED OFF) +endif() + +find_package(Threads REQUIRED) + +# Find triSYCL directory +if(NOT TRISYCL_INCLUDE_DIR) + message(FATAL_ERROR + "triSYCL include directory - Not found! (please set TRISYCL_INCLUDE_DIR") +else() + message(STATUS "triSYCL include directory - Found ${TRISYCL_INCLUDE_DIR}") +endif() + +####################### +# add_sycl_to_target +####################### +# +# Sets the proper flags and includes for the target compilation. +# +# targetName : Name of the target to add a SYCL to. +# sourceFile : Source file to be compiled for SYCL. +# binaryDir : Intermediate directory to output the integration header. +# +function(add_sycl_to_target targetName sourceFile binaryDir) + + # Add include directories to the "#include <>" paths + target_include_directories (${targetName} PUBLIC + ${TRISYCL_INCLUDE_DIR} + ${Boost_INCLUDE_DIRS} + $<$<BOOL:${TRISYCL_OPENCL}>:${OpenCL_INCLUDE_DIRS}> + $<$<BOOL:${TRISYCL_OPENCL}>:${BOOST_COMPUTE_INCPATH}>) + + + # Link dependencies + target_link_libraries(${targetName} PUBLIC + $<$<BOOL:${TRISYCL_OPENCL}>:${OpenCL_LIBRARIES}> + Threads::Threads + $<$<BOOL:${LOG_NEEDED}>:Boost::log> + Boost::chrono) + + + # Compile definitions + target_compile_definitions(${targetName} PUBLIC + $<$<BOOL:${TRISYCL_NO_ASYNC}>:TRISYCL_NO_ASYNC> + $<$<BOOL:${TRISYCL_OPENCL}>:TRISYCL_OPENCL> + $<$<BOOL:${TRISYCL_DEBUG}>:TRISYCL_DEBUG> + $<$<BOOL:${TRISYCL_DEBUG_STRUCTORS}>:TRISYCL_DEBUG_STRUCTORS> + $<$<BOOL:${TRISYCL_TRACE_KERNEL}>:TRISYCL_TRACE_KERNEL> + $<$<BOOL:${LOG_NEEDED}>:BOOST_LOG_DYN_LINK>) + + # C++ and OpenMP requirements + target_compile_options(${targetName} PUBLIC + ${TRISYCL_COMPILE_OPTIONS} + $<$<BOOL:${TRISYCL_OPENMP}>:${OpenMP_CXX_FLAGS}>) + + if(${TRISYCL_OPENMP} AND (NOT WIN32)) + # Does not support generator expressions + set_target_properties(${targetName} + PROPERTIES + LINK_FLAGS ${OpenMP_CXX_FLAGS}) + endif(${TRISYCL_OPENMP} AND (NOT WIN32)) + +endfunction(add_sycl_to_target)
diff --git a/cmake/FindXsmm.cmake b/cmake/FindXsmm.cmake new file mode 100644 index 0000000..809d6f4 --- /dev/null +++ b/cmake/FindXsmm.cmake
@@ -0,0 +1,25 @@ +# libxsmm support. +# libxsmm provides matrix multiplication kernels optimized for +# the latest Intel architectures. +# Download the library from https://github.com/hfp/libxsmm +# Compile with make BLAS=0 + +if (LIBXSMM) + set(XSMM_FIND_QUIETLY TRUE) + set(XSMM_INCLUDES ${LIBXSMM}/include) + set(XSMM_LIBRARIES ${LIBXSMM}/lib) +endif (LIBXSMM) + +find_path(LIBXSMM + NAMES + libxsmm.h + PATHS + $ENV{XSMMDIR}/include + ${INCLUDE_INSTALL_DIR} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(XSMM DEFAULT_MSG + LIBXSMM) + +mark_as_advanced(LIBXSMM)
diff --git a/cmake/language_support.cmake b/cmake/language_support.cmake index 2f14f30..ddba509 100644 --- a/cmake/language_support.cmake +++ b/cmake/language_support.cmake
@@ -26,7 +26,7 @@ cmake_minimum_required(VERSION 2.8.0) set (CMAKE_Fortran_FLAGS \"${CMAKE_Fortran_FLAGS}\") set (CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS}\") - enable_language(${language} OPTIONAL) + enable_language(${language}) ") file(REMOVE_RECURSE ${CMAKE_BINARY_DIR}/language_tests/${language}) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/language_tests/${language})
diff --git a/debug/gdb/printers.py b/debug/gdb/printers.py index 0d67a5f..24961d1 100644 --- a/debug/gdb/printers.py +++ b/debug/gdb/printers.py
@@ -10,8 +10,7 @@ # Pretty printers for Eigen::Matrix # This is still pretty basic as the python extension to gdb is still pretty basic. -# It cannot handle complex eigen types and it doesn't support any of the other eigen types -# Such as quaternion or some other type. +# It cannot handle complex eigen types and it doesn't support many of the other eigen types # This code supports fixed size as well as dynamic size matrices # To use it: @@ -29,7 +28,45 @@ import gdb import re import itertools +from bisect import bisect_left +# Basic row/column iteration code for use with Sparse and Dense matrices +class _MatrixEntryIterator(object): + + def __init__ (self, rows, cols, rowMajor): + self.rows = rows + self.cols = cols + self.currentRow = 0 + self.currentCol = 0 + self.rowMajor = rowMajor + + def __iter__ (self): + return self + + def next(self): + return self.__next__() # Python 2.x compatibility + + def __next__(self): + row = self.currentRow + col = self.currentCol + if self.rowMajor == 0: + if self.currentCol >= self.cols: + raise StopIteration + + self.currentRow = self.currentRow + 1 + if self.currentRow >= self.rows: + self.currentRow = 0 + self.currentCol = self.currentCol + 1 + else: + if self.currentRow >= self.rows: + raise StopIteration + + self.currentCol = self.currentCol + 1 + if self.currentCol >= self.cols: + self.currentCol = 0 + self.currentRow = self.currentRow + 1 + + return (row, col) class EigenMatrixPrinter: "Print Eigen Matrix or Array of some kind" @@ -77,42 +114,15 @@ self.data = self.data['array'] self.data = self.data.cast(self.innerType.pointer()) - class _iterator: + class _iterator(_MatrixEntryIterator): def __init__ (self, rows, cols, dataPtr, rowMajor): - self.rows = rows - self.cols = cols - self.dataPtr = dataPtr - self.currentRow = 0 - self.currentCol = 0 - self.rowMajor = rowMajor - - def __iter__ (self): - return self + super(EigenMatrixPrinter._iterator, self).__init__(rows, cols, rowMajor) - def next(self): - return self.__next__() # Python 2.x compatibility + self.dataPtr = dataPtr def __next__(self): - row = self.currentRow - col = self.currentCol - if self.rowMajor == 0: - if self.currentCol >= self.cols: - raise StopIteration - - self.currentRow = self.currentRow + 1 - if self.currentRow >= self.rows: - self.currentRow = 0 - self.currentCol = self.currentCol + 1 - else: - if self.currentRow >= self.rows: - raise StopIteration - - self.currentCol = self.currentCol + 1 - if self.currentCol >= self.cols: - self.currentCol = 0 - self.currentRow = self.currentRow + 1 - + row, col = super(EigenMatrixPrinter._iterator, self).__next__() item = self.dataPtr.dereference() self.dataPtr = self.dataPtr + 1 @@ -129,6 +139,95 @@ def to_string(self): return "Eigen::%s<%s,%d,%d,%s> (data ptr: %s)" % (self.variety, self.innerType, self.rows, self.cols, "RowMajor" if self.rowMajor else "ColMajor", self.data) +class EigenSparseMatrixPrinter: + "Print an Eigen SparseMatrix" + + def __init__(self, val): + "Extract all the necessary information" + + type = val.type + if type.code == gdb.TYPE_CODE_REF: + type = type.target() + self.type = type.unqualified().strip_typedefs() + tag = self.type.tag + regex = re.compile('\<.*\>') + m = regex.findall(tag)[0][1:-1] + template_params = m.split(',') + template_params = [x.replace(" ", "") for x in template_params] + + self.options = 0 + if len(template_params) > 1: + self.options = template_params[1]; + + self.rowMajor = (int(self.options) & 0x1) + + self.innerType = self.type.template_argument(0) + + self.val = val + + self.data = self.val['m_data'] + self.data = self.data.cast(self.innerType.pointer()) + + class _iterator(_MatrixEntryIterator): + def __init__ (self, rows, cols, val, rowMajor): + super(EigenSparseMatrixPrinter._iterator, self).__init__(rows, cols, rowMajor) + + self.val = val + + def __next__(self): + + row, col = super(EigenSparseMatrixPrinter._iterator, self).__next__() + + # repeat calculations from SparseMatrix.h: + outer = row if self.rowMajor else col + inner = col if self.rowMajor else row + start = self.val['m_outerIndex'][outer] + end = ((start + self.val['m_innerNonZeros'][outer]) if self.val['m_innerNonZeros'] else + self.val['m_outerIndex'][outer+1]) + + # and from CompressedStorage.h: + data = self.val['m_data'] + if start >= end: + item = 0 + elif (end > start) and (inner == data['m_indices'][end-1]): + item = data['m_values'][end-1] + else: + # create Python index list from the target range within m_indices + indices = [data['m_indices'][x] for x in range(int(start), int(end)-1)] + # find the index with binary search + idx = int(start) + bisect_left(indices, inner) + if ((idx < end) and (data['m_indices'][idx] == inner)): + item = data['m_values'][idx] + else: + item = 0 + + return ('[%d,%d]' % (row, col), item) + + def children(self): + if self.data: + return self._iterator(self.rows(), self.cols(), self.val, self.rowMajor) + + return iter([]) # empty matrix, for now + + + def rows(self): + return self.val['m_outerSize'] if self.rowMajor else self.val['m_innerSize'] + + def cols(self): + return self.val['m_innerSize'] if self.rowMajor else self.val['m_outerSize'] + + def to_string(self): + + if self.data: + status = ("not compressed" if self.val['m_innerNonZeros'] else "compressed") + else: + status = "empty" + dimensions = "%d x %d" % (self.rows(), self.cols()) + layout = "row" if self.rowMajor else "column" + + return "Eigen::SparseMatrix<%s>, %s, %s major, %s" % ( + self.innerType, dimensions, layout, status ) + class EigenQuaternionPrinter: "Print an Eigen Quaternion" @@ -156,7 +255,7 @@ return self def next(self): - return self.__next__() # Python 2.x compatibility + return self.__next__() # Python 2.x compatibility def __next__(self): element = self.currentElement @@ -180,6 +279,7 @@ def build_eigen_dictionary (): pretty_printers_dict[re.compile('^Eigen::Quaternion<.*>$')] = lambda val: EigenQuaternionPrinter(val) pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter("Matrix", val) + pretty_printers_dict[re.compile('^Eigen::SparseMatrix<.*>$')] = lambda val: EigenSparseMatrixPrinter(val) pretty_printers_dict[re.compile('^Eigen::Array<.*>$')] = lambda val: EigenMatrixPrinter("Array", val) def register_eigen_printers(obj):
diff --git a/debug/msvc/eigen_autoexp_part.dat b/debug/msvc/eigen_autoexp_part.dat index 07aa437..35ef580 100644 --- a/debug/msvc/eigen_autoexp_part.dat +++ b/debug/msvc/eigen_autoexp_part.dat
@@ -14,7 +14,7 @@ ; * - Eigen::Matrix<*,-1,+,*,*,*> ; * - Eigen::Matrix<*,+,+,*,*,*> ; * -; * Matrices are displayed properly independantly of the memory +; * Matrices are displayed properly independently of the memory ; * alignment (RowMajor vs. ColMajor). ; * ; * This file is distributed WITHOUT ANY WARRANTY. Please ensure
diff --git a/doc/A05_PortingFrom2To3.dox b/doc/A05_PortingFrom2To3.dox deleted file mode 100644 index 0dbddb9..0000000 --- a/doc/A05_PortingFrom2To3.dox +++ /dev/null
@@ -1,299 +0,0 @@ -namespace Eigen { - -/** \page Eigen2ToEigen3 Porting from Eigen2 to Eigen3 - -This page lists the most important API changes between Eigen2 and Eigen3, -and gives tips to help porting your application from Eigen2 to Eigen3. - -\eigenAutoToc - -\section CompatibilitySupport Eigen2 compatibility support - -Up to version 3.2 %Eigen provides <a href="http://eigen.tuxfamily.org/dox-3.2/Eigen2SupportModes.html">Eigen2 support modes</a>. These are removed now, because they were barely used anymore and became hard to maintain after internal re-designs. -You can still use them by first <a href="http://eigen.tuxfamily.org/dox-3.2/Eigen2ToEigen3.html">porting your code to Eigen 3.2</a>. - -\section Using The USING_PART_OF_NAMESPACE_EIGEN macro - -The USING_PART_OF_NAMESPACE_EIGEN macro has been removed. In Eigen 3, just do: -\code -using namespace Eigen; -\endcode - -\section ComplexDot Dot products over complex numbers - -This is the single trickiest change between Eigen 2 and Eigen 3. It only affects code using \c std::complex numbers as scalar type. - -Eigen 2's dot product was linear in the first variable. Eigen 3's dot product is linear in the second variable. In other words, the Eigen 2 code \code x.dot(y) \endcode is equivalent to the Eigen 3 code \code y.dot(x) \endcode In yet other words, dot products are complex-conjugated in Eigen 3 compared to Eigen 2. The switch to the new convention was commanded by common usage, especially with the notation \f$ x^Ty \f$ for dot products of column-vectors. - -\section VectorBlocks Vector blocks - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th></th> -<tr><td>\code -vector.start(length) -vector.start<length>() -vector.end(length) -vector.end<length>() -\endcode</td><td>\code -vector.head(length) -vector.head<length>() -vector.tail(length) -vector.tail<length>() -\endcode</td></tr> -</table> - - -\section Corners Matrix Corners - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th></th> -<tr><td>\code -matrix.corner(TopLeft,r,c) -matrix.corner(TopRight,r,c) -matrix.corner(BottomLeft,r,c) -matrix.corner(BottomRight,r,c) -matrix.corner<r,c>(TopLeft) -matrix.corner<r,c>(TopRight) -matrix.corner<r,c>(BottomLeft) -matrix.corner<r,c>(BottomRight) -\endcode</td><td>\code -matrix.topLeftCorner(r,c) -matrix.topRightCorner(r,c) -matrix.bottomLeftCorner(r,c) -matrix.bottomRightCorner(r,c) -matrix.topLeftCorner<r,c>() -matrix.topRightCorner<r,c>() -matrix.bottomLeftCorner<r,c>() -matrix.bottomRightCorner<r,c>() -\endcode</td> -</tr> -</table> - -Notice that Eigen3 also provides these new convenience methods: topRows(), bottomRows(), leftCols(), rightCols(). See in class DenseBase. - -\section CoefficientWiseOperations Coefficient wise operations - -In Eigen2, coefficient wise operations which have no proper mathematical definition (as a coefficient wise product) -were achieved using the .cwise() prefix, e.g.: -\code a.cwise() * b \endcode -In Eigen3 this .cwise() prefix has been superseded by a new kind of matrix type called -Array for which all operations are performed coefficient wise. You can easily view a matrix as an array and vice versa using -the MatrixBase::array() and ArrayBase::matrix() functions respectively. Here is an example: -\code -Vector4f a, b, c; -c = a.array() * b.array(); -\endcode -Note that the .array() function is not at all a synonym of the deprecated .cwise() prefix. -While the .cwise() prefix changed the behavior of the following operator, the array() function performs -a permanent conversion to the array world. Therefore, for binary operations such as the coefficient wise product, -both sides must be converted to an \em array as in the above example. On the other hand, when you -concatenate multiple coefficient wise operations you only have to do the conversion once, e.g.: -\code -Vector4f a, b, c; -c = a.array().abs().pow(3) * b.array().abs().sin(); -\endcode -With Eigen2 you would have written: -\code -c = (a.cwise().abs().cwise().pow(3)).cwise() * (b.cwise().abs().cwise().sin()); -\endcode - -\section PartAndExtract Triangular and self-adjoint matrices - -In Eigen 2 you had to play with the part, extract, and marked functions to deal with triangular and selfadjoint matrices. In Eigen 3, all these functions have been removed in favor of the concept of \em views: - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th></tr> -<tr><td>\code -A.part<UpperTriangular>(); -A.part<StrictlyLowerTriangular>(); \endcode</td> -<td>\code -A.triangularView<Upper>() -A.triangularView<StrictlyLower>()\endcode</td></tr> -<tr><td>\code -A.extract<UpperTriangular>(); -A.extract<StrictlyLowerTriangular>();\endcode</td> -<td>\code -A.triangularView<Upper>() -A.triangularView<StrictlyLower>()\endcode</td></tr> -<tr><td>\code -A.marked<UpperTriangular>(); -A.marked<StrictlyLowerTriangular>();\endcode</td> -<td>\code -A.triangularView<Upper>() -A.triangularView<StrictlyLower>()\endcode</td></tr> -<tr><td colspan="2"></td></tr> -<tr><td>\code -A.part<SelfAdfjoint|UpperTriangular>(); -A.extract<SelfAdfjoint|LowerTriangular>();\endcode</td> -<td>\code -A.selfadjointView<Upper>() -A.selfadjointView<Lower>()\endcode</td></tr> -<tr><td colspan="2"></td></tr> -<tr><td>\code -UpperTriangular -LowerTriangular -UnitUpperTriangular -UnitLowerTriangular -StrictlyUpperTriangular -StrictlyLowerTriangular -\endcode</td><td>\code -Upper -Lower -UnitUpper -UnitLower -StrictlyUpper -StrictlyLower -\endcode</td> -</tr> -</table> - -\sa class TriangularView, class SelfAdjointView - -\section TriangularSolveInPlace Triangular in-place solving - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th></tr> -<tr><td>\code A.triangularSolveInPlace<XxxTriangular>(Y);\endcode</td><td>\code A.triangularView<Xxx>().solveInPlace(Y);\endcode</td></tr> -</table> - - -\section Decompositions Matrix decompositions - -Some of Eigen 2's matrix decompositions have been renamed in Eigen 3, while some others have been removed and are replaced by other decompositions in Eigen 3. - -<table class="manual"> - <tr> - <th>Eigen 2</th> - <th>Eigen 3</th> - <th>Notes</th> - </tr> - <tr> - <td>LU</td> - <td>FullPivLU</td> - <td class="alt">See also the new PartialPivLU, it's much faster</td> - </tr> - <tr> - <td>QR</td> - <td>HouseholderQR</td> - <td class="alt">See also the new ColPivHouseholderQR, it's more reliable</td> - </tr> - <tr> - <td>SVD</td> - <td>JacobiSVD</td> - <td class="alt">We currently don't have a bidiagonalizing SVD; of course this is planned.</td> - </tr> - <tr> - <td>EigenSolver and friends</td> - <td>\code #include<Eigen/Eigenvalues> \endcode </td> - <td class="alt">Moved to separate module</td> - </tr> -</table> - -\section LinearSolvers Linear solvers - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th><th>Notes</th></tr> -<tr><td>\code A.lu();\endcode</td> -<td>\code A.fullPivLu();\endcode</td> -<td class="alt">Now A.lu() returns a PartialPivLU</td></tr> -<tr><td>\code A.lu().solve(B,&X);\endcode</td> -<td>\code X = A.lu().solve(B); - X = A.fullPivLu().solve(B);\endcode</td> -<td class="alt">The returned by value is fully optimized</td></tr> -<tr><td>\code A.llt().solve(B,&X);\endcode</td> -<td>\code X = A.llt().solve(B); - X = A.selfadjointView<Lower>.llt().solve(B); - X = A.selfadjointView<Upper>.llt().solve(B);\endcode</td> -<td class="alt">The returned by value is fully optimized and \n -the selfadjointView API allows you to select the \n -triangular part to work on (default is lower part)</td></tr> -<tr><td>\code A.llt().solveInPlace(B);\endcode</td> -<td>\code B = A.llt().solve(B); - B = A.selfadjointView<Lower>.llt().solve(B); - B = A.selfadjointView<Upper>.llt().solve(B);\endcode</td> -<td class="alt">In place solving</td></tr> -<tr><td>\code A.ldlt().solve(B,&X);\endcode</td> -<td>\code X = A.ldlt().solve(B); - X = A.selfadjointView<Lower>.ldlt().solve(B); - X = A.selfadjointView<Upper>.ldlt().solve(B);\endcode</td> -<td class="alt">The returned by value is fully optimized and \n -the selfadjointView API allows you to select the \n -triangular part to work on</td></tr> -</table> - -\section GeometryModule Changes in the Geometry module - -The Geometry module is the one that changed the most. If you rely heavily on it, it's probably a good idea to use the <a href="http://eigen.tuxfamily.org/dox-3.2/Eigen2SupportModes.html">"Eigen 2 support modes"</a> to perform your migration. - -\section Transform The Transform class - -In Eigen 2, the Transform class didn't really know whether it was a projective or affine transformation. In Eigen 3, it takes a new \a Mode template parameter, which indicates whether it's \a Projective or \a Affine transform. There is no default value. - -The Transform3f (etc) typedefs are no more. In Eigen 3, the Transform typedefs explicitly refer to the \a Projective and \a Affine modes: - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th><th>Notes</th></tr> -<tr> - <td> Transform3f </td> - <td> Affine3f or Projective3f </td> - <td> Of course 3f is just an example here </td> -</tr> -</table> - - -\section LazyVsNoalias Lazy evaluation and noalias - -In Eigen all operations are performed in a lazy fashion except the matrix products which are always evaluated into a temporary by default. -In Eigen2, lazy evaluation could be enforced by tagging a product using the .lazy() function. However, in complex expressions it was not -easy to determine where to put the lazy() function. In Eigen3, the lazy() feature has been superseded by the MatrixBase::noalias() function -which can be used on the left hand side of an assignment when no aliasing can occur. Here is an example: -\code -MatrixXf a, b, c; -... -c.noalias() += 2 * a.transpose() * b; -\endcode -However, the noalias mechanism does not cover all the features of the old .lazy(). Indeed, in some extremely rare cases, -it might be useful to explicit request for a lay product, i.e., for a product which will be evaluated one coefficient at once, on request, -just like any other expressions. To this end you can use the MatrixBase::lazyProduct() function, however we strongly discourage you to -use it unless you are sure of what you are doing, i.e., you have rigourosly measured a speed improvement. - -\section AlignMacros Alignment-related macros - -The EIGEN_ALIGN_128 macro has been renamed to EIGEN_ALIGN16. Don't be surprised, it's just that we switched to counting in bytes ;-) - -The EIGEN_DONT_ALIGN option still exists in Eigen 3, but it has a new cousin: EIGEN_DONT_ALIGN_STATICALLY. It allows to get rid of all static alignment issues while keeping alignment of dynamic-size heap-allocated arrays, thus keeping vectorization for dynamic-size objects. - -\section AlignedMap Aligned Map objects - -A common issue with Eigen 2 was that when mapping an array with Map, there was no way to tell Eigen that your array was aligned. There was a ForceAligned option but it didn't mean that; it was just confusing and has been removed. - -New in Eigen3 is the #Aligned option. See the documentation of class Map. Use it like this: -\code -Map<Vector4f, Aligned> myMappedVector(some_aligned_array); -\endcode -There also are related convenience static methods, which actually are the preferred way as they take care of such things as constness: -\code -result = Vector4f::MapAligned(some_aligned_array); -\endcode - -\section StdContainers STL Containers - -In Eigen2, <tt>\#include\<Eigen/StdVector\></tt> tweaked std::vector to automatically align elements. The problem was that that was quite invasive. In Eigen3, we only override standard behavior if you use Eigen::aligned_allocator<T> as your allocator type. So for example, if you use std::vector<Matrix4f>, you need to do the following change (note that aligned_allocator is under namespace Eigen): - -<table class="manual"> -<tr><th>Eigen 2</th><th>Eigen 3</th></tr> -<tr> - <td> \code std::vector<Matrix4f> \endcode </td> - <td> \code std::vector<Matrix4f, aligned_allocator<Matrix4f> > \endcode </td> -</tr> -</table> - -\section eiPrefix Internal ei_ prefix - -In Eigen2, global internal functions and structures were prefixed by \c ei_. In Eigen3, they all have been moved into the more explicit \c internal namespace. So, e.g., \c ei_sqrt(x) now becomes \c internal::sqrt(x). Of course it is not recommended to rely on Eigen's internal features. - - - -*/ - -}
diff --git a/doc/AsciiQuickReference.txt b/doc/AsciiQuickReference.txt index 9599df6..18b4446 100644 --- a/doc/AsciiQuickReference.txt +++ b/doc/AsciiQuickReference.txt
@@ -36,7 +36,7 @@ MatrixXd::Identity(rows,cols) // eye(rows,cols) C.setIdentity(rows,cols) // C = eye(rows,cols) MatrixXd::Zero(rows,cols) // zeros(rows,cols) -C.setZero(rows,cols) // C = ones(rows,cols) +C.setZero(rows,cols) // C = zeros(rows,cols) MatrixXd::Ones(rows,cols) // ones(rows,cols) C.setOnes(rows,cols) // C = ones(rows,cols) MatrixXd::Random(rows,cols) // rand(rows,cols)*2-1 // MatrixXd::Random returns uniform random numbers in (-1, 1). @@ -50,6 +50,12 @@ // Matrix slicing and blocks. All expressions listed here are read/write. // Templated size versions are faster. Note that Matlab is 1-based (a size N // vector is x(1)...x(N)). +/******************************************************************************/ +/* PLEASE HELP US IMPROVING THIS SECTION */ +/* Eigen 3.4 supports a much improved API for sub-matrices, including, */ +/* slicing and indexing from arrays: */ +/* http://eigen.tuxfamily.org/dox-devel/group__TutorialSlicingIndexing.html */ +/******************************************************************************/ // Eigen // Matlab x.head(n) // x(1:n) x.head<n>() // x(1:n) @@ -84,10 +90,15 @@ // Of particular note is Eigen's swap function which is highly optimized. // Eigen // Matlab -R.row(i) = P.col(j); // R(i, :) = P(:, i) +R.row(i) = P.col(j); // R(i, :) = P(:, j) R.col(j1).swap(mat1.col(j2)); // R(:, [j1 j2]) = R(:, [j2, j1]) // Views, transpose, etc; +/******************************************************************************/ +/* PLEASE HELP US IMPROVING THIS SECTION */ +/* Eigen 3.4 supports a new API for reshaping: */ +/* http://eigen.tuxfamily.org/dox-devel/group__TutorialReshape.html */ +/******************************************************************************/ // Eigen // Matlab R.adjoint() // R' R.transpose() // R.' or conj(R') // Read-write @@ -140,7 +151,7 @@ R.cwiseAbs2() // abs(P.^2) R.array().abs2() // abs(P.^2) (R.array() < s).select(P,Q ); // (R < s ? P : Q) -R = (Q.array()==0).select(P,A) // R(Q==0) = P(Q==0) +R = (Q.array()==0).select(P,R) // R(Q==0) = P(Q==0) R = P.unaryExpr(ptr_fun(func)) // R = arrayfun(func, P) // with: scalar func(const scalar &x);
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 4d01a04..aa36d78 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt
@@ -10,6 +10,9 @@ endif(CMAKE_SYSTEM_NAME MATCHES Linux) endif(CMAKE_COMPILER_IS_GNUCXX) +# some examples and snippets needs c++11, so let's check it once +check_cxx_compiler_flag("-std=c++11" EIGEN_COMPILER_SUPPORT_CPP11) + option(EIGEN_INTERNAL_DOCUMENTATION "Build internal documentation" OFF) @@ -34,8 +37,8 @@ set(EIGEN_DOXY_OUTPUT_DIRECTORY_SUFFIX "/unsupported") set(EIGEN_DOXY_INPUT "\"${Eigen_SOURCE_DIR}/unsupported/Eigen\" \"${Eigen_SOURCE_DIR}/unsupported/doc\"") set(EIGEN_DOXY_HTML_COLORSTYLE_HUE "0") -# set(EIGEN_DOXY_TAGFILES "\"${Eigen_BINARY_DIR}/doc/eigen.doxytags =../\"") -set(EIGEN_DOXY_TAGFILES "") +set(EIGEN_DOXY_TAGFILES "\"${Eigen_BINARY_DIR}/doc/Eigen.doxytags=..\"") +#set(EIGEN_DOXY_TAGFILES "") configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in @@ -78,6 +81,8 @@ COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/html/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/eigen_navtree_hacks.js ${CMAKE_CURRENT_BINARY_DIR}/html/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/Eigen_Silly_Professor_64x64.png ${CMAKE_CURRENT_BINARY_DIR}/html/ + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ftv2pnode.png ${CMAKE_CURRENT_BINARY_DIR}/html/ + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ftv2node.png ${CMAKE_CURRENT_BINARY_DIR}/html/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/AsciiQuickReference.txt ${CMAKE_CURRENT_BINARY_DIR}/html/ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) @@ -88,6 +93,8 @@ COMMAND ${CMAKE_COMMAND} -E make_directory ${Eigen_BINARY_DIR}/doc/html/unsupported COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/eigen_navtree_hacks.js ${CMAKE_CURRENT_BINARY_DIR}/html/unsupported/ COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/Eigen_Silly_Professor_64x64.png ${CMAKE_CURRENT_BINARY_DIR}/html/unsupported/ + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ftv2pnode.png ${CMAKE_CURRENT_BINARY_DIR}/html/unsupported/ + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ftv2node.png ${CMAKE_CURRENT_BINARY_DIR}/html/unsupported/ WORKING_DIRECTORY ${Eigen_BINARY_DIR}/doc )
diff --git a/doc/CoeffwiseMathFunctionsTable.dox b/doc/CoeffwiseMathFunctionsTable.dox new file mode 100644 index 0000000..080e056 --- /dev/null +++ b/doc/CoeffwiseMathFunctionsTable.dox
@@ -0,0 +1,563 @@ +namespace Eigen { + +/** \eigenManualPage CoeffwiseMathFunctions Catalog of coefficient-wise math functions + + +<!-- <span style="font-size:300%; color:red; font-weight: 900;">!WORK IN PROGRESS!</span> --> + +This table presents a catalog of the coefficient-wise math functions supported by %Eigen. +In this table, \c a, \c b, refer to Array objects or expressions, and \c m refers to a linear algebra Matrix/Vector object. Standard scalar types are abbreviated as follows: + - \c int: \c i32 + - \c float: \c f + - \c double: \c d + - \c std::complex<float>: \c cf + - \c std::complex<double>: \c cd + +For each row, the first column list the equivalent calls for arrays, and matrices when supported. Of course, all functions are available for matrices by first casting it as an array: \c m.array(). + +The third column gives some hints in the underlying scalar implementation. In most cases, %Eigen does not implement itself the math function but relies on the STL for standard scalar types, or user-provided functions for custom scalar types. +For instance, some simply calls the respective function of the STL while preserving <a href="http://en.cppreference.com/w/cpp/language/adl">argument-dependent lookup</a> for custom types. +The following: +\code +using std::foo; +foo(a[i]); +\endcode +means that the STL's function \c std::foo will be potentially called if it is compatible with the underlying scalar type. If not, then the user must ensure that an overload of the function foo is available for the given scalar type (usually defined in the same namespace as the given scalar type). +This also means that, unless specified, if the function \c std::foo is available only in some recent c++ versions (e.g., c++11), then the respective %Eigen's function/method will be usable on standard types only if the compiler support the required c++ version. + +<table class="manual-hl"> +<tr> +<th>API</th><th>Description</th><th>Default scalar implementation</th><th>SIMD</th> +</tr> +<tr><td colspan="4"></td></tr> +<tr><th colspan="4">Basic operations</th></tr> +<tr> + <td class="code"> + \anchor cwisetable_abs + a.\link ArrayBase::abs abs\endlink(); \n + \link Eigen::abs abs\endlink(a); \n + m.\link MatrixBase::cwiseAbs cwiseAbs\endlink(); + </td> + <td>absolute value (\f$ |a_i| \f$) </td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/fabs">std::abs</a>; \n + abs(a[i]); + </td> + <td>SSE2, AVX (i32,f,d)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_inverse + a.\link ArrayBase::inverse inverse\endlink(); \n + \link Eigen::inverse inverse\endlink(a); \n + m.\link MatrixBase::cwiseInverse cwiseInverse\endlink(); + </td> + <td>inverse value (\f$ 1/a_i \f$) </td> + <td class="code"> + 1/a[i]; + </td> + <td>All engines (f,d,fc,fd)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_conj + a.\link ArrayBase::conjugate conjugate\endlink(); \n + \link Eigen::conj conj\endlink(a); \n + m.\link MatrixBase::conjugate conjugate\endlink(); + </td> + <td><a href="https://en.wikipedia.org/wiki/Complex_conjugate">complex conjugate</a> (\f$ \bar{a_i} \f$),\n + no-op for real </td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/complex/conj">std::conj</a>; \n + conj(a[i]); + </td> + <td>All engines (fc,fd)</td> +</tr> +<tr> +<th colspan="4">Exponential functions</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_exp + a.\link ArrayBase::exp exp\endlink(); \n + \link Eigen::exp exp\endlink(a); + </td> + <td>\f$ e \f$ raised to the given power (\f$ e^{a_i} \f$) </td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/exp">std::exp</a>; \n + exp(a[i]); + </td> + <td>SSE2, AVX (f,d)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_log + a.\link ArrayBase::log log\endlink(); \n + \link Eigen::log log\endlink(a); + </td> + <td>natural (base \f$ e \f$) logarithm (\f$ \ln({a_i}) \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/log">std::log</a>; \n + log(a[i]); + </td> + <td>SSE2, AVX (f)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_log1p + a.\link ArrayBase::log1p log1p\endlink(); \n + \link Eigen::log1p log1p\endlink(a); + </td> + <td>natural (base \f$ e \f$) logarithm of 1 plus \n the given number (\f$ \ln({1+a_i}) \f$)</td> + <td>built-in generic implementation based on \c log,\n + plus \c using <a href="http://en.cppreference.com/w/cpp/numeric/math/log1p">\c std::log1p </a>; \cpp11</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_log10 + a.\link ArrayBase::log10 log10\endlink(); \n + \link Eigen::log10 log10\endlink(a); + </td> + <td>base 10 logarithm (\f$ \log_{10}({a_i}) \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/log10">std::log10</a>; \n + log10(a[i]); + </td> + <td></td> +</tr> +<tr> +<th colspan="4">Power functions</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_pow + a.\link ArrayBase::pow pow\endlink(b); \n + \link ArrayBase::pow(const Eigen::ArrayBase< Derived > &x, const Eigen::ArrayBase< ExponentDerived > &exponents) pow\endlink(a,b); + </td> + <!-- For some reason Doxygen thinks that pow is in ArrayBase namespace --> + <td>raises a number to the given power (\f$ a_i ^ {b_i} \f$) \n \c a and \c b can be either an array or scalar.</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/pow">std::pow</a>; \n + pow(a[i],b[i]);\n + (plus builtin for integer types)</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_sqrt + a.\link ArrayBase::sqrt sqrt\endlink(); \n + \link Eigen::sqrt sqrt\endlink(a);\n + m.\link MatrixBase::cwiseSqrt cwiseSqrt\endlink(); + </td> + <td>computes square root (\f$ \sqrt a_i \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/sqrt">std::sqrt</a>; \n + sqrt(a[i]);</td> + <td>SSE2, AVX (f,d)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_rsqrt + a.\link ArrayBase::rsqrt rsqrt\endlink(); \n + \link Eigen::rsqrt rsqrt\endlink(a); + </td> + <td><a href="https://en.wikipedia.org/wiki/Fast_inverse_square_root">reciprocal square root</a> (\f$ 1/{\sqrt a_i} \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/sqrt">std::sqrt</a>; \n + 1/sqrt(a[i]); \n + </td> + <td>SSE2, AVX, AltiVec, ZVector (f,d)\n + (approx + 1 Newton iteration)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_square + a.\link ArrayBase::square square\endlink(); \n + \link Eigen::square square\endlink(a); + </td> + <td>computes square power (\f$ a_i^2 \f$)</td> + <td class="code"> + a[i]*a[i]</td> + <td>All (i32,f,d,cf,cd)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_cube + a.\link ArrayBase::cube cube\endlink(); \n + \link Eigen::cube cube\endlink(a); + </td> + <td>computes cubic power (\f$ a_i^3 \f$)</td> + <td class="code"> + a[i]*a[i]*a[i]</td> + <td>All (i32,f,d,cf,cd)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_abs2 + a.\link ArrayBase::abs2 abs2\endlink(); \n + \link Eigen::abs2 abs2\endlink(a);\n + m.\link MatrixBase::cwiseAbs2 cwiseAbs2\endlink(); + </td> + <td>computes the squared absolute value (\f$ |a_i|^2 \f$)</td> + <td class="code"> + real: a[i]*a[i] \n + complex: real(a[i])*real(a[i]) \n + + imag(a[i])*imag(a[i])</td> + <td>All (i32,f,d)</td> +</tr> +<tr> +<th colspan="4">Trigonometric functions</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_sin + a.\link ArrayBase::sin sin\endlink(); \n + \link Eigen::sin sin\endlink(a); + </td> + <td>computes sine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/sin">std::sin</a>; \n + sin(a[i]);</td> + <td>SSE2, AVX (f)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_cos + a.\link ArrayBase::cos cos\endlink(); \n + \link Eigen::cos cos\endlink(a); + </td> + <td>computes cosine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/cos">std::cos</a>; \n + cos(a[i]);</td> + <td>SSE2, AVX (f)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_tan + a.\link ArrayBase::tan tan\endlink(); \n + \link Eigen::tan tan\endlink(a); + </td> + <td>computes tangent</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/tan">std::tan</a>; \n + tan(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_asin + a.\link ArrayBase::asin asin\endlink(); \n + \link Eigen::asin asin\endlink(a); + </td> + <td>computes arc sine (\f$ \sin^{-1} a_i \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/asin">std::asin</a>; \n + asin(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_acos + a.\link ArrayBase::acos acos\endlink(); \n + \link Eigen::acos acos\endlink(a); + </td> + <td>computes arc cosine (\f$ \cos^{-1} a_i \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/acos">std::acos</a>; \n + acos(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_atan + a.\link ArrayBase::atan atan\endlink(); \n + \link Eigen::atan atan\endlink(a); + </td> + <td>computes arc tangent (\f$ \tan^{-1} a_i \f$)</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/atan">std::atan</a>; \n + atan(a[i]);</td> + <td></td> +</tr> +<tr> +<th colspan="4">Hyperbolic functions</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_sinh + a.\link ArrayBase::sinh sinh\endlink(); \n + \link Eigen::sinh sinh\endlink(a); + </td> + <td>computes hyperbolic sine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/sinh">std::sinh</a>; \n + sinh(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_cosh + a.\link ArrayBase::cosh cohs\endlink(); \n + \link Eigen::cosh cosh\endlink(a); + </td> + <td>computes hyperbolic cosine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/cosh">std::cosh</a>; \n + cosh(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_tanh + a.\link ArrayBase::tanh tanh\endlink(); \n + \link Eigen::tanh tanh\endlink(a); + </td> + <td>computes hyperbolic tangent</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/tanh">std::tanh</a>; \n + tanh(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_asinh + a.\link ArrayBase::asinh asinh\endlink(); \n + \link Eigen::asinh asinh\endlink(a); + </td> + <td>computes inverse hyperbolic sine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/asinh">std::asinh</a>; \n + asinh(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_acosh + a.\link ArrayBase::acosh cohs\endlink(); \n + \link Eigen::acosh acosh\endlink(a); + </td> + <td>computes hyperbolic cosine</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/acosh">std::acosh</a>; \n + acosh(a[i]);</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_atanh + a.\link ArrayBase::atanh atanh\endlink(); \n + \link Eigen::atanh atanh\endlink(a); + </td> + <td>computes hyperbolic tangent</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/atanh">std::atanh</a>; \n + atanh(a[i]);</td> + <td></td> +</tr> +<tr> +<th colspan="4">Nearest integer floating point operations</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_ceil + a.\link ArrayBase::ceil ceil\endlink(); \n + \link Eigen::ceil ceil\endlink(a); + </td> + <td>nearest integer not less than the given value</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/ceil">std::ceil</a>; \n + ceil(a[i]);</td> + <td>SSE4,AVX,ZVector (f,d)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_floor + a.\link ArrayBase::floor floor\endlink(); \n + \link Eigen::floor floor\endlink(a); + </td> + <td>nearest integer not greater than the given value</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/floor">std::floor</a>; \n + floor(a[i]);</td> + <td>SSE4,AVX,ZVector (f,d)</td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_round + a.\link ArrayBase::round round\endlink(); \n + \link Eigen::round round\endlink(a); + </td> + <td>nearest integer, \n rounding away from zero in halfway cases</td> + <td>built-in generic implementation \n based on \c floor and \c ceil,\n + plus \c using <a href="http://en.cppreference.com/w/cpp/numeric/math/round">\c std::round </a>; \cpp11</td> + <td>SSE4,AVX,ZVector (f,d)</td> +</tr> +<tr> +<th colspan="4">Floating point manipulation functions</th> +</tr> +<tr> +<th colspan="4">Classification and comparison</th> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_isfinite + a.\link ArrayBase::isFinite isFinite\endlink(); \n + \link Eigen::isfinite isfinite\endlink(a); + </td> + <td>checks if the given number has finite value</td> + <td>built-in generic implementation,\n + plus \c using <a href="http://en.cppreference.com/w/cpp/numeric/math/isfinite">\c std::isfinite </a>; \cpp11</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_isinf + a.\link ArrayBase::isInf isInf\endlink(); \n + \link Eigen::isinf isinf\endlink(a); + </td> + <td>checks if the given number is infinite</td> + <td>built-in generic implementation,\n + plus \c using <a href="http://en.cppreference.com/w/cpp/numeric/math/isinf">\c std::isinf </a>; \cpp11</td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_isnan + a.\link ArrayBase::isNaN isNaN\endlink(); \n + \link Eigen::isnan isnan\endlink(a); + </td> + <td>checks if the given number is not a number</td> + <td>built-in generic implementation,\n + plus \c using <a href="http://en.cppreference.com/w/cpp/numeric/math/isnan">\c std::isnan </a>; \cpp11</td> + <td></td> +</tr> +<tr> +<th colspan="4">Error and gamma functions</th> +</tr> +<tr> <td colspan="4"> Require \c \#include \c <unsupported/Eigen/SpecialFunctions> </td></tr> +<tr> + <td class="code"> + \anchor cwisetable_erf + a.\link ArrayBase::erf erf\endlink(); \n + \link Eigen::erf erf\endlink(a); + </td> + <td>error function</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/erf">std::erf</a>; \cpp11 \n + erf(a[i]); + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_erfc + a.\link ArrayBase::erfc erfc\endlink(); \n + \link Eigen::erfc erfc\endlink(a); + </td> + <td>complementary error function</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/erfc">std::erfc</a>; \cpp11 \n + erfc(a[i]); + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_lgamma + a.\link ArrayBase::lgamma lgamma\endlink(); \n + \link Eigen::lgamma lgamma\endlink(a); + </td> + <td>natural logarithm of the gamma function</td> + <td class="code"> + using <a href="http://en.cppreference.com/w/cpp/numeric/math/lgamma">std::lgamma</a>; \cpp11 \n + lgamma(a[i]); + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_digamma + a.\link ArrayBase::digamma digamma\endlink(); \n + \link Eigen::digamma digamma\endlink(a); + </td> + <td><a href="https://en.wikipedia.org/wiki/Digamma_function">logarithmic derivative of the gamma function</a></td> + <td> + built-in for float and double + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_igamma + \link Eigen::igamma igamma\endlink(a,x); + </td> + <td><a href="https://en.wikipedia.org/wiki/Incomplete_gamma_function">lower incomplete gamma integral</a> + \n \f$ \gamma(a_i,x_i)= \frac{1}{|a_i|} \int_{0}^{x_i}e^{\text{-}t} t^{a_i-1} \mathrm{d} t \f$</td> + <td> + built-in for float and double,\n but requires \cpp11 + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_igammac + \link Eigen::igammac igammac\endlink(a,x); + </td> + <td><a href="https://en.wikipedia.org/wiki/Incomplete_gamma_function">upper incomplete gamma integral</a> + \n \f$ \Gamma(a_i,x_i) = \frac{1}{|a_i|} \int_{x_i}^{\infty}e^{\text{-}t} t^{a_i-1} \mathrm{d} t \f$</td> + <td> + built-in for float and double,\n but requires \cpp11 + </td> + <td></td> +</tr> +<tr> +<th colspan="4">Special functions</th> +</tr> +<tr> <td colspan="4"> Require \c \#include \c <unsupported/Eigen/SpecialFunctions> </td></tr> +<tr> + <td class="code"> + \anchor cwisetable_polygamma + \link Eigen::polygamma polygamma\endlink(n,x); + </td> + <td><a href="https://en.wikipedia.org/wiki/Polygamma_function">n-th derivative of digamma at x</a></td> + <td> + built-in generic based on\n <a href="#cwisetable_lgamma">\c lgamma </a>, + <a href="#cwisetable_digamma"> \c digamma </a> + and <a href="#cwisetable_zeta">\c zeta </a>. + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_betainc + \link Eigen::betainc betainc\endlink(a,b,x); + </td> + <td><a href="https://en.wikipedia.org/wiki/Beta_function#Incomplete_beta_function">Incomplete beta function</a></td> + <td> + built-in for float and double,\n but requires \cpp11 + </td> + <td></td> +</tr> +<tr> + <td class="code"> + \anchor cwisetable_zeta + \link Eigen::zeta zeta\endlink(a,b); \n + a.\link ArrayBase::zeta zeta\endlink(b); + </td> + <td><a href="https://en.wikipedia.org/wiki/Hurwitz_zeta_function">Hurwitz zeta function</a> + \n \f$ \zeta(a_i,b_i)=\sum_{k=0}^{\infty}(b_i+k)^{\text{-}a_i} \f$</td> + <td> + built-in for float and double + </td> + <td></td> +</tr> +<tr><td colspan="4"></td></tr> +</table> + +\n + +*/ + +}
diff --git a/doc/CustomizingEigen.dox b/doc/CustomizingEigen.dox deleted file mode 100644 index cb25f4e..0000000 --- a/doc/CustomizingEigen.dox +++ /dev/null
@@ -1,226 +0,0 @@ -namespace Eigen { - -/** \page TopicCustomizingEigen Customizing/Extending Eigen - -Eigen can be extended in several ways, for instance, by defining global methods, \ref ExtendingMatrixBase "by adding custom methods to MatrixBase", adding support to \ref CustomScalarType "custom types" etc. - -\eigenAutoToc - -\section ExtendingMatrixBase Extending MatrixBase (and other classes) - -In this section we will see how to add custom methods to MatrixBase. Since all expressions and matrix types inherit MatrixBase, adding a method to MatrixBase make it immediately available to all expressions ! A typical use case is, for instance, to make Eigen compatible with another API. - -You certainly know that in C++ it is not possible to add methods to an existing class. So how that's possible ? Here the trick is to include in the declaration of MatrixBase a file defined by the preprocessor token \c EIGEN_MATRIXBASE_PLUGIN: -\code -class MatrixBase { - // ... - #ifdef EIGEN_MATRIXBASE_PLUGIN - #include EIGEN_MATRIXBASE_PLUGIN - #endif -}; -\endcode -Therefore to extend MatrixBase with your own methods you just have to create a file with your method declaration and define EIGEN_MATRIXBASE_PLUGIN before you include any Eigen's header file. - -You can extend many of the other classes used in Eigen by defining similarly named preprocessor symbols. For instance, define \c EIGEN_ARRAYBASE_PLUGIN if you want to extend the ArrayBase class. A full list of classes that can be extended in this way and the corresponding preprocessor symbols can be found on our page \ref TopicPreprocessorDirectives. - -Here is an example of an extension file for adding methods to MatrixBase: \n -\b MatrixBaseAddons.h -\code -inline Scalar at(uint i, uint j) const { return this->operator()(i,j); } -inline Scalar& at(uint i, uint j) { return this->operator()(i,j); } -inline Scalar at(uint i) const { return this->operator[](i); } -inline Scalar& at(uint i) { return this->operator[](i); } - -inline RealScalar squaredLength() const { return squaredNorm(); } -inline RealScalar length() const { return norm(); } -inline RealScalar invLength(void) const { return fast_inv_sqrt(squaredNorm()); } - -template<typename OtherDerived> -inline Scalar squaredDistanceTo(const MatrixBase<OtherDerived>& other) const -{ return (derived() - other.derived()).squaredNorm(); } - -template<typename OtherDerived> -inline RealScalar distanceTo(const MatrixBase<OtherDerived>& other) const -{ return internal::sqrt(derived().squaredDistanceTo(other)); } - -inline void scaleTo(RealScalar l) { RealScalar vl = norm(); if (vl>1e-9) derived() *= (l/vl); } - -inline Transpose<Derived> transposed() {return this->transpose();} -inline const Transpose<Derived> transposed() const {return this->transpose();} - -inline uint minComponentId(void) const { int i; this->minCoeff(&i); return i; } -inline uint maxComponentId(void) const { int i; this->maxCoeff(&i); return i; } - -template<typename OtherDerived> -void makeFloor(const MatrixBase<OtherDerived>& other) { derived() = derived().cwiseMin(other.derived()); } -template<typename OtherDerived> -void makeCeil(const MatrixBase<OtherDerived>& other) { derived() = derived().cwiseMax(other.derived()); } - -const CwiseUnaryOp<internal::scalar_add_op<Scalar>, Derived> -operator+(const Scalar& scalar) const -{ return CwiseUnaryOp<internal::scalar_add_op<Scalar>, Derived>(derived(), internal::scalar_add_op<Scalar>(scalar)); } - -friend const CwiseUnaryOp<internal::scalar_add_op<Scalar>, Derived> -operator+(const Scalar& scalar, const MatrixBase<Derived>& mat) -{ return CwiseUnaryOp<internal::scalar_add_op<Scalar>, Derived>(mat.derived(), internal::scalar_add_op<Scalar>(scalar)); } -\endcode - -Then one can the following declaration in the config.h or whatever prerequisites header file of his project: -\code -#define EIGEN_MATRIXBASE_PLUGIN "MatrixBaseAddons.h" -\endcode - -\section InheritingFromMatrix Inheriting from Matrix - -Before inheriting from Matrix, be really, I mean REALLY, sure that using -EIGEN_MATRIX_PLUGIN is not what you really want (see previous section). -If you just need to add few members to Matrix, this is the way to go. - -An example of when you actually need to inherit Matrix, is when you -have several layers of heritage such as -MyVerySpecificVector1, MyVerySpecificVector2 -> MyVector1 -> Matrix and -MyVerySpecificVector3, MyVerySpecificVector4 -> MyVector2 -> Matrix. - -In order for your object to work within the %Eigen framework, you need to -define a few members in your inherited class. - -Here is a minimalistic example: - -\include CustomizingEigen_Inheritance.cpp - -Output: \verbinclude CustomizingEigen_Inheritance.out - -This is the kind of error you can get if you don't provide those methods -\verbatim -error: no match for ‘operator=’ in ‘v = Eigen::operator*( -const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1, 0, -0x000000001, 1> >::Scalar&, -const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType&) -(((const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType&) -((const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType*)(& v))))’ -\endverbatim - -\anchor user_defined_scalars \section CustomScalarType Using custom scalar types - -By default, Eigen currently supports standard floating-point types (\c float, \c double, \c std::complex<float>, \c std::complex<double>, \c long \c double), as well as all native integer types (e.g., \c int, \c unsigned \c int, \c short, etc.), and \c bool. -On x86-64 systems, \c long \c double permits to locally enforces the use of x87 registers with extended accuracy (in comparison to SSE). - -In order to add support for a custom type \c T you need: --# make sure the common operator (+,-,*,/,etc.) are supported by the type \c T --# add a specialization of struct Eigen::NumTraits<T> (see \ref NumTraits) --# define the math functions that makes sense for your type. This includes standard ones like sqrt, pow, sin, tan, conj, real, imag, etc, as well as abs2 which is Eigen specific. - (see the file Eigen/src/Core/MathFunctions.h) - -The math function should be defined in the same namespace than \c T, or in the \c std namespace though that second approach is not recommended. - -Here is a concrete example adding support for the Adolc's \c adouble type. <a href="https://projects.coin-or.org/ADOL-C">Adolc</a> is an automatic differentiation library. The type \c adouble is basically a real value tracking the values of any number of partial derivatives. - -\code -#ifndef ADOLCSUPPORT_H -#define ADOLCSUPPORT_H - -#define ADOLC_TAPELESS -#include <adolc/adouble.h> -#include <Eigen/Core> - -namespace Eigen { - -template<> struct NumTraits<adtl::adouble> - : NumTraits<double> // permits to get the epsilon, dummy_precision, lowest, highest functions -{ - typedef adtl::adouble Real; - typedef adtl::adouble NonInteger; - typedef adtl::adouble Nested; - - enum { - IsComplex = 0, - IsInteger = 0, - IsSigned = 1, - RequireInitialization = 1, - ReadCost = 1, - AddCost = 3, - MulCost = 3 - }; -}; - -} - -namespace adtl { - -inline const adouble& conj(const adouble& x) { return x; } -inline const adouble& real(const adouble& x) { return x; } -inline adouble imag(const adouble&) { return 0.; } -inline adouble abs(const adouble& x) { return fabs(x); } -inline adouble abs2(const adouble& x) { return x*x; } - -} - -#endif // ADOLCSUPPORT_H -\endcode - -This other example adds support for the \c mpq_class type from <a href="https://gmplib.org/">GMP</a>. It shows in particular how to change the way Eigen picks the best pivot during LU factorization. It selects the coefficient with the highest score, where the score is by default the absolute value of a number, but we can define a different score, for instance to prefer pivots with a more compact representation (this is an example, not a recommendation). Note that the scores should always be non-negative and only zero is allowed to have a score of zero. Also, this can interact badly with thresholds for inexact scalar types. - -\code -#include <gmpxx.h> -#include <Eigen/Core> -#include <boost/operators.hpp> - -namespace Eigen { - template<class> struct NumTraits; - template<> struct NumTraits<mpq_class> - { - typedef mpq_class Real; - typedef mpq_class NonInteger; - typedef mpq_class Nested; - - static inline Real epsilon() { return 0; } - static inline Real dummy_precision() { return 0; } - - enum { - IsInteger = 0, - IsSigned = 1, - IsComplex = 0, - RequireInitialization = 1, - ReadCost = 6, - AddCost = 150, - MulCost = 100 - }; - }; - - namespace internal { - template<> - struct significant_decimals_impl<mpq_class> - { - // Infinite precision when printing - static inline int run() { return 0; } - }; - - template<> struct scalar_score_coeff_op<mpq_class> { - struct result_type : boost::totally_ordered1<result_type> { - std::size_t len; - result_type(int i = 0) : len(i) {} // Eigen uses Score(0) and Score() - result_type(mpq_class const& q) : - len(mpz_size(q.get_num_mpz_t())+ - mpz_size(q.get_den_mpz_t())-1) {} - friend bool operator<(result_type x, result_type y) { - // 0 is the worst possible pivot - if (x.len == 0) return y.len > 0; - if (y.len == 0) return false; - // Prefer a pivot with a small representation - return x.len > y.len; - } - friend bool operator==(result_type x, result_type y) { - // Only used to test if the score is 0 - return x.len == y.len; - } - }; - result_type operator()(mpq_class const& x) const { return x; } - }; - } -} -\endcode - -\sa \ref TopicPreprocessorDirectives - -*/ - -}
diff --git a/doc/CustomizingEigen_CustomScalar.dox b/doc/CustomizingEigen_CustomScalar.dox new file mode 100644 index 0000000..1ee78cb --- /dev/null +++ b/doc/CustomizingEigen_CustomScalar.dox
@@ -0,0 +1,120 @@ +namespace Eigen { + +/** \page TopicCustomizing_CustomScalar Using custom scalar types +\anchor user_defined_scalars + +By default, Eigen currently supports standard floating-point types (\c float, \c double, \c std::complex<float>, \c std::complex<double>, \c long \c double), as well as all native integer types (e.g., \c int, \c unsigned \c int, \c short, etc.), and \c bool. +On x86-64 systems, \c long \c double permits to locally enforces the use of x87 registers with extended accuracy (in comparison to SSE). + +In order to add support for a custom type \c T you need: +-# make sure the common operator (+,-,*,/,etc.) are supported by the type \c T +-# add a specialization of struct Eigen::NumTraits<T> (see \ref NumTraits) +-# define the math functions that makes sense for your type. This includes standard ones like sqrt, pow, sin, tan, conj, real, imag, etc, as well as abs2 which is Eigen specific. + (see the file Eigen/src/Core/MathFunctions.h) + +The math function should be defined in the same namespace than \c T, or in the \c std namespace though that second approach is not recommended. + +Here is a concrete example adding support for the Adolc's \c adouble type. <a href="https://projects.coin-or.org/ADOL-C">Adolc</a> is an automatic differentiation library. The type \c adouble is basically a real value tracking the values of any number of partial derivatives. + +\code +#ifndef ADOLCSUPPORT_H +#define ADOLCSUPPORT_H + +#define ADOLC_TAPELESS +#include <adolc/adouble.h> +#include <Eigen/Core> + +namespace Eigen { + +template<> struct NumTraits<adtl::adouble> + : NumTraits<double> // permits to get the epsilon, dummy_precision, lowest, highest functions +{ + typedef adtl::adouble Real; + typedef adtl::adouble NonInteger; + typedef adtl::adouble Nested; + + enum { + IsComplex = 0, + IsInteger = 0, + IsSigned = 1, + RequireInitialization = 1, + ReadCost = 1, + AddCost = 3, + MulCost = 3 + }; +}; + +} + +namespace adtl { + +inline const adouble& conj(const adouble& x) { return x; } +inline const adouble& real(const adouble& x) { return x; } +inline adouble imag(const adouble&) { return 0.; } +inline adouble abs(const adouble& x) { return fabs(x); } +inline adouble abs2(const adouble& x) { return x*x; } + +} + +#endif // ADOLCSUPPORT_H +\endcode + +This other example adds support for the \c mpq_class type from <a href="https://gmplib.org/">GMP</a>. It shows in particular how to change the way Eigen picks the best pivot during LU factorization. It selects the coefficient with the highest score, where the score is by default the absolute value of a number, but we can define a different score, for instance to prefer pivots with a more compact representation (this is an example, not a recommendation). Note that the scores should always be non-negative and only zero is allowed to have a score of zero. Also, this can interact badly with thresholds for inexact scalar types. + +\code +#include <gmpxx.h> +#include <Eigen/Core> +#include <boost/operators.hpp> + +namespace Eigen { + template<> struct NumTraits<mpq_class> : GenericNumTraits<mpq_class> + { + typedef mpq_class Real; + typedef mpq_class NonInteger; + typedef mpq_class Nested; + + static inline Real epsilon() { return 0; } + static inline Real dummy_precision() { return 0; } + static inline Real digits10() { return 0; } + + enum { + IsInteger = 0, + IsSigned = 1, + IsComplex = 0, + RequireInitialization = 1, + ReadCost = 6, + AddCost = 150, + MulCost = 100 + }; + }; + + namespace internal { + + template<> struct scalar_score_coeff_op<mpq_class> { + struct result_type : boost::totally_ordered1<result_type> { + std::size_t len; + result_type(int i = 0) : len(i) {} // Eigen uses Score(0) and Score() + result_type(mpq_class const& q) : + len(mpz_size(q.get_num_mpz_t())+ + mpz_size(q.get_den_mpz_t())-1) {} + friend bool operator<(result_type x, result_type y) { + // 0 is the worst possible pivot + if (x.len == 0) return y.len > 0; + if (y.len == 0) return false; + // Prefer a pivot with a small representation + return x.len > y.len; + } + friend bool operator==(result_type x, result_type y) { + // Only used to test if the score is 0 + return x.len == y.len; + } + }; + result_type operator()(mpq_class const& x) const { return x; } + }; + } +} +\endcode + +*/ + +}
diff --git a/doc/CustomizingEigen_InheritingMatrix.dox b/doc/CustomizingEigen_InheritingMatrix.dox new file mode 100644 index 0000000..b21e554 --- /dev/null +++ b/doc/CustomizingEigen_InheritingMatrix.dox
@@ -0,0 +1,34 @@ +namespace Eigen { + +/** \page TopicCustomizing_InheritingMatrix Inheriting from Matrix + +Before inheriting from Matrix, be really, I mean REALLY, sure that using +EIGEN_MATRIX_PLUGIN is not what you really want (see previous section). +If you just need to add few members to Matrix, this is the way to go. + +An example of when you actually need to inherit Matrix, is when you +have several layers of heritage such as +MyVerySpecificVector1, MyVerySpecificVector2 -> MyVector1 -> Matrix and +MyVerySpecificVector3, MyVerySpecificVector4 -> MyVector2 -> Matrix. + +In order for your object to work within the %Eigen framework, you need to +define a few members in your inherited class. + +Here is a minimalistic example: + +\include CustomizingEigen_Inheritance.cpp + +Output: \verbinclude CustomizingEigen_Inheritance.out + +This is the kind of error you can get if you don't provide those methods +\verbatim +error: no match for ‘operator=’ in ‘v = Eigen::operator*( +const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1, 0, -0x000000001, 1> >::Scalar&, +const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType&) +(((const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType&) +((const Eigen::MatrixBase<Eigen::Matrix<double, -0x000000001, 1> >::StorageBaseType*)(& v))))’ +\endverbatim + +*/ + +}
diff --git a/doc/CustomizingEigen_NullaryExpr.dox b/doc/CustomizingEigen_NullaryExpr.dox new file mode 100644 index 0000000..37c8dcd --- /dev/null +++ b/doc/CustomizingEigen_NullaryExpr.dox
@@ -0,0 +1,86 @@ +namespace Eigen { + +/** \page TopicCustomizing_NullaryExpr Matrix manipulation via nullary-expressions + + +The main purpose of the class CwiseNullaryOp is to define \em procedural matrices such as constant or random matrices as returned by the Ones(), Zero(), Constant(), Identity() and Random() methods. +Nevertheless, with some imagination it is possible to accomplish very sophisticated matrix manipulation with minimal efforts such that \ref TopicNewExpressionType "implementing new expression" is rarely needed. + +\section NullaryExpr_Circulant Example 1: circulant matrix + +To explore these possibilities let us start with the \em circulant example of the \ref TopicNewExpressionType "implementing new expression" topic. +Let us recall that a circulant matrix is a matrix where each column is the same as the +column to the left, except that it is cyclically shifted downwards. +For example, here is a 4-by-4 circulant matrix: +\f[ \begin{bmatrix} + 1 & 8 & 4 & 2 \\ + 2 & 1 & 8 & 4 \\ + 4 & 2 & 1 & 8 \\ + 8 & 4 & 2 & 1 +\end{bmatrix} \f] +A circulant matrix is uniquely determined by its first column. We wish +to write a function \c makeCirculant which, given the first column, +returns an expression representing the circulant matrix. + +For this exercise, the return type of \c makeCirculant will be a CwiseNullaryOp that we need to instantiate with: +1 - a proper \c circulant_functor storing the input vector and implementing the adequate coefficient accessor \c operator(i,j) +2 - a template instantiation of class Matrix conveying compile-time information such as the scalar type, sizes, and preferred storage layout. + +Calling \c ArgType the type of the input vector, we can construct the equivalent squared Matrix type as follows: + +\snippet make_circulant2.cpp square + +This little helper structure will help us to implement our \c makeCirculant function as follows: + +\snippet make_circulant2.cpp makeCirculant + +As usual, our function takes as argument a \c MatrixBase (see this \ref TopicFunctionTakingEigenTypes "page" for more details). +Then, the CwiseNullaryOp object is constructed through the DenseBase::NullaryExpr static method with the adequate runtime sizes. + +Then, we need to implement our \c circulant_functor, which is a straightforward exercise: + +\snippet make_circulant2.cpp circulant_func + +We are now all set to try our new feature: + +\snippet make_circulant2.cpp main + + +If all the fragments are combined, the following output is produced, +showing that the program works as expected: + +\include make_circulant2.out + +This implementation of \c makeCirculant is much simpler than \ref TopicNewExpressionType "defining a new expression" from scratch. + + +\section NullaryExpr_Indexing Example 2: indexing rows and columns + +The goal here is to mimic MatLab's ability to index a matrix through two vectors of indices referencing the rows and columns to be picked respectively, like this: + +\snippet nullary_indexing.out main1 + +To this end, let us first write a nullary-functor storing references to the input matrix and to the two arrays of indices, and implementing the required \c operator()(i,j): + +\snippet nullary_indexing.cpp functor + +Then, let's create an \c indexing(A,rows,cols) function creating the nullary expression: + +\snippet nullary_indexing.cpp function + +Finally, here is an example of how this function can be used: + +\snippet nullary_indexing.cpp main1 + +This straightforward implementation is already quite powerful as the row or column index arrays can also be expressions to perform offsetting, modulo, striding, reverse, etc. + +\snippet nullary_indexing.cpp main2 + +and the output is: + +\snippet nullary_indexing.out main2 + +*/ + +} +
diff --git a/doc/CustomizingEigen_Plugins.dox b/doc/CustomizingEigen_Plugins.dox new file mode 100644 index 0000000..d88f240 --- /dev/null +++ b/doc/CustomizingEigen_Plugins.dox
@@ -0,0 +1,69 @@ +namespace Eigen { + +/** \page TopicCustomizing_Plugins Extending MatrixBase (and other classes) + +In this section we will see how to add custom methods to MatrixBase. Since all expressions and matrix types inherit MatrixBase, adding a method to MatrixBase make it immediately available to all expressions ! A typical use case is, for instance, to make Eigen compatible with another API. + +You certainly know that in C++ it is not possible to add methods to an existing class. So how that's possible ? Here the trick is to include in the declaration of MatrixBase a file defined by the preprocessor token \c EIGEN_MATRIXBASE_PLUGIN: +\code +class MatrixBase { + // ... + #ifdef EIGEN_MATRIXBASE_PLUGIN + #include EIGEN_MATRIXBASE_PLUGIN + #endif +}; +\endcode +Therefore to extend MatrixBase with your own methods you just have to create a file with your method declaration and define EIGEN_MATRIXBASE_PLUGIN before you include any Eigen's header file. + +You can extend many of the other classes used in Eigen by defining similarly named preprocessor symbols. For instance, define \c EIGEN_ARRAYBASE_PLUGIN if you want to extend the ArrayBase class. A full list of classes that can be extended in this way and the corresponding preprocessor symbols can be found on our page \ref TopicPreprocessorDirectives. + +Here is an example of an extension file for adding methods to MatrixBase: \n +\b MatrixBaseAddons.h +\code +inline Scalar at(uint i, uint j) const { return this->operator()(i,j); } +inline Scalar& at(uint i, uint j) { return this->operator()(i,j); } +inline Scalar at(uint i) const { return this->operator[](i); } +inline Scalar& at(uint i) { return this->operator[](i); } + +inline RealScalar squaredLength() const { return squaredNorm(); } +inline RealScalar length() const { return norm(); } +inline RealScalar invLength(void) const { return fast_inv_sqrt(squaredNorm()); } + +template<typename OtherDerived> +inline Scalar squaredDistanceTo(const MatrixBase<OtherDerived>& other) const +{ return (derived() - other.derived()).squaredNorm(); } + +template<typename OtherDerived> +inline RealScalar distanceTo(const MatrixBase<OtherDerived>& other) const +{ return internal::sqrt(derived().squaredDistanceTo(other)); } + +inline void scaleTo(RealScalar l) { RealScalar vl = norm(); if (vl>1e-9) derived() *= (l/vl); } + +inline Transpose<Derived> transposed() {return this->transpose();} +inline const Transpose<Derived> transposed() const {return this->transpose();} + +inline uint minComponentId(void) const { int i; this->minCoeff(&i); return i; } +inline uint maxComponentId(void) const { int i; this->maxCoeff(&i); return i; } + +template<typename OtherDerived> +void makeFloor(const MatrixBase<OtherDerived>& other) { derived() = derived().cwiseMin(other.derived()); } +template<typename OtherDerived> +void makeCeil(const MatrixBase<OtherDerived>& other) { derived() = derived().cwiseMax(other.derived()); } + +const CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const Derived, const ConstantReturnType> +operator+(const Scalar& scalar) const +{ return CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const Derived, const ConstantReturnType>(derived(), Constant(rows(),cols(),scalar)); } + +friend const CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const ConstantReturnType, Derived> +operator+(const Scalar& scalar, const MatrixBase<Derived>& mat) +{ return CwiseBinaryOp<internal::scalar_sum_op<Scalar>, const ConstantReturnType, Derived>(Constant(rows(),cols(),scalar), mat.derived()); } +\endcode + +Then one can the following declaration in the config.h or whatever prerequisites header file of his project: +\code +#define EIGEN_MATRIXBASE_PLUGIN "MatrixBaseAddons.h" +\endcode + +*/ + +}
diff --git a/doc/DenseDecompositionBenchmark.dox b/doc/DenseDecompositionBenchmark.dox new file mode 100644 index 0000000..7be9c70 --- /dev/null +++ b/doc/DenseDecompositionBenchmark.dox
@@ -0,0 +1,42 @@ +namespace Eigen { + +/** \eigenManualPage DenseDecompositionBenchmark Benchmark of dense decompositions + +This page presents a speed comparison of the dense matrix decompositions offered by %Eigen for a wide range of square matrices and overconstrained problems. + +For a more general overview on the features and numerical robustness of linear solvers and decompositions, check this \link TopicLinearAlgebraDecompositions table \endlink. + +This benchmark has been run on a laptop equipped with an Intel core i7 \@ 2,6 GHz, and compiled with clang with \b AVX and \b FMA instruction sets enabled but without multi-threading. +It uses \b single \b precision \b float numbers. For double, you can get a good estimate by multiplying the timings by a factor 2. + +The square matrices are symmetric, and for the overconstrained matrices, the reported timmings include the cost to compute the symmetric covariance matrix \f$ A^T A \f$ for the first four solvers based on Cholesky and LU, as denoted by the \b * symbol (top-right corner part of the table). +Timings are in \b milliseconds, and factors are relative to the LLT decomposition which is the fastest but also the least general and robust. + +<table class="manual"> +<tr><th>solver/size</th> + <th>8x8</th> <th>100x100</th> <th>1000x1000</th> <th>4000x4000</th> <th>10000x8</th> <th>10000x100</th> <th>10000x1000</th> <th>10000x4000</th></tr> +<tr><td>LLT</td><td>0.05</td><td>0.42</td><td>5.83</td><td>374.55</td><td>6.79 <sup><a href="#note_ls">*</a></sup></td><td>30.15 <sup><a href="#note_ls">*</a></sup></td><td>236.34 <sup><a href="#note_ls">*</a></sup></td><td>3847.17 <sup><a href="#note_ls">*</a></sup></td></tr> +<tr class="alt"><td>LDLT</td><td>0.07 (x1.3)</td><td>0.65 (x1.5)</td><td>26.86 (x4.6)</td><td>2361.18 (x6.3)</td><td>6.81 (x1) <sup><a href="#note_ls">*</a></sup></td><td>31.91 (x1.1) <sup><a href="#note_ls">*</a></sup></td><td>252.61 (x1.1) <sup><a href="#note_ls">*</a></sup></td><td>5807.66 (x1.5) <sup><a href="#note_ls">*</a></sup></td></tr> +<tr><td>PartialPivLU</td><td>0.08 (x1.5)</td><td>0.69 (x1.6)</td><td>15.63 (x2.7)</td><td>709.32 (x1.9)</td><td>6.81 (x1) <sup><a href="#note_ls">*</a></sup></td><td>31.32 (x1) <sup><a href="#note_ls">*</a></sup></td><td>241.68 (x1) <sup><a href="#note_ls">*</a></sup></td><td>4270.48 (x1.1) <sup><a href="#note_ls">*</a></sup></td></tr> +<tr class="alt"><td>FullPivLU</td><td>0.1 (x1.9)</td><td>4.48 (x10.6)</td><td>281.33 (x48.2)</td><td>-</td><td>6.83 (x1) <sup><a href="#note_ls">*</a></sup></td><td>32.67 (x1.1) <sup><a href="#note_ls">*</a></sup></td><td>498.25 (x2.1) <sup><a href="#note_ls">*</a></sup></td><td>-</td></tr> +<tr><td>HouseholderQR</td><td>0.19 (x3.5)</td><td>2.18 (x5.2)</td><td>23.42 (x4)</td><td>1337.52 (x3.6)</td><td>34.26 (x5)</td><td>129.01 (x4.3)</td><td>377.37 (x1.6)</td><td>4839.1 (x1.3)</td></tr> +<tr class="alt"><td>ColPivHouseholderQR</td><td>0.23 (x4.3)</td><td>2.23 (x5.3)</td><td>103.34 (x17.7)</td><td>9987.16 (x26.7)</td><td>36.05 (x5.3)</td><td>163.18 (x5.4)</td><td>2354.08 (x10)</td><td>37860.5 (x9.8)</td></tr> +<tr><td>CompleteOrthogonalDecomposition</td><td>0.23 (x4.3)</td><td>2.22 (x5.2)</td><td>99.44 (x17.1)</td><td>10555.3 (x28.2)</td><td>35.75 (x5.3)</td><td>169.39 (x5.6)</td><td>2150.56 (x9.1)</td><td>36981.8 (x9.6)</td></tr> +<tr class="alt"><td>FullPivHouseholderQR</td><td>0.23 (x4.3)</td><td>4.64 (x11)</td><td>289.1 (x49.6)</td><td>-</td><td>69.38 (x10.2)</td><td>446.73 (x14.8)</td><td>4852.12 (x20.5)</td><td>-</td></tr> +<tr><td>JacobiSVD</td><td>1.01 (x18.6)</td><td>71.43 (x168.4)</td><td>-</td><td>-</td><td>113.81 (x16.7)</td><td>1179.66 (x39.1)</td><td>-</td><td>-</td></tr> +<tr class="alt"><td>BDCSVD</td><td>1.07 (x19.7)</td><td>21.83 (x51.5)</td><td>331.77 (x56.9)</td><td>18587.9 (x49.6)</td><td>110.53 (x16.3)</td><td>397.67 (x13.2)</td><td>2975 (x12.6)</td><td>48593.2 (x12.6)</td></tr> +</table> + +<a name="note_ls">\b *: </a> This decomposition do not support direct least-square solving for over-constrained problems, and the reported timing include the cost to form the symmetric covariance matrix \f$ A^T A \f$. + +\b Observations: + + LLT is always the fastest solvers. + + For largely over-constrained problems, the cost of Cholesky/LU decompositions is dominated by the computation of the symmetric covariance matrix. + + For large problem sizes, only the decomposition implementing a cache-friendly blocking strategy scale well. Those include LLT, PartialPivLU, HouseholderQR, and BDCSVD. This explain why for a 4k x 4k matrix, HouseholderQR is faster than LDLT. In the future, LDLT and ColPivHouseholderQR will also implement blocking strategies. + + CompleteOrthogonalDecomposition is based on ColPivHouseholderQR and they thus achieve the same level of performance. + +The above table has been generated by the <a href="https://bitbucket.org/eigen/eigen/raw/default/bench/dense_solvers.cpp">bench/dense_solvers.cpp</a> file, feel-free to hack it to generate a table matching your hardware, compiler, and favorite problem sizes. + +*/ + +}
diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 0a43c7c..5671986 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in
@@ -125,7 +125,7 @@ # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. -INLINE_INHERITED_MEMB = YES +INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set @@ -216,6 +216,7 @@ "lu_module=This is defined in the %LU module. \code #include <Eigen/LU> \endcode" \ "qr_module=This is defined in the %QR module. \code #include <Eigen/QR> \endcode" \ "svd_module=This is defined in the %SVD module. \code #include <Eigen/SVD> \endcode" \ + "specialfunctions_module=This is defined in the \b unsupported SpecialFunctions module. \code #include <Eigen/SpecialFunctions> \endcode" \ "label=\bug" \ "matrixworld=<a href='#matrixonly' style='color:green;text-decoration: none;'>*</a>" \ "arrayworld=<a href='#arrayonly' style='color:blue;text-decoration: none;'>*</a>" \ @@ -225,7 +226,11 @@ "note_try_to_help_rvo=This function returns the result by value. In order to make that efficient, it is implemented as just a return statement using a special constructor, hopefully allowing the compiler to perform a RVO (return value optimization)." \ "nonstableyet=\warning This is not considered to be part of the stable public API yet. Changes may happen in future releases. See \ref Experimental \"Experimental parts of Eigen\"" \ "implsparsesolverconcept=This class follows the \link TutorialSparseSolverConcept sparse solver concept \endlink." \ - "blank= " + "blank= " \ + "cpp11=<span class='cpp11'>[c++11]</span>" \ + "cpp14=<span class='cpp14'>[c++14]</span>" \ + "cpp17=<span class='cpp17'>[c++17]</span>" \ + "newin{1}=<span class='newin3x'>New in %Eigen \1.</span>" ALIASES += "eigenAutoToc= " @@ -405,7 +410,7 @@ # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. -EXTRACT_STATIC = NO +EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. @@ -723,7 +728,8 @@ # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = "${Eigen_SOURCE_DIR}/Eigen/Eigen2Support" \ +EXCLUDE = "${Eigen_SOURCE_DIR}/Eigen/src/Core/products" \ + "${Eigen_SOURCE_DIR}/Eigen/Eigen2Support" \ "${Eigen_SOURCE_DIR}/Eigen/src/Eigen2Support" \ "${Eigen_SOURCE_DIR}/doc/examples" \ "${Eigen_SOURCE_DIR}/doc/special_examples" \ @@ -1586,8 +1592,16 @@ EIGEN_QT_SUPPORT \ EIGEN_STRONG_INLINE=inline \ EIGEN_DEVICE_FUNC= \ + EIGEN_HAS_CXX11=1 \ + EIGEN_HAS_CXX11_MATH=1 \ "EIGEN_MAKE_CWISE_BINARY_OP(METHOD,FUNCTOR)=template<typename OtherDerived> const CwiseBinaryOp<FUNCTOR<Scalar>, const Derived, const OtherDerived> METHOD(const EIGEN_CURRENT_STORAGE_BASE_CLASS<OtherDerived> &other) const;" \ - "EIGEN_CWISE_PRODUCT_RETURN_TYPE(LHS,RHS)=CwiseBinaryOp<internal::scalar_product_op<typename LHS::Scalar, typename RHS::Scalar >, const LHS, const RHS>" + "EIGEN_CWISE_PRODUCT_RETURN_TYPE(LHS,RHS)=CwiseBinaryOp<internal::scalar_product_op<LHS::Scalar,RHS::Scalar>, const LHS, const RHS>"\ + "EIGEN_CAT2(a,b)= a ## b"\ + "EIGEN_CAT(a,b)=EIGEN_CAT2(a,b)"\ + "EIGEN_CWISE_BINARY_RETURN_TYPE(LHS,RHS,OPNAME)=CwiseBinaryOp<EIGEN_CAT(EIGEN_CAT(internal::scalar_,OPNAME),_op)<LHS::Scalar, RHS::Scalar>, const LHS, const RHS>"\ + "EIGEN_ALIGN_TO_BOUNDARY(x)="\ + DOXCOMMA=, + # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. @@ -1602,7 +1616,19 @@ EIGEN_CWISE_BINOP_RETURN_TYPE \ EIGEN_CURRENT_STORAGE_BASE_CLASS \ EIGEN_MATHFUNC_IMPL \ - _EIGEN_GENERIC_PUBLIC_INTERFACE + _EIGEN_GENERIC_PUBLIC_INTERFACE \ + EIGEN_ARRAY_DECLARE_GLOBAL_UNARY \ + EIGEN_EMPTY \ + EIGEN_EULER_ANGLES_TYPEDEFS \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF \ + EIGEN_EULER_SYSTEM_TYPEDEF \ + EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY \ + EIGEN_MATRIX_FUNCTION \ + EIGEN_MATRIX_FUNCTION_1 \ + EIGEN_DOC_UNARY_ADDONS \ + EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL \ + EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF + # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all references to function-like macros @@ -1646,7 +1672,7 @@ # in the modules index. If set to NO, only the current project's groups will # be listed. -EXTERNAL_GROUPS = YES +EXTERNAL_GROUPS = NO # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). @@ -1744,7 +1770,7 @@ # the class node. If there are many fields or methods and many nodes the # graph may become too big to be useful. The UML_LIMIT_NUM_FIELDS # threshold limits the number of items for each type to make the size more -# managable. Set this to 0 for no limit. Note that the threshold may be +# manageable. Set this to 0 for no limit. Note that the threshold may be # exceeded by 50% before the limit is enforced. UML_LIMIT_NUM_FIELDS = 10
diff --git a/doc/FixedSizeVectorizable.dox b/doc/FixedSizeVectorizable.dox index 49e38af..0012465 100644 --- a/doc/FixedSizeVectorizable.dox +++ b/doc/FixedSizeVectorizable.dox
@@ -1,6 +1,6 @@ namespace Eigen { -/** \eigenManualPage TopicFixedSizeVectorizable Fixed-size vectorizable Eigen objects +/** \eigenManualPage TopicFixedSizeVectorizable Fixed-size vectorizable %Eigen objects The goal of this page is to explain what we mean by "fixed-size vectorizable". @@ -23,15 +23,15 @@ \section FixedSizeVectorizable_explanation Explanation -First, "fixed-size" should be clear: an Eigen object has fixed size if its number of rows and its number of columns are fixed at compile-time. So for example Matrix3f has fixed size, but MatrixXf doesn't (the opposite of fixed-size is dynamic-size). +First, "fixed-size" should be clear: an %Eigen object has fixed size if its number of rows and its number of columns are fixed at compile-time. So for example \ref Matrix3f has fixed size, but \ref MatrixXf doesn't (the opposite of fixed-size is dynamic-size). -The array of coefficients of a fixed-size Eigen object is a plain "static array", it is not dynamically allocated. For example, the data behind a Matrix4f is just a "float array[16]". +The array of coefficients of a fixed-size %Eigen object is a plain "static array", it is not dynamically allocated. For example, the data behind a \ref Matrix4f is just a "float array[16]". Fixed-size objects are typically very small, which means that we want to handle them with zero runtime overhead -- both in terms of memory usage and of speed. -Now, vectorization (both SSE and AltiVec) works with 128-bit packets. Moreover, for performance reasons, these packets need to be have 128-bit alignment. +Now, vectorization works with 128-bit packets (e.g., SSE, AltiVec, NEON), 256-bit packets (e.g., AVX), or 512-bit packets (e.g., AVX512). Moreover, for performance reasons, these packets are most efficiently read and written if they have the same alignment as the packet size, that is 16 bytes, 32 bytes, and 64 bytes respectively. -So it turns out that the only way that fixed-size Eigen objects can be vectorized, is if their size is a multiple of 128 bits, or 16 bytes. Eigen will then request 16-byte alignment for these objects, and henceforth rely on these objects being aligned so no runtime check for alignment is performed. +So it turns out that the best way that fixed-size %Eigen objects can be vectorized, is if their size is a multiple of 16 bytes (or more). %Eigen will then request 16-byte alignment (or more) for these objects, and henceforth rely on these objects being aligned to achieve maximal efficiency. */
diff --git a/doc/FunctionsTakingEigenTypes.dox b/doc/FunctionsTakingEigenTypes.dox index 152dda4..6b4e492 100644 --- a/doc/FunctionsTakingEigenTypes.dox +++ b/doc/FunctionsTakingEigenTypes.dox
@@ -79,7 +79,7 @@ \section TopicUsingRefClass How to write generic, but non-templated function? -In all the previous examples, the functions had to be template functions. This approach allows to write very generic code, but it is often desirable to write non templated function and still keep some level of genericity to avoid stupid copies of the arguments. The typical example is to write functions accepting both a MatrixXf or a block of a MatrixXf. This exactly the purpose of the Ref class. Here is a simple example: +In all the previous examples, the functions had to be template functions. This approach allows to write very generic code, but it is often desirable to write non templated functions and still keep some level of genericity to avoid stupid copies of the arguments. The typical example is to write functions accepting both a MatrixXf or a block of a MatrixXf. This is exactly the purpose of the Ref class. Here is a simple example: <table class="example"> <tr><th>Example:</th><th>Output:</th></tr> @@ -133,7 +133,7 @@ \section TopicPlainFunctionsFailing In which cases do functions taking a plain Matrix or Array argument fail? -Here, we consider a slightly modified version of the function given above. This time, we do not want to return the result but pass an additional non-const paramter which allows us to store the result. A first naive implementation might look as follows. +Here, we consider a slightly modified version of the function given above. This time, we do not want to return the result but pass an additional non-const parameter which allows us to store the result. A first naive implementation might look as follows. \code // Note: This code is flawed! void cov(const MatrixXf& x, const MatrixXf& y, MatrixXf& C) @@ -176,7 +176,7 @@ \section TopicResizingInGenericImplementations How to resize matrices in generic implementations? -One might think we are done now, right? This is not completely true because in order for our covariance function to be generically applicable, we want the follwing code to work +One might think we are done now, right? This is not completely true because in order for our covariance function to be generically applicable, we want the following code to work \code MatrixXf x = MatrixXf::Random(100,3); MatrixXf y = MatrixXf::Random(100,3);
diff --git a/doc/HiPerformance.dox b/doc/HiPerformance.dox index ab6cdfd..9cee335 100644 --- a/doc/HiPerformance.dox +++ b/doc/HiPerformance.dox
@@ -105,7 +105,7 @@ <td>First of all, here the .noalias() in the first expression is useless because m2*m3 will be evaluated anyway. However, note how this expression can be rewritten so that no temporary is required. (tip: for very small fixed size matrix - it is slighlty better to rewrite it like this: m1.noalias() = m2 * m3; m1 += m4;</td> + it is slightly better to rewrite it like this: m1.noalias() = m2 * m3; m1 += m4;</td> </tr> <tr class="alt"> <td>\code
diff --git a/doc/InplaceDecomposition.dox b/doc/InplaceDecomposition.dox new file mode 100644 index 0000000..cb1c6d4 --- /dev/null +++ b/doc/InplaceDecomposition.dox
@@ -0,0 +1,115 @@ +namespace Eigen { + +/** \eigenManualPage InplaceDecomposition Inplace matrix decompositions + +Starting from %Eigen 3.3, the LU, Cholesky, and QR decompositions can operate \em inplace, that is, directly within the given input matrix. +This feature is especially useful when dealing with huge matrices, and or when the available memory is very limited (embedded systems). + +To this end, the respective decomposition class must be instantiated with a Ref<> matrix type, and the decomposition object must be constructed with the input matrix as argument. As an example, let us consider an inplace LU decomposition with partial pivoting. + +Let's start with the basic inclusions, and declaration of a 2x2 matrix \c A: + +<table class="example"> +<tr><th>code</th><th>output</th></tr> +<tr> + <td>\snippet TutorialInplaceLU.cpp init + </td> + <td>\snippet TutorialInplaceLU.out init + </td> +</tr> +</table> + +No surprise here! Then, let's declare our inplace LU object \c lu, and check the content of the matrix \c A: + +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp declaration + </td> + <td>\snippet TutorialInplaceLU.out declaration + </td> +</tr> +</table> + +Here, the \c lu object computes and stores the \c L and \c U factors within the memory held by the matrix \c A. +The coefficients of \c A have thus been destroyed during the factorization, and replaced by the L and U factors as one can verify: + +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp matrixLU + </td> + <td>\snippet TutorialInplaceLU.out matrixLU + </td> +</tr> +</table> + +Then, one can use the \c lu object as usual, for instance to solve the Ax=b problem: +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp solve + </td> + <td>\snippet TutorialInplaceLU.out solve + </td> +</tr> +</table> + +Here, since the content of the original matrix \c A has been lost, we had to declared a new matrix \c A0 to verify the result. + +Since the memory is shared between \c A and \c lu, modifying the matrix \c A will make \c lu invalid. +This can easily be verified by modifying the content of \c A and trying to solve the initial problem again: + +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp modifyA + </td> + <td>\snippet TutorialInplaceLU.out modifyA + </td> +</tr> +</table> + +Note that there is no shared pointer under the hood, it is the \b responsibility \b of \b the \b user to keep the input matrix \c A in life as long as \c lu is living. + +If one wants to update the factorization with the modified A, one has to call the compute method as usual: +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp recompute + </td> + <td>\snippet TutorialInplaceLU.out recompute + </td> +</tr> +</table> + +Note that calling compute does not change the memory which is referenced by the \c lu object. Therefore, if the compute method is called with another matrix \c A1 different than \c A, then the content of \c A1 won't be modified. This is still the content of \c A that will be used to store the L and U factors of the matrix \c A1. +This can easily be verified as follows: +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp recompute_bis0 + </td> + <td>\snippet TutorialInplaceLU.out recompute_bis0 + </td> +</tr> +</table> +The matrix \c A1 is unchanged, and one can thus solve A1*x=b, and directly check the residual without any copy of \c A1: +<table class="example"> +<tr> + <td>\snippet TutorialInplaceLU.cpp recompute_bis1 + </td> + <td>\snippet TutorialInplaceLU.out recompute_bis1 + </td> +</tr> +</table> + + +Here is the list of matrix decompositions supporting this inplace mechanism: + +- class LLT +- class LDLT +- class PartialPivLU +- class FullPivLU +- class HouseholderQR +- class ColPivHouseholderQR +- class FullPivHouseholderQR +- class CompleteOrthogonalDecomposition + +*/ + +} \ No newline at end of file
diff --git a/doc/InsideEigenExample.dox b/doc/InsideEigenExample.dox index ed053c6..ea2275b 100644 --- a/doc/InsideEigenExample.dox +++ b/doc/InsideEigenExample.dox
@@ -212,6 +212,11 @@ \section Assignment The assignment +<div class="warningbox"> +<strong>PLEASE HELP US IMPROVING THIS SECTION.</strong> +This page reflects how %Eigen worked until 3.2, but since %Eigen 3.3 the assignment is more sophisticated as it involves an Assignment expression, and the creation of so called evaluator which are responsible for the evaluation of each kind of expressions. +</div> + At this point, the expression \a v + \a w has finished evaluating, so, in the process of compiling the line of code \code u = v + w;
diff --git a/doc/LeastSquares.dox b/doc/LeastSquares.dox index e2191a2..24dfe4b 100644 --- a/doc/LeastSquares.dox +++ b/doc/LeastSquares.dox
@@ -16,7 +16,7 @@ \section LeastSquaresSVD Using the SVD decomposition -The \link JacobiSVD::solve() solve() \endlink method in the JacobiSVD class can be directly used to +The \link BDCSVD::solve() solve() \endlink method in the BDCSVD class can be directly used to solve linear squares systems. It is not enough to compute only the singular values (the default for this class); you also need the singular vectors but the thin SVD decomposition suffices for computing least squares solutions:
diff --git a/doc/Manual.dox b/doc/Manual.dox index 70aaa9a..84f0db6 100644 --- a/doc/Manual.dox +++ b/doc/Manual.dox
@@ -3,21 +3,31 @@ namespace Eigen { +/** \page UserManual_CustomizingEigen Extending/Customizing Eigen + %Eigen can be extended in several ways, for instance, by defining global methods, by inserting custom methods within main %Eigen's classes through the \ref TopicCustomizing_Plugins "plugin" mechanism, by adding support to \ref TopicCustomizing_CustomScalar "custom scalar types" etc. See below for the respective sub-topics. + - \subpage TopicCustomizing_Plugins + - \subpage TopicCustomizing_InheritingMatrix + - \subpage TopicCustomizing_CustomScalar + - \subpage TopicCustomizing_NullaryExpr + - \subpage TopicNewExpressionType + \sa \ref TopicPreprocessorDirectives +*/ + + /** \page UserManual_Generalities General topics - - \subpage Eigen2ToEigen3 - \subpage TopicFunctionTakingEigenTypes - \subpage TopicPreprocessorDirectives - \subpage TopicAssertions - - \subpage TopicCustomizingEigen - \subpage TopicMultiThreading + - \subpage TopicUsingBlasLapack - \subpage TopicUsingIntelMKL - \subpage TopicCUDA - \subpage TopicPitfalls - \subpage TopicTemplateKeyword - - \subpage TopicNewExpressionType - \subpage UserManual_UnderstandingEigen + - \subpage TopicCMakeGuide */ - + /** \page UserManual_UnderstandingEigen Understanding Eigen - \subpage TopicInsideEigenExample - \subpage TopicClassHierarchy @@ -53,42 +63,49 @@ \ingroup DenseMatrixManipulation_chapter */ /** \addtogroup TutorialBlockOperations \ingroup DenseMatrixManipulation_chapter */ +/** \addtogroup TutorialSlicingIndexing + \ingroup DenseMatrixManipulation_chapter */ /** \addtogroup TutorialAdvancedInitialization \ingroup DenseMatrixManipulation_chapter */ /** \addtogroup TutorialReductionsVisitorsBroadcasting \ingroup DenseMatrixManipulation_chapter */ -/** \addtogroup TutorialMapClass +/** \addtogroup TutorialReshape \ingroup DenseMatrixManipulation_chapter */ -/** \addtogroup TutorialReshapeSlicing +/** \addtogroup TutorialSTL + \ingroup DenseMatrixManipulation_chapter */ +/** \addtogroup TutorialMapClass \ingroup DenseMatrixManipulation_chapter */ /** \addtogroup TopicAliasing \ingroup DenseMatrixManipulation_chapter */ /** \addtogroup TopicStorageOrders \ingroup DenseMatrixManipulation_chapter */ - + /** \addtogroup DenseMatrixManipulation_Alignement - \ingroup DenseMatrixManipulation_chapter */ -/** \addtogroup TopicUnalignedArrayAssert - \ingroup DenseMatrixManipulation_Alignement */ -/** \addtogroup TopicFixedSizeVectorizable - \ingroup DenseMatrixManipulation_Alignement */ -/** \addtogroup TopicStructHavingEigenMembers - \ingroup DenseMatrixManipulation_Alignement */ -/** \addtogroup TopicStlContainers - \ingroup DenseMatrixManipulation_Alignement */ -/** \addtogroup TopicPassingByValue - \ingroup DenseMatrixManipulation_Alignement */ -/** \addtogroup TopicWrongStackAlignment - \ingroup DenseMatrixManipulation_Alignement */ + \ingroup DenseMatrixManipulation_chapter */ +/** \addtogroup TopicUnalignedArrayAssert + \ingroup DenseMatrixManipulation_Alignement */ +/** \addtogroup TopicFixedSizeVectorizable + \ingroup DenseMatrixManipulation_Alignement */ +/** \addtogroup TopicStructHavingEigenMembers + \ingroup DenseMatrixManipulation_Alignement */ +/** \addtogroup TopicStlContainers + \ingroup DenseMatrixManipulation_Alignement */ +/** \addtogroup TopicPassingByValue + \ingroup DenseMatrixManipulation_Alignement */ +/** \addtogroup TopicWrongStackAlignment + \ingroup DenseMatrixManipulation_Alignement */ /** \addtogroup DenseMatrixManipulation_Reference + \ingroup DenseMatrixManipulation_chapter */ +/** \addtogroup Core_Module + \ingroup DenseMatrixManipulation_Reference */ +/** \addtogroup Jacobi_Module + \ingroup DenseMatrixManipulation_Reference */ +/** \addtogroup Householder_Module + \ingroup DenseMatrixManipulation_Reference */ + +/** \addtogroup CoeffwiseMathFunctions \ingroup DenseMatrixManipulation_chapter */ -/** \addtogroup Core_Module - \ingroup DenseMatrixManipulation_Reference */ -/** \addtogroup Jacobi_Module - \ingroup DenseMatrixManipulation_Reference */ -/** \addtogroup Householder_Module - \ingroup DenseMatrixManipulation_Reference */ /** \addtogroup QuickRefPage \ingroup DenseMatrixManipulation_chapter */ @@ -103,6 +120,10 @@ \ingroup DenseLinearSolvers_chapter */ /** \addtogroup LeastSquares \ingroup DenseLinearSolvers_chapter */ +/** \addtogroup InplaceDecomposition + \ingroup DenseLinearSolvers_chapter */ +/** \addtogroup DenseDecompositionBenchmark + \ingroup DenseLinearSolvers_chapter */ /** \addtogroup DenseLinearSolvers_Reference \ingroup DenseLinearSolvers_chapter */
diff --git a/doc/MatrixfreeSolverExample.dox b/doc/MatrixfreeSolverExample.dox index 000cb0b..3efa292 100644 --- a/doc/MatrixfreeSolverExample.dox +++ b/doc/MatrixfreeSolverExample.dox
@@ -6,12 +6,12 @@ \eigenManualPage MatrixfreeSolverExample Matrix-free solvers Iterative solvers such as ConjugateGradient and BiCGSTAB can be used in a matrix free context. To this end, user must provide a wrapper class inheriting EigenBase<> and implementing the following methods: - - Index rows() and Index cols(): returns number of rows and columns respectively - - operator* with and %Eigen dense column vector (its actual implementation goes in a specialization of the internal::generic_product_impl class) + - \c Index \c rows() and \c Index \c cols(): returns number of rows and columns respectively + - \c operator* with your type and an %Eigen dense column vector (its actual implementation goes in a specialization of the internal::generic_product_impl class) -Eigen::internal::traits<> must also be specialized for the wrapper type. +\c Eigen::internal::traits<> must also be specialized for the wrapper type. -Here is a complete example wrapping a Eigen::SparseMatrix: +Here is a complete example wrapping an Eigen::SparseMatrix: \include matrixfree_cg.cpp Output: \verbinclude matrixfree_cg.out
diff --git a/doc/NewExpressionType.dox b/doc/NewExpressionType.dox index ad8b7f8..c2f2433 100644 --- a/doc/NewExpressionType.dox +++ b/doc/NewExpressionType.dox
@@ -2,6 +2,12 @@ /** \page TopicNewExpressionType Adding a new expression type +<!--<span style="font-size:130%; color:red; font-weight: 900;"></span>--> +\warning +Disclaimer: this page is tailored to very advanced users who are not afraid of dealing with some %Eigen's internal aspects. +In most cases, a custom expression can be avoided by either using custom \ref MatrixBase::unaryExpr "unary" or \ref MatrixBase::binaryExpr "binary" functors, +while extremely complex matrix manipulations can be achieved by a nullary functors as described in the \ref TopicCustomizing_NullaryExpr "previous page". + This page describes with the help of an example how to implement a new light-weight expression type in %Eigen. This consists of three parts: the expression type itself, a traits class containing compile-time @@ -130,7 +136,7 @@ If all the fragments are combined, the following output is produced, showing that the program works as expected: -\verbinclude make_circulant.out +\include make_circulant.out */ }
diff --git a/doc/Overview.dox b/doc/Overview.dox index 9ab9623..43a1287 100644 --- a/doc/Overview.dox +++ b/doc/Overview.dox
@@ -4,8 +4,6 @@ This is the API documentation for Eigen3. You can <a href="eigen-doc.tgz">download</a> it as a tgz archive for offline reading. -You're already an Eigen2 user? Here is a \link Eigen2ToEigen3 Eigen2 to Eigen3 guide \endlink to help porting your application. - For a first contact with Eigen, the best place is to have a look at the \link GettingStarted getting started \endlink page that show you how to write and compile your first program with Eigen. Then, the \b quick \b reference \b pages give you a quite complete description of the API in a very condensed format that is specially useful to recall the syntax of a particular feature, or to have a quick look at the API. They currently cover the two following feature sets, and more will come in the future: @@ -17,7 +15,9 @@ The \b main \b documentation is organized into \em chapters covering different domains of features. They are themselves composed of \em user \em manual pages describing the different features in a comprehensive way, and \em reference pages that gives you access to the API documentation through the related Eigen's \em modules and \em classes. -Under the \subpage UserManual_Generalities section, you will find documentation on more general topics such as preprocessor directives, controlling assertions, multi-threading, MKL support, some Eigen's internal insights, and much more... +Under the \subpage UserManual_CustomizingEigen section, you will find discussions and examples on extending %Eigen's features and supporting custom scalar types. + +Under the \subpage UserManual_Generalities section, you will find documentation on more general topics such as preprocessor directives, controlling assertions, multi-threading, MKL support, some Eigen's internal insights, and much more... Finally, do not miss the search engine, useful to quickly get to the documentation of a given class or function.
diff --git a/doc/PassingByValue.dox b/doc/PassingByValue.dox index bf4d0ef..9254fe6 100644 --- a/doc/PassingByValue.dox +++ b/doc/PassingByValue.dox
@@ -4,21 +4,21 @@ Passing objects by value is almost always a very bad idea in C++, as this means useless copies, and one should pass them by reference instead. -With Eigen, this is even more important: passing \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen objects" by value is not only inefficient, it can be illegal or make your program crash! And the reason is that these Eigen objects have alignment modifiers that aren't respected when they are passed by value. +With %Eigen, this is even more important: passing \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen objects" by value is not only inefficient, it can be illegal or make your program crash! And the reason is that these %Eigen objects have alignment modifiers that aren't respected when they are passed by value. -So for example, a function like this, where v is passed by value: +For example, a function like this, where \c v is passed by value: \code void my_function(Eigen::Vector2d v); \endcode -needs to be rewritten as follows, passing v by reference: +needs to be rewritten as follows, passing \c v by const reference: \code void my_function(const Eigen::Vector2d& v); \endcode -Likewise if you have a class having a Eigen object as member: +Likewise if you have a class having an %Eigen object as member: \code struct Foo
diff --git a/doc/Pitfalls.dox b/doc/Pitfalls.dox index cf42eff..fda4025 100644 --- a/doc/Pitfalls.dox +++ b/doc/Pitfalls.dox
@@ -2,13 +2,35 @@ /** \page TopicPitfalls Common pitfalls + \section TopicPitfalls_template_keyword Compilation error with template methods See this \link TopicTemplateKeyword page \endlink. + +\section TopicPitfalls_aliasing Aliasing + +Don't miss this \link TopicAliasing page \endlink on aliasing, +especially if you got wrong results in statements where the destination appears on the right hand side of the expression. + + +\section TopicPitfalls_alignment_issue Alignment Issues (runtime assertion) + +%Eigen does explicit vectorization, and while that is appreciated by many users, that also leads to some issues in special situations where data alignment is compromised. +Indeed, since C++17, C++ does not have quite good enough support for explicit data alignment. +In that case your program hits an assertion failure (that is, a "controlled crash") with a message that tells you to consult this page: +\code +http://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html +\endcode +Have a look at \link TopicUnalignedArrayAssert it \endlink and see for yourself if that's something that you can cope with. +It contains detailed information about how to deal with each known cause for that issue. + +Now what if you don't care about vectorization and so don't want to be annoyed with these alignment issues? Then read \link getrid how to get rid of them \endlink. + + \section TopicPitfalls_auto_keyword C++11 and the auto keyword -In short: do not use the auto keywords with Eigen's expressions, unless you are 100% sure about what you are doing. In particular, do not use the auto keyword as a replacement for a Matrix<> type. Here is an example: +In short: do not use the auto keywords with %Eigen's expressions, unless you are 100% sure about what you are doing. In particular, do not use the auto keyword as a replacement for a \c Matrix<> type. Here is an example: \code MatrixXd A, B; @@ -16,23 +38,81 @@ for(...) { ... w = C * v; ...} \endcode -In this example, the type of C is not a MatrixXd but an abstract expression representing a matrix product and storing references to A and B. Therefore, the product of A*B will be carried out multiple times, once per iteration of the for loop. Moreover, if the coefficients of A or B change during the iteration, then C will evaluate to different values. +In this example, the type of C is not a \c MatrixXd but an abstract expression representing a matrix product and storing references to \c A and \c B. +Therefore, the product of \c A*B will be carried out multiple times, once per iteration of the for loop. +Moreover, if the coefficients of A or B change during the iteration, then C will evaluate to different values. Here is another example leading to a segfault: \code auto C = ((A+B).eval()).transpose(); // do something with C \endcode -The problem is that eval() returns a temporary object (in this case a MatrixXd) which is then referenced by the Transpose<> expression. However, this temporary is deleted right after the first line, and there the C expression reference a dead object. The same issue might occur when sub expressions are automatically evaluated by Eigen as in the following example: +The problem is that \c eval() returns a temporary object (in this case a \c MatrixXd) which is then referenced by the \c Transpose<> expression. +However, this temporary is deleted right after the first line, and then the \c C expression references a dead object. +One possible fix consists in applying \c eval() on the whole expression: +\code +auto C = (A+B).transpose().eval(); +\endcode + +The same issue might occur when sub expressions are automatically evaluated by %Eigen as in the following example: \code VectorXd u, v; auto C = u + (A*v).normalized(); // do something with C \endcode -where the normalized() method has to evaluate the expensive product A*v to avoid evaluating it twice. On the other hand, the following example is perfectly fine: +Here the \c normalized() method has to evaluate the expensive product \c A*v to avoid evaluating it twice. +Again, one possible fix is to call \c .eval() on the whole expression: \code auto C = (u + (A*v).normalized()).eval(); \endcode -In this case, C will be a regular VectorXd object. +In this case, \c C will be a regular \c VectorXd object. +Note that DenseBase::eval() is smart enough to avoid copies when the underlying expression is already a plain \c Matrix<>. + + +\section TopicPitfalls_header_issues Header Issues (failure to compile) + +With all libraries, one must check the documentation for which header to include. +The same is true with %Eigen, but slightly worse: with %Eigen, a method in a class may require an additional <code>#include</code> over what the class itself requires! +For example, if you want to use the \c cross() method on a vector (it computes a cross-product) then you need to: +\code +#include<Eigen/Geometry> +\endcode +We try to always document this, but do tell us if we forgot an occurrence. + + +\section TopicPitfalls_ternary_operator Ternary operator + +In short: avoid the use of the ternary operator <code>(COND ? THEN : ELSE)</code> with %Eigen's expressions for the \c THEN and \c ELSE statements. +To see why, let's consider the following example: +\code +Vector3f A; +A << 1, 2, 3; +Vector3f B = ((1 < 0) ? (A.reverse()) : A); +\endcode +This example will return <code>B = 3, 2, 1</code>. Do you see why? +The reason is that in c++ the type of the \c ELSE statement is inferred from the type of the \c THEN expression such that both match. +Since \c THEN is a <code>Reverse<Vector3f></code>, the \c ELSE statement A is converted to a <code>Reverse<Vector3f></code>, and the compiler thus generates: +\code +Vector3f B = ((1 < 0) ? (A.reverse()) : Reverse<Vector3f>(A)); +\endcode +In this very particular case, a workaround would be to call A.reverse().eval() for the \c THEN statement, but the safest and fastest is really to avoid this ternary operator with %Eigen's expressions and use a if/else construct. + + +\section TopicPitfalls_pass_by_value Pass-by-value + +If you don't know why passing-by-value is wrong with %Eigen, read this \link TopicPassingByValue page \endlink first. + +While you may be extremely careful and use care to make sure that all of your code that explicitly uses %Eigen types is pass-by-reference you have to watch out for templates which define the argument types at compile time. + +If a template has a function that takes arguments pass-by-value, and the relevant template parameter ends up being an %Eigen type, then you will of course have the same alignment problems that you would in an explicitly defined function passing %Eigen types by reference. + +Using %Eigen types with other third party libraries or even the STL can present the same problem. +<code>boost::bind</code> for example uses pass-by-value to store arguments in the returned functor. +This will of course be a problem. + +There are at least two ways around this: + - If the value you are passing is guaranteed to be around for the life of the functor, you can use boost::ref() to wrap the value as you pass it to boost::bind. Generally this is not a solution for values on the stack as if the functor ever gets passed to a lower or independent scope, the object may be gone by the time it's attempted to be used. + - The other option is to make your functions take a reference counted pointer like boost::shared_ptr as the argument. This avoids needing to worry about managing the lifetime of the object being passed. + */ }
diff --git a/doc/PreprocessorDirectives.dox b/doc/PreprocessorDirectives.dox index 14e84bc..ffd2c66 100644 --- a/doc/PreprocessorDirectives.dox +++ b/doc/PreprocessorDirectives.dox
@@ -49,6 +49,37 @@ the correct size. Not defined by default. +\section TopicPreprocessorDirectivesCppVersion C++ standard features + +By default, %Eigen strive to automatically detect and enable language features at compile-time based on +the information provided by the compiler. + + - \b EIGEN_MAX_CPP_VER - disables usage of C++ features requiring a version greater than EIGEN_MAX_CPP_VER. + Possible values are: 03, 11, 14, 17, etc. If not defined (the default), %Eigen enables all features supported + by the compiler. + +Individual features can be explicitly enabled or disabled by defining the following token to 0 or 1 respectively. +For instance, one might limit the C++ version to C++03 by defining EIGEN_MAX_CPP_VER=03, but still enable C99 math +functions by defining EIGEN_HAS_C99_MATH=1. + + - \b EIGEN_HAS_C99_MATH - controls the usage of C99 math functions such as erf, erfc, lgamma, etc. + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_CXX11_MATH - controls the implementation of some functions such as round, logp1, isinf, isnan, etc. + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_RVALUE_REFERENCES - defines whether rvalue references are supported + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_STD_RESULT_OF - defines whether std::result_of is supported + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_VARIADIC_TEMPLATES - defines whether variadic templates are supported + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_CONSTEXPR - defines whether relaxed const expression are supported + Automatic detection disabled if EIGEN_MAX_CPP_VER<14. + - \b EIGEN_HAS_CXX11_CONTAINERS - defines whether STL's containers follows C++11 specifications + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_HAS_CXX11_NOEXCEPT - defines whether noexcept is supported + Automatic detection disabled if EIGEN_MAX_CPP_VER<11. + - \b EIGEN_NO_IO - Disables any usage and support for `<iostreams>`. + \section TopicPreprocessorDirectivesAssertions Assertions The %Eigen library contains many assertions to guard against programming errors, both at compile time and at @@ -67,32 +98,45 @@ \section TopicPreprocessorDirectivesPerformance Alignment, vectorization and performance tweaking - - \b EIGEN_MALLOC_ALREADY_ALIGNED - Can be set to 0 or 1 to tell whether default system \c malloc already + - \b \c EIGEN_MALLOC_ALREADY_ALIGNED - Can be set to 0 or 1 to tell whether default system \c malloc already returns aligned buffers. In not defined, then this information is automatically deduced from the compiler and system preprocessor tokens. - - \b EIGEN_DONT_ALIGN - disables alignment completely. %Eigen will not try to align its objects and does not - expect that any objects passed to it are aligned. This will turn off vectorization. Not defined by default. - - \b EIGEN_DONT_ALIGN_STATICALLY - disables alignment of arrays on the stack. Not defined by default, unless - \c EIGEN_DONT_ALIGN is defined. - - \b EIGEN_DONT_PARALLELIZE - if defined, this disables multi-threading. This is only relevant if you enabled OpenMP. + - \b \c EIGEN_MAX_ALIGN_BYTES - Must be a power of two, or 0. Defines an upper bound on the memory boundary in bytes on which dynamically and statically allocated data may be aligned by %Eigen. If not defined, a default value is automatically computed based on architecture, compiler, and OS. + This option is typically used to enforce binary compatibility between code/libraries compiled with different SIMD options. For instance, one may compile AVX code and enforce ABI compatibility with existing SSE code by defining \c EIGEN_MAX_ALIGN_BYTES=16. In the other way round, since by default AVX implies 32 bytes alignment for best performance, one can compile SSE code to be ABI compatible with AVX code by defining \c EIGEN_MAX_ALIGN_BYTES=32. + - \b \c EIGEN_MAX_STATIC_ALIGN_BYTES - Same as \c EIGEN_MAX_ALIGN_BYTES but for statically allocated data only. By default, if only \c EIGEN_MAX_ALIGN_BYTES is defined, then \c EIGEN_MAX_STATIC_ALIGN_BYTES == \c EIGEN_MAX_ALIGN_BYTES, otherwise a default value is automatically computed based on architecture, compiler, and OS (can be smaller than the default value of EIGEN_MAX_ALIGN_BYTES on architectures that do not support stack alignment). + Let us emphasize that \c EIGEN_MAX_*_ALIGN_BYTES define only a diserable upper bound. In practice data is aligned to largest power-of-two common divisor of \c EIGEN_MAX_STATIC_ALIGN_BYTES and the size of the data, such that memory is not wasted. + - \b \c EIGEN_DONT_PARALLELIZE - if defined, this disables multi-threading. This is only relevant if you enabled OpenMP. See \ref TopicMultiThreading for details. - \b EIGEN_DONT_VECTORIZE - disables explicit vectorization when defined. Not defined by default, unless alignment is disabled by %Eigen's platform test or the user defining \c EIGEN_DONT_ALIGN. - - \b EIGEN_FAST_MATH - enables some optimizations which might affect the accuracy of the result. This currently + - \b \c EIGEN_UNALIGNED_VECTORIZE - disables/enables vectorization with unaligned stores. Default is 1 (enabled). + If set to 0 (disabled), then expression for which the destination cannot be aligned are not vectorized (e.g., unaligned + small fixed size vectors or matrices) + - \b \c EIGEN_FAST_MATH - enables some optimizations which might affect the accuracy of the result. This currently enables the SSE vectorization of sin() and cos(), and speedups sqrt() for single precision. Defined to 1 by default. Define it to 0 to disable. - - \b EIGEN_UNROLLING_LIMIT - defines the size of a loop to enable meta unrolling. Set it to zero to disable + - \b \c EIGEN_UNROLLING_LIMIT - defines the size of a loop to enable meta unrolling. Set it to zero to disable unrolling. The size of a loop here is expressed in %Eigen's own notion of "number of FLOPS", it does not - correspond to the number of iterations or the number of instructions. The default is value 100. - - \b EIGEN_STACK_ALLOCATION_LIMIT - defines the maximum bytes for a buffer to be allocated on the stack. For internal + correspond to the number of iterations or the number of instructions. The default is value 110. + - \b \c EIGEN_STACK_ALLOCATION_LIMIT - defines the maximum bytes for a buffer to be allocated on the stack. For internal temporary buffers, dynamic memory allocation is employed as a fall back. For fixed-size matrices or arrays, exceeding this threshold raises a compile time assertion. Use 0 to set no limit. Default is 128 KB. + - \b \c EIGEN_NO_CUDA - disables CUDA support when defined. Might be useful in .cu files for which Eigen is used on the host only, + and never called from device code. + - \b \c EIGEN_STRONG_INLINE - This macro is used to qualify critical functions and methods that we expect the compiler to inline. + By default it is defined to \c __forceinline for MSVC and ICC, and to \c inline for other compilers. A tipical usage is to + define it to \c inline for MSVC users wanting faster compilation times, at the risk of performance degradations in some rare + cases for which MSVC inliner fails to do a good job. + + + - \c EIGEN_DONT_ALIGN - Deprecated, it is a synonym for \c EIGEN_MAX_ALIGN_BYTES=0. It disables alignment completely. %Eigen will not try to align its objects and does not expect that any objects passed to it are aligned. This will turn off vectorization if \b EIGEN_UNALIGNED_VECTORIZE=1. Not defined by default. + - \c EIGEN_DONT_ALIGN_STATICALLY - Deprecated, it is a synonym for \c EIGEN_MAX_STATIC_ALIGN_BYTES=0. It disables alignment of arrays on the stack. Not defined by default, unless \c EIGEN_DONT_ALIGN is defined. \section TopicPreprocessorDirectivesPlugins Plugins It is possible to add new methods to many fundamental classes in %Eigen by writing a plugin. As explained in -the section \ref ExtendingMatrixBase, the plugin is specified by defining a \c EIGEN_xxx_PLUGIN macro. The +the section \ref TopicCustomizing_Plugins, the plugin is specified by defining a \c EIGEN_xxx_PLUGIN macro. The following macros are supported; none of them are defined by default. - \b EIGEN_ARRAY_PLUGIN - filename of plugin for extending the Array class.
diff --git a/doc/QuickReference.dox b/doc/QuickReference.dox index e19c7e3..9c8e6fb 100644 --- a/doc/QuickReference.dox +++ b/doc/QuickReference.dox
@@ -68,7 +68,7 @@ Conversion between the matrix and array worlds: \code -Array44f a1, a1; +Array44f a1, a2; Matrix4f m1, m2; m1 = a1 * a2; // coeffwise product, implicit conversion from array to matrix. a1 = m1 * m2; // matrix product, implicit conversion from matrix to array. @@ -261,6 +261,8 @@ Vector3f::UnitX() // 1 0 0 Vector3f::UnitY() // 0 1 0 Vector3f::UnitZ() // 0 0 1 +Vector4f::Unit(i) +x.setUnit(i); \endcode </td> <td> @@ -278,6 +280,7 @@ VectorXf::Unit(size,i) +x.setUnit(size,i); VectorXf::Unit(4,1) == Vector4f(0,1,0,0) == Vector4f::UnitY() \endcode @@ -285,7 +288,12 @@ </tr> </table> - +Note that it is allowed to call any of the \c set* functions to a dynamic-sized vector or matrix without passing new sizes. +For instance: +\code +MatrixXi M(3,3); +M.setIdentity(); +\endcode \subsection QuickRef_Map Mapping external arrays @@ -340,7 +348,7 @@ \endcode </td></tr> <tr><td> -\link MatrixBase::dot() dot \endlink product \n inner product \matrixworld</td><td>\code +\link MatrixBase::dot dot \endlink product \n inner product \matrixworld</td><td>\code scalar = vec1.dot(vec2); scalar = col1.adjoint() * col2; scalar = (col1.adjoint() * col2).value();\endcode @@ -521,6 +529,12 @@ <a href="#" class="top">top</a>\section QuickRef_Blocks Sub-matrices +<div class="warningbox"> +<strong>PLEASE HELP US IMPROVING THIS SECTION.</strong> +%Eigen 3.4 supports a much improved API for sub-matrices, including, +slicing and indexing from arrays: \ref TutorialSlicingIndexing +</div> + Read-write access to a \link DenseBase::col(Index) column \endlink or a \link DenseBase::row(Index) row \endlink of a matrix (or array): \code @@ -576,6 +590,11 @@ <a href="#" class="top">top</a>\section QuickRef_Misc Miscellaneous operations +<div class="warningbox"> +<strong>PLEASE HELP US IMPROVING THIS SECTION.</strong> +%Eigen 3.4 supports a new API for reshaping: \ref TutorialReshape +</div> + \subsection QuickRef_Reverse Reverse Vectors, rows, and/or columns of a matrix can be reversed (see DenseBase::reverse(), DenseBase::reverseInPlace(), VectorwiseOp::reverse()). \code
diff --git a/doc/QuickStartGuide.dox b/doc/QuickStartGuide.dox index ea32c3b..4192b28 100644 --- a/doc/QuickStartGuide.dox +++ b/doc/QuickStartGuide.dox
@@ -66,9 +66,9 @@ \section GettingStartedExplanation2 Explanation of the second example -The second example starts by declaring a 3-by-3 matrix \c m which is initialized using the \link DenseBase::Random(Index,Index) Random() \endlink method with random values between -1 and 1. The next line applies a linear mapping such that the values are between 10 and 110. The function call \link DenseBase::Constant(Index,Index,const Scalar&) MatrixXd::Constant\endlink(3,3,1.2) returns a 3-by-3 matrix expression having all coefficients equal to 1.2. The rest is standard arithmetics. +The second example starts by declaring a 3-by-3 matrix \c m which is initialized using the \link DenseBase::Random(Index,Index) Random() \endlink method with random values between -1 and 1. The next line applies a linear mapping such that the values are between 10 and 110. The function call \link DenseBase::Constant(Index,Index,const Scalar&) MatrixXd::Constant\endlink(3,3,1.2) returns a 3-by-3 matrix expression having all coefficients equal to 1.2. The rest is standard arithmetic. -The next line of the \c main function introduces a new type: \c VectorXd. This represents a (column) vector of arbitrary size. Here, the vector \c v is created to contain \c 3 coefficients which are left unitialized. The one but last line uses the so-called comma-initializer, explained in \ref TutorialAdvancedInitialization, to set all coefficients of the vector \c v to be as follows: +The next line of the \c main function introduces a new type: \c VectorXd. This represents a (column) vector of arbitrary size. Here, the vector \c v is created to contain \c 3 coefficients which are left uninitialized. The one but last line uses the so-called comma-initializer, explained in \ref TutorialAdvancedInitialization, to set all coefficients of the vector \c v to be as follows: \f[ v =
diff --git a/doc/SparseLinearSystems.dox b/doc/SparseLinearSystems.dox index ee4f53a..38754e4 100644 --- a/doc/SparseLinearSystems.dox +++ b/doc/SparseLinearSystems.dox
@@ -70,12 +70,18 @@ <tr><td>UmfPackLU</td><td>\link UmfPackSupport_Module UmfPackSupport \endlink</td><td>Direct LU factorization</td><td>Square</td><td>Fill-in reducing, Leverage fast dense algebra</td> <td>Requires the <a href="http://www.suitesparse.com">SuiteSparse</a> package, \b GPL </td> <td></td></tr> +<tr><td>KLU</td><td>\link KLUSupport_Module KLUSupport \endlink</td><td>Direct LU factorization</td><td>Square</td><td>Fill-in reducing, suitted for circuit simulation</td> + <td>Requires the <a href="http://www.suitesparse.com">SuiteSparse</a> package, \b GPL </td> + <td></td></tr> <tr><td>SuperLU</td><td>\link SuperLUSupport_Module SuperLUSupport \endlink</td><td>Direct LU factorization</td><td>Square</td><td>Fill-in reducing, Leverage fast dense algebra</td> <td>Requires the <a href="http://crd-legacy.lbl.gov/~xiaoye/SuperLU/">SuperLU</a> library, (BSD-like)</td> <td></td></tr> <tr><td>SPQR</td><td>\link SPQRSupport_Module SPQRSupport \endlink </td> <td> QR factorization </td> <td> Any, rectangular</td><td>fill-in reducing, multithreaded, fast dense algebra</td> <td> requires the <a href="http://www.suitesparse.com">SuiteSparse</a> package, \b GPL </td><td>recommended for linear least-squares problems, has a rank-revealing feature</tr> +<tr><td>PardisoLLT \n PardisoLDLT \n PardisoLU</td><td>\link PardisoSupport_Module PardisoSupport \endlink</td><td>Direct LLt, LDLt, LU factorizations</td><td>SPD \n SPD \n Square</td><td>Fill-in reducing, Leverage fast dense algebra, Multithreading</td> + <td>Requires the <a href="http://eigen.tuxfamily.org/Counter/redirect_to_mkl.php">Intel MKL</a> package, \b Proprietary </td> + <td>optimized for tough problems patterns, see also \link TopicUsingIntelMKL using MKL with Eigen \endlink</td></tr> </table> Here \c SPD means symmetric positive definite.
diff --git a/doc/SparseQuickReference.dox b/doc/SparseQuickReference.dox index e0a30ed..81a73ee 100644 --- a/doc/SparseQuickReference.dox +++ b/doc/SparseQuickReference.dox
@@ -80,7 +80,7 @@ \section SparseBasicInfos Matrix properties -Beyond the basic functions rows() and cols(), there are some useful functions that are available to easily get some informations from the matrix. +Beyond the basic functions rows() and cols(), there are some useful functions that are available to easily get some information from the matrix. <table class="manual"> <tr> <td> \code @@ -206,7 +206,7 @@ sm1.innerVectors(start, size); // RW sm1.leftCols(size); // RW sm2.rightCols(size); // RO because sm2 is row-major - sm1.middleRows(start, numRows); // RO becasue sm1 is column-major + sm1.middleRows(start, numRows); // RO because sm1 is column-major sm1.middleCols(start, numCols); // RW sm1.col(j); // RW \endcode @@ -253,6 +253,20 @@ Note that these functions are mostly provided for interoperability purposes with external libraries.\n A better access to the values of the matrix is done by using the InnerIterator class as described in \link TutorialSparse the Tutorial Sparse \endlink section</td> </tr> +<tr class="alt"><td colspan="2">Mapping external buffers</td></tr> +<tr class="alt"> +<td> +\code +int outerIndexPtr[cols+1]; +int innerIndices[nnz]; +double values[nnz]; +Map<SparseMatrix<double> > sm1(rows,cols,nnz,outerIndexPtr, // read-write + innerIndices,values); +Map<const SparseMatrix<double> > sm2(...); // read-only +\endcode +</td> +<td>As for dense matrices, class Map<SparseMatrixType> can be used to see external buffers as an %Eigen's SparseMatrix object. </td> +</tr> </table> */ }
diff --git a/doc/StlContainers.dox b/doc/StlContainers.dox index e0f8714..665a547 100644 --- a/doc/StlContainers.dox +++ b/doc/StlContainers.dox
@@ -6,31 +6,39 @@ \section StlContainers_summary Executive summary -Using STL containers on \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", or classes having members of such types, requires taking the following two steps: +If you're compiling in \cpp17 mode only with a sufficiently recent compiler (e.g., GCC>=7, clang>=5, MSVC>=19.12), then everything is taken care by the compiler and you can stop reading. -\li A 16-byte-aligned allocator must be used. Eigen does provide one ready for use: aligned_allocator. -\li If you want to use the std::vector container, you need to \#include <Eigen/StdVector>. +Otherwise, using STL containers on \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", or classes having members of such types, requires the use of an over-aligned allocator. +That is, an allocator capable of allocating buffers with 16, 32, or even 64 bytes alignment. +%Eigen does provide one ready for use: aligned_allocator. -These issues arise only with \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types" and \ref TopicStructHavingEigenMembers "structures having such Eigen objects as member". For other Eigen types, such as Vector3f or MatrixXd, no special care is needed when using STL containers. +Prior to \cpp11, if you want to use the `std::vector` container, then you also have to `#include <Eigen/StdVector>`. + +These issues arise only with \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types" and \ref TopicStructHavingEigenMembers "structures having such Eigen objects as member". +For other %Eigen types, such as Vector3f or MatrixXd, no special care is needed when using STL containers. \section allocator Using an aligned allocator -STL containers take an optional template parameter, the allocator type. When using STL containers on \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", you need tell the container to use an allocator that will always allocate memory at 16-byte-aligned locations. Fortunately, Eigen does provide such an allocator: Eigen::aligned_allocator. +STL containers take an optional template parameter, the allocator type. When using STL containers on \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", you need tell the container to use an allocator that will always allocate memory at 16-byte-aligned (or more) locations. Fortunately, %Eigen does provide such an allocator: Eigen::aligned_allocator. For example, instead of \code -std::map<int, Eigen::Vector4f> +std::map<int, Eigen::Vector4d> \endcode you need to use \code -std::map<int, Eigen::Vector4f, std::less<int>, - Eigen::aligned_allocator<std::pair<const int, Eigen::Vector4f> > > +std::map<int, Eigen::Vector4d, std::less<int>, + Eigen::aligned_allocator<std::pair<const int, Eigen::Vector4d> > > \endcode -Note that the third parameter "std::less<int>" is just the default value, but we have to include it because we want to specify the fourth parameter, which is the allocator type. +Note that the third parameter `std::less<int>` is just the default value, but we have to include it because we want to specify the fourth parameter, which is the allocator type. \section StlContainers_vector The case of std::vector -The situation with std::vector was even worse (explanation below) so we had to specialize it for the Eigen::aligned_allocator type. In practice you \b must use the Eigen::aligned_allocator (not another aligned allocator), \b and \#include <Eigen/StdVector>. +This section is for c++98/03 users only. \cpp11 (or above) users can stop reading here. + +So in c++98/03, the situation with `std::vector` is more complicated because of a bug in the standard (explanation below). +To workaround the issue, we had to specialize it for the Eigen::aligned_allocator type. +In practice you \b must use the Eigen::aligned_allocator (not another aligned allocator), \b and \#include <Eigen/StdVector>. Here is an example: \code @@ -39,12 +47,16 @@ std::vector<Eigen::Vector4f,Eigen::aligned_allocator<Eigen::Vector4f> > \endcode +<span class="note">\b Explanation: The `resize()` method of `std::vector` takes a `value_type` argument (defaulting to `value_type()`). So with `std::vector<Eigen::Vector4d>`, some Eigen::Vector4d objects will be passed by value, which discards any alignment modifiers, so a Eigen::Vector4d can be created at an unaligned location. +In order to avoid that, the only solution we saw was to specialize `std::vector` to make it work on a slight modification of, here, Eigen::Vector4d, that is able to deal properly with this situation. +</span> + \subsection vector_spec An alternative - specializing std::vector for Eigen types As an alternative to the recommended approach described above, you have the option to specialize std::vector for Eigen types requiring alignment. -The advantage is that you won't need to declare std::vector all over with Eigen::allocator. One drawback on the other hand side is that -the specialization needs to be defined before all code pieces in which e.g. std::vector<Vector2d> is used. Otherwise, without knowing the specialization -the compiler will compile that particular instance with the default std::allocator and you program is most likely to crash. +The advantage is that you won't need to declare std::vector all over with Eigen::aligned_allocator. One drawback on the other hand side is that +the specialization needs to be defined before all code pieces in which e.g. `std::vector<Vector2d>` is used. Otherwise, without knowing the specialization +the compiler will compile that particular instance with the default `std::allocator` and you program is most likely to crash. Here is an example: \code @@ -54,8 +66,7 @@ std::vector<Eigen::Vector2d> \endcode -<span class="note">\b Explanation: The resize() method of std::vector takes a value_type argument (defaulting to value_type()). So with std::vector<Eigen::Vector4f>, some Eigen::Vector4f objects will be passed by value, which discards any alignment modifiers, so a Eigen::Vector4f can be created at an unaligned location. In order to avoid that, the only solution we saw was to specialize std::vector to make it work on a slight modification of, here, Eigen::Vector4f, that is able to deal properly with this situation. -</span> + */
diff --git a/doc/StructHavingEigenMembers.dox b/doc/StructHavingEigenMembers.dox index 7fbed0e..87016cd 100644 --- a/doc/StructHavingEigenMembers.dox +++ b/doc/StructHavingEigenMembers.dox
@@ -6,7 +6,12 @@ \section StructHavingEigenMembers_summary Executive Summary -If you define a structure having members of \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", you must overload its "operator new" so that it generates 16-bytes-aligned pointers. Fortunately, %Eigen provides you with a macro EIGEN_MAKE_ALIGNED_OPERATOR_NEW that does that for you. + +If you define a structure having members of \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types", you must ensure that calling operator new on it allocates properly aligned buffers. +If you're compiling in \cpp17 mode only with a sufficiently recent compiler (e.g., GCC>=7, clang>=5, MSVC>=19.12), then everything is taken care by the compiler and you can stop reading. + +Otherwise, you have to overload its `operator new` so that it generates properly aligned pointers (e.g., 32-bytes-aligned for Vector4d and AVX). +Fortunately, %Eigen provides you with a macro `EIGEN_MAKE_ALIGNED_OPERATOR_NEW` that does that for you. \section StructHavingEigenMembers_what What kind of code needs to be changed? @@ -29,13 +34,13 @@ \section StructHavingEigenMembers_how How should such code be modified? -Very easy, you just need to put a EIGEN_MAKE_ALIGNED_OPERATOR_NEW macro in a public part of your class, like this: +Very easy, you just need to put a `EIGEN_MAKE_ALIGNED_OPERATOR_NEW` macro in a public part of your class, like this: \code class Foo { ... - Eigen::Vector2d v; + Eigen::Vector4d v; ... public: EIGEN_MAKE_ALIGNED_OPERATOR_NEW @@ -46,7 +51,9 @@ Foo *foo = new Foo; \endcode -This macro makes "new Foo" always return an aligned pointer. +This macro makes `new Foo` always return an aligned pointer. + +In \cpp17, this macro is empty. If this approach is too intrusive, see also the \ref StructHavingEigenMembers_othersolutions "other solutions". @@ -58,7 +65,7 @@ class Foo { ... - Eigen::Vector2d v; + Eigen::Vector4d v; ... }; @@ -67,45 +74,59 @@ Foo *foo = new Foo; \endcode -A Eigen::Vector2d consists of 2 doubles, which is 128 bits. Which is exactly the size of a SSE packet, which makes it possible to use SSE for all sorts of operations on this vector. But SSE instructions (at least the ones that %Eigen uses, which are the fast ones) require 128-bit alignment. Otherwise you get a segmentation fault. +A Eigen::Vector4d consists of 4 doubles, which is 256 bits. +This is exactly the size of an AVX register, which makes it possible to use AVX for all sorts of operations on this vector. +But AVX instructions (at least the ones that %Eigen uses, which are the fast ones) require 256-bit alignment. +Otherwise you get a segmentation fault. -For this reason, Eigen takes care by itself to require 128-bit alignment for Eigen::Vector2d, by doing two things: -\li Eigen requires 128-bit alignment for the Eigen::Vector2d's array (of 2 doubles). With GCC, this is done with a __attribute__ ((aligned(16))). -\li Eigen overloads the "operator new" of Eigen::Vector2d so it will always return 128-bit aligned pointers. +For this reason, %Eigen takes care by itself to require 256-bit alignment for Eigen::Vector4d, by doing two things: +\li %Eigen requires 256-bit alignment for the Eigen::Vector4d's array (of 4 doubles). With \cpp11 this is done with the <a href="https://en.cppreference.com/w/cpp/keyword/alignas">alignas</a> keyword, or compiler's extensions for c++98/03. +\li %Eigen overloads the `operator new` of Eigen::Vector4d so it will always return 256-bit aligned pointers. (removed in \cpp17) -Thus, normally, you don't have to worry about anything, Eigen handles alignment for you... +Thus, normally, you don't have to worry about anything, %Eigen handles alignment of operator new for you... -... except in one case. When you have a class Foo like above, and you dynamically allocate a new Foo as above, then, since Foo doesn't have aligned "operator new", the returned pointer foo is not necessarily 128-bit aligned. +... except in one case. When you have a `class Foo` like above, and you dynamically allocate a new `Foo` as above, then, since `Foo` doesn't have aligned `operator new`, the returned pointer foo is not necessarily 256-bit aligned. -The alignment attribute of the member v is then relative to the start of the class, foo. If the foo pointer wasn't aligned, then foo->v won't be aligned either! +The alignment attribute of the member `v` is then relative to the start of the class `Foo`. If the `foo` pointer wasn't aligned, then `foo->v` won't be aligned either! -The solution is to let class Foo have an aligned "operator new", as we showed in the previous section. +The solution is to let `class Foo` have an aligned `operator new`, as we showed in the previous section. + +This explanation also holds for SSE/NEON/MSA/Altivec/VSX targets, which require 16-bytes alignment, and AVX512 which requires 64-bytes alignment for fixed-size objects multiple of 64 bytes (e.g., Eigen::Matrix4d). \section StructHavingEigenMembers_movetotop Should I then put all the members of Eigen types at the beginning of my class? -That's not required. Since Eigen takes care of declaring 128-bit alignment, all members that need it are automatically 128-bit aligned relatively to the class. So code like this works fine: +That's not required. Since %Eigen takes care of declaring adequate alignment, all members that need it are automatically aligned relatively to the class. So code like this works fine: \code class Foo { double x; - Eigen::Vector2d v; + Eigen::Vector4d v; public: EIGEN_MAKE_ALIGNED_OPERATOR_NEW }; \endcode +That said, as usual, it is recommended to sort the members so that alignment does not waste memory. +In the above example, with AVX, the compiler will have to reserve 24 empty bytes between `x` and `v`. + + \section StructHavingEigenMembers_dynamicsize What about dynamic-size matrices and vectors? Dynamic-size matrices and vectors, such as Eigen::VectorXd, allocate dynamically their own array of coefficients, so they take care of requiring absolute alignment automatically. So they don't cause this issue. The issue discussed here is only with \ref TopicFixedSizeVectorizable "fixed-size vectorizable matrices and vectors". + \section StructHavingEigenMembers_bugineigen So is this a bug in Eigen? -No, it's not our bug. It's more like an inherent problem of the C++98 language specification, and seems to be taken care of in the upcoming language revision: <a href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2007/n2341.pdf">see this document</a>. +No, it's not our bug. It's more like an inherent problem of the c++ language specification that has been solved in c++17 through the feature known as <a href="http://wg21.link/p0035r4">dynamic memory allocation for over-aligned data</a>. -\section StructHavingEigenMembers_conditional What if I want to do this conditionnally (depending on template parameters) ? -For this situation, we offer the macro EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign). It will generate aligned operators like EIGEN_MAKE_ALIGNED_OPERATOR_NEW if NeedsToAlign is true. It will generate operators with the default alignment if NeedsToAlign is false. +\section StructHavingEigenMembers_conditional What if I want to do this conditionally (depending on template parameters) ? + +For this situation, we offer the macro `EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)`. +It will generate aligned operators like `EIGEN_MAKE_ALIGNED_OPERATOR_NEW` if `NeedsToAlign` is true. +It will generate operators with the default alignment if `NeedsToAlign` is false. +In \cpp17, this macro is empty. Example: @@ -130,7 +151,7 @@ \section StructHavingEigenMembers_othersolutions Other solutions -In case putting the EIGEN_MAKE_ALIGNED_OPERATOR_NEW macro everywhere is too intrusive, there exists at least two other solutions. +In case putting the `EIGEN_MAKE_ALIGNED_OPERATOR_NEW` macro everywhere is too intrusive, there exists at least two other solutions. \subsection othersolutions1 Disabling alignment @@ -139,22 +160,13 @@ class Foo { ... - Eigen::Matrix<double,2,1,Eigen::DontAlign> v; + Eigen::Matrix<double,4,1,Eigen::DontAlign> v; ... }; \endcode -This has for effect to disable vectorization when using \c v. -If a function of Foo uses it several times, then it still possible to re-enable vectorization by copying it into an aligned temporary vector: -\code -void Foo::bar() -{ - Eigen::Vector2d av(v); - // use av instead of v - ... - // if av changed, then do: - v = av; -} -\endcode +This `v` is fully compatible with aligned Eigen::Vector4d. +This has only for effect to make load/stores to `v` more expensive (usually slightly, but that's hardware dependent). + \subsection othersolutions2 Private structure @@ -164,7 +176,7 @@ struct Foo_d { EIGEN_MAKE_ALIGNED_OPERATOR_NEW - Vector2d v; + Vector4d v; ... }; @@ -183,7 +195,8 @@ }; \endcode -The clear advantage here is that the class Foo remains unchanged regarding alignment issues. The drawback is that a heap allocation will be required whatsoever. +The clear advantage here is that the class `Foo` remains unchanged regarding alignment issues. +The drawback is that an additional heap allocation will be required whatsoever. */
diff --git a/doc/TemplateKeyword.dox b/doc/TemplateKeyword.dox index b84cfda..fbf2c70 100644 --- a/doc/TemplateKeyword.dox +++ b/doc/TemplateKeyword.dox
@@ -76,7 +76,7 @@ and \c Derived2 in the example). That means that the compiler cannot know that <tt>dst.triangularView</tt> is a member template and that the following < symbol is part of the delimiter for the template parameter. Another possibility would be that <tt>dst.triangularView</tt> is a member variable with the < -symbol refering to the <tt>operator<()</tt> function. In fact, the compiler should choose the second +symbol referring to the <tt>operator<()</tt> function. In fact, the compiler should choose the second possibility, according to the standard. If <tt>dst.triangularView</tt> is a member template (as in our case), the programmer should specify this explicitly with the \c template keyword and write <tt>dst.template triangularView</tt>.
diff --git a/doc/TopicAssertions.dox b/doc/TopicAssertions.dox index 4ead401..c8b4d84 100644 --- a/doc/TopicAssertions.dox +++ b/doc/TopicAssertions.dox
@@ -16,7 +16,7 @@ #include <stdexcept> #undef eigen_assert #define eigen_assert(x) \ - if (!x) { throw (std::runtime_error("Put your message here")); } + if (!(x)) { throw (std::runtime_error("Put your message here")); } \endcode \subsection DisableAssert Disabling assertions
diff --git a/doc/TopicCMakeGuide.dox b/doc/TopicCMakeGuide.dox new file mode 100644 index 0000000..cf767d0 --- /dev/null +++ b/doc/TopicCMakeGuide.dox
@@ -0,0 +1,56 @@ +namespace Eigen { + +/** + +\page TopicCMakeGuide Using %Eigen in CMake Projects + +%Eigen provides native CMake support which allows the library to be easily +used in CMake projects. + +\note %CMake 3.0 (or later) is required to enable this functionality. + +%Eigen exports a CMake target called `Eigen3::Eigen` which can be imported +using the `find_package` CMake command and used by calling +`target_link_libraries` as in the following example: +\code{.cmake} +cmake_minimum_required (VERSION 3.0) +project (myproject) + +find_package (Eigen3 3.3 REQUIRED NO_MODULE) + +add_executable (example example.cpp) +target_link_libraries (example Eigen3::Eigen) +\endcode + +The above code snippet must be placed in a file called `CMakeLists.txt` alongside +`example.cpp`. After running +\code{.sh} +$ cmake path-to-example-directory +\endcode +CMake will produce project files that generate an executable called `example` +which requires at least version 3.3 of %Eigen. Here, `path-to-example-directory` +is the path to the directory that contains both `CMakeLists.txt` and +`example.cpp`. + +Do not forget to set the <a href="https://cmake.org/cmake/help/v3.7/variable/CMAKE_PREFIX_PATH.html">\c CMAKE_PREFIX_PATH </a> variable if Eigen is not installed in a default location or if you want to pick a specific version. For instance: +\code{.sh} +$ cmake path-to-example-directory -DCMAKE_PREFIX_PATH=$HOME/mypackages +\endcode +An alternative is to set the \c Eigen3_DIR cmake's variable to the respective path containing the \c Eigen3*.cmake files. For instance: +\code{.sh} +$ cmake path-to-example-directory -DEigen3_DIR=$HOME/mypackages/share/eigen3/cmake/ +\endcode + +If the `REQUIRED` option is omitted when locating %Eigen using +`find_package`, one can check whether the package was found as follows: +\code{.cmake} +find_package (Eigen3 3.3 NO_MODULE) + +if (TARGET Eigen3::Eigen) + # Use the imported target +endif (TARGET Eigen3::Eigen) +\endcode + +*/ + +}
diff --git a/doc/TopicLazyEvaluation.dox b/doc/TopicLazyEvaluation.dox index 101ef8c..d2a704f 100644 --- a/doc/TopicLazyEvaluation.dox +++ b/doc/TopicLazyEvaluation.dox
@@ -2,63 +2,95 @@ /** \page TopicLazyEvaluation Lazy Evaluation and Aliasing -Executive summary: Eigen has intelligent compile-time mechanisms to enable lazy evaluation and removing temporaries where appropriate. +Executive summary: %Eigen has intelligent compile-time mechanisms to enable lazy evaluation and removing temporaries where appropriate. It will handle aliasing automatically in most cases, for example with matrix products. The automatic behavior can be overridden manually by using the MatrixBase::eval() and MatrixBase::noalias() methods. When you write a line of code involving a complex expression such as -\code mat1 = mat2 + mat3 * (mat4 + mat5); \endcode +\code mat1 = mat2 + mat3 * (mat4 + mat5); +\endcode -Eigen determines automatically, for each sub-expression, whether to evaluate it into a temporary variable. Indeed, in certain cases it is better to evaluate immediately a sub-expression into a temporary variable, while in other cases it is better to avoid that. +%Eigen determines automatically, for each sub-expression, whether to evaluate it into a temporary variable. Indeed, in certain cases it is better to evaluate a sub-expression into a temporary variable, while in other cases it is better to avoid that. A traditional math library without expression templates always evaluates all sub-expressions into temporaries. So with this code, -\code vec1 = vec2 + vec3; \endcode +\code vec1 = vec2 + vec3; +\endcode a traditional library would evaluate \c vec2 + vec3 into a temporary \c vec4 and then copy \c vec4 into \c vec1. This is of course inefficient: the arrays are traversed twice, so there are a lot of useless load/store operations. -Expression-templates-based libraries can avoid evaluating sub-expressions into temporaries, which in many cases results in large speed improvements. This is called <i>lazy evaluation</i> as an expression is getting evaluated as late as possible, instead of immediately. However, most other expression-templates-based libraries <i>always</i> choose lazy evaluation. There are two problems with that: first, lazy evaluation is not always a good choice for performance; second, lazy evaluation can be very dangerous, for example with matrix products: doing <tt>matrix = matrix*matrix</tt> gives a wrong result if the matrix product is lazy-evaluated, because of the way matrix product works. +Expression-templates-based libraries can avoid evaluating sub-expressions into temporaries, which in many cases results in large speed improvements. +This is called <i>lazy evaluation</i> as an expression is getting evaluated as late as possible. +In %Eigen <b>all expressions are lazy-evaluated</b>. +More precisely, an expression starts to be evaluated once it is assigned to a matrix. +Until then nothing happens beyond constructing the abstract expression tree. +In contrast to most other expression-templates-based libraries, however, <b>%Eigen might choose to evaluate some sub-expressions into temporaries</b>. +There are two reasons for that: first, pure lazy evaluation is not always a good choice for performance; second, pure lazy evaluation can be very dangerous, for example with matrix products: doing <tt>mat = mat*mat</tt> gives a wrong result if the matrix product is directly evaluated within the destination matrix, because of the way matrix product works. -For these reasons, Eigen has intelligent compile-time mechanisms to determine automatically when to use lazy evaluation, and when on the contrary it should evaluate immediately into a temporary variable. +For these reasons, %Eigen has intelligent compile-time mechanisms to determine automatically which sub-expression should be evaluated into a temporary variable. So in the basic example, -\code matrix1 = matrix2 + matrix3; \endcode +\code mat1 = mat2 + mat3; +\endcode -Eigen chooses lazy evaluation. Thus the arrays are traversed only once, producing optimized code. If you really want to force immediate evaluation, use \link MatrixBase::eval() eval()\endlink: +%Eigen chooses not to introduce any temporary. Thus the arrays are traversed only once, producing optimized code. +If you really want to force immediate evaluation, use \link MatrixBase::eval() eval()\endlink: -\code matrix1 = (matrix2 + matrix3).eval(); \endcode +\code mat1 = (mat2 + mat3).eval(); +\endcode Here is now a more involved example: -\code matrix1 = -matrix2 + matrix3 + 5 * matrix4; \endcode +\code mat1 = -mat2 + mat3 + 5 * mat4; +\endcode -Eigen chooses lazy evaluation at every stage in that example, which is clearly the correct choice. In fact, lazy evaluation is the "default choice" and Eigen will choose it except in a few circumstances. +Here again %Eigen won't introduce any temporary, thus producing a single <b>fused</b> evaluation loop, which is clearly the correct choice. -<b>The first circumstance</b> in which Eigen chooses immediate evaluation, is when it sees an assignment <tt>a = b;</tt> and the expression \c b has the evaluate-before-assigning \link flags flag\endlink. The most important example of such an expression is the \link Product matrix product expression\endlink. For example, when you do +\section TopicLazyEvaluationWhichExpr Which sub-expressions are evaluated into temporaries? -\code matrix = matrix * matrix; \endcode +The default evaluation strategy is to fuse the operations in a single loop, and %Eigen will choose it except in a few circumstances. -Eigen first evaluates <tt>matrix * matrix</tt> into a temporary matrix, and then copies it into the original \c matrix. This guarantees a correct result as we saw above that lazy evaluation gives wrong results with matrix products. It also doesn't cost much, as the cost of the matrix product itself is much higher. +<b>The first circumstance</b> in which %Eigen chooses to evaluate a sub-expression is when it sees an assignment <tt>a = b;</tt> and the expression \c b has the evaluate-before-assigning \link flags flag\endlink. +The most important example of such an expression is the \link Product matrix product expression\endlink. For example, when you do + +\code mat = mat * mat; +\endcode + +%Eigen will evaluate <tt>mat * mat</tt> into a temporary matrix, and then copies it into the original \c mat. +This guarantees a correct result as we saw above that lazy evaluation gives wrong results with matrix products. +It also doesn't cost much, as the cost of the matrix product itself is much higher. +Note that this temporary is introduced at evaluation time only, that is, within operator= in this example. +The expression <tt>mat * mat</tt> still return a abstract product type. What if you know that the result does no alias the operand of the product and want to force lazy evaluation? Then use \link MatrixBase::noalias() .noalias()\endlink instead. Here is an example: -\code matrix1.noalias() = matrix2 * matrix2; \endcode +\code mat1.noalias() = mat2 * mat2; +\endcode -Here, since we know that matrix2 is not the same matrix as matrix1, we know that lazy evaluation is not dangerous, so we may force lazy evaluation. Concretely, the effect of noalias() here is to bypass the evaluate-before-assigning \link flags flag\endlink. +Here, since we know that mat2 is not the same matrix as mat1, we know that lazy evaluation is not dangerous, so we may force lazy evaluation. Concretely, the effect of noalias() here is to bypass the evaluate-before-assigning \link flags flag\endlink. -<b>The second circumstance</b> in which Eigen chooses immediate evaluation, is when it sees a nested expression such as <tt>a + b</tt> where \c b is already an expression having the evaluate-before-nesting \link flags flag\endlink. Again, the most important example of such an expression is the \link Product matrix product expression\endlink. For example, when you do +<b>The second circumstance</b> in which %Eigen chooses to evaluate a sub-expression, is when it sees a nested expression such as <tt>a + b</tt> where \c b is already an expression having the evaluate-before-nesting \link flags flag\endlink. +Again, the most important example of such an expression is the \link Product matrix product expression\endlink. +For example, when you do -\code matrix1 = matrix2 + matrix3 * matrix4; \endcode +\code mat1 = mat2 * mat3 + mat4 * mat5; +\endcode -the product <tt>matrix3 * matrix4</tt> gets evaluated immediately into a temporary matrix. Indeed, experiments showed that it is often beneficial for performance to evaluate immediately matrix products when they are nested into bigger expressions. +the products <tt>mat2 * mat3</tt> and <tt>mat4 * mat5</tt> gets evaluated separately into temporary matrices before being summed up in <tt>mat1</tt>. +Indeed, to be efficient matrix products need to be evaluated within a destination matrix at hand, and not as simple "dot products". +For small matrices, however, you might want to enforce a "dot-product" based lazy evaluation with lazyProduct(). +Again, it is important to understand that those temporaries are created at evaluation time only, that is in operator =. +See TopicPitfalls_auto_keyword for common pitfalls regarding this remark. -<b>The third circumstance</b> in which Eigen chooses immediate evaluation, is when its cost model shows that the total cost of an operation is reduced if a sub-expression gets evaluated into a temporary. Indeed, in certain cases, an intermediate result is sufficiently costly to compute and is reused sufficiently many times, that is worth "caching". Here is an example: +<b>The third circumstance</b> in which %Eigen chooses to evaluate a sub-expression, is when its cost model shows that the total cost of an operation is reduced if a sub-expression gets evaluated into a temporary. +Indeed, in certain cases, an intermediate result is sufficiently costly to compute and is reused sufficiently many times, that is worth "caching". Here is an example: -\code matrix1 = matrix2 * (matrix3 + matrix4); \endcode +\code mat1 = mat2 * (mat3 + mat4); +\endcode -Here, provided the matrices have at least 2 rows and 2 columns, each coefficienct of the expression <tt>matrix3 + matrix4</tt> is going to be used several times in the matrix product. Instead of computing the sum everytime, it is much better to compute it once and store it in a temporary variable. Eigen understands this and evaluates <tt>matrix3 + matrix4</tt> into a temporary variable before evaluating the product. +Here, provided the matrices have at least 2 rows and 2 columns, each coefficient of the expression <tt>mat3 + mat4</tt> is going to be used several times in the matrix product. Instead of computing the sum every time, it is much better to compute it once and store it in a temporary variable. %Eigen understands this and evaluates <tt>mat3 + mat4</tt> into a temporary variable before evaluating the product. */
diff --git a/doc/TopicLinearAlgebraDecompositions.dox b/doc/TopicLinearAlgebraDecompositions.dox index 5bcff2c..0965da8 100644 --- a/doc/TopicLinearAlgebraDecompositions.dox +++ b/doc/TopicLinearAlgebraDecompositions.dox
@@ -4,6 +4,7 @@ This page presents a catalogue of the dense matrix decompositions offered by Eigen. For an introduction on linear solvers and decompositions, check this \link TutorialLinearAlgebra page \endlink. +To get an overview of the true relative speed of the different decompositions, check this \link DenseDecompositionBenchmark benchmark \endlink. \section TopicLinAlgBigTable Catalogue of decompositions offered by Eigen @@ -113,6 +114,18 @@ <tr><th class="inter" colspan="9">\n Singular values and eigenvalues decompositions</th></tr> <tr> + <td>BDCSVD (divide \& conquer)</td> + <td>-</td> + <td>One of the fastest SVD algorithms</td> + <td>Excellent</td> + <td>Yes</td> + <td>Singular values/vectors, least squares</td> + <td>Yes (and does least squares)</td> + <td>Excellent</td> + <td>Blocked bidiagonalization</td> + </tr> + + <tr> <td>JacobiSVD (two-sided)</td> <td>-</td> <td>Slow (but fast for small matrices)</td> @@ -247,7 +260,7 @@ <dt><b>Blocking</b></dt> <dd>Means the algorithm can work per block, whence guaranteeing a good scaling of the performance for large matrices.</dd> <dt><b>Implicit Multi Threading (MT)</b></dt> - <dd>Means the algorithm can take advantage of multicore processors via OpenMP. "Implicit" means the algortihm itself is not parallelized, but that it relies on parallelized matrix-matrix product rountines.</dd> + <dd>Means the algorithm can take advantage of multicore processors via OpenMP. "Implicit" means the algortihm itself is not parallelized, but that it relies on parallelized matrix-matrix product routines.</dd> <dt><b>Explicit Multi Threading (MT)</b></dt> <dd>Means the algorithm is explicitly parallelized to take advantage of multicore processors via OpenMP.</dd> <dt><b>Meta-unroller</b></dt> @@ -256,6 +269,7 @@ <dd></dd> </dl> + */ }
diff --git a/doc/TopicMultithreading.dox b/doc/TopicMultithreading.dox index 47c9b26..97190b5 100644 --- a/doc/TopicMultithreading.dox +++ b/doc/TopicMultithreading.dox
@@ -4,22 +4,25 @@ \section TopicMultiThreading_MakingEigenMT Make Eigen run in parallel -Some Eigen's algorithms can exploit the multiple cores present in your hardware. To this end, it is enough to enable OpenMP on your compiler, for instance: - * GCC: \c -fopenmp - * ICC: \c -openmp - * MSVC: check the respective option in the build properties. -You can control the number of thread that will be used using either the OpenMP API or Eigen's API using the following priority: +Some %Eigen's algorithms can exploit the multiple cores present in your hardware. +To this end, it is enough to enable OpenMP on your compiler, for instance: + - GCC: \c -fopenmp + - ICC: \c -openmp + - MSVC: check the respective option in the build properties. + +You can control the number of threads that will be used using either the OpenMP API or %Eigen's API using the following priority: \code OMP_NUM_THREADS=n ./my_program omp_set_num_threads(n); Eigen::setNbThreads(n); \endcode -Unless setNbThreads has been called, Eigen uses the number of threads specified by OpenMP. You can restore this behavior by calling \code setNbThreads(0); \endcode +Unless `setNbThreads` has been called, %Eigen uses the number of threads specified by OpenMP. +You can restore this behavior by calling `setNbThreads(0);`. You can query the number of threads that will be used with: \code n = Eigen::nbThreads( ); \endcode -You can disable Eigen's multi threading at compile time by defining the EIGEN_DONT_PARALLELIZE preprocessor token. +You can disable %Eigen's multi threading at compile time by defining the \link TopicPreprocessorDirectivesPerformance EIGEN_DONT_PARALLELIZE \endlink preprocessor token. Currently, the following algorithms can make use of multi-threading: - general dense matrix - matrix products @@ -29,9 +32,17 @@ - BiCGSTAB with a row-major sparse matrix format. - LeastSquaresConjugateGradient +\warning On most OS it is <strong>very important</strong> to limit the number of threads to the number of physical cores, otherwise significant slowdowns are expected, especially for operations involving dense matrices. + +Indeed, the principle of hyper-threading is to run multiple threads (in most cases 2) on a single core in an interleaved manner. +However, %Eigen's matrix-matrix product kernel is fully optimized and already exploits nearly 100% of the CPU capacity. +Consequently, there is no room for running multiple such threads on a single core, and the performance would drops significantly because of cache pollution and other sources of overheads. +At this stage of reading you're probably wondering why %Eigen does not limit itself to the number of physical cores? +This is simply because OpenMP does not allow to know the number of physical cores, and thus %Eigen will launch as many threads as <i>cores</i> reported by OpenMP. + \section TopicMultiThreading_UsingEigenWithMT Using Eigen in a multi-threaded application -In the case your own application is multithreaded, and multiple threads make calls to Eigen, then you have to initialize Eigen by calling the following routine \b before creating the threads: +In the case your own application is multithreaded, and multiple threads make calls to %Eigen, then you have to initialize %Eigen by calling the following routine \b before creating the threads: \code #include <Eigen/Core> @@ -43,11 +54,12 @@ } \endcode -\note With Eigen 3.3, and a fully C++11 compliant compiler (i.e., <a href="http://en.cppreference.com/w/cpp/language/storage_duration#Static_local_variables">thread-safe static local variable initialization</a>), then calling \c initParallel() is optional. +\note With %Eigen 3.3, and a fully C++11 compliant compiler (i.e., <a href="http://en.cppreference.com/w/cpp/language/storage_duration#Static_local_variables">thread-safe static local variable initialization</a>), then calling \c initParallel() is optional. -\warning note that all functions generating random matrices are \b not re-entrant nor thread-safe. Those include DenseBase::Random(), and DenseBase::setRandom() despite a call to Eigen::initParallel(). This is because these functions are based on std::rand which is not re-entrant. For thread-safe random generator, we recommend the use of boost::random or c++11 random feature. +\warning Note that all functions generating random matrices are \b not re-entrant nor thread-safe. Those include DenseBase::Random(), and DenseBase::setRandom() despite a call to `Eigen::initParallel()`. This is because these functions are based on `std::rand` which is not re-entrant. +For thread-safe random generator, we recommend the use of c++11 random generators (\link DenseBase::NullaryExpr(Index, const CustomNullaryOp&) example \endlink) or `boost::random`. -In the case your application is parallelized with OpenMP, you might want to disable Eigen's own parallization as detailed in the previous section. +In the case your application is parallelized with OpenMP, you might want to disable %Eigen's own parallelization as detailed in the previous section. */
diff --git a/doc/TutorialGeometry.dox b/doc/TutorialGeometry.dox index 2e1420f..86e9f1e 100644 --- a/doc/TutorialGeometry.dox +++ b/doc/TutorialGeometry.dox
@@ -111,7 +111,7 @@ <a href="#" class="top">top</a>\section TutorialGeoTransform Affine transformations -Generic affine transformations are represented by the Transform class which internaly +Generic affine transformations are represented by the Transform class which internally is a (Dim+1)^2 matrix. In Eigen we have chosen to not distinghish between points and vectors such that all points are actually represented by displacement vectors from the origin ( \f$ \mathbf{p} \equiv \mathbf{p}-0 \f$ ). With that in mind, real points and
diff --git a/doc/TutorialLinearAlgebra.dox b/doc/TutorialLinearAlgebra.dox index cb92cee..a727241 100644 --- a/doc/TutorialLinearAlgebra.dox +++ b/doc/TutorialLinearAlgebra.dox
@@ -73,7 +73,7 @@ <td>ColPivHouseholderQR</td> <td>colPivHouseholderQr()</td> <td>None</td> - <td>++</td> + <td>+</td> <td>-</td> <td>+++</td> </tr> @@ -86,6 +86,14 @@ <td>+++</td> </tr> <tr class="alt"> + <td>CompleteOrthogonalDecomposition</td> + <td>completeOrthogonalDecomposition()</td> + <td>None</td> + <td>+</td> + <td>-</td> + <td>+++</td> + </tr> + <tr class="alt"> <td>LLT</td> <td>llt()</td> <td>Positive definite</td> @@ -102,14 +110,23 @@ <td>++</td> </tr> <tr class="alt"> + <td>BDCSVD</td> + <td>bdcSvd()</td> + <td>None</td> + <td>-</td> + <td>-</td> + <td>+++</td> + </tr> + <tr class="alt"> <td>JacobiSVD</td> <td>jacobiSvd()</td> <td>None</td> - <td>- -</td> + <td>-</td> <td>- - -</td> <td>+++</td> </tr> </table> +To get an overview of the true relative speed of the different decompositions, check this \link DenseDecompositionBenchmark benchmark \endlink. All of these decompositions offer a solve() method that works as in the above example. @@ -183,8 +200,11 @@ \section TutorialLinAlgLeastsquares Least squares solving -The most accurate method to do least squares solving is with a SVD decomposition. Eigen provides one -as the JacobiSVD class, and its solve() is doing least-squares solving. +The most accurate method to do least squares solving is with a SVD decomposition. +Eigen provides two implementations. +The recommended one is the BDCSVD class, which scale well for large problems +and automatically fall-back to the JacobiSVD class for smaller problems. +For both classes, their solve() method is doing least-squares solving. Here is an example: <table class="example">
diff --git a/doc/TutorialMapClass.dox b/doc/TutorialMapClass.dox index f8fb0fd..caa2539 100644 --- a/doc/TutorialMapClass.dox +++ b/doc/TutorialMapClass.dox
@@ -29,9 +29,9 @@ \endcode where \c pi is an \c int \c *. In this case the size does not have to be passed to the constructor, because it is already specified by the Matrix/Array type. -Note that Map does not have a default constructor; you \em must pass a pointer to intialize the object. However, you can work around this requirement (see \ref TutorialMapPlacementNew). +Note that Map does not have a default constructor; you \em must pass a pointer to initialize the object. However, you can work around this requirement (see \ref TutorialMapPlacementNew). -Map is flexible enough to accomodate a variety of different data representations. There are two other (optional) template parameters: +Map is flexible enough to accommodate a variety of different data representations. There are two other (optional) template parameters: \code Map<typename MatrixType, int MapOptions,
diff --git a/doc/TutorialMatrixClass.dox b/doc/TutorialMatrixClass.dox index 7ea0cd7..2c45222 100644 --- a/doc/TutorialMatrixClass.dox +++ b/doc/TutorialMatrixClass.dox
@@ -101,13 +101,41 @@ \endcode and is a no-operation. -Finally, we also offer some constructors to initialize the coefficients of small fixed-size vectors up to size 4: +Matrices and vectors can also be initialized from lists of coefficients. +Prior to C++11, this feature is limited to small fixed-size column or vectors up to size 4: \code Vector2d a(5.0, 6.0); Vector3d b(5.0, 6.0, 7.0); Vector4d c(5.0, 6.0, 7.0, 8.0); \endcode +If C++11 is enabled, fixed-size column or row vectors of arbitrary size can be initialized by passing an arbitrary number of coefficients: +\code +Vector2i a(1, 2); // A column vector containing the elements {1, 2} +Matrix<int, 5, 1> b {1, 2, 3, 4, 5}; // A row-vector containing the elements {1, 2, 3, 4, 5} +Matrix<int, 1, 5> c = {1, 2, 3, 4, 5}; // A column vector containing the elements {1, 2, 3, 4, 5} +\endcode + +In the general case of matrices and vectors with either fixed or runtime sizes, +coefficients have to be grouped by rows and passed as an initializer list of initializer list (\link Matrix::Matrix(const std::initializer_list<std::initializer_list<Scalar>>&) details \endlink): +\code +MatrixXi a { // construct a 2x2 matrix + {1, 2}, // first row + {3, 4} // second row +}; +Matrix<double, 2, 3> b { + {2, 3, 4}, + {5, 6, 7}, +}; +\endcode + +For column or row vectors, implicit transposition is allowed. +This means that a column vector can be initialized from a single row: +\code +VectorXd a {{1.5, 2.5, 3.5}}; // A column-vector with 3 coefficients +RowVectorXd b {{1.0, 2.0, 3.0, 4.0}}; // A row-vector with 4 coefficients +\endcode + \section TutorialMatrixCoeffAccessors Coefficient accessors The primary coefficient accessors and mutators in Eigen are the overloaded parenthesis operators.
diff --git a/doc/TutorialReshape.dox b/doc/TutorialReshape.dox new file mode 100644 index 0000000..5b4022a --- /dev/null +++ b/doc/TutorialReshape.dox
@@ -0,0 +1,82 @@ +namespace Eigen { + +/** \eigenManualPage TutorialReshape Reshape + +Since the version 3.4, %Eigen exposes convenient methods to reshape a matrix to another matrix of different sizes or vector. +All cases are handled via the DenseBase::reshaped(NRowsType,NColsType) and DenseBase::reshaped() functions. +Those functions do not perform in-place reshaping, but instead return a <i> view </i> on the input expression. + +\eigenAutoToc + +\section TutorialReshapeMat2Mat Reshaped 2D views + +The more general reshaping transformation is handled via: `reshaped(nrows,ncols)`. +Here is an example reshaping a 4x4 matrix to a 2x8 one: + +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include MatrixBase_reshaped_int_int.cpp +</td> +<td> +\verbinclude MatrixBase_reshaped_int_int.out +</td></tr></table> + +By default, the input coefficients are always interpreted in column-major order regardless of the storage order of the input expression. +For more control on ordering, compile-time sizes, and automatic size deduction, please see de documentation of DenseBase::reshaped(NRowsType,NColsType) that contains all the details with many examples. + + +\section TutorialReshapeMat2Vec 1D linear views + +A very common usage of reshaping is to create a 1D linear view over a given 2D matrix or expression. +In this case, sizes can be deduced and thus omitted as in the following example: + +<table class="example"> +<tr><th>Example:</th></tr> +<tr><td> +\include MatrixBase_reshaped_to_vector.cpp +</td></tr> +<tr><th>Output:</th></tr> +<tr><td> +\verbinclude MatrixBase_reshaped_to_vector.out +</td></tr></table> + +This shortcut always returns a column vector and by default input coefficients are always interpreted in column-major order. +Again, see the documentation of DenseBase::reshaped() for more control on the ordering. + +\section TutorialReshapeInPlace + +The above examples create reshaped views, but what about reshaping inplace a given matrix? +Of course this task in only conceivable for matrix and arrays having runtime dimensions. +In many cases, this can be accomplished via PlainObjectBase::resize(Index,Index): + +<table class="example"> +<tr><th>Example:</th></tr> +<tr><td> +\include Tutorial_reshaped_vs_resize_1.cpp +</td></tr> +<tr><th>Output:</th></tr> +<tr><td> +\verbinclude Tutorial_reshaped_vs_resize_1.out +</td></tr></table> + +However beware that unlike \c reshaped, the result of \c resize depends on the input storage order. +It thus behaves similarly to `reshaped<AutoOrder>`: + +<table class="example"> +<tr><th>Example:</th></tr> +<tr><td> +\include Tutorial_reshaped_vs_resize_2.cpp +</td></tr> +<tr><th>Output:</th></tr> +<tr><td> +\verbinclude Tutorial_reshaped_vs_resize_2.out +</td></tr></table> + +Finally, assigning a reshaped matrix to itself is currently not supported and will result to undefined-behavior because of \link TopicAliasing aliasing \endlink. +The following is forbidden: \code A = A.reshaped(2,8); \endcode +This is OK: \code A = A.reshaped(2,8).eval(); \endcode + +*/ + +}
diff --git a/doc/TutorialReshapeSlicing.dox b/doc/TutorialReshapeSlicing.dox deleted file mode 100644 index 3730a5d..0000000 --- a/doc/TutorialReshapeSlicing.dox +++ /dev/null
@@ -1,65 +0,0 @@ -namespace Eigen { - -/** \eigenManualPage TutorialReshapeSlicing Reshape and Slicing - -%Eigen does not expose convenient methods to take slices or to reshape a matrix yet. -Nonetheless, such features can easily be emulated using the Map class. - -\eigenAutoToc - -\section TutorialReshape Reshape - -A reshape operation consists in modifying the sizes of a matrix while keeping the same coefficients. -Instead of modifying the input matrix itself, which is not possible for compile-time sizes, the approach consist in creating a different \em view on the storage using class Map. -Here is a typical example creating a 1D linear view of a matrix: - -<table class="example"> -<tr><th>Example:</th><th>Output:</th></tr> -<tr><td> -\include Tutorial_ReshapeMat2Vec.cpp -</td> -<td> -\verbinclude Tutorial_ReshapeMat2Vec.out -</td></tr></table> - -Remark how the storage order of the input matrix modifies the order of the coefficients in the linear view. -Here is another example reshaping a 2x6 matrix to a 6x2 one: -<table class="example"> -<tr><th>Example:</th><th>Output:</th></tr> -<tr><td> -\include Tutorial_ReshapeMat2Mat.cpp -</td> -<td> -\verbinclude Tutorial_ReshapeMat2Mat.out -</td></tr></table> - - - -\section TutorialSlicing Slicing - -Slicing consists in taking a set of rows, columns, or elements, uniformly spaced within a matrix. -Again, the class Map allows to easily mimic this feature. - -For instance, one can skip every P elements in a vector: -<table class="example"> -<tr><th>Example:</th><th>Output:</th></tr> -<tr><td> -\include Tutorial_SlicingVec.cpp -</td> -<td> -\verbinclude Tutorial_SlicingVec.out -</td></tr></table> - -One can olso take one column over three using an adequate outer-stride or inner-stride depending on the actual storage order: -<table class="example"> -<tr><th>Example:</th><th>Output:</th></tr> -<tr><td> -\include Tutorial_SlicingCol.cpp -</td> -<td> -\verbinclude Tutorial_SlicingCol.out -</td></tr></table> - -*/ - -}
diff --git a/doc/TutorialSTL.dox b/doc/TutorialSTL.dox new file mode 100644 index 0000000..9a825bc --- /dev/null +++ b/doc/TutorialSTL.dox
@@ -0,0 +1,66 @@ +namespace Eigen { + +/** \eigenManualPage TutorialSTL STL iterators and algorithms + +Since the version 3.4, %Eigen's dense matrices and arrays provide STL compatible iterators. +As demonstrated below, this makes them naturally compatible with range-for-loops and STL's algorithms. + +\eigenAutoToc + +\section TutorialSTLVectors Iterating over 1D arrays and vectors + +Any dense 1D expressions exposes the pair of `begin()/end()` methods to iterate over them. + +This directly enables c++11 range for loops: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Tutorial_range_for_loop_1d_cxx11.cpp +</td> +<td> +\verbinclude Tutorial_range_for_loop_1d_cxx11.out +</td></tr></table> + +One dimensional expressions can also easily be passed to STL algorithms: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Tutorial_std_sort.cpp +</td> +<td> +\verbinclude Tutorial_std_sort.out +</td></tr></table> + +Similar to `std::vector`, 1D expressions also exposes the pair of `cbegin()/cend()` methods to conveniently get const iterators on non-const object. + +\section TutorialSTLMatrices Iterating over coefficients of 2D arrays and matrices + +STL iterators are intrinsically designed to iterate over 1D structures. +This is why `begin()/end()` methods are disabled for 2D expressions. +Iterating over all coefficients of a 2D expressions is still easily accomplished by creating a 1D linear view through `reshaped()`: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Tutorial_range_for_loop_2d_cxx11.cpp +</td> +<td> +\verbinclude Tutorial_range_for_loop_2d_cxx11.out +</td></tr></table> + +\section TutorialSTLRowsColumns Iterating over rows or columns of 2D arrays and matrices + +It is also possible to get iterators over rows or columns of 2D expressions. +Those are available through the `rowwise()` and `colwise()` proxies. +Here is an example sorting each row of a matrix: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Tutorial_std_sort_rows_cxx11.cpp +</td> +<td> +\verbinclude Tutorial_std_sort_rows_cxx11.out +</td></tr></table> + +*/ + +}
diff --git a/doc/TutorialSlicingIndexing.dox b/doc/TutorialSlicingIndexing.dox new file mode 100644 index 0000000..98ace43 --- /dev/null +++ b/doc/TutorialSlicingIndexing.dox
@@ -0,0 +1,244 @@ +namespace Eigen { + +/** \eigenManualPage TutorialSlicingIndexing Slicing and Indexing + +This page presents the numerous possibilities offered by `operator()` to index sub-set of rows and columns. +This API has been introduced in %Eigen 3.4. +It supports all the feature proposed by the \link TutorialBlockOperations block API \endlink, and much more. +In particular, it supports \b slicing that consists in taking a set of rows, columns, or elements, uniformly spaced within a matrix or indexed from an array of indices. + +\eigenAutoToc + +\section TutorialSlicingOverview Overview + +All the aforementioned operations are handled through the generic DenseBase::operator()(const RowIndices&, const ColIndices&) method. +Each argument can be: + - An integer indexing a single row or column, including symbolic indices. + - The symbol Eigen::all representing the whole set of respective rows or columns in increasing order. + - An ArithmeticSequence as constructed by the Eigen::seq, Eigen::seqN, or Eigen::lastN functions. + - Any 1D vector/array of integers including %Eigen's vector/array, expressions, std::vector, std::array, as well as plain C arrays: `int[N]`. + +More generally, it can accepts any object exposing the following two member functions: + \code + <integral type> operator[](<integral type>) const; + <integral type> size() const; + \endcode +where `<integral type>` stands for any integer type compatible with Eigen::Index (i.e. `std::ptrdiff_t`). + +\section TutorialSlicingBasic Basic slicing + +Taking a set of rows, columns, or elements, uniformly spaced within a matrix or vector is achieved through the Eigen::seq or Eigen::seqN functions where "seq" stands for arithmetic sequence. Their signatures are summarized below: + +<table class="manual"> +<tr> + <th>function</th> + <th>description</th> + <th>example</th> +</tr> +<tr> + <td>\code seq(firstIdx,lastIdx) \endcode</td> + <td>represents the sequence of integers ranging from \c firstIdx to \c lastIdx</td> + <td>\code seq(2,5) <=> {2,3,4,5} \endcode</td> +</tr> +<tr> + <td>\code seq(firstIdx,lastIdx,incr) \endcode</td> + <td>same but using the increment \c incr to advance from one index to the next</td> + <td>\code seq(2,8,2) <=> {2,4,6,8} \endcode</td> +</tr> +<tr> + <td>\code seqN(firstIdx,size) \endcode</td> + <td>represents the sequence of \c size integers starting from \c firstIdx</td> + <td>\code seqN(2,5) <=> {2,3,4,5,6} \endcode</td> +</tr> +<tr> + <td>\code seqN(firstIdx,size,incr) \endcode</td> + <td>same but using the increment \c incr to advance from one index to the next</td> + <td>\code seqN(2,3,3) <=> {2,5,8} \endcode</td> +</tr> +</table> + +The \c firstIdx and \c lastIdx parameters can also be defined with the help of the Eigen::last symbol representing the index of the last row, column or element of the underlying matrix/vector once the arithmetic sequence is passed to it through operator(). +Here are some examples for a 2D array/matrix \c A and a 1D array/vector \c v. +<table class="manual"> +<tr> + <th>Intent</th> + <th>Code</th> + <th>Block-API equivalence</th> +</tr> +<tr> + <td>Bottom-left corner starting at row \c i with \c n columns</td> + <td>\code A(seq(i,last), seqN(0,n)) \endcode</td> + <td>\code A.bottomLeftCorner(A.rows()-i,n) \endcode</td> +</tr> +<tr> + <td>%Block starting at \c i,j having \c m rows, and \c n columns</td> + <td>\code A(seqN(i,m), seqN(i,n) \endcode</td> + <td>\code A.block(i,j,m,n) \endcode</td> +</tr> +<tr> + <td>%Block starting at \c i0,j0 and ending at \c i1,j1</td> + <td>\code A(seq(i0,i1), seq(j0,j1) \endcode</td> + <td>\code A.block(i0,j0,i1-i0+1,j1-j0+1) \endcode</td> +</tr> +<tr> + <td>Even columns of A</td> + <td>\code A(all, seq(0,last,2)) \endcode</td> + <td></td> +</tr> +<tr> + <td>First \c n odd rows A</td> + <td>\code A(seqN(1,n,2), all) \endcode</td> + <td></td> +</tr> +<tr> + <td>The last past one column</td> + <td>\code A(all, last-1) \endcode</td> + <td>\code A.col(A.cols()-2) \endcode</td> +</tr> +<tr> + <td>The middle row</td> + <td>\code A(last/2,all) \endcode</td> + <td>\code A.row((A.rows()-1)/2) \endcode</td> +</tr> +<tr> + <td>Last elements of v starting at i</td> + <td>\code v(seq(i,last)) \endcode</td> + <td>\code v.tail(v.size()-i) \endcode</td> +</tr> +<tr> + <td>Last \c n elements of v</td> + <td>\code v(seq(last+1-n,last)) \endcode</td> + <td>\code v.tail(n) \endcode</td> +</tr> +</table> + +As seen in the last exemple, referencing the <i> last n </i> elements (or rows/columns) is a bit cumbersome to write. +This becomes even more tricky and error prone with a non-default increment. +Here comes \link Eigen::lastN(SizeType) Eigen::lastN(size) \endlink, and \link Eigen::lastN(SizeType,IncrType) Eigen::lastN(size,incr) \endlink: + +<table class="manual"> +<tr> + <th>Intent</th> + <th>Code</th> + <th>Block-API equivalence</th> +</tr> +<tr> + <td>Last \c n elements of v</td> + <td>\code v(lastN(n)) \endcode</td> + <td>\code v.tail(n) \endcode</td> +</tr> +<tr> + <td>Bottom-right corner of A of size \c m times \c n</td> + <td>\code v(lastN(m), lastN(n)) \endcode</td> + <td>\code A.bottomRightCorner(m,n) \endcode</td> +</tr> +<tr> + <td>Bottom-right corner of A of size \c m times \c n</td> + <td>\code v(lastN(m), lastN(n)) \endcode</td> + <td>\code A.bottomRightCorner(m,n) \endcode</td> +</tr> +<tr> + <td>Last \c n columns taking 1 column over 3</td> + <td>\code A(all, lastN(n,3)) \endcode</td> + <td></td> +</tr> +</table> + +\section TutorialSlicingFixed Compile time size and increment + +In terms of performance, %Eigen and the compiler can take advantage of compile-time size and increment. +To this end, you can enforce compile-time parameters using Eigen::fix<val>. +Such compile-time value can be combined with the Eigen::last symbol: +\code v(seq(last-fix<7>, last-fix<2>)) +\endcode +In this example %Eigen knowns at compile-time that the returned expression has 6 elements. +It is equivalent to: +\code v(seqN(last-7, fix<6>)) +\endcode + +We can revisit the <i>even columns of A</i> example as follows: +\code A(all, seq(0,last,fix<2>)) +\endcode + + +\section TutorialSlicingReverse Reverse order + +Row/column indices can also be enumerated in decreasing order using a negative increment. +For instance, one over two columns of A from the column 20 to 10: +\code A(all, seq(20, 10, fix<-2>)) +\endcode +The last \c n rows starting from the last one: +\code A(seqN(last, n, fix<-1>), all) +\endcode +You can also use the ArithmeticSequence::reverse() method to reverse its order. +The previous example can thus also be written as: +\code A(lastN(n).reverse(), all) +\endcode + + +\section TutorialSlicingArray Array of indices + +The generic `operator()` can also takes as input an arbitrary list of row or column indices stored as either an `ArrayXi`, a `std::vector<int>`, `std::array<int,N>`, etc. + +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Slicing_stdvector_cxx11.cpp +</td> +<td> +\verbinclude Slicing_stdvector_cxx11.out +</td></tr></table> + +You can also directly pass a static array: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Slicing_rawarray_cxx11.cpp +</td> +<td> +\verbinclude Slicing_rawarray_cxx11.out +</td></tr></table> + +or expressions: +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Slicing_arrayexpr.cpp +</td> +<td> +\verbinclude Slicing_arrayexpr.out +</td></tr></table> + +When passing an object with a compile-time size such as `Array4i`, `std::array<int,N>`, or a static array, then the returned expression also exhibit compile-time dimensions. + +\section TutorialSlicingCustomArray Custom index list + +More generally, `operator()` can accept as inputs any object \c ind of type \c T compatible with: +\code +Index s = ind.size(); or Index s = size(ind); +Index i; +i = ind[i]; +\endcode + +This means you can easily build your own fancy sequence generator and pass it to `operator()`. +Here is an exemple enlarging a given matrix while padding the additional first rows and columns through repetition: + +<table class="example"> +<tr><th>Example:</th><th>Output:</th></tr> +<tr><td> +\include Slicing_custom_padding_cxx11.cpp +</td> +<td> +\verbinclude Slicing_custom_padding_cxx11.out +</td></tr></table> + +<br> + +*/ + +/* +TODO add: +so_repeat_inner.cpp +so_repeleme.cpp +*/ +}
diff --git a/doc/TutorialSparse.dox b/doc/TutorialSparse.dox index 3529074..350ea11 100644 --- a/doc/TutorialSparse.dox +++ b/doc/TutorialSparse.dox
@@ -57,7 +57,7 @@ Assuming no reallocation is needed, the insertion of a random element is therefore in O(nnz_j) where nnz_j is the number of nonzeros of the respective inner vector. On the other hand, inserting elements with increasing inner indices in a given inner vector is much more efficient since this only requires to increase the respective \c InnerNNZs entry that is a O(1) operation. -The case where no empty space is available is a special case, and is refered as the \em compressed mode. +The case where no empty space is available is a special case, and is referred as the \em compressed mode. It corresponds to the widely used Compressed Column (or Row) Storage schemes (CCS or CRS). Any SparseMatrix can be turned to this form by calling the SparseMatrix::makeCompressed() function. In this case, one can remark that the \c InnerNNZs array is redundant with \c OuterStarts because we the equality: \c InnerNNZs[j] = \c OuterStarts[j+1]-\c OuterStarts[j]. @@ -212,7 +212,7 @@ In some cases, however, slightly higher performance, and lower memory consumption can be reached by directly inserting the non-zeros into the destination matrix. -A typical scenario of this approach is illustrated bellow: +A typical scenario of this approach is illustrated below: \code 1: SparseMatrix<double> mat(rows,cols); // default is column major 2: mat.reserve(VectorXi::Constant(cols,6));
diff --git a/doc/UnalignedArrayAssert.dox b/doc/UnalignedArrayAssert.dox index f0f84d2..410c8a5 100644 --- a/doc/UnalignedArrayAssert.dox +++ b/doc/UnalignedArrayAssert.dox
@@ -12,7 +12,9 @@ **** READ THIS WEB PAGE !!! ****"' failed. </pre> -There are 4 known causes for this issue. Please read on to understand them and learn how to fix them. +There are 4 known causes for this issue. +If you can target \cpp17 only with a recent compiler (e.g., GCC>=7, clang>=5, MSVC>=19.12), then you're lucky: enabling c++17 should be enough (if not, please <a href="http://eigen.tuxfamily.org/bz/">report</a> to us). +Otherwise, please read on to understand those issues and learn how to fix them. \eigenAutoToc @@ -35,7 +37,7 @@ class Foo { //... - Eigen::Vector2d v; + Eigen::Vector4d v; //... }; //... @@ -44,27 +46,27 @@ then you need to read this separate page: \ref TopicStructHavingEigenMembers "Structures Having Eigen Members". -Note that here, Eigen::Vector2d is only used as an example, more generally the issue arises for all \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types". +Note that here, Eigen::Vector4d is only used as an example, more generally the issue arises for all \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types". \section c2 Cause 2: STL Containers or manual memory allocation If you use STL Containers such as std::vector, std::map, ..., with %Eigen objects, or with classes containing %Eigen objects, like this, \code -std::vector<Eigen::Matrix2f> my_vector; -struct my_class { ... Eigen::Matrix2f m; ... }; +std::vector<Eigen::Matrix2d> my_vector; +struct my_class { ... Eigen::Matrix2d m; ... }; std::map<int, my_class> my_map; \endcode then you need to read this separate page: \ref TopicStlContainers "Using STL Containers with Eigen". -Note that here, Eigen::Matrix2f is only used as an example, more generally the issue arises for all \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types" and \ref TopicStructHavingEigenMembers "structures having such Eigen objects as member". +Note that here, Eigen::Matrix2d is only used as an example, more generally the issue arises for all \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types" and \ref TopicStructHavingEigenMembers "structures having such Eigen objects as member". -The same issue will be exhibited by any classes/functions by-passing operator new to allocate memory, that is, by performing custom memory allocation followed by calls to the placement new operator. This is for instance typically the case of \c std::make_shared or \c std::allocate_shared for which is the solution is to use an \ref aligned_allocator "aligned allocator" as detailed in the \ref TopicStlContainers "solution for STL containers". +The same issue will be exhibited by any classes/functions by-passing operator new to allocate memory, that is, by performing custom memory allocation followed by calls to the placement new operator. This is for instance typically the case of \c `std::make_shared` or `std::allocate_shared` for which is the solution is to use an \ref aligned_allocator "aligned allocator" as detailed in the \ref TopicStlContainers "solution for STL containers". \section c3 Cause 3: Passing Eigen objects by value -If some function in your code is getting an Eigen object passed by value, like this, +If some function in your code is getting an %Eigen object passed by value, like this, \code void func(Eigen::Vector4d v); @@ -90,29 +92,41 @@ Note that here, Eigen::Quaternionf is only used as an example, more generally the issue arises for all \ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen types". + \section explanation General explanation of this assertion -\ref TopicFixedSizeVectorizable "fixed-size vectorizable Eigen objects" must absolutely be created at 16-byte-aligned locations, otherwise SIMD instructions adressing them will crash. +\ref TopicFixedSizeVectorizable "Fixed-size vectorizable Eigen objects" must absolutely be created at properly aligned locations, otherwise SIMD instructions addressing them will crash. +For instance, SSE/NEON/MSA/Altivec/VSX targets will require 16-byte-alignment, whereas AVX and AVX512 targets may require up to 32 and 64 byte alignment respectively. -Eigen normally takes care of these alignment issues for you, by setting an alignment attribute on them and by overloading their "operator new". +%Eigen normally takes care of these alignment issues for you, by setting an alignment attribute on them and by overloading their `operator new`. However there are a few corner cases where these alignment settings get overridden: they are the possible causes for this assertion. -\section getrid I don't care about vectorization, how do I get rid of that stuff? +\section getrid I don't care about optimal vectorization, how do I get rid of that stuff? -Two possibilities: +Three possibilities: <ul> - <li>Define EIGEN_DONT_ALIGN_STATICALLY. That disables all 128-bit static alignment code, while keeping 128-bit heap alignment. This has the effect of - disabling vectorization for fixed-size objects (like Matrix4d) while keeping vectorization of dynamic-size objects - (like MatrixXd). But do note that this breaks ABI compatibility with the default behavior of 128-bit static alignment.</li> - <li>Or define both EIGEN_DONT_VECTORIZE and EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT. This keeps the - 128-bit alignment code and thus preserves ABI compatibility, but completely disables vectorization.</li> + <li>Use the \c DontAlign option to Matrix, Array, Quaternion, etc. objects that gives you trouble. This way %Eigen won't try to over-align them, and thus won"t assume any special alignment. On the down side, you will pay the cost of unaligned loads/stores for them, but on modern CPUs, the overhead is either null or marginal. See \link StructHavingEigenMembers_othersolutions here \endlink for an example.</li> + <li>Define \link TopicPreprocessorDirectivesPerformance EIGEN_MAX_STATIC_ALIGN_BYTES \endlink to 0. That disables all 16-byte (and above) static alignment code, while keeping 16-byte (or above) heap alignment. This has the effect of + vectorizing fixed-size objects (like Matrix4d) through unaligned stores (as controlled by \link TopicPreprocessorDirectivesPerformance EIGEN_UNALIGNED_VECTORIZE \endlink), while keeping unchanged the vectorization of dynamic-size objects + (like MatrixXd). On 64 bytes systems, you might also define it 16 to disable only 32 and 64 bytes of over-alignment. But do note that this breaks ABI compatibility with the default behavior of static alignment.</li> + <li>Or define both \link TopicPreprocessorDirectivesPerformance EIGEN_DONT_VECTORIZE \endlink and `EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT`. This keeps the + 16-byte (or above) alignment code and thus preserves ABI compatibility, but completely disables vectorization.</li> </ul> -If you want to know why defining EIGEN_DONT_VECTORIZE does not by itself disable 128-bit alignment and the assertion, here's the explanation: +If you want to know why defining `EIGEN_DONT_VECTORIZE` does not by itself disable 16-byte (or above) alignment and the assertion, here's the explanation: It doesn't disable the assertion, because otherwise code that runs fine without vectorization would suddenly crash when enabling vectorization. -It doesn't disable 128bit alignment, because that would mean that vectorized and non-vectorized code are not mutually ABI-compatible. This ABI compatibility is very important, even for people who develop only an in-house application, as for instance one may want to have in the same application a vectorized path and a non-vectorized path. +It doesn't disable 16-byte (or above) alignment, because that would mean that vectorized and non-vectorized code are not mutually ABI-compatible. This ABI compatibility is very important, even for people who develop only an in-house application, as for instance one may want to have in the same application a vectorized path and a non-vectorized path. + +\section checkmycode How can I check my code is safe regarding alignment issues? + +Unfortunately, there is no possibility in c++ to detect any of the aforementioned shortcoming at compile time (though static analyzers are becoming more and more powerful and could detect some of them). +Even at runtime, all we can do is to catch invalid unaligned allocation and trigger the explicit assertion mentioned at the beginning of this page. +Therefore, if your program runs fine on a given system with some given compilation flags, then this does not guarantee that your code is safe. For instance, on most 64 bits systems buffer are aligned on 16 bytes boundary and so, if you do not enable AVX instruction set, then your code will run fine. On the other hand, the same code may assert if moving to a more exotic platform, or enabling AVX instructions that required 32 bytes alignment by default. + +The situation is not hopeless though. Assuming your code is well covered by unit test, then you can check its alignment safety by linking it to a custom malloc library returning 8 bytes aligned buffers only. This way all alignment shortcomings should pop-up. To this end, you must also compile your program with \link TopicPreprocessorDirectivesPerformance EIGEN_MALLOC_ALREADY_ALIGNED=0 \endlink. + */
diff --git a/doc/UsingBlasLapackBackends.dox b/doc/UsingBlasLapackBackends.dox new file mode 100644 index 0000000..caa5971 --- /dev/null +++ b/doc/UsingBlasLapackBackends.dox
@@ -0,0 +1,133 @@ +/* + Copyright (c) 2011, Intel Corporation. All rights reserved. + Copyright (C) 2011-2016 Gael Guennebaud <gael.guennebaud@inria.fr> + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ******************************************************************************** + * Content : Documentation on the use of BLAS/LAPACK libraries through Eigen + ******************************************************************************** +*/ + +namespace Eigen { + +/** \page TopicUsingBlasLapack Using BLAS/LAPACK from %Eigen + + +Since %Eigen version 3.3 and later, any F77 compatible BLAS or LAPACK libraries can be used as backends for dense matrix products and dense matrix decompositions. +For instance, one can use <a href="http://eigen.tuxfamily.org/Counter/redirect_to_mkl.php">Intel® MKL</a>, Apple's Accelerate framework on OSX, <a href="http://www.openblas.net/">OpenBLAS</a>, <a href="http://www.netlib.org/lapack">Netlib LAPACK</a>, etc. + +Do not miss this \link TopicUsingIntelMKL page \endlink for further discussions on the specific use of Intel® MKL (also includes VML, PARDISO, etc.) + +In order to use an external BLAS and/or LAPACK library, you must link you own application to the respective libraries and their dependencies. +For LAPACK, you must also link to the standard <a href="http://www.netlib.org/lapack/lapacke.html">Lapacke</a> library, which is used as a convenient think layer between %Eigen's C++ code and LAPACK F77 interface. Then you must activate their usage by defining one or multiple of the following macros (\b before including any %Eigen's header): + +\note For Mac users, in order to use the lapack version shipped with the Accelerate framework, you also need the lapacke library. +Using <a href="https://www.macports.org/">MacPorts</a>, this is as easy as: +\code +sudo port install lapack +\endcode +and then use the following link flags: \c -framework \c Accelerate \c /opt/local/lib/lapack/liblapacke.dylib + +<table class="manual"> +<tr><td>\c EIGEN_USE_BLAS </td><td>Enables the use of external BLAS level 2 and 3 routines (compatible with any F77 BLAS interface)</td></tr> +<tr class="alt"><td>\c EIGEN_USE_LAPACKE </td><td>Enables the use of external Lapack routines via the <a href="http://www.netlib.org/lapack/lapacke.html">Lapacke</a> C interface to Lapack (compatible with any F77 LAPACK interface)</td></tr> +<tr><td>\c EIGEN_USE_LAPACKE_STRICT </td><td>Same as \c EIGEN_USE_LAPACKE but algorithms of lower numerical robustness are disabled. \n This currently concerns only JacobiSVD which otherwise would be replaced by \c gesvd that is less robust than Jacobi rotations.</td></tr> +</table> + +When doing so, a number of %Eigen's algorithms are silently substituted with calls to BLAS or LAPACK routines. +These substitutions apply only for \b Dynamic \b or \b large enough objects with one of the following four standard scalar types: \c float, \c double, \c complex<float>, and \c complex<double>. +Operations on other scalar types or mixing reals and complexes will continue to use the built-in algorithms. + +The breadth of %Eigen functionality that can be substituted is listed in the table below. +<table class="manual"> +<tr><th>Functional domain</th><th>Code example</th><th>BLAS/LAPACK routines</th></tr> +<tr><td>Matrix-matrix operations \n \c EIGEN_USE_BLAS </td><td>\code +m1*m2.transpose(); +m1.selfadjointView<Lower>()*m2; +m1*m2.triangularView<Upper>(); +m1.selfadjointView<Lower>().rankUpdate(m2,1.0); +\endcode</td><td>\code +?gemm +?symm/?hemm +?trmm +dsyrk/ssyrk +\endcode</td></tr> +<tr class="alt"><td>Matrix-vector operations \n \c EIGEN_USE_BLAS </td><td>\code +m1.adjoint()*b; +m1.selfadjointView<Lower>()*b; +m1.triangularView<Upper>()*b; +\endcode</td><td>\code +?gemv +?symv/?hemv +?trmv +\endcode</td></tr> +<tr><td>LU decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code +v1 = m1.lu().solve(v2); +\endcode</td><td>\code +?getrf +\endcode</td></tr> +<tr class="alt"><td>Cholesky decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code +v1 = m2.selfadjointView<Upper>().llt().solve(v2); +\endcode</td><td>\code +?potrf +\endcode</td></tr> +<tr><td>QR decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code +m1.householderQr(); +m1.colPivHouseholderQr(); +\endcode</td><td>\code +?geqrf +?geqp3 +\endcode</td></tr> +<tr class="alt"><td>Singular value decomposition \n \c EIGEN_USE_LAPACKE </td><td>\code +JacobiSVD<MatrixXd> svd; +svd.compute(m1, ComputeThinV); +\endcode</td><td>\code +?gesvd +\endcode</td></tr> +<tr><td>Eigen-value decompositions \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code +EigenSolver<MatrixXd> es(m1); +ComplexEigenSolver<MatrixXcd> ces(m1); +SelfAdjointEigenSolver<MatrixXd> saes(m1+m1.transpose()); +GeneralizedSelfAdjointEigenSolver<MatrixXd> + gsaes(m1+m1.transpose(),m2+m2.transpose()); +\endcode</td><td>\code +?gees +?gees +?syev/?heev +?syev/?heev, +?potrf +\endcode</td></tr> +<tr class="alt"><td>Schur decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code +RealSchur<MatrixXd> schurR(m1); +ComplexSchur<MatrixXcd> schurC(m1); +\endcode</td><td>\code +?gees +\endcode</td></tr> +</table> +In the examples, m1 and m2 are dense matrices and v1 and v2 are dense vectors. + +*/ + +}
diff --git a/doc/UsingIntelMKL.dox b/doc/UsingIntelMKL.dox index dbe559e..fc35c3c 100644 --- a/doc/UsingIntelMKL.dox +++ b/doc/UsingIntelMKL.dox
@@ -32,107 +32,51 @@ namespace Eigen { -/** \page TopicUsingIntelMKL Using Intel® Math Kernel Library from Eigen +/** \page TopicUsingIntelMKL Using Intel® MKL from %Eigen -\section TopicUsingIntelMKL_Intro Eigen and Intel® Math Kernel Library (Intel® MKL) +<!-- \section TopicUsingIntelMKL_Intro Eigen and Intel® Math Kernel Library (Intel® MKL) --> -Since Eigen version 3.1 and later, users can benefit from built-in Intel MKL optimizations with an installed copy of Intel MKL 10.3 (or later). +Since %Eigen version 3.1 and later, users can benefit from built-in Intel® Math Kernel Library (MKL) optimizations with an installed copy of Intel MKL 10.3 (or later). + <a href="http://eigen.tuxfamily.org/Counter/redirect_to_mkl.php"> Intel MKL </a> provides highly optimized multi-threaded mathematical routines for x86-compatible architectures. Intel MKL is available on Linux, Mac and Windows for both Intel64 and IA32 architectures. \note Intel® MKL is a proprietary software and it is the responsibility of users to buy or register for community (free) Intel MKL licenses for their products. Moreover, the license of the user product has to allow linking to proprietary software that excludes any unmodified versions of the GPL. -Using Intel MKL through Eigen is easy: --# define the \c EIGEN_USE_MKL_ALL macro before including any Eigen's header +Using Intel MKL through %Eigen is easy: +-# define the \c EIGEN_USE_MKL_ALL macro before including any %Eigen's header -# link your program to MKL libraries (see the <a href="http://software.intel.com/en-us/articles/intel-mkl-link-line-advisor/">MKL linking advisor</a>) -# on a 64bits system, you must use the LP64 interface (not the ILP64 one) -When doing so, a number of Eigen's algorithms are silently substituted with calls to Intel MKL routines. +When doing so, a number of %Eigen's algorithms are silently substituted with calls to Intel MKL routines. These substitutions apply only for \b Dynamic \b or \b large enough objects with one of the following four standard scalar types: \c float, \c double, \c complex<float>, and \c complex<double>. Operations on other scalar types or mixing reals and complexes will continue to use the built-in algorithms. In addition you can choose which parts will be substituted by defining one or multiple of the following macros: <table class="manual"> -<tr><td>\c EIGEN_USE_BLAS </td><td>Enables the use of external BLAS level 2 and 3 routines (compatible with any F77 BLAS interface, not only Intel MKL)</td></tr> -<tr class="alt"><td>\c EIGEN_USE_LAPACKE </td><td>Enables the use of external Lapack routines via the <a href="http://www.netlib.org/lapack/lapacke.html">Intel Lapacke</a> C interface to Lapack (currently works with Intel MKL only)</td></tr> -<tr><td>\c EIGEN_USE_LAPACKE_STRICT </td><td>Same as \c EIGEN_USE_LAPACKE but algorithm of lower robustness are disabled. This currently concerns only JacobiSVD which otherwise would be replaced by \c gesvd that is less robust than Jacobi rotations.</td></tr> +<tr><td>\c EIGEN_USE_BLAS </td><td>Enables the use of external BLAS level 2 and 3 routines</td></tr> +<tr class="alt"><td>\c EIGEN_USE_LAPACKE </td><td>Enables the use of external Lapack routines via the <a href="http://www.netlib.org/lapack/lapacke.html">Lapacke</a> C interface to Lapack</td></tr> +<tr><td>\c EIGEN_USE_LAPACKE_STRICT </td><td>Same as \c EIGEN_USE_LAPACKE but algorithm of lower robustness are disabled. \n This currently concerns only JacobiSVD which otherwise would be replaced by \c gesvd that is less robust than Jacobi rotations.</td></tr> <tr class="alt"><td>\c EIGEN_USE_MKL_VML </td><td>Enables the use of Intel VML (vector operations)</td></tr> <tr><td>\c EIGEN_USE_MKL_ALL </td><td>Defines \c EIGEN_USE_BLAS, \c EIGEN_USE_LAPACKE, and \c EIGEN_USE_MKL_VML </td></tr> </table> +The \c EIGEN_USE_BLAS and \c EIGEN_USE_LAPACKE* macros can be combined with \c EIGEN_USE_MKL to explicitly tell Eigen that the underlying BLAS/Lapack implementation is Intel MKL. +The main effect is to enable MKL direct call feature (\c MKL_DIRECT_CALL). +This may help to increase performance of some MKL BLAS (?GEMM, ?GEMV, ?TRSM, ?AXPY and ?DOT) and LAPACK (LU, Cholesky and QR) routines for very small matrices. +MKL direct call can be disabled by defining \c EIGEN_MKL_NO_DIRECT_CALL. + + +Note that the BLAS and LAPACKE backends can be enabled for any F77 compatible BLAS and LAPACK libraries. See this \link TopicUsingBlasLapack page \endlink for the details. + Finally, the PARDISO sparse solver shipped with Intel MKL can be used through the \ref PardisoLU, \ref PardisoLLT and \ref PardisoLDLT classes of the \ref PardisoSupport_Module. - -\section TopicUsingIntelMKL_SupportedFeatures List of supported features - -The breadth of Eigen functionality covered by Intel MKL is listed in the table below. +The following table summarizes the list of functions covered by \c EIGEN_USE_MKL_VML: <table class="manual"> -<tr><th>Functional domain</th><th>Code example</th><th>MKL routines</th></tr> -<tr><td>Matrix-matrix operations \n \c EIGEN_USE_BLAS </td><td>\code -m1*m2.transpose(); -m1.selfadjointView<Lower>()*m2; -m1*m2.triangularView<Upper>(); -m1.selfadjointView<Lower>().rankUpdate(m2,1.0); -\endcode</td><td>\code -?gemm -?symm/?hemm -?trmm -dsyrk/ssyrk -\endcode</td></tr> -<tr class="alt"><td>Matrix-vector operations \n \c EIGEN_USE_BLAS </td><td>\code -m1.adjoint()*b; -m1.selfadjointView<Lower>()*b; -m1.triangularView<Upper>()*b; -\endcode</td><td>\code -?gemv -?symv/?hemv -?trmv -\endcode</td></tr> -<tr><td>LU decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code -v1 = m1.lu().solve(v2); -\endcode</td><td>\code -?getrf -\endcode</td></tr> -<tr class="alt"><td>Cholesky decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code -v1 = m2.selfadjointView<Upper>().llt().solve(v2); -\endcode</td><td>\code -?potrf -\endcode</td></tr> -<tr><td>QR decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code -m1.householderQr(); -m1.colPivHouseholderQr(); -\endcode</td><td>\code -?geqrf -?geqp3 -\endcode</td></tr> -<tr class="alt"><td>Singular value decomposition \n \c EIGEN_USE_LAPACKE </td><td>\code -JacobiSVD<MatrixXd> svd; -svd.compute(m1, ComputeThinV); -\endcode</td><td>\code -?gesvd -\endcode</td></tr> -<tr><td>Eigen-value decompositions \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code -EigenSolver<MatrixXd> es(m1); -ComplexEigenSolver<MatrixXcd> ces(m1); -SelfAdjointEigenSolver<MatrixXd> saes(m1+m1.transpose()); -GeneralizedSelfAdjointEigenSolver<MatrixXd> - gsaes(m1+m1.transpose(),m2+m2.transpose()); -\endcode</td><td>\code -?gees -?gees -?syev/?heev -?syev/?heev, -?potrf -\endcode</td></tr> -<tr class="alt"><td>Schur decomposition \n \c EIGEN_USE_LAPACKE \n \c EIGEN_USE_LAPACKE_STRICT </td><td>\code -RealSchur<MatrixXd> schurR(m1); -ComplexSchur<MatrixXcd> schurC(m1); -\endcode</td><td>\code -?gees -\endcode</td></tr> -<tr><td>Vector Math \n \c EIGEN_USE_MKL_VML </td><td>\code +<tr><th>Code example</th><th>MKL routines</th></tr> +<tr><td>\code v2=v1.array().sin(); v2=v1.array().asin(); v2=v1.array().cos(); @@ -156,7 +100,7 @@ v?Powx \endcode</td></tr> </table> -In the examples, m1 and m2 are dense matrices and v1 and v2 are dense vectors. +In the examples, v1 and v2 are dense vectors. \section TopicUsingIntelMKL_Links Links
diff --git a/doc/UsingNVCC.dox b/doc/UsingNVCC.dox index f8e755b..36beb2d 100644 --- a/doc/UsingNVCC.dox +++ b/doc/UsingNVCC.dox
@@ -3,18 +3,16 @@ /** \page TopicCUDA Using Eigen in CUDA kernels -\b Disclaimer: this page is about an \b experimental feature in %Eigen. - -Staring from CUDA 5.0, the CUDA compiler, \c nvcc, is able to properly parse %Eigen's code (almost). -A few adaptations of the %Eigen's code already allows to use some parts of %Eigen in your own CUDA kernels. -To this end you need the devel branch of %Eigen, CUDA 5.0 or greater with GCC. +Staring from CUDA 5.5 and Eigen 3.3, it is possible to use Eigen's matrices, vectors, and arrays for fixed size within CUDA kernels. This is especially useful when working on numerous but small problems. By default, when Eigen's headers are included within a .cu file compiled by nvcc most Eigen's functions and methods are prefixed by the \c __device__ \c __host__ keywords making them callable from both host and device code. +This support can be disabled by defining \c EIGEN_NO_CUDA before including any Eigen's header. +This might be useful to disable some warnings when a .cu file makes use of Eigen on the host side only. +However, in both cases, host's SIMD vectorization has to be disabled in .cu files. +It is thus \b strongly \b recommended to properly move all costly host computation from your .cu files to regular .cpp files. Known issues: - \c nvcc with MS Visual Studio does not work (patch welcome) - - \c nvcc with \c clang does not work (patch welcome) - - \c nvcc 5.5 with gcc-4.7 (or greater) has issues with the standard \c \<limits\> header file. To workaround this, you can add the following before including any other files: \code // workaround issue between gcc >= 4.7 and cuda 5.5
diff --git a/doc/eigen_navtree_hacks.js b/doc/eigen_navtree_hacks.js index bd7e02b..a6f8c34 100644 --- a/doc/eigen_navtree_hacks.js +++ b/doc/eigen_navtree_hacks.js
@@ -64,14 +64,20 @@ // Overloaded to adjust the size of the navtree wrt the toc function resizeHeight() { - var toc = $("#nav-toc"); - var tocHeight = toc.height(); // <- we added this line - var headerHeight = header.height(); - var footerHeight = footer.height(); + var header = $("#top"); + var sidenav = $("#side-nav"); + var content = $("#doc-content"); + var navtree = $("#nav-tree"); + var footer = $("#nav-path"); + var toc = $("#nav-toc"); + + var headerHeight = header.outerHeight(); + var footerHeight = footer.outerHeight(); + var tocHeight = toc.height(); var windowHeight = $(window).height() - headerHeight - footerHeight; content.css({height:windowHeight + "px"}); - navtree.css({height:(windowHeight-tocHeight) + "px"}); // <- we modified this line - sidenav.css({height:(windowHeight) + "px",top: headerHeight+"px"}); + navtree.css({height:(windowHeight-tocHeight) + "px"}); + sidenav.css({height:windowHeight + "px"}); } // Overloaded to save the root node into global_navtree_object @@ -155,19 +161,18 @@ var level=-2; // <- we replaced level=-1 by level=-2 var n = node; while (n.parentNode) { level++; n=n.parentNode; } - var imgNode = document.createElement("img"); - imgNode.style.paddingLeft=(16*(level)).toString()+'px'; - imgNode.width = 16; - imgNode.height = 22; - imgNode.border = 0; if (checkChildrenData(node)) { // <- we modified this line to use checkChildrenData(node) instead of node.childrenData + var imgNode = document.createElement("span"); + imgNode.className = 'arrow'; + imgNode.style.paddingLeft=(16*level).toString()+'px'; + imgNode.innerHTML=arrowRight; node.plus_img = imgNode; node.expandToggle = document.createElement("a"); node.expandToggle.href = "javascript:void(0)"; node.expandToggle.onclick = function() { if (node.expanded) { $(node.getChildrenUL()).slideUp("fast"); - node.plus_img.src = node.relpath+"ftv2pnode.png"; + node.plus_img.innerHTML=arrowRight; node.expanded = false; } else { expandNode(o, node, false, false); @@ -175,11 +180,13 @@ } node.expandToggle.appendChild(imgNode); domNode.appendChild(node.expandToggle); - imgNode.src = node.relpath+"ftv2pnode.png"; } else { - imgNode.src = node.relpath+"ftv2node.png"; - domNode.appendChild(imgNode); - } + var span = document.createElement("span"); + span.className = 'arrow'; + span.style.width = 16*(level+1)+'px'; + span.innerHTML = ' '; + domNode.appendChild(span); + } } // Overloaded to automatically expand the selected node @@ -233,8 +240,7 @@ setTimeout(arguments.callee, 10); } })(); + + $(window).load(resizeHeight); }); -$(window).load(function() { - resizeHeight(); -});
diff --git a/doc/eigendoxy.css b/doc/eigendoxy.css index 60243d8..6148655 100644 --- a/doc/eigendoxy.css +++ b/doc/eigendoxy.css
@@ -45,7 +45,7 @@ /* Common style for all Eigen's tables */ -table.example, table.manual, table.manual-vl { +table.example, table.manual, table.manual-vl, table.manual-hl { max-width:100%; border-collapse: collapse; border-style: solid; @@ -58,7 +58,7 @@ -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); } -table.example th, table.manual th, table.manual-vl th { +table.example th, table.manual th, table.manual-vl th, table.manual-hl th { padding: 0.5em 0.5em 0.5em 0.5em; text-align: left; padding-right: 1em; @@ -70,7 +70,7 @@ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#FFFFFF', endColorstr='#F4F4E5'); } -table.example td, table.manual td, table.manual-vl td { +table.example td, table.manual td, table.manual-vl td, table.manual-hl td { vertical-align:top; border-width: 1px; border-color: #cccccc; @@ -93,7 +93,7 @@ border-color: #cccccc; } -/** class for exemple / output tables **/ +/** class for example / output tables **/ table.example { } @@ -108,15 +108,15 @@ /* standard class for the manual */ -table.manual, table.manual-vl { +table.manual, table.manual-vl, table.manual-hl { padding: 0.2em 0em 0.5em 0em; } -table.manual th, table.manual-vl th { +table.manual th, table.manual-vl th, table.manual-hl th { margin: 0em 0em 0.3em 0em; } -table.manual td, table.manual-vl td { +table.manual td, table.manual-vl td, table.manual-hl td { padding: 0.3em 0.5em 0.3em 0.5em; vertical-align:top; border-width: 1px; @@ -136,6 +136,16 @@ border-style: solid solid solid solid; } +table.manual-hl td { + border-color: #cccccc; + border-width: 1px; + border-style: solid none solid none; +} + +table td.code { + font-family: monospace; +} + h2 { margin-top:2em; border-style: none none solid none; @@ -155,6 +165,8 @@ bottom:0; border-radius:0px; border-style: solid none none none; + max-height:50%; + overflow-y: scroll; } div.toc h3 { @@ -166,6 +178,23 @@ margin: 0.2em 0 0.4em 0.5em; } +span.cpp11,span.cpp14,span.cpp17 { + color: #119911; + font-weight: bold; +} + +.newin3x { + color: #a37c1a; + font-weight: bold; +} + +div.warningbox { + max-width:60em; + border-style: solid solid solid solid; + border-color: red; + border-width: 3px; +} + /**** old Eigen's styles ****/ @@ -177,8 +206,8 @@ /* Whenever doxygen meets a '\n' or a '<BR/>', it will put - * the text containing the characted into a <p class="starttd">. - * This little hack togehter with table.tutorial_code td.note + * the text containing the character into a <p class="starttd">. + * This little hack together with table.tutorial_code td.note * aims at fixing this issue. */ table.tutorial_code td.note p.starttd { margin: 0px; @@ -199,3 +228,8 @@ td.width20em p.endtd { width: 20em; } + +/* needed for huge screens */ +.ui-resizable-e { + background-repeat: repeat-y; +} \ No newline at end of file
diff --git a/doc/eigendoxy_footer.html.in b/doc/eigendoxy_footer.html.in index 878244a..9ac0596 100644 --- a/doc/eigendoxy_footer.html.in +++ b/doc/eigendoxy_footer.html.in
@@ -5,14 +5,14 @@ $navpath <li class="footer">$generatedby <a href="http://www.doxygen.org/index.html"> - <img class="footer" src="$relpath$doxygen.png" alt="doxygen"/></a> $doxygenversion </li> + <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/></a> $doxygenversion </li> </ul> </div> <!--END GENERATE_TREEVIEW--> <!--BEGIN !GENERATE_TREEVIEW--> <hr class="footer"/><address class="footer"><small> $generatedby  <a href="http://www.doxygen.org/index.html"> -<img class="footer" src="$relpath$doxygen.png" alt="doxygen"/> +<img class="footer" src="$relpath^doxygen.png" alt="doxygen"/> </a> $doxygenversion </small></address> <!--END !GENERATE_TREEVIEW-->
diff --git a/doc/eigendoxy_header.html.in b/doc/eigendoxy_header.html.in index 0f3859f..bb149f8 100644 --- a/doc/eigendoxy_header.html.in +++ b/doc/eigendoxy_header.html.in
@@ -4,25 +4,23 @@ <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> <meta http-equiv="X-UA-Compatible" content="IE=9"/> <meta name="generator" content="Doxygen $doxygenversion"/> +<meta name="viewport" content="width=device-width, initial-scale=1"/> <!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME--> <!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME--> -<link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> -<script type="text/javascript" src="$relpath$jquery.js"></script> -<script type="text/javascript" src="$relpath$dynsections.js"></script> +<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/> +<script type="text/javascript" src="$relpath^jquery.js"></script> +<script type="text/javascript" src="$relpath^dynsections.js"></script> $treeview $search $mathjax -<link href="$relpath$$stylesheet" rel="stylesheet" type="text/css" /> +<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" /> <link href="$relpath$eigendoxy.css" rel="stylesheet" type="text/css"> <!-- $extrastylesheet --> <script type="text/javascript" src="$relpath$eigen_navtree_hacks.js"></script> -<!-- <script type="text/javascript"> --> -<!-- </script> --> </head> <body> <div id="top"><!-- do not remove this div, it is closed by doxygen! --> -<!-- <a name="top"></a> --> <!--BEGIN TITLEAREA--> <div id="titlearea"> @@ -30,10 +28,10 @@ <tbody> <tr style="height: 56px;"> <!--BEGIN PROJECT_LOGO--> - <td id="projectlogo"><img alt="Logo" src="$relpath$$projectlogo"/></td> + <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td> <!--END PROJECT_LOGO--> <!--BEGIN PROJECT_NAME--> - <td style="padding-left: 0.5em;"> + <td id="projectalign" style="padding-left: 0.5em;"> <div id="projectname"><a href="http://eigen.tuxfamily.org">$projectname</a> <!--BEGIN PROJECT_NUMBER--> <span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER--> </div> @@ -42,7 +40,7 @@ <!--END PROJECT_NAME--> <!--BEGIN !PROJECT_NAME--> <!--BEGIN PROJECT_BRIEF--> - <td style="padding-left: 0.5em;"> + <td id="projectalign" style="padding-left: 0.5em;"> <div id="projectbrief">$projectbrief</div> </td> <!--END PROJECT_BRIEF-->
diff --git a/doc/examples/CMakeLists.txt b/doc/examples/CMakeLists.txt index 08cf8ef..4fa0b10 100644 --- a/doc/examples/CMakeLists.txt +++ b/doc/examples/CMakeLists.txt
@@ -14,3 +14,7 @@ ) add_dependencies(all_examples ${example}) endforeach(example_src) + +if(EIGEN_COMPILER_SUPPORT_CPP11) +ei_add_target_property(nullary_indexing COMPILE_FLAGS "-std=c++11") +endif() \ No newline at end of file
diff --git a/doc/examples/Cwise_erf.cpp b/doc/examples/Cwise_erf.cpp new file mode 100644 index 0000000..e7cd2c1 --- /dev/null +++ b/doc/examples/Cwise_erf.cpp
@@ -0,0 +1,9 @@ +#include <Eigen/Core> +#include <unsupported/Eigen/SpecialFunctions> +#include <iostream> +using namespace Eigen; +int main() +{ + Array4d v(-0.5,2,0,-7); + std::cout << v.erf() << std::endl; +}
diff --git a/doc/examples/Cwise_erfc.cpp b/doc/examples/Cwise_erfc.cpp new file mode 100644 index 0000000..d8bb04c --- /dev/null +++ b/doc/examples/Cwise_erfc.cpp
@@ -0,0 +1,9 @@ +#include <Eigen/Core> +#include <unsupported/Eigen/SpecialFunctions> +#include <iostream> +using namespace Eigen; +int main() +{ + Array4d v(-0.5,2,0,-7); + std::cout << v.erfc() << std::endl; +}
diff --git a/doc/examples/Cwise_lgamma.cpp b/doc/examples/Cwise_lgamma.cpp new file mode 100644 index 0000000..6bfaccb --- /dev/null +++ b/doc/examples/Cwise_lgamma.cpp
@@ -0,0 +1,9 @@ +#include <Eigen/Core> +#include <unsupported/Eigen/SpecialFunctions> +#include <iostream> +using namespace Eigen; +int main() +{ + Array4d v(0.5,10,0,-1); + std::cout << v.lgamma() << std::endl; +}
diff --git a/doc/examples/TutorialInplaceLU.cpp b/doc/examples/TutorialInplaceLU.cpp new file mode 100644 index 0000000..cb9c59b --- /dev/null +++ b/doc/examples/TutorialInplaceLU.cpp
@@ -0,0 +1,61 @@ +#include <iostream> +struct init { + init() { std::cout << "[" << "init" << "]" << std::endl; } +}; +init init_obj; +// [init] +#include <iostream> +#include <Eigen/Dense> + +using namespace std; +using namespace Eigen; + +int main() +{ + MatrixXd A(2,2); + A << 2, -1, 1, 3; + cout << "Here is the input matrix A before decomposition:\n" << A << endl; +cout << "[init]" << endl; + +cout << "[declaration]" << endl; + PartialPivLU<Ref<MatrixXd> > lu(A); + cout << "Here is the input matrix A after decomposition:\n" << A << endl; +cout << "[declaration]" << endl; + +cout << "[matrixLU]" << endl; + cout << "Here is the matrix storing the L and U factors:\n" << lu.matrixLU() << endl; +cout << "[matrixLU]" << endl; + +cout << "[solve]" << endl; + MatrixXd A0(2,2); A0 << 2, -1, 1, 3; + VectorXd b(2); b << 1, 2; + VectorXd x = lu.solve(b); + cout << "Residual: " << (A0 * x - b).norm() << endl; +cout << "[solve]" << endl; + +cout << "[modifyA]" << endl; + A << 3, 4, -2, 1; + x = lu.solve(b); + cout << "Residual: " << (A0 * x - b).norm() << endl; +cout << "[modifyA]" << endl; + +cout << "[recompute]" << endl; + A0 = A; // save A + lu.compute(A); + x = lu.solve(b); + cout << "Residual: " << (A0 * x - b).norm() << endl; +cout << "[recompute]" << endl; + +cout << "[recompute_bis0]" << endl; + MatrixXd A1(2,2); + A1 << 5,-2,3,4; + lu.compute(A1); + cout << "Here is the input matrix A1 after decomposition:\n" << A1 << endl; +cout << "[recompute_bis0]" << endl; + +cout << "[recompute_bis1]" << endl; + x = lu.solve(b); + cout << "Residual: " << (A1 * x - b).norm() << endl; +cout << "[recompute_bis1]" << endl; + +}
diff --git a/doc/examples/TutorialLinAlgSVDSolve.cpp b/doc/examples/TutorialLinAlgSVDSolve.cpp index 9fbc031..f109f04 100644 --- a/doc/examples/TutorialLinAlgSVDSolve.cpp +++ b/doc/examples/TutorialLinAlgSVDSolve.cpp
@@ -11,5 +11,5 @@ VectorXf b = VectorXf::Random(3); cout << "Here is the right hand side b:\n" << b << endl; cout << "The least-squares solution is:\n" - << A.jacobiSvd(ComputeThinU | ComputeThinV).solve(b) << endl; + << A.bdcSvd(ComputeThinU | ComputeThinV).solve(b) << endl; }
diff --git a/doc/examples/Tutorial_simple_example_dynamic_size.cpp b/doc/examples/Tutorial_simple_example_dynamic_size.cpp index 0f0280e..defcb1e 100644 --- a/doc/examples/Tutorial_simple_example_dynamic_size.cpp +++ b/doc/examples/Tutorial_simple_example_dynamic_size.cpp
@@ -10,7 +10,7 @@ MatrixXi m(size,size+1); // a (size)x(size+1)-matrix of int's for (int j=0; j<m.cols(); ++j) // loop over columns for (int i=0; i<m.rows(); ++i) // loop over rows - m(i,j) = i+j*m.rows(); // to access matrix coefficients, + m(i,j) = i+j*size; // to access matrix coefficients, // use operator()(int,int) std::cout << m << "\n\n"; }
diff --git a/doc/examples/class_FixedReshaped.cpp b/doc/examples/class_FixedReshaped.cpp new file mode 100644 index 0000000..b6d4085 --- /dev/null +++ b/doc/examples/class_FixedReshaped.cpp
@@ -0,0 +1,22 @@ +#include <Eigen/Core> +#include <iostream> +using namespace Eigen; +using namespace std; + +template<typename Derived> +Eigen::Reshaped<Derived, 4, 2> +reshape_helper(MatrixBase<Derived>& m) +{ + return Eigen::Reshaped<Derived, 4, 2>(m.derived()); +} + +int main(int, char**) +{ + MatrixXd m(2, 4); + m << 1, 2, 3, 4, + 5, 6, 7, 8; + MatrixXd n = reshape_helper(m); + cout << "matrix m is:" << endl << m << endl; + cout << "matrix n is:" << endl << n << endl; + return 0; +}
diff --git a/doc/examples/class_Reshaped.cpp b/doc/examples/class_Reshaped.cpp new file mode 100644 index 0000000..18fb454 --- /dev/null +++ b/doc/examples/class_Reshaped.cpp
@@ -0,0 +1,23 @@ +#include <Eigen/Core> +#include <iostream> +using namespace std; +using namespace Eigen; + +template<typename Derived> +const Reshaped<const Derived> +reshape_helper(const MatrixBase<Derived>& m, int rows, int cols) +{ + return Reshaped<const Derived>(m.derived(), rows, cols); +} + +int main(int, char**) +{ + MatrixXd m(3, 4); + m << 1, 4, 7, 10, + 2, 5, 8, 11, + 3, 6, 9, 12; + cout << m << endl; + Ref<const MatrixXd> n = reshape_helper(m, 2, 6); + cout << "Matrix m is:" << endl << m << endl; + cout << "Matrix n is:" << endl << n << endl; +}
diff --git a/doc/examples/make_circulant2.cpp b/doc/examples/make_circulant2.cpp new file mode 100644 index 0000000..95d3dd3 --- /dev/null +++ b/doc/examples/make_circulant2.cpp
@@ -0,0 +1,52 @@ +#include <Eigen/Core> +#include <iostream> + +using namespace Eigen; + +// [circulant_func] +template<class ArgType> +class circulant_functor { + const ArgType &m_vec; +public: + circulant_functor(const ArgType& arg) : m_vec(arg) {} + + const typename ArgType::Scalar& operator() (Index row, Index col) const { + Index index = row - col; + if (index < 0) index += m_vec.size(); + return m_vec(index); + } +}; +// [circulant_func] + +// [square] +template<class ArgType> +struct circulant_helper { + typedef Matrix<typename ArgType::Scalar, + ArgType::SizeAtCompileTime, + ArgType::SizeAtCompileTime, + ColMajor, + ArgType::MaxSizeAtCompileTime, + ArgType::MaxSizeAtCompileTime> MatrixType; +}; +// [square] + +// [makeCirculant] +template <class ArgType> +CwiseNullaryOp<circulant_functor<ArgType>, typename circulant_helper<ArgType>::MatrixType> +makeCirculant(const Eigen::MatrixBase<ArgType>& arg) +{ + typedef typename circulant_helper<ArgType>::MatrixType MatrixType; + return MatrixType::NullaryExpr(arg.size(), arg.size(), circulant_functor<ArgType>(arg.derived())); +} +// [makeCirculant] + +// [main] +int main() +{ + Eigen::VectorXd vec(4); + vec << 1, 2, 4, 8; + Eigen::MatrixXd mat; + mat = makeCirculant(vec); + std::cout << mat << std::endl; +} +// [main]
diff --git a/doc/examples/matrixfree_cg.cpp b/doc/examples/matrixfree_cg.cpp index 6a205ae..7469938 100644 --- a/doc/examples/matrixfree_cg.cpp +++ b/doc/examples/matrixfree_cg.cpp
@@ -67,6 +67,7 @@ // This method should implement "dst += alpha * lhs * rhs" inplace, // however, for iterative solvers, alpha is always equal to 1, so let's not bother about it. assert(alpha==Scalar(1) && "scaling is not implemented"); + EIGEN_ONLY_USED_FOR_DEBUG(alpha); // Here we could simply call dst.noalias() += lhs.my_matrix() * rhs, // but let's do something fancier (and less efficient):
diff --git a/doc/examples/nullary_indexing.cpp b/doc/examples/nullary_indexing.cpp new file mode 100644 index 0000000..ca17456 --- /dev/null +++ b/doc/examples/nullary_indexing.cpp
@@ -0,0 +1,66 @@ +#include <Eigen/Core> +#include <iostream> + +using namespace Eigen; + +// [functor] +template<class ArgType, class RowIndexType, class ColIndexType> +class indexing_functor { + const ArgType &m_arg; + const RowIndexType &m_rowIndices; + const ColIndexType &m_colIndices; +public: + typedef Matrix<typename ArgType::Scalar, + RowIndexType::SizeAtCompileTime, + ColIndexType::SizeAtCompileTime, + ArgType::Flags&RowMajorBit?RowMajor:ColMajor, + RowIndexType::MaxSizeAtCompileTime, + ColIndexType::MaxSizeAtCompileTime> MatrixType; + + indexing_functor(const ArgType& arg, const RowIndexType& row_indices, const ColIndexType& col_indices) + : m_arg(arg), m_rowIndices(row_indices), m_colIndices(col_indices) + {} + + const typename ArgType::Scalar& operator() (Index row, Index col) const { + return m_arg(m_rowIndices[row], m_colIndices[col]); + } +}; +// [functor] + +// [function] +template <class ArgType, class RowIndexType, class ColIndexType> +CwiseNullaryOp<indexing_functor<ArgType,RowIndexType,ColIndexType>, typename indexing_functor<ArgType,RowIndexType,ColIndexType>::MatrixType> +mat_indexing(const Eigen::MatrixBase<ArgType>& arg, const RowIndexType& row_indices, const ColIndexType& col_indices) +{ + typedef indexing_functor<ArgType,RowIndexType,ColIndexType> Func; + typedef typename Func::MatrixType MatrixType; + return MatrixType::NullaryExpr(row_indices.size(), col_indices.size(), Func(arg.derived(), row_indices, col_indices)); +} +// [function] + + +int main() +{ + std::cout << "[main1]\n"; + Eigen::MatrixXi A = Eigen::MatrixXi::Random(4,4); + Array3i ri(1,2,1); + ArrayXi ci(6); ci << 3,2,1,0,0,2; + Eigen::MatrixXi B = mat_indexing(A, ri, ci); + std::cout << "A =" << std::endl; + std::cout << A << std::endl << std::endl; + std::cout << "A([" << ri.transpose() << "], [" << ci.transpose() << "]) =" << std::endl; + std::cout << B << std::endl; + std::cout << "[main1]\n"; + + std::cout << "[main2]\n"; + B = mat_indexing(A, ri+1, ci); + std::cout << "A(ri+1,ci) =" << std::endl; + std::cout << B << std::endl << std::endl; +#if __cplusplus >= 201103L + B = mat_indexing(A, ArrayXi::LinSpaced(13,0,12).unaryExpr([](int x){return x%4;}), ArrayXi::LinSpaced(4,0,3)); + std::cout << "A(ArrayXi::LinSpaced(13,0,12).unaryExpr([](int x){return x%4;}), ArrayXi::LinSpaced(4,0,3)) =" << std::endl; + std::cout << B << std::endl << std::endl; +#endif + std::cout << "[main2]\n"; +} +
diff --git a/doc/ftv2node.png b/doc/ftv2node.png new file mode 100644 index 0000000..63c605b --- /dev/null +++ b/doc/ftv2node.png Binary files differ
diff --git a/doc/ftv2pnode.png b/doc/ftv2pnode.png new file mode 100644 index 0000000..c6ee22f --- /dev/null +++ b/doc/ftv2pnode.png Binary files differ
diff --git a/doc/snippets/Array_initializer_list_23_cxx11.cpp b/doc/snippets/Array_initializer_list_23_cxx11.cpp new file mode 100644 index 0000000..1ea32dd --- /dev/null +++ b/doc/snippets/Array_initializer_list_23_cxx11.cpp
@@ -0,0 +1,5 @@ +ArrayXXi a { + {1, 2, 3}, + {3, 4, 5} +}; +cout << a << endl; \ No newline at end of file
diff --git a/doc/snippets/Array_initializer_list_vector_cxx11.cpp b/doc/snippets/Array_initializer_list_vector_cxx11.cpp new file mode 100644 index 0000000..e38b61e --- /dev/null +++ b/doc/snippets/Array_initializer_list_vector_cxx11.cpp
@@ -0,0 +1,2 @@ +Array<int, Dynamic, 1> v {{1, 2, 3, 4, 5}}; +cout << v << endl; \ No newline at end of file
diff --git a/doc/snippets/Array_variadic_ctor_cxx11.cpp b/doc/snippets/Array_variadic_ctor_cxx11.cpp new file mode 100644 index 0000000..234c7a7 --- /dev/null +++ b/doc/snippets/Array_variadic_ctor_cxx11.cpp
@@ -0,0 +1,3 @@ +Array<int, 1, 6> a(1, 2, 3, 4, 5, 6); +Array<int, 3, 1> b {1, 2, 3}; +cout << a << "\n\n" << b << endl; \ No newline at end of file
diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index 1135900..cf3f6de 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt
@@ -6,23 +6,28 @@ get_filename_component(snippet ${snippet_src} NAME_WE) set(compile_snippet_target compile_${snippet}) set(compile_snippet_src ${compile_snippet_target}.cpp) - file(READ ${snippet_src} snippet_source_code) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/compile_snippet.cpp.in - ${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src}) - add_executable(${compile_snippet_target} - ${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src}) - if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) - target_link_libraries(${compile_snippet_target} ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO}) + if((NOT ${snippet_src} MATCHES "cxx11") OR EIGEN_COMPILER_SUPPORT_CPP11) + file(READ ${snippet_src} snippet_source_code) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/compile_snippet.cpp.in + ${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src}) + add_executable(${compile_snippet_target} + ${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src}) + if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) + target_link_libraries(${compile_snippet_target} ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO}) + endif() + if(${snippet_src} MATCHES "cxx11") + set_target_properties(${compile_snippet_target} PROPERTIES COMPILE_FLAGS "-std=c++11") + endif() + add_custom_command( + TARGET ${compile_snippet_target} + POST_BUILD + COMMAND ${compile_snippet_target} + ARGS >${CMAKE_CURRENT_BINARY_DIR}/${snippet}.out + ) + add_dependencies(all_snippets ${compile_snippet_target}) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src} + PROPERTIES OBJECT_DEPENDS ${snippet_src}) + else() + message("skip snippet ${snippet_src} because compiler does not support C++11") endif() - add_custom_command( - TARGET ${compile_snippet_target} - POST_BUILD - COMMAND ${compile_snippet_target} - ARGS >${CMAKE_CURRENT_BINARY_DIR}/${snippet}.out - ) - add_dependencies(all_snippets ${compile_snippet_target}) - set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${compile_snippet_src} - PROPERTIES OBJECT_DEPENDS ${snippet_src}) endforeach(snippet_src) - -ei_add_target_property(compile_tut_arithmetic_transpose_aliasing COMPILE_FLAGS -DEIGEN_NO_DEBUG)
diff --git a/doc/snippets/Cwise_boolean_xor.cpp b/doc/snippets/Cwise_boolean_xor.cpp new file mode 100644 index 0000000..fafbec8 --- /dev/null +++ b/doc/snippets/Cwise_boolean_xor.cpp
@@ -0,0 +1,2 @@ +Array3d v(-1,2,1), w(-3,2,3); +cout << ((v<w) ^ (v<0)) << endl;
diff --git a/doc/snippets/Cwise_erf.cpp b/doc/snippets/Cwise_erf.cpp deleted file mode 100644 index 7f51c1b..0000000 --- a/doc/snippets/Cwise_erf.cpp +++ /dev/null
@@ -1,2 +0,0 @@ -Array4d v(-0.5,2,0,-7); -cout << v.erf() << endl;
diff --git a/doc/snippets/Cwise_erfc.cpp b/doc/snippets/Cwise_erfc.cpp deleted file mode 100644 index f0453d4..0000000 --- a/doc/snippets/Cwise_erfc.cpp +++ /dev/null
@@ -1,2 +0,0 @@ -Array4d v(-0.5,2,0,-7); -cout << v.erfc() << endl;
diff --git a/doc/snippets/Cwise_lgamma.cpp b/doc/snippets/Cwise_lgamma.cpp deleted file mode 100644 index cbc69b9..0000000 --- a/doc/snippets/Cwise_lgamma.cpp +++ /dev/null
@@ -1,2 +0,0 @@ -Array4d v(0.5,10,0,-1); -cout << v.lgamma() << endl; \ No newline at end of file
diff --git a/doc/snippets/DenseBase_LinSpacedInt.cpp b/doc/snippets/DenseBase_LinSpacedInt.cpp new file mode 100644 index 0000000..0d7ae06 --- /dev/null +++ b/doc/snippets/DenseBase_LinSpacedInt.cpp
@@ -0,0 +1,8 @@ +cout << "Even spacing inputs:" << endl; +cout << VectorXi::LinSpaced(8,1,4).transpose() << endl; +cout << VectorXi::LinSpaced(8,1,8).transpose() << endl; +cout << VectorXi::LinSpaced(8,1,15).transpose() << endl; +cout << "Uneven spacing inputs:" << endl; +cout << VectorXi::LinSpaced(8,1,7).transpose() << endl; +cout << VectorXi::LinSpaced(8,1,9).transpose() << endl; +cout << VectorXi::LinSpaced(8,1,16).transpose() << endl;
diff --git a/doc/snippets/DirectionWise_hnormalized.cpp b/doc/snippets/DirectionWise_hnormalized.cpp index 3410790..2451f6e 100644 --- a/doc/snippets/DirectionWise_hnormalized.cpp +++ b/doc/snippets/DirectionWise_hnormalized.cpp
@@ -1,7 +1,6 @@ -typedef Matrix<double,4,Dynamic> Matrix4Xd; Matrix4Xd M = Matrix4Xd::Random(4,5); Projective3d P(Matrix4d::Random()); cout << "The matrix M is:" << endl << M << endl << endl; cout << "M.colwise().hnormalized():" << endl << M.colwise().hnormalized() << endl << endl; cout << "P*M:" << endl << P*M << endl << endl; -cout << "(P*M).colwise().hnormalized():" << endl << (P*M).colwise().hnormalized() << endl << endl; \ No newline at end of file +cout << "(P*M).colwise().hnormalized():" << endl << (P*M).colwise().hnormalized() << endl << endl;
diff --git a/doc/snippets/MatrixBase_colwise_iterator_cxx11.cpp b/doc/snippets/MatrixBase_colwise_iterator_cxx11.cpp new file mode 100644 index 0000000..88a9cd3 --- /dev/null +++ b/doc/snippets/MatrixBase_colwise_iterator_cxx11.cpp
@@ -0,0 +1,12 @@ +Matrix3i m = Matrix3i::Random(); +cout << "Here is the initial matrix m:" << endl << m << endl; +int i = -1; +for(auto c: m.colwise()) { + c *= i; + ++i; +} +cout << "Here is the matrix m after the for-range-loop:" << endl << m << endl; +auto cols = m.colwise(); +auto it = std::find_if(cols.cbegin(), cols.cend(), + [](Matrix3i::ConstColXpr x) { return x.squaredNorm() == 0; }); +cout << "The first empty column is: " << distance(cols.cbegin(),it) << endl; \ No newline at end of file
diff --git a/doc/snippets/MatrixBase_cwiseEqual.cpp b/doc/snippets/MatrixBase_cwiseEqual.cpp index eb3656f..469af64 100644 --- a/doc/snippets/MatrixBase_cwiseEqual.cpp +++ b/doc/snippets/MatrixBase_cwiseEqual.cpp
@@ -3,5 +3,5 @@ 1, 1; cout << "Comparing m with identity matrix:" << endl; cout << m.cwiseEqual(MatrixXi::Identity(2,2)) << endl; -int count = m.cwiseEqual(MatrixXi::Identity(2,2)).count(); +Index count = m.cwiseEqual(MatrixXi::Identity(2,2)).count(); cout << "Number of coefficients that are equal: " << count << endl;
diff --git a/doc/snippets/MatrixBase_cwiseNotEqual.cpp b/doc/snippets/MatrixBase_cwiseNotEqual.cpp index 6a2e4fb..7f0a105 100644 --- a/doc/snippets/MatrixBase_cwiseNotEqual.cpp +++ b/doc/snippets/MatrixBase_cwiseNotEqual.cpp
@@ -3,5 +3,5 @@ 1, 1; cout << "Comparing m with identity matrix:" << endl; cout << m.cwiseNotEqual(MatrixXi::Identity(2,2)) << endl; -int count = m.cwiseNotEqual(MatrixXi::Identity(2,2)).count(); +Index count = m.cwiseNotEqual(MatrixXi::Identity(2,2)).count(); cout << "Number of coefficients that are not equal: " << count << endl;
diff --git a/doc/snippets/MatrixBase_reshaped_auto.cpp b/doc/snippets/MatrixBase_reshaped_auto.cpp new file mode 100644 index 0000000..59f9d3f --- /dev/null +++ b/doc/snippets/MatrixBase_reshaped_auto.cpp
@@ -0,0 +1,4 @@ +Matrix4i m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped(2, AutoSize):" << endl << m.reshaped(2, AutoSize) << endl; +cout << "Here is m.reshaped<RowMajor>(AutoSize, fix<8>):" << endl << m.reshaped<RowMajor>(AutoSize, fix<8>) << endl;
diff --git a/doc/snippets/MatrixBase_reshaped_fixed.cpp b/doc/snippets/MatrixBase_reshaped_fixed.cpp new file mode 100644 index 0000000..3e9e2cf --- /dev/null +++ b/doc/snippets/MatrixBase_reshaped_fixed.cpp
@@ -0,0 +1,3 @@ +Matrix4i m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped(fix<2>,fix<8>):" << endl << m.reshaped(fix<2>,fix<8>) << endl;
diff --git a/doc/snippets/MatrixBase_reshaped_int_int.cpp b/doc/snippets/MatrixBase_reshaped_int_int.cpp new file mode 100644 index 0000000..af4ca59 --- /dev/null +++ b/doc/snippets/MatrixBase_reshaped_int_int.cpp
@@ -0,0 +1,3 @@ +Matrix4i m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped(2, 8):" << endl << m.reshaped(2, 8) << endl;
diff --git a/doc/snippets/MatrixBase_reshaped_to_vector.cpp b/doc/snippets/MatrixBase_reshaped_to_vector.cpp new file mode 100644 index 0000000..37f65f7 --- /dev/null +++ b/doc/snippets/MatrixBase_reshaped_to_vector.cpp
@@ -0,0 +1,4 @@ +Matrix4i m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped().transpose():" << endl << m.reshaped().transpose() << endl; +cout << "Here is m.reshaped<RowMajor>().transpose(): " << endl << m.reshaped<RowMajor>().transpose() << endl;
diff --git a/doc/snippets/MatrixBase_selfadjointView.cpp b/doc/snippets/MatrixBase_selfadjointView.cpp new file mode 100644 index 0000000..4bd3c7e --- /dev/null +++ b/doc/snippets/MatrixBase_selfadjointView.cpp
@@ -0,0 +1,6 @@ +Matrix3i m = Matrix3i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is the symmetric matrix extracted from the upper part of m:" << endl + << Matrix3i(m.selfadjointView<Upper>()) << endl; +cout << "Here is the symmetric matrix extracted from the lower part of m:" << endl + << Matrix3i(m.selfadjointView<Lower>()) << endl;
diff --git a/doc/snippets/Matrix_Map_stride.cpp b/doc/snippets/Matrix_Map_stride.cpp new file mode 100644 index 0000000..ae42a12 --- /dev/null +++ b/doc/snippets/Matrix_Map_stride.cpp
@@ -0,0 +1,7 @@ +Matrix4i A; +A << 1, 2, 3, 4, + 5, 6, 7, 8, + 9, 10, 11, 12, + 13, 14, 15, 16; + +std::cout << Matrix2i::Map(&A(1,1),Stride<8,2>()) << std::endl;
diff --git a/doc/snippets/Matrix_initializer_list_23_cxx11.cpp b/doc/snippets/Matrix_initializer_list_23_cxx11.cpp new file mode 100644 index 0000000..d338d02 --- /dev/null +++ b/doc/snippets/Matrix_initializer_list_23_cxx11.cpp
@@ -0,0 +1,5 @@ +MatrixXd m { + {1, 2, 3}, + {4, 5, 6} +}; +cout << m << endl; \ No newline at end of file
diff --git a/doc/snippets/Matrix_initializer_list_vector_cxx11.cpp b/doc/snippets/Matrix_initializer_list_vector_cxx11.cpp new file mode 100644 index 0000000..8872e2c --- /dev/null +++ b/doc/snippets/Matrix_initializer_list_vector_cxx11.cpp
@@ -0,0 +1,2 @@ +VectorXi v {{1, 2}}; +cout << v << endl; \ No newline at end of file
diff --git a/doc/snippets/Matrix_variadic_ctor_cxx11.cpp b/doc/snippets/Matrix_variadic_ctor_cxx11.cpp new file mode 100644 index 0000000..fcb4ccf --- /dev/null +++ b/doc/snippets/Matrix_variadic_ctor_cxx11.cpp
@@ -0,0 +1,3 @@ +Matrix<int, 1, 6> a(1, 2, 3, 4, 5, 6); +Matrix<int, 3, 1> b {1, 2, 3}; +cout << a << "\n\n" << b << endl; \ No newline at end of file
diff --git a/doc/snippets/Slicing_arrayexpr.cpp b/doc/snippets/Slicing_arrayexpr.cpp new file mode 100644 index 0000000..00b689b --- /dev/null +++ b/doc/snippets/Slicing_arrayexpr.cpp
@@ -0,0 +1,4 @@ +ArrayXi ind(5); ind<<4,2,5,5,3; +MatrixXi A = MatrixXi::Random(4,6); +cout << "Initial matrix A:\n" << A << "\n\n"; +cout << "A(all,ind-1):\n" << A(all,ind-1) << "\n\n"; \ No newline at end of file
diff --git a/doc/snippets/Slicing_custom_padding_cxx11.cpp b/doc/snippets/Slicing_custom_padding_cxx11.cpp new file mode 100644 index 0000000..5dac3d7 --- /dev/null +++ b/doc/snippets/Slicing_custom_padding_cxx11.cpp
@@ -0,0 +1,12 @@ +struct pad { + Index size() const { return out_size; } + Index operator[] (Index i) const { return std::max<Index>(0,i-(out_size-in_size)); } + Index in_size, out_size; +}; + +Matrix3i A; +A.reshaped() = VectorXi::LinSpaced(9,1,9); +cout << "Initial matrix A:\n" << A << "\n\n"; +MatrixXi B(5,5); +B = A(pad{3,5}, pad{3,5}); +cout << "A(pad{3,N}, pad{3,N}):\n" << B << "\n\n"; \ No newline at end of file
diff --git a/doc/snippets/Slicing_rawarray_cxx11.cpp b/doc/snippets/Slicing_rawarray_cxx11.cpp new file mode 100644 index 0000000..0d3287a --- /dev/null +++ b/doc/snippets/Slicing_rawarray_cxx11.cpp
@@ -0,0 +1,5 @@ +#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE +MatrixXi A = MatrixXi::Random(4,6); +cout << "Initial matrix A:\n" << A << "\n\n"; +cout << "A(all,{4,2,5,5,3}):\n" << A(all,{4,2,5,5,3}) << "\n\n"; +#endif \ No newline at end of file
diff --git a/doc/snippets/Slicing_stdvector_cxx11.cpp b/doc/snippets/Slicing_stdvector_cxx11.cpp new file mode 100644 index 0000000..469651e --- /dev/null +++ b/doc/snippets/Slicing_stdvector_cxx11.cpp
@@ -0,0 +1,4 @@ +std::vector<int> ind{4,2,5,5,3}; +MatrixXi A = MatrixXi::Random(4,6); +cout << "Initial matrix A:\n" << A << "\n\n"; +cout << "A(all,ind):\n" << A(all,ind) << "\n\n"; \ No newline at end of file
diff --git a/doc/snippets/SparseMatrix_coeffs.cpp b/doc/snippets/SparseMatrix_coeffs.cpp new file mode 100644 index 0000000..f71a69b --- /dev/null +++ b/doc/snippets/SparseMatrix_coeffs.cpp
@@ -0,0 +1,9 @@ +SparseMatrix<double> A(3,3); +A.insert(1,2) = 0; +A.insert(0,1) = 1; +A.insert(2,0) = 2; +A.makeCompressed(); +cout << "The matrix A is:" << endl << MatrixXd(A) << endl; +cout << "it has " << A.nonZeros() << " stored non zero coefficients that are: " << A.coeffs().transpose() << endl; +A.coeffs() += 10; +cout << "After adding 10 to every stored non zero coefficient, the matrix A is:" << endl << MatrixXd(A) << endl;
diff --git a/doc/snippets/Tutorial_range_for_loop_1d_cxx11.cpp b/doc/snippets/Tutorial_range_for_loop_1d_cxx11.cpp new file mode 100644 index 0000000..d8e582a --- /dev/null +++ b/doc/snippets/Tutorial_range_for_loop_1d_cxx11.cpp
@@ -0,0 +1,4 @@ +VectorXi v = VectorXi::Random(4); +cout << "Here is the vector v:\n"; +for(auto x : v) cout << x << " "; +cout << "\n"; \ No newline at end of file
diff --git a/doc/snippets/Tutorial_range_for_loop_2d_cxx11.cpp b/doc/snippets/Tutorial_range_for_loop_2d_cxx11.cpp new file mode 100644 index 0000000..7e532d3 --- /dev/null +++ b/doc/snippets/Tutorial_range_for_loop_2d_cxx11.cpp
@@ -0,0 +1,5 @@ +Matrix2i A = Matrix2i::Random(); +cout << "Here are the coeffs of the 2x2 matrix A:\n"; +for(auto x : A.reshaped()) + cout << x << " "; +cout << "\n"; \ No newline at end of file
diff --git a/doc/snippets/Tutorial_reshaped_vs_resize_1.cpp b/doc/snippets/Tutorial_reshaped_vs_resize_1.cpp new file mode 100644 index 0000000..686b96c --- /dev/null +++ b/doc/snippets/Tutorial_reshaped_vs_resize_1.cpp
@@ -0,0 +1,5 @@ +MatrixXi m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped(2, 8):" << endl << m.reshaped(2, 8) << endl; +m.resize(2,8); +cout << "Here is the matrix m after m.resize(2,8):" << endl << m << endl; \ No newline at end of file
diff --git a/doc/snippets/Tutorial_reshaped_vs_resize_2.cpp b/doc/snippets/Tutorial_reshaped_vs_resize_2.cpp new file mode 100644 index 0000000..50dc454 --- /dev/null +++ b/doc/snippets/Tutorial_reshaped_vs_resize_2.cpp
@@ -0,0 +1,6 @@ +Matrix<int,Dynamic,Dynamic,RowMajor> m = Matrix4i::Random(); +cout << "Here is the matrix m:" << endl << m << endl; +cout << "Here is m.reshaped(2, 8):" << endl << m.reshaped(2, 8) << endl; +cout << "Here is m.reshaped<AutoOrder>(2, 8):" << endl << m.reshaped<AutoOrder>(2, 8) << endl; +m.resize(2,8); +cout << "Here is the matrix m after m.resize(2,8):" << endl << m << endl;
diff --git a/doc/snippets/Tutorial_std_sort.cpp b/doc/snippets/Tutorial_std_sort.cpp new file mode 100644 index 0000000..8f89368 --- /dev/null +++ b/doc/snippets/Tutorial_std_sort.cpp
@@ -0,0 +1,4 @@ +Array4i v = Array4i::Random().abs(); +cout << "Here is the initial vector v:\n" << v.transpose() << "\n"; +std::sort(v.begin(), v.end()); +cout << "Here is the sorted vector v:\n" << v.transpose() << "\n"; \ No newline at end of file
diff --git a/doc/snippets/Tutorial_std_sort_rows_cxx11.cpp b/doc/snippets/Tutorial_std_sort_rows_cxx11.cpp new file mode 100644 index 0000000..fdd850d --- /dev/null +++ b/doc/snippets/Tutorial_std_sort_rows_cxx11.cpp
@@ -0,0 +1,5 @@ +ArrayXXi A = ArrayXXi::Random(4,4).abs(); +cout << "Here is the initial matrix A:\n" << A << "\n"; +for(auto row : A.rowwise()) + std::sort(row.begin(), row.end()); +cout << "Here is the sorted matrix A:\n" << A << "\n"; \ No newline at end of file
diff --git a/doc/snippets/VectorwiseOp_homogeneous.cpp b/doc/snippets/VectorwiseOp_homogeneous.cpp index aba4fed..67cf573 100644 --- a/doc/snippets/VectorwiseOp_homogeneous.cpp +++ b/doc/snippets/VectorwiseOp_homogeneous.cpp
@@ -1,7 +1,6 @@ -typedef Matrix<double,3,Dynamic> Matrix3Xd; Matrix3Xd M = Matrix3Xd::Random(3,5); Projective3d P(Matrix4d::Random()); cout << "The matrix M is:" << endl << M << endl << endl; cout << "M.colwise().homogeneous():" << endl << M.colwise().homogeneous() << endl << endl; cout << "P * M.colwise().homogeneous():" << endl << P * M.colwise().homogeneous() << endl << endl; -cout << "P * M.colwise().homogeneous().hnormalized(): " << endl << (P * M.colwise().homogeneous()).colwise().hnormalized() << endl << endl; \ No newline at end of file +cout << "P * M.colwise().homogeneous().hnormalized(): " << endl << (P * M.colwise().homogeneous()).colwise().hnormalized() << endl << endl;
diff --git a/doc/snippets/compile_snippet.cpp.in b/doc/snippets/compile_snippet.cpp.in index fdae39b..d63f371 100644 --- a/doc/snippets/compile_snippet.cpp.in +++ b/doc/snippets/compile_snippet.cpp.in
@@ -1,5 +1,8 @@ -#include <Eigen/Eigen> +static bool eigen_did_assert = false; +#define eigen_assert(X) if(!eigen_did_assert && !(X)){ std::cout << "### Assertion raised in " << __FILE__ << ":" << __LINE__ << ":\n" #X << "\n### The following would happen without assertions:\n"; eigen_did_assert = true;} + #include <iostream> +#include <Eigen/Eigen> #ifndef M_PI #define M_PI 3.1415926535897932384626433832795
diff --git a/doc/special_examples/CMakeLists.txt b/doc/special_examples/CMakeLists.txt index 101fbc5..66ba4de 100644 --- a/doc/special_examples/CMakeLists.txt +++ b/doc/special_examples/CMakeLists.txt
@@ -19,7 +19,6 @@ add_dependencies(all_examples Tutorial_sparse_example) endif(QT4_FOUND) -check_cxx_compiler_flag("-std=c++11" EIGEN_COMPILER_SUPPORT_CPP11) if(EIGEN_COMPILER_SUPPORT_CPP11) add_executable(random_cpp11 random_cpp11.cpp) target_link_libraries(random_cpp11 ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO})
diff --git a/doc/special_examples/Tutorial_sparse_example.cpp b/doc/special_examples/Tutorial_sparse_example.cpp index 830e196..8850db0 100644 --- a/doc/special_examples/Tutorial_sparse_example.cpp +++ b/doc/special_examples/Tutorial_sparse_example.cpp
@@ -1,5 +1,6 @@ #include <Eigen/Sparse> #include <vector> +#include <iostream> typedef Eigen::SparseMatrix<double> SpMat; // declares a column-major sparse matrix type of double typedef Eigen::Triplet<double> T; @@ -9,10 +10,13 @@ int main(int argc, char** argv) { - assert(argc==2); + if(argc!=2) { + std::cerr << "Error: expected one and only one argument.\n"; + return -1; + } int n = 300; // size of the image - int m = n*n; // number of unknows (=number of pixels) + int m = n*n; // number of unknowns (=number of pixels) // Assembly: std::vector<T> coefficients; // list of non-zeros coefficients
diff --git a/doc/special_examples/random_cpp11.cpp b/doc/special_examples/random_cpp11.cpp index adc3c11..33744c0 100644 --- a/doc/special_examples/random_cpp11.cpp +++ b/doc/special_examples/random_cpp11.cpp
@@ -7,7 +7,7 @@ int main() { std::default_random_engine generator; std::poisson_distribution<int> distribution(4.1); - auto poisson = [&] (Eigen::Index) {return distribution(generator);}; + auto poisson = [&] () {return distribution(generator);}; RowVectorXi v = RowVectorXi::NullaryExpr(10, poisson ); std::cout << v << "\n";
diff --git a/failtest/CMakeLists.txt b/failtest/CMakeLists.txt index 1a73f05..256e541 100644 --- a/failtest/CMakeLists.txt +++ b/failtest/CMakeLists.txt
@@ -1,4 +1,3 @@ -message(STATUS "Running the failtests") ei_add_failtest("failtest_sanity_check") @@ -64,12 +63,8 @@ ei_add_failtest("eigensolver_int") ei_add_failtest("eigensolver_cplx") -if (EIGEN_FAILTEST_FAILURE_COUNT) - message(FATAL_ERROR - "${EIGEN_FAILTEST_FAILURE_COUNT} out of ${EIGEN_FAILTEST_COUNT} failtests FAILED. " - "To debug these failures, manually compile these programs in ${CMAKE_CURRENT_SOURCE_DIR}, " - "with and without #define EIGEN_SHOULD_FAIL_TO_BUILD.") -else() - message(STATUS "Failtest SUCCESS: all ${EIGEN_FAILTEST_COUNT} failtests passed.") - message(STATUS "") +if(EIGEN_TEST_CXX11) + ei_add_failtest("initializer_list_1") + ei_add_failtest("initializer_list_2") endif() +
diff --git a/failtest/initializer_list_1.cpp b/failtest/initializer_list_1.cpp new file mode 100644 index 0000000..92dfd1f --- /dev/null +++ b/failtest/initializer_list_1.cpp
@@ -0,0 +1,14 @@ +#include "../Eigen/Core" + +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD +#define ROWS Dynamic +#else +#define ROWS 3 +#endif + +using namespace Eigen; + +int main() +{ + Matrix<int, ROWS, 1> {1, 2, 3}; +}
diff --git a/failtest/initializer_list_2.cpp b/failtest/initializer_list_2.cpp new file mode 100644 index 0000000..1996050 --- /dev/null +++ b/failtest/initializer_list_2.cpp
@@ -0,0 +1,16 @@ +#include "../Eigen/Core" + +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD +#define ROWS Dynamic +#define COLS Dynamic +#else +#define ROWS 3 +#define COLS 1 +#endif + +using namespace Eigen; + +int main() +{ + Matrix<int, ROWS, COLS> {1, 2, 3}; +}
diff --git a/lapack/CMakeLists.txt b/lapack/CMakeLists.txt index 9883d4c..9b0d863 100644 --- a/lapack/CMakeLists.txt +++ b/lapack/CMakeLists.txt
@@ -35,7 +35,7 @@ second_NONE.f dsecnd_NONE.f ) -option(EIGEN_ENABLE_LAPACK_TESTS OFF "Enbale the Lapack unit tests") +option(EIGEN_ENABLE_LAPACK_TESTS OFF "Enable the Lapack unit tests") if(EIGEN_ENABLE_LAPACK_TESTS) @@ -49,7 +49,7 @@ INACTIVITY_TIMEOUT 15 TIMEOUT 240 STATUS download_status - EXPECTED_MD5 5758ce55afcf79da98de8b9de1615ad5 + EXPECTED_MD5 ab5742640617e3221a873aba44bbdc93 SHOW_PROGRESS) message(STATUS ${download_status}) @@ -59,7 +59,7 @@ message(STATUS "Setup lapack reference and lapack unit tests") execute_process(COMMAND tar xzf "lapack_addons_3.4.1.tgz" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) else() - message(STATUS "Download of lapack_addons_3.4.1.tgz failed, LAPACK unit tests wont be enabled") + message(STATUS "Download of lapack_addons_3.4.1.tgz failed, LAPACK unit tests won't be enabled") set(EIGEN_ENABLE_LAPACK_TESTS false) endif() @@ -136,6 +136,7 @@ add_subdirectory(testing/MATGEN) add_subdirectory(testing/LIN) add_subdirectory(testing/EIG) + cmake_policy(SET CMP0026 OLD) macro(add_lapack_test output input target) set(TEST_INPUT "${LAPACK_SOURCE_DIR}/testing/${input}") set(TEST_OUTPUT "${LAPACK_BINARY_DIR}/testing/${output}")
diff --git a/lapack/svd.cpp b/lapack/svd.cpp index df77a37..77b302b 100644 --- a/lapack/svd.cpp +++ b/lapack/svd.cpp
@@ -124,14 +124,15 @@ JacobiSVD<PlainMatrixType> svd(mat,option); make_vector(s,diag_size) = svd.singularValues().head(diag_size); - + { if(*jobu=='A') matrix(u,*m,*m,*ldu) = svd.matrixU(); else if(*jobu=='S') matrix(u,*m,diag_size,*ldu) = svd.matrixU(); - else if(*jobu=='O') matrix(a,*m,diag_size,*lda) = svd.matrixU(); - + else if(*jobu=='O') matrix(a,*m,diag_size,*lda) = svd.matrixU(); + } + { if(*jobv=='A') matrix(vt,*n,*n,*ldvt) = svd.matrixV().adjoint(); else if(*jobv=='S') matrix(vt,diag_size,*n,*ldvt) = svd.matrixV().adjoint(); else if(*jobv=='O') matrix(a,diag_size,*n,*lda) = svd.matrixV().adjoint(); - + } return 0; }
diff --git a/scripts/buildtests.in b/scripts/buildtests.in index d2fd102..ab9c18f 100755 --- a/scripts/buildtests.in +++ b/scripts/buildtests.in
@@ -2,7 +2,7 @@ if [[ $# != 1 || $1 == *help ]] then - echo "usage: ./check regexp" + echo "usage: $0 regexp" echo " Builds tests matching the regexp." echo " The EIGEN_MAKE_ARGS environment variable allows to pass args to 'make'." echo " For example, to launch 5 concurrent builds, use EIGEN_MAKE_ARGS='-j5'" @@ -10,7 +10,7 @@ fi TESTSLIST="@EIGEN_TESTS_LIST@" -targets_to_make=`echo "$TESTSLIST" | egrep "$1" | xargs echo` +targets_to_make=$(echo "$TESTSLIST" | grep -E "$1" | xargs echo) if [ -n "${EIGEN_MAKE_ARGS:+x}" ] then
diff --git a/scripts/check.in b/scripts/check.in index a90061a..7717e2d 100755 --- a/scripts/check.in +++ b/scripts/check.in
@@ -3,7 +3,7 @@ if [[ $# != 1 || $1 == *help ]] then - echo "usage: ./check regexp" + echo "usage: $0 regexp" echo " Builds and runs tests matching the regexp." echo " The EIGEN_MAKE_ARGS environment variable allows to pass args to 'make'." echo " For example, to launch 5 concurrent builds, use EIGEN_MAKE_ARGS='-j5'"
diff --git a/scripts/eigen_gen_split_test_help.cmake b/scripts/eigen_gen_split_test_help.cmake new file mode 100644 index 0000000..e43f5aa --- /dev/null +++ b/scripts/eigen_gen_split_test_help.cmake
@@ -0,0 +1,11 @@ +#!cmake -P +file(WRITE split_test_helper.h "") +foreach(i RANGE 1 999) + file(APPEND split_test_helper.h + "#if defined(EIGEN_TEST_PART_${i}) || defined(EIGEN_TEST_PART_ALL)\n" + "#define CALL_SUBTEST_${i}(FUNC) CALL_SUBTEST(FUNC)\n" + "#else\n" + "#define CALL_SUBTEST_${i}(FUNC)\n" + "#endif\n\n" + ) +endforeach() \ No newline at end of file
diff --git a/scripts/eigen_monitor_perf.sh b/scripts/eigen_monitor_perf.sh new file mode 100755 index 0000000..8f3425d --- /dev/null +++ b/scripts/eigen_monitor_perf.sh
@@ -0,0 +1,25 @@ +#!/bin/bash + +# This is a script example to automatically update and upload performance unit tests. +# The following five variables must be adjusted to match your settings. + +USER='ggael' +UPLOAD_DIR=perf_monitoring/ggaelmacbook26 +EIGEN_SOURCE_PATH=$HOME/Eigen/eigen +export PREFIX="haswell-fma" +export CXX_FLAGS="-mfma -w" + +#### + +BENCH_PATH=$EIGEN_SOURCE_PATH/bench/perf_monitoring/$PREFIX +PREVPATH=$(pwd) +cd $EIGEN_SOURCE_PATH/bench/perf_monitoring && ./runall.sh "Haswell 2.6GHz, FMA, Apple's clang" "$@" +cd $PREVPATH || exit 1 + +ALLFILES="$BENCH_PATH/*.png $BENCH_PATH/*.html $BENCH_PATH/index.html $BENCH_PATH/s1.js $BENCH_PATH/s2.js" + +# (the '/' at the end of path is very important, see rsync documentation) +rsync -az --no-p --delete $ALLFILES $USER@ssh.tuxfamily.org:eigen/eigen.tuxfamily.org-web/htdocs/$UPLOAD_DIR/ || { echo "upload failed"; exit 1; } + +# fix the perm +ssh $USER@ssh.tuxfamily.org "chmod -R g+w /home/eigen/eigen.tuxfamily.org-web/htdocs/perf_monitoring" || { echo "perm failed"; exit 1; }
diff --git a/test/AnnoyingScalar.h b/test/AnnoyingScalar.h new file mode 100644 index 0000000..2b6544a --- /dev/null +++ b/test/AnnoyingScalar.h
@@ -0,0 +1,154 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2011-2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TEST_ANNOYING_SCALAR_H +#define EIGEN_TEST_ANNOYING_SCALAR_H + +#include <ostream> + +#ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW +struct my_exception +{ + my_exception() {} + ~my_exception() {} +}; +#endif + +// An AnnoyingScalar is a pseudo scalar type that: +// - can randomly through an exception in operator + +// - randomly allocate on the heap or initialize a reference to itself making it non trivially copyable, nor movable, nor relocatable. + +class AnnoyingScalar +{ + public: + AnnoyingScalar() { init(); *v = 0; } + AnnoyingScalar(long double _v) { init(); *v = _v; } + AnnoyingScalar(double _v) { init(); *v = _v; } + AnnoyingScalar(float _v) { init(); *v = _v; } + AnnoyingScalar(int _v) { init(); *v = _v; } + AnnoyingScalar(long _v) { init(); *v = _v; } + #if EIGEN_HAS_CXX11 + AnnoyingScalar(long long _v) { init(); *v = _v; } + #endif + AnnoyingScalar(const AnnoyingScalar& other) { init(); *v = *(other.v); } + ~AnnoyingScalar() { + if(v!=&data) + delete v; + instances--; + } + + void init() { + if(internal::random<bool>()) + v = new float; + else + v = &data; + instances++; + } + + AnnoyingScalar operator+(const AnnoyingScalar& other) const + { + #ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW + countdown--; + if(countdown<=0 && !dont_throw) + throw my_exception(); + #endif + return AnnoyingScalar(*v+*other.v); + } + + AnnoyingScalar operator-() const + { return AnnoyingScalar(-*v); } + + AnnoyingScalar operator-(const AnnoyingScalar& other) const + { return AnnoyingScalar(*v-*other.v); } + + AnnoyingScalar operator*(const AnnoyingScalar& other) const + { return AnnoyingScalar((*v)*(*other.v)); } + + AnnoyingScalar operator/(const AnnoyingScalar& other) const + { return AnnoyingScalar((*v)/(*other.v)); } + + AnnoyingScalar& operator+=(const AnnoyingScalar& other) { *v += *other.v; return *this; } + AnnoyingScalar& operator-=(const AnnoyingScalar& other) { *v -= *other.v; return *this; } + AnnoyingScalar& operator*=(const AnnoyingScalar& other) { *v *= *other.v; return *this; } + AnnoyingScalar& operator/=(const AnnoyingScalar& other) { *v /= *other.v; return *this; } + AnnoyingScalar& operator= (const AnnoyingScalar& other) { *v = *other.v; return *this; } + + bool operator==(const AnnoyingScalar& other) const { return *v == *other.v; } + bool operator!=(const AnnoyingScalar& other) const { return *v != *other.v; } + bool operator<=(const AnnoyingScalar& other) const { return *v <= *other.v; } + bool operator< (const AnnoyingScalar& other) const { return *v < *other.v; } + bool operator>=(const AnnoyingScalar& other) const { return *v >= *other.v; } + bool operator> (const AnnoyingScalar& other) const { return *v > *other.v; } + + float* v; + float data; + static int instances; +#ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW + static int countdown; + static bool dont_throw; +#endif +}; + +AnnoyingScalar real(const AnnoyingScalar &x) { return x; } +AnnoyingScalar imag(const AnnoyingScalar & ) { return 0; } +AnnoyingScalar conj(const AnnoyingScalar &x) { return x; } +AnnoyingScalar sqrt(const AnnoyingScalar &x) { return std::sqrt(*x.v); } +AnnoyingScalar abs (const AnnoyingScalar &x) { return std::abs(*x.v); } +AnnoyingScalar cos (const AnnoyingScalar &x) { return std::cos(*x.v); } +AnnoyingScalar sin (const AnnoyingScalar &x) { return std::sin(*x.v); } +AnnoyingScalar acos(const AnnoyingScalar &x) { return std::acos(*x.v); } +AnnoyingScalar atan2(const AnnoyingScalar &y,const AnnoyingScalar &x) { return std::atan2(*y.v,*x.v); } + +std::ostream& operator<<(std::ostream& stream,const AnnoyingScalar& x) { + stream << (*(x.v)); + return stream; +} + +int AnnoyingScalar::instances = 0; + +#ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW +int AnnoyingScalar::countdown = 0; +bool AnnoyingScalar::dont_throw = false; +#endif + +namespace Eigen { +template<> +struct NumTraits<AnnoyingScalar> : NumTraits<float> +{ + enum { + RequireInitialization = true + }; + typedef AnnoyingScalar Real; + typedef AnnoyingScalar Nested; + typedef AnnoyingScalar Literal; + typedef AnnoyingScalar NonInteger; +}; + +template<> inline AnnoyingScalar test_precision<AnnoyingScalar>() { return test_precision<float>(); } + +namespace internal { + template<> double cast(const AnnoyingScalar& x) { return double(*x.v); } + template<> float cast(const AnnoyingScalar& x) { return *x.v; } +} + +} + +AnnoyingScalar get_test_precision(const AnnoyingScalar&) +{ return Eigen::test_precision<AnnoyingScalar>(); } + +AnnoyingScalar test_relative_error(const AnnoyingScalar &a, const AnnoyingScalar &b) +{ return test_relative_error(*a.v, *b.v); } + +inline bool test_isApprox(const AnnoyingScalar &a, const AnnoyingScalar &b) +{ return internal::isApprox(*a.v, *b.v, test_precision<float>()); } + +inline bool test_isMuchSmallerThan(const AnnoyingScalar &a, const AnnoyingScalar &b) +{ return test_isMuchSmallerThan(*a.v, *b.v); } + +#endif // EIGEN_TEST_ANNOYING_SCALAR_H
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7bed6a4..3dbb426 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt
@@ -1,16 +1,7 @@ -# generate split test header file only if it does not yet exist -# in order to prevent a rebuild everytime cmake is configured -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h "") - foreach(i RANGE 1 999) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h - "#ifdef EIGEN_TEST_PART_${i}\n" - "#define CALL_SUBTEST_${i}(FUNC) CALL_SUBTEST(FUNC)\n" - "#else\n" - "#define CALL_SUBTEST_${i}(FUNC)\n" - "#endif\n\n" - ) - endforeach() +# The file split_test_helper.h was generated at first run, +# it is now included in test/ +if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) endif() # check if we have a Fortran compiler @@ -27,9 +18,18 @@ if(NOT EIGEN_Fortran_COMPILER_WORKS) # search for a default Lapack library to complete Eigen's one - find_package(LAPACK) + find_package(LAPACK QUIET) endif() +# TODO do the same for EXTERNAL_LAPACK +option(EIGEN_TEST_EXTERNAL_BLAS "Use external BLAS library for testsuite" OFF) +if(EIGEN_TEST_EXTERNAL_BLAS) + find_package(BLAS REQUIRED) + message(STATUS "BLAS_COMPILER_FLAGS: ${BLAS_COMPILER_FLAGS}") + add_definitions("-DEIGEN_USE_BLAS") # is adding ${BLAS_COMPILER_FLAGS} necessary? + list(APPEND EXTERNAL_LIBS "${BLAS_LIBRARIES}") +endif(EIGEN_TEST_EXTERNAL_BLAS) + # configure blas/lapack (use Eigen's ones) set(EIGEN_BLAS_LIBRARIES eigen_blas) set(EIGEN_LAPACK_LIBRARIES eigen_lapack) @@ -68,6 +68,17 @@ ei_add_property(EIGEN_MISSING_BACKENDS "UmfPack, ") endif() +find_package(KLU) +if(KLU_FOUND) + add_definitions("-DEIGEN_KLU_SUPPORT") + include_directories(${KLU_INCLUDES}) + set(SPARSE_LIBS ${SPARSE_LIBS} ${KLU_LIBRARIES} ${EIGEN_BLAS_LIBRARIES}) + set(KLU_ALL_LIBS ${KLU_LIBRARIES} ${EIGEN_BLAS_LIBRARIES}) + ei_add_property(EIGEN_TESTED_BACKENDS "KLU, ") +else() + ei_add_property(EIGEN_MISSING_BACKENDS "KLU, ") +endif() + find_package(SuperLU 4.0) if(SUPERLU_FOUND) add_definitions("-DEIGEN_SUPERLU_SUPPORT") @@ -80,23 +91,30 @@ endif() -find_package(Pastix) -find_package(Scotch) -find_package(Metis 5.0 REQUIRED) -if(PASTIX_FOUND) +find_package(PASTIX QUIET COMPONENTS METIS SEQ) +# check that the PASTIX found is a version without MPI +find_path(PASTIX_pastix_nompi.h_INCLUDE_DIRS + NAMES pastix_nompi.h + HINTS ${PASTIX_INCLUDE_DIRS} +) +if (NOT PASTIX_pastix_nompi.h_INCLUDE_DIRS) + message(STATUS "A version of Pastix has been found but pastix_nompi.h does not exist in the include directory." + " Because Eigen tests require a version without MPI, we disable the Pastix backend.") +endif() +if(PASTIX_FOUND AND PASTIX_pastix_nompi.h_INCLUDE_DIRS) add_definitions("-DEIGEN_PASTIX_SUPPORT") - include_directories(${PASTIX_INCLUDES}) + include_directories(${PASTIX_INCLUDE_DIRS_DEP}) if(SCOTCH_FOUND) - include_directories(${SCOTCH_INCLUDES}) + include_directories(${SCOTCH_INCLUDE_DIRS}) set(PASTIX_LIBRARIES ${PASTIX_LIBRARIES} ${SCOTCH_LIBRARIES}) elseif(METIS_FOUND) - include_directories(${METIS_INCLUDES}) + include_directories(${METIS_INCLUDE_DIRS}) set(PASTIX_LIBRARIES ${PASTIX_LIBRARIES} ${METIS_LIBRARIES}) else(SCOTCH_FOUND) ei_add_property(EIGEN_MISSING_BACKENDS "PaStiX, ") endif(SCOTCH_FOUND) - set(SPARSE_LIBS ${SPARSE_LIBS} ${PASTIX_LIBRARIES} ${ORDERING_LIBRARIES} ${EIGEN_BLAS_LIBRARIES}) - set(PASTIX_ALL_LIBS ${PASTIX_LIBRARIES} ${EIGEN_BLAS_LIBRARIES}) + set(SPARSE_LIBS ${SPARSE_LIBS} ${PASTIX_LIBRARIES_DEP} ${ORDERING_LIBRARIES}) + set(PASTIX_ALL_LIBS ${PASTIX_LIBRARIES_DEP}) ei_add_property(EIGEN_TESTED_BACKENDS "PaStiX, ") else() ei_add_property(EIGEN_MISSING_BACKENDS "PaStiX, ") @@ -104,7 +122,7 @@ if(METIS_FOUND) add_definitions("-DEIGEN_METIS_SUPPORT") - include_directories(${METIS_INCLUDES}) + include_directories(${METIS_INCLUDE_DIRS}) ei_add_property(EIGEN_TESTED_BACKENDS "METIS, ") else() ei_add_property(EIGEN_MISSING_BACKENDS "METIS, ") @@ -141,16 +159,18 @@ ei_add_test(rand) ei_add_test(meta) +ei_add_test(numext) ei_add_test(sizeof) ei_add_test(dynalloc) ei_add_test(nomalloc) ei_add_test(first_aligned) ei_add_test(nullary) ei_add_test(mixingtypes) -ei_add_test(packetmath) +ei_add_test(packetmath "-DEIGEN_FAST_MATH=1") ei_add_test(unalignedassert) ei_add_test(vectorization_logic) ei_add_test(basicstuff) +ei_add_test(constructor) ei_add_test(linearstructure) ei_add_test(integer_types) ei_add_test(unalignedcount) @@ -161,6 +181,9 @@ ei_add_test(visitor) ei_add_test(block) ei_add_test(corners) +ei_add_test(symbolic_index) +ei_add_test(indexed_view) +ei_add_test(reshape) ei_add_test(swap) ei_add_test(resize) ei_add_test(conservative_resize) @@ -176,7 +199,7 @@ ei_add_test(mapped_matrix) ei_add_test(mapstride) ei_add_test(mapstaticmethods) -ei_add_test(array) +ei_add_test(array_cwise) ei_add_test(array_for_matrix) ei_add_test(array_replicate) ei_add_test(array_reverse) @@ -245,6 +268,7 @@ ei_add_test(sparseqr) ei_add_test(umeyama) ei_add_test(nesting_ops "${CMAKE_CXX_FLAGS_DEBUG}") +ei_add_test(nestbyvalue) ei_add_test(zerosized) ei_add_test(dontalign) ei_add_test(evaluators) @@ -258,6 +282,16 @@ ei_add_test(dense_storage) ei_add_test(ctorleak) ei_add_test(mpl2only) +ei_add_test(inplace_decomposition) +ei_add_test(half_float) +ei_add_test(array_of_string) +ei_add_test(num_dimensions) +ei_add_test(stl_iterators) +if(EIGEN_TEST_CXX11) + ei_add_test(initializer_list_construction) +endif() + +add_executable(bug1213 bug1213.cpp bug1213_main.cpp) check_cxx_compiler_flag("-ffast-math" COMPILER_SUPPORT_FASTMATH) if(COMPILER_SUPPORT_FASTMATH) @@ -281,6 +315,10 @@ ei_add_test(umfpack_support "" "${UMFPACK_ALL_LIBS}") endif() +if(KLU_FOUND OR SuiteSparse_FOUND) + ei_add_test(klu_support "" "${KLU_ALL_LIBS}") +endif() + if(SUPERLU_FOUND) ei_add_test(superlu_support "" "${SUPERLU_ALL_LIBS}") endif() @@ -324,6 +362,16 @@ message(WARNING "The Eigen2 test suite has been removed") endif() +# boost MP unit test +find_package(Boost 1.53.0) +if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + ei_add_test(boostmultiprec "" "${Boost_LIBRARIES}") + ei_add_property(EIGEN_TESTED_BACKENDS "Boost.Multiprecision, ") +else() + ei_add_property(EIGEN_MISSING_BACKENDS "Boost.Multiprecision, ") +endif() + # CUDA unit tests option(EIGEN_TEST_CUDA "Enable CUDA support in unit tests" OFF) @@ -340,15 +388,14 @@ set(CUDA_PROPAGATE_HOST_FLAGS OFF) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - set(CUDA_NVCC_FLAGS "-ccbin /usr/bin/clang" CACHE STRING "nvcc flags" FORCE) + set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_C_COMPILER}" CACHE STRING "nvcc flags" FORCE) endif() if(EIGEN_TEST_CUDA_CLANG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 --cuda-gpu-arch=sm_30") endif() - cuda_include_directories(${CMAKE_CURRENT_BINARY_DIR}) set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu") - ei_add_test(cuda_basic) + ei_add_test(gpu_basic) unset(EIGEN_ADD_TEST_FILENAME_EXTENSION) @@ -357,8 +404,44 @@ endif(EIGEN_TEST_CUDA) -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/failtests) -add_test(NAME failtests WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/failtests COMMAND ${CMAKE_COMMAND} ${Eigen_SOURCE_DIR} -G "${CMAKE_GENERATOR}" -DEIGEN_FAILTEST=ON) +# HIP unit tests +option(EIGEN_TEST_HIP "Add HIP support." OFF) +if (EIGEN_TEST_HIP) + + set(HIP_PATH "/opt/rocm/hip" CACHE STRING "Path to the HIP installation.") + + if (EXISTS ${HIP_PATH}) + + list(APPEND CMAKE_MODULE_PATH ${HIP_PATH}/cmake) + + find_package(HIP REQUIRED) + if (HIP_FOUND) + + execute_process(COMMAND ${HIP_PATH}/bin/hipconfig --platform OUTPUT_VARIABLE HIP_PLATFORM) + + if (${HIP_PLATFORM} STREQUAL "hcc") + + include_directories(${HIP_PATH}/include) + + set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu") + ei_add_test(gpu_basic) + unset(EIGEN_ADD_TEST_FILENAME_EXTENSION) + + elseif (${HIP_PLATFORM} STREQUAL "nvcc") + message(FATAL_ERROR "HIP_PLATFORM = nvcc is not supported within Eigen") + else () + message(FATAL_ERROR "Unknown HIP_PLATFORM = ${HIP_PLATFORM}") + endif() + + endif(HIP_FOUND) + + else () + + message(FATAL_ERROR "EIGEN_TEST_HIP is ON, but the specified HIP_PATH (${HIP_PATH}) does not exist") + + endif() + +endif(EIGEN_TEST_HIP) option(EIGEN_TEST_BUILD_DOCUMENTATION "Test building the doxygen documentation" OFF) IF(EIGEN_TEST_BUILD_DOCUMENTATION)
diff --git a/test/adjoint.cpp b/test/adjoint.cpp index 9c895e0..4c4f98b 100644 --- a/test/adjoint.cpp +++ b/test/adjoint.cpp
@@ -70,7 +70,6 @@ Transpose.h Conjugate.h Dot.h */ using std::abs; - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; @@ -144,9 +143,55 @@ RealVectorType rv1 = RealVectorType::Random(rows); VERIFY_IS_APPROX(v1.dot(rv1.template cast<Scalar>()), v1.dot(rv1)); VERIFY_IS_APPROX(rv1.template cast<Scalar>().dot(v1), rv1.dot(v1)); + + VERIFY( is_same_type(m1,m1.template conjugateIf<false>()) ); + VERIFY( is_same_type(m1.conjugate(),m1.template conjugateIf<true>()) ); } -void test_adjoint() +template<int> +void adjoint_extra() +{ + MatrixXcf a(10,10), b(10,10); + VERIFY_RAISES_ASSERT(a = a.transpose()); + VERIFY_RAISES_ASSERT(a = a.transpose() + b); + VERIFY_RAISES_ASSERT(a = b + a.transpose()); + VERIFY_RAISES_ASSERT(a = a.conjugate().transpose()); + VERIFY_RAISES_ASSERT(a = a.adjoint()); + VERIFY_RAISES_ASSERT(a = a.adjoint() + b); + VERIFY_RAISES_ASSERT(a = b + a.adjoint()); + + // no assertion should be triggered for these cases: + a.transpose() = a.transpose(); + a.transpose() += a.transpose(); + a.transpose() += a.transpose() + b; + a.transpose() = a.adjoint(); + a.transpose() += a.adjoint(); + a.transpose() += a.adjoint() + b; + + // regression tests for check_for_aliasing + MatrixXd c(10,10); + c = 1.0 * MatrixXd::Ones(10,10) + c; + c = MatrixXd::Ones(10,10) * 1.0 + c; + c = c + MatrixXd::Ones(10,10) .cwiseProduct( MatrixXd::Zero(10,10) ); + c = MatrixXd::Ones(10,10) * MatrixXd::Zero(10,10); + + // regression for bug 1646 + for (int j = 0; j < 10; ++j) { + c.col(j).head(j) = c.row(j).head(j); + } + + for (int j = 0; j < 10; ++j) { + c.col(j) = c.row(j); + } + + a.conservativeResize(1,1); + a = a.transpose(); + + a.conservativeResize(0,0); + a = a.transpose(); +} + +EIGEN_DECLARE_TEST(adjoint) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( adjoint(Matrix<float, 1, 1>()) ); @@ -169,25 +214,6 @@ // test a large static matrix only once CALL_SUBTEST_7( adjoint(Matrix<float, 100, 100>()) ); -#ifdef EIGEN_TEST_PART_4 - { - MatrixXcf a(10,10), b(10,10); - VERIFY_RAISES_ASSERT(a = a.transpose()); - VERIFY_RAISES_ASSERT(a = a.transpose() + b); - VERIFY_RAISES_ASSERT(a = b + a.transpose()); - VERIFY_RAISES_ASSERT(a = a.conjugate().transpose()); - VERIFY_RAISES_ASSERT(a = a.adjoint()); - VERIFY_RAISES_ASSERT(a = a.adjoint() + b); - VERIFY_RAISES_ASSERT(a = b + a.adjoint()); - - // no assertion should be triggered for these cases: - a.transpose() = a.transpose(); - a.transpose() += a.transpose(); - a.transpose() += a.transpose() + b; - a.transpose() = a.adjoint(); - a.transpose() += a.adjoint(); - a.transpose() += a.adjoint() + b; - } -#endif + CALL_SUBTEST_13( adjoint_extra<0>() ); }
diff --git a/test/array.cpp b/test/array_cwise.cpp similarity index 68% rename from test/array.cpp rename to test/array_cwise.cpp index beaa622..9e4adb7 100644 --- a/test/array.cpp +++ b/test/array_cwise.cpp
@@ -11,13 +11,13 @@ template<typename ArrayType> void array(const ArrayType& m) { - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; + typedef typename ArrayType::RealScalar RealScalar; typedef Array<Scalar, ArrayType::RowsAtCompileTime, 1> ColVectorType; typedef Array<Scalar, 1, ArrayType::ColsAtCompileTime> RowVectorType; Index rows = m.rows(); - Index cols = m.cols(); + Index cols = m.cols(); ArrayType m1 = ArrayType::Random(rows, cols), m2 = ArrayType::Random(rows, cols), @@ -43,25 +43,25 @@ VERIFY_IS_APPROX(m3, m1 + s2); m3 = m1; m3 -= s1; - VERIFY_IS_APPROX(m3, m1 - s1); - + VERIFY_IS_APPROX(m3, m1 - s1); + // scalar operators via Maps m3 = m1; ArrayType::Map(m1.data(), m1.rows(), m1.cols()) -= ArrayType::Map(m2.data(), m2.rows(), m2.cols()); VERIFY_IS_APPROX(m1, m3 - m2); - + m3 = m1; ArrayType::Map(m1.data(), m1.rows(), m1.cols()) += ArrayType::Map(m2.data(), m2.rows(), m2.cols()); VERIFY_IS_APPROX(m1, m3 + m2); - + m3 = m1; ArrayType::Map(m1.data(), m1.rows(), m1.cols()) *= ArrayType::Map(m2.data(), m2.rows(), m2.cols()); VERIFY_IS_APPROX(m1, m3 * m2); - + m3 = m1; m2 = ArrayType::Random(rows,cols); m2 = (m2==0).select(1,m2); - ArrayType::Map(m1.data(), m1.rows(), m1.cols()) /= ArrayType::Map(m2.data(), m2.rows(), m2.cols()); + ArrayType::Map(m1.data(), m1.rows(), m1.cols()) /= ArrayType::Map(m2.data(), m2.rows(), m2.cols()); VERIFY_IS_APPROX(m1, m3 / m2); // reductions @@ -72,7 +72,7 @@ VERIFY_IS_MUCH_SMALLER_THAN(abs(m1.rowwise().sum().sum() - m1.sum()), m1.abs().sum()); if (!internal::isMuchSmallerThan(abs(m1.sum() - (m1+m2).sum()), m1.abs().sum(), test_precision<Scalar>())) VERIFY_IS_NOT_APPROX(((m1+m2).rowwise().sum()).sum(), m1.sum()); - VERIFY_IS_APPROX(m1.colwise().sum(), m1.colwise().redux(internal::scalar_sum_op<Scalar>())); + VERIFY_IS_APPROX(m1.colwise().sum(), m1.colwise().redux(internal::scalar_sum_op<Scalar,Scalar>())); // vector-wise ops m3 = m1; @@ -83,7 +83,7 @@ VERIFY_IS_APPROX(m3.rowwise() += rv1, m1.rowwise() + rv1); m3 = m1; VERIFY_IS_APPROX(m3.rowwise() -= rv1, m1.rowwise() - rv1); - + // Conversion from scalar VERIFY_IS_APPROX((m3 = s1), ArrayType::Constant(rows,cols,s1)); VERIFY_IS_APPROX((m3 = 1), ArrayType::Constant(rows,cols,1)); @@ -92,28 +92,99 @@ ArrayType::RowsAtCompileTime==Dynamic?2:ArrayType::RowsAtCompileTime, ArrayType::ColsAtCompileTime==Dynamic?2:ArrayType::ColsAtCompileTime, ArrayType::Options> FixedArrayType; - FixedArrayType f1(s1); - VERIFY_IS_APPROX(f1, FixedArrayType::Constant(s1)); - FixedArrayType f2(numext::real(s1)); - VERIFY_IS_APPROX(f2, FixedArrayType::Constant(numext::real(s1))); - FixedArrayType f3((int)100*numext::real(s1)); - VERIFY_IS_APPROX(f3, FixedArrayType::Constant((int)100*numext::real(s1))); - f1.setRandom(); - FixedArrayType f4(f1.data()); - VERIFY_IS_APPROX(f4, f1); - + { + FixedArrayType f1(s1); + VERIFY_IS_APPROX(f1, FixedArrayType::Constant(s1)); + FixedArrayType f2(numext::real(s1)); + VERIFY_IS_APPROX(f2, FixedArrayType::Constant(numext::real(s1))); + FixedArrayType f3((int)100*numext::real(s1)); + VERIFY_IS_APPROX(f3, FixedArrayType::Constant((int)100*numext::real(s1))); + f1.setRandom(); + FixedArrayType f4(f1.data()); + VERIFY_IS_APPROX(f4, f1); + } + #if EIGEN_HAS_CXX11 + { + FixedArrayType f1{s1}; + VERIFY_IS_APPROX(f1, FixedArrayType::Constant(s1)); + FixedArrayType f2{numext::real(s1)}; + VERIFY_IS_APPROX(f2, FixedArrayType::Constant(numext::real(s1))); + FixedArrayType f3{(int)100*numext::real(s1)}; + VERIFY_IS_APPROX(f3, FixedArrayType::Constant((int)100*numext::real(s1))); + f1.setRandom(); + FixedArrayType f4{f1.data()}; + VERIFY_IS_APPROX(f4, f1); + } + #endif + + // pow + VERIFY_IS_APPROX(m1.pow(2), m1.square()); + VERIFY_IS_APPROX(pow(m1,2), m1.square()); + VERIFY_IS_APPROX(m1.pow(3), m1.cube()); + VERIFY_IS_APPROX(pow(m1,3), m1.cube()); + VERIFY_IS_APPROX((-m1).pow(3), -m1.cube()); + VERIFY_IS_APPROX(pow(2*m1,3), 8*m1.cube()); + ArrayType exponents = ArrayType::Constant(rows, cols, RealScalar(2)); + VERIFY_IS_APPROX(Eigen::pow(m1,exponents), m1.square()); + VERIFY_IS_APPROX(m1.pow(exponents), m1.square()); + VERIFY_IS_APPROX(Eigen::pow(2*m1,exponents), 4*m1.square()); + VERIFY_IS_APPROX((2*m1).pow(exponents), 4*m1.square()); + VERIFY_IS_APPROX(Eigen::pow(m1,2*exponents), m1.square().square()); + VERIFY_IS_APPROX(m1.pow(2*exponents), m1.square().square()); + VERIFY_IS_APPROX(Eigen::pow(m1(0,0), exponents), ArrayType::Constant(rows,cols,m1(0,0)*m1(0,0))); + // Check possible conflicts with 1D ctor typedef Array<Scalar, Dynamic, 1> OneDArrayType; - OneDArrayType o1(rows); - VERIFY(o1.size()==rows); - OneDArrayType o4((int)rows); - VERIFY(o4.size()==rows); + { + OneDArrayType o1(rows); + VERIFY(o1.size()==rows); + OneDArrayType o2(static_cast<int>(rows)); + VERIFY(o2.size()==rows); + } + #if EIGEN_HAS_CXX11 + { + OneDArrayType o1{rows}; + VERIFY(o1.size()==rows); + OneDArrayType o4{int(rows)}; + VERIFY(o4.size()==rows); + } + #endif + // Check possible conflicts with 2D ctor + typedef Array<Scalar, Dynamic, Dynamic> TwoDArrayType; + typedef Array<Scalar, 2, 1> ArrayType2; + { + TwoDArrayType o1(rows,cols); + VERIFY(o1.rows()==rows); + VERIFY(o1.cols()==cols); + TwoDArrayType o2(static_cast<int>(rows),static_cast<int>(cols)); + VERIFY(o2.rows()==rows); + VERIFY(o2.cols()==cols); + + ArrayType2 o3(rows,cols); + VERIFY(o3(0)==Scalar(rows) && o3(1)==Scalar(cols)); + ArrayType2 o4(static_cast<int>(rows),static_cast<int>(cols)); + VERIFY(o4(0)==Scalar(rows) && o4(1)==Scalar(cols)); + } + #if EIGEN_HAS_CXX11 + { + TwoDArrayType o1{rows,cols}; + VERIFY(o1.rows()==rows); + VERIFY(o1.cols()==cols); + TwoDArrayType o2{int(rows),int(cols)}; + VERIFY(o2.rows()==rows); + VERIFY(o2.cols()==cols); + + ArrayType2 o3{rows,cols}; + VERIFY(o3(0)==Scalar(rows) && o3(1)==Scalar(cols)); + ArrayType2 o4{int(rows),int(cols)}; + VERIFY(o4(0)==Scalar(rows) && o4(1)==Scalar(cols)); + } + #endif } template<typename ArrayType> void comparisons(const ArrayType& m) { using std::abs; - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; @@ -127,7 +198,7 @@ m2 = ArrayType::Random(rows, cols), m3(rows, cols), m4 = m1; - + m4 = (m4.abs()==Scalar(0)).select(1,m4); VERIFY(((m1 + Scalar(1)) > m1).all()); @@ -180,7 +251,7 @@ RealScalar a = m1.abs().mean(); VERIFY( (m1<-a || m1>a).count() == (m1.abs()>a).count()); - typedef Array<typename ArrayType::Index, Dynamic, 1> ArrayOfIndices; + typedef Array<Index, Dynamic, 1> ArrayOfIndices; // TODO allows colwise/rowwise for array VERIFY_IS_APPROX(((m1.abs()+1)>RealScalar(0.1)).colwise().count(), ArrayOfIndices::Constant(cols,rows).transpose()); @@ -191,7 +262,6 @@ { using std::abs; using std::sqrt; - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; @@ -217,12 +287,13 @@ VERIFY_IS_APPROX(m1.sinh(), sinh(m1)); VERIFY_IS_APPROX(m1.cosh(), cosh(m1)); VERIFY_IS_APPROX(m1.tanh(), tanh(m1)); -#ifdef EIGEN_HAS_C99_MATH - VERIFY_IS_APPROX(m1.lgamma(), lgamma(m1)); - VERIFY_IS_APPROX(m1.digamma(), digamma(m1)); - VERIFY_IS_APPROX(m1.erf(), erf(m1)); - VERIFY_IS_APPROX(m1.erfc(), erfc(m1)); -#endif // EIGEN_HAS_C99_MATH +#if EIGEN_HAS_CXX11_MATH + VERIFY_IS_APPROX(m1.tanh().atanh(), atanh(tanh(m1))); + VERIFY_IS_APPROX(m1.sinh().asinh(), asinh(sinh(m1))); + VERIFY_IS_APPROX(m1.cosh().acosh(), acosh(cosh(m1))); +#endif + VERIFY_IS_APPROX(m1.logistic(), logistic(m1)); + VERIFY_IS_APPROX(m1.arg(), arg(m1)); VERIFY_IS_APPROX(m1.round(), round(m1)); VERIFY_IS_APPROX(m1.floor(), floor(m1)); @@ -243,7 +314,9 @@ m3 = m1.abs(); VERIFY_IS_APPROX(m3.sqrt(), sqrt(abs(m1))); VERIFY_IS_APPROX(m3.rsqrt(), Scalar(1)/sqrt(abs(m1))); + VERIFY_IS_APPROX(rsqrt(m3), Scalar(1)/sqrt(abs(m1))); VERIFY_IS_APPROX(m3.log(), log(m3)); + VERIFY_IS_APPROX(m3.log1p(), log1p(m3)); VERIFY_IS_APPROX(m3.log10(), log10(m3)); @@ -255,6 +328,7 @@ VERIFY_IS_APPROX(sinh(m1), 0.5*(exp(m1)-exp(-m1))); VERIFY_IS_APPROX(cosh(m1), 0.5*(exp(m1)+exp(-m1))); VERIFY_IS_APPROX(tanh(m1), (0.5*(exp(m1)-exp(-m1)))/(0.5*(exp(m1)+exp(-m1)))); + VERIFY_IS_APPROX(logistic(m1), (1.0/(1.0+exp(-m1)))); VERIFY_IS_APPROX(arg(m1), ((m1<0).template cast<Scalar>())*std::acos(-1.0)); VERIFY((round(m1) <= ceil(m1) && round(m1) >= floor(m1)).all()); VERIFY((Eigen::isnan)((m1*0.0)/0.0).all()); @@ -275,26 +349,14 @@ // shift argument of logarithm so that it is not zero Scalar smallNumber = NumTraits<Scalar>::dummy_precision(); VERIFY_IS_APPROX((m3 + smallNumber).log() , log(abs(m1) + smallNumber)); + VERIFY_IS_APPROX((m3 + smallNumber + 1).log() , log1p(abs(m1) + smallNumber)); VERIFY_IS_APPROX(m1.exp() * m2.exp(), exp(m1+m2)); VERIFY_IS_APPROX(m1.exp(), exp(m1)); VERIFY_IS_APPROX(m1.exp() / m2.exp(),(m1-m2).exp()); - VERIFY_IS_APPROX(m1.pow(2), m1.square()); - VERIFY_IS_APPROX(pow(m1,2), m1.square()); - VERIFY_IS_APPROX(m1.pow(3), m1.cube()); - VERIFY_IS_APPROX(pow(m1,3), m1.cube()); - VERIFY_IS_APPROX((-m1).pow(3), -m1.cube()); - VERIFY_IS_APPROX(pow(2*m1,3), 8*m1.cube()); - - ArrayType exponents = ArrayType::Constant(rows, cols, RealScalar(2)); - VERIFY_IS_APPROX(Eigen::pow(m1,exponents), m1.square()); - VERIFY_IS_APPROX(m1.pow(exponents), m1.square()); - VERIFY_IS_APPROX(Eigen::pow(2*m1,exponents), 4*m1.square()); - VERIFY_IS_APPROX((2*m1).pow(exponents), 4*m1.square()); - VERIFY_IS_APPROX(Eigen::pow(m1,2*exponents), m1.square().square()); - VERIFY_IS_APPROX(m1.pow(2*exponents), m1.square().square()); - VERIFY_IS_APPROX(pow(m1(0,0), exponents), ArrayType::Constant(rows,cols,m1(0,0)*m1(0,0))); + VERIFY_IS_APPROX(m1.expm1(), expm1(m1)); + VERIFY_IS_APPROX((m3 + smallNumber).exp() - 1, expm1(abs(m3) + smallNumber)); VERIFY_IS_APPROX(m3.pow(RealScalar(0.5)), m3.sqrt()); VERIFY_IS_APPROX(pow(m3,RealScalar(0.5)), m3.sqrt()); @@ -310,122 +372,6 @@ m1 += ArrayType::Constant(rows,cols,Scalar(tiny)); VERIFY_IS_APPROX(s1/m1, s1 * m1.inverse()); -#ifdef EIGEN_HAS_C99_MATH - // check special functions (comparing against numpy implementation) - if (!NumTraits<Scalar>::IsComplex) { - VERIFY_IS_APPROX(numext::digamma(Scalar(1)), RealScalar(-0.5772156649015329)); - VERIFY_IS_APPROX(numext::digamma(Scalar(1.5)), RealScalar(0.03648997397857645)); - VERIFY_IS_APPROX(numext::digamma(Scalar(4)), RealScalar(1.2561176684318)); - VERIFY_IS_APPROX(numext::digamma(Scalar(-10.5)), RealScalar(2.398239129535781)); - VERIFY_IS_APPROX(numext::digamma(Scalar(10000.5)), RealScalar(9.210340372392849)); - VERIFY_IS_EQUAL(numext::digamma(Scalar(0)), - std::numeric_limits<RealScalar>::infinity()); - VERIFY_IS_EQUAL(numext::digamma(Scalar(-1)), - std::numeric_limits<RealScalar>::infinity()); - - // Check the zeta function against scipy.special.zeta - VERIFY_IS_APPROX(numext::zeta(Scalar(1.5), Scalar(2)), RealScalar(1.61237534869)); - VERIFY_IS_APPROX(numext::zeta(Scalar(4), Scalar(1.5)), RealScalar(0.234848505667)); - VERIFY_IS_APPROX(numext::zeta(Scalar(10.5), Scalar(3)), RealScalar(1.03086757337e-5)); - VERIFY_IS_APPROX(numext::zeta(Scalar(10000.5), Scalar(1.0001)), RealScalar(0.367879440865)); - VERIFY_IS_APPROX(numext::zeta(Scalar(3), Scalar(-2.5)), RealScalar(0.054102025820864097)); - VERIFY_IS_EQUAL(numext::zeta(Scalar(1), Scalar(1.2345)), // The second scalar does not matter - std::numeric_limits<RealScalar>::infinity()); - VERIFY((numext::isnan)(numext::zeta(Scalar(0.9), Scalar(1.2345)))); // The second scalar does not matter - - // Check the polygamma against scipy.special.polygamma examples - VERIFY_IS_APPROX(numext::polygamma(Scalar(1), Scalar(2)), RealScalar(0.644934066848)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(1), Scalar(3)), RealScalar(0.394934066848)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(1), Scalar(25.5)), RealScalar(0.0399946696496)); - VERIFY((numext::isnan)(numext::polygamma(Scalar(1.5), Scalar(1.2345)))); // The second scalar does not matter - - // Check the polygamma function over a larger range of values - VERIFY_IS_APPROX(numext::polygamma(Scalar(17), Scalar(4.7)), RealScalar(293.334565435)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(31), Scalar(11.8)), RealScalar(0.445487887616)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(28), Scalar(17.7)), RealScalar(-2.47810300902e-07)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(8), Scalar(30.2)), RealScalar(-8.29668781082e-09)); - /* The following tests only pass for doubles because floats cannot handle the large values of - the gamma function. - VERIFY_IS_APPROX(numext::polygamma(Scalar(42), Scalar(15.8)), RealScalar(-0.434562276666)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(147), Scalar(54.1)), RealScalar(0.567742190178)); - VERIFY_IS_APPROX(numext::polygamma(Scalar(170), Scalar(64)), RealScalar(-0.0108615497927)); - */ - - { - // Test various propreties of igamma & igammac. These are normalized - // gamma integrals where - // igammac(a, x) = Gamma(a, x) / Gamma(a) - // igamma(a, x) = gamma(a, x) / Gamma(a) - // where Gamma and gamma are considered the standard unnormalized - // upper and lower incomplete gamma functions, respectively. - ArrayType a = m1.abs() + 2; - ArrayType x = m2.abs() + 2; - ArrayType zero = ArrayType::Zero(rows, cols); - ArrayType one = ArrayType::Constant(rows, cols, Scalar(1.0)); - ArrayType a_m1 = a - one; - ArrayType Gamma_a_x = Eigen::igammac(a, x) * a.lgamma().exp(); - ArrayType Gamma_a_m1_x = Eigen::igammac(a_m1, x) * a_m1.lgamma().exp(); - ArrayType gamma_a_x = Eigen::igamma(a, x) * a.lgamma().exp(); - ArrayType gamma_a_m1_x = Eigen::igamma(a_m1, x) * a_m1.lgamma().exp(); - - // Gamma(a, 0) == Gamma(a) - VERIFY_IS_APPROX(Eigen::igammac(a, zero), one); - - // Gamma(a, x) + gamma(a, x) == Gamma(a) - VERIFY_IS_APPROX(Gamma_a_x + gamma_a_x, a.lgamma().exp()); - - // Gamma(a, x) == (a - 1) * Gamma(a-1, x) + x^(a-1) * exp(-x) - VERIFY_IS_APPROX(Gamma_a_x, (a - 1) * Gamma_a_m1_x + x.pow(a-1) * (-x).exp()); - - // gamma(a, x) == (a - 1) * gamma(a-1, x) - x^(a-1) * exp(-x) - VERIFY_IS_APPROX(gamma_a_x, (a - 1) * gamma_a_m1_x - x.pow(a-1) * (-x).exp()); - } - - // Check exact values of igamma and igammac against a third party calculation. - Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - - // location i*6+j corresponds to a_s[i], x_s[j]. - Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); - Scalar igamma_s[][6] = {{0.0, nan, nan, nan, nan, nan}, - {0.0, 0.6321205588285578, 0.7768698398515702, - 0.9816843611112658, 9.999500016666262e-05, 1.0}, - {0.0, 0.4275932955291202, 0.608374823728911, - 0.9539882943107686, 7.522076445089201e-07, 1.0}, - {0.0, 0.01898815687615381, 0.06564245437845008, - 0.5665298796332909, 4.166333347221828e-18, 1.0}, - {0.0, 0.9999780593618628, 0.9999899967080838, - 0.9999996219837988, 0.9991370418689945, 1.0}, - {0.0, 0.0, 0.0, 0.0, 0.0, 0.5042041932513908}}; - Scalar igammac_s[][6] = {{nan, nan, nan, nan, nan, nan}, - {1.0, 0.36787944117144233, 0.22313016014842982, - 0.018315638888734182, 0.9999000049998333, 0.0}, - {1.0, 0.5724067044708798, 0.3916251762710878, - 0.04601170568923136, 0.9999992477923555, 0.0}, - {1.0, 0.9810118431238462, 0.9343575456215499, - 0.4334701203667089, 1.0, 0.0}, - {1.0, 2.1940638138146658e-05, 1.0003291916285e-05, - 3.7801620118431334e-07, 0.0008629581310054535, - 0.0}, - {1.0, 1.0, 1.0, 1.0, 1.0, 0.49579580674813944}}; - for (int i = 0; i < 6; ++i) { - for (int j = 0; j < 6; ++j) { - if ((std::isnan)(igamma_s[i][j])) { - VERIFY((std::isnan)(numext::igamma(a_s[i], x_s[j]))); - } else { - VERIFY_IS_APPROX(numext::igamma(a_s[i], x_s[j]), igamma_s[i][j]); - } - - if ((std::isnan)(igammac_s[i][j])) { - VERIFY((std::isnan)(numext::igammac(a_s[i], x_s[j]))); - } else { - VERIFY_IS_APPROX(numext::igammac(a_s[i], x_s[j]), igammac_s[i][j]); - } - } - } - } -#endif // EIGEN_HAS_C99_MATH - // check inplace transpose m3 = m1; m3.transposeInPlace(); @@ -436,7 +382,6 @@ template<typename ArrayType> void array_complex(const ArrayType& m) { - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; @@ -446,7 +391,7 @@ ArrayType m1 = ArrayType::Random(rows, cols), m2(rows, cols), m4 = m1; - + m4.real() = (m4.real().abs()==RealScalar(0)).select(RealScalar(1),m4.real()); m4.imag() = (m4.imag().abs()==RealScalar(0)).select(RealScalar(1),m4.imag()); @@ -463,6 +408,7 @@ VERIFY_IS_APPROX(m1.sinh(), sinh(m1)); VERIFY_IS_APPROX(m1.cosh(), cosh(m1)); VERIFY_IS_APPROX(m1.tanh(), tanh(m1)); + VERIFY_IS_APPROX(m1.logistic(), logistic(m1)); VERIFY_IS_APPROX(m1.arg(), arg(m1)); VERIFY((m1.isNaN() == (Eigen::isnan)(m1)).all()); VERIFY((m1.isInf() == (Eigen::isinf)(m1)).all()); @@ -486,6 +432,7 @@ VERIFY_IS_APPROX(sinh(m1), 0.5*(exp(m1)-exp(-m1))); VERIFY_IS_APPROX(cosh(m1), 0.5*(exp(m1)+exp(-m1))); VERIFY_IS_APPROX(tanh(m1), (0.5*(exp(m1)-exp(-m1)))/(0.5*(exp(m1)+exp(-m1)))); + VERIFY_IS_APPROX(logistic(m1), (1.0/(1.0 + exp(-m1)))); for (Index i = 0; i < m.rows(); ++i) for (Index j = 0; j < m.cols(); ++j) @@ -525,7 +472,7 @@ // scalar by array division Scalar s1 = internal::random<Scalar>(); - const RealScalar tiny = sqrt(std::numeric_limits<RealScalar>::epsilon()); + const RealScalar tiny = std::sqrt(std::numeric_limits<RealScalar>::epsilon()); s1 += Scalar(tiny); m1 += ArrayType::Constant(rows,cols,Scalar(tiny)); VERIFY_IS_APPROX(s1/m1, s1 * m1.inverse()); @@ -541,7 +488,6 @@ template<typename ArrayType> void min_max(const ArrayType& m) { - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; Index rows = m.rows(); @@ -568,7 +514,7 @@ } -void test_array() +EIGEN_DECLARE_TEST(array_cwise) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( array(Array<float, 1, 1>()) ); @@ -577,6 +523,7 @@ CALL_SUBTEST_4( array(ArrayXXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_5( array(ArrayXXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_6( array(ArrayXXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_6( array(Array<Index,Dynamic,Dynamic>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( comparisons(Array<float, 1, 1>()) ); @@ -605,7 +552,7 @@ VERIFY((internal::is_same< internal::global_math_functions_filtering_base<int>::type, int >::value)); VERIFY((internal::is_same< internal::global_math_functions_filtering_base<float>::type, float >::value)); VERIFY((internal::is_same< internal::global_math_functions_filtering_base<Array2i>::type, ArrayBase<Array2i> >::value)); - typedef CwiseUnaryOp<internal::scalar_multiple_op<double>, ArrayXd > Xpr; + typedef CwiseUnaryOp<internal::scalar_abs_op<double>, ArrayXd > Xpr; VERIFY((internal::is_same< internal::global_math_functions_filtering_base<Xpr>::type, ArrayBase<Xpr> >::value));
diff --git a/test/array_for_matrix.cpp b/test/array_for_matrix.cpp index db5f3b3..fb6be35 100644 --- a/test/array_for_matrix.cpp +++ b/test/array_for_matrix.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void array_for_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> ColVectorType; typedef Matrix<Scalar, 1, MatrixType::ColsAtCompileTime> RowVectorType; @@ -45,7 +44,7 @@ VERIFY_IS_MUCH_SMALLER_THAN(m1.rowwise().sum().sum() - m1.sum(), m1.squaredNorm()); VERIFY_IS_MUCH_SMALLER_THAN(m1.colwise().sum() + m2.colwise().sum() - (m1+m2).colwise().sum(), (m1+m2).squaredNorm()); VERIFY_IS_MUCH_SMALLER_THAN(m1.rowwise().sum() - m2.rowwise().sum() - (m1-m2).rowwise().sum(), (m1-m2).squaredNorm()); - VERIFY_IS_APPROX(m1.colwise().sum(), m1.colwise().redux(internal::scalar_sum_op<Scalar>())); + VERIFY_IS_APPROX(m1.colwise().sum(), m1.colwise().redux(internal::scalar_sum_op<Scalar,Scalar>())); // vector-wise ops m3 = m1; @@ -58,7 +57,14 @@ VERIFY_IS_APPROX(m3.rowwise() -= rv1, m1.rowwise() - rv1); // empty objects - VERIFY_IS_APPROX(m1.block(0,0,0,cols).colwise().sum(), RowVectorType::Zero(cols)); + VERIFY_IS_APPROX((m1.template block<0,Dynamic>(0,0,0,cols).colwise().sum()), RowVectorType::Zero(cols)); + VERIFY_IS_APPROX((m1.template block<Dynamic,0>(0,0,rows,0).rowwise().sum()), ColVectorType::Zero(rows)); + VERIFY_IS_APPROX((m1.template block<0,Dynamic>(0,0,0,cols).colwise().prod()), RowVectorType::Ones(cols)); + VERIFY_IS_APPROX((m1.template block<Dynamic,0>(0,0,rows,0).rowwise().prod()), ColVectorType::Ones(rows)); + + VERIFY_IS_APPROX(m1.block(0,0,0,cols).colwise().sum(), RowVectorType::Zero(cols)); + VERIFY_IS_APPROX(m1.block(0,0,rows,0).rowwise().sum(), ColVectorType::Zero(rows)); + VERIFY_IS_APPROX(m1.block(0,0,0,cols).colwise().prod(), RowVectorType::Ones(cols)); VERIFY_IS_APPROX(m1.block(0,0,rows,0).rowwise().prod(), ColVectorType::Ones(rows)); // verify the const accessors exist @@ -83,7 +89,6 @@ template<typename MatrixType> void comparisons(const MatrixType& m) { using std::abs; - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; @@ -134,7 +139,13 @@ // count VERIFY(((m1.array().abs()+1)>RealScalar(0.1)).count() == rows*cols); - typedef Matrix<typename MatrixType::Index, Dynamic, 1> VectorOfIndices; + // and/or + VERIFY( ((m1.array()<RealScalar(0)).matrix() && (m1.array()>RealScalar(0)).matrix()).count() == 0); + VERIFY( ((m1.array()<RealScalar(0)).matrix() || (m1.array()>=RealScalar(0)).matrix()).count() == rows*cols); + RealScalar a = m1.cwiseAbs().mean(); + VERIFY( ((m1.array()<-a).matrix() || (m1.array()>a).matrix()).count() == (m1.cwiseAbs().array()>a).count()); + + typedef Matrix<Index, Dynamic, 1> VectorOfIndices; // TODO allows colwise/rowwise for array VERIFY_IS_APPROX(((m1.array().abs()+1)>RealScalar(0.1)).matrix().colwise().count(), VectorOfIndices::Constant(cols,rows).transpose()); @@ -144,9 +155,21 @@ template<typename VectorType> void lpNorm(const VectorType& v) { using std::sqrt; + typedef typename VectorType::RealScalar RealScalar; VectorType u = VectorType::Random(v.size()); - VERIFY_IS_APPROX(u.template lpNorm<Infinity>(), u.cwiseAbs().maxCoeff()); + if(v.size()==0) + { + VERIFY_IS_APPROX(u.template lpNorm<Infinity>(), RealScalar(0)); + VERIFY_IS_APPROX(u.template lpNorm<1>(), RealScalar(0)); + VERIFY_IS_APPROX(u.template lpNorm<2>(), RealScalar(0)); + VERIFY_IS_APPROX(u.template lpNorm<5>(), RealScalar(0)); + } + else + { + VERIFY_IS_APPROX(u.template lpNorm<Infinity>(), u.cwiseAbs().maxCoeff()); + } + VERIFY_IS_APPROX(u.template lpNorm<1>(), u.cwiseAbs().sum()); VERIFY_IS_APPROX(u.template lpNorm<2>(), sqrt(u.array().abs().square().sum())); VERIFY_IS_APPROX(numext::pow(u.template lpNorm<5>(), typename VectorType::RealScalar(5)), u.array().abs().pow(5).sum()); @@ -154,7 +177,6 @@ template<typename MatrixType> void cwise_min_max(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(); @@ -193,7 +215,6 @@ template<typename MatrixTraits> void resize(const MatrixTraits& t) { - typedef typename MatrixTraits::Index Index; typedef typename MatrixTraits::Scalar Scalar; typedef Matrix<Scalar,Dynamic,Dynamic> MatrixType; typedef Array<Scalar,Dynamic,Dynamic> Array2DType; @@ -217,13 +238,32 @@ VERIFY(a1.size()==cols); } +template<int> void regression_bug_654() { ArrayXf a = RowVectorXf(3); VectorXf v = Array<float,1,Dynamic>(3); } -void test_array_for_matrix() +// Check propagation of LvalueBit through Array/Matrix-Wrapper +template<int> +void regrrssion_bug_1410() +{ + const Matrix4i M; + const Array4i A; + ArrayWrapper<const Matrix4i> MA = M.array(); + MA.row(0); + MatrixWrapper<const Array4i> AM = A.matrix(); + AM.row(0); + + VERIFY((internal::traits<ArrayWrapper<const Matrix4i> >::Flags&LvalueBit)==0); + VERIFY((internal::traits<MatrixWrapper<const Array4i> >::Flags&LvalueBit)==0); + + VERIFY((internal::traits<ArrayWrapper<Matrix4i> >::Flags&LvalueBit)==LvalueBit); + VERIFY((internal::traits<MatrixWrapper<Array4i> >::Flags&LvalueBit)==LvalueBit); +} + +EIGEN_DECLARE_TEST(array_for_matrix) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( array_for_matrix(Matrix<float, 1, 1>()) ); @@ -255,10 +295,13 @@ CALL_SUBTEST_5( lpNorm(VectorXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_4( lpNorm(VectorXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } + CALL_SUBTEST_5( lpNorm(VectorXf(0)) ); + CALL_SUBTEST_4( lpNorm(VectorXcf(0)) ); for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_4( resize(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_5( resize(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_6( resize(MatrixXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } - CALL_SUBTEST_6( regression_bug_654() ); + CALL_SUBTEST_6( regression_bug_654<0>() ); + CALL_SUBTEST_6( regrrssion_bug_1410<0>() ); }
diff --git a/test/array_of_string.cpp b/test/array_of_string.cpp new file mode 100644 index 0000000..23e5152 --- /dev/null +++ b/test/array_of_string.cpp
@@ -0,0 +1,32 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +EIGEN_DECLARE_TEST(array_of_string) +{ + typedef Array<std::string,1,Dynamic> ArrayXs; + ArrayXs a1(3), a2(3), a3(3), a3ref(3); + a1 << "one", "two", "three"; + a2 << "1", "2", "3"; + a3ref << "one (1)", "two (2)", "three (3)"; + std::stringstream s1; + s1 << a1; + VERIFY_IS_EQUAL(s1.str(), std::string(" one two three")); + a3 = a1 + std::string(" (") + a2 + std::string(")"); + VERIFY((a3==a3ref).all()); + + a3 = a1; + a3 += std::string(" (") + a2 + std::string(")"); + VERIFY((a3==a3ref).all()); + + a1.swap(a3); + VERIFY((a1==a3ref).all()); + VERIFY((a3!=a3ref).all()); +}
diff --git a/test/array_replicate.cpp b/test/array_replicate.cpp index 779c8fc..057c3c7 100644 --- a/test/array_replicate.cpp +++ b/test/array_replicate.cpp
@@ -14,7 +14,6 @@ /* this test covers the following files: Replicate.cpp */ - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; typedef Matrix<Scalar, Dynamic, Dynamic> MatrixX; @@ -69,7 +68,7 @@ VERIFY_IS_APPROX(vx1, v1.colwise().replicate(f2)); } -void test_array_replicate() +EIGEN_DECLARE_TEST(array_replicate) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( replicate(Matrix<float, 1, 1>()) );
diff --git a/test/array_reverse.cpp b/test/array_reverse.cpp index a5c0d37..e23159d 100644 --- a/test/array_reverse.cpp +++ b/test/array_reverse.cpp
@@ -15,7 +15,6 @@ template<typename MatrixType> void reverse(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; @@ -117,16 +116,23 @@ m2.colwise().reverseInPlace(); VERIFY_IS_APPROX(m2,m1.colwise().reverse().eval()); - /* m1.colwise().reverse()(r, c) = x; VERIFY_IS_APPROX(x, m1(rows - 1 - r, c)); m1.rowwise().reverse()(r, c) = x; VERIFY_IS_APPROX(x, m1(r, cols - 1 - c)); - */ } -void test_array_reverse() +template<int> +void array_reverse_extra() +{ + Vector4f x; x << 1, 2, 3, 4; + Vector4f y; y << 4, 3, 2, 1; + VERIFY(x.reverse()[1] == 3); + VERIFY(x.reverse() == y); +} + +EIGEN_DECLARE_TEST(array_reverse) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( reverse(Matrix<float, 1, 1>()) ); @@ -139,10 +145,5 @@ CALL_SUBTEST_8( reverse(Matrix<float, 100, 100>()) ); CALL_SUBTEST_9( reverse(Matrix<float,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } -#ifdef EIGEN_TEST_PART_3 - Vector4f x; x << 1, 2, 3, 4; - Vector4f y; y << 4, 3, 2, 1; - VERIFY(x.reverse()[1] == 3); - VERIFY(x.reverse() == y); -#endif + CALL_SUBTEST_3( array_reverse_extra<0>() ); }
diff --git a/test/bandmatrix.cpp b/test/bandmatrix.cpp index f8c38f7..66a1b0d 100644 --- a/test/bandmatrix.cpp +++ b/test/bandmatrix.cpp
@@ -59,7 +59,7 @@ using Eigen::internal::BandMatrix; -void test_bandmatrix() +EIGEN_DECLARE_TEST(bandmatrix) { for(int i = 0; i < 10*g_repeat ; i++) { Index rows = internal::random<Index>(1,10);
diff --git a/test/basicstuff.cpp b/test/basicstuff.cpp index 99d91f9..85af603 100644 --- a/test/basicstuff.cpp +++ b/test/basicstuff.cpp
@@ -13,7 +13,6 @@ template<typename MatrixType> void basicStuff(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::RowsAtCompileTime> SquareMatrixType; @@ -49,6 +48,22 @@ v1[r] = x; VERIFY_IS_APPROX(x, v1[r]); + // test fetching with various index types. + Index r1 = internal::random<Index>(0, numext::mini(Index(127),rows-1)); + x = v1(static_cast<char>(r1)); + x = v1(static_cast<signed char>(r1)); + x = v1(static_cast<unsigned char>(r1)); + x = v1(static_cast<signed short>(r1)); + x = v1(static_cast<unsigned short>(r1)); + x = v1(static_cast<signed int>(r1)); + x = v1(static_cast<unsigned int>(r1)); + x = v1(static_cast<signed long>(r1)); + x = v1(static_cast<unsigned long>(r1)); +#if EIGEN_HAS_CXX11 + x = v1(static_cast<long long int>(r1)); + x = v1(static_cast<unsigned long long int>(r1)); +#endif + VERIFY_IS_APPROX( v1, v1); VERIFY_IS_NOT_APPROX( v1, 2*v1); VERIFY_IS_MUCH_SMALLER_THAN( vzero, v1); @@ -108,22 +123,22 @@ // check automatic transposition sm2.setZero(); - for(typename MatrixType::Index i=0;i<rows;++i) + for(Index i=0;i<rows;++i) sm2.col(i) = sm1.row(i); VERIFY_IS_APPROX(sm2,sm1.transpose()); sm2.setZero(); - for(typename MatrixType::Index i=0;i<rows;++i) + for(Index i=0;i<rows;++i) sm2.col(i).noalias() = sm1.row(i); VERIFY_IS_APPROX(sm2,sm1.transpose()); sm2.setZero(); - for(typename MatrixType::Index i=0;i<rows;++i) + for(Index i=0;i<rows;++i) sm2.col(i).noalias() += sm1.row(i); VERIFY_IS_APPROX(sm2,sm1.transpose()); sm2.setZero(); - for(typename MatrixType::Index i=0;i<rows;++i) + for(Index i=0;i<rows;++i) sm2.col(i).noalias() -= sm1.row(i); VERIFY_IS_APPROX(sm2,-sm1.transpose()); @@ -144,7 +159,6 @@ template<typename MatrixType> void basicStuffComplex(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<RealScalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime> RealMatrixType; @@ -180,7 +194,7 @@ VERIFY(!static_cast<const MatrixType&>(cm).imag().isZero()); } -#ifdef EIGEN_TEST_PART_2 +template<int> void casting() { Matrix4f m = Matrix4f::Random(), m2; @@ -189,7 +203,6 @@ m2 = m.cast<float>(); // check the specialization when NewType == Type VERIFY(m.isApprox(m2)); } -#endif template <typename Scalar> void fixedSizeMatrixConstruction() @@ -254,7 +267,7 @@ } } -void test_basicstuff() +EIGEN_DECLARE_TEST(basicstuff) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( basicStuff(Matrix<float, 1, 1>()) ); @@ -276,5 +289,5 @@ CALL_SUBTEST_1(fixedSizeMatrixConstruction<long int>()); CALL_SUBTEST_1(fixedSizeMatrixConstruction<std::ptrdiff_t>()); - CALL_SUBTEST_2(casting()); + CALL_SUBTEST_2(casting<0>()); }
diff --git a/test/bdcsvd.cpp b/test/bdcsvd.cpp index f9f687a..85a80d6 100644 --- a/test/bdcsvd.cpp +++ b/test/bdcsvd.cpp
@@ -46,6 +46,8 @@ VERIFY_RAISES_ASSERT(m.bdcSvd().matrixU()); VERIFY_RAISES_ASSERT(m.bdcSvd().matrixV()); VERIFY_IS_APPROX(m.bdcSvd(ComputeFullU|ComputeFullV).solve(m), m); + VERIFY_IS_APPROX(m.bdcSvd(ComputeFullU|ComputeFullV).transpose().solve(m), m); + VERIFY_IS_APPROX(m.bdcSvd(ComputeFullU|ComputeFullV).adjoint().solve(m), m); } // compare the Singular values returned with Jacobi and Bdc @@ -62,7 +64,7 @@ if(computationOptions & ComputeThinV) VERIFY_IS_APPROX(bdc_svd.matrixV(), jacobi_svd.matrixV()); } -void test_bdcsvd() +EIGEN_DECLARE_TEST(bdcsvd) { CALL_SUBTEST_3(( svd_verify_assert<BDCSVD<Matrix3f> >(Matrix3f()) )); CALL_SUBTEST_4(( svd_verify_assert<BDCSVD<Matrix4d> >(Matrix4d()) )); @@ -104,7 +106,8 @@ CALL_SUBTEST_7( BDCSVD<MatrixXf>(10,10) ); // Check that preallocation avoids subsequent mallocs - CALL_SUBTEST_9( svd_preallocate<void>() ); + // Disabled because not supported by BDCSVD + // CALL_SUBTEST_9( svd_preallocate<void>() ); CALL_SUBTEST_2( svd_underoverflow<void>() ); }
diff --git a/test/bicgstab.cpp b/test/bicgstab.cpp index 4cc0dd3..59c4b50 100644 --- a/test/bicgstab.cpp +++ b/test/bicgstab.cpp
@@ -10,11 +10,11 @@ #include "sparse_solver.h" #include <Eigen/IterativeLinearSolvers> -template<typename T, typename I> void test_bicgstab_T() +template<typename T, typename I_> void test_bicgstab_T() { - BiCGSTAB<SparseMatrix<T,0,I>, DiagonalPreconditioner<T> > bicgstab_colmajor_diag; - BiCGSTAB<SparseMatrix<T,0,I>, IdentityPreconditioner > bicgstab_colmajor_I; - BiCGSTAB<SparseMatrix<T,0,I>, IncompleteLUT<T,I> > bicgstab_colmajor_ilut; + BiCGSTAB<SparseMatrix<T,0,I_>, DiagonalPreconditioner<T> > bicgstab_colmajor_diag; + BiCGSTAB<SparseMatrix<T,0,I_>, IdentityPreconditioner > bicgstab_colmajor_I; + BiCGSTAB<SparseMatrix<T,0,I_>, IncompleteLUT<T,I_> > bicgstab_colmajor_ilut; //BiCGSTAB<SparseMatrix<T>, SSORPreconditioner<T> > bicgstab_colmajor_ssor; bicgstab_colmajor_diag.setTolerance(NumTraits<T>::epsilon()*4); @@ -26,7 +26,7 @@ //CALL_SUBTEST( check_sparse_square_solving(bicgstab_colmajor_ssor) ); } -void test_bicgstab() +EIGEN_DECLARE_TEST(bicgstab) { CALL_SUBTEST_1((test_bicgstab_T<double,int>()) ); CALL_SUBTEST_2((test_bicgstab_T<std::complex<double>, int>()));
diff --git a/test/block.cpp b/test/block.cpp index 1eeb2da..84124ab 100644 --- a/test/block.cpp +++ b/test/block.cpp
@@ -29,15 +29,21 @@ return Scalar(0); } +// Check at compile-time that T1==T2, and at runtime-time that a==b +template<typename T1,typename T2> +typename internal::enable_if<internal::is_same<T1,T2>::value,bool>::type +is_same_block(const T1& a, const T2& b) +{ + return a.isApprox(b); +} template<typename MatrixType> void block(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; typedef Matrix<Scalar, 1, MatrixType::ColsAtCompileTime> RowVectorType; - typedef Matrix<Scalar, Dynamic, Dynamic> DynamicMatrixType; + typedef Matrix<Scalar, Dynamic, Dynamic, MatrixType::IsRowMajor?RowMajor:ColMajor> DynamicMatrixType; typedef Matrix<Scalar, Dynamic, 1> DynamicVectorType; Index rows = m.rows(); @@ -87,10 +93,9 @@ m1.block(r1,c1,r2-r1+1,c2-c1+1) = s1 * m2.block(0, 0, r2-r1+1,c2-c1+1); m1.block(r1,c1,r2-r1+1,c2-c1+1)(r2-r1,c2-c1) = m2.block(0, 0, r2-r1+1,c2-c1+1)(0,0); - enum { - BlockRows = 2, - BlockCols = 5 - }; + const Index BlockRows = 2; + const Index BlockCols = 5; + if (rows>=5 && cols>=8) { // test fixed block() as lvalue @@ -106,6 +111,11 @@ m1.template block<BlockRows,Dynamic>(1,1,BlockRows,BlockCols)(0,3) = m1.template block<2,5>(1,1)(1,2); Matrix<Scalar,Dynamic,Dynamic> b2 = m1.template block<Dynamic,BlockCols>(3,3,2,5); VERIFY_IS_EQUAL(b2, m1.block(3,3,BlockRows,BlockCols)); + + VERIFY(is_same_block(m1.block(3,3,BlockRows,BlockCols), m1.block(3,3,fix<Dynamic>(BlockRows),fix<Dynamic>(BlockCols)))); + VERIFY(is_same_block(m1.template block<BlockRows,Dynamic>(1,1,BlockRows,BlockCols), m1.block(1,1,fix<BlockRows>,BlockCols))); + VERIFY(is_same_block(m1.template block<BlockRows,BlockCols>(1,1,BlockRows,BlockCols), m1.block(1,1,fix<BlockRows>(),fix<BlockCols>))); + VERIFY(is_same_block(m1.template block<BlockRows,BlockCols>(1,1,BlockRows,BlockCols), m1.block(1,1,fix<BlockRows>,fix<BlockCols>(BlockCols)))); } if (rows>2) @@ -131,7 +141,7 @@ VERIFY(numext::real(ones.col(c1).dot(ones.col(c2))) == RealScalar(rows)); VERIFY(numext::real(ones.row(r1).dot(ones.row(r2))) == RealScalar(cols)); - // chekc that linear acccessors works on blocks + // check that linear acccessors works on blocks m1 = m1_copy; if((MatrixType::Flags&RowMajorBit)==0) VERIFY_IS_EQUAL(m1.leftCols(c1).coeff(r1+c1*rows), m1(r1,c1)); @@ -151,9 +161,25 @@ // expressions without direct access VERIFY_IS_APPROX( ((m1+m2).block(r1,c1,rows-r1,cols-c1).block(r2-r1,c2-c1,rows-r2,cols-c2)) , ((m1+m2).block(r2,c2,rows-r2,cols-c2)) ); VERIFY_IS_APPROX( ((m1+m2).block(r1,c1,r2-r1+1,c2-c1+1).row(0)) , ((m1+m2).row(r1).segment(c1,c2-c1+1)) ); + VERIFY_IS_APPROX( ((m1+m2).block(r1,c1,r2-r1+1,c2-c1+1).row(0)) , ((m1+m2).eval().row(r1).segment(c1,c2-c1+1)) ); VERIFY_IS_APPROX( ((m1+m2).block(r1,c1,r2-r1+1,c2-c1+1).col(0)) , ((m1+m2).col(c1).segment(r1,r2-r1+1)) ); VERIFY_IS_APPROX( ((m1+m2).block(r1,c1,r2-r1+1,c2-c1+1).transpose().col(0)) , ((m1+m2).row(r1).segment(c1,c2-c1+1)).transpose() ); VERIFY_IS_APPROX( ((m1+m2).transpose().block(c1,r1,c2-c1+1,r2-r1+1).col(0)) , ((m1+m2).row(r1).segment(c1,c2-c1+1)).transpose() ); + VERIFY_IS_APPROX( ((m1+m2).template block<Dynamic,1>(r1,c1,r2-r1+1,1)) , ((m1+m2).eval().col(c1).eval().segment(r1,r2-r1+1)) ); + VERIFY_IS_APPROX( ((m1+m2).template block<1,Dynamic>(r1,c1,1,c2-c1+1)) , ((m1+m2).eval().row(r1).eval().segment(c1,c2-c1+1)) ); + VERIFY_IS_APPROX( ((m1+m2).transpose().template block<1,Dynamic>(c1,r1,1,r2-r1+1)) , ((m1+m2).eval().col(c1).eval().segment(r1,r2-r1+1)).transpose() ); + VERIFY_IS_APPROX( (m1+m2).row(r1).eval(), (m1+m2).eval().row(r1) ); + VERIFY_IS_APPROX( (m1+m2).adjoint().col(r1).eval(), (m1+m2).adjoint().eval().col(r1) ); + VERIFY_IS_APPROX( (m1+m2).adjoint().row(c1).eval(), (m1+m2).adjoint().eval().row(c1) ); + VERIFY_IS_APPROX( (m1*1).row(r1).segment(c1,c2-c1+1).eval(), m1.row(r1).eval().segment(c1,c2-c1+1).eval() ); + VERIFY_IS_APPROX( m1.col(c1).reverse().segment(r1,r2-r1+1).eval(),m1.col(c1).reverse().eval().segment(r1,r2-r1+1).eval() ); + + VERIFY_IS_APPROX( (m1*1).topRows(r1), m1.topRows(r1) ); + VERIFY_IS_APPROX( (m1*1).leftCols(c1), m1.leftCols(c1) ); + VERIFY_IS_APPROX( (m1*1).transpose().topRows(c1), m1.transpose().topRows(c1) ); + VERIFY_IS_APPROX( (m1*1).transpose().leftCols(r1), m1.transpose().leftCols(r1) ); + VERIFY_IS_APPROX( (m1*1).transpose().middleRows(c1,c2-c1+1), m1.transpose().middleRows(c1,c2-c1+1) ); + VERIFY_IS_APPROX( (m1*1).transpose().middleCols(r1,r2-r1+1), m1.transpose().middleCols(r1,r2-r1+1) ); // evaluation into plain matrices from expressions with direct access (stress MapBase) DynamicMatrixType dm; @@ -186,13 +212,37 @@ VERIFY_IS_EQUAL( (m1.template block<1,Dynamic>(0,1,1,0)), m1.block(0,1,1,0)); VERIFY_IS_EQUAL( ((m1*1).template block<Dynamic,1>(1,0,0,1)), m1.block(1,0,0,1)); VERIFY_IS_EQUAL( ((m1*1).template block<1,Dynamic>(0,1,1,0)), m1.block(0,1,1,0)); + + if (rows>=2 && cols>=2) + { + VERIFY_RAISES_ASSERT( m1 += m1.col(0) ); + VERIFY_RAISES_ASSERT( m1 -= m1.col(0) ); + VERIFY_RAISES_ASSERT( m1.array() *= m1.col(0).array() ); + VERIFY_RAISES_ASSERT( m1.array() /= m1.col(0).array() ); + } + + VERIFY_IS_EQUAL( m1.template subVector<Horizontal>(r1), m1.row(r1) ); + VERIFY_IS_APPROX( (m1+m1).template subVector<Horizontal>(r1), (m1+m1).row(r1) ); + VERIFY_IS_EQUAL( m1.template subVector<Vertical>(c1), m1.col(c1) ); + VERIFY_IS_APPROX( (m1+m1).template subVector<Vertical>(c1), (m1+m1).col(c1) ); + VERIFY_IS_EQUAL( m1.template subVectors<Horizontal>(), m1.rows() ); + VERIFY_IS_EQUAL( m1.template subVectors<Vertical>(), m1.cols() ); + + if (rows>=2 || cols>=2) { + VERIFY_IS_EQUAL( int(m1.middleCols(0,0).IsRowMajor), int(m1.IsRowMajor) ); + VERIFY_IS_EQUAL( m1.middleCols(0,0).outerSize(), m1.IsRowMajor ? rows : 0); + VERIFY_IS_EQUAL( m1.middleCols(0,0).innerSize(), m1.IsRowMajor ? 0 : rows); + + VERIFY_IS_EQUAL( int(m1.middleRows(0,0).IsRowMajor), int(m1.IsRowMajor) ); + VERIFY_IS_EQUAL( m1.middleRows(0,0).outerSize(), m1.IsRowMajor ? 0 : cols); + VERIFY_IS_EQUAL( m1.middleRows(0,0).innerSize(), m1.IsRowMajor ? cols : 0); + } } template<typename MatrixType> void compare_using_data_and_stride(const MatrixType& m) { - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); Index size = m.size(); @@ -226,7 +276,6 @@ template<typename MatrixType> void data_and_stride(const MatrixType& m) { - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -244,15 +293,18 @@ compare_using_data_and_stride(m1.col(c1).transpose()); } -void test_block() +EIGEN_DECLARE_TEST(block) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( block(Matrix<float, 1, 1>()) ); + CALL_SUBTEST_1( block(Matrix<float, 1, Dynamic>(internal::random(2,50))) ); + CALL_SUBTEST_1( block(Matrix<float, Dynamic, 1>(internal::random(2,50))) ); CALL_SUBTEST_2( block(Matrix4d()) ); - CALL_SUBTEST_3( block(MatrixXcf(3, 3)) ); - CALL_SUBTEST_4( block(MatrixXi(8, 12)) ); - CALL_SUBTEST_5( block(MatrixXcd(20, 20)) ); - CALL_SUBTEST_6( block(MatrixXf(20, 20)) ); + CALL_SUBTEST_3( block(MatrixXcf(internal::random(2,50), internal::random(2,50))) ); + CALL_SUBTEST_4( block(MatrixXi(internal::random(2,50), internal::random(2,50))) ); + CALL_SUBTEST_5( block(MatrixXcd(internal::random(2,50), internal::random(2,50))) ); + CALL_SUBTEST_6( block(MatrixXf(internal::random(2,50), internal::random(2,50))) ); + CALL_SUBTEST_7( block(Matrix<int,Dynamic,Dynamic,RowMajor>(internal::random(2,50), internal::random(2,50))) ); CALL_SUBTEST_8( block(Matrix<float,Dynamic,4>(3, 4)) );
diff --git a/test/boostmultiprec.cpp b/test/boostmultiprec.cpp new file mode 100644 index 0000000..1d1441a --- /dev/null +++ b/test/boostmultiprec.cpp
@@ -0,0 +1,208 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include <sstream> + +#ifdef EIGEN_TEST_MAX_SIZE +#undef EIGEN_TEST_MAX_SIZE +#endif + +#define EIGEN_TEST_MAX_SIZE 50 + +#ifdef EIGEN_TEST_PART_1 +#include "cholesky.cpp" +#endif + +#ifdef EIGEN_TEST_PART_2 +#include "lu.cpp" +#endif + +#ifdef EIGEN_TEST_PART_3 +#include "qr.cpp" +#endif + +#ifdef EIGEN_TEST_PART_4 +#include "qr_colpivoting.cpp" +#endif + +#ifdef EIGEN_TEST_PART_5 +#include "qr_fullpivoting.cpp" +#endif + +#ifdef EIGEN_TEST_PART_6 +#include "eigensolver_selfadjoint.cpp" +#endif + +#ifdef EIGEN_TEST_PART_7 +#include "eigensolver_generic.cpp" +#endif + +#ifdef EIGEN_TEST_PART_8 +#include "eigensolver_generalized_real.cpp" +#endif + +#ifdef EIGEN_TEST_PART_9 +#include "jacobisvd.cpp" +#endif + +#ifdef EIGEN_TEST_PART_10 +#include "bdcsvd.cpp" +#endif + +#ifdef EIGEN_TEST_PART_11 +#include "simplicial_cholesky.cpp" +#endif + +#include <Eigen/Dense> + +#undef min +#undef max +#undef isnan +#undef isinf +#undef isfinite +#undef I + +#include <boost/multiprecision/cpp_dec_float.hpp> +#include <boost/multiprecision/number.hpp> +#include <boost/math/special_functions.hpp> +#include <boost/math/complex.hpp> + +namespace mp = boost::multiprecision; +typedef mp::number<mp::cpp_dec_float<100>, mp::et_on> Real; + +namespace Eigen { + template<> struct NumTraits<Real> : GenericNumTraits<Real> { + static inline Real dummy_precision() { return 1e-50; } + }; + + template<typename T1,typename T2,typename T3,typename T4,typename T5> + struct NumTraits<boost::multiprecision::detail::expression<T1,T2,T3,T4,T5> > : NumTraits<Real> {}; + + template<> + Real test_precision<Real>() { return 1e-50; } + + // needed in C++93 mode where number does not support explicit cast. + namespace internal { + template<typename NewType> + struct cast_impl<Real,NewType> { + static inline NewType run(const Real& x) { + return x.template convert_to<NewType>(); + } + }; + + template<> + struct cast_impl<Real,std::complex<Real> > { + static inline std::complex<Real> run(const Real& x) { + return std::complex<Real>(x); + } + }; + } +} + +namespace boost { +namespace multiprecision { + // to make ADL works as expected: + using boost::math::isfinite; + using boost::math::isnan; + using boost::math::isinf; + using boost::math::copysign; + using boost::math::hypot; + + // The following is needed for std::complex<Real>: + Real fabs(const Real& a) { return abs EIGEN_NOT_A_MACRO (a); } + Real fmax(const Real& a, const Real& b) { using std::max; return max(a,b); } + + // some specialization for the unit tests: + inline bool test_isMuchSmallerThan(const Real& a, const Real& b) { + return internal::isMuchSmallerThan(a, b, test_precision<Real>()); + } + + inline bool test_isApprox(const Real& a, const Real& b) { + return internal::isApprox(a, b, test_precision<Real>()); + } + + inline bool test_isApproxOrLessThan(const Real& a, const Real& b) { + return internal::isApproxOrLessThan(a, b, test_precision<Real>()); + } + + Real get_test_precision(const Real&) { + return test_precision<Real>(); + } + + Real test_relative_error(const Real &a, const Real &b) { + using Eigen::numext::abs2; + return sqrt(abs2<Real>(a-b)/Eigen::numext::mini<Real>(abs2(a),abs2(b))); + } +} +} + +namespace Eigen { + +} + +EIGEN_DECLARE_TEST(boostmultiprec) +{ + typedef Matrix<Real,Dynamic,Dynamic> Mat; + typedef Matrix<std::complex<Real>,Dynamic,Dynamic> MatC; + + std::cout << "NumTraits<Real>::epsilon() = " << NumTraits<Real>::epsilon() << std::endl; + std::cout << "NumTraits<Real>::dummy_precision() = " << NumTraits<Real>::dummy_precision() << std::endl; + std::cout << "NumTraits<Real>::lowest() = " << NumTraits<Real>::lowest() << std::endl; + std::cout << "NumTraits<Real>::highest() = " << NumTraits<Real>::highest() << std::endl; + std::cout << "NumTraits<Real>::digits10() = " << NumTraits<Real>::digits10() << std::endl; + + // check stream output + { + Mat A(10,10); + A.setRandom(); + std::stringstream ss; + ss << A; + } + { + MatC A(10,10); + A.setRandom(); + std::stringstream ss; + ss << A; + } + + for(int i = 0; i < g_repeat; i++) { + int s = internal::random<int>(1,EIGEN_TEST_MAX_SIZE); + + CALL_SUBTEST_1( cholesky(Mat(s,s)) ); + + CALL_SUBTEST_2( lu_non_invertible<Mat>() ); + CALL_SUBTEST_2( lu_invertible<Mat>() ); + CALL_SUBTEST_2( lu_non_invertible<MatC>() ); + CALL_SUBTEST_2( lu_invertible<MatC>() ); + + CALL_SUBTEST_3( qr(Mat(internal::random<int>(1,EIGEN_TEST_MAX_SIZE),internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_3( qr_invertible<Mat>() ); + + CALL_SUBTEST_4( qr<Mat>() ); + CALL_SUBTEST_4( cod<Mat>() ); + CALL_SUBTEST_4( qr_invertible<Mat>() ); + + CALL_SUBTEST_5( qr<Mat>() ); + CALL_SUBTEST_5( qr_invertible<Mat>() ); + + CALL_SUBTEST_6( selfadjointeigensolver(Mat(s,s)) ); + + CALL_SUBTEST_7( eigensolver(Mat(s,s)) ); + + CALL_SUBTEST_8( generalized_eigensolver_real(Mat(s,s)) ); + + TEST_SET_BUT_UNUSED_VARIABLE(s) + } + + CALL_SUBTEST_9(( jacobisvd(Mat(internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE), internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE/2))) )); + CALL_SUBTEST_10(( bdcsvd(Mat(internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE), internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE/2))) )); + + CALL_SUBTEST_11(( test_simplicial_cholesky_T<Real,int>() )); +} +
diff --git a/test/bug1213.cpp b/test/bug1213.cpp new file mode 100644 index 0000000..581760c --- /dev/null +++ b/test/bug1213.cpp
@@ -0,0 +1,13 @@ + +// This anonymous enum is essential to trigger the linking issue +enum { + Foo +}; + +#include "bug1213.h" + +bool bug1213_1(const Eigen::Vector3f& x) +{ + return bug1213_2(x); +} +
diff --git a/test/bug1213.h b/test/bug1213.h new file mode 100644 index 0000000..040e5a4 --- /dev/null +++ b/test/bug1213.h
@@ -0,0 +1,8 @@ + +#include <Eigen/Core> + +template<typename T, int dim> +bool bug1213_2(const Eigen::Matrix<T,dim,1>& x); + +bool bug1213_1(const Eigen::Vector3f& x); +
diff --git a/test/bug1213_main.cpp b/test/bug1213_main.cpp new file mode 100644 index 0000000..4802c00 --- /dev/null +++ b/test/bug1213_main.cpp
@@ -0,0 +1,18 @@ + +// This is a regression unit regarding a weird linking issue with gcc. + +#include "bug1213.h" + +int main() +{ + return 0; +} + + +template<typename T, int dim> +bool bug1213_2(const Eigen::Matrix<T,dim,1>& ) +{ + return true; +} + +template bool bug1213_2<float,3>(const Eigen::Vector3f&);
diff --git a/test/cholesky.cpp b/test/cholesky.cpp index b7abc23..0b1a7b4 100644 --- a/test/cholesky.cpp +++ b/test/cholesky.cpp
@@ -7,18 +7,16 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_NO_ASSERTION_CHECKING -#define EIGEN_NO_ASSERTION_CHECKING -#endif - #define TEST_ENABLE_TEMPORARY_TRACKING #include "main.h" #include <Eigen/Cholesky> #include <Eigen/QR> +#include "solverbase.h" template<typename MatrixType, int UpLo> typename MatrixType::RealScalar matrix_l1_norm(const MatrixType& m) { + if(m.cols()==0) return typename MatrixType::RealScalar(0); MatrixType symm = m.template selfadjointView<UpLo>(); return symm.cwiseAbs().colwise().sum().maxCoeff(); } @@ -57,7 +55,6 @@ template<typename MatrixType> void cholesky(const MatrixType& m) { - typedef typename MatrixType::Index Index; /* this test covers the following files: LLT.h LDLT.h */ @@ -81,15 +78,17 @@ } { + STATIC_CHECK(( internal::is_same<typename LLT<MatrixType,Lower>::StorageIndex,int>::value )); + STATIC_CHECK(( internal::is_same<typename LLT<MatrixType,Upper>::StorageIndex,int>::value )); + SquareMatrixType symmUp = symm.template triangularView<Upper>(); SquareMatrixType symmLo = symm.template triangularView<Lower>(); LLT<SquareMatrixType,Lower> chollo(symmLo); VERIFY_IS_APPROX(symm, chollo.reconstructedMatrix()); - vecX = chollo.solve(vecB); - VERIFY_IS_APPROX(symm * vecX, vecB); - matX = chollo.solve(matB); - VERIFY_IS_APPROX(symm * matX, matB); + + check_solverbase<VectorType, VectorType>(symm, chollo, rows, rows, 1); + check_solverbase<MatrixType, MatrixType>(symm, chollo, rows, cols, rows); const MatrixType symmLo_inverse = chollo.solve(MatrixType::Identity(rows,cols)); RealScalar rcond = (RealScalar(1) / matrix_l1_norm<MatrixType, Lower>(symmLo)) / @@ -97,7 +96,7 @@ RealScalar rcond_est = chollo.rcond(); // Verify that the estimated condition number is within a factor of 10 of the // truth. - VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); + VERIFY(rcond_est >= rcond / 10 && rcond_est <= rcond * 10); // test the upper mode LLT<SquareMatrixType,Upper> cholup(symmUp); @@ -113,12 +112,12 @@ rcond = (RealScalar(1) / matrix_l1_norm<MatrixType, Upper>(symmUp)) / matrix_l1_norm<MatrixType, Upper>(symmUp_inverse); rcond_est = cholup.rcond(); - VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); + VERIFY(rcond_est >= rcond / 10 && rcond_est <= rcond * 10); MatrixType neg = -symmLo; chollo.compute(neg); - VERIFY(chollo.info()==NumericalIssue); + VERIFY(neg.size()==0 || chollo.info()==NumericalIssue); VERIFY_IS_APPROX(MatrixType(chollo.matrixL().transpose().conjugate()), MatrixType(chollo.matrixU())); VERIFY_IS_APPROX(MatrixType(chollo.matrixU().transpose().conjugate()), MatrixType(chollo.matrixL())); @@ -143,6 +142,9 @@ // LDLT { + STATIC_CHECK(( internal::is_same<typename LDLT<MatrixType,Lower>::StorageIndex,int>::value )); + STATIC_CHECK(( internal::is_same<typename LDLT<MatrixType,Upper>::StorageIndex,int>::value )); + int sign = internal::random<int>()%2 ? 1 : -1; if(sign == -1) @@ -154,11 +156,11 @@ SquareMatrixType symmLo = symm.template triangularView<Lower>(); LDLT<SquareMatrixType,Lower> ldltlo(symmLo); + VERIFY(ldltlo.info()==Success); VERIFY_IS_APPROX(symm, ldltlo.reconstructedMatrix()); - vecX = ldltlo.solve(vecB); - VERIFY_IS_APPROX(symm * vecX, vecB); - matX = ldltlo.solve(matB); - VERIFY_IS_APPROX(symm * matX, matB); + + check_solverbase<VectorType, VectorType>(symm, ldltlo, rows, rows, 1); + check_solverbase<MatrixType, MatrixType>(symm, ldltlo, rows, cols, rows); const MatrixType symmLo_inverse = ldltlo.solve(MatrixType::Identity(rows,cols)); RealScalar rcond = (RealScalar(1) / matrix_l1_norm<MatrixType, Lower>(symmLo)) / @@ -166,10 +168,11 @@ RealScalar rcond_est = ldltlo.rcond(); // Verify that the estimated condition number is within a factor of 10 of the // truth. - VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); + VERIFY(rcond_est >= rcond / 10 && rcond_est <= rcond * 10); LDLT<SquareMatrixType,Upper> ldltup(symmUp); + VERIFY(ldltup.info()==Success); VERIFY_IS_APPROX(symm, ldltup.reconstructedMatrix()); vecX = ldltup.solve(vecB); VERIFY_IS_APPROX(symm * vecX, vecB); @@ -182,7 +185,7 @@ rcond = (RealScalar(1) / matrix_l1_norm<MatrixType, Upper>(symmUp)) / matrix_l1_norm<MatrixType, Upper>(symmUp_inverse); rcond_est = ldltup.rcond(); - VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); + VERIFY(rcond_est >= rcond / 10 && rcond_est <= rcond * 10); VERIFY_IS_APPROX(MatrixType(ldltlo.matrixL().transpose().conjugate()), MatrixType(ldltlo.matrixU())); VERIFY_IS_APPROX(MatrixType(ldltlo.matrixU().transpose().conjugate()), MatrixType(ldltlo.matrixL())); @@ -243,11 +246,13 @@ // check matrices with a wide spectrum if(rows>=3) { + using std::pow; + using std::sqrt; RealScalar s = (std::min)(16,std::numeric_limits<RealScalar>::max_exponent10/8); Matrix<Scalar,Dynamic,Dynamic> a = Matrix<Scalar,Dynamic,Dynamic>::Random(rows,rows); Matrix<RealScalar,Dynamic,1> d = Matrix<RealScalar,Dynamic,1>::Random(rows); for(Index k=0; k<rows; ++k) - d(k) = d(k)*std::pow(RealScalar(10),internal::random<RealScalar>(-s,s)); + d(k) = d(k)*pow(RealScalar(10),internal::random<RealScalar>(-s,s)); SquareMatrixType A = a * d.asDiagonal() * a.adjoint(); // Make sure a solution exists: vecX.setRandom(); @@ -263,7 +268,7 @@ } else { - RealScalar large_tol = std::sqrt(test_precision<RealScalar>()); + RealScalar large_tol = sqrt(test_precision<RealScalar>()); VERIFY((A * vecX).isApprox(vecB, large_tol)); ++g_test_level; @@ -285,8 +290,6 @@ // test mixing real/scalar types - typedef typename MatrixType::Index Index; - Index rows = m.rows(); Index cols = m.cols(); @@ -311,10 +314,9 @@ LLT<RealMatrixType,Lower> chollo(symmLo); VERIFY_IS_APPROX(symm, chollo.reconstructedMatrix()); - vecX = chollo.solve(vecB); - VERIFY_IS_APPROX(symm * vecX, vecB); -// matX = chollo.solve(matB); -// VERIFY_IS_APPROX(symm * matX, matB); + + check_solverbase<VectorType, VectorType>(symm, chollo, rows, rows, 1); + //check_solverbase<MatrixType, MatrixType>(symm, chollo, rows, cols, rows); } // LDLT @@ -329,11 +331,11 @@ RealMatrixType symmLo = symm.template triangularView<Lower>(); LDLT<RealMatrixType,Lower> ldltlo(symmLo); + VERIFY(ldltlo.info()==Success); VERIFY_IS_APPROX(symm, ldltlo.reconstructedMatrix()); - vecX = ldltlo.solve(vecB); - VERIFY_IS_APPROX(symm * vecX, vecB); -// matX = ldltlo.solve(matB); -// VERIFY_IS_APPROX(symm * matX, matB); + + check_solverbase<VectorType, VectorType>(symm, ldltlo, rows, rows, 1); + //check_solverbase<MatrixType, MatrixType>(symm, ldltlo, rows, cols, rows); } } @@ -365,32 +367,104 @@ { mat << 1, 0, 0, -1; ldlt.compute(mat); + VERIFY(ldlt.info()==Success); VERIFY(!ldlt.isNegative()); VERIFY(!ldlt.isPositive()); + VERIFY_IS_APPROX(mat,ldlt.reconstructedMatrix()); } { mat << 1, 2, 2, 1; ldlt.compute(mat); + VERIFY(ldlt.info()==Success); VERIFY(!ldlt.isNegative()); VERIFY(!ldlt.isPositive()); + VERIFY_IS_APPROX(mat,ldlt.reconstructedMatrix()); } { mat << 0, 0, 0, 0; ldlt.compute(mat); + VERIFY(ldlt.info()==Success); VERIFY(ldlt.isNegative()); VERIFY(ldlt.isPositive()); + VERIFY_IS_APPROX(mat,ldlt.reconstructedMatrix()); } { mat << 0, 0, 0, 1; ldlt.compute(mat); + VERIFY(ldlt.info()==Success); VERIFY(!ldlt.isNegative()); VERIFY(ldlt.isPositive()); + VERIFY_IS_APPROX(mat,ldlt.reconstructedMatrix()); } { mat << -1, 0, 0, 0; ldlt.compute(mat); + VERIFY(ldlt.info()==Success); VERIFY(ldlt.isNegative()); VERIFY(!ldlt.isPositive()); + VERIFY_IS_APPROX(mat,ldlt.reconstructedMatrix()); + } +} + +template<typename> +void cholesky_faillure_cases() +{ + MatrixXd mat; + LDLT<MatrixXd> ldlt; + + { + mat.resize(2,2); + mat << 0, 1, 1, 0; + ldlt.compute(mat); + VERIFY_IS_NOT_APPROX(mat,ldlt.reconstructedMatrix()); + VERIFY(ldlt.info()==NumericalIssue); + } +#if (!EIGEN_ARCH_i386) || defined(EIGEN_VECTORIZE_SSE2) + { + mat.resize(3,3); + mat << -1, -3, 3, + -3, -8.9999999999999999999, 1, + 3, 1, 0; + ldlt.compute(mat); + VERIFY(ldlt.info()==NumericalIssue); + VERIFY_IS_NOT_APPROX(mat,ldlt.reconstructedMatrix()); + } +#endif + { + mat.resize(3,3); + mat << 1, 2, 3, + 2, 4, 1, + 3, 1, 0; + ldlt.compute(mat); + VERIFY(ldlt.info()==NumericalIssue); + VERIFY_IS_NOT_APPROX(mat,ldlt.reconstructedMatrix()); + } + + { + mat.resize(8,8); + mat << 0.1, 0, -0.1, 0, 0, 0, 1, 0, + 0, 4.24667, 0, 2.00333, 0, 0, 0, 0, + -0.1, 0, 0.2, 0, -0.1, 0, 0, 0, + 0, 2.00333, 0, 8.49333, 0, 2.00333, 0, 0, + 0, 0, -0.1, 0, 0.1, 0, 0, 1, + 0, 0, 0, 2.00333, 0, 4.24667, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0; + ldlt.compute(mat); + VERIFY(ldlt.info()==NumericalIssue); + VERIFY_IS_NOT_APPROX(mat,ldlt.reconstructedMatrix()); + } + + // bug 1479 + { + mat.resize(4,4); + mat << 1, 2, 0, 1, + 2, 4, 0, 2, + 0, 0, 0, 1, + 1, 2, 1, 1; + ldlt.compute(mat); + VERIFY(ldlt.info()==NumericalIssue); + VERIFY_IS_NOT_APPROX(mat,ldlt.reconstructedMatrix()); } } @@ -402,19 +476,23 @@ VERIFY_RAISES_ASSERT(llt.matrixL()) VERIFY_RAISES_ASSERT(llt.matrixU()) VERIFY_RAISES_ASSERT(llt.solve(tmp)) - VERIFY_RAISES_ASSERT(llt.solveInPlace(&tmp)) + VERIFY_RAISES_ASSERT(llt.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(llt.adjoint().solve(tmp)) + VERIFY_RAISES_ASSERT(llt.solveInPlace(tmp)) LDLT<MatrixType> ldlt; VERIFY_RAISES_ASSERT(ldlt.matrixL()) - VERIFY_RAISES_ASSERT(ldlt.permutationP()) + VERIFY_RAISES_ASSERT(ldlt.transpositionsP()) VERIFY_RAISES_ASSERT(ldlt.vectorD()) VERIFY_RAISES_ASSERT(ldlt.isPositive()) VERIFY_RAISES_ASSERT(ldlt.isNegative()) VERIFY_RAISES_ASSERT(ldlt.solve(tmp)) - VERIFY_RAISES_ASSERT(ldlt.solveInPlace(&tmp)) + VERIFY_RAISES_ASSERT(ldlt.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(ldlt.adjoint().solve(tmp)) + VERIFY_RAISES_ASSERT(ldlt.solveInPlace(tmp)) } -void test_cholesky() +EIGEN_DECLARE_TEST(cholesky) { int s = 0; for(int i = 0; i < g_repeat; i++) { @@ -433,6 +511,11 @@ CALL_SUBTEST_6( cholesky_cplx(MatrixXcd(s,s)) ); TEST_SET_BUT_UNUSED_VARIABLE(s) } + // empty matrix, regression test for Bug 785: + CALL_SUBTEST_2( cholesky(MatrixXd(0,0)) ); + + // This does not work yet: + // CALL_SUBTEST_2( cholesky(Matrix<double,0,0>()) ); CALL_SUBTEST_4( cholesky_verify_assert<Matrix3f>() ); CALL_SUBTEST_7( cholesky_verify_assert<Matrix3d>() ); @@ -443,5 +526,7 @@ CALL_SUBTEST_9( LLT<MatrixXf>(10) ); CALL_SUBTEST_9( LDLT<MatrixXf>(10) ); + CALL_SUBTEST_2( cholesky_faillure_cases<void>() ); + TEST_SET_BUT_UNUSED_VARIABLE(nb_temporaries) }
diff --git a/test/cholmod_support.cpp b/test/cholmod_support.cpp index a7eda28..89b9cf4 100644 --- a/test/cholmod_support.cpp +++ b/test/cholmod_support.cpp
@@ -12,21 +12,21 @@ #include <Eigen/CholmodSupport> -template<typename T> void test_cholmod_T() +template<typename SparseType> void test_cholmod_ST() { - CholmodDecomposition<SparseMatrix<T>, Lower> g_chol_colmajor_lower; g_chol_colmajor_lower.setMode(CholmodSupernodalLLt); - CholmodDecomposition<SparseMatrix<T>, Upper> g_chol_colmajor_upper; g_chol_colmajor_upper.setMode(CholmodSupernodalLLt); - CholmodDecomposition<SparseMatrix<T>, Lower> g_llt_colmajor_lower; g_llt_colmajor_lower.setMode(CholmodSimplicialLLt); - CholmodDecomposition<SparseMatrix<T>, Upper> g_llt_colmajor_upper; g_llt_colmajor_upper.setMode(CholmodSimplicialLLt); - CholmodDecomposition<SparseMatrix<T>, Lower> g_ldlt_colmajor_lower; g_ldlt_colmajor_lower.setMode(CholmodLDLt); - CholmodDecomposition<SparseMatrix<T>, Upper> g_ldlt_colmajor_upper; g_ldlt_colmajor_upper.setMode(CholmodLDLt); + CholmodDecomposition<SparseType, Lower> g_chol_colmajor_lower; g_chol_colmajor_lower.setMode(CholmodSupernodalLLt); + CholmodDecomposition<SparseType, Upper> g_chol_colmajor_upper; g_chol_colmajor_upper.setMode(CholmodSupernodalLLt); + CholmodDecomposition<SparseType, Lower> g_llt_colmajor_lower; g_llt_colmajor_lower.setMode(CholmodSimplicialLLt); + CholmodDecomposition<SparseType, Upper> g_llt_colmajor_upper; g_llt_colmajor_upper.setMode(CholmodSimplicialLLt); + CholmodDecomposition<SparseType, Lower> g_ldlt_colmajor_lower; g_ldlt_colmajor_lower.setMode(CholmodLDLt); + CholmodDecomposition<SparseType, Upper> g_ldlt_colmajor_upper; g_ldlt_colmajor_upper.setMode(CholmodLDLt); - CholmodSupernodalLLT<SparseMatrix<T>, Lower> chol_colmajor_lower; - CholmodSupernodalLLT<SparseMatrix<T>, Upper> chol_colmajor_upper; - CholmodSimplicialLLT<SparseMatrix<T>, Lower> llt_colmajor_lower; - CholmodSimplicialLLT<SparseMatrix<T>, Upper> llt_colmajor_upper; - CholmodSimplicialLDLT<SparseMatrix<T>, Lower> ldlt_colmajor_lower; - CholmodSimplicialLDLT<SparseMatrix<T>, Upper> ldlt_colmajor_upper; + CholmodSupernodalLLT<SparseType, Lower> chol_colmajor_lower; + CholmodSupernodalLLT<SparseType, Upper> chol_colmajor_upper; + CholmodSimplicialLLT<SparseType, Lower> llt_colmajor_lower; + CholmodSimplicialLLT<SparseType, Upper> llt_colmajor_upper; + CholmodSimplicialLDLT<SparseType, Lower> ldlt_colmajor_lower; + CholmodSimplicialLDLT<SparseType, Upper> ldlt_colmajor_upper; check_sparse_spd_solving(g_chol_colmajor_lower); check_sparse_spd_solving(g_chol_colmajor_upper); @@ -50,8 +50,20 @@ check_sparse_spd_determinant(ldlt_colmajor_upper); } -void test_cholmod_support() +template<typename T, int flags, typename IdxType> void test_cholmod_T() { - CALL_SUBTEST_1(test_cholmod_T<double>()); - CALL_SUBTEST_2(test_cholmod_T<std::complex<double> >()); + test_cholmod_ST<SparseMatrix<T, flags, IdxType> >(); +} + +EIGEN_DECLARE_TEST(cholmod_support) +{ + CALL_SUBTEST_11( (test_cholmod_T<double , ColMajor, int >()) ); + CALL_SUBTEST_12( (test_cholmod_T<double , ColMajor, long>()) ); + CALL_SUBTEST_13( (test_cholmod_T<double , RowMajor, int >()) ); + CALL_SUBTEST_14( (test_cholmod_T<double , RowMajor, long>()) ); + CALL_SUBTEST_21( (test_cholmod_T<std::complex<double>, ColMajor, int >()) ); + CALL_SUBTEST_22( (test_cholmod_T<std::complex<double>, ColMajor, long>()) ); + // TODO complex row-major matrices do not work at the moment: + // CALL_SUBTEST_23( (test_cholmod_T<std::complex<double>, RowMajor, int >()) ); + // CALL_SUBTEST_24( (test_cholmod_T<std::complex<double>, RowMajor, long>()) ); }
diff --git a/test/commainitializer.cpp b/test/commainitializer.cpp index 99102b9..3cb94da 100644 --- a/test/commainitializer.cpp +++ b/test/commainitializer.cpp
@@ -9,7 +9,63 @@ #include "main.h" -void test_commainitializer() + +template<int M1, int M2, int N1, int N2> +void test_blocks() +{ + Matrix<int, M1+M2, N1+N2> m_fixed; + MatrixXi m_dynamic(M1+M2, N1+N2); + + Matrix<int, M1, N1> mat11; mat11.setRandom(); + Matrix<int, M1, N2> mat12; mat12.setRandom(); + Matrix<int, M2, N1> mat21; mat21.setRandom(); + Matrix<int, M2, N2> mat22; mat22.setRandom(); + + MatrixXi matx11 = mat11, matx12 = mat12, matx21 = mat21, matx22 = mat22; + + { + VERIFY_IS_EQUAL((m_fixed << mat11, mat12, mat21, matx22).finished(), (m_dynamic << mat11, matx12, mat21, matx22).finished()); + VERIFY_IS_EQUAL((m_fixed.template topLeftCorner<M1,N1>()), mat11); + VERIFY_IS_EQUAL((m_fixed.template topRightCorner<M1,N2>()), mat12); + VERIFY_IS_EQUAL((m_fixed.template bottomLeftCorner<M2,N1>()), mat21); + VERIFY_IS_EQUAL((m_fixed.template bottomRightCorner<M2,N2>()), mat22); + VERIFY_IS_EQUAL((m_fixed << mat12, mat11, matx21, mat22).finished(), (m_dynamic << mat12, matx11, matx21, mat22).finished()); + } + + if(N1 > 0) + { + VERIFY_RAISES_ASSERT((m_fixed << mat11, mat12, mat11, mat21, mat22)); + VERIFY_RAISES_ASSERT((m_fixed << mat11, mat12, mat21, mat21, mat22)); + } + else + { + // allow insertion of zero-column blocks: + VERIFY_IS_EQUAL((m_fixed << mat11, mat12, mat11, mat11, mat21, mat21, mat22).finished(), (m_dynamic << mat12, mat22).finished()); + } + if(M1 != M2) + { + VERIFY_RAISES_ASSERT((m_fixed << mat11, mat21, mat12, mat22)); + } +} + + +template<int N> +struct test_block_recursion +{ + static void run() + { + test_blocks<(N>>6)&3, (N>>4)&3, (N>>2)&3, N & 3>(); + test_block_recursion<N-1>::run(); + } +}; + +template<> +struct test_block_recursion<-1> +{ + static void run() { } +}; + +EIGEN_DECLARE_TEST(commainitializer) { Matrix3d m3; Matrix4d m4; @@ -43,4 +99,8 @@ 4, 5, 6, vec[2].transpose(); VERIFY_IS_APPROX(m3, ref); + + + // recursively test all block-sizes from 0 to 3: + test_block_recursion<(1<<8) - 1>(); }
diff --git a/test/conjugate_gradient.cpp b/test/conjugate_gradient.cpp index 9622fd8..b076a12 100644 --- a/test/conjugate_gradient.cpp +++ b/test/conjugate_gradient.cpp
@@ -10,9 +10,9 @@ #include "sparse_solver.h" #include <Eigen/IterativeLinearSolvers> -template<typename T, typename I> void test_conjugate_gradient_T() +template<typename T, typename I_> void test_conjugate_gradient_T() { - typedef SparseMatrix<T,0,I> SparseMatrixType; + typedef SparseMatrix<T,0,I_> SparseMatrixType; ConjugateGradient<SparseMatrixType, Lower > cg_colmajor_lower_diag; ConjugateGradient<SparseMatrixType, Upper > cg_colmajor_upper_diag; ConjugateGradient<SparseMatrixType, Lower|Upper> cg_colmajor_loup_diag; @@ -26,7 +26,7 @@ CALL_SUBTEST( check_sparse_spd_solving(cg_colmajor_upper_I) ); } -void test_conjugate_gradient() +EIGEN_DECLARE_TEST(conjugate_gradient) { CALL_SUBTEST_1(( test_conjugate_gradient_T<double,int>() )); CALL_SUBTEST_2(( test_conjugate_gradient_T<std::complex<double>, int>() ));
diff --git a/test/conservative_resize.cpp b/test/conservative_resize.cpp index 498421b..5dc5000 100644 --- a/test/conservative_resize.cpp +++ b/test/conservative_resize.cpp
@@ -10,6 +10,7 @@ #include "main.h" #include <Eigen/Core> +#include "AnnoyingScalar.h" using namespace Eigen; @@ -17,7 +18,6 @@ void run_matrix_tests() { typedef Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Storage> MatrixType; - typedef typename MatrixType::Index Index; MatrixType m, n; @@ -110,7 +110,31 @@ } } -void test_conservative_resize() +// Basic memory leak check with a non-copyable scalar type +template<int> void noncopyable() +{ + typedef Eigen::Matrix<AnnoyingScalar,Dynamic,1> VectorType; + typedef Eigen::Matrix<AnnoyingScalar,Dynamic,Dynamic> MatrixType; + + { + AnnoyingScalar::dont_throw = true; + int n = 50; + VectorType v0(n), v1(n); + MatrixType m0(n,n), m1(n,n), m2(n,n); + v0.setOnes(); v1.setOnes(); + m0.setOnes(); m1.setOnes(); m2.setOnes(); + VERIFY(m0==m1); + m0.conservativeResize(2*n,2*n); + VERIFY(m0.topLeftCorner(n,n) == m1); + + VERIFY(v0.head(n) == v1); + v0.conservativeResize(2*n); + VERIFY(v0.head(n) == v1); + } + VERIFY(AnnoyingScalar::instances==0 && "global memory leak detected in noncopyable"); +} + +EIGEN_DECLARE_TEST(conservative_resize) { for(int i=0; i<g_repeat; ++i) { @@ -123,12 +147,16 @@ CALL_SUBTEST_4((run_matrix_tests<std::complex<float>, Eigen::RowMajor>())); CALL_SUBTEST_4((run_matrix_tests<std::complex<float>, Eigen::ColMajor>())); CALL_SUBTEST_5((run_matrix_tests<std::complex<double>, Eigen::RowMajor>())); - CALL_SUBTEST_6((run_matrix_tests<std::complex<double>, Eigen::ColMajor>())); + CALL_SUBTEST_5((run_matrix_tests<std::complex<double>, Eigen::ColMajor>())); CALL_SUBTEST_1((run_vector_tests<int>())); CALL_SUBTEST_2((run_vector_tests<float>())); CALL_SUBTEST_3((run_vector_tests<double>())); CALL_SUBTEST_4((run_vector_tests<std::complex<float> >())); CALL_SUBTEST_5((run_vector_tests<std::complex<double> >())); + + AnnoyingScalar::dont_throw = true; + CALL_SUBTEST_6(( run_vector_tests<AnnoyingScalar>() )); + CALL_SUBTEST_6(( noncopyable<0>() )); } }
diff --git a/test/constructor.cpp b/test/constructor.cpp new file mode 100644 index 0000000..1dd3bc3 --- /dev/null +++ b/test/constructor.cpp
@@ -0,0 +1,84 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define TEST_ENABLE_TEMPORARY_TRACKING + +#include "main.h" + +template<typename MatrixType> struct Wrapper +{ + MatrixType m_mat; + inline Wrapper(const MatrixType &x) : m_mat(x) {} + inline operator const MatrixType& () const { return m_mat; } + inline operator MatrixType& () { return m_mat; } +}; + +template<typename MatrixType> void ctor_init1(const MatrixType& m) +{ + // Check logic in PlainObjectBase::_init1 + Index rows = m.rows(); + Index cols = m.cols(); + + MatrixType m0 = MatrixType::Random(rows,cols); + + VERIFY_EVALUATION_COUNT( MatrixType m1(m0), 1); + VERIFY_EVALUATION_COUNT( MatrixType m2(m0+m0), 1); + VERIFY_EVALUATION_COUNT( MatrixType m2(m0.block(0,0,rows,cols)) , 1); + + Wrapper<MatrixType> wrapper(m0); + VERIFY_EVALUATION_COUNT( MatrixType m3(wrapper) , 1); +} + + +EIGEN_DECLARE_TEST(constructor) +{ + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1( ctor_init1(Matrix<float, 1, 1>()) ); + CALL_SUBTEST_1( ctor_init1(Matrix4d()) ); + CALL_SUBTEST_1( ctor_init1(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_1( ctor_init1(MatrixXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + } + { + Matrix<Index,1,1> a(123); + VERIFY_IS_EQUAL(a[0], 123); + } + { + Matrix<Index,1,1> a(123.0); + VERIFY_IS_EQUAL(a[0], 123); + } + { + Matrix<float,1,1> a(123); + VERIFY_IS_EQUAL(a[0], 123.f); + } + { + Array<Index,1,1> a(123); + VERIFY_IS_EQUAL(a[0], 123); + } + { + Array<Index,1,1> a(123.0); + VERIFY_IS_EQUAL(a[0], 123); + } + { + Array<float,1,1> a(123); + VERIFY_IS_EQUAL(a[0], 123.f); + } + { + Array<Index,3,3> a(123); + VERIFY_IS_EQUAL(a(4), 123); + } + { + Array<Index,3,3> a(123.0); + VERIFY_IS_EQUAL(a(4), 123); + } + { + Array<float,3,3> a(123); + VERIFY_IS_EQUAL(a(4), 123.f); + } +}
diff --git a/test/corners.cpp b/test/corners.cpp index 3c64c32..73342a8 100644 --- a/test/corners.cpp +++ b/test/corners.cpp
@@ -15,7 +15,6 @@ template<typename MatrixType> void corners(const MatrixType& m) { - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -102,7 +101,7 @@ VERIFY_IS_EQUAL((const_matrix.template rightCols<c>()), (const_matrix.template block<rows,c>(0,cols-c))); } -void test_corners() +EIGEN_DECLARE_TEST(corners) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( corners(Matrix<float, 1, 1>()) );
diff --git a/test/ctorleak.cpp b/test/ctorleak.cpp index c158f5e..7390417 100644 --- a/test/ctorleak.cpp +++ b/test/ctorleak.cpp
@@ -8,7 +8,7 @@ static Index object_limit; int dummy; - Foo() + Foo() : dummy(0) { #ifdef EIGEN_EXCEPTIONS // TODO: Is this the correct way to handle this? @@ -33,26 +33,37 @@ #undef EIGEN_TEST_MAX_SIZE #define EIGEN_TEST_MAX_SIZE 3 -void test_ctorleak() +EIGEN_DECLARE_TEST(ctorleak) { typedef Matrix<Foo, Dynamic, Dynamic> MatrixX; typedef Matrix<Foo, Dynamic, 1> VectorX; + Foo::object_count = 0; for(int i = 0; i < g_repeat; i++) { Index rows = internal::random<Index>(2,EIGEN_TEST_MAX_SIZE), cols = internal::random<Index>(2,EIGEN_TEST_MAX_SIZE); - Foo::object_limit = internal::random<Index>(0, rows*cols - 2); + Foo::object_limit = rows*cols; + { + MatrixX r(rows, cols); + Foo::object_limit = r.size()+internal::random<Index>(0, rows*cols - 2); std::cout << "object_limit =" << Foo::object_limit << std::endl; #ifdef EIGEN_EXCEPTIONS try { #endif - std::cout << "\nMatrixX m(" << rows << ", " << cols << ");\n"; - MatrixX m(rows, cols); + if(internal::random<bool>()) { + std::cout << "\nMatrixX m(" << rows << ", " << cols << ");\n"; + MatrixX m(rows, cols); + } + else { + std::cout << "\nMatrixX m(r);\n"; + MatrixX m(r); + } #ifdef EIGEN_EXCEPTIONS VERIFY(false); // not reached if exceptions are enabled } catch (const Foo::Fail&) { /* ignore */ } #endif + } VERIFY_IS_EQUAL(Index(0), Foo::object_count); { @@ -66,4 +77,5 @@ } VERIFY_IS_EQUAL(Index(0), Foo::object_count); } + std::cout << "\n"; }
diff --git a/test/cuda_basic.cu b/test/cuda_basic.cu deleted file mode 100644 index b36ed88..0000000 --- a/test/cuda_basic.cu +++ /dev/null
@@ -1,161 +0,0 @@ - - -// workaround issue between gcc >= 4.7 and cuda 5.5 -#if (defined __GNUC__) && (__GNUC__>4 || __GNUC_MINOR__>=7) - #undef _GLIBCXX_ATOMIC_BUILTINS - #undef _GLIBCXX_USE_INT128 -#endif - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cuda_basic -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int - -#include <math_constants.h> -#include "main.h" -#include "cuda_common.h" - -#include <Eigen/Eigenvalues> - -// struct Foo{ -// EIGEN_DEVICE_FUNC -// void operator()(int i, const float* mats, float* vecs) const { -// using namespace Eigen; -// // Matrix3f M(data); -// // Vector3f x(data+9); -// // Map<Vector3f>(data+9) = M.inverse() * x; -// Matrix3f M(mats+i/16); -// Vector3f x(vecs+i*3); -// // using std::min; -// // using std::sqrt; -// Map<Vector3f>(vecs+i*3) << x.minCoeff(), 1, 2;// / x.dot(x);//(M.inverse() * x) / x.x(); -// //x = x*2 + x.y() * x + x * x.maxCoeff() - x / x.sum(); -// } -// }; - -template<typename T> -struct coeff_wise { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const - { - using namespace Eigen; - T x1(in+i); - T x2(in+i+1); - T x3(in+i+2); - Map<T> res(out+i*T::MaxSizeAtCompileTime); - - res.array() += (in[0] * x1 + x2).array() * x3.array(); - } -}; - -template<typename T> -struct replicate { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const - { - using namespace Eigen; - T x1(in+i); - int step = x1.size() * 4; - int stride = 3 * step; - - typedef Map<Array<typename T::Scalar,Dynamic,Dynamic> > MapType; - MapType(out+i*stride+0*step, x1.rows()*2, x1.cols()*2) = x1.replicate(2,2); - MapType(out+i*stride+1*step, x1.rows()*3, x1.cols()) = in[i] * x1.colwise().replicate(3); - MapType(out+i*stride+2*step, x1.rows(), x1.cols()*3) = in[i] * x1.rowwise().replicate(3); - } -}; - -template<typename T> -struct redux { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const - { - using namespace Eigen; - int N = 10; - T x1(in+i); - out[i*N+0] = x1.minCoeff(); - out[i*N+1] = x1.maxCoeff(); - out[i*N+2] = x1.sum(); - out[i*N+3] = x1.prod(); - out[i*N+4] = x1.matrix().squaredNorm(); - out[i*N+5] = x1.matrix().norm(); - out[i*N+6] = x1.colwise().sum().maxCoeff(); - out[i*N+7] = x1.rowwise().maxCoeff().sum(); - out[i*N+8] = x1.matrix().colwise().squaredNorm().sum(); - } -}; - -template<typename T1, typename T2> -struct prod_test { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T1::Scalar* in, typename T1::Scalar* out) const - { - using namespace Eigen; - typedef Matrix<typename T1::Scalar, T1::RowsAtCompileTime, T2::ColsAtCompileTime> T3; - T1 x1(in+i); - T2 x2(in+i+1); - Map<T3> res(out+i*T3::MaxSizeAtCompileTime); - res += in[i] * x1 * x2; - } -}; - -template<typename T1, typename T2> -struct diagonal { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T1::Scalar* in, typename T1::Scalar* out) const - { - using namespace Eigen; - T1 x1(in+i); - Map<T2> res(out+i*T2::MaxSizeAtCompileTime); - res += x1.diagonal(); - } -}; - -template<typename T> -struct eigenvalues { - EIGEN_DEVICE_FUNC - void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const - { - using namespace Eigen; - typedef Matrix<typename T::Scalar, T::RowsAtCompileTime, 1> Vec; - T M(in+i); - Map<Vec> res(out+i*Vec::MaxSizeAtCompileTime); - T A = M*M.adjoint(); - SelfAdjointEigenSolver<T> eig; - eig.computeDirect(M); - res = eig.eigenvalues(); - } -}; - -void test_cuda_basic() -{ - ei_test_init_cuda(); - - int nthreads = 100; - Eigen::VectorXf in, out; - - #ifndef __CUDA_ARCH__ - int data_size = nthreads * 512; - in.setRandom(data_size); - out.setRandom(data_size); - #endif - - CALL_SUBTEST( run_and_compare_to_cuda(coeff_wise<Vector3f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(coeff_wise<Array44f>(), nthreads, in, out) ); - - CALL_SUBTEST( run_and_compare_to_cuda(replicate<Array4f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(replicate<Array33f>(), nthreads, in, out) ); - - CALL_SUBTEST( run_and_compare_to_cuda(redux<Array4f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(redux<Matrix3f>(), nthreads, in, out) ); - - CALL_SUBTEST( run_and_compare_to_cuda(prod_test<Matrix3f,Matrix3f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(prod_test<Matrix4f,Vector4f>(), nthreads, in, out) ); - - CALL_SUBTEST( run_and_compare_to_cuda(diagonal<Matrix3f,Vector3f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(diagonal<Matrix4f,Vector4f>(), nthreads, in, out) ); - - CALL_SUBTEST( run_and_compare_to_cuda(eigenvalues<Matrix3f>(), nthreads, in, out) ); - CALL_SUBTEST( run_and_compare_to_cuda(eigenvalues<Matrix2f>(), nthreads, in, out) ); - -}
diff --git a/test/cuda_common.h b/test/cuda_common.h deleted file mode 100644 index 9737693..0000000 --- a/test/cuda_common.h +++ /dev/null
@@ -1,101 +0,0 @@ - -#ifndef EIGEN_TEST_CUDA_COMMON_H -#define EIGEN_TEST_CUDA_COMMON_H - -#include <cuda.h> -#include <cuda_runtime.h> -#include <cuda_runtime_api.h> -#include <iostream> - -#ifndef __CUDACC__ -dim3 threadIdx, blockDim, blockIdx; -#endif - -template<typename Kernel, typename Input, typename Output> -void run_on_cpu(const Kernel& ker, int n, const Input& in, Output& out) -{ - for(int i=0; i<n; i++) - ker(i, in.data(), out.data()); -} - - -template<typename Kernel, typename Input, typename Output> -__global__ -void run_on_cuda_meta_kernel(const Kernel ker, int n, const Input* in, Output* out) -{ - int i = threadIdx.x + blockIdx.x*blockDim.x; - if(i<n) { - ker(i, in, out); - } -} - - -template<typename Kernel, typename Input, typename Output> -void run_on_cuda(const Kernel& ker, int n, const Input& in, Output& out) -{ - typename Input::Scalar* d_in; - typename Output::Scalar* d_out; - std::ptrdiff_t in_bytes = in.size() * sizeof(typename Input::Scalar); - std::ptrdiff_t out_bytes = out.size() * sizeof(typename Output::Scalar); - - cudaMalloc((void**)(&d_in), in_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_in, in.data(), in_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_out, out.data(), out_bytes, cudaMemcpyHostToDevice); - - // Simple and non-optimal 1D mapping assuming n is not too large - // That's only for unit testing! - dim3 Blocks(128); - dim3 Grids( (n+int(Blocks.x)-1)/int(Blocks.x) ); - - cudaThreadSynchronize(); - run_on_cuda_meta_kernel<<<Grids,Blocks>>>(ker, n, d_in, d_out); - cudaThreadSynchronize(); - - // check inputs have not been modified - cudaMemcpy(const_cast<typename Input::Scalar*>(in.data()), d_in, in_bytes, cudaMemcpyDeviceToHost); - cudaMemcpy(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost); - - cudaFree(d_in); - cudaFree(d_out); -} - - -template<typename Kernel, typename Input, typename Output> -void run_and_compare_to_cuda(const Kernel& ker, int n, const Input& in, Output& out) -{ - Input in_ref, in_cuda; - Output out_ref, out_cuda; - #ifndef __CUDA_ARCH__ - in_ref = in_cuda = in; - out_ref = out_cuda = out; - #endif - run_on_cpu (ker, n, in_ref, out_ref); - run_on_cuda(ker, n, in_cuda, out_cuda); - #ifndef __CUDA_ARCH__ - VERIFY_IS_APPROX(in_ref, in_cuda); - VERIFY_IS_APPROX(out_ref, out_cuda); - #endif -} - - -void ei_test_init_cuda() -{ - int device = 0; - cudaDeviceProp deviceProp; - cudaGetDeviceProperties(&deviceProp, device); - std::cout << "CUDA device info:\n"; - std::cout << " name: " << deviceProp.name << "\n"; - std::cout << " capability: " << deviceProp.major << "." << deviceProp.minor << "\n"; - std::cout << " multiProcessorCount: " << deviceProp.multiProcessorCount << "\n"; - std::cout << " maxThreadsPerMultiProcessor: " << deviceProp.maxThreadsPerMultiProcessor << "\n"; - std::cout << " warpSize: " << deviceProp.warpSize << "\n"; - std::cout << " regsPerBlock: " << deviceProp.regsPerBlock << "\n"; - std::cout << " concurrentKernels: " << deviceProp.concurrentKernels << "\n"; - std::cout << " clockRate: " << deviceProp.clockRate << "\n"; - std::cout << " canMapHostMemory: " << deviceProp.canMapHostMemory << "\n"; - std::cout << " computeMode: " << deviceProp.computeMode << "\n"; -} - -#endif // EIGEN_TEST_CUDA_COMMON_H
diff --git a/test/denseLM.cpp b/test/denseLM.cpp index 0aa736e..afb8004 100644 --- a/test/denseLM.cpp +++ b/test/denseLM.cpp
@@ -182,7 +182,7 @@ } -void test_denseLM() +EIGEN_DECLARE_TEST(denseLM) { CALL_SUBTEST_2(test_denseLM_T<double>());
diff --git a/test/dense_storage.cpp b/test/dense_storage.cpp index e63712b..7fa2585 100644 --- a/test/dense_storage.cpp +++ b/test/dense_storage.cpp
@@ -52,7 +52,33 @@ VERIFY_IS_EQUAL(raw_reference[i], raw_copied_reference[i]); } -void test_dense_storage() +template<typename T, int Size, std::size_t Alignment> +void dense_storage_alignment() +{ + #if EIGEN_HAS_ALIGNAS + + struct alignas(Alignment) Empty1 {}; + VERIFY_IS_EQUAL(std::alignment_of<Empty1>::value, Alignment); + + struct EIGEN_ALIGN_TO_BOUNDARY(Alignment) Empty2 {}; + VERIFY_IS_EQUAL(std::alignment_of<Empty2>::value, Alignment); + + struct Nested1 { EIGEN_ALIGN_TO_BOUNDARY(Alignment) T data[Size]; }; + VERIFY_IS_EQUAL(std::alignment_of<Nested1>::value, Alignment); + + VERIFY_IS_EQUAL( (std::alignment_of<internal::plain_array<T,Size,AutoAlign,Alignment> >::value), Alignment); + + const std::size_t default_alignment = internal::compute_default_alignment<T,Size>::value; + + VERIFY_IS_EQUAL( (std::alignment_of<DenseStorage<T,Size,1,1,AutoAlign> >::value), default_alignment); + VERIFY_IS_EQUAL( (std::alignment_of<Matrix<T,Size,1,AutoAlign> >::value), default_alignment); + struct Nested2 { Matrix<T,Size,1,AutoAlign> mat; }; + VERIFY_IS_EQUAL(std::alignment_of<Nested2>::value, default_alignment); + + #endif +} + +EIGEN_DECLARE_TEST(dense_storage) { dense_storage_copy<int,Dynamic,Dynamic>(); dense_storage_copy<int,Dynamic,3>(); @@ -72,5 +98,10 @@ dense_storage_assignment<float,Dynamic,Dynamic>(); dense_storage_assignment<float,Dynamic,3>(); dense_storage_assignment<float,4,Dynamic>(); - dense_storage_assignment<float,4,3>(); + dense_storage_assignment<float,4,3>(); + + dense_storage_alignment<float,16,8>(); + dense_storage_alignment<float,16,16>(); + dense_storage_alignment<float,16,32>(); + dense_storage_alignment<float,16,64>(); }
diff --git a/test/determinant.cpp b/test/determinant.cpp index 758f3af..7dd33c3 100644 --- a/test/determinant.cpp +++ b/test/determinant.cpp
@@ -16,7 +16,6 @@ /* this test covers the following files: Determinant.h */ - typedef typename MatrixType::Index Index; Index size = m.rows(); MatrixType m1(size, size), m2(size, size); @@ -51,7 +50,7 @@ VERIFY_IS_APPROX(m2.block(0,0,0,0).determinant(), Scalar(1)); } -void test_determinant() +EIGEN_DECLARE_TEST(determinant) { for(int i = 0; i < g_repeat; i++) { int s = 0;
diff --git a/test/diagonal.cpp b/test/diagonal.cpp index ee00cad..4e8c4b3 100644 --- a/test/diagonal.cpp +++ b/test/diagonal.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void diagonal(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(); @@ -66,9 +65,30 @@ m2.diagonal(N2).coeffRef(0) = Scalar(2)*s1; VERIFY_IS_APPROX(m2.diagonal(N2).coeff(0), Scalar(2)*s1); } + + VERIFY( m1.diagonal( cols).size()==0 ); + VERIFY( m1.diagonal(-rows).size()==0 ); } -void test_diagonal() +template<typename MatrixType> void diagonal_assert(const MatrixType& m) { + Index rows = m.rows(); + Index cols = m.cols(); + + MatrixType m1 = MatrixType::Random(rows, cols); + + if (rows>=2 && cols>=2) + { + VERIFY_RAISES_ASSERT( m1 += m1.diagonal() ); + VERIFY_RAISES_ASSERT( m1 -= m1.diagonal() ); + VERIFY_RAISES_ASSERT( m1.array() *= m1.diagonal().array() ); + VERIFY_RAISES_ASSERT( m1.array() /= m1.diagonal().array() ); + } + + VERIFY_RAISES_ASSERT( m1.diagonal(cols+1) ); + VERIFY_RAISES_ASSERT( m1.diagonal(-(rows+1)) ); +} + +EIGEN_DECLARE_TEST(diagonal) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( diagonal(Matrix<float, 1, 1>()) ); @@ -80,5 +100,6 @@ CALL_SUBTEST_2( diagonal(MatrixXcd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_1( diagonal(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_1( diagonal(Matrix<float,Dynamic,4>(3, 4)) ); + CALL_SUBTEST_1( diagonal_assert(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } }
diff --git a/test/diagonalmatrices.cpp b/test/diagonalmatrices.cpp index cd6dc8c..276bead 100644 --- a/test/diagonalmatrices.cpp +++ b/test/diagonalmatrices.cpp
@@ -11,7 +11,6 @@ using namespace std; template<typename MatrixType> void diagonalmatrices(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; enum { Rows = MatrixType::RowsAtCompileTime, Cols = MatrixType::ColsAtCompileTime }; typedef Matrix<Scalar, Rows, 1> VectorType; @@ -30,6 +29,7 @@ v2 = VectorType::Random(rows); RowVectorType rv1 = RowVectorType::Random(cols), rv2 = RowVectorType::Random(cols); + LeftDiagonalMatrix ldm1(v1), ldm2(v2); RightDiagonalMatrix rdm1(rv1), rdm2(rv2); @@ -99,6 +99,45 @@ VERIFY_IS_APPROX( (sq_m1 += (s1*v1).asDiagonal()), sq_m2 += (s1*v1).asDiagonal().toDenseMatrix() ); VERIFY_IS_APPROX( (sq_m1 -= (s1*v1).asDiagonal()), sq_m2 -= (s1*v1).asDiagonal().toDenseMatrix() ); VERIFY_IS_APPROX( (sq_m1 = (s1*v1).asDiagonal()), (s1*v1).asDiagonal().toDenseMatrix() ); + + sq_m1.setRandom(); + sq_m2 = v1.asDiagonal(); + sq_m2 = sq_m1 * sq_m2; + VERIFY_IS_APPROX( (sq_m1*v1.asDiagonal()).col(i), sq_m2.col(i) ); + VERIFY_IS_APPROX( (sq_m1*v1.asDiagonal()).row(i), sq_m2.row(i) ); + + sq_m1 = v1.asDiagonal(); + sq_m2 = v2.asDiagonal(); + SquareMatrixType sq_m3 = v1.asDiagonal(); + VERIFY_IS_APPROX( sq_m3 = v1.asDiagonal() + v2.asDiagonal(), sq_m1 + sq_m2); + VERIFY_IS_APPROX( sq_m3 = v1.asDiagonal() - v2.asDiagonal(), sq_m1 - sq_m2); + VERIFY_IS_APPROX( sq_m3 = v1.asDiagonal() - 2*v2.asDiagonal() + v1.asDiagonal(), sq_m1 - 2*sq_m2 + sq_m1); +} + +template<typename MatrixType> void as_scalar_product(const MatrixType& m) +{ + typedef typename MatrixType::Scalar Scalar; + typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; + typedef Matrix<Scalar, Dynamic, Dynamic> DynMatrixType; + typedef Matrix<Scalar, Dynamic, 1> DynVectorType; + typedef Matrix<Scalar, 1, Dynamic> DynRowVectorType; + + Index rows = m.rows(); + Index depth = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + + VectorType v1 = VectorType::Random(rows); + DynVectorType dv1 = DynVectorType::Random(depth); + DynRowVectorType drv1 = DynRowVectorType::Random(depth); + DynMatrixType dm1 = dv1; + DynMatrixType drm1 = drv1; + + Scalar s = v1(0); + + VERIFY_IS_APPROX( v1.asDiagonal() * drv1, s*drv1 ); + VERIFY_IS_APPROX( dv1 * v1.asDiagonal(), dv1*s ); + + VERIFY_IS_APPROX( v1.asDiagonal() * drm1, s*drm1 ); + VERIFY_IS_APPROX( dm1 * v1.asDiagonal(), dm1*s ); } template<int> @@ -112,18 +151,23 @@ VERIFY_IS_APPROX(( res1 = points.topLeftCorner<2,2>()*diag.asDiagonal()) , res2 = tmp2*diag.asDiagonal() ); } -void test_diagonalmatrices() +EIGEN_DECLARE_TEST(diagonalmatrices) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( diagonalmatrices(Matrix<float, 1, 1>()) ); + CALL_SUBTEST_1( as_scalar_product(Matrix<float, 1, 1>()) ); + CALL_SUBTEST_2( diagonalmatrices(Matrix3f()) ); CALL_SUBTEST_3( diagonalmatrices(Matrix<double,3,3,RowMajor>()) ); CALL_SUBTEST_4( diagonalmatrices(Matrix4d()) ); CALL_SUBTEST_5( diagonalmatrices(Matrix<float,4,4,RowMajor>()) ); CALL_SUBTEST_6( diagonalmatrices(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_6( as_scalar_product(MatrixXcf(1,1)) ); CALL_SUBTEST_7( diagonalmatrices(MatrixXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_8( diagonalmatrices(Matrix<double,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_9( diagonalmatrices(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_9( diagonalmatrices(MatrixXf(1,1)) ); + CALL_SUBTEST_9( as_scalar_product(MatrixXf(1,1)) ); } CALL_SUBTEST_10( bug987<0>() ); }
diff --git a/test/dontalign.cpp b/test/dontalign.cpp index 4643cfe..2e4102b 100644 --- a/test/dontalign.cpp +++ b/test/dontalign.cpp
@@ -19,7 +19,6 @@ template<typename MatrixType> void dontalign(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::RowsAtCompileTime> SquareMatrixType; @@ -45,7 +44,7 @@ internal::aligned_delete(array, rows); } -void test_dontalign() +EIGEN_DECLARE_TEST(dontalign) { #if defined EIGEN_TEST_PART_1 || defined EIGEN_TEST_PART_5 dontalign(Matrix3d());
diff --git a/test/dynalloc.cpp b/test/dynalloc.cpp index 5f58700..23c90a7 100644 --- a/test/dynalloc.cpp +++ b/test/dynalloc.cpp
@@ -15,6 +15,7 @@ #define ALIGNMENT 1 #endif +typedef Matrix<float,16,1> Vector16f; typedef Matrix<float,8,1> Vector8f; void check_handmade_aligned_malloc() @@ -22,7 +23,7 @@ for(int i = 1; i < 1000; i++) { char *p = (char*)internal::handmade_aligned_malloc(i); - VERIFY(size_t(p)%ALIGNMENT==0); + VERIFY(internal::UIntPtr(p)%ALIGNMENT==0); // if the buffer is wrongly allocated this will give a bad write --> check with valgrind for(int j = 0; j < i; j++) p[j]=0; internal::handmade_aligned_free(p); @@ -34,7 +35,7 @@ for(int i = ALIGNMENT; i < 1000; i++) { char *p = (char*)internal::aligned_malloc(i); - VERIFY(size_t(p)%ALIGNMENT==0); + VERIFY(internal::UIntPtr(p)%ALIGNMENT==0); // if the buffer is wrongly allocated this will give a bad write --> check with valgrind for(int j = 0; j < i; j++) p[j]=0; internal::aligned_free(p); @@ -46,7 +47,7 @@ for(int i = ALIGNMENT; i < 1000; i++) { float *p = internal::aligned_new<float>(i); - VERIFY(size_t(p)%ALIGNMENT==0); + VERIFY(internal::UIntPtr(p)%ALIGNMENT==0); // if the buffer is wrongly allocated this will give a bad write --> check with valgrind for(int j = 0; j < i; j++) p[j]=0; internal::aligned_delete(p,i); @@ -58,7 +59,7 @@ for(int i = ALIGNMENT; i < 400; i++) { ei_declare_aligned_stack_constructed_variable(float,p,i,0); - VERIFY(size_t(p)%ALIGNMENT==0); + VERIFY(internal::UIntPtr(p)%ALIGNMENT==0); // if the buffer is wrongly allocated this will give a bad write --> check with valgrind for(int j = 0; j < i; j++) p[j]=0; } @@ -70,7 +71,7 @@ { EIGEN_MAKE_ALIGNED_OPERATOR_NEW char dummychar; - Vector8f avec; + Vector16f avec; }; class MyClassA @@ -78,7 +79,7 @@ public: EIGEN_MAKE_ALIGNED_OPERATOR_NEW char dummychar; - Vector8f avec; + Vector16f avec; }; template<typename T> void check_dynaligned() @@ -88,7 +89,7 @@ { T* obj = new T; VERIFY(T::NeedsToAlign==1); - VERIFY(size_t(obj)%ALIGNMENT==0); + VERIFY(internal::UIntPtr(obj)%ALIGNMENT==0); delete obj; } } @@ -106,7 +107,7 @@ delete[] t; } -#if EIGEN_MAX_ALIGN_BYTES>0 +#if EIGEN_MAX_ALIGN_BYTES>0 && (!EIGEN_HAS_CXX17_OVERALIGN) { T* t = static_cast<T *>((T::operator new)(sizeof(T))); (T::operator delete)(t, sizeof(T)); @@ -119,7 +120,7 @@ #endif } -void test_dynalloc() +EIGEN_DECLARE_TEST(dynalloc) { // low level dynamic memory allocation CALL_SUBTEST(check_handmade_aligned_malloc()); @@ -145,18 +146,19 @@ CALL_SUBTEST(check_dynaligned<Vector4d>() ); CALL_SUBTEST(check_dynaligned<Vector4i>() ); CALL_SUBTEST(check_dynaligned<Vector8f>() ); + CALL_SUBTEST(check_dynaligned<Vector16f>() ); } { - MyStruct foo0; VERIFY(size_t(foo0.avec.data())%ALIGNMENT==0); - MyClassA fooA; VERIFY(size_t(fooA.avec.data())%ALIGNMENT==0); + MyStruct foo0; VERIFY(internal::UIntPtr(foo0.avec.data())%ALIGNMENT==0); + MyClassA fooA; VERIFY(internal::UIntPtr(fooA.avec.data())%ALIGNMENT==0); } // dynamic allocation, single object for (int i=0; i<g_repeat*100; ++i) { - MyStruct *foo0 = new MyStruct(); VERIFY(size_t(foo0->avec.data())%ALIGNMENT==0); - MyClassA *fooA = new MyClassA(); VERIFY(size_t(fooA->avec.data())%ALIGNMENT==0); + MyStruct *foo0 = new MyStruct(); VERIFY(internal::UIntPtr(foo0->avec.data())%ALIGNMENT==0); + MyClassA *fooA = new MyClassA(); VERIFY(internal::UIntPtr(fooA->avec.data())%ALIGNMENT==0); delete foo0; delete fooA; } @@ -165,8 +167,8 @@ const int N = 10; for (int i=0; i<g_repeat*100; ++i) { - MyStruct *foo0 = new MyStruct[N]; VERIFY(size_t(foo0->avec.data())%ALIGNMENT==0); - MyClassA *fooA = new MyClassA[N]; VERIFY(size_t(fooA->avec.data())%ALIGNMENT==0); + MyStruct *foo0 = new MyStruct[N]; VERIFY(internal::UIntPtr(foo0->avec.data())%ALIGNMENT==0); + MyClassA *fooA = new MyClassA[N]; VERIFY(internal::UIntPtr(fooA->avec.data())%ALIGNMENT==0); delete[] foo0; delete[] fooA; }
diff --git a/test/eigen2support.cpp b/test/eigen2support.cpp index ad1d980..49d7328 100644 --- a/test/eigen2support.cpp +++ b/test/eigen2support.cpp
@@ -13,7 +13,6 @@ template<typename MatrixType> void eigen2support(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(); @@ -53,7 +52,7 @@ m1.minor(0,0); } -void test_eigen2support() +EIGEN_DECLARE_TEST(eigen2support) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( eigen2support(Matrix<double,1,1>()) );
diff --git a/test/eigensolver_complex.cpp b/test/eigensolver_complex.cpp index 8e2bb9e..c5373f4 100644 --- a/test/eigensolver_complex.cpp +++ b/test/eigensolver_complex.cpp
@@ -47,7 +47,7 @@ return false; } -/* Check that two column vectors are approximately equal upto permutations. +/* Check that two column vectors are approximately equal up to permutations. * Initially, this method checked that the k-th power sums are equal for all k = 1, ..., vec1.rows(), * however this strategy is numerically inacurate because of numerical cancellation issues. */ @@ -71,7 +71,6 @@ template<typename MatrixType> void eigensolver(const MatrixType& m) { - typedef typename MatrixType::Index Index; /* this test covers the following files: ComplexEigenSolver.h, and indirectly ComplexSchur.h */ @@ -131,6 +130,15 @@ ComplexEigenSolver<MatrixType> eig(a.adjoint() * a); eig.compute(a.adjoint() * a); } + + // regression test for bug 478 + { + a.setZero(); + ComplexEigenSolver<MatrixType> ei3(a); + VERIFY_IS_EQUAL(ei3.info(), Success); + VERIFY_IS_MUCH_SMALLER_THAN(ei3.eigenvalues().norm(),RealScalar(1)); + VERIFY((ei3.eigenvectors().transpose()*ei3.eigenvectors().transpose()).eval().isIdentity()); + } } template<typename MatrixType> void eigensolver_verify_assert(const MatrixType& m) @@ -144,7 +152,7 @@ VERIFY_RAISES_ASSERT(eig.eigenvectors()); } -void test_eigensolver_complex() +EIGEN_DECLARE_TEST(eigensolver_complex) { int s = 0; for(int i = 0; i < g_repeat; i++) {
diff --git a/test/eigensolver_generalized_real.cpp b/test/eigensolver_generalized_real.cpp index a46a2e5..95ed431 100644 --- a/test/eigensolver_generalized_real.cpp +++ b/test/eigensolver_generalized_real.cpp
@@ -1,19 +1,20 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2012 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2012-2016 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#define EIGEN_RUNTIME_NO_MALLOC #include "main.h" #include <limits> #include <Eigen/Eigenvalues> +#include <Eigen/LU> template<typename MatrixType> void generalized_eigensolver_real(const MatrixType& m) { - typedef typename MatrixType::Index Index; /* this test covers the following files: GeneralizedEigenSolver.h */ @@ -21,6 +22,7 @@ Index cols = m.cols(); typedef typename MatrixType::Scalar Scalar; + typedef std::complex<Scalar> ComplexScalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; MatrixType a = MatrixType::Random(rows,cols); @@ -31,14 +33,41 @@ MatrixType spdB = b.adjoint() * b + b1.adjoint() * b1; // lets compare to GeneralizedSelfAdjointEigenSolver - GeneralizedSelfAdjointEigenSolver<MatrixType> symmEig(spdA, spdB); - GeneralizedEigenSolver<MatrixType> eig(spdA, spdB); + { + GeneralizedSelfAdjointEigenSolver<MatrixType> symmEig(spdA, spdB); + GeneralizedEigenSolver<MatrixType> eig(spdA, spdB); - VERIFY_IS_EQUAL(eig.eigenvalues().imag().cwiseAbs().maxCoeff(), 0); + VERIFY_IS_EQUAL(eig.eigenvalues().imag().cwiseAbs().maxCoeff(), 0); - VectorType realEigenvalues = eig.eigenvalues().real(); - std::sort(realEigenvalues.data(), realEigenvalues.data()+realEigenvalues.size()); - VERIFY_IS_APPROX(realEigenvalues, symmEig.eigenvalues()); + VectorType realEigenvalues = eig.eigenvalues().real(); + std::sort(realEigenvalues.data(), realEigenvalues.data()+realEigenvalues.size()); + VERIFY_IS_APPROX(realEigenvalues, symmEig.eigenvalues()); + + // check eigenvectors + typename GeneralizedEigenSolver<MatrixType>::EigenvectorsType D = eig.eigenvalues().asDiagonal(); + typename GeneralizedEigenSolver<MatrixType>::EigenvectorsType V = eig.eigenvectors(); + VERIFY_IS_APPROX(spdA*V, spdB*V*D); + } + + // non symmetric case: + { + GeneralizedEigenSolver<MatrixType> eig(rows); + // TODO enable full-prealocation of required memory, this probably requires an in-place mode for HessenbergDecomposition + //Eigen::internal::set_is_malloc_allowed(false); + eig.compute(a,b); + //Eigen::internal::set_is_malloc_allowed(true); + for(Index k=0; k<cols; ++k) + { + Matrix<ComplexScalar,Dynamic,Dynamic> tmp = (eig.betas()(k)*a).template cast<ComplexScalar>() - eig.alphas()(k)*b; + if(tmp.size()>1 && tmp.norm()>(std::numeric_limits<Scalar>::min)()) + tmp /= tmp.norm(); + VERIFY_IS_MUCH_SMALLER_THAN( std::abs(tmp.determinant()), Scalar(1) ); + } + // check eigenvectors + typename GeneralizedEigenSolver<MatrixType>::EigenvectorsType D = eig.eigenvalues().asDiagonal(); + typename GeneralizedEigenSolver<MatrixType>::EigenvectorsType V = eig.eigenvectors(); + VERIFY_IS_APPROX(a*V, b*V*D); + } // regression test for bug 1098 { @@ -47,9 +76,16 @@ GeneralizedEigenSolver<MatrixType> eig2(a.adjoint() * a,b.adjoint() * b); eig2.compute(a.adjoint() * a,b.adjoint() * b); } + + // check without eigenvectors + { + GeneralizedEigenSolver<MatrixType> eig1(spdA, spdB, true); + GeneralizedEigenSolver<MatrixType> eig2(spdA, spdB, false); + VERIFY_IS_APPROX(eig1.eigenvalues(), eig2.eigenvalues()); + } } -void test_eigensolver_generalized_real() +EIGEN_DECLARE_TEST(eigensolver_generalized_real) { for(int i = 0; i < g_repeat; i++) { int s = 0; @@ -57,7 +93,7 @@ s = internal::random<int>(1,EIGEN_TEST_MAX_SIZE/4); CALL_SUBTEST_2( generalized_eigensolver_real(MatrixXd(s,s)) ); - // some trivial but implementation-wise tricky cases + // some trivial but implementation-wise special cases CALL_SUBTEST_2( generalized_eigensolver_real(MatrixXd(1,1)) ); CALL_SUBTEST_2( generalized_eigensolver_real(MatrixXd(2,2)) ); CALL_SUBTEST_3( generalized_eigensolver_real(Matrix<double,1,1>()) );
diff --git a/test/eigensolver_generic.cpp b/test/eigensolver_generic.cpp index 5665463..7adb986 100644 --- a/test/eigensolver_generic.cpp +++ b/test/eigensolver_generic.cpp
@@ -12,9 +12,23 @@ #include <limits> #include <Eigen/Eigenvalues> +template<typename EigType,typename MatType> +void check_eigensolver_for_given_mat(const EigType &eig, const MatType& a) +{ + typedef typename NumTraits<typename MatType::Scalar>::Real RealScalar; + typedef Matrix<RealScalar, MatType::RowsAtCompileTime, 1> RealVectorType; + typedef typename std::complex<RealScalar> Complex; + Index n = a.rows(); + VERIFY_IS_EQUAL(eig.info(), Success); + VERIFY_IS_APPROX(a * eig.pseudoEigenvectors(), eig.pseudoEigenvectors() * eig.pseudoEigenvalueMatrix()); + VERIFY_IS_APPROX(a.template cast<Complex>() * eig.eigenvectors(), + eig.eigenvectors() * eig.eigenvalues().asDiagonal()); + VERIFY_IS_APPROX(eig.eigenvectors().colwise().norm(), RealVectorType::Ones(n).transpose()); + VERIFY_IS_APPROX(a.eigenvalues(), eig.eigenvalues()); +} + template<typename MatrixType> void eigensolver(const MatrixType& m) { - typedef typename MatrixType::Index Index; /* this test covers the following files: EigenSolver.h */ @@ -23,8 +37,7 @@ typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; - typedef Matrix<RealScalar, MatrixType::RowsAtCompileTime, 1> RealVectorType; - typedef typename std::complex<typename NumTraits<typename MatrixType::Scalar>::Real> Complex; + typedef typename std::complex<RealScalar> Complex; MatrixType a = MatrixType::Random(rows,cols); MatrixType a1 = MatrixType::Random(rows,cols); @@ -37,12 +50,7 @@ (ei0.pseudoEigenvectors().template cast<Complex>()) * (ei0.eigenvalues().asDiagonal())); EigenSolver<MatrixType> ei1(a); - VERIFY_IS_EQUAL(ei1.info(), Success); - VERIFY_IS_APPROX(a * ei1.pseudoEigenvectors(), ei1.pseudoEigenvectors() * ei1.pseudoEigenvalueMatrix()); - VERIFY_IS_APPROX(a.template cast<Complex>() * ei1.eigenvectors(), - ei1.eigenvectors() * ei1.eigenvalues().asDiagonal()); - VERIFY_IS_APPROX(ei1.eigenvectors().colwise().norm(), RealVectorType::Ones(rows).transpose()); - VERIFY_IS_APPROX(a.eigenvalues(), ei1.eigenvalues()); + CALL_SUBTEST( check_eigensolver_for_given_mat(ei1,a) ); EigenSolver<MatrixType> ei2; ei2.setMaxIterations(RealSchur<MatrixType>::m_maxIterationsPerRow * rows).compute(a); @@ -68,7 +76,7 @@ // Test matrix with NaN a(0,0) = std::numeric_limits<typename MatrixType::RealScalar>::quiet_NaN(); EigenSolver<MatrixType> eiNaN(a); - VERIFY_IS_EQUAL(eiNaN.info(), NoConvergence); + VERIFY_IS_NOT_EQUAL(eiNaN.info(), Success); } // regression test for bug 1098 @@ -76,6 +84,15 @@ EigenSolver<MatrixType> eig(a.adjoint() * a); eig.compute(a.adjoint() * a); } + + // regression test for bug 478 + { + a.setZero(); + EigenSolver<MatrixType> ei3(a); + VERIFY_IS_EQUAL(ei3.info(), Success); + VERIFY_IS_MUCH_SMALLER_THAN(ei3.eigenvalues().norm(),RealScalar(1)); + VERIFY((ei3.eigenvectors().transpose()*ei3.eigenvectors().transpose()).eval().isIdentity()); + } } template<typename MatrixType> void eigensolver_verify_assert(const MatrixType& m) @@ -92,7 +109,104 @@ VERIFY_RAISES_ASSERT(eig.pseudoEigenvectors()); } -void test_eigensolver_generic() + +template<typename CoeffType> +Matrix<typename CoeffType::Scalar,Dynamic,Dynamic> +make_companion(const CoeffType& coeffs) +{ + Index n = coeffs.size()-1; + Matrix<typename CoeffType::Scalar,Dynamic,Dynamic> res(n,n); + res.setZero(); + res.row(0) = -coeffs.tail(n) / coeffs(0); + res.diagonal(-1).setOnes(); + return res; +} + +template<int> +void eigensolver_generic_extra() +{ + { + // regression test for bug 793 + MatrixXd a(3,3); + a << 0, 0, 1, + 1, 1, 1, + 1, 1e+200, 1; + Eigen::EigenSolver<MatrixXd> eig(a); + double scale = 1e-200; // scale to avoid overflow during the comparisons + VERIFY_IS_APPROX(a * eig.pseudoEigenvectors()*scale, eig.pseudoEigenvectors() * eig.pseudoEigenvalueMatrix()*scale); + VERIFY_IS_APPROX(a * eig.eigenvectors()*scale, eig.eigenvectors() * eig.eigenvalues().asDiagonal()*scale); + } + { + // check a case where all eigenvalues are null. + MatrixXd a(2,2); + a << 1, 1, + -1, -1; + Eigen::EigenSolver<MatrixXd> eig(a); + VERIFY_IS_APPROX(eig.pseudoEigenvectors().squaredNorm(), 2.); + VERIFY_IS_APPROX((a * eig.pseudoEigenvectors()).norm()+1., 1.); + VERIFY_IS_APPROX((eig.pseudoEigenvectors() * eig.pseudoEigenvalueMatrix()).norm()+1., 1.); + VERIFY_IS_APPROX((a * eig.eigenvectors()).norm()+1., 1.); + VERIFY_IS_APPROX((eig.eigenvectors() * eig.eigenvalues().asDiagonal()).norm()+1., 1.); + } + + // regression test for bug 933 + { + { + VectorXd coeffs(5); coeffs << 1, -3, -175, -225, 2250; + MatrixXd C = make_companion(coeffs); + EigenSolver<MatrixXd> eig(C); + CALL_SUBTEST( check_eigensolver_for_given_mat(eig,C) ); + } + { + // this test is tricky because it requires high accuracy in smallest eigenvalues + VectorXd coeffs(5); coeffs << 6.154671e-15, -1.003870e-10, -9.819570e-01, 3.995715e+03, 2.211511e+08; + MatrixXd C = make_companion(coeffs); + EigenSolver<MatrixXd> eig(C); + CALL_SUBTEST( check_eigensolver_for_given_mat(eig,C) ); + Index n = C.rows(); + for(Index i=0;i<n;++i) + { + typedef std::complex<double> Complex; + MatrixXcd ac = C.cast<Complex>(); + ac.diagonal().array() -= eig.eigenvalues()(i); + VectorXd sv = ac.jacobiSvd().singularValues(); + // comparing to sv(0) is not enough here to catch the "bug", + // the hard-coded 1.0 is important! + VERIFY_IS_MUCH_SMALLER_THAN(sv(n-1), 1.0); + } + } + } + // regression test for bug 1557 + { + // this test is interesting because it contains zeros on the diagonal. + MatrixXd A_bug1557(3,3); + A_bug1557 << 0, 0, 0, 1, 0, 0.5887907064808635127, 0, 1, 0; + EigenSolver<MatrixXd> eig(A_bug1557); + CALL_SUBTEST( check_eigensolver_for_given_mat(eig,A_bug1557) ); + } + + // regression test for bug 1174 + { + Index n = 12; + MatrixXf A_bug1174(n,n); + A_bug1174 << 262144, 0, 0, 262144, 786432, 0, 0, 0, 0, 0, 0, 786432, + 262144, 0, 0, 262144, 786432, 0, 0, 0, 0, 0, 0, 786432, + 262144, 0, 0, 262144, 786432, 0, 0, 0, 0, 0, 0, 786432, + 262144, 0, 0, 262144, 786432, 0, 0, 0, 0, 0, 0, 786432, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0, + 0, 262144, 262144, 0, 0, 262144, 262144, 262144, 262144, 262144, 262144, 0; + EigenSolver<MatrixXf> eig(A_bug1174); + CALL_SUBTEST( check_eigensolver_for_given_mat(eig,A_bug1174) ); + } +} + +EIGEN_DECLARE_TEST(eigensolver_generic) { int s = 0; for(int i = 0; i < g_repeat; i++) { @@ -127,18 +241,7 @@ } ); - // regression test for bug 793 -#ifdef EIGEN_TEST_PART_2 - { - MatrixXd a(3,3); - a << 0, 0, 1, - 1, 1, 1, - 1, 1e+200, 1; - Eigen::EigenSolver<MatrixXd> eig(a); - VERIFY_IS_APPROX(a * eig.pseudoEigenvectors(), eig.pseudoEigenvectors() * eig.pseudoEigenvalueMatrix()); - VERIFY_IS_APPROX(a * eig.eigenvectors(), eig.eigenvectors() * eig.eigenvalues().asDiagonal()); - } -#endif + CALL_SUBTEST_2( eigensolver_generic_extra<0>() ); TEST_SET_BUT_UNUSED_VARIABLE(s) }
diff --git a/test/eigensolver_selfadjoint.cpp b/test/eigensolver_selfadjoint.cpp index f909761..65b80c3 100644 --- a/test/eigensolver_selfadjoint.cpp +++ b/test/eigensolver_selfadjoint.cpp
@@ -12,18 +12,29 @@ #include "svd_fill.h" #include <limits> #include <Eigen/Eigenvalues> +#include <Eigen/SparseCore> template<typename MatrixType> void selfadjointeigensolver_essential_check(const MatrixType& m) { typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; - RealScalar eival_eps = (std::min)(test_precision<RealScalar>(), NumTraits<Scalar>::dummy_precision()*20000); + RealScalar eival_eps = numext::mini<RealScalar>(test_precision<RealScalar>(), NumTraits<Scalar>::dummy_precision()*20000); SelfAdjointEigenSolver<MatrixType> eiSymm(m); VERIFY_IS_EQUAL(eiSymm.info(), Success); - VERIFY_IS_APPROX(m.template selfadjointView<Lower>() * eiSymm.eigenvectors(), - eiSymm.eigenvectors() * eiSymm.eigenvalues().asDiagonal()); + + RealScalar scaling = m.cwiseAbs().maxCoeff(); + + if(scaling<(std::numeric_limits<RealScalar>::min)()) + { + VERIFY(eiSymm.eigenvalues().cwiseAbs().maxCoeff() <= (std::numeric_limits<RealScalar>::min)()); + } + else + { + VERIFY_IS_APPROX((m.template selfadjointView<Lower>() * eiSymm.eigenvectors())/scaling, + (eiSymm.eigenvectors() * eiSymm.eigenvalues().asDiagonal())/scaling); + } VERIFY_IS_APPROX(m.template selfadjointView<Lower>().eigenvalues(), eiSymm.eigenvalues()); VERIFY_IS_UNITARY(eiSymm.eigenvectors()); @@ -32,7 +43,6 @@ SelfAdjointEigenSolver<MatrixType> eiDirect; eiDirect.computeDirect(m); VERIFY_IS_EQUAL(eiDirect.info(), Success); - VERIFY_IS_APPROX(eiSymm.eigenvalues(), eiDirect.eigenvalues()); if(! eiSymm.eigenvalues().isApprox(eiDirect.eigenvalues(), eival_eps) ) { std::cerr << "reference eigenvalues: " << eiSymm.eigenvalues().transpose() << "\n" @@ -40,17 +50,24 @@ << "diff: " << (eiSymm.eigenvalues()-eiDirect.eigenvalues()).transpose() << "\n" << "error (eps): " << (eiSymm.eigenvalues()-eiDirect.eigenvalues()).norm() / eiSymm.eigenvalues().norm() << " (" << eival_eps << ")\n"; } - VERIFY(eiSymm.eigenvalues().isApprox(eiDirect.eigenvalues(), eival_eps)); - VERIFY_IS_APPROX(m.template selfadjointView<Lower>() * eiDirect.eigenvectors(), - eiDirect.eigenvectors() * eiDirect.eigenvalues().asDiagonal()); - VERIFY_IS_APPROX(m.template selfadjointView<Lower>().eigenvalues(), eiDirect.eigenvalues()); + if(scaling<(std::numeric_limits<RealScalar>::min)()) + { + VERIFY(eiDirect.eigenvalues().cwiseAbs().maxCoeff() <= (std::numeric_limits<RealScalar>::min)()); + } + else + { + VERIFY_IS_APPROX(eiSymm.eigenvalues()/scaling, eiDirect.eigenvalues()/scaling); + VERIFY_IS_APPROX((m.template selfadjointView<Lower>() * eiDirect.eigenvectors())/scaling, + (eiDirect.eigenvectors() * eiDirect.eigenvalues().asDiagonal())/scaling); + VERIFY_IS_APPROX(m.template selfadjointView<Lower>().eigenvalues()/scaling, eiDirect.eigenvalues()/scaling); + } + VERIFY_IS_UNITARY(eiDirect.eigenvectors()); } } template<typename MatrixType> void selfadjointeigensolver(const MatrixType& m) { - typedef typename MatrixType::Index Index; /* this test covers the following files: EigenSolver.h, SelfAdjointEigenSolver.h (and indirectly: Tridiagonalization.h) */ @@ -162,8 +179,18 @@ SelfAdjointEigenSolver<MatrixType> eig(a.adjoint() * a); eig.compute(a.adjoint() * a); } + + // regression test for bug 478 + { + a.setZero(); + SelfAdjointEigenSolver<MatrixType> ei3(a); + VERIFY_IS_EQUAL(ei3.info(), Success); + VERIFY_IS_MUCH_SMALLER_THAN(ei3.eigenvalues().norm(),RealScalar(1)); + VERIFY((ei3.eigenvectors().transpose()*ei3.eigenvectors().transpose()).eval().isIdentity()); + } } +template<int> void bug_854() { Matrix3d m; @@ -173,6 +200,7 @@ selfadjointeigensolver_essential_check(m); } +template<int> void bug_1014() { Matrix3d m; @@ -182,7 +210,27 @@ selfadjointeigensolver_essential_check(m); } -void test_eigensolver_selfadjoint() +template<int> +void bug_1225() +{ + Matrix3d m1, m2; + m1.setRandom(); + m1 = m1*m1.transpose(); + m2 = m1.triangularView<Upper>(); + SelfAdjointEigenSolver<Matrix3d> eig1(m1); + SelfAdjointEigenSolver<Matrix3d> eig2(m2.selfadjointView<Upper>()); + VERIFY_IS_APPROX(eig1.eigenvalues(), eig2.eigenvalues()); +} + +template<int> +void bug_1204() +{ + SparseMatrix<double> A(2,2); + A.setIdentity(); + SelfAdjointEigenSolver<Eigen::SparseMatrix<double> > eig(A); +} + +EIGEN_DECLARE_TEST(eigensolver_selfadjoint) { int s = 0; for(int i = 0; i < g_repeat; i++) { @@ -210,8 +258,10 @@ CALL_SUBTEST_7( selfadjointeigensolver(Matrix<double,2,2>()) ); } - CALL_SUBTEST_13( bug_854() ); - CALL_SUBTEST_13( bug_1014() ); + CALL_SUBTEST_13( bug_854<0>() ); + CALL_SUBTEST_13( bug_1014<0>() ); + CALL_SUBTEST_13( bug_1204<0>() ); + CALL_SUBTEST_13( bug_1225<0>() ); // Test problem size constructors s = internal::random<int>(1,EIGEN_TEST_MAX_SIZE/4);
diff --git a/test/evaluators.cpp b/test/evaluators.cpp index 876dffe..2810cd2 100644 --- a/test/evaluators.cpp +++ b/test/evaluators.cpp
@@ -21,7 +21,7 @@ EIGEN_STRONG_INLINE DstXprType& copy_using_evaluator(const EigenBase<DstXprType> &dst, const SrcXprType &src) { - call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>()); return dst.const_cast_derived(); } @@ -29,7 +29,7 @@ EIGEN_STRONG_INLINE const DstXprType& copy_using_evaluator(const NoAlias<DstXprType, StorageBase>& dst, const SrcXprType &src) { - call_assignment(dst, src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + call_assignment(dst, src.derived(), internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>()); return dst.expression(); } @@ -45,7 +45,7 @@ dst.const_cast_derived().resizeLike(src.derived()); #endif - call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>()); return dst.const_cast_derived(); } @@ -53,28 +53,28 @@ void add_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) { typedef typename DstXprType::Scalar Scalar; - call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::add_assign_op<Scalar>()); + call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::add_assign_op<Scalar,typename SrcXprType::Scalar>()); } template<typename DstXprType, typename SrcXprType> void subtract_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) { typedef typename DstXprType::Scalar Scalar; - call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::sub_assign_op<Scalar>()); + call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::sub_assign_op<Scalar,typename SrcXprType::Scalar>()); } template<typename DstXprType, typename SrcXprType> void multiply_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) { typedef typename DstXprType::Scalar Scalar; - call_assignment(dst.const_cast_derived(), src.derived(), internal::mul_assign_op<Scalar>()); + call_assignment(dst.const_cast_derived(), src.derived(), internal::mul_assign_op<Scalar,typename SrcXprType::Scalar>()); } template<typename DstXprType, typename SrcXprType> void divide_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) { typedef typename DstXprType::Scalar Scalar; - call_assignment(dst.const_cast_derived(), src.derived(), internal::div_assign_op<Scalar>()); + call_assignment(dst.const_cast_derived(), src.derived(), internal::div_assign_op<Scalar,typename SrcXprType::Scalar>()); } template<typename DstXprType, typename SrcXprType> @@ -90,6 +90,12 @@ { call_assignment_no_alias(dst.expression(), src, func); } + + template<typename Dst, template <typename> class StorageBase, typename Src, typename Func> + EIGEN_DEVICE_FUNC void call_restricted_packet_assignment(const NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func) + { + call_restricted_packet_assignment_no_alias(dst.expression(), src, func); + } } } @@ -101,7 +107,7 @@ #define VERIFY_IS_APPROX_EVALUATOR(DEST,EXPR) VERIFY_IS_APPROX(copy_using_evaluator(DEST,(EXPR)), (EXPR).eval()); #define VERIFY_IS_APPROX_EVALUATOR2(DEST,EXPR,REF) VERIFY_IS_APPROX(copy_using_evaluator(DEST,(EXPR)), (REF).eval()); -void test_evaluators() +EIGEN_DECLARE_TEST(evaluators) { // Testing Matrix evaluator and Transpose Vector2d v = Vector2d::Random(); @@ -496,4 +502,24 @@ VERIFY_IS_EQUAL( get_cost(a*(a+b)), 1); VERIFY_IS_EQUAL( get_cost(a.lazyProduct(a+b)), 15); } + + // regression test for PR 544 and bug 1622 (introduced in #71609c4) + { + // test restricted_packet_assignment with an unaligned destination + const size_t M = 2; + const size_t K = 2; + const size_t N = 5; + float *destMem = new float[(M*N) + 1]; + float *dest = (internal::UIntPtr(destMem)%EIGEN_MAX_ALIGN_BYTES) == 0 ? destMem+1 : destMem; + + const Matrix<float, Dynamic, Dynamic, RowMajor> a = Matrix<float, Dynamic, Dynamic, RowMajor>::Random(M, K); + const Matrix<float, Dynamic, Dynamic, RowMajor> b = Matrix<float, Dynamic, Dynamic, RowMajor>::Random(K, N); + + Map<Matrix<float, Dynamic, Dynamic, RowMajor> > z(dest, M, N);; + Product<Matrix<float, Dynamic, Dynamic, RowMajor>, Matrix<float, Dynamic, Dynamic, RowMajor>, LazyProduct> tmp(a,b); + internal::call_restricted_packet_assignment(z.noalias(), tmp.derived(), internal::assign_op<float, float>()); + + VERIFY_IS_APPROX(z, a*b); + delete[] destMem; + } }
diff --git a/test/exceptions.cpp b/test/exceptions.cpp index b83fb82..3d93060 100644 --- a/test/exceptions.cpp +++ b/test/exceptions.cpp
@@ -8,93 +8,34 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// Various sanity tests with exceptions: +// Various sanity tests with exceptions and non trivially copyable scalar type. // - no memory leak when a custom scalar type trow an exceptions // - todo: complete the list of tests! #define EIGEN_STACK_ALLOCATION_LIMIT 100000000 #include "main.h" - -struct my_exception -{ - my_exception() {} - ~my_exception() {} -}; - -class ScalarWithExceptions -{ - public: - ScalarWithExceptions() { init(); } - ScalarWithExceptions(const float& _v) { init(); *v = _v; } - ScalarWithExceptions(const ScalarWithExceptions& other) { init(); *v = *(other.v); } - ~ScalarWithExceptions() { - delete v; - instances--; - } - - void init() { - v = new float; - instances++; - } - - ScalarWithExceptions operator+(const ScalarWithExceptions& other) const - { - countdown--; - if(countdown<=0) - throw my_exception(); - return ScalarWithExceptions(*v+*other.v); - } - - ScalarWithExceptions operator-(const ScalarWithExceptions& other) const - { return ScalarWithExceptions(*v-*other.v); } - - ScalarWithExceptions operator*(const ScalarWithExceptions& other) const - { return ScalarWithExceptions((*v)*(*other.v)); } - - ScalarWithExceptions& operator+=(const ScalarWithExceptions& other) - { *v+=*other.v; return *this; } - ScalarWithExceptions& operator-=(const ScalarWithExceptions& other) - { *v-=*other.v; return *this; } - ScalarWithExceptions& operator=(const ScalarWithExceptions& other) - { *v = *(other.v); return *this; } - - bool operator==(const ScalarWithExceptions& other) const - { return *v==*other.v; } - bool operator!=(const ScalarWithExceptions& other) const - { return *v!=*other.v; } - - float* v; - static int instances; - static int countdown; -}; - -ScalarWithExceptions real(const ScalarWithExceptions &x) { return x; } -ScalarWithExceptions imag(const ScalarWithExceptions & ) { return 0; } -ScalarWithExceptions conj(const ScalarWithExceptions &x) { return x; } - -int ScalarWithExceptions::instances = 0; -int ScalarWithExceptions::countdown = 0; - +#include "AnnoyingScalar.h" #define CHECK_MEMLEAK(OP) { \ - ScalarWithExceptions::countdown = 100; \ - int before = ScalarWithExceptions::instances; \ - bool exception_thrown = false; \ - try { OP; } \ + AnnoyingScalar::countdown = 100; \ + int before = AnnoyingScalar::instances; \ + bool exception_thrown = false; \ + try { OP; } \ catch (my_exception) { \ exception_thrown = true; \ - VERIFY(ScalarWithExceptions::instances==before && "memory leak detected in " && EIGEN_MAKESTRING(OP)); \ + VERIFY(AnnoyingScalar::instances==before && "memory leak detected in " && EIGEN_MAKESTRING(OP)); \ } \ - VERIFY(exception_thrown && " no exception thrown in " && EIGEN_MAKESTRING(OP)); \ + VERIFY( (AnnoyingScalar::dont_throw) || (exception_thrown && " no exception thrown in " && EIGEN_MAKESTRING(OP)) ); \ } -void memoryleak() +EIGEN_DECLARE_TEST(exceptions) { - typedef Eigen::Matrix<ScalarWithExceptions,Dynamic,1> VectorType; - typedef Eigen::Matrix<ScalarWithExceptions,Dynamic,Dynamic> MatrixType; + typedef Eigen::Matrix<AnnoyingScalar,Dynamic,1> VectorType; + typedef Eigen::Matrix<AnnoyingScalar,Dynamic,Dynamic> MatrixType; { + AnnoyingScalar::dont_throw = false; int n = 50; VectorType v0(n), v1(n); MatrixType m0(n,n), m1(n,n), m2(n,n); @@ -104,10 +45,5 @@ CHECK_MEMLEAK(m2 = m0 * m1 * m2); CHECK_MEMLEAK((v0+v1).dot(v0+v1)); } - VERIFY(ScalarWithExceptions::instances==0 && "global memory leak detected in " && EIGEN_MAKESTRING(OP)); \ -} - -void test_exceptions() -{ - CALL_SUBTEST( memoryleak() ); + VERIFY(AnnoyingScalar::instances==0 && "global memory leak detected in " && EIGEN_MAKESTRING(OP)); }
diff --git a/test/fastmath.cpp b/test/fastmath.cpp index efdd5b3..00a1a59 100644 --- a/test/fastmath.cpp +++ b/test/fastmath.cpp
@@ -43,13 +43,14 @@ } else { - VERIFY( !(numext::isfinite)(m(3)) ); - VERIFY( !(numext::isinf)(m(3)) ); - VERIFY( (numext::isnan)(m(3)) ); - VERIFY( !m.allFinite() ); - VERIFY( m.hasNaN() ); + if( (std::isfinite)(m(3))) g_test_level=1; VERIFY( !(numext::isfinite)(m(3)) ); g_test_level=0; + if( (std::isinf) (m(3))) g_test_level=1; VERIFY( !(numext::isinf)(m(3)) ); g_test_level=0; + if(!(std::isnan) (m(3))) g_test_level=1; VERIFY( (numext::isnan)(m(3)) ); g_test_level=0; + if( (std::isfinite)(m(3))) g_test_level=1; VERIFY( !m.allFinite() ); g_test_level=0; + if(!(std::isnan) (m(3))) g_test_level=1; VERIFY( m.hasNaN() ); g_test_level=0; } - m(4) /= 0.0; + T hidden_zero = (std::numeric_limits<T>::min)()*(std::numeric_limits<T>::min)(); + m(4) /= hidden_zero; if(dryrun) { std::cout << "std::isfinite(" << m(4) << ") = "; check((std::isfinite)(m(4)),false); std::cout << " ; numext::isfinite = "; check((numext::isfinite)(m(4)), false); std::cout << "\n"; @@ -61,33 +62,33 @@ } else { - VERIFY( !(numext::isfinite)(m(4)) ); - VERIFY( (numext::isinf)(m(4)) ); - VERIFY( !(numext::isnan)(m(4)) ); - VERIFY( !m.allFinite() ); - VERIFY( m.hasNaN() ); + if( (std::isfinite)(m(3))) g_test_level=1; VERIFY( !(numext::isfinite)(m(4)) ); g_test_level=0; + if(!(std::isinf) (m(3))) g_test_level=1; VERIFY( (numext::isinf)(m(4)) ); g_test_level=0; + if( (std::isnan) (m(3))) g_test_level=1; VERIFY( !(numext::isnan)(m(4)) ); g_test_level=0; + if( (std::isfinite)(m(3))) g_test_level=1; VERIFY( !m.allFinite() ); g_test_level=0; + if(!(std::isnan) (m(3))) g_test_level=1; VERIFY( m.hasNaN() ); g_test_level=0; } m(3) = 0; if(dryrun) { std::cout << "std::isfinite(" << m(3) << ") = "; check((std::isfinite)(m(3)),true); std::cout << " ; numext::isfinite = "; check((numext::isfinite)(m(3)), true); std::cout << "\n"; - std::cout << "std::isinf(" << m(3) << ") = "; check((std::isinf)(m(3)),false); std::cout << " ; numext::isinf = "; check((numext::isinf)(m(3)), false); std::cout << "\n"; - std::cout << "std::isnan(" << m(3) << ") = "; check((std::isnan)(m(3)),false); std::cout << " ; numext::isnan = "; check((numext::isnan)(m(3)), false); std::cout << "\n"; + std::cout << "std::isinf(" << m(3) << ") = "; check((std::isinf)(m(3)),false); std::cout << " ; numext::isinf = "; check((numext::isinf)(m(3)), false); std::cout << "\n"; + std::cout << "std::isnan(" << m(3) << ") = "; check((std::isnan)(m(3)),false); std::cout << " ; numext::isnan = "; check((numext::isnan)(m(3)), false); std::cout << "\n"; std::cout << "allFinite: "; check(m.allFinite(), 0); std::cout << "\n"; std::cout << "hasNaN: "; check(m.hasNaN(), 0); std::cout << "\n"; std::cout << "\n\n"; } else { - VERIFY( (numext::isfinite)(m(3)) ); - VERIFY( !(numext::isinf)(m(3)) ); - VERIFY( !(numext::isnan)(m(3)) ); - VERIFY( !m.allFinite() ); - VERIFY( !m.hasNaN() ); + if(!(std::isfinite)(m(3))) g_test_level=1; VERIFY( (numext::isfinite)(m(3)) ); g_test_level=0; + if( (std::isinf) (m(3))) g_test_level=1; VERIFY( !(numext::isinf)(m(3)) ); g_test_level=0; + if( (std::isnan) (m(3))) g_test_level=1; VERIFY( !(numext::isnan)(m(3)) ); g_test_level=0; + if( (std::isfinite)(m(3))) g_test_level=1; VERIFY( !m.allFinite() ); g_test_level=0; + if( (std::isnan) (m(3))) g_test_level=1; VERIFY( !m.hasNaN() ); g_test_level=0; } } -void test_fastmath() { +EIGEN_DECLARE_TEST(fastmath) { std::cout << "*** float *** \n\n"; check_inf_nan<float>(true); std::cout << "*** double ***\n\n"; check_inf_nan<double>(true); std::cout << "*** long double *** \n\n"; check_inf_nan<long double>(true);
diff --git a/test/first_aligned.cpp b/test/first_aligned.cpp index bf22f6b..ed99450 100644 --- a/test/first_aligned.cpp +++ b/test/first_aligned.cpp
@@ -26,7 +26,7 @@ struct some_non_vectorizable_type { float x; }; -void test_first_aligned() +EIGEN_DECLARE_TEST(first_aligned) { EIGEN_ALIGN16 float array_float[100]; test_first_aligned_helper(array_float, 50); @@ -41,7 +41,7 @@ test_first_aligned_helper(array_double+1, 50); test_first_aligned_helper(array_double+2, 50); - double *array_double_plus_4_bytes = (double*)(size_t(array_double)+4); + double *array_double_plus_4_bytes = (double*)(internal::UIntPtr(array_double)+4); test_none_aligned_helper(array_double_plus_4_bytes, 50); test_none_aligned_helper(array_double_plus_4_bytes+1, 50);
diff --git a/test/geo_alignedbox.cpp b/test/geo_alignedbox.cpp index 2bdb4b7..c6c051c 100644 --- a/test/geo_alignedbox.cpp +++ b/test/geo_alignedbox.cpp
@@ -15,8 +15,17 @@ #include<iostream> using namespace std; +// NOTE the following workaround was needed on some 32 bits builds to kill extra precision of x87 registers. +// It seems that it os not needed anymore, but let's keep it here, just in case... + template<typename T> EIGEN_DONT_INLINE -void kill_extra_precision(T& x) { eigen_assert((void*)(&x) != (void*)0); } +void kill_extra_precision(T& /* x */) { + // This one worked but triggered a warning: + /* eigen_assert((void*)(&x) != (void*)0); */ + // An alternative could be: + /* volatile T tmp = x; */ + /* x = tmp; */ +} template<typename BoxType> void alignedbox(const BoxType& _box) @@ -24,7 +33,6 @@ /* this test covers the following files: AlignedBox.h */ - typedef typename BoxType::Index Index; typedef typename BoxType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, BoxType::AmbientDimAtCompileTime, 1> VectorType; @@ -48,6 +56,8 @@ b0.extend(p0); b0.extend(p1); VERIFY(b0.contains(p0*s1+(Scalar(1)-s1)*p1)); + VERIFY(b0.contains(b0.center())); + VERIFY_IS_APPROX(b0.center(),(p0+p1)/Scalar(2)); (b2 = b0).extend(b1); VERIFY(b2.contains(b0)); @@ -84,7 +94,6 @@ void alignedboxCastTests(const BoxType& _box) { // casting - typedef typename BoxType::Index Index; typedef typename BoxType::Scalar Scalar; typedef Matrix<Scalar, BoxType::AmbientDimAtCompileTime, 1> VectorType; @@ -160,7 +169,7 @@ } -void test_geo_alignedbox() +EIGEN_DECLARE_TEST(geo_alignedbox) { for(int i = 0; i < g_repeat; i++) {
diff --git a/test/geo_eulerangles.cpp b/test/geo_eulerangles.cpp index 932ebe7..693c627 100644 --- a/test/geo_eulerangles.cpp +++ b/test/geo_eulerangles.cpp
@@ -103,7 +103,7 @@ check_all_var(ea); } -void test_geo_eulerangles() +EIGEN_DECLARE_TEST(geo_eulerangles) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( eulerangles<float>() );
diff --git a/test/geo_homogeneous.cpp b/test/geo_homogeneous.cpp index bf63c69..9aebe62 100644 --- a/test/geo_homogeneous.cpp +++ b/test/geo_homogeneous.cpp
@@ -58,6 +58,8 @@ T2MatrixType t2 = T2MatrixType::Random(); VERIFY_IS_APPROX(t2 * (v0.homogeneous().eval()), t2 * v0.homogeneous()); VERIFY_IS_APPROX(t2 * (m0.colwise().homogeneous().eval()), t2 * m0.colwise().homogeneous()); + VERIFY_IS_APPROX(t2 * (v0.homogeneous().asDiagonal()), t2 * hv0.asDiagonal()); + VERIFY_IS_APPROX((v0.homogeneous().asDiagonal()) * t2, hv0.asDiagonal() * t2); VERIFY_IS_APPROX((v0.transpose().rowwise().homogeneous().eval()) * t2, v0.transpose().rowwise().homogeneous() * t2); @@ -109,9 +111,11 @@ VERIFY_IS_APPROX( (v0.transpose().homogeneous() .lazyProduct( t2 )).hnormalized(), (v0.transpose().homogeneous()*t2).hnormalized() ); VERIFY_IS_APPROX( (pts.transpose().rowwise().homogeneous() .lazyProduct( t2 )).rowwise().hnormalized(), (pts1.transpose()*t2).rowwise().hnormalized() ); + + VERIFY_IS_APPROX( (t2.template triangularView<Lower>() * v0.homogeneous()).eval(), (t2.template triangularView<Lower>()*hv0) ); } -void test_geo_homogeneous() +EIGEN_DECLARE_TEST(geo_homogeneous) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(( homogeneous<float,1>() ));
diff --git a/test/geo_hyperplane.cpp b/test/geo_hyperplane.cpp index c1cc691..a267093 100644 --- a/test/geo_hyperplane.cpp +++ b/test/geo_hyperplane.cpp
@@ -19,7 +19,6 @@ Hyperplane.h */ using std::abs; - typedef typename HyperplaneType::Index Index; const Index dim = _plane.dim(); enum { Options = HyperplaneType::Options }; typedef typename HyperplaneType::Scalar Scalar; @@ -66,12 +65,15 @@ VERIFY_IS_MUCH_SMALLER_THAN( pl2.transform(rot,Isometry).absDistance(rot * p1), Scalar(1) ); pl2 = pl1; VERIFY_IS_MUCH_SMALLER_THAN( pl2.transform(rot*scaling).absDistance((rot*scaling) * p1), Scalar(1) ); + VERIFY_IS_APPROX( pl2.normal().norm(), RealScalar(1) ); pl2 = pl1; VERIFY_IS_MUCH_SMALLER_THAN( pl2.transform(rot*scaling*translation) .absDistance((rot*scaling*translation) * p1), Scalar(1) ); + VERIFY_IS_APPROX( pl2.normal().norm(), RealScalar(1) ); pl2 = pl1; VERIFY_IS_MUCH_SMALLER_THAN( pl2.transform(rot*translation,Isometry) .absDistance((rot*translation) * p1), Scalar(1) ); + VERIFY_IS_APPROX( pl2.normal().norm(), RealScalar(1) ); } // casting @@ -97,9 +99,9 @@ Vector u = Vector::Random(); Vector v = Vector::Random(); Scalar a = internal::random<Scalar>(); - while (abs(a-1) < 1e-4) a = internal::random<Scalar>(); - while (u.norm() < 1e-4) u = Vector::Random(); - while (v.norm() < 1e-4) v = Vector::Random(); + while (abs(a-1) < Scalar(1e-4)) a = internal::random<Scalar>(); + while (u.norm() < Scalar(1e-4)) u = Vector::Random(); + while (v.norm() < Scalar(1e-4)) v = Vector::Random(); HLine line_u = HLine::Through(center + u, center + a*u); HLine line_v = HLine::Through(center + v, center + a*v); @@ -111,14 +113,14 @@ Vector result = line_u.intersection(line_v); // the lines should intersect at the point we called "center" - if(abs(a-1) > 1e-2 && abs(v.normalized().dot(u.normalized()))<0.9) + if(abs(a-1) > Scalar(1e-2) && abs(v.normalized().dot(u.normalized()))<Scalar(0.9)) VERIFY_IS_APPROX(result, center); // check conversions between two types of lines PLine pl(line_u); // gcc 3.3 will commit suicide if we don't name this variable HLine line_u2(pl); CoeffsType converted_coeffs = line_u2.coeffs(); - if(line_u2.normal().dot(line_u.normal())<0.) + if(line_u2.normal().dot(line_u.normal())<Scalar(0)) converted_coeffs = -line_u2.coeffs(); VERIFY(line_u.coeffs().isApprox(converted_coeffs)); } @@ -178,7 +180,7 @@ } -void test_geo_hyperplane() +EIGEN_DECLARE_TEST(geo_hyperplane) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( hyperplane(Hyperplane<float,2>()) );
diff --git a/test/geo_orthomethods.cpp b/test/geo_orthomethods.cpp index e178df2..b7b6607 100644 --- a/test/geo_orthomethods.cpp +++ b/test/geo_orthomethods.cpp
@@ -115,7 +115,7 @@ VERIFY_IS_APPROX(mcrossN3.row(i), matN3.row(i).cross(vec3)); } -void test_geo_orthomethods() +EIGEN_DECLARE_TEST(geo_orthomethods) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( orthomethods_3<float>() );
diff --git a/test/geo_parametrizedline.cpp b/test/geo_parametrizedline.cpp index 9bf5f3c..7135c8f 100644 --- a/test/geo_parametrizedline.cpp +++ b/test/geo_parametrizedline.cpp
@@ -19,12 +19,13 @@ ParametrizedLine.h */ using std::abs; - typedef typename LineType::Index Index; const Index dim = _line.dim(); typedef typename LineType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, LineType::AmbientDimAtCompileTime, 1> VectorType; typedef Hyperplane<Scalar,LineType::AmbientDimAtCompileTime> HyperplaneType; + typedef Matrix<Scalar, HyperplaneType::AmbientDimAtCompileTime, + HyperplaneType::AmbientDimAtCompileTime> MatrixType; VectorType p0 = VectorType::Random(dim); VectorType p1 = VectorType::Random(dim); @@ -59,6 +60,31 @@ VERIFY_IS_MUCH_SMALLER_THAN(hp.signedDistance(pi), RealScalar(1)); VERIFY_IS_MUCH_SMALLER_THAN(l0.distance(pi), RealScalar(1)); VERIFY_IS_APPROX(l0.intersectionPoint(hp), pi); + + // transform + if (!NumTraits<Scalar>::IsComplex) + { + MatrixType rot = MatrixType::Random(dim,dim).householderQr().householderQ(); + DiagonalMatrix<Scalar,LineType::AmbientDimAtCompileTime> scaling(VectorType::Random()); + Translation<Scalar,LineType::AmbientDimAtCompileTime> translation(VectorType::Random()); + + while(scaling.diagonal().cwiseAbs().minCoeff()<RealScalar(1e-4)) scaling.diagonal() = VectorType::Random(); + + LineType l1 = l0; + VectorType p3 = l0.pointAt(Scalar(1)); + VERIFY_IS_MUCH_SMALLER_THAN( l1.transform(rot).distance(rot * p3), Scalar(1) ); + l1 = l0; + VERIFY_IS_MUCH_SMALLER_THAN( l1.transform(rot,Isometry).distance(rot * p3), Scalar(1) ); + l1 = l0; + VERIFY_IS_MUCH_SMALLER_THAN( l1.transform(rot*scaling).distance((rot*scaling) * p3), Scalar(1) ); + l1 = l0; + VERIFY_IS_MUCH_SMALLER_THAN( l1.transform(rot*scaling*translation) + .distance((rot*scaling*translation) * p3), Scalar(1) ); + l1 = l0; + VERIFY_IS_MUCH_SMALLER_THAN( l1.transform(rot*translation,Isometry) + .distance((rot*translation) * p3), Scalar(1) ); + } + } template<typename Scalar> void parametrizedline_alignment() @@ -91,7 +117,7 @@ #endif } -void test_geo_parametrizedline() +EIGEN_DECLARE_TEST(geo_parametrizedline) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( parametrizedline(ParametrizedLine<float,2>()) );
diff --git a/test/geo_quaternion.cpp b/test/geo_quaternion.cpp index 761bb52..27219db 100644 --- a/test/geo_quaternion.cpp +++ b/test/geo_quaternion.cpp
@@ -12,6 +12,7 @@ #include <Eigen/Geometry> #include <Eigen/LU> #include <Eigen/SVD> +#include "AnnoyingScalar.h" template<typename T> T bounded_acos(T v) { @@ -30,8 +31,8 @@ Scalar largeEps = test_precision<Scalar>(); Scalar theta_tot = AA(q1*q0.inverse()).angle(); - if(theta_tot>EIGEN_PI) - theta_tot = Scalar(2.*EIGEN_PI)-theta_tot; + if(theta_tot>Scalar(EIGEN_PI)) + theta_tot = Scalar(2.)*Scalar(EIGEN_PI)-theta_tot; for(Scalar t=0; t<=Scalar(1.001); t+=Scalar(0.1)) { QuatType q = q0.slerp(t,q1); @@ -50,13 +51,12 @@ using std::abs; typedef Matrix<Scalar,3,1> Vector3; typedef Matrix<Scalar,3,3> Matrix3; - typedef Matrix<Scalar,4,1> Vector4; typedef Quaternion<Scalar,Options> Quaternionx; typedef AngleAxis<Scalar> AngleAxisx; Scalar largeEps = test_precision<Scalar>(); if (internal::is_same<Scalar,float>::value) - largeEps = 1e-3f; + largeEps = Scalar(1e-3); Scalar eps = internal::random<Scalar>() * Scalar(1e-2); @@ -86,7 +86,7 @@ if (refangle>Scalar(EIGEN_PI)) refangle = Scalar(2)*Scalar(EIGEN_PI) - refangle; - if((q1.coeffs()-q2.coeffs()).norm() > 10*largeEps) + if((q1.coeffs()-q2.coeffs()).norm() > Scalar(10)*largeEps) { VERIFY_IS_MUCH_SMALLER_THAN(abs(q1.angularDistance(q2) - refangle), Scalar(1)); } @@ -114,9 +114,9 @@ // Do not execute the test if the rotation angle is almost zero, or // the rotation axis and v1 are almost parallel. - if (abs(aa.angle()) > 5*test_precision<Scalar>() - && (aa.axis() - v1.normalized()).norm() < 1.99 - && (aa.axis() + v1.normalized()).norm() < 1.99) + if (abs(aa.angle()) > Scalar(5)*test_precision<Scalar>() + && (aa.axis() - v1.normalized()).norm() < Scalar(1.99) + && (aa.axis() + v1.normalized()).norm() < Scalar(1.99)) { VERIFY_IS_NOT_APPROX(q1 * v1, Quaternionx(AngleAxisx(aa.angle()*2,aa.axis())) * v1); } @@ -157,8 +157,8 @@ Quaternionx *q = new Quaternionx; delete q; - q1 = AngleAxisx(a, v0.normalized()); - q2 = AngleAxisx(b, v1.normalized()); + q1 = Quaternionx::UnitRandom(); + q2 = Quaternionx::UnitRandom(); check_slerp(q1,q2); q1 = AngleAxisx(b, v1.normalized()); @@ -169,7 +169,7 @@ q2 = AngleAxisx(-b, -v1.normalized()); check_slerp(q1,q2); - q1.coeffs() = Vector4::Random().normalized(); + q1 = Quaternionx::UnitRandom(); q2.coeffs() = -q1.coeffs(); check_slerp(q1,q2); } @@ -232,6 +232,19 @@ VERIFY_IS_APPROX(mq3*mq2, q3*q2); VERIFY_IS_APPROX(mcq1*mq2, q1*q2); VERIFY_IS_APPROX(mcq3*mq2, q3*q2); + + // Bug 1461, compilation issue with Map<const Quat>::w(), and other reference/constness checks: + VERIFY_IS_APPROX(mcq3.coeffs().x() + mcq3.coeffs().y() + mcq3.coeffs().z() + mcq3.coeffs().w(), mcq3.coeffs().sum()); + VERIFY_IS_APPROX(mcq3.x() + mcq3.y() + mcq3.z() + mcq3.w(), mcq3.coeffs().sum()); + mq3.w() = 1; + const Quaternionx& cq3(q3); + VERIFY( &cq3.x() == &q3.x() ); + const MQuaternionUA& cmq3(mq3); + VERIFY( &cmq3.x() == &mq3.x() ); + // FIXME the following should be ok. The problem is that currently the LValueBit flag + // is used to determine whether we can return a coeff by reference or not, which is not enough for Map<const ...>. + //const MCQuaternionUA& cmcq3(mcq3); + //VERIFY( &cmcq3.x() == &mcq3.x() ); } template<typename Scalar> void quaternionAlignment(void){ @@ -273,18 +286,38 @@ VERIFY( !(Map<ConstPlainObjectType, Aligned>::Flags & LvalueBit) ); } -void test_geo_quaternion() +#if EIGEN_HAS_RVALUE_REFERENCES + +// Regression for bug 1573 +struct MovableClass { + // The following line is a workaround for gcc 4.7 and 4.8 (see bug 1573 comments). + static_assert(std::is_nothrow_move_constructible<Quaternionf>::value,""); + MovableClass() = default; + MovableClass(const MovableClass&) = default; + MovableClass(MovableClass&&) noexcept = default; + MovableClass& operator=(const MovableClass&) = default; + MovableClass& operator=(MovableClass&&) = default; + Quaternionf m_quat; +}; + +#endif + +EIGEN_DECLARE_TEST(geo_quaternion) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(( quaternion<float,AutoAlign>() )); CALL_SUBTEST_1( check_const_correctness(Quaternionf()) ); + CALL_SUBTEST_1(( quaternion<float,DontAlign>() )); + CALL_SUBTEST_1(( quaternionAlignment<float>() )); + CALL_SUBTEST_1( mapQuaternion<float>() ); + CALL_SUBTEST_2(( quaternion<double,AutoAlign>() )); CALL_SUBTEST_2( check_const_correctness(Quaterniond()) ); - CALL_SUBTEST_3(( quaternion<float,DontAlign>() )); - CALL_SUBTEST_4(( quaternion<double,DontAlign>() )); - CALL_SUBTEST_5(( quaternionAlignment<float>() )); - CALL_SUBTEST_6(( quaternionAlignment<double>() )); - CALL_SUBTEST_1( mapQuaternion<float>() ); + CALL_SUBTEST_2(( quaternion<double,DontAlign>() )); + CALL_SUBTEST_2(( quaternionAlignment<double>() )); CALL_SUBTEST_2( mapQuaternion<double>() ); + + AnnoyingScalar::dont_throw = true; + CALL_SUBTEST_3(( quaternion<AnnoyingScalar,AutoAlign>() )); } }
diff --git a/test/geo_transformations.cpp b/test/geo_transformations.cpp old mode 100644 new mode 100755 index 51f9003..c722679 --- a/test/geo_transformations.cpp +++ b/test/geo_transformations.cpp
@@ -18,6 +18,11 @@ return Matrix<T,2,1>(std::cos(a), std::sin(a)); } +// This permits to workaround a bug in clang/llvm code generation. +template<typename T> +EIGEN_DONT_INLINE +void dont_over_optimize(T& x) { volatile typename T::Scalar tmp = x(0); x(0) = tmp; } + template<typename Scalar, int Mode, int Options> void non_projective_only() { /* this test covers the following files: @@ -224,12 +229,13 @@ do { v3 = Vector3::Random(); + dont_over_optimize(v3); } while (v3.cwiseAbs().minCoeff()<NumTraits<Scalar>::epsilon()); Translation3 tv3(v3); Transform3 t5(tv3); t4 = tv3; VERIFY_IS_APPROX(t5.matrix(), t4.matrix()); - t4.translate(-v3); + t4.translate((-v3).eval()); VERIFY_IS_APPROX(t4.matrix(), MatrixType::Identity()); t4 *= tv3; VERIFY_IS_APPROX(t5.matrix(), t4.matrix()); @@ -328,6 +334,9 @@ t0.scale(v0); t1 *= AlignedScaling3(v0); VERIFY_IS_APPROX(t0.matrix(), t1.matrix()); + t1 = AlignedScaling3(v0) * (Translation3(v0) * Transform3(q1)); + t1 = t1 * v0.asDiagonal(); + VERIFY_IS_APPROX(t0.matrix(), t1.matrix()); // transformation * translation t0.translate(v0); t1 = t1 * Translation3(v0); @@ -466,7 +475,7 @@ Scalar a2 = R0.slerp(Scalar(k+1)/Scalar(path_steps), R1).angle(); l += std::abs(a2-a1); } - VERIFY(l<=EIGEN_PI*(Scalar(1)+NumTraits<Scalar>::epsilon()*Scalar(path_steps/2))); + VERIFY(l<=Scalar(EIGEN_PI)*(Scalar(1)+NumTraits<Scalar>::epsilon()*Scalar(path_steps/2))); // check basic features { @@ -476,6 +485,79 @@ Rotation2D<Scalar> r2(r1); // copy ctor VERIFY_IS_APPROX(r2.angle(),s0); } + + { + Transform3 t32(Matrix4::Random()), t33, t34; + t34 = t33 = t32; + t32.scale(v0); + t33*=AlignedScaling3(v0); + VERIFY_IS_APPROX(t32.matrix(), t33.matrix()); + t33 = t34 * AlignedScaling3(v0); + VERIFY_IS_APPROX(t32.matrix(), t33.matrix()); + } + +} + +template<typename A1, typename A2, typename P, typename Q, typename V, typename H> +void transform_associativity_left(const A1& a1, const A2& a2, const P& p, const Q& q, const V& v, const H& h) +{ + VERIFY_IS_APPROX( q*(a1*v), (q*a1)*v ); + VERIFY_IS_APPROX( q*(a2*v), (q*a2)*v ); + VERIFY_IS_APPROX( q*(p*h).hnormalized(), ((q*p)*h).hnormalized() ); +} + +template<typename A1, typename A2, typename P, typename Q, typename V, typename H> +void transform_associativity2(const A1& a1, const A2& a2, const P& p, const Q& q, const V& v, const H& h) +{ + VERIFY_IS_APPROX( a1*(q*v), (a1*q)*v ); + VERIFY_IS_APPROX( a2*(q*v), (a2*q)*v ); + VERIFY_IS_APPROX( p *(q*v).homogeneous(), (p *q)*v.homogeneous() ); + + transform_associativity_left(a1, a2,p, q, v, h); +} + +template<typename Scalar, int Dim, int Options,typename RotationType> +void transform_associativity(const RotationType& R) +{ + typedef Matrix<Scalar,Dim,1> VectorType; + typedef Matrix<Scalar,Dim+1,1> HVectorType; + typedef Matrix<Scalar,Dim,Dim> LinearType; + typedef Matrix<Scalar,Dim+1,Dim+1> MatrixType; + typedef Transform<Scalar,Dim,AffineCompact,Options> AffineCompactType; + typedef Transform<Scalar,Dim,Affine,Options> AffineType; + typedef Transform<Scalar,Dim,Projective,Options> ProjectiveType; + typedef DiagonalMatrix<Scalar,Dim> ScalingType; + typedef Translation<Scalar,Dim> TranslationType; + + AffineCompactType A1c; A1c.matrix().setRandom(); + AffineCompactType A2c; A2c.matrix().setRandom(); + AffineType A1(A1c); + AffineType A2(A2c); + ProjectiveType P1; P1.matrix().setRandom(); + VectorType v1 = VectorType::Random(); + VectorType v2 = VectorType::Random(); + HVectorType h1 = HVectorType::Random(); + Scalar s1 = internal::random<Scalar>(); + LinearType L = LinearType::Random(); + MatrixType M = MatrixType::Random(); + + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, A2, v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, A2c, v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, v1.asDiagonal(), v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, ScalingType(v1), v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, Scaling(v1), v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, Scaling(s1), v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, TranslationType(v1), v2, h1) ); + CALL_SUBTEST( transform_associativity_left(A1c, A1, P1, L, v2, h1) ); + CALL_SUBTEST( transform_associativity2(A1c, A1, P1, R, v2, h1) ); + + VERIFY_IS_APPROX( A1*(M*h1), (A1*M)*h1 ); + VERIFY_IS_APPROX( A1c*(M*h1), (A1c*M)*h1 ); + VERIFY_IS_APPROX( P1*(M*h1), (P1*M)*h1 ); + + VERIFY_IS_APPROX( M*(A1*h1), (M*A1)*h1 ); + VERIFY_IS_APPROX( M*(A1c*h1), (M*A1c)*h1 ); + VERIFY_IS_APPROX( M*(P1*h1), ((M*P1)*h1) ); } template<typename Scalar> void transform_alignment() @@ -530,7 +612,67 @@ VERIFY_IS_APPROX((ac*p).matrix(), a_m*p_m); } -void test_geo_transformations() +template<typename Scalar, int Mode, int Options> void transformations_no_scale() +{ + /* this test covers the following files: + Cross.h Quaternion.h, Transform.h + */ + typedef Matrix<Scalar,3,1> Vector3; + typedef Matrix<Scalar,4,1> Vector4; + typedef Quaternion<Scalar> Quaternionx; + typedef AngleAxis<Scalar> AngleAxisx; + typedef Transform<Scalar,3,Mode,Options> Transform3; + typedef Translation<Scalar,3> Translation3; + typedef Matrix<Scalar,4,4> Matrix4; + + Vector3 v0 = Vector3::Random(), + v1 = Vector3::Random(); + + Transform3 t0, t1, t2; + + Scalar a = internal::random<Scalar>(-Scalar(EIGEN_PI), Scalar(EIGEN_PI)); + + Quaternionx q1, q2; + + q1 = AngleAxisx(a, v0.normalized()); + + t0 = Transform3::Identity(); + VERIFY_IS_APPROX(t0.matrix(), Transform3::MatrixType::Identity()); + + t0.setIdentity(); + t1.setIdentity(); + v1 = Vector3::Ones(); + t0.linear() = q1.toRotationMatrix(); + t0.pretranslate(v0); + t1.linear() = q1.conjugate().toRotationMatrix(); + t1.translate(-v0); + + VERIFY((t0 * t1).matrix().isIdentity(test_precision<Scalar>())); + + t1.fromPositionOrientationScale(v0, q1, v1); + VERIFY_IS_APPROX(t1.matrix(), t0.matrix()); + VERIFY_IS_APPROX(t1*v1, t0*v1); + + // translation * vector + t0.setIdentity(); + t0.translate(v0); + VERIFY_IS_APPROX((t0 * v1).template head<3>(), Translation3(v0) * v1); + + // Conversion to matrix. + Transform3 t3; + t3.linear() = q1.toRotationMatrix(); + t3.translation() = v1; + Matrix4 m3 = t3.matrix(); + VERIFY((m3 * m3.inverse()).isIdentity(test_precision<Scalar>())); + // Verify implicit last row is initialized. + VERIFY_IS_APPROX(Vector4(m3.row(3)), Vector4(0.0, 0.0, 0.0, 1.0)); + + VERIFY_IS_APPROX(t3.rotation(), t3.linear()); + if(Mode==Isometry) + VERIFY(t3.rotation().data()==t3.linear().data()); +} + +EIGEN_DECLARE_TEST(geo_transformations) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(( transformations<double,Affine,AutoAlign>() )); @@ -543,7 +685,7 @@ CALL_SUBTEST_3(( transformations<double,Projective,AutoAlign>() )); CALL_SUBTEST_3(( transformations<double,Projective,DontAlign>() )); CALL_SUBTEST_3(( transform_alignment<double>() )); - + CALL_SUBTEST_4(( transformations<float,Affine,RowMajor|AutoAlign>() )); CALL_SUBTEST_4(( non_projective_only<float,Affine,RowMajor>() )); @@ -556,5 +698,11 @@ CALL_SUBTEST_7(( transform_products<double,3,RowMajor|AutoAlign>() )); CALL_SUBTEST_7(( transform_products<float,2,AutoAlign>() )); + + CALL_SUBTEST_8(( transform_associativity<double,2,ColMajor>(Rotation2D<double>(internal::random<double>()*double(EIGEN_PI))) )); + CALL_SUBTEST_8(( transform_associativity<double,3,ColMajor>(Quaterniond::UnitRandom()) )); + + CALL_SUBTEST_9(( transformations_no_scale<double,Affine,AutoAlign>() )); + CALL_SUBTEST_9(( transformations_no_scale<double,Isometry,AutoAlign>() )); } }
diff --git a/test/gpu_basic.cu b/test/gpu_basic.cu new file mode 100644 index 0000000..e8069f1 --- /dev/null +++ b/test/gpu_basic.cu
@@ -0,0 +1,214 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015-2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// workaround issue between gcc >= 4.7 and cuda 5.5 +#if (defined __GNUC__) && (__GNUC__>4 || __GNUC_MINOR__>=7) + #undef _GLIBCXX_ATOMIC_BUILTINS + #undef _GLIBCXX_USE_INT128 +#endif + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int + +#include "main.h" +#include "gpu_common.h" + +// Check that dense modules can be properly parsed by nvcc +#include <Eigen/Dense> + +// struct Foo{ +// EIGEN_DEVICE_FUNC +// void operator()(int i, const float* mats, float* vecs) const { +// using namespace Eigen; +// // Matrix3f M(data); +// // Vector3f x(data+9); +// // Map<Vector3f>(data+9) = M.inverse() * x; +// Matrix3f M(mats+i/16); +// Vector3f x(vecs+i*3); +// // using std::min; +// // using std::sqrt; +// Map<Vector3f>(vecs+i*3) << x.minCoeff(), 1, 2;// / x.dot(x);//(M.inverse() * x) / x.x(); +// //x = x*2 + x.y() * x + x * x.maxCoeff() - x / x.sum(); +// } +// }; + +template<typename T> +struct coeff_wise { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + T x1(in+i); + T x2(in+i+1); + T x3(in+i+2); + Map<T> res(out+i*T::MaxSizeAtCompileTime); + + res.array() += (in[0] * x1 + x2).array() * x3.array(); + } +}; + +template<typename T> +struct replicate { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + T x1(in+i); + int step = x1.size() * 4; + int stride = 3 * step; + + typedef Map<Array<typename T::Scalar,Dynamic,Dynamic> > MapType; + MapType(out+i*stride+0*step, x1.rows()*2, x1.cols()*2) = x1.replicate(2,2); + MapType(out+i*stride+1*step, x1.rows()*3, x1.cols()) = in[i] * x1.colwise().replicate(3); + MapType(out+i*stride+2*step, x1.rows(), x1.cols()*3) = in[i] * x1.rowwise().replicate(3); + } +}; + +template<typename T> +struct redux { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + int N = 10; + T x1(in+i); + out[i*N+0] = x1.minCoeff(); + out[i*N+1] = x1.maxCoeff(); + out[i*N+2] = x1.sum(); + out[i*N+3] = x1.prod(); + out[i*N+4] = x1.matrix().squaredNorm(); + out[i*N+5] = x1.matrix().norm(); + out[i*N+6] = x1.colwise().sum().maxCoeff(); + out[i*N+7] = x1.rowwise().maxCoeff().sum(); + out[i*N+8] = x1.matrix().colwise().squaredNorm().sum(); + } +}; + +template<typename T1, typename T2> +struct prod_test { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T1::Scalar* in, typename T1::Scalar* out) const + { + using namespace Eigen; + typedef Matrix<typename T1::Scalar, T1::RowsAtCompileTime, T2::ColsAtCompileTime> T3; + T1 x1(in+i); + T2 x2(in+i+1); + Map<T3> res(out+i*T3::MaxSizeAtCompileTime); + res += in[i] * x1 * x2; + } +}; + +template<typename T1, typename T2> +struct diagonal { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T1::Scalar* in, typename T1::Scalar* out) const + { + using namespace Eigen; + T1 x1(in+i); + Map<T2> res(out+i*T2::MaxSizeAtCompileTime); + res += x1.diagonal(); + } +}; + +template<typename T> +struct eigenvalues_direct { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + typedef Matrix<typename T::Scalar, T::RowsAtCompileTime, 1> Vec; + T M(in+i); + Map<Vec> res(out+i*Vec::MaxSizeAtCompileTime); + T A = M*M.adjoint(); + SelfAdjointEigenSolver<T> eig; + eig.computeDirect(A); + res = eig.eigenvalues(); + } +}; + +template<typename T> +struct eigenvalues { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + typedef Matrix<typename T::Scalar, T::RowsAtCompileTime, 1> Vec; + T M(in+i); + Map<Vec> res(out+i*Vec::MaxSizeAtCompileTime); + T A = M*M.adjoint(); + SelfAdjointEigenSolver<T> eig; + eig.compute(A); + res = eig.eigenvalues(); + } +}; + +template<typename T> +struct matrix_inverse { + EIGEN_DEVICE_FUNC + void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const + { + using namespace Eigen; + T M(in+i); + Map<T> res(out+i*T::MaxSizeAtCompileTime); + res = M.inverse(); + } +}; + +EIGEN_DECLARE_TEST(gpu_basic) +{ + ei_test_init_gpu(); + + int nthreads = 100; + Eigen::VectorXf in, out; + + #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) + int data_size = nthreads * 512; + in.setRandom(data_size); + out.setRandom(data_size); + #endif + + CALL_SUBTEST( run_and_compare_to_gpu(coeff_wise<Vector3f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(coeff_wise<Array44f>(), nthreads, in, out) ); + +#if !defined(EIGEN_USE_HIP) + // FIXME + // These subtests result in a compile failure on the HIP platform + // + // eigen-upstream/Eigen/src/Core/Replicate.h:61:65: error: + // base class 'internal::dense_xpr_base<Replicate<Array<float, 4, 1, 0, 4, 1>, -1, -1> >::type' + // (aka 'ArrayBase<Eigen::Replicate<Eigen::Array<float, 4, 1, 0, 4, 1>, -1, -1> >') has protected default constructor + CALL_SUBTEST( run_and_compare_to_gpu(replicate<Array4f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(replicate<Array33f>(), nthreads, in, out) ); +#endif + + CALL_SUBTEST( run_and_compare_to_gpu(redux<Array4f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(redux<Matrix3f>(), nthreads, in, out) ); + + CALL_SUBTEST( run_and_compare_to_gpu(prod_test<Matrix3f,Matrix3f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(prod_test<Matrix4f,Vector4f>(), nthreads, in, out) ); + + CALL_SUBTEST( run_and_compare_to_gpu(diagonal<Matrix3f,Vector3f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(diagonal<Matrix4f,Vector4f>(), nthreads, in, out) ); + + CALL_SUBTEST( run_and_compare_to_gpu(matrix_inverse<Matrix2f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(matrix_inverse<Matrix3f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(matrix_inverse<Matrix4f>(), nthreads, in, out) ); + + CALL_SUBTEST( run_and_compare_to_gpu(eigenvalues_direct<Matrix3f>(), nthreads, in, out) ); + CALL_SUBTEST( run_and_compare_to_gpu(eigenvalues_direct<Matrix2f>(), nthreads, in, out) ); + +#if defined(__NVCC__) + // FIXME + // These subtests compiles only with nvcc and fail with HIPCC and clang-cuda + CALL_SUBTEST( run_and_compare_to_gpu(eigenvalues<Matrix4f>(), nthreads, in, out) ); + typedef Matrix<float,6,6> Matrix6f; + CALL_SUBTEST( run_and_compare_to_gpu(eigenvalues<Matrix6f>(), nthreads, in, out) ); +#endif +}
diff --git a/test/gpu_common.h b/test/gpu_common.h new file mode 100644 index 0000000..79d4ea6 --- /dev/null +++ b/test/gpu_common.h
@@ -0,0 +1,157 @@ + +#ifndef EIGEN_TEST_GPU_COMMON_H +#define EIGEN_TEST_GPU_COMMON_H + +#ifdef EIGEN_USE_HIP + #include <hip/hip_runtime.h> + #include <hip/hip_runtime_api.h> +#else + #include <cuda.h> + #include <cuda_runtime.h> + #include <cuda_runtime_api.h> +#endif + +#include <iostream> + +#define EIGEN_USE_GPU +#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + +#if !defined(__CUDACC__) && !defined(__HIPCC__) +dim3 threadIdx, blockDim, blockIdx; +#endif + +template<typename Kernel, typename Input, typename Output> +void run_on_cpu(const Kernel& ker, int n, const Input& in, Output& out) +{ + for(int i=0; i<n; i++) + ker(i, in.data(), out.data()); +} + + +template<typename Kernel, typename Input, typename Output> +__global__ +void run_on_gpu_meta_kernel(const Kernel ker, int n, const Input* in, Output* out) +{ + int i = threadIdx.x + blockIdx.x*blockDim.x; + if(i<n) { + ker(i, in, out); + } +} + + +template<typename Kernel, typename Input, typename Output> +void run_on_gpu(const Kernel& ker, int n, const Input& in, Output& out) +{ + typename Input::Scalar* d_in; + typename Output::Scalar* d_out; + std::ptrdiff_t in_bytes = in.size() * sizeof(typename Input::Scalar); + std::ptrdiff_t out_bytes = out.size() * sizeof(typename Output::Scalar); + + gpuMalloc((void**)(&d_in), in_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_in, in.data(), in_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_out, out.data(), out_bytes, gpuMemcpyHostToDevice); + + // Simple and non-optimal 1D mapping assuming n is not too large + // That's only for unit testing! + dim3 Blocks(128); + dim3 Grids( (n+int(Blocks.x)-1)/int(Blocks.x) ); + + gpuDeviceSynchronize(); + +#ifdef EIGEN_USE_HIP + hipLaunchKernelGGL(HIP_KERNEL_NAME(run_on_gpu_meta_kernel<Kernel, + typename std::decay<decltype(*d_in)>::type, + typename std::decay<decltype(*d_out)>::type>), + dim3(Grids), dim3(Blocks), 0, 0, ker, n, d_in, d_out); +#else + run_on_gpu_meta_kernel<<<Grids,Blocks>>>(ker, n, d_in, d_out); +#endif + + gpuDeviceSynchronize(); + + // check inputs have not been modified + gpuMemcpy(const_cast<typename Input::Scalar*>(in.data()), d_in, in_bytes, gpuMemcpyDeviceToHost); + gpuMemcpy(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost); + + gpuFree(d_in); + gpuFree(d_out); +} + + +template<typename Kernel, typename Input, typename Output> +void run_and_compare_to_gpu(const Kernel& ker, int n, const Input& in, Output& out) +{ + Input in_ref, in_gpu; + Output out_ref, out_gpu; + #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) + in_ref = in_gpu = in; + out_ref = out_gpu = out; + #else + EIGEN_UNUSED_VARIABLE(in); + EIGEN_UNUSED_VARIABLE(out); + #endif + run_on_cpu (ker, n, in_ref, out_ref); + run_on_gpu(ker, n, in_gpu, out_gpu); + #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) + VERIFY_IS_APPROX(in_ref, in_gpu); + VERIFY_IS_APPROX(out_ref, out_gpu); + #endif +} + +struct compile_time_device_info { + EIGEN_DEVICE_FUNC + void operator()(int /*i*/, const int* /*in*/, int* info) const + { + #if defined(__CUDA_ARCH__) + info[0] = int(__CUDA_ARCH__ +0); + #endif + #if defined(EIGEN_HIP_DEVICE_COMPILE) + info[1] = int(EIGEN_HIP_DEVICE_COMPILE +0); + #endif + } +}; + +void ei_test_init_gpu() +{ + int device = 0; + gpuDeviceProp_t deviceProp; + gpuGetDeviceProperties(&deviceProp, device); + + ArrayXi dummy(1), info(10); + info = -1; + run_on_gpu(compile_time_device_info(),10,dummy,info); + + + std::cout << "GPU compile-time info:\n"; + + #ifdef EIGEN_CUDACC + std::cout << " EIGEN_CUDACC: " << int(EIGEN_CUDACC) << "\n"; + #endif + + #ifdef EIGEN_CUDACC_VER + std::cout << " EIGEN_CUDACC_VER: " << int(EIGEN_CUDACC_VER) << "\n"; + #endif + + #ifdef EIGEN_HIPCC + std::cout << " EIGEN_HIPCC: " << int(EIGEN_HIPCC) << "\n"; + #endif + + std::cout << " EIGEN_CUDA_ARCH: " << info[0] << "\n"; + std::cout << " EIGEN_HIP_DEVICE_COMPILE: " << info[1] << "\n"; + + std::cout << "GPU device info:\n"; + std::cout << " name: " << deviceProp.name << "\n"; + std::cout << " capability: " << deviceProp.major << "." << deviceProp.minor << "\n"; + std::cout << " multiProcessorCount: " << deviceProp.multiProcessorCount << "\n"; + std::cout << " maxThreadsPerMultiProcessor: " << deviceProp.maxThreadsPerMultiProcessor << "\n"; + std::cout << " warpSize: " << deviceProp.warpSize << "\n"; + std::cout << " regsPerBlock: " << deviceProp.regsPerBlock << "\n"; + std::cout << " concurrentKernels: " << deviceProp.concurrentKernels << "\n"; + std::cout << " clockRate: " << deviceProp.clockRate << "\n"; + std::cout << " canMapHostMemory: " << deviceProp.canMapHostMemory << "\n"; + std::cout << " computeMode: " << deviceProp.computeMode << "\n"; +} + +#endif // EIGEN_TEST_GPU_COMMON_H
diff --git a/test/half_float.cpp b/test/half_float.cpp new file mode 100644 index 0000000..2a7f9b4 --- /dev/null +++ b/test/half_float.cpp
@@ -0,0 +1,287 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include <sstream> + +#include "main.h" + +#include <Eigen/src/Core/arch/GPU/Half.h> + +// Make sure it's possible to forward declare Eigen::half +namespace Eigen { +struct half; +} + +using Eigen::half; + +void test_conversion() +{ + using Eigen::half_impl::__half_raw; + + // Conversion from float. + VERIFY_IS_EQUAL(half(1.0f).x, 0x3c00); + VERIFY_IS_EQUAL(half(0.5f).x, 0x3800); + VERIFY_IS_EQUAL(half(0.33333f).x, 0x3555); + VERIFY_IS_EQUAL(half(0.0f).x, 0x0000); + VERIFY_IS_EQUAL(half(-0.0f).x, 0x8000); + VERIFY_IS_EQUAL(half(65504.0f).x, 0x7bff); + VERIFY_IS_EQUAL(half(65536.0f).x, 0x7c00); // Becomes infinity. + + // Denormals. + VERIFY_IS_EQUAL(half(-5.96046e-08f).x, 0x8001); + VERIFY_IS_EQUAL(half(5.96046e-08f).x, 0x0001); + VERIFY_IS_EQUAL(half(1.19209e-07f).x, 0x0002); + + // Verify round-to-nearest-even behavior. + float val1 = float(half(__half_raw(0x3c00))); + float val2 = float(half(__half_raw(0x3c01))); + float val3 = float(half(__half_raw(0x3c02))); + VERIFY_IS_EQUAL(half(0.5f * (val1 + val2)).x, 0x3c00); + VERIFY_IS_EQUAL(half(0.5f * (val2 + val3)).x, 0x3c02); + + // Conversion from int. + VERIFY_IS_EQUAL(half(-1).x, 0xbc00); + VERIFY_IS_EQUAL(half(0).x, 0x0000); + VERIFY_IS_EQUAL(half(1).x, 0x3c00); + VERIFY_IS_EQUAL(half(2).x, 0x4000); + VERIFY_IS_EQUAL(half(3).x, 0x4200); + + // Conversion from bool. + VERIFY_IS_EQUAL(half(false).x, 0x0000); + VERIFY_IS_EQUAL(half(true).x, 0x3c00); + + // Conversion to float. + VERIFY_IS_EQUAL(float(half(__half_raw(0x0000))), 0.0f); + VERIFY_IS_EQUAL(float(half(__half_raw(0x3c00))), 1.0f); + + // Denormals. + VERIFY_IS_APPROX(float(half(__half_raw(0x8001))), -5.96046e-08f); + VERIFY_IS_APPROX(float(half(__half_raw(0x0001))), 5.96046e-08f); + VERIFY_IS_APPROX(float(half(__half_raw(0x0002))), 1.19209e-07f); + + // NaNs and infinities. + VERIFY(!(numext::isinf)(float(half(65504.0f)))); // Largest finite number. + VERIFY(!(numext::isnan)(float(half(0.0f)))); + VERIFY((numext::isinf)(float(half(__half_raw(0xfc00))))); + VERIFY((numext::isnan)(float(half(__half_raw(0xfc01))))); + VERIFY((numext::isinf)(float(half(__half_raw(0x7c00))))); + VERIFY((numext::isnan)(float(half(__half_raw(0x7c01))))); + +#if !EIGEN_COMP_MSVC + // Visual Studio errors out on divisions by 0 + VERIFY((numext::isnan)(float(half(0.0 / 0.0)))); + VERIFY((numext::isinf)(float(half(1.0 / 0.0)))); + VERIFY((numext::isinf)(float(half(-1.0 / 0.0)))); +#endif + + // Exactly same checks as above, just directly on the half representation. + VERIFY(!(numext::isinf)(half(__half_raw(0x7bff)))); + VERIFY(!(numext::isnan)(half(__half_raw(0x0000)))); + VERIFY((numext::isinf)(half(__half_raw(0xfc00)))); + VERIFY((numext::isnan)(half(__half_raw(0xfc01)))); + VERIFY((numext::isinf)(half(__half_raw(0x7c00)))); + VERIFY((numext::isnan)(half(__half_raw(0x7c01)))); + +#if !EIGEN_COMP_MSVC + // Visual Studio errors out on divisions by 0 + VERIFY((numext::isnan)(half(0.0 / 0.0))); + VERIFY((numext::isinf)(half(1.0 / 0.0))); + VERIFY((numext::isinf)(half(-1.0 / 0.0))); +#endif +} + +void test_numtraits() +{ + std::cout << "epsilon = " << NumTraits<half>::epsilon() << " (0x" << std::hex << NumTraits<half>::epsilon().x << ")" << std::endl; + std::cout << "highest = " << NumTraits<half>::highest() << " (0x" << std::hex << NumTraits<half>::highest().x << ")" << std::endl; + std::cout << "lowest = " << NumTraits<half>::lowest() << " (0x" << std::hex << NumTraits<half>::lowest().x << ")" << std::endl; + std::cout << "min = " << (std::numeric_limits<half>::min)() << " (0x" << std::hex << half((std::numeric_limits<half>::min)()).x << ")" << std::endl; + std::cout << "denorm min = " << (std::numeric_limits<half>::denorm_min)() << " (0x" << std::hex << half((std::numeric_limits<half>::denorm_min)()).x << ")" << std::endl; + std::cout << "infinity = " << NumTraits<half>::infinity() << " (0x" << std::hex << NumTraits<half>::infinity().x << ")" << std::endl; + std::cout << "quiet nan = " << NumTraits<half>::quiet_NaN() << " (0x" << std::hex << NumTraits<half>::quiet_NaN().x << ")" << std::endl; + std::cout << "signaling nan = " << std::numeric_limits<half>::signaling_NaN() << " (0x" << std::hex << std::numeric_limits<half>::signaling_NaN().x << ")" << std::endl; + + VERIFY(NumTraits<half>::IsSigned); + + VERIFY_IS_EQUAL( std::numeric_limits<half>::infinity().x, half(std::numeric_limits<float>::infinity()).x ); + VERIFY_IS_EQUAL( std::numeric_limits<half>::quiet_NaN().x, half(std::numeric_limits<float>::quiet_NaN()).x ); + VERIFY_IS_EQUAL( std::numeric_limits<half>::signaling_NaN().x, half(std::numeric_limits<float>::signaling_NaN()).x ); + VERIFY( (std::numeric_limits<half>::min)() > half(0.f) ); + VERIFY( (std::numeric_limits<half>::denorm_min)() > half(0.f) ); + VERIFY( (std::numeric_limits<half>::min)()/half(2) > half(0.f) ); + VERIFY_IS_EQUAL( (std::numeric_limits<half>::denorm_min)()/half(2), half(0.f) ); +} + +void test_arithmetic() +{ + VERIFY_IS_EQUAL(float(half(2) + half(2)), 4); + VERIFY_IS_EQUAL(float(half(2) + half(-2)), 0); + VERIFY_IS_APPROX(float(half(0.33333f) + half(0.66667f)), 1.0f); + VERIFY_IS_EQUAL(float(half(2.0f) * half(-5.5f)), -11.0f); + VERIFY_IS_APPROX(float(half(1.0f) / half(3.0f)), 0.33333f); + VERIFY_IS_EQUAL(float(-half(4096.0f)), -4096.0f); + VERIFY_IS_EQUAL(float(-half(-4096.0f)), 4096.0f); +} + +void test_comparison() +{ + VERIFY(half(1.0f) > half(0.5f)); + VERIFY(half(0.5f) < half(1.0f)); + VERIFY(!(half(1.0f) < half(0.5f))); + VERIFY(!(half(0.5f) > half(1.0f))); + + VERIFY(!(half(4.0f) > half(4.0f))); + VERIFY(!(half(4.0f) < half(4.0f))); + + VERIFY(!(half(0.0f) < half(-0.0f))); + VERIFY(!(half(-0.0f) < half(0.0f))); + VERIFY(!(half(0.0f) > half(-0.0f))); + VERIFY(!(half(-0.0f) > half(0.0f))); + + VERIFY(half(0.2f) > half(-1.0f)); + VERIFY(half(-1.0f) < half(0.2f)); + VERIFY(half(-16.0f) < half(-15.0f)); + + VERIFY(half(1.0f) == half(1.0f)); + VERIFY(half(1.0f) != half(2.0f)); + + // Comparisons with NaNs and infinities. +#if !EIGEN_COMP_MSVC + // Visual Studio errors out on divisions by 0 + VERIFY(!(half(0.0 / 0.0) == half(0.0 / 0.0))); + VERIFY(half(0.0 / 0.0) != half(0.0 / 0.0)); + + VERIFY(!(half(1.0) == half(0.0 / 0.0))); + VERIFY(!(half(1.0) < half(0.0 / 0.0))); + VERIFY(!(half(1.0) > half(0.0 / 0.0))); + VERIFY(half(1.0) != half(0.0 / 0.0)); + + VERIFY(half(1.0) < half(1.0 / 0.0)); + VERIFY(half(1.0) > half(-1.0 / 0.0)); +#endif +} + +void test_basic_functions() +{ + VERIFY_IS_EQUAL(float(numext::abs(half(3.5f))), 3.5f); + VERIFY_IS_EQUAL(float(abs(half(3.5f))), 3.5f); + VERIFY_IS_EQUAL(float(numext::abs(half(-3.5f))), 3.5f); + VERIFY_IS_EQUAL(float(abs(half(-3.5f))), 3.5f); + + VERIFY_IS_EQUAL(float(numext::floor(half(3.5f))), 3.0f); + VERIFY_IS_EQUAL(float(floor(half(3.5f))), 3.0f); + VERIFY_IS_EQUAL(float(numext::floor(half(-3.5f))), -4.0f); + VERIFY_IS_EQUAL(float(floor(half(-3.5f))), -4.0f); + + VERIFY_IS_EQUAL(float(numext::ceil(half(3.5f))), 4.0f); + VERIFY_IS_EQUAL(float(ceil(half(3.5f))), 4.0f); + VERIFY_IS_EQUAL(float(numext::ceil(half(-3.5f))), -3.0f); + VERIFY_IS_EQUAL(float(ceil(half(-3.5f))), -3.0f); + + VERIFY_IS_APPROX(float(numext::sqrt(half(0.0f))), 0.0f); + VERIFY_IS_APPROX(float(sqrt(half(0.0f))), 0.0f); + VERIFY_IS_APPROX(float(numext::sqrt(half(4.0f))), 2.0f); + VERIFY_IS_APPROX(float(sqrt(half(4.0f))), 2.0f); + + VERIFY_IS_APPROX(float(numext::pow(half(0.0f), half(1.0f))), 0.0f); + VERIFY_IS_APPROX(float(pow(half(0.0f), half(1.0f))), 0.0f); + VERIFY_IS_APPROX(float(numext::pow(half(2.0f), half(2.0f))), 4.0f); + VERIFY_IS_APPROX(float(pow(half(2.0f), half(2.0f))), 4.0f); + + VERIFY_IS_EQUAL(float(numext::exp(half(0.0f))), 1.0f); + VERIFY_IS_EQUAL(float(exp(half(0.0f))), 1.0f); + VERIFY_IS_APPROX(float(numext::exp(half(EIGEN_PI))), 20.f + float(EIGEN_PI)); + VERIFY_IS_APPROX(float(exp(half(EIGEN_PI))), 20.f + float(EIGEN_PI)); + + VERIFY_IS_EQUAL(float(numext::expm1(half(0.0f))), 0.0f); + VERIFY_IS_EQUAL(float(expm1(half(0.0f))), 0.0f); + VERIFY_IS_APPROX(float(numext::expm1(half(2.0f))), 6.3890561f); + VERIFY_IS_APPROX(float(expm1(half(2.0f))), 6.3890561f); + + VERIFY_IS_EQUAL(float(numext::log(half(1.0f))), 0.0f); + VERIFY_IS_EQUAL(float(log(half(1.0f))), 0.0f); + VERIFY_IS_APPROX(float(numext::log(half(10.0f))), 2.30273f); + VERIFY_IS_APPROX(float(log(half(10.0f))), 2.30273f); + + VERIFY_IS_EQUAL(float(numext::log1p(half(0.0f))), 0.0f); + VERIFY_IS_EQUAL(float(log1p(half(0.0f))), 0.0f); + VERIFY_IS_APPROX(float(numext::log1p(half(10.0f))), 2.3978953f); + VERIFY_IS_APPROX(float(log1p(half(10.0f))), 2.3978953f); +} + +void test_trigonometric_functions() +{ + VERIFY_IS_APPROX(numext::cos(half(0.0f)), half(cosf(0.0f))); + VERIFY_IS_APPROX(cos(half(0.0f)), half(cosf(0.0f))); + VERIFY_IS_APPROX(numext::cos(half(EIGEN_PI)), half(cosf(EIGEN_PI))); + //VERIFY_IS_APPROX(numext::cos(half(EIGEN_PI/2)), half(cosf(EIGEN_PI/2))); + //VERIFY_IS_APPROX(numext::cos(half(3*EIGEN_PI/2)), half(cosf(3*EIGEN_PI/2))); + VERIFY_IS_APPROX(numext::cos(half(3.5f)), half(cosf(3.5f))); + + VERIFY_IS_APPROX(numext::sin(half(0.0f)), half(sinf(0.0f))); + VERIFY_IS_APPROX(sin(half(0.0f)), half(sinf(0.0f))); + // VERIFY_IS_APPROX(numext::sin(half(EIGEN_PI)), half(sinf(EIGEN_PI))); + VERIFY_IS_APPROX(numext::sin(half(EIGEN_PI/2)), half(sinf(EIGEN_PI/2))); + VERIFY_IS_APPROX(numext::sin(half(3*EIGEN_PI/2)), half(sinf(3*EIGEN_PI/2))); + VERIFY_IS_APPROX(numext::sin(half(3.5f)), half(sinf(3.5f))); + + VERIFY_IS_APPROX(numext::tan(half(0.0f)), half(tanf(0.0f))); + VERIFY_IS_APPROX(tan(half(0.0f)), half(tanf(0.0f))); + // VERIFY_IS_APPROX(numext::tan(half(EIGEN_PI)), half(tanf(EIGEN_PI))); + // VERIFY_IS_APPROX(numext::tan(half(EIGEN_PI/2)), half(tanf(EIGEN_PI/2))); + //VERIFY_IS_APPROX(numext::tan(half(3*EIGEN_PI/2)), half(tanf(3*EIGEN_PI/2))); + VERIFY_IS_APPROX(numext::tan(half(3.5f)), half(tanf(3.5f))); +} + +void test_array() +{ + typedef Array<half,1,Dynamic> ArrayXh; + Index size = internal::random<Index>(1,10); + Index i = internal::random<Index>(0,size-1); + ArrayXh a1 = ArrayXh::Random(size), a2 = ArrayXh::Random(size); + VERIFY_IS_APPROX( a1+a1, half(2)*a1 ); + VERIFY( (a1.abs() >= half(0)).all() ); + VERIFY_IS_APPROX( (a1*a1).sqrt(), a1.abs() ); + + VERIFY( ((a1.min)(a2) <= (a1.max)(a2)).all() ); + a1(i) = half(-10.); + VERIFY_IS_EQUAL( a1.minCoeff(), half(-10.) ); + a1(i) = half(10.); + VERIFY_IS_EQUAL( a1.maxCoeff(), half(10.) ); + + std::stringstream ss; + ss << a1; +} + +void test_product() +{ + typedef Matrix<half,Dynamic,Dynamic> MatrixXh; + Index rows = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + Index cols = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + Index depth = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + MatrixXh Ah = MatrixXh::Random(rows,depth); + MatrixXh Bh = MatrixXh::Random(depth,cols); + MatrixXh Ch = MatrixXh::Random(rows,cols); + MatrixXf Af = Ah.cast<float>(); + MatrixXf Bf = Bh.cast<float>(); + MatrixXf Cf = Ch.cast<float>(); + VERIFY_IS_APPROX(Ch.noalias()+=Ah*Bh, (Cf.noalias()+=Af*Bf).cast<half>()); +} + +EIGEN_DECLARE_TEST(half_float) +{ + CALL_SUBTEST(test_numtraits()); + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST(test_conversion()); + CALL_SUBTEST(test_arithmetic()); + CALL_SUBTEST(test_comparison()); + CALL_SUBTEST(test_basic_functions()); + CALL_SUBTEST(test_trigonometric_functions()); + CALL_SUBTEST(test_array()); + CALL_SUBTEST(test_product()); + } +}
diff --git a/test/hessenberg.cpp b/test/hessenberg.cpp index 96bc19e..0e1b009 100644 --- a/test/hessenberg.cpp +++ b/test/hessenberg.cpp
@@ -49,7 +49,7 @@ // TODO: Add tests for packedMatrix() and householderCoefficients() } -void test_hessenberg() +EIGEN_DECLARE_TEST(hessenberg) { CALL_SUBTEST_1(( hessenberg<std::complex<double>,1>() )); CALL_SUBTEST_2(( hessenberg<std::complex<double>,2>() ));
diff --git a/test/householder.cpp b/test/householder.cpp index c5f6b5e..cad8138 100644 --- a/test/householder.cpp +++ b/test/householder.cpp
@@ -12,7 +12,6 @@ template<typename MatrixType> void householder(const MatrixType& m) { - typedef typename MatrixType::Index Index; static bool even = true; even = !even; /* this test covers the following files: @@ -49,6 +48,17 @@ v1.applyHouseholderOnTheLeft(essential,beta,tmp); VERIFY_IS_APPROX(v1.norm(), v2.norm()); + // reconstruct householder matrix: + SquareMatrixType id, H1, H2; + id.setIdentity(rows, rows); + H1 = H2 = id; + VectorType vv(rows); + vv << Scalar(1), essential; + H1.applyHouseholderOnTheLeft(essential, beta, tmp); + H2.applyHouseholderOnTheRight(essential, beta, tmp); + VERIFY_IS_APPROX(H1, H2); + VERIFY_IS_APPROX(H1, id - beta * vv*vv.adjoint()); + MatrixType m1(rows, cols), m2(rows, cols); @@ -69,7 +79,7 @@ m3.rowwise() = v1.transpose(); m4 = m3; m3.row(0).makeHouseholder(essential, beta, alpha); - m3.applyHouseholderOnTheRight(essential,beta,tmp); + m3.applyHouseholderOnTheRight(essential.conjugate(),beta,tmp); VERIFY_IS_APPROX(m3.norm(), m4.norm()); if(rows>=2) VERIFY_IS_MUCH_SMALLER_THAN(m3.block(0,1,rows,rows-1).norm(), m3.norm()); VERIFY_IS_MUCH_SMALLER_THAN(numext::imag(m3(0,0)), numext::real(m3(0,0))); @@ -104,14 +114,14 @@ VERIFY_IS_APPROX(hseq_mat.adjoint(), hseq_mat_adj); VERIFY_IS_APPROX(hseq_mat.conjugate(), hseq_mat_conj); VERIFY_IS_APPROX(hseq_mat.transpose(), hseq_mat_trans); - VERIFY_IS_APPROX(hseq_mat * m6, hseq_mat * m6); - VERIFY_IS_APPROX(hseq_mat.adjoint() * m6, hseq_mat_adj * m6); - VERIFY_IS_APPROX(hseq_mat.conjugate() * m6, hseq_mat_conj * m6); - VERIFY_IS_APPROX(hseq_mat.transpose() * m6, hseq_mat_trans * m6); - VERIFY_IS_APPROX(m6 * hseq_mat, m6 * hseq_mat); - VERIFY_IS_APPROX(m6 * hseq_mat.adjoint(), m6 * hseq_mat_adj); - VERIFY_IS_APPROX(m6 * hseq_mat.conjugate(), m6 * hseq_mat_conj); - VERIFY_IS_APPROX(m6 * hseq_mat.transpose(), m6 * hseq_mat_trans); + VERIFY_IS_APPROX(hseq * m6, hseq_mat * m6); + VERIFY_IS_APPROX(hseq.adjoint() * m6, hseq_mat_adj * m6); + VERIFY_IS_APPROX(hseq.conjugate() * m6, hseq_mat_conj * m6); + VERIFY_IS_APPROX(hseq.transpose() * m6, hseq_mat_trans * m6); + VERIFY_IS_APPROX(m6 * hseq, m6 * hseq_mat); + VERIFY_IS_APPROX(m6 * hseq.adjoint(), m6 * hseq_mat_adj); + VERIFY_IS_APPROX(m6 * hseq.conjugate(), m6 * hseq_mat_conj); + VERIFY_IS_APPROX(m6 * hseq.transpose(), m6 * hseq_mat_trans); // test householder sequence on the right with a shift @@ -123,7 +133,7 @@ VERIFY_IS_APPROX(m3 * m5, m1); // test evaluating rhseq to a dense matrix, then applying } -void test_householder() +EIGEN_DECLARE_TEST(householder) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( householder(Matrix<double,2,2>()) );
diff --git a/test/incomplete_cholesky.cpp b/test/incomplete_cholesky.cpp index 59ffe92..ecc17f5 100644 --- a/test/incomplete_cholesky.cpp +++ b/test/incomplete_cholesky.cpp
@@ -12,14 +12,14 @@ #include <Eigen/IterativeLinearSolvers> #include <unsupported/Eigen/IterativeSolvers> -template<typename T, typename I> void test_incomplete_cholesky_T() +template<typename T, typename I_> void test_incomplete_cholesky_T() { - typedef SparseMatrix<T,0,I> SparseMatrixType; - ConjugateGradient<SparseMatrixType, Lower, IncompleteCholesky<T, Lower, AMDOrdering<I> > > cg_illt_lower_amd; - ConjugateGradient<SparseMatrixType, Lower, IncompleteCholesky<T, Lower, NaturalOrdering<I> > > cg_illt_lower_nat; - ConjugateGradient<SparseMatrixType, Upper, IncompleteCholesky<T, Upper, AMDOrdering<I> > > cg_illt_upper_amd; - ConjugateGradient<SparseMatrixType, Upper, IncompleteCholesky<T, Upper, NaturalOrdering<I> > > cg_illt_upper_nat; - ConjugateGradient<SparseMatrixType, Upper|Lower, IncompleteCholesky<T, Lower, AMDOrdering<I> > > cg_illt_uplo_amd; + typedef SparseMatrix<T,0,I_> SparseMatrixType; + ConjugateGradient<SparseMatrixType, Lower, IncompleteCholesky<T, Lower, AMDOrdering<I_> > > cg_illt_lower_amd; + ConjugateGradient<SparseMatrixType, Lower, IncompleteCholesky<T, Lower, NaturalOrdering<I_> > > cg_illt_lower_nat; + ConjugateGradient<SparseMatrixType, Upper, IncompleteCholesky<T, Upper, AMDOrdering<I_> > > cg_illt_upper_amd; + ConjugateGradient<SparseMatrixType, Upper, IncompleteCholesky<T, Upper, NaturalOrdering<I_> > > cg_illt_upper_nat; + ConjugateGradient<SparseMatrixType, Upper|Lower, IncompleteCholesky<T, Lower, AMDOrdering<I_> > > cg_illt_uplo_amd; CALL_SUBTEST( check_sparse_spd_solving(cg_illt_lower_amd) ); @@ -29,14 +29,10 @@ CALL_SUBTEST( check_sparse_spd_solving(cg_illt_uplo_amd) ); } -void test_incomplete_cholesky() +template<int> +void bug1150() { - CALL_SUBTEST_1(( test_incomplete_cholesky_T<double,int>() )); - CALL_SUBTEST_2(( test_incomplete_cholesky_T<std::complex<double>, int>() )); - CALL_SUBTEST_3(( test_incomplete_cholesky_T<double,long int>() )); - -#ifdef EIGEN_TEST_PART_1 - // regression for bug 1150 + // regression for bug 1150 for(int N = 1; N<20; ++N) { Eigen::MatrixXd b( N, N ); @@ -61,5 +57,13 @@ VERIFY(solver.preconditioner().info() == Eigen::Success); VERIFY(solver.info() == Eigen::Success); } -#endif +} + +EIGEN_DECLARE_TEST(incomplete_cholesky) +{ + CALL_SUBTEST_1(( test_incomplete_cholesky_T<double,int>() )); + CALL_SUBTEST_2(( test_incomplete_cholesky_T<std::complex<double>, int>() )); + CALL_SUBTEST_3(( test_incomplete_cholesky_T<double,long int>() )); + + CALL_SUBTEST_1(( bug1150<0>() )); }
diff --git a/test/indexed_view.cpp b/test/indexed_view.cpp new file mode 100644 index 0000000..5f1e01f --- /dev/null +++ b/test/indexed_view.cpp
@@ -0,0 +1,442 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifdef EIGEN_TEST_PART_2 +// Make sure we also check c++11 max implementation +#define EIGEN_MAX_CPP_VER 11 +#endif + +#ifdef EIGEN_TEST_PART_3 +// Make sure we also check c++98 max implementation +#define EIGEN_MAX_CPP_VER 03 + +// We need to disable this warning when compiling with c++11 while limiting Eigen to c++98 +// Ideally we would rather configure the compiler to build in c++98 mode but this needs +// to be done at the CMakeLists.txt level. +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) + #pragma GCC diagnostic ignored "-Wdeprecated" +#endif + +#endif + +#include <valarray> +#include <vector> +#include "main.h" + +#if EIGEN_HAS_CXX11 +#include <array> +#endif + +typedef std::pair<Index,Index> IndexPair; + +int encode(Index i, Index j) { + return int(i*100 + j); +} + +IndexPair decode(Index ij) { + return IndexPair(ij / 100, ij % 100); +} + +template<typename T> +bool match(const T& xpr, std::string ref, std::string str_xpr = "") { + EIGEN_UNUSED_VARIABLE(str_xpr); + std::stringstream str; + str << xpr; + if(!(str.str() == ref)) + std::cout << str_xpr << "\n" << xpr << "\n\n"; + return str.str() == ref; +} + +#define MATCH(X,R) match(X, R, #X) + +template<typename T1,typename T2> +typename internal::enable_if<internal::is_same<T1,T2>::value,bool>::type +is_same_eq(const T1& a, const T2& b) +{ + return (a == b).all(); +} + +template<typename T1,typename T2> +bool is_same_seq(const T1& a, const T2& b) +{ + bool ok = a.first()==b.first() && a.size() == b.size() && Index(a.incrObject())==Index(b.incrObject());; + if(!ok) + { + std::cerr << "seqN(" << a.first() << ", " << a.size() << ", " << Index(a.incrObject()) << ") != "; + std::cerr << "seqN(" << b.first() << ", " << b.size() << ", " << Index(b.incrObject()) << ")\n"; + } + return ok; +} + +template<typename T1,typename T2> +typename internal::enable_if<internal::is_same<T1,T2>::value,bool>::type +is_same_seq_type(const T1& a, const T2& b) +{ + return is_same_seq(a,b); +} + + + +#define VERIFY_EQ_INT(A,B) VERIFY_IS_APPROX(int(A),int(B)) + +// C++03 does not allow local or unnamed enums as index +enum DummyEnum { XX=0, YY=1 }; + +void check_indexed_view() +{ + Index n = 10; + + ArrayXd a = ArrayXd::LinSpaced(n,0,n-1); + Array<double,1,Dynamic> b = a.transpose(); + + #if EIGEN_COMP_CXXVER>=14 + ArrayXXi A = ArrayXXi::NullaryExpr(n,n, std::ref(encode)); + #else + ArrayXXi A = ArrayXXi::NullaryExpr(n,n, std::ptr_fun(&encode)); + #endif + + for(Index i=0; i<n; ++i) + for(Index j=0; j<n; ++j) + VERIFY( decode(A(i,j)) == IndexPair(i,j) ); + + Array4i eii(4); eii << 3, 1, 6, 5; + std::valarray<int> vali(4); Map<ArrayXi>(&vali[0],4) = eii; + std::vector<int> veci(4); Map<ArrayXi>(veci.data(),4) = eii; + + VERIFY( MATCH( A(3, seq(9,3,-1)), + "309 308 307 306 305 304 303") + ); + + VERIFY( MATCH( A(seqN(2,5), seq(9,3,-1)), + "209 208 207 206 205 204 203\n" + "309 308 307 306 305 304 303\n" + "409 408 407 406 405 404 403\n" + "509 508 507 506 505 504 503\n" + "609 608 607 606 605 604 603") + ); + + VERIFY( MATCH( A(seqN(2,5), 5), + "205\n" + "305\n" + "405\n" + "505\n" + "605") + ); + + VERIFY( MATCH( A(seqN(last,5,-1), seq(2,last)), + "902 903 904 905 906 907 908 909\n" + "802 803 804 805 806 807 808 809\n" + "702 703 704 705 706 707 708 709\n" + "602 603 604 605 606 607 608 609\n" + "502 503 504 505 506 507 508 509") + ); + + VERIFY( MATCH( A(eii, veci), + "303 301 306 305\n" + "103 101 106 105\n" + "603 601 606 605\n" + "503 501 506 505") + ); + + VERIFY( MATCH( A(eii, all), + "300 301 302 303 304 305 306 307 308 309\n" + "100 101 102 103 104 105 106 107 108 109\n" + "600 601 602 603 604 605 606 607 608 609\n" + "500 501 502 503 504 505 506 507 508 509") + ); + + // take row number 3, and repeat it 5 times + VERIFY( MATCH( A(seqN(3,5,0), all), + "300 301 302 303 304 305 306 307 308 309\n" + "300 301 302 303 304 305 306 307 308 309\n" + "300 301 302 303 304 305 306 307 308 309\n" + "300 301 302 303 304 305 306 307 308 309\n" + "300 301 302 303 304 305 306 307 308 309") + ); + + VERIFY( MATCH( a(seqN(3,3),0), "3\n4\n5" ) ); + VERIFY( MATCH( a(seq(3,5)), "3\n4\n5" ) ); + VERIFY( MATCH( a(seqN(3,3,1)), "3\n4\n5" ) ); + VERIFY( MATCH( a(seqN(5,3,-1)), "5\n4\n3" ) ); + + VERIFY( MATCH( b(0,seqN(3,3)), "3 4 5" ) ); + VERIFY( MATCH( b(seq(3,5)), "3 4 5" ) ); + VERIFY( MATCH( b(seqN(3,3,1)), "3 4 5" ) ); + VERIFY( MATCH( b(seqN(5,3,-1)), "5 4 3" ) ); + + VERIFY( MATCH( b(all), "0 1 2 3 4 5 6 7 8 9" ) ); + VERIFY( MATCH( b(eii), "3 1 6 5" ) ); + + Array44i B; + B.setRandom(); + VERIFY( (A(seqN(2,5), 5)).ColsAtCompileTime == 1); + VERIFY( (A(seqN(2,5), 5)).RowsAtCompileTime == Dynamic); + VERIFY_EQ_INT( (A(seqN(2,5), 5)).InnerStrideAtCompileTime , A.InnerStrideAtCompileTime); + VERIFY_EQ_INT( (A(seqN(2,5), 5)).OuterStrideAtCompileTime , A.col(5).OuterStrideAtCompileTime); + + VERIFY_EQ_INT( (A(5,seqN(2,5))).InnerStrideAtCompileTime , A.row(5).InnerStrideAtCompileTime); + VERIFY_EQ_INT( (A(5,seqN(2,5))).OuterStrideAtCompileTime , A.row(5).OuterStrideAtCompileTime); + VERIFY_EQ_INT( (B(1,seqN(1,2))).InnerStrideAtCompileTime , B.row(1).InnerStrideAtCompileTime); + VERIFY_EQ_INT( (B(1,seqN(1,2))).OuterStrideAtCompileTime , B.row(1).OuterStrideAtCompileTime); + + VERIFY_EQ_INT( (A(seqN(2,5), seq(1,3))).InnerStrideAtCompileTime , A.InnerStrideAtCompileTime); + VERIFY_EQ_INT( (A(seqN(2,5), seq(1,3))).OuterStrideAtCompileTime , A.OuterStrideAtCompileTime); + VERIFY_EQ_INT( (B(seqN(1,2), seq(1,3))).InnerStrideAtCompileTime , B.InnerStrideAtCompileTime); + VERIFY_EQ_INT( (B(seqN(1,2), seq(1,3))).OuterStrideAtCompileTime , B.OuterStrideAtCompileTime); + VERIFY_EQ_INT( (A(seqN(2,5,2), seq(1,3,2))).InnerStrideAtCompileTime , Dynamic); + VERIFY_EQ_INT( (A(seqN(2,5,2), seq(1,3,2))).OuterStrideAtCompileTime , Dynamic); + VERIFY_EQ_INT( (A(seqN(2,5,fix<2>), seq(1,3,fix<3>))).InnerStrideAtCompileTime , 2); + VERIFY_EQ_INT( (A(seqN(2,5,fix<2>), seq(1,3,fix<3>))).OuterStrideAtCompileTime , Dynamic); + VERIFY_EQ_INT( (B(seqN(1,2,fix<2>), seq(1,3,fix<3>))).InnerStrideAtCompileTime , 2); + VERIFY_EQ_INT( (B(seqN(1,2,fix<2>), seq(1,3,fix<3>))).OuterStrideAtCompileTime , 3*4); + + VERIFY_EQ_INT( (A(seqN(2,fix<5>), seqN(1,fix<3>))).RowsAtCompileTime, 5); + VERIFY_EQ_INT( (A(seqN(2,fix<5>), seqN(1,fix<3>))).ColsAtCompileTime, 3); + VERIFY_EQ_INT( (A(seqN(2,fix<5>(5)), seqN(1,fix<3>(3)))).RowsAtCompileTime, 5); + VERIFY_EQ_INT( (A(seqN(2,fix<5>(5)), seqN(1,fix<3>(3)))).ColsAtCompileTime, 3); + VERIFY_EQ_INT( (A(seqN(2,fix<Dynamic>(5)), seqN(1,fix<Dynamic>(3)))).RowsAtCompileTime, Dynamic); + VERIFY_EQ_INT( (A(seqN(2,fix<Dynamic>(5)), seqN(1,fix<Dynamic>(3)))).ColsAtCompileTime, Dynamic); + VERIFY_EQ_INT( (A(seqN(2,fix<Dynamic>(5)), seqN(1,fix<Dynamic>(3)))).rows(), 5); + VERIFY_EQ_INT( (A(seqN(2,fix<Dynamic>(5)), seqN(1,fix<Dynamic>(3)))).cols(), 3); + + VERIFY( is_same_seq_type( seqN(2,5,fix<-1>), seqN(2,5,fix<-1>(-1)) ) ); + VERIFY( is_same_seq_type( seqN(2,5), seqN(2,5,fix<1>(1)) ) ); + VERIFY( is_same_seq_type( seqN(2,5,3), seqN(2,5,fix<DynamicIndex>(3)) ) ); + VERIFY( is_same_seq_type( seq(2,7,fix<3>), seqN(2,2,fix<3>) ) ); + VERIFY( is_same_seq_type( seqN(2,fix<Dynamic>(5),3), seqN(2,5,fix<DynamicIndex>(3)) ) ); + VERIFY( is_same_seq_type( seqN(2,fix<5>(5),fix<-2>), seqN(2,fix<5>,fix<-2>()) ) ); + + VERIFY( is_same_seq_type( seq(2,fix<5>), seqN(2,4) ) ); +#if EIGEN_HAS_CXX11 + VERIFY( is_same_seq_type( seq(fix<2>,fix<5>), seqN(fix<2>,fix<4>) ) ); + VERIFY( is_same_seq( seqN(2,std::integral_constant<int,5>(),std::integral_constant<int,-2>()), seqN(2,fix<5>,fix<-2>()) ) ); + VERIFY( is_same_seq( seq(std::integral_constant<int,1>(),std::integral_constant<int,5>(),std::integral_constant<int,2>()), + seq(fix<1>,fix<5>,fix<2>()) ) ); + VERIFY( is_same_seq_type( seqN(2,std::integral_constant<int,5>(),std::integral_constant<int,-2>()), seqN(2,fix<5>,fix<-2>()) ) ); + VERIFY( is_same_seq_type( seq(std::integral_constant<int,1>(),std::integral_constant<int,5>(),std::integral_constant<int,2>()), + seq(fix<1>,fix<5>,fix<2>()) ) ); + + VERIFY( is_same_seq_type( seqN(2,std::integral_constant<int,5>()), seqN(2,fix<5>) ) ); + VERIFY( is_same_seq_type( seq(std::integral_constant<int,1>(),std::integral_constant<int,5>()), seq(fix<1>,fix<5>) ) ); +#else + // sorry, no compile-time size recovery in c++98/03 + VERIFY( is_same_seq( seq(fix<2>,fix<5>), seqN(fix<2>,fix<4>) ) ); +#endif + + VERIFY( (A(seqN(2,fix<5>), 5)).RowsAtCompileTime == 5); + VERIFY( (A(4, all)).ColsAtCompileTime == Dynamic); + VERIFY( (A(4, all)).RowsAtCompileTime == 1); + VERIFY( (B(1, all)).ColsAtCompileTime == 4); + VERIFY( (B(1, all)).RowsAtCompileTime == 1); + VERIFY( (B(all,1)).ColsAtCompileTime == 1); + VERIFY( (B(all,1)).RowsAtCompileTime == 4); + + VERIFY(int( (A(all, eii)).ColsAtCompileTime) == int(eii.SizeAtCompileTime)); + VERIFY_EQ_INT( (A(eii, eii)).Flags&DirectAccessBit, (unsigned int)(0)); + VERIFY_EQ_INT( (A(eii, eii)).InnerStrideAtCompileTime, 0); + VERIFY_EQ_INT( (A(eii, eii)).OuterStrideAtCompileTime, 0); + + VERIFY_IS_APPROX( A(seq(n-1,2,-2), seqN(n-1-6,3,-1)), A(seq(last,2,fix<-2>), seqN(last-6,3,fix<-1>)) ); + + VERIFY_IS_APPROX( A(seq(n-1,2,-2), seqN(n-1-6,4)), A(seq(last,2,-2), seqN(last-6,4)) ); + VERIFY_IS_APPROX( A(seq(n-1-6,n-1-2), seqN(n-1-6,4)), A(seq(last-6,last-2), seqN(6+last-6-6,4)) ); + VERIFY_IS_APPROX( A(seq((n-1)/2,(n)/2+3), seqN(2,4)), A(seq(last/2,(last+1)/2+3), seqN(last+2-last,4)) ); + VERIFY_IS_APPROX( A(seq(n-2,2,-2), seqN(n-8,4)), A(seq(lastp1-2,2,-2), seqN(lastp1-8,4)) ); + + // Check all combinations of seq: + VERIFY_IS_APPROX( A(seq(1,n-1-2,2), seq(1,n-1-2,2)), A(seq(1,last-2,2), seq(1,last-2,fix<2>)) ); + VERIFY_IS_APPROX( A(seq(n-1-5,n-1-2,2), seq(n-1-5,n-1-2,2)), A(seq(last-5,last-2,2), seq(last-5,last-2,fix<2>)) ); + VERIFY_IS_APPROX( A(seq(n-1-5,7,2), seq(n-1-5,7,2)), A(seq(last-5,7,2), seq(last-5,7,fix<2>)) ); + VERIFY_IS_APPROX( A(seq(1,n-1-2), seq(n-1-5,7)), A(seq(1,last-2), seq(last-5,7)) ); + VERIFY_IS_APPROX( A(seq(n-1-5,n-1-2), seq(n-1-5,n-1-2)), A(seq(last-5,last-2), seq(last-5,last-2)) ); + + VERIFY_IS_APPROX( A.col(A.cols()-1), A(all,last) ); + VERIFY_IS_APPROX( A(A.rows()-2, A.cols()/2), A(last-1, lastp1/2) ); + VERIFY_IS_APPROX( a(a.size()-2), a(last-1) ); + VERIFY_IS_APPROX( a(a.size()/2), a((last+1)/2) ); + + // Check fall-back to Block + { + VERIFY( is_same_eq(A.col(0), A(all,0)) ); + VERIFY( is_same_eq(A.row(0), A(0,all)) ); + VERIFY( is_same_eq(A.block(0,0,2,2), A(seqN(0,2),seq(0,1))) ); + VERIFY( is_same_eq(A.middleRows(2,4), A(seqN(2,4),all)) ); + VERIFY( is_same_eq(A.middleCols(2,4), A(all,seqN(2,4))) ); + + VERIFY( is_same_eq(A.col(A.cols()-1), A(all,last)) ); + + const ArrayXXi& cA(A); + VERIFY( is_same_eq(cA.col(0), cA(all,0)) ); + VERIFY( is_same_eq(cA.row(0), cA(0,all)) ); + VERIFY( is_same_eq(cA.block(0,0,2,2), cA(seqN(0,2),seq(0,1))) ); + VERIFY( is_same_eq(cA.middleRows(2,4), cA(seqN(2,4),all)) ); + VERIFY( is_same_eq(cA.middleCols(2,4), cA(all,seqN(2,4))) ); + + VERIFY( is_same_eq(a.head(4), a(seq(0,3))) ); + VERIFY( is_same_eq(a.tail(4), a(seqN(last-3,4))) ); + VERIFY( is_same_eq(a.tail(4), a(seq(lastp1-4,last))) ); + VERIFY( is_same_eq(a.segment<4>(3), a(seqN(3,fix<4>))) ); + } + + ArrayXXi A1=A, A2 = ArrayXXi::Random(4,4); + ArrayXi range25(4); range25 << 3,2,4,5; + A1(seqN(3,4),seq(2,5)) = A2; + VERIFY_IS_APPROX( A1.block(3,2,4,4), A2 ); + A1 = A; + A2.setOnes(); + A1(seq(6,3,-1),range25) = A2; + VERIFY_IS_APPROX( A1.block(3,2,4,4), A2 ); + + // check reverse + { + VERIFY( is_same_seq_type( seq(3,7).reverse(), seqN(7,5,fix<-1>) ) ); + VERIFY( is_same_seq_type( seq(7,3,fix<-2>).reverse(), seqN(3,3,fix<2>) ) ); + VERIFY_IS_APPROX( a(seqN(2,last/2).reverse()), a(seqN(2+(last/2-1)*1,last/2,fix<-1>)) ); + VERIFY_IS_APPROX( a(seqN(last/2,fix<4>).reverse()),a(seqN(last/2,fix<4>)).reverse() ); + VERIFY_IS_APPROX( A(seq(last-5,last-1,2).reverse(), seqN(last-3,3,fix<-2>).reverse()), + A(seq(last-5,last-1,2), seqN(last-3,3,fix<-2>)).reverse() ); + } + +#if EIGEN_HAS_CXX11 + // check lastN + VERIFY_IS_APPROX( a(lastN(3)), a.tail(3) ); + VERIFY( MATCH( a(lastN(3)), "7\n8\n9" ) ); + VERIFY_IS_APPROX( a(lastN(fix<3>())), a.tail<3>() ); + VERIFY( MATCH( a(lastN(3,2)), "5\n7\n9" ) ); + VERIFY( MATCH( a(lastN(3,fix<2>())), "5\n7\n9" ) ); + VERIFY( a(lastN(fix<3>())).SizeAtCompileTime == 3 ); + + VERIFY( (A(all, std::array<int,4>{{1,3,2,4}})).ColsAtCompileTime == 4); + + VERIFY_IS_APPROX( (A(std::array<int,3>{{1,3,5}}, std::array<int,4>{{9,6,3,0}})), A(seqN(1,3,2), seqN(9,4,-3)) ); + +#if EIGEN_HAS_STATIC_ARRAY_TEMPLATE + VERIFY_IS_APPROX( A({3, 1, 6, 5}, all), A(std::array<int,4>{{3, 1, 6, 5}}, all) ); + VERIFY_IS_APPROX( A(all,{3, 1, 6, 5}), A(all,std::array<int,4>{{3, 1, 6, 5}}) ); + VERIFY_IS_APPROX( A({1,3,5},{3, 1, 6, 5}), A(std::array<int,3>{{1,3,5}},std::array<int,4>{{3, 1, 6, 5}}) ); + + VERIFY_IS_EQUAL( A({1,3,5},{3, 1, 6, 5}).RowsAtCompileTime, 3 ); + VERIFY_IS_EQUAL( A({1,3,5},{3, 1, 6, 5}).ColsAtCompileTime, 4 ); + + VERIFY_IS_APPROX( a({3, 1, 6, 5}), a(std::array<int,4>{{3, 1, 6, 5}}) ); + VERIFY_IS_EQUAL( a({1,3,5}).SizeAtCompileTime, 3 ); + + VERIFY_IS_APPROX( b({3, 1, 6, 5}), b(std::array<int,4>{{3, 1, 6, 5}}) ); + VERIFY_IS_EQUAL( b({1,3,5}).SizeAtCompileTime, 3 ); +#endif + +#endif + + // check mat(i,j) with weird types for i and j + { + VERIFY_IS_APPROX( A(B.RowsAtCompileTime-1, 1), A(3,1) ); + VERIFY_IS_APPROX( A(B.RowsAtCompileTime, 1), A(4,1) ); + VERIFY_IS_APPROX( A(B.RowsAtCompileTime-1, B.ColsAtCompileTime-1), A(3,3) ); + VERIFY_IS_APPROX( A(B.RowsAtCompileTime, B.ColsAtCompileTime), A(4,4) ); + const Index I_ = 3, J_ = 4; + VERIFY_IS_APPROX( A(I_,J_), A(3,4) ); + } + + // check extended block API + { + VERIFY( is_same_eq( A.block<3,4>(1,1), A.block(1,1,fix<3>,fix<4>)) ); + VERIFY( is_same_eq( A.block<3,4>(1,1,3,4), A.block(1,1,fix<3>(),fix<4>(4))) ); + VERIFY( is_same_eq( A.block<3,Dynamic>(1,1,3,4), A.block(1,1,fix<3>,4)) ); + VERIFY( is_same_eq( A.block<Dynamic,4>(1,1,3,4), A.block(1,1,fix<Dynamic>(3),fix<4>)) ); + VERIFY( is_same_eq( A.block(1,1,3,4), A.block(1,1,fix<Dynamic>(3),fix<Dynamic>(4))) ); + + VERIFY( is_same_eq( A.topLeftCorner<3,4>(), A.topLeftCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( A.bottomLeftCorner<3,4>(), A.bottomLeftCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( A.bottomRightCorner<3,4>(), A.bottomRightCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( A.topRightCorner<3,4>(), A.topRightCorner(fix<3>,fix<4>)) ); + + VERIFY( is_same_eq( A.leftCols<3>(), A.leftCols(fix<3>)) ); + VERIFY( is_same_eq( A.rightCols<3>(), A.rightCols(fix<3>)) ); + VERIFY( is_same_eq( A.middleCols<3>(1), A.middleCols(1,fix<3>)) ); + + VERIFY( is_same_eq( A.topRows<3>(), A.topRows(fix<3>)) ); + VERIFY( is_same_eq( A.bottomRows<3>(), A.bottomRows(fix<3>)) ); + VERIFY( is_same_eq( A.middleRows<3>(1), A.middleRows(1,fix<3>)) ); + + VERIFY( is_same_eq( a.segment<3>(1), a.segment(1,fix<3>)) ); + VERIFY( is_same_eq( a.head<3>(), a.head(fix<3>)) ); + VERIFY( is_same_eq( a.tail<3>(), a.tail(fix<3>)) ); + + const ArrayXXi& cA(A); + VERIFY( is_same_eq( cA.block<Dynamic,4>(1,1,3,4), cA.block(1,1,fix<Dynamic>(3),fix<4>)) ); + + VERIFY( is_same_eq( cA.topLeftCorner<3,4>(), cA.topLeftCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( cA.bottomLeftCorner<3,4>(), cA.bottomLeftCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( cA.bottomRightCorner<3,4>(), cA.bottomRightCorner(fix<3>,fix<4>)) ); + VERIFY( is_same_eq( cA.topRightCorner<3,4>(), cA.topRightCorner(fix<3>,fix<4>)) ); + + VERIFY( is_same_eq( cA.leftCols<3>(), cA.leftCols(fix<3>)) ); + VERIFY( is_same_eq( cA.rightCols<3>(), cA.rightCols(fix<3>)) ); + VERIFY( is_same_eq( cA.middleCols<3>(1), cA.middleCols(1,fix<3>)) ); + + VERIFY( is_same_eq( cA.topRows<3>(), cA.topRows(fix<3>)) ); + VERIFY( is_same_eq( cA.bottomRows<3>(), cA.bottomRows(fix<3>)) ); + VERIFY( is_same_eq( cA.middleRows<3>(1), cA.middleRows(1,fix<3>)) ); + } + + // Check compilation of enums as index type: + a(XX) = 1; + A(XX,YY) = 1; + // Anonymous enums only work with C++11 +#if EIGEN_HAS_CXX11 + enum { X=0, Y=1 }; + a(X) = 1; + A(X,Y) = 1; + A(XX,Y) = 1; + A(X,YY) = 1; +#endif + + // Check compilation of varying integer types as index types: + Index i = n/2; + short i_short(i); + std::size_t i_sizet(i); + VERIFY_IS_EQUAL( a(i), a.coeff(i_short) ); + VERIFY_IS_EQUAL( a(i), a.coeff(i_sizet) ); + + VERIFY_IS_EQUAL( A(i,i), A.coeff(i_short, i_short) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(i_short, i) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(i, i_short) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(i, i_sizet) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(i_sizet, i) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(i_sizet, i_short) ); + VERIFY_IS_EQUAL( A(i,i), A.coeff(5, i_sizet) ); + + // Regression test for Max{Rows,Cols}AtCompileTime + { + Matrix3i A3 = Matrix3i::Random(); + ArrayXi ind(5); ind << 1,1,1,1,1; + VERIFY_IS_EQUAL( A3(ind,ind).eval(), MatrixXi::Constant(5,5,A3(1,1)) ); + } + +} + +EIGEN_DECLARE_TEST(indexed_view) +{ +// for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1( check_indexed_view() ); + CALL_SUBTEST_2( check_indexed_view() ); + CALL_SUBTEST_3( check_indexed_view() ); +// } + + // static checks of some internals: + STATIC_CHECK(( internal::is_valid_index_type<int>::value )); + STATIC_CHECK(( internal::is_valid_index_type<unsigned int>::value )); + STATIC_CHECK(( internal::is_valid_index_type<short>::value )); + STATIC_CHECK(( internal::is_valid_index_type<std::ptrdiff_t>::value )); + STATIC_CHECK(( internal::is_valid_index_type<std::size_t>::value )); + STATIC_CHECK(( !internal::valid_indexed_view_overload<int,int>::value )); + STATIC_CHECK(( !internal::valid_indexed_view_overload<int,std::ptrdiff_t>::value )); + STATIC_CHECK(( !internal::valid_indexed_view_overload<std::ptrdiff_t,int>::value )); + STATIC_CHECK(( !internal::valid_indexed_view_overload<std::size_t,int>::value )); +}
diff --git a/test/initializer_list_construction.cpp b/test/initializer_list_construction.cpp new file mode 100644 index 0000000..0d1c6f2 --- /dev/null +++ b/test/initializer_list_construction.cpp
@@ -0,0 +1,390 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2019 David Tellenbach <david.tellenbach@tellnotes.org> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_NO_STATIC_ASSERT + +#include "main.h" + +template<typename Scalar, bool is_integer = NumTraits<Scalar>::IsInteger> +struct TestMethodDispatching { + static void run() {} +}; + +template<typename Scalar> +struct TestMethodDispatching<Scalar, 1> { + static void run() + { + { + Matrix<Scalar, Dynamic, Dynamic> m {3, 4}; + Array<Scalar, Dynamic, Dynamic> a {3, 4}; + VERIFY(m.rows() == 3); + VERIFY(m.cols() == 4); + VERIFY(a.rows() == 3); + VERIFY(a.cols() == 4); + } + { + Matrix<Scalar, 1, 2> m {3, 4}; + Array<Scalar, 1, 2> a {3, 4}; + VERIFY(m(0) == 3); + VERIFY(m(1) == 4); + VERIFY(a(0) == 3); + VERIFY(a(1) == 4); + } + { + Matrix<Scalar, 2, 1> m {3, 4}; + Array<Scalar, 2, 1> a {3, 4}; + VERIFY(m(0) == 3); + VERIFY(m(1) == 4); + VERIFY(a(0) == 3); + VERIFY(a(1) == 4); + } + } +}; + +template<typename Vec4, typename Vec5> void fixedsizeVariadicVectorConstruction2() +{ + { + Vec4 ref = Vec4::Random(); + Vec4 v{ ref[0], ref[1], ref[2], ref[3] }; + VERIFY_IS_APPROX(v, ref); + VERIFY_IS_APPROX(v, (Vec4( ref[0], ref[1], ref[2], ref[3] ))); + VERIFY_IS_APPROX(v, (Vec4({ref[0], ref[1], ref[2], ref[3]}))); + + Vec4 v2 = { ref[0], ref[1], ref[2], ref[3] }; + VERIFY_IS_APPROX(v2, ref); + } + { + Vec5 ref = Vec5::Random(); + Vec5 v{ ref[0], ref[1], ref[2], ref[3], ref[4] }; + VERIFY_IS_APPROX(v, ref); + VERIFY_IS_APPROX(v, (Vec5( ref[0], ref[1], ref[2], ref[3], ref[4] ))); + VERIFY_IS_APPROX(v, (Vec5({ref[0], ref[1], ref[2], ref[3], ref[4]}))); + + Vec5 v2 = { ref[0], ref[1], ref[2], ref[3], ref[4] }; + VERIFY_IS_APPROX(v2, ref); + } +} + +#define CHECK_MIXSCALAR_V5_APPROX(V, A0, A1, A2, A3, A4) { \ + VERIFY_IS_APPROX(V[0], Scalar(A0) ); \ + VERIFY_IS_APPROX(V[1], Scalar(A1) ); \ + VERIFY_IS_APPROX(V[2], Scalar(A2) ); \ + VERIFY_IS_APPROX(V[3], Scalar(A3) ); \ + VERIFY_IS_APPROX(V[4], Scalar(A4) ); \ +} + +#define CHECK_MIXSCALAR_V5(VEC5, A0, A1, A2, A3, A4) { \ + typedef VEC5::Scalar Scalar; \ + VEC5 v = { A0 , A1 , A2 , A3 , A4 }; \ + CHECK_MIXSCALAR_V5_APPROX(v, A0 , A1 , A2 , A3 , A4); \ +} + +template<int> void fixedsizeVariadicVectorConstruction3() +{ + typedef Matrix<double,5,1> Vec5; + typedef Array<float,5,1> Arr5; + CHECK_MIXSCALAR_V5(Vec5, 1, 2., -3, 4.121, 5.53252); + CHECK_MIXSCALAR_V5(Arr5, 1, 2., 3.12f, 4.121, 5.53252); +} + +template<typename Scalar> void fixedsizeVariadicVectorConstruction() +{ + CALL_SUBTEST(( fixedsizeVariadicVectorConstruction2<Matrix<Scalar,4,1>, Matrix<Scalar,5,1> >() )); + CALL_SUBTEST(( fixedsizeVariadicVectorConstruction2<Matrix<Scalar,1,4>, Matrix<Scalar,1,5> >() )); + CALL_SUBTEST(( fixedsizeVariadicVectorConstruction2<Array<Scalar,4,1>, Array<Scalar,5,1> >() )); + CALL_SUBTEST(( fixedsizeVariadicVectorConstruction2<Array<Scalar,1,4>, Array<Scalar,1,5> >() )); +} + + +template<typename Scalar> void initializerListVectorConstruction() +{ + Scalar raw[4]; + for(int k = 0; k < 4; ++k) { + raw[k] = internal::random<Scalar>(); + } + { + Matrix<Scalar, 4, 1> m { {raw[0]}, {raw[1]},{raw[2]},{raw[3]} }; + Array<Scalar, 4, 1> a { {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }; + for(int k = 0; k < 4; ++k) { + VERIFY(m(k) == raw[k]); + } + for(int k = 0; k < 4; ++k) { + VERIFY(a(k) == raw[k]); + } + VERIFY_IS_EQUAL(m, (Matrix<Scalar,4,1>({ {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }))); + VERIFY((a == (Array<Scalar,4,1>({ {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }))).all()); + } + { + Matrix<Scalar, 1, 4> m { {raw[0], raw[1], raw[2], raw[3]} }; + Array<Scalar, 1, 4> a { {raw[0], raw[1], raw[2], raw[3]} }; + for(int k = 0; k < 4; ++k) { + VERIFY(m(k) == raw[k]); + } + for(int k = 0; k < 4; ++k) { + VERIFY(a(k) == raw[k]); + } + VERIFY_IS_EQUAL(m, (Matrix<Scalar, 1, 4>({{raw[0],raw[1],raw[2],raw[3]}}))); + VERIFY((a == (Array<Scalar, 1, 4>({{raw[0],raw[1],raw[2],raw[3]}}))).all()); + } + { + Matrix<Scalar, 4, Dynamic> m { {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }; + Array<Scalar, 4, Dynamic> a { {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }; + for(int k=0; k < 4; ++k) { + VERIFY(m(k) == raw[k]); + } + for(int k=0; k < 4; ++k) { + VERIFY(a(k) == raw[k]); + } + VERIFY_IS_EQUAL(m, (Matrix<Scalar, 4, Dynamic>({ {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }))); + VERIFY((a == (Array<Scalar, 4, Dynamic>({ {raw[0]}, {raw[1]}, {raw[2]}, {raw[3]} }))).all()); + } + { + Matrix<Scalar, Dynamic, 4> m {{raw[0],raw[1],raw[2],raw[3]}}; + Array<Scalar, Dynamic, 4> a {{raw[0],raw[1],raw[2],raw[3]}}; + for(int k=0; k < 4; ++k) { + VERIFY(m(k) == raw[k]); + } + for(int k=0; k < 4; ++k) { + VERIFY(a(k) == raw[k]); + } + VERIFY_IS_EQUAL(m, (Matrix<Scalar, Dynamic, 4>({{raw[0],raw[1],raw[2],raw[3]}}))); + VERIFY((a == (Array<Scalar, Dynamic, 4>({{raw[0],raw[1],raw[2],raw[3]}}))).all()); + } +} + +template<typename Scalar> void initializerListMatrixConstruction() +{ + const Index RowsAtCompileTime = 5; + const Index ColsAtCompileTime = 4; + const Index SizeAtCompileTime = RowsAtCompileTime * ColsAtCompileTime; + + Scalar raw[SizeAtCompileTime]; + for (int i = 0; i < SizeAtCompileTime; ++i) { + raw[i] = internal::random<Scalar>(); + } + { + Matrix<Scalar, Dynamic, Dynamic> m {}; + VERIFY(m.cols() == 0); + VERIFY(m.rows() == 0); + VERIFY_IS_EQUAL(m, (Matrix<Scalar, Dynamic, Dynamic>())); + } + { + Matrix<Scalar, 5, 4> m { + {raw[0], raw[1], raw[2], raw[3]}, + {raw[4], raw[5], raw[6], raw[7]}, + {raw[8], raw[9], raw[10], raw[11]}, + {raw[12], raw[13], raw[14], raw[15]}, + {raw[16], raw[17], raw[18], raw[19]} + }; + + Matrix<Scalar, 5, 4> m2; + m2 << raw[0], raw[1], raw[2], raw[3], + raw[4], raw[5], raw[6], raw[7], + raw[8], raw[9], raw[10], raw[11], + raw[12], raw[13], raw[14], raw[15], + raw[16], raw[17], raw[18], raw[19]; + + int k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + VERIFY(m(i, j) == raw[k]); + ++k; + } + } + VERIFY_IS_EQUAL(m, m2); + } + { + Matrix<Scalar, Dynamic, Dynamic> m{ + {raw[0], raw[1], raw[2], raw[3]}, + {raw[4], raw[5], raw[6], raw[7]}, + {raw[8], raw[9], raw[10], raw[11]}, + {raw[12], raw[13], raw[14], raw[15]}, + {raw[16], raw[17], raw[18], raw[19]} + }; + + VERIFY(m.cols() == 4); + VERIFY(m.rows() == 5); + int k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + VERIFY(m(i, j) == raw[k]); + ++k; + } + } + + Matrix<Scalar, Dynamic, Dynamic> m2(RowsAtCompileTime, ColsAtCompileTime); + k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + m2(i, j) = raw[k]; + ++k; + } + } + VERIFY_IS_EQUAL(m, m2); + } +} + +template<typename Scalar> void initializerListArrayConstruction() +{ + const Index RowsAtCompileTime = 5; + const Index ColsAtCompileTime = 4; + const Index SizeAtCompileTime = RowsAtCompileTime * ColsAtCompileTime; + + Scalar raw[SizeAtCompileTime]; + for (int i = 0; i < SizeAtCompileTime; ++i) { + raw[i] = internal::random<Scalar>(); + } + { + Array<Scalar, Dynamic, Dynamic> a {}; + VERIFY(a.cols() == 0); + VERIFY(a.rows() == 0); + } + { + Array<Scalar, 5, 4> m { + {raw[0], raw[1], raw[2], raw[3]}, + {raw[4], raw[5], raw[6], raw[7]}, + {raw[8], raw[9], raw[10], raw[11]}, + {raw[12], raw[13], raw[14], raw[15]}, + {raw[16], raw[17], raw[18], raw[19]} + }; + + Array<Scalar, 5, 4> m2; + m2 << raw[0], raw[1], raw[2], raw[3], + raw[4], raw[5], raw[6], raw[7], + raw[8], raw[9], raw[10], raw[11], + raw[12], raw[13], raw[14], raw[15], + raw[16], raw[17], raw[18], raw[19]; + + int k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + VERIFY(m(i, j) == raw[k]); + ++k; + } + } + VERIFY_IS_APPROX(m, m2); + } + { + Array<Scalar, Dynamic, Dynamic> m { + {raw[0], raw[1], raw[2], raw[3]}, + {raw[4], raw[5], raw[6], raw[7]}, + {raw[8], raw[9], raw[10], raw[11]}, + {raw[12], raw[13], raw[14], raw[15]}, + {raw[16], raw[17], raw[18], raw[19]} + }; + + VERIFY(m.cols() == 4); + VERIFY(m.rows() == 5); + int k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + VERIFY(m(i, j) == raw[k]); + ++k; + } + } + + Array<Scalar, Dynamic, Dynamic> m2(RowsAtCompileTime, ColsAtCompileTime); + k = 0; + for(int i = 0; i < RowsAtCompileTime; ++i) { + for (int j = 0; j < ColsAtCompileTime; ++j) { + m2(i, j) = raw[k]; + ++k; + } + } + VERIFY_IS_APPROX(m, m2); + } +} + +template<typename Scalar> void dynamicVectorConstruction() +{ + const Index size = 4; + Scalar raw[size]; + for (int i = 0; i < size; ++i) { + raw[i] = internal::random<Scalar>(); + } + + typedef Matrix<Scalar, Dynamic, 1> VectorX; + + { + VectorX v {{raw[0], raw[1], raw[2], raw[3]}}; + for (int i = 0; i < size; ++i) { + VERIFY(v(i) == raw[i]); + } + VERIFY(v.rows() == size); + VERIFY(v.cols() == 1); + VERIFY_IS_EQUAL(v, (VectorX {{raw[0], raw[1], raw[2], raw[3]}})); + } + + { + VERIFY_RAISES_ASSERT((VectorX {raw[0], raw[1], raw[2], raw[3]})); + } + { + VERIFY_RAISES_ASSERT((VectorX { + {raw[0], raw[1], raw[2], raw[3]}, + {raw[0], raw[1], raw[2], raw[3]}, + })); + } +} + +EIGEN_DECLARE_TEST(initializer_list_construction) +{ + CALL_SUBTEST_1(initializerListVectorConstruction<unsigned char>()); + CALL_SUBTEST_1(initializerListVectorConstruction<float>()); + CALL_SUBTEST_1(initializerListVectorConstruction<double>()); + CALL_SUBTEST_1(initializerListVectorConstruction<int>()); + CALL_SUBTEST_1(initializerListVectorConstruction<long int>()); + CALL_SUBTEST_1(initializerListVectorConstruction<std::ptrdiff_t>()); + CALL_SUBTEST_1(initializerListVectorConstruction<std::complex<int>>()); + CALL_SUBTEST_1(initializerListVectorConstruction<std::complex<double>>()); + CALL_SUBTEST_1(initializerListVectorConstruction<std::complex<float>>()); + + CALL_SUBTEST_2(initializerListMatrixConstruction<unsigned char>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<float>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<double>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<int>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<long int>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<std::ptrdiff_t>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<std::complex<int>>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<std::complex<double>>()); + CALL_SUBTEST_2(initializerListMatrixConstruction<std::complex<float>>()); + + CALL_SUBTEST_3(initializerListArrayConstruction<unsigned char>()); + CALL_SUBTEST_3(initializerListArrayConstruction<float>()); + CALL_SUBTEST_3(initializerListArrayConstruction<double>()); + CALL_SUBTEST_3(initializerListArrayConstruction<int>()); + CALL_SUBTEST_3(initializerListArrayConstruction<long int>()); + CALL_SUBTEST_3(initializerListArrayConstruction<std::ptrdiff_t>()); + CALL_SUBTEST_3(initializerListArrayConstruction<std::complex<int>>()); + CALL_SUBTEST_3(initializerListArrayConstruction<std::complex<double>>()); + CALL_SUBTEST_3(initializerListArrayConstruction<std::complex<float>>()); + + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<unsigned char>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<float>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<double>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<int>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<long int>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<std::ptrdiff_t>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<std::complex<int>>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<std::complex<double>>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction<std::complex<float>>()); + CALL_SUBTEST_4(fixedsizeVariadicVectorConstruction3<0>()); + + CALL_SUBTEST_5(TestMethodDispatching<int>::run()); + CALL_SUBTEST_5(TestMethodDispatching<long int>::run()); + + CALL_SUBTEST_6(dynamicVectorConstruction<unsigned char>()); + CALL_SUBTEST_6(dynamicVectorConstruction<float>()); + CALL_SUBTEST_6(dynamicVectorConstruction<double>()); + CALL_SUBTEST_6(dynamicVectorConstruction<int>()); + CALL_SUBTEST_6(dynamicVectorConstruction<long int>()); + CALL_SUBTEST_6(dynamicVectorConstruction<std::ptrdiff_t>()); + CALL_SUBTEST_6(dynamicVectorConstruction<std::complex<int>>()); + CALL_SUBTEST_6(dynamicVectorConstruction<std::complex<double>>()); + CALL_SUBTEST_6(dynamicVectorConstruction<std::complex<float>>()); +} \ No newline at end of file
diff --git a/test/inplace_decomposition.cpp b/test/inplace_decomposition.cpp new file mode 100644 index 0000000..e3aa995 --- /dev/null +++ b/test/inplace_decomposition.cpp
@@ -0,0 +1,110 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" +#include <Eigen/LU> +#include <Eigen/Cholesky> +#include <Eigen/QR> + +// This file test inplace decomposition through Ref<>, as supported by Cholesky, LU, and QR decompositions. + +template<typename DecType,typename MatrixType> void inplace(bool square = false, bool SPD = false) +{ + typedef typename MatrixType::Scalar Scalar; + typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> RhsType; + typedef Matrix<Scalar, MatrixType::ColsAtCompileTime, 1> ResType; + + Index rows = MatrixType::RowsAtCompileTime==Dynamic ? internal::random<Index>(2,EIGEN_TEST_MAX_SIZE/2) : Index(MatrixType::RowsAtCompileTime); + Index cols = MatrixType::ColsAtCompileTime==Dynamic ? (square?rows:internal::random<Index>(2,rows)) : Index(MatrixType::ColsAtCompileTime); + + MatrixType A = MatrixType::Random(rows,cols); + RhsType b = RhsType::Random(rows); + ResType x(cols); + + if(SPD) + { + assert(square); + A.topRows(cols) = A.topRows(cols).adjoint() * A.topRows(cols); + A.diagonal().array() += 1e-3; + } + + MatrixType A0 = A; + MatrixType A1 = A; + + DecType dec(A); + + // Check that the content of A has been modified + VERIFY_IS_NOT_APPROX( A, A0 ); + + // Check that the decomposition is correct: + if(rows==cols) + { + VERIFY_IS_APPROX( A0 * (x = dec.solve(b)), b ); + } + else + { + VERIFY_IS_APPROX( A0.transpose() * A0 * (x = dec.solve(b)), A0.transpose() * b ); + } + + // Check that modifying A breaks the current dec: + A.setRandom(); + if(rows==cols) + { + VERIFY_IS_NOT_APPROX( A0 * (x = dec.solve(b)), b ); + } + else + { + VERIFY_IS_NOT_APPROX( A0.transpose() * A0 * (x = dec.solve(b)), A0.transpose() * b ); + } + + // Check that calling compute(A1) does not modify A1: + A = A0; + dec.compute(A1); + VERIFY_IS_EQUAL(A0,A1); + VERIFY_IS_NOT_APPROX( A, A0 ); + if(rows==cols) + { + VERIFY_IS_APPROX( A0 * (x = dec.solve(b)), b ); + } + else + { + VERIFY_IS_APPROX( A0.transpose() * A0 * (x = dec.solve(b)), A0.transpose() * b ); + } +} + + +EIGEN_DECLARE_TEST(inplace_decomposition) +{ + EIGEN_UNUSED typedef Matrix<double,4,3> Matrix43d; + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1(( inplace<LLT<Ref<MatrixXd> >, MatrixXd>(true,true) )); + CALL_SUBTEST_1(( inplace<LLT<Ref<Matrix4d> >, Matrix4d>(true,true) )); + + CALL_SUBTEST_2(( inplace<LDLT<Ref<MatrixXd> >, MatrixXd>(true,true) )); + CALL_SUBTEST_2(( inplace<LDLT<Ref<Matrix4d> >, Matrix4d>(true,true) )); + + CALL_SUBTEST_3(( inplace<PartialPivLU<Ref<MatrixXd> >, MatrixXd>(true,false) )); + CALL_SUBTEST_3(( inplace<PartialPivLU<Ref<Matrix4d> >, Matrix4d>(true,false) )); + + CALL_SUBTEST_4(( inplace<FullPivLU<Ref<MatrixXd> >, MatrixXd>(true,false) )); + CALL_SUBTEST_4(( inplace<FullPivLU<Ref<Matrix4d> >, Matrix4d>(true,false) )); + + CALL_SUBTEST_5(( inplace<HouseholderQR<Ref<MatrixXd> >, MatrixXd>(false,false) )); + CALL_SUBTEST_5(( inplace<HouseholderQR<Ref<Matrix43d> >, Matrix43d>(false,false) )); + + CALL_SUBTEST_6(( inplace<ColPivHouseholderQR<Ref<MatrixXd> >, MatrixXd>(false,false) )); + CALL_SUBTEST_6(( inplace<ColPivHouseholderQR<Ref<Matrix43d> >, Matrix43d>(false,false) )); + + CALL_SUBTEST_7(( inplace<FullPivHouseholderQR<Ref<MatrixXd> >, MatrixXd>(false,false) )); + CALL_SUBTEST_7(( inplace<FullPivHouseholderQR<Ref<Matrix43d> >, Matrix43d>(false,false) )); + + CALL_SUBTEST_8(( inplace<CompleteOrthogonalDecomposition<Ref<MatrixXd> >, MatrixXd>(false,false) )); + CALL_SUBTEST_8(( inplace<CompleteOrthogonalDecomposition<Ref<Matrix43d> >, Matrix43d>(false,false) )); + } +}
diff --git a/test/integer_types.cpp b/test/integer_types.cpp index 950f8e9..3f9030d 100644 --- a/test/integer_types.cpp +++ b/test/integer_types.cpp
@@ -18,7 +18,6 @@ template<typename MatrixType> void signed_integer_type_tests(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; enum { is_signed = (Scalar(-1) > Scalar(0)) ? 0 : 1 }; @@ -49,7 +48,6 @@ template<typename MatrixType> void integer_type_tests(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; VERIFY(NumTraits<Scalar>::IsInteger); @@ -133,7 +131,18 @@ VERIFY_IS_APPROX((m1 * m2.transpose()) * m1, m1 * (m2.transpose() * m1)); } -void test_integer_types() +template<int> +void integer_types_extra() +{ + VERIFY_IS_EQUAL(int(internal::scalar_div_cost<int>::value), 8); + VERIFY_IS_EQUAL(int(internal::scalar_div_cost<unsigned int>::value), 8); + if(sizeof(long)>sizeof(int)) { + VERIFY(int(internal::scalar_div_cost<long>::value) > int(internal::scalar_div_cost<int>::value)); + VERIFY(int(internal::scalar_div_cost<unsigned long>::value) > int(internal::scalar_div_cost<int>::value)); + } +} + +EIGEN_DECLARE_TEST(integer_types) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( integer_type_tests(Matrix<unsigned int, 1, 1>()) ); @@ -158,4 +167,5 @@ CALL_SUBTEST_8( integer_type_tests(Matrix<unsigned long long, Dynamic, 5>(1, 5)) ); } + CALL_SUBTEST_9( integer_types_extra<0>() ); }
diff --git a/test/inverse.cpp b/test/inverse.cpp index 5c6777a..99f9e0c 100644 --- a/test/inverse.cpp +++ b/test/inverse.cpp
@@ -11,10 +11,59 @@ #include "main.h" #include <Eigen/LU> -template<typename MatrixType> void inverse(const MatrixType& m) +template<typename MatrixType> +void inverse_for_fixed_size(const MatrixType&, typename internal::enable_if<MatrixType::SizeAtCompileTime==Dynamic>::type* = 0) +{ +} + +template<typename MatrixType> +void inverse_for_fixed_size(const MatrixType& m1, typename internal::enable_if<MatrixType::SizeAtCompileTime!=Dynamic>::type* = 0) { using std::abs; - typedef typename MatrixType::Index Index; + + MatrixType m2, identity = MatrixType::Identity(); + + typedef typename MatrixType::Scalar Scalar; + typedef typename NumTraits<Scalar>::Real RealScalar; + typedef Matrix<Scalar, MatrixType::ColsAtCompileTime, 1> VectorType; + + //computeInverseAndDetWithCheck tests + //First: an invertible matrix + bool invertible; + Scalar det; + + m2.setZero(); + m1.computeInverseAndDetWithCheck(m2, det, invertible); + VERIFY(invertible); + VERIFY_IS_APPROX(identity, m1*m2); + VERIFY_IS_APPROX(det, m1.determinant()); + + m2.setZero(); + m1.computeInverseWithCheck(m2, invertible); + VERIFY(invertible); + VERIFY_IS_APPROX(identity, m1*m2); + + //Second: a rank one matrix (not invertible, except for 1x1 matrices) + VectorType v3 = VectorType::Random(); + MatrixType m3 = v3*v3.transpose(), m4; + m3.computeInverseAndDetWithCheck(m4, det, invertible); + VERIFY( m1.rows()==1 ? invertible : !invertible ); + VERIFY_IS_MUCH_SMALLER_THAN(abs(det-m3.determinant()), RealScalar(1)); + m3.computeInverseWithCheck(m4, invertible); + VERIFY( m1.rows()==1 ? invertible : !invertible ); + + // check with submatrices + { + Matrix<Scalar, MatrixType::RowsAtCompileTime+1, MatrixType::RowsAtCompileTime+1, MatrixType::Options> m5; + m5.setRandom(); + m5.topLeftCorner(m1.rows(),m1.rows()) = m1; + m2 = m5.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>().inverse(); + VERIFY_IS_APPROX( (m5.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>()), m2.inverse() ); + } +} + +template<typename MatrixType> void inverse(const MatrixType& m) +{ /* this test covers the following files: Inverse.h */ @@ -40,44 +89,7 @@ // since for the general case we implement separately row-major and col-major, test that VERIFY_IS_APPROX(MatrixType(m1.transpose().inverse()), MatrixType(m1.inverse().transpose())); -#if !defined(EIGEN_TEST_PART_5) && !defined(EIGEN_TEST_PART_6) - typedef typename NumTraits<Scalar>::Real RealScalar; - typedef Matrix<Scalar, MatrixType::ColsAtCompileTime, 1> VectorType; - - //computeInverseAndDetWithCheck tests - //First: an invertible matrix - bool invertible; - RealScalar det; - - m2.setZero(); - m1.computeInverseAndDetWithCheck(m2, det, invertible); - VERIFY(invertible); - VERIFY_IS_APPROX(identity, m1*m2); - VERIFY_IS_APPROX(det, m1.determinant()); - - m2.setZero(); - m1.computeInverseWithCheck(m2, invertible); - VERIFY(invertible); - VERIFY_IS_APPROX(identity, m1*m2); - - //Second: a rank one matrix (not invertible, except for 1x1 matrices) - VectorType v3 = VectorType::Random(rows); - MatrixType m3 = v3*v3.transpose(), m4(rows,cols); - m3.computeInverseAndDetWithCheck(m4, det, invertible); - VERIFY( rows==1 ? invertible : !invertible ); - VERIFY_IS_MUCH_SMALLER_THAN(abs(det-m3.determinant()), RealScalar(1)); - m3.computeInverseWithCheck(m4, invertible); - VERIFY( rows==1 ? invertible : !invertible ); - - // check with submatrices - { - Matrix<Scalar, MatrixType::RowsAtCompileTime+1, MatrixType::RowsAtCompileTime+1, MatrixType::Options> m5; - m5.setRandom(); - m5.topLeftCorner(rows,rows) = m1; - m2 = m5.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>().inverse(); - VERIFY_IS_APPROX( (m5.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>()), m2.inverse() ); - } -#endif + inverse_for_fixed_size(m1); // check in-place inversion if(MatrixType::RowsAtCompileTime>=2 && MatrixType::RowsAtCompileTime<=4) @@ -93,7 +105,23 @@ } } -void test_inverse() +template<typename Scalar> +void inverse_zerosized() +{ + Matrix<Scalar,Dynamic,Dynamic> A(0,0); + { + Matrix<Scalar,0,1> b, x; + x = A.inverse() * b; + } + { + Matrix<Scalar,Dynamic,Dynamic> b(0,1), x; + x = A.inverse() * b; + VERIFY_IS_EQUAL(x.rows(), 0); + VERIFY_IS_EQUAL(x.cols(), 1); + } +} + +EIGEN_DECLARE_TEST(inverse) { int s = 0; for(int i = 0; i < g_repeat; i++) { @@ -106,6 +134,7 @@ s = internal::random<int>(50,320); CALL_SUBTEST_5( inverse(MatrixXf(s,s)) ); TEST_SET_BUT_UNUSED_VARIABLE(s) + CALL_SUBTEST_5( inverse_zerosized<float>() ); s = internal::random<int>(25,100); CALL_SUBTEST_6( inverse(MatrixXcd(s,s)) ); @@ -113,5 +142,7 @@ CALL_SUBTEST_7( inverse(Matrix4d()) ); CALL_SUBTEST_7( inverse(Matrix<double,4,4,DontAlign>()) ); + + CALL_SUBTEST_8( inverse(Matrix4cd()) ); } }
diff --git a/test/is_same_dense.cpp b/test/is_same_dense.cpp index 6d7904b..23dd806 100644 --- a/test/is_same_dense.cpp +++ b/test/is_same_dense.cpp
@@ -9,12 +9,18 @@ #include "main.h" -void test_is_same_dense() +using internal::is_same_dense; + +EIGEN_DECLARE_TEST(is_same_dense) { typedef Matrix<double,Dynamic,Dynamic,ColMajor> ColMatrixXd; + typedef Matrix<std::complex<double>,Dynamic,Dynamic,ColMajor> ColMatrixXcd; ColMatrixXd m1(10,10); + ColMatrixXcd m2(10,10); Ref<ColMatrixXd> ref_m1(m1); + Ref<ColMatrixXd,0, Stride<Dynamic,Dynamic> > ref_m2_real(m2.real()); Ref<const ColMatrixXd> const_ref_m1(m1); + VERIFY(is_same_dense(m1,m1)); VERIFY(is_same_dense(m1,ref_m1)); VERIFY(is_same_dense(const_ref_m1,m1)); @@ -28,4 +34,8 @@ Ref<const ColMatrixXd> const_ref_m1_col(m1.col(1)); VERIFY(is_same_dense(m1.col(1),const_ref_m1_col)); + + + VERIFY(!is_same_dense(m1, ref_m2_real)); + VERIFY(!is_same_dense(m2, ref_m2_real)); }
diff --git a/test/jacobi.cpp b/test/jacobi.cpp index 7ccd412..5604797 100644 --- a/test/jacobi.cpp +++ b/test/jacobi.cpp
@@ -14,7 +14,6 @@ template<typename MatrixType, typename JacobiScalar> void jacobi(const MatrixType& m = MatrixType()) { - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -58,7 +57,7 @@ } } -void test_jacobi() +EIGEN_DECLARE_TEST(jacobi) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(( jacobi<Matrix3f, float>() ));
diff --git a/test/jacobisvd.cpp b/test/jacobisvd.cpp index 3d8d020..89484d9 100644 --- a/test/jacobisvd.cpp +++ b/test/jacobisvd.cpp
@@ -36,7 +36,6 @@ template<typename MatrixType> void jacobisvd_verify_assert(const MatrixType& m) { svd_verify_assert<JacobiSVD<MatrixType> >(m); - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -68,9 +67,26 @@ VERIFY_RAISES_ASSERT(m.jacobiSvd().matrixU()); VERIFY_RAISES_ASSERT(m.jacobiSvd().matrixV()); VERIFY_IS_APPROX(m.jacobiSvd(ComputeFullU|ComputeFullV).solve(m), m); + VERIFY_IS_APPROX(m.jacobiSvd(ComputeFullU|ComputeFullV).transpose().solve(m), m); + VERIFY_IS_APPROX(m.jacobiSvd(ComputeFullU|ComputeFullV).adjoint().solve(m), m); } -void test_jacobisvd() +namespace Foo { +// older compiler require a default constructor for Bar +// cf: https://stackoverflow.com/questions/7411515/ +class Bar {public: Bar() {}}; +bool operator<(const Bar&, const Bar&) { return true; } +} +// regression test for a very strange MSVC issue for which simply +// including SVDBase.h messes up with std::max and custom scalar type +void msvc_workaround() +{ + const Foo::Bar a; + const Foo::Bar b; + std::max EIGEN_NOT_A_MACRO (a,b); +} + +EIGEN_DECLARE_TEST(jacobisvd) { CALL_SUBTEST_3(( jacobisvd_verify_assert(Matrix3f()) )); CALL_SUBTEST_4(( jacobisvd_verify_assert(Matrix4d()) )); @@ -101,6 +117,12 @@ // Test on inf/nan matrix CALL_SUBTEST_7( (svd_inf_nan<JacobiSVD<MatrixXf>, MatrixXf>()) ); CALL_SUBTEST_10( (svd_inf_nan<JacobiSVD<MatrixXd>, MatrixXd>()) ); + + // bug1395 test compile-time vectors as input + CALL_SUBTEST_13(( jacobisvd_verify_assert(Matrix<double,6,1>()) )); + CALL_SUBTEST_13(( jacobisvd_verify_assert(Matrix<double,1,6>()) )); + CALL_SUBTEST_13(( jacobisvd_verify_assert(Matrix<double,Dynamic,1>(r)) )); + CALL_SUBTEST_13(( jacobisvd_verify_assert(Matrix<double,1,Dynamic>(c)) )); } CALL_SUBTEST_7(( jacobisvd<MatrixXf>(MatrixXf(internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE/2), internal::random<int>(EIGEN_TEST_MAX_SIZE/4, EIGEN_TEST_MAX_SIZE/2))) )); @@ -117,4 +139,6 @@ CALL_SUBTEST_9( svd_preallocate<void>() ); CALL_SUBTEST_2( svd_underoverflow<void>() ); + + msvc_workaround(); }
diff --git a/test/klu_support.cpp b/test/klu_support.cpp new file mode 100644 index 0000000..f806ad5 --- /dev/null +++ b/test/klu_support.cpp
@@ -0,0 +1,32 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2011 Gael Guennebaud <g.gael@free.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_NO_DEBUG_SMALL_PRODUCT_BLOCKS +#include "sparse_solver.h" + +#include <Eigen/KLUSupport> + +template<typename T> void test_klu_support_T() +{ + KLU<SparseMatrix<T, ColMajor> > klu_colmajor; + KLU<SparseMatrix<T, RowMajor> > klu_rowmajor; + + check_sparse_square_solving(klu_colmajor); + check_sparse_square_solving(klu_rowmajor); + + //check_sparse_square_determinant(umfpack_colmajor); + //check_sparse_square_determinant(umfpack_rowmajor); +} + +EIGEN_DECLARE_TEST(klu_support) +{ + CALL_SUBTEST_1(test_klu_support_T<double>()); + CALL_SUBTEST_2(test_klu_support_T<std::complex<double> >()); +} +
diff --git a/test/linearstructure.cpp b/test/linearstructure.cpp index 292f339..46ee516 100644 --- a/test/linearstructure.cpp +++ b/test/linearstructure.cpp
@@ -9,7 +9,7 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. static bool g_called; -#define EIGEN_SPECIAL_SCALAR_MULTIPLE_PLUGIN { g_called = true; } +#define EIGEN_SCALAR_BINARY_OP_PLUGIN { g_called |= (!internal::is_same<LhsScalar,RhsScalar>::value); } #include "main.h" @@ -19,8 +19,8 @@ /* this test covers the following files: CwiseUnaryOp.h, CwiseBinaryOp.h, SelfCwiseBinaryOp.h */ - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; Index rows = m.rows(); Index cols = m.cols(); @@ -32,7 +32,7 @@ m3(rows, cols); Scalar s1 = internal::random<Scalar>(); - while (abs(s1)<1e-3) s1 = internal::random<Scalar>(); + while (abs(s1)<RealScalar(1e-3)) s1 = internal::random<Scalar>(); Index r = internal::random<Index>(0, rows-1), c = internal::random<Index>(0, cols-1); @@ -92,9 +92,38 @@ g_called = false; VERIFY_IS_APPROX(m1/s, m1/Scalar(s)); VERIFY(g_called && "matrix<complex> / real not properly optimized"); + + g_called = false; + VERIFY_IS_APPROX(s+m1.array(), Scalar(s)+m1.array()); + VERIFY(g_called && "real + matrix<complex> not properly optimized"); + + g_called = false; + VERIFY_IS_APPROX(m1.array()+s, m1.array()+Scalar(s)); + VERIFY(g_called && "matrix<complex> + real not properly optimized"); + + g_called = false; + VERIFY_IS_APPROX(s-m1.array(), Scalar(s)-m1.array()); + VERIFY(g_called && "real - matrix<complex> not properly optimized"); + + g_called = false; + VERIFY_IS_APPROX(m1.array()-s, m1.array()-Scalar(s)); + VERIFY(g_called && "matrix<complex> - real not properly optimized"); } -void test_linearstructure() +template<int> +void linearstructure_overflow() +{ + // make sure that /=scalar and /scalar do not overflow + // rational: 1.0/4.94e-320 overflow, but m/4.94e-320 should not + Matrix4d m2, m3; + m3 = m2 = Matrix4d::Random()*1e-20; + m2 = m2 / 4.9e-320; + VERIFY_IS_APPROX(m2.cwiseQuotient(m2), Matrix4d::Ones()); + m3 /= 4.9e-320; + VERIFY_IS_APPROX(m3.cwiseQuotient(m3), Matrix4d::Ones()); +} + +EIGEN_DECLARE_TEST(linearstructure) { g_called = true; VERIFY(g_called); // avoid `unneeded-internal-declaration` warning. @@ -114,19 +143,5 @@ CALL_SUBTEST_11( real_complex<MatrixXcf>(10,10) ); CALL_SUBTEST_11( real_complex<ArrayXXcf>(10,10) ); } - -#ifdef EIGEN_TEST_PART_4 - { - // make sure that /=scalar and /scalar do not overflow - // rational: 1.0/4.94e-320 overflow, but m/4.94e-320 should not - Matrix4d m2, m3; - m3 = m2 = Matrix4d::Random()*1e-20; - m2 = m2 / 4.9e-320; - VERIFY_IS_APPROX(m2.cwiseQuotient(m2), Matrix4d::Ones()); - m3 /= 4.9e-320; - VERIFY_IS_APPROX(m3.cwiseQuotient(m3), Matrix4d::Ones()); - - - } -#endif + CALL_SUBTEST_4( linearstructure_overflow<0>() ); }
diff --git a/test/lscg.cpp b/test/lscg.cpp index daa62a9..feb2347 100644 --- a/test/lscg.cpp +++ b/test/lscg.cpp
@@ -14,15 +14,23 @@ { LeastSquaresConjugateGradient<SparseMatrix<T> > lscg_colmajor_diag; LeastSquaresConjugateGradient<SparseMatrix<T>, IdentityPreconditioner> lscg_colmajor_I; + LeastSquaresConjugateGradient<SparseMatrix<T,RowMajor> > lscg_rowmajor_diag; + LeastSquaresConjugateGradient<SparseMatrix<T,RowMajor>, IdentityPreconditioner> lscg_rowmajor_I; CALL_SUBTEST( check_sparse_square_solving(lscg_colmajor_diag) ); CALL_SUBTEST( check_sparse_square_solving(lscg_colmajor_I) ); CALL_SUBTEST( check_sparse_leastsquare_solving(lscg_colmajor_diag) ); CALL_SUBTEST( check_sparse_leastsquare_solving(lscg_colmajor_I) ); + + CALL_SUBTEST( check_sparse_square_solving(lscg_rowmajor_diag) ); + CALL_SUBTEST( check_sparse_square_solving(lscg_rowmajor_I) ); + + CALL_SUBTEST( check_sparse_leastsquare_solving(lscg_rowmajor_diag) ); + CALL_SUBTEST( check_sparse_leastsquare_solving(lscg_rowmajor_I) ); } -void test_lscg() +EIGEN_DECLARE_TEST(lscg) { CALL_SUBTEST_1(test_lscg_T<double>()); CALL_SUBTEST_2(test_lscg_T<std::complex<double> >());
diff --git a/test/lu.cpp b/test/lu.cpp index 9787f4d..1bbadcb 100644 --- a/test/lu.cpp +++ b/test/lu.cpp
@@ -9,6 +9,7 @@ #include "main.h" #include <Eigen/LU> +#include "solverbase.h" using namespace std; template<typename MatrixType> @@ -18,7 +19,8 @@ template<typename MatrixType> void lu_non_invertible() { - typedef typename MatrixType::Index Index; + STATIC_CHECK(( internal::is_same<typename FullPivLU<MatrixType>::StorageIndex,int>::value )); + typedef typename MatrixType::RealScalar RealScalar; /* this test covers the following files: LU.h @@ -58,6 +60,10 @@ // The image of the zero matrix should consist of a single (zero) column vector VERIFY((MatrixType::Zero(rows,cols).fullPivLu().image(MatrixType::Zero(rows,cols)).cols() == 1)); + // The kernel of the zero matrix is the entire space, and thus is an invertible matrix of dimensions cols. + KernelMatrixType kernel = MatrixType::Zero(rows,cols).fullPivLu().kernel(); + VERIFY((kernel.fullPivLu().isInvertible())); + MatrixType m1(rows, cols), m3(rows, cols2); CMatrixType m2(cols, cols2); createRandomPIMatrixOfRank(rank, rows, cols, m1); @@ -87,42 +93,24 @@ VERIFY(!lu.isInjective()); VERIFY(!lu.isInvertible()); VERIFY(!lu.isSurjective()); - VERIFY((m1 * m1kernel).isMuchSmallerThan(m1)); + VERIFY_IS_MUCH_SMALLER_THAN((m1 * m1kernel), m1); VERIFY(m1image.fullPivLu().rank() == rank); VERIFY_IS_APPROX(m1 * m1.adjoint() * m1image, m1image); + check_solverbase<CMatrixType, MatrixType>(m1, lu, rows, cols, cols2); + m2 = CMatrixType::Random(cols,cols2); m3 = m1*m2; m2 = CMatrixType::Random(cols,cols2); // test that the code, which does resize(), may be applied to an xpr m2.block(0,0,m2.rows(),m2.cols()) = lu.solve(m3); VERIFY_IS_APPROX(m3, m1*m2); - - // test solve with transposed - m3 = MatrixType::Random(rows,cols2); - m2 = m1.transpose()*m3; - m3 = MatrixType::Random(rows,cols2); - lu.template _solve_impl_transposed<false>(m2, m3); - VERIFY_IS_APPROX(m2, m1.transpose()*m3); - m3 = MatrixType::Random(rows,cols2); - m3 = lu.transpose().solve(m2); - VERIFY_IS_APPROX(m2, m1.transpose()*m3); - - // test solve with conjugate transposed - m3 = MatrixType::Random(rows,cols2); - m2 = m1.adjoint()*m3; - m3 = MatrixType::Random(rows,cols2); - lu.template _solve_impl_transposed<true>(m2, m3); - VERIFY_IS_APPROX(m2, m1.adjoint()*m3); - m3 = MatrixType::Random(rows,cols2); - m3 = lu.adjoint().solve(m2); - VERIFY_IS_APPROX(m2, m1.adjoint()*m3); } template<typename MatrixType> void lu_invertible() { /* this test covers the following files: - LU.h + FullPivLU.h */ typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; Index size = MatrixType::RowsAtCompileTime; @@ -145,10 +133,12 @@ VERIFY(lu.isSurjective()); VERIFY(lu.isInvertible()); VERIFY(lu.image(m1).fullPivLu().isInvertible()); + + check_solverbase<MatrixType, MatrixType>(m1, lu, size, size, size); + + MatrixType m1_inverse = lu.inverse(); m3 = MatrixType::Random(size,size); m2 = lu.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); - MatrixType m1_inverse = lu.inverse(); VERIFY_IS_APPROX(m2, m1_inverse*m3); RealScalar rcond = (RealScalar(1) / matrix_l1_norm(m1)) / matrix_l1_norm(m1_inverse); @@ -157,64 +147,37 @@ // truth. VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); - // test solve with transposed - lu.template _solve_impl_transposed<false>(m3, m2); - VERIFY_IS_APPROX(m3, m1.transpose()*m2); - m3 = MatrixType::Random(size,size); - m3 = lu.transpose().solve(m2); - VERIFY_IS_APPROX(m2, m1.transpose()*m3); - - // test solve with conjugate transposed - lu.template _solve_impl_transposed<true>(m3, m2); - VERIFY_IS_APPROX(m3, m1.adjoint()*m2); - m3 = MatrixType::Random(size,size); - m3 = lu.adjoint().solve(m2); - VERIFY_IS_APPROX(m2, m1.adjoint()*m3); - // Regression test for Bug 302 MatrixType m4 = MatrixType::Random(size,size); VERIFY_IS_APPROX(lu.solve(m3*m4), lu.solve(m3)*m4); } -template<typename MatrixType> void lu_partial_piv() +template<typename MatrixType> void lu_partial_piv(Index size = MatrixType::ColsAtCompileTime) { /* this test covers the following files: PartialPivLU.h */ - typedef typename MatrixType::Index Index; typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; - Index size = internal::random<Index>(1,4); MatrixType m1(size, size), m2(size, size), m3(size, size); m1.setRandom(); PartialPivLU<MatrixType> plu(m1); + STATIC_CHECK(( internal::is_same<typename PartialPivLU<MatrixType>::StorageIndex,int>::value )); + VERIFY_IS_APPROX(m1, plu.reconstructedMatrix()); + check_solverbase<MatrixType, MatrixType>(m1, plu, size, size, size); + + MatrixType m1_inverse = plu.inverse(); m3 = MatrixType::Random(size,size); m2 = plu.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); - MatrixType m1_inverse = plu.inverse(); VERIFY_IS_APPROX(m2, m1_inverse*m3); RealScalar rcond = (RealScalar(1) / matrix_l1_norm(m1)) / matrix_l1_norm(m1_inverse); const RealScalar rcond_est = plu.rcond(); // Verify that the estimate is within a factor of 10 of the truth. VERIFY(rcond_est > rcond / 10 && rcond_est < rcond * 10); - - // test solve with transposed - plu.template _solve_impl_transposed<false>(m3, m2); - VERIFY_IS_APPROX(m3, m1.transpose()*m2); - m3 = MatrixType::Random(size,size); - m3 = plu.transpose().solve(m2); - VERIFY_IS_APPROX(m2, m1.transpose()*m3); - - // test solve with conjugate transposed - plu.template _solve_impl_transposed<true>(m3, m2); - VERIFY_IS_APPROX(m3, m1.adjoint()*m2); - m3 = MatrixType::Random(size,size); - m3 = plu.adjoint().solve(m2); - VERIFY_IS_APPROX(m2, m1.adjoint()*m3); } template<typename MatrixType> void lu_verify_assert() @@ -228,6 +191,8 @@ VERIFY_RAISES_ASSERT(lu.kernel()) VERIFY_RAISES_ASSERT(lu.image(tmp)) VERIFY_RAISES_ASSERT(lu.solve(tmp)) + VERIFY_RAISES_ASSERT(lu.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(lu.adjoint().solve(tmp)) VERIFY_RAISES_ASSERT(lu.determinant()) VERIFY_RAISES_ASSERT(lu.rank()) VERIFY_RAISES_ASSERT(lu.dimensionOfKernel()) @@ -240,19 +205,25 @@ VERIFY_RAISES_ASSERT(plu.matrixLU()) VERIFY_RAISES_ASSERT(plu.permutationP()) VERIFY_RAISES_ASSERT(plu.solve(tmp)) + VERIFY_RAISES_ASSERT(plu.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(plu.adjoint().solve(tmp)) VERIFY_RAISES_ASSERT(plu.determinant()) VERIFY_RAISES_ASSERT(plu.inverse()) } -void test_lu() +EIGEN_DECLARE_TEST(lu) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( lu_non_invertible<Matrix3f>() ); CALL_SUBTEST_1( lu_invertible<Matrix3f>() ); CALL_SUBTEST_1( lu_verify_assert<Matrix3f>() ); + CALL_SUBTEST_1( lu_partial_piv<Matrix3f>() ); CALL_SUBTEST_2( (lu_non_invertible<Matrix<double, 4, 6> >()) ); CALL_SUBTEST_2( (lu_verify_assert<Matrix<double, 4, 6> >()) ); + CALL_SUBTEST_2( lu_partial_piv<Matrix2d>() ); + CALL_SUBTEST_2( lu_partial_piv<Matrix4d>() ); + CALL_SUBTEST_2( (lu_partial_piv<Matrix<double,6,6> >()) ); CALL_SUBTEST_3( lu_non_invertible<MatrixXf>() ); CALL_SUBTEST_3( lu_invertible<MatrixXf>() ); @@ -260,7 +231,7 @@ CALL_SUBTEST_4( lu_non_invertible<MatrixXd>() ); CALL_SUBTEST_4( lu_invertible<MatrixXd>() ); - CALL_SUBTEST_4( lu_partial_piv<MatrixXd>() ); + CALL_SUBTEST_4( lu_partial_piv<MatrixXd>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE)) ); CALL_SUBTEST_4( lu_verify_assert<MatrixXd>() ); CALL_SUBTEST_5( lu_non_invertible<MatrixXcf>() ); @@ -269,7 +240,7 @@ CALL_SUBTEST_6( lu_non_invertible<MatrixXcd>() ); CALL_SUBTEST_6( lu_invertible<MatrixXcd>() ); - CALL_SUBTEST_6( lu_partial_piv<MatrixXcd>() ); + CALL_SUBTEST_6( lu_partial_piv<MatrixXcd>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE)) ); CALL_SUBTEST_6( lu_verify_assert<MatrixXcd>() ); CALL_SUBTEST_7(( lu_non_invertible<Matrix<float,Dynamic,16> >() ));
diff --git a/test/main.h b/test/main.h index b0e3b78..1fe631c 100644 --- a/test/main.h +++ b/test/main.h
@@ -17,6 +17,7 @@ #include <sstream> #include <vector> #include <typeinfo> +#include <functional> // The following includes of STL headers have to be done _before_ the // definition of macros min() and max(). The reason is that many STL @@ -41,6 +42,7 @@ #include <complex> #include <deque> #include <queue> +#include <cassert> #include <list> #if __cplusplus >= 201103L #include <random> @@ -49,15 +51,44 @@ #endif #endif +// Same for cuda_fp16.h +#if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9) +#define EIGEN_TEST_CUDACC_VER ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100)) +#elif defined(__CUDACC_VER__) +#define EIGEN_TEST_CUDACC_VER __CUDACC_VER__ +#else +#define EIGEN_TEST_CUDACC_VER 0 +#endif + +#if EIGEN_TEST_CUDACC_VER >= 70500 +#include <cuda_fp16.h> +#endif + // To test that all calls from Eigen code to std::min() and std::max() are // protected by parenthesis against macro expansion, the min()/max() macros // are defined here and any not-parenthesized min/max call will cause a // compiler error. -#define min(A,B) please_protect_your_min_with_parentheses -#define max(A,B) please_protect_your_max_with_parentheses -#define isnan(X) please_protect_your_isnan_with_parentheses -#define isinf(X) please_protect_your_isinf_with_parentheses -#define isfinite(X) please_protect_your_isfinite_with_parentheses +#if !defined(__HIPCC__) + // + // HIP header files include the following files + // <thread> + // <regex> + // <unordered_map> + // which seem to contain not-parenthesized calls to "max"/"min", triggering the following check and causing the compile to fail + // + // Including those header files before the following macro definition for "min" / "max", only partially resolves the issue + // This is because other HIP header files also define "isnan" / "isinf" / "isfinite" functions, which are needed in other + // headers. + // + // So instead choosing to simply disable this check for HIP + // + #define min(A,B) please_protect_your_min_with_parentheses + #define max(A,B) please_protect_your_max_with_parentheses + #define isnan(X) please_protect_your_isnan_with_parentheses + #define isinf(X) please_protect_your_isinf_with_parentheses + #define isfinite(X) please_protect_your_isfinite_with_parentheses +#endif + #ifdef M_PI #undef M_PI #endif @@ -66,6 +97,8 @@ #define FORBIDDEN_IDENTIFIER (this_identifier_is_forbidden_to_avoid_clashes) this_identifier_is_forbidden_to_avoid_clashes // B0 is defined in POSIX header termios.h #define B0 FORBIDDEN_IDENTIFIER +// `I` may be defined by complex.h: +#define I FORBIDDEN_IDENTIFIER // Unit tests calling Eigen's blas library must preserve the default blocking size // to avoid troubles. @@ -79,10 +112,12 @@ #ifdef TEST_ENABLE_TEMPORARY_TRACKING static long int nb_temporaries; +static long int nb_temporaries_on_assert = -1; inline void on_temporary_creation(long int size) { // here's a great place to set a breakpoint when debugging failures in this test! if(size!=0) nb_temporaries++; + if(nb_temporaries_on_assert>0) assert(nb_temporaries<nb_temporaries_on_assert); } #define EIGEN_DENSE_STORAGE_CTOR_PLUGIN { on_temporary_creation(size); } @@ -90,13 +125,12 @@ #define VERIFY_EVALUATION_COUNT(XPR,N) {\ nb_temporaries = 0; \ XPR; \ - if(nb_temporaries!=N) std::cerr << "nb_temporaries == " << nb_temporaries << "\n"; \ - VERIFY( (#XPR) && nb_temporaries==N ); \ + if(nb_temporaries!=(N)) { std::cerr << "nb_temporaries == " << nb_temporaries << "\n"; }\ + VERIFY( (#XPR) && nb_temporaries==(N) ); \ } - + #endif -// the following file is automatically generated by cmake #include "split_test_helper.h" #ifdef NDEBUG @@ -113,10 +147,6 @@ #define EIGEN_MAKING_DOCS #endif -#ifndef EIGEN_TEST_FUNC -#error EIGEN_TEST_FUNC must be defined -#endif - #define DEFAULT_REPEAT 10 namespace Eigen @@ -125,20 +155,48 @@ // level == 0 <=> abort if test fail // level >= 1 <=> warning message to std::cerr if test fail static int g_test_level = 0; - static int g_repeat; - static unsigned int g_seed; - static bool g_has_set_repeat, g_has_set_seed; + static int g_repeat = 1; + static unsigned int g_seed = 0; + static bool g_has_set_repeat = false, g_has_set_seed = false; + + class EigenTest + { + public: + EigenTest() : m_func(0) {} + EigenTest(const char* a_name, void (*func)(void)) + : m_name(a_name), m_func(func) + { + ms_registered_tests.push_back(this); + } + const std::string& name() const { return m_name; } + void operator()() const { m_func(); } + + static const std::vector<EigenTest*>& all() { return ms_registered_tests; } + protected: + std::string m_name; + void (*m_func)(void); + static std::vector<EigenTest*> ms_registered_tests; + }; + + std::vector<EigenTest*> EigenTest::ms_registered_tests; + + // Declare and register a test, e.g.: + // EIGEN_DECLARE_TEST(mytest) { ... } + // will create a function: + // void test_mytest() { ... } + // that will be automatically called. + #define EIGEN_DECLARE_TEST(X) \ + void EIGEN_CAT(test_,X) (); \ + static EigenTest EIGEN_CAT(test_handler_,X) (EIGEN_MAKESTRING(X), & EIGEN_CAT(test_,X)); \ + void EIGEN_CAT(test_,X) () } #define TRACK std::cerr << __FILE__ << " " << __LINE__ << std::endl // #define TRACK while() -#define EI_PP_MAKE_STRING2(S) #S -#define EI_PP_MAKE_STRING(S) EI_PP_MAKE_STRING2(S) - #define EIGEN_DEFAULT_IO_FORMAT IOFormat(4, 0, " ", "\n", "", "", "", "") -#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) +#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) && !defined(__SYCL_DEVICE_ONLY__) #define EIGEN_EXCEPTIONS #endif @@ -159,9 +217,15 @@ eigen_assert_exception(void) {} ~eigen_assert_exception() { Eigen::no_more_assert = false; } }; + + struct eigen_static_assert_exception + { + eigen_static_assert_exception(void) {} + ~eigen_static_assert_exception() { Eigen::no_more_assert = false; } + }; } // If EIGEN_DEBUG_ASSERTS is defined and if no assertion is triggered while - // one should have been, then the list of excecuted assertions is printed out. + // one should have been, then the list of executed assertions is printed out. // // EIGEN_DEBUG_ASSERTS is not enabled by default as it // significantly increases the compilation time @@ -187,7 +251,7 @@ } \ else if (Eigen::internal::push_assert) \ { \ - eigen_assert_list.push_back(std::string(EI_PP_MAKE_STRING(__FILE__) " (" EI_PP_MAKE_STRING(__LINE__) ") : " #a) ); \ + eigen_assert_list.push_back(std::string(EIGEN_MAKESTRING(__FILE__) " (" EIGEN_MAKESTRING(__LINE__) ") : " #a) ); \ } #ifdef EIGEN_EXCEPTIONS @@ -211,7 +275,7 @@ } #endif //EIGEN_EXCEPTIONS - #elif !defined(__CUDACC__) // EIGEN_DEBUG_ASSERTS + #elif !defined(__CUDACC__) && !defined(__HIPCC__) && !defined(__SYCL_DEVICE_ONLY__) // EIGEN_DEBUG_ASSERTS // see bug 89. The copy_bool here is working around a bug in gcc <= 4.3 #define eigen_assert(a) \ if( (!Eigen::internal::copy_bool(a)) && (!no_more_assert) )\ @@ -222,6 +286,7 @@ else \ EIGEN_THROW_X(Eigen::eigen_assert_exception()); \ } + #ifdef EIGEN_EXCEPTIONS #define VERIFY_RAISES_ASSERT(a) { \ Eigen::no_more_assert = false; \ @@ -233,25 +298,51 @@ catch (Eigen::eigen_assert_exception&) { VERIFY(true); } \ Eigen::report_on_cerr_on_assert_failure = true; \ } - #endif //EIGEN_EXCEPTIONS + #endif // EIGEN_EXCEPTIONS #endif // EIGEN_DEBUG_ASSERTS + #if defined(TEST_CHECK_STATIC_ASSERTIONS) && defined(EIGEN_EXCEPTIONS) + #define EIGEN_STATIC_ASSERT(a,MSG) \ + if( (!Eigen::internal::copy_bool(a)) && (!no_more_assert) )\ + { \ + Eigen::no_more_assert = true; \ + if(report_on_cerr_on_assert_failure) \ + eigen_plain_assert((a) && #MSG); \ + else \ + EIGEN_THROW_X(Eigen::eigen_static_assert_exception()); \ + } + #define VERIFY_RAISES_STATIC_ASSERT(a) { \ + Eigen::no_more_assert = false; \ + Eigen::report_on_cerr_on_assert_failure = false; \ + try { \ + a; \ + VERIFY(Eigen::should_raise_an_assert && # a); \ + } \ + catch (Eigen::eigen_static_assert_exception&) { VERIFY(true); } \ + Eigen::report_on_cerr_on_assert_failure = true; \ + } + #endif // TEST_CHECK_STATIC_ASSERTIONS + #ifndef VERIFY_RAISES_ASSERT #define VERIFY_RAISES_ASSERT(a) \ std::cout << "Can't VERIFY_RAISES_ASSERT( " #a " ) with exceptions disabled\n"; #endif - - #if !defined(__CUDACC__) +#ifndef VERIFY_RAISES_STATIC_ASSERT + #define VERIFY_RAISES_STATIC_ASSERT(a) \ + std::cout << "Can't VERIFY_RAISES_STATIC_ASSERT( " #a " ) with exceptions disabled\n"; +#endif + + #if !defined(__CUDACC__) && !defined(__HIPCC__) && !defined(__SYCL_DEVICE_ONLY__) #define EIGEN_USE_CUSTOM_ASSERT #endif #else // EIGEN_NO_ASSERTION_CHECKING #define VERIFY_RAISES_ASSERT(a) {} + #define VERIFY_RAISES_STATIC_ASSERT(a) {} #endif // EIGEN_NO_ASSERTION_CHECKING - #define EIGEN_INTERNAL_DEBUGGING #include <Eigen/QR> // required for createRandomPIMatrixOfRank @@ -273,14 +364,14 @@ } } -#define VERIFY(a) ::verify_impl(a, g_test_stack.back().c_str(), __FILE__, __LINE__, EI_PP_MAKE_STRING(a)) +#define VERIFY(a) ::verify_impl(a, g_test_stack.back().c_str(), __FILE__, __LINE__, EIGEN_MAKESTRING(a)) -#define VERIFY_GE(a, b) ::verify_impl(a >= b, g_test_stack.back().c_str(), __FILE__, __LINE__, EI_PP_MAKE_STRING(a >= b)) -#define VERIFY_LE(a, b) ::verify_impl(a <= b, g_test_stack.back().c_str(), __FILE__, __LINE__, EI_PP_MAKE_STRING(a <= b)) +#define VERIFY_GE(a, b) ::verify_impl(a >= b, g_test_stack.back().c_str(), __FILE__, __LINE__, EIGEN_MAKESTRING(a >= b)) +#define VERIFY_LE(a, b) ::verify_impl(a <= b, g_test_stack.back().c_str(), __FILE__, __LINE__, EIGEN_MAKESTRING(a <= b)) -#define VERIFY_IS_EQUAL(a, b) VERIFY(test_is_equal(a, b)) -#define VERIFY_IS_NOT_EQUAL(a, b) VERIFY(!test_is_equal(a, b)) +#define VERIFY_IS_EQUAL(a, b) VERIFY(test_is_equal(a, b, true)) +#define VERIFY_IS_NOT_EQUAL(a, b) VERIFY(test_is_equal(a, b, false)) #define VERIFY_IS_APPROX(a, b) VERIFY(verifyIsApprox(a, b)) #define VERIFY_IS_NOT_APPROX(a, b) VERIFY(!test_isApprox(a, b)) #define VERIFY_IS_MUCH_SMALLER_THAN(a, b) VERIFY(test_isMuchSmallerThan(a, b)) @@ -290,8 +381,10 @@ #define VERIFY_IS_UNITARY(a) VERIFY(test_isUnitary(a)) +#define STATIC_CHECK(COND) EIGEN_STATIC_ASSERT( (COND) , EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT ) + #define CALL_SUBTEST(FUNC) do { \ - g_test_stack.push_back(EI_PP_MAKE_STRING(FUNC)); \ + g_test_stack.push_back(EIGEN_MAKESTRING(FUNC)); \ FUNC; \ g_test_stack.pop_back(); \ } while (0) @@ -299,34 +392,44 @@ namespace Eigen { +template<typename T1,typename T2> +typename internal::enable_if<internal::is_same<T1,T2>::value,bool>::type +is_same_type(const T1&, const T2&) +{ + return true; +} + template<typename T> inline typename NumTraits<T>::Real test_precision() { return NumTraits<T>::dummy_precision(); } template<> inline float test_precision<float>() { return 1e-3f; } template<> inline double test_precision<double>() { return 1e-6; } -template<> inline long double test_precision<long double>() { return 1e-6; } +template<> inline long double test_precision<long double>() { return 1e-6l; } template<> inline float test_precision<std::complex<float> >() { return test_precision<float>(); } template<> inline double test_precision<std::complex<double> >() { return test_precision<double>(); } template<> inline long double test_precision<std::complex<long double> >() { return test_precision<long double>(); } -inline bool test_isApprox(const int& a, const int& b) -{ return internal::isApprox(a, b, test_precision<int>()); } -inline bool test_isMuchSmallerThan(const int& a, const int& b) -{ return internal::isMuchSmallerThan(a, b, test_precision<int>()); } -inline bool test_isApproxOrLessThan(const int& a, const int& b) -{ return internal::isApproxOrLessThan(a, b, test_precision<int>()); } +#define EIGEN_TEST_SCALAR_TEST_OVERLOAD(TYPE) \ + inline bool test_isApprox(TYPE a, TYPE b) \ + { return internal::isApprox(a, b, test_precision<TYPE>()); } \ + inline bool test_isMuchSmallerThan(TYPE a, TYPE b) \ + { return internal::isMuchSmallerThan(a, b, test_precision<TYPE>()); } \ + inline bool test_isApproxOrLessThan(TYPE a, TYPE b) \ + { return internal::isApproxOrLessThan(a, b, test_precision<TYPE>()); } -inline bool test_isApprox(const float& a, const float& b) -{ return internal::isApprox(a, b, test_precision<float>()); } -inline bool test_isMuchSmallerThan(const float& a, const float& b) -{ return internal::isMuchSmallerThan(a, b, test_precision<float>()); } -inline bool test_isApproxOrLessThan(const float& a, const float& b) -{ return internal::isApproxOrLessThan(a, b, test_precision<float>()); } +EIGEN_TEST_SCALAR_TEST_OVERLOAD(short) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned short) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(int) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned int) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(long) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned long) +#if EIGEN_HAS_CXX11 +EIGEN_TEST_SCALAR_TEST_OVERLOAD(long long) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned long long) +#endif +EIGEN_TEST_SCALAR_TEST_OVERLOAD(float) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(double) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(half) -inline bool test_isApprox(const double& a, const double& b) -{ return internal::isApprox(a, b, test_precision<double>()); } -inline bool test_isMuchSmallerThan(const double& a, const double& b) -{ return internal::isMuchSmallerThan(a, b, test_precision<double>()); } -inline bool test_isApproxOrLessThan(const double& a, const double& b) -{ return internal::isApproxOrLessThan(a, b, test_precision<double>()); } +#undef EIGEN_TEST_SCALAR_TEST_OVERLOAD #ifndef EIGEN_TEST_NO_COMPLEX inline bool test_isApprox(const std::complex<float>& a, const std::complex<float>& b) @@ -363,19 +466,12 @@ { return internal::isApproxOrLessThan(a, b, test_precision<long double>()); } #endif // EIGEN_TEST_NO_LONGDOUBLE -inline bool test_isApprox(const half& a, const half& b) -{ return internal::isApprox(a, b, test_precision<half>()); } -inline bool test_isMuchSmallerThan(const half& a, const half& b) -{ return internal::isMuchSmallerThan(a, b, test_precision<half>()); } -inline bool test_isApproxOrLessThan(const half& a, const half& b) -{ return internal::isApproxOrLessThan(a, b, test_precision<half>()); } - // test_relative_error returns the relative difference between a and b as a real scalar as used in isApprox. template<typename T1,typename T2> -typename T1::RealScalar test_relative_error(const EigenBase<T1> &a, const EigenBase<T2> &b) +typename NumTraits<typename T1::RealScalar>::NonInteger test_relative_error(const EigenBase<T1> &a, const EigenBase<T2> &b) { using std::sqrt; - typedef typename T1::RealScalar RealScalar; + typedef typename NumTraits<typename T1::RealScalar>::NonInteger RealScalar; typename internal::nested_eval<T1,2>::type ea(a.derived()); typename internal::nested_eval<T2,2>::type eb(b.derived()); return sqrt(RealScalar((ea-eb).cwiseAbs2().sum()) / RealScalar((std::min)(eb.cwiseAbs2().sum(),ea.cwiseAbs2().sum()))); @@ -433,9 +529,9 @@ } template<typename T1,typename T2> -typename NumTraits<T1>::Real test_relative_error(const T1 &a, const T2 &b, typename internal::enable_if<internal::is_arithmetic<typename NumTraits<T1>::Real>::value, T1>::type* = 0) +typename NumTraits<typename NumTraits<T1>::Real>::NonInteger test_relative_error(const T1 &a, const T2 &b, typename internal::enable_if<internal::is_arithmetic<typename NumTraits<T1>::Real>::value, T1>::type* = 0) { - typedef typename NumTraits<T1>::Real RealScalar; + typedef typename NumTraits<typename NumTraits<T1>::Real>::NonInteger RealScalar; return numext::sqrt(RealScalar(numext::abs2(a-b))/RealScalar((numext::mini)(numext::abs2(a),numext::abs2(b)))); } @@ -452,20 +548,20 @@ } template<typename Type1, typename Type2> -inline bool test_isApprox(const Type1& a, const Type2& b) +inline bool test_isApprox(const Type1& a, const Type2& b, typename Type1::Scalar* = 0) // Enabled for Eigen's type only { return a.isApprox(b, test_precision<typename Type1::Scalar>()); } // get_test_precision is a small wrapper to test_precision allowing to return the scalar precision for either scalars or expressions template<typename T> -typename NumTraits<typename T::Scalar>::Real get_test_precision(const typename T::Scalar* = 0) +typename NumTraits<typename T::Scalar>::Real get_test_precision(const T&, const typename T::Scalar* = 0) { return test_precision<typename NumTraits<typename T::Scalar>::Real>(); } template<typename T> -typename NumTraits<T>::Real get_test_precision(typename internal::enable_if<internal::is_arithmetic<typename NumTraits<T>::Real>::value, T>::type* = 0) +typename NumTraits<T>::Real get_test_precision(const T&,typename internal::enable_if<internal::is_arithmetic<typename NumTraits<T>::Real>::value, T>::type* = 0) { return test_precision<typename NumTraits<T>::Real>(); } @@ -477,7 +573,7 @@ bool ret = test_isApprox(a,b); if(!ret) { - std::cerr << "Difference too large wrt tolerance " << get_test_precision<Type1>() << ", relative error is: " << test_relative_error(a,b) << std::endl; + std::cerr << "Difference too large wrt tolerance " << get_test_precision(a) << ", relative error is: " << test_relative_error(a,b) << std::endl; } return ret; } @@ -517,17 +613,17 @@ // Forward declaration to avoid ICC warning template<typename T, typename U> -bool test_is_equal(const T& actual, const U& expected); +bool test_is_equal(const T& actual, const U& expected, bool expect_equal=true); template<typename T, typename U> -bool test_is_equal(const T& actual, const U& expected) +bool test_is_equal(const T& actual, const U& expected, bool expect_equal) { - if (actual==expected) + if ((actual==expected) == expect_equal) return true; // false: std::cerr - << std::endl << " actual = " << actual - << std::endl << " expected = " << expected << std::endl << std::endl; + << "\n actual = " << actual + << "\n expected " << (expect_equal ? "= " : "!=") << expected << "\n\n"; return false; } @@ -631,9 +727,6 @@ template<> std::string type_name<std::complex<long double> >() { return "complex<long double>"; } template<> std::string type_name<std::complex<int> >() { return "complex<int>"; } -// forward declaration of the main test function -void EIGEN_CAT(test_,EIGEN_TEST_FUNC)(); - using namespace Eigen; inline void set_repeat_from_string(const char *str) @@ -720,9 +813,16 @@ srand(g_seed); std::cout << "Repeating each test " << g_repeat << " times" << std::endl; - Eigen::g_test_stack.push_back(std::string(EI_PP_MAKE_STRING(EIGEN_TEST_FUNC))); + VERIFY(EigenTest::all().size()>0); - EIGEN_CAT(test_,EIGEN_TEST_FUNC)(); + for(std::size_t i=0; i<EigenTest::all().size(); ++i) + { + const EigenTest& current_test = *EigenTest::all()[i]; + Eigen::g_test_stack.push_back(current_test.name()); + current_test(); + Eigen::g_test_stack.pop_back(); + } + return 0; } @@ -736,3 +836,8 @@ // remark #1572: floating-point equality and inequality comparisons are unreliable #pragma warning disable 279 383 1418 1572 #endif + +#ifdef _MSC_VER + // 4503 - decorated name length exceeded, name was truncated + #pragma warning( disable : 4503) +#endif
diff --git a/test/mapped_matrix.cpp b/test/mapped_matrix.cpp index 88653e8..0ea136a 100644 --- a/test/mapped_matrix.cpp +++ b/test/mapped_matrix.cpp
@@ -17,7 +17,6 @@ template<typename VectorType> void map_class_vector(const VectorType& m) { - typedef typename VectorType::Index Index; typedef typename VectorType::Scalar Scalar; Index size = m.size(); @@ -25,7 +24,7 @@ Scalar* array1 = internal::aligned_new<Scalar>(size); Scalar* array2 = internal::aligned_new<Scalar>(size); Scalar* array3 = new Scalar[size+1]; - Scalar* array3unaligned = (std::size_t(array3)%EIGEN_MAX_ALIGN_BYTES) == 0 ? array3+1 : array3; + Scalar* array3unaligned = (internal::UIntPtr(array3)%EIGEN_MAX_ALIGN_BYTES) == 0 ? array3+1 : array3; Scalar array4[EIGEN_TESTMAP_MAX_SIZE]; Map<VectorType, AlignedMax>(array1, size) = VectorType::Random(size); @@ -51,7 +50,6 @@ template<typename MatrixType> void map_class_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(), cols = m.cols(), size = rows*cols; @@ -64,8 +62,9 @@ for(int i = 0; i < size; i++) array2[i] = Scalar(1); // array3unaligned -> unaligned pointer to heap Scalar* array3 = new Scalar[size+1]; - for(int i = 0; i < size+1; i++) array3[i] = Scalar(1); - Scalar* array3unaligned = size_t(array3)%EIGEN_MAX_ALIGN_BYTES == 0 ? array3+1 : array3; + Index sizep1 = size + 1; // <- without this temporary MSVC 2103 generates bad code + for(Index i = 0; i < sizep1; i++) array3[i] = Scalar(1); + Scalar* array3unaligned = (internal::UIntPtr(array3)%EIGEN_MAX_ALIGN_BYTES) == 0 ? array3+1 : array3; Scalar array4[256]; if(size<=256) for(int i = 0; i < size; i++) array4[i] = Scalar(1); @@ -121,7 +120,6 @@ template<typename VectorType> void map_static_methods(const VectorType& m) { - typedef typename VectorType::Index Index; typedef typename VectorType::Scalar Scalar; Index size = m.size(); @@ -129,7 +127,7 @@ Scalar* array1 = internal::aligned_new<Scalar>(size); Scalar* array2 = internal::aligned_new<Scalar>(size); Scalar* array3 = new Scalar[size+1]; - Scalar* array3unaligned = size_t(array3)%EIGEN_MAX_ALIGN_BYTES == 0 ? array3+1 : array3; + Scalar* array3unaligned = internal::UIntPtr(array3)%EIGEN_MAX_ALIGN_BYTES == 0 ? array3+1 : array3; VectorType::MapAligned(array1, size) = VectorType::Random(size); VectorType::Map(array2, size) = VectorType::Map(array1, size); @@ -163,7 +161,6 @@ void map_not_aligned_on_scalar() { typedef Matrix<Scalar,Dynamic,Dynamic> MatrixType; - typedef typename MatrixType::Index Index; Index size = 11; Scalar* array1 = internal::aligned_new<Scalar>((size+1)*(size+1)+1); Scalar* array2 = reinterpret_cast<Scalar*>(sizeof(Scalar)/2+std::size_t(array1)); @@ -181,7 +178,7 @@ internal::aligned_delete(array1, (size+1)*(size+1)+1); } -void test_mapped_matrix() +EIGEN_DECLARE_TEST(mapped_matrix) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( map_class_vector(Matrix<float, 1, 1>()) ); @@ -205,7 +202,6 @@ CALL_SUBTEST_8( map_static_methods(RowVector3d()) ); CALL_SUBTEST_9( map_static_methods(VectorXcd(8)) ); CALL_SUBTEST_10( map_static_methods(VectorXf(12)) ); - CALL_SUBTEST_11( map_not_aligned_on_scalar<double>() ); } }
diff --git a/test/mapstaticmethods.cpp b/test/mapstaticmethods.cpp index 06272d1..d0128ba 100644 --- a/test/mapstaticmethods.cpp +++ b/test/mapstaticmethods.cpp
@@ -9,8 +9,12 @@ #include "main.h" +// GCC<=4.8 has spurious shadow warnings, because `ptr` re-appears inside template instantiations +// workaround: put these in an anonymous namespace +namespace { float *ptr; const float *const_ptr; +} template<typename PlainObjectType, bool IsDynamicSize = PlainObjectType::SizeAtCompileTime == Dynamic, @@ -69,7 +73,6 @@ { static void run(const PlainObjectType& m) { - typedef typename PlainObjectType::Index Index; Index rows = m.rows(), cols = m.cols(); int i = internal::random<int>(2,5), j = internal::random<int>(2,5); @@ -116,7 +119,6 @@ { static void run(const PlainObjectType& v) { - typedef typename PlainObjectType::Index Index; Index size = v.size(); int i = internal::random<int>(2,5); @@ -145,7 +147,7 @@ VERIFY(true); // just to avoid 'unused function' warning } -void test_mapstaticmethods() +EIGEN_DECLARE_TEST(mapstaticmethods) { ptr = internal::aligned_new<float>(1000); for(int i = 0; i < 1000; i++) ptr[i] = float(i);
diff --git a/test/mapstride.cpp b/test/mapstride.cpp index ee24142..0919660 100644 --- a/test/mapstride.cpp +++ b/test/mapstride.cpp
@@ -11,7 +11,6 @@ template<int Alignment,typename VectorType> void map_class_vector(const VectorType& m) { - typedef typename VectorType::Index Index; typedef typename VectorType::Scalar Scalar; Index size = m.size(); @@ -23,7 +22,7 @@ Scalar* a_array = internal::aligned_new<Scalar>(arraysize+1); Scalar* array = a_array; if(Alignment!=Aligned) - array = (Scalar*)(ptrdiff_t(a_array) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); + array = (Scalar*)(internal::IntPtr(a_array) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); { Map<VectorType, Alignment, InnerStride<3> > map(array, size); @@ -50,7 +49,6 @@ template<int Alignment,typename MatrixType> void map_class_matrix(const MatrixType& _m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = _m.rows(), cols = _m.cols(); @@ -58,19 +56,19 @@ MatrixType m = MatrixType::Random(rows,cols); Scalar s1 = internal::random<Scalar>(); - Index arraysize = 2*(rows+4)*(cols+4); + Index arraysize = 4*(rows+4)*(cols+4); Scalar* a_array1 = internal::aligned_new<Scalar>(arraysize+1); Scalar* array1 = a_array1; if(Alignment!=Aligned) - array1 = (Scalar*)(std::ptrdiff_t(a_array1) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); + array1 = (Scalar*)(internal::IntPtr(a_array1) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); Scalar a_array2[256]; Scalar* array2 = a_array2; if(Alignment!=Aligned) - array2 = (Scalar*)(std::ptrdiff_t(a_array2) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); + array2 = (Scalar*)(internal::IntPtr(a_array2) + (internal::packet_traits<Scalar>::AlignedOnScalar?sizeof(Scalar):sizeof(typename NumTraits<Scalar>::Real))); else - array2 = (Scalar*)(((std::size_t(a_array2)+EIGEN_MAX_ALIGN_BYTES-1)/EIGEN_MAX_ALIGN_BYTES)*EIGEN_MAX_ALIGN_BYTES); + array2 = (Scalar*)(((internal::UIntPtr(a_array2)+EIGEN_MAX_ALIGN_BYTES-1)/EIGEN_MAX_ALIGN_BYTES)*EIGEN_MAX_ALIGN_BYTES); Index maxsize2 = a_array2 - array2 + 256; // test no inner stride and some dynamic outer stride @@ -143,10 +141,63 @@ VERIFY_IS_APPROX(map,s1*m); } + // test inner stride and no outer stride + for(int k=0; k<2; ++k) + { + if(k==1 && (m.innerSize()*2)*m.outerSize() > maxsize2) + break; + Scalar* array = (k==0 ? array1 : array2); + + Map<MatrixType, Alignment, InnerStride<Dynamic> > map(array, rows, cols, InnerStride<Dynamic>(2)); + map = m; + VERIFY(map.outerStride() == map.innerSize()*2); + for(int i = 0; i < m.outerSize(); ++i) + for(int j = 0; j < m.innerSize(); ++j) + { + VERIFY(array[map.innerSize()*i*2+j*2] == m.coeffByOuterInner(i,j)); + VERIFY(map.coeffByOuterInner(i,j) == m.coeffByOuterInner(i,j)); + } + VERIFY_IS_APPROX(s1*map,s1*m); + map *= s1; + VERIFY_IS_APPROX(map,s1*m); + } + internal::aligned_delete(a_array1, arraysize+1); } -void test_mapstride() +// Additional tests for inner-stride but no outer-stride +template<int> +void bug1453() +{ + const int data[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + typedef Matrix<int,Dynamic,Dynamic,RowMajor> RowMatrixXi; + typedef Matrix<int,2,3,ColMajor> ColMatrix23i; + typedef Matrix<int,3,2,ColMajor> ColMatrix32i; + typedef Matrix<int,2,3,RowMajor> RowMatrix23i; + typedef Matrix<int,3,2,RowMajor> RowMatrix32i; + + VERIFY_IS_APPROX(MatrixXi::Map(data, 2, 3, InnerStride<2>()), MatrixXi::Map(data, 2, 3, Stride<4,2>())); + VERIFY_IS_APPROX(MatrixXi::Map(data, 2, 3, InnerStride<>(2)), MatrixXi::Map(data, 2, 3, Stride<4,2>())); + VERIFY_IS_APPROX(MatrixXi::Map(data, 3, 2, InnerStride<2>()), MatrixXi::Map(data, 3, 2, Stride<6,2>())); + VERIFY_IS_APPROX(MatrixXi::Map(data, 3, 2, InnerStride<>(2)), MatrixXi::Map(data, 3, 2, Stride<6,2>())); + + VERIFY_IS_APPROX(RowMatrixXi::Map(data, 2, 3, InnerStride<2>()), RowMatrixXi::Map(data, 2, 3, Stride<6,2>())); + VERIFY_IS_APPROX(RowMatrixXi::Map(data, 2, 3, InnerStride<>(2)), RowMatrixXi::Map(data, 2, 3, Stride<6,2>())); + VERIFY_IS_APPROX(RowMatrixXi::Map(data, 3, 2, InnerStride<2>()), RowMatrixXi::Map(data, 3, 2, Stride<4,2>())); + VERIFY_IS_APPROX(RowMatrixXi::Map(data, 3, 2, InnerStride<>(2)), RowMatrixXi::Map(data, 3, 2, Stride<4,2>())); + + VERIFY_IS_APPROX(ColMatrix23i::Map(data, InnerStride<2>()), MatrixXi::Map(data, 2, 3, Stride<4,2>())); + VERIFY_IS_APPROX(ColMatrix23i::Map(data, InnerStride<>(2)), MatrixXi::Map(data, 2, 3, Stride<4,2>())); + VERIFY_IS_APPROX(ColMatrix32i::Map(data, InnerStride<2>()), MatrixXi::Map(data, 3, 2, Stride<6,2>())); + VERIFY_IS_APPROX(ColMatrix32i::Map(data, InnerStride<>(2)), MatrixXi::Map(data, 3, 2, Stride<6,2>())); + + VERIFY_IS_APPROX(RowMatrix23i::Map(data, InnerStride<2>()), RowMatrixXi::Map(data, 2, 3, Stride<6,2>())); + VERIFY_IS_APPROX(RowMatrix23i::Map(data, InnerStride<>(2)), RowMatrixXi::Map(data, 2, 3, Stride<6,2>())); + VERIFY_IS_APPROX(RowMatrix32i::Map(data, InnerStride<2>()), RowMatrixXi::Map(data, 3, 2, Stride<4,2>())); + VERIFY_IS_APPROX(RowMatrix32i::Map(data, InnerStride<>(2)), RowMatrixXi::Map(data, 3, 2, Stride<4,2>())); +} + +EIGEN_DECLARE_TEST(mapstride) { for(int i = 0; i < g_repeat; i++) { int maxn = 30; @@ -175,6 +226,8 @@ CALL_SUBTEST_5( map_class_matrix<Unaligned>(MatrixXi(internal::random<int>(1,maxn),internal::random<int>(1,maxn))) ); CALL_SUBTEST_6( map_class_matrix<Aligned>(MatrixXcd(internal::random<int>(1,maxn),internal::random<int>(1,maxn))) ); CALL_SUBTEST_6( map_class_matrix<Unaligned>(MatrixXcd(internal::random<int>(1,maxn),internal::random<int>(1,maxn))) ); + + CALL_SUBTEST_5( bug1453<0>() ); TEST_SET_BUT_UNUSED_VARIABLE(maxn); }
diff --git a/test/meta.cpp b/test/meta.cpp index b8dea68..ea9607f 100644 --- a/test/meta.cpp +++ b/test/meta.cpp
@@ -15,7 +15,19 @@ return internal::is_convertible<From,To>::value; } -void test_meta() +struct FooReturnType { + typedef int ReturnType; +}; + +struct MyInterface { + virtual void func() = 0; + virtual ~MyInterface() {} +}; +struct MyImpl : public MyInterface { + void func() {} +}; + +EIGEN_DECLARE_TEST(meta) { VERIFY((internal::conditional<(3<4),internal::true_type, internal::false_type>::type::value)); VERIFY(( internal::is_same<float,float>::value)); @@ -58,14 +70,27 @@ VERIFY(( internal::is_same<const float,internal::remove_pointer<const float*>::type >::value)); VERIFY(( internal::is_same<float,internal::remove_pointer<float* const >::type >::value)); - VERIFY(( internal::is_convertible<float,double>::value )); - VERIFY(( internal::is_convertible<int,double>::value )); - VERIFY(( internal::is_convertible<double,int>::value )); - VERIFY((!internal::is_convertible<std::complex<double>,double>::value )); - VERIFY(( internal::is_convertible<Array33f,Matrix3f>::value )); -// VERIFY((!internal::is_convertible<Matrix3f,Matrix3d>::value )); //does not work because the conversion is prevented by a static assertion - VERIFY((!internal::is_convertible<Array33f,int>::value )); - VERIFY((!internal::is_convertible<MatrixXf,float>::value )); + + // is_convertible + STATIC_CHECK(( internal::is_convertible<float,double>::value )); + STATIC_CHECK(( internal::is_convertible<int,double>::value )); + STATIC_CHECK(( internal::is_convertible<int, short>::value )); + STATIC_CHECK(( internal::is_convertible<short, int>::value )); + STATIC_CHECK(( internal::is_convertible<double,int>::value )); + STATIC_CHECK(( internal::is_convertible<double,std::complex<double> >::value )); + STATIC_CHECK((!internal::is_convertible<std::complex<double>,double>::value )); + STATIC_CHECK(( internal::is_convertible<Array33f,Matrix3f>::value )); + STATIC_CHECK(( internal::is_convertible<Matrix3f&,Matrix3f>::value )); + STATIC_CHECK(( internal::is_convertible<Matrix3f&,Matrix3f&>::value )); + STATIC_CHECK(( internal::is_convertible<Matrix3f&,const Matrix3f&>::value )); + STATIC_CHECK(( internal::is_convertible<const Matrix3f&,Matrix3f>::value )); + STATIC_CHECK(( internal::is_convertible<const Matrix3f&,const Matrix3f&>::value )); + STATIC_CHECK((!internal::is_convertible<const Matrix3f&,Matrix3f&>::value )); + STATIC_CHECK((!internal::is_convertible<const Matrix3f,Matrix3f&>::value )); + STATIC_CHECK(( internal::is_convertible<Matrix3f,Matrix3f&>::value )); // std::is_convertible returns false here though Matrix3f from; Matrix3f& to = from; is valid. + //STATIC_CHECK((!internal::is_convertible<Matrix3f,Matrix3d>::value )); //does not work because the conversion is prevented by a static assertion + STATIC_CHECK((!internal::is_convertible<Array33f,int>::value )); + STATIC_CHECK((!internal::is_convertible<MatrixXf,float>::value )); { float f; MatrixXf A, B; @@ -75,6 +100,26 @@ VERIFY((!check_is_convertible(A*B, f) )); VERIFY(( check_is_convertible(A*B, A) )); } + + STATIC_CHECK(( !internal::is_convertible<MyInterface, MyImpl>::value )); + #if (!EIGEN_COMP_GNUC_STRICT) || (EIGEN_GNUC_AT_LEAST(4,8)) + // GCC prior to 4.8 fails to compile this test: + // error: cannot allocate an object of abstract type 'MyInterface' + // In other word, it does not obey SFINAE. + // Nevertheless, we don't really care about supporting abstract type as scalar type! + STATIC_CHECK(( !internal::is_convertible<MyImpl, MyInterface>::value )); + #endif + STATIC_CHECK(( internal::is_convertible<MyImpl, const MyInterface&>::value )); + { + int i; + VERIFY(( check_is_convertible(fix<3>(), i) )); + VERIFY((!check_is_convertible(i, fix<DynamicIndex>()) )); + } + + VERIFY(( internal::has_ReturnType<FooReturnType>::value )); + VERIFY(( internal::has_ReturnType<ScalarBinaryOpTraits<int,int> >::value )); + VERIFY(( !internal::has_ReturnType<MatrixXf>::value )); + VERIFY(( !internal::has_ReturnType<int>::value )); VERIFY(internal::meta_sqrt<1>::ret == 1); #define VERIFY_META_SQRT(X) VERIFY(internal::meta_sqrt<X>::ret == int(std::sqrt(double(X))))
diff --git a/test/metis_support.cpp b/test/metis_support.cpp index d87c56a..b490dac 100644 --- a/test/metis_support.cpp +++ b/test/metis_support.cpp
@@ -19,7 +19,7 @@ check_sparse_square_solving(sparselu_metis); } -void test_metis_support() +EIGEN_DECLARE_TEST(metis_support) { CALL_SUBTEST_1(test_metis_T<double>()); }
diff --git a/test/miscmatrices.cpp b/test/miscmatrices.cpp index ef20dc7..e71712f 100644 --- a/test/miscmatrices.cpp +++ b/test/miscmatrices.cpp
@@ -14,7 +14,6 @@ /* this test covers the following files: DiagonalMatrix.h Ones.h */ - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; Index rows = m.rows(); @@ -35,7 +34,7 @@ VERIFY_IS_APPROX(square, MatrixType::Identity(rows, rows)); } -void test_miscmatrices() +EIGEN_DECLARE_TEST(miscmatrices) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( miscMatrices(Matrix<float, 1, 1>()) );
diff --git a/test/mixingtypes.cpp b/test/mixingtypes.cpp index 0b381ec..aad63ec 100644 --- a/test/mixingtypes.cpp +++ b/test/mixingtypes.cpp
@@ -8,13 +8,27 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// work around "uninitialized" warnings and give that option some testing -#define EIGEN_INITIALIZE_MATRICES_BY_ZERO +#if defined(EIGEN_TEST_PART_7) #ifndef EIGEN_NO_STATIC_ASSERT #define EIGEN_NO_STATIC_ASSERT // turn static asserts into runtime asserts in order to check them #endif +// ignore double-promotion diagnostic for clang and gcc, if we check for static assertion anyway: +// TODO do the same for MSVC? +#if defined(__clang__) +# if (__clang_major__ * 100 + __clang_minor__) >= 308 +# pragma clang diagnostic ignored "-Wdouble-promotion" +# endif +#elif defined(__GNUC__) + // TODO is there a minimal GCC version for this? At least g++-4.7 seems to be fine with this. +# pragma GCC diagnostic ignored "-Wdouble-promotion" +#endif + +#endif + + + #if defined(EIGEN_TEST_PART_1) || defined(EIGEN_TEST_PART_2) || defined(EIGEN_TEST_PART_3) #ifndef EIGEN_DONT_VECTORIZE @@ -23,10 +37,40 @@ #endif +static bool g_called; +#define EIGEN_SCALAR_BINARY_OP_PLUGIN { g_called |= (!internal::is_same<LhsScalar,RhsScalar>::value); } + #include "main.h" using namespace std; +#define VERIFY_MIX_SCALAR(XPR,REF) \ + g_called = false; \ + VERIFY_IS_APPROX(XPR,REF); \ + VERIFY( g_called && #XPR" not properly optimized"); + +template<int SizeAtCompileType> +void raise_assertion(Index size = SizeAtCompileType) +{ + // VERIFY_RAISES_ASSERT(mf+md); // does not even compile + Matrix<float, SizeAtCompileType, 1> vf; vf.setRandom(size); + Matrix<double, SizeAtCompileType, 1> vd; vd.setRandom(size); + VERIFY_RAISES_ASSERT(vf=vd); + VERIFY_RAISES_ASSERT(vf+=vd); + VERIFY_RAISES_ASSERT(vf-=vd); + VERIFY_RAISES_ASSERT(vd=vf); + VERIFY_RAISES_ASSERT(vd+=vf); + VERIFY_RAISES_ASSERT(vd-=vf); + + // vd.asDiagonal() * mf; // does not even compile + // vcd.asDiagonal() * mf; // does not even compile + +#if 0 // we get other compilation errors here than just static asserts + VERIFY_RAISES_ASSERT(vd.dot(vf)); +#endif +} + + template<int SizeAtCompileType> void mixingtypes(int size = SizeAtCompileType) { typedef std::complex<float> CF; @@ -42,6 +86,7 @@ Mat_f mf = Mat_f::Random(size,size); Mat_d md = mf.template cast<double>(); + //Mat_d rd = md; Mat_cf mcf = Mat_cf::Random(size,size); Mat_cd mcd = mcf.template cast<complex<double> >(); Mat_cd rcd = mcd; @@ -54,31 +99,55 @@ complex<float> scf = internal::random<complex<float> >(); complex<double> scd = internal::random<complex<double> >(); - mf+mf; - VERIFY_RAISES_ASSERT(mf+md); -#ifndef EIGEN_HAS_STD_RESULT_OF - // this one does not even compile with C++11 - VERIFY_RAISES_ASSERT(mf+mcf); -#endif -#ifdef EIGEN_DONT_VECTORIZE - VERIFY_RAISES_ASSERT(vf=vd); - VERIFY_RAISES_ASSERT(vf+=vd); - VERIFY_RAISES_ASSERT(mcd=md); -#endif - + float epsf = std::sqrt(std::numeric_limits<float> ::min EIGEN_EMPTY ()); + double epsd = std::sqrt(std::numeric_limits<double>::min EIGEN_EMPTY ()); + + while(std::abs(sf )<epsf) sf = internal::random<float>(); + while(std::abs(sd )<epsd) sd = internal::random<double>(); + while(std::abs(scf)<epsf) scf = internal::random<CF>(); + while(std::abs(scd)<epsd) scd = internal::random<CD>(); + // check scalar products - VERIFY_IS_APPROX(vcf * sf , vcf * complex<float>(sf)); - VERIFY_IS_APPROX(sd * vcd, complex<double>(sd) * vcd); - VERIFY_IS_APPROX(vf * scf , vf.template cast<complex<float> >() * scf); - VERIFY_IS_APPROX(scd * vd, scd * vd.template cast<complex<double> >()); + VERIFY_MIX_SCALAR(vcf * sf , vcf * complex<float>(sf)); + VERIFY_MIX_SCALAR(sd * vcd , complex<double>(sd) * vcd); + VERIFY_MIX_SCALAR(vf * scf , vf.template cast<complex<float> >() * scf); + VERIFY_MIX_SCALAR(scd * vd , scd * vd.template cast<complex<double> >()); + + VERIFY_MIX_SCALAR(vcf * 2 , vcf * complex<float>(2)); + VERIFY_MIX_SCALAR(vcf * 2.1 , vcf * complex<float>(2.1)); + VERIFY_MIX_SCALAR(2 * vcf, vcf * complex<float>(2)); + VERIFY_MIX_SCALAR(2.1 * vcf , vcf * complex<float>(2.1)); + + // check scalar quotients + VERIFY_MIX_SCALAR(vcf / sf , vcf / complex<float>(sf)); + VERIFY_MIX_SCALAR(vf / scf , vf.template cast<complex<float> >() / scf); + VERIFY_MIX_SCALAR(vf.array() / scf, vf.template cast<complex<float> >().array() / scf); + VERIFY_MIX_SCALAR(scd / vd.array() , scd / vd.template cast<complex<double> >().array()); + + // check scalar increment + VERIFY_MIX_SCALAR(vcf.array() + sf , vcf.array() + complex<float>(sf)); + VERIFY_MIX_SCALAR(sd + vcd.array(), complex<double>(sd) + vcd.array()); + VERIFY_MIX_SCALAR(vf.array() + scf, vf.template cast<complex<float> >().array() + scf); + VERIFY_MIX_SCALAR(scd + vd.array() , scd + vd.template cast<complex<double> >().array()); + + // check scalar subtractions + VERIFY_MIX_SCALAR(vcf.array() - sf , vcf.array() - complex<float>(sf)); + VERIFY_MIX_SCALAR(sd - vcd.array(), complex<double>(sd) - vcd.array()); + VERIFY_MIX_SCALAR(vf.array() - scf, vf.template cast<complex<float> >().array() - scf); + VERIFY_MIX_SCALAR(scd - vd.array() , scd - vd.template cast<complex<double> >().array()); + + // check scalar powers + VERIFY_MIX_SCALAR( pow(vcf.array(), sf), Eigen::pow(vcf.array(), complex<float>(sf)) ); + VERIFY_MIX_SCALAR( vcf.array().pow(sf) , Eigen::pow(vcf.array(), complex<float>(sf)) ); + VERIFY_MIX_SCALAR( pow(sd, vcd.array()), Eigen::pow(complex<double>(sd), vcd.array()) ); + VERIFY_MIX_SCALAR( Eigen::pow(vf.array(), scf), Eigen::pow(vf.template cast<complex<float> >().array(), scf) ); + VERIFY_MIX_SCALAR( vf.array().pow(scf) , Eigen::pow(vf.template cast<complex<float> >().array(), scf) ); + VERIFY_MIX_SCALAR( Eigen::pow(scd, vd.array()), Eigen::pow(scd, vd.template cast<complex<double> >().array()) ); // check dot product vf.dot(vf); -#if 0 // we get other compilation errors here than just static asserts - VERIFY_RAISES_ASSERT(vd.dot(vf)); -#endif VERIFY_IS_APPROX(vcf.dot(vf), vcf.dot(vf.template cast<complex<float> >())); // check diagonal product @@ -87,9 +156,6 @@ VERIFY_IS_APPROX(mcf * vf.asDiagonal(), mcf * vf.template cast<complex<float> >().asDiagonal()); VERIFY_IS_APPROX(md * vcd.asDiagonal(), md.template cast<complex<double> >() * vcd.asDiagonal()); -// vd.asDiagonal() * mf; // does not even compile -// vcd.asDiagonal() * mf; // does not even compile - // check inner product VERIFY_IS_APPROX((vf.transpose() * vcf).value(), (vf.template cast<complex<float> >().transpose() * vcf).value()); @@ -184,9 +250,66 @@ Mat_cd((scd * mcd * md.template cast<CD>().eval()).template triangularView<Upper>())); VERIFY_IS_APPROX(Mat_cd(rcd.template triangularView<Upper>() = scd * md * mcd), Mat_cd((scd * md.template cast<CD>().eval() * mcd).template triangularView<Upper>())); + + + VERIFY_IS_APPROX( md.array() * mcd.array(), md.template cast<CD>().eval().array() * mcd.array() ); + VERIFY_IS_APPROX( mcd.array() * md.array(), mcd.array() * md.template cast<CD>().eval().array() ); + + VERIFY_IS_APPROX( md.array() + mcd.array(), md.template cast<CD>().eval().array() + mcd.array() ); + VERIFY_IS_APPROX( mcd.array() + md.array(), mcd.array() + md.template cast<CD>().eval().array() ); + + VERIFY_IS_APPROX( md.array() - mcd.array(), md.template cast<CD>().eval().array() - mcd.array() ); + VERIFY_IS_APPROX( mcd.array() - md.array(), mcd.array() - md.template cast<CD>().eval().array() ); + + if(mcd.array().abs().minCoeff()>epsd) + { + VERIFY_IS_APPROX( md.array() / mcd.array(), md.template cast<CD>().eval().array() / mcd.array() ); + } + if(md.array().abs().minCoeff()>epsd) + { + VERIFY_IS_APPROX( mcd.array() / md.array(), mcd.array() / md.template cast<CD>().eval().array() ); + } + + if(md.array().abs().minCoeff()>epsd || mcd.array().abs().minCoeff()>epsd) + { + VERIFY_IS_APPROX( md.array().pow(mcd.array()), md.template cast<CD>().eval().array().pow(mcd.array()) ); + VERIFY_IS_APPROX( mcd.array().pow(md.array()), mcd.array().pow(md.template cast<CD>().eval().array()) ); + + VERIFY_IS_APPROX( pow(md.array(),mcd.array()), md.template cast<CD>().eval().array().pow(mcd.array()) ); + VERIFY_IS_APPROX( pow(mcd.array(),md.array()), mcd.array().pow(md.template cast<CD>().eval().array()) ); + } + + rcd = mcd; + VERIFY_IS_APPROX( rcd = md, md.template cast<CD>().eval() ); + rcd = mcd; + VERIFY_IS_APPROX( rcd += md, mcd + md.template cast<CD>().eval() ); + rcd = mcd; + VERIFY_IS_APPROX( rcd -= md, mcd - md.template cast<CD>().eval() ); + rcd = mcd; + VERIFY_IS_APPROX( rcd.array() *= md.array(), mcd.array() * md.template cast<CD>().eval().array() ); + rcd = mcd; + if(md.array().abs().minCoeff()>epsd) + { + VERIFY_IS_APPROX( rcd.array() /= md.array(), mcd.array() / md.template cast<CD>().eval().array() ); + } + + rcd = mcd; + VERIFY_IS_APPROX( rcd.noalias() += md + mcd*md, mcd + (md.template cast<CD>().eval()) + mcd*(md.template cast<CD>().eval())); + + VERIFY_IS_APPROX( rcd.noalias() = md*md, ((md*md).eval().template cast<CD>()) ); + rcd = mcd; + VERIFY_IS_APPROX( rcd.noalias() += md*md, mcd + ((md*md).eval().template cast<CD>()) ); + rcd = mcd; + VERIFY_IS_APPROX( rcd.noalias() -= md*md, mcd - ((md*md).eval().template cast<CD>()) ); + + VERIFY_IS_APPROX( rcd.noalias() = mcd + md*md, mcd + ((md*md).eval().template cast<CD>()) ); + rcd = mcd; + VERIFY_IS_APPROX( rcd.noalias() += mcd + md*md, mcd + mcd + ((md*md).eval().template cast<CD>()) ); + rcd = mcd; + VERIFY_IS_APPROX( rcd.noalias() -= mcd + md*md, - ((md*md).eval().template cast<CD>()) ); } -void test_mixingtypes() +EIGEN_DECLARE_TEST(mixingtypes) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(mixingtypes<3>()); @@ -196,5 +319,10 @@ CALL_SUBTEST_4(mixingtypes<3>()); CALL_SUBTEST_5(mixingtypes<4>()); CALL_SUBTEST_6(mixingtypes<Dynamic>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE))); + CALL_SUBTEST_7(raise_assertion<Dynamic>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE))); } + CALL_SUBTEST_7(raise_assertion<0>()); + CALL_SUBTEST_7(raise_assertion<3>()); + CALL_SUBTEST_7(raise_assertion<4>()); + CALL_SUBTEST_7(raise_assertion<Dynamic>(0)); }
diff --git a/test/mpl2only.cpp b/test/mpl2only.cpp index 5ef0d2b..7d04d6b 100644 --- a/test/mpl2only.cpp +++ b/test/mpl2only.cpp
@@ -12,7 +12,9 @@ #include <Eigen/SparseCore> #include <Eigen/SparseLU> #include <Eigen/SparseQR> +#include <Eigen/Sparse> #include <Eigen/IterativeLinearSolvers> +#include <Eigen/Eigen> int main() {
diff --git a/test/nestbyvalue.cpp b/test/nestbyvalue.cpp new file mode 100644 index 0000000..c5356bc --- /dev/null +++ b/test/nestbyvalue.cpp
@@ -0,0 +1,37 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2019 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define TEST_ENABLE_TEMPORARY_TRACKING + +#include "main.h" + +typedef NestByValue<MatrixXd> CpyMatrixXd; +typedef CwiseBinaryOp<internal::scalar_sum_op<double,double>,const CpyMatrixXd,const CpyMatrixXd> XprType; + +XprType get_xpr_with_temps(const MatrixXd& a) +{ + MatrixXd t1 = a.rowwise().reverse(); + MatrixXd t2 = a+a; + return t1.nestByValue() + t2.nestByValue(); +} + +EIGEN_DECLARE_TEST(nestbyvalue) +{ + for(int i = 0; i < g_repeat; i++) { + Index rows = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + Index cols = internal::random<Index>(1,EIGEN_TEST_MAX_SIZE); + MatrixXd a = MatrixXd(rows,cols); + nb_temporaries = 0; + XprType x = get_xpr_with_temps(a); + VERIFY_IS_EQUAL(nb_temporaries,6); + MatrixXd b = x; + VERIFY_IS_EQUAL(nb_temporaries,6+1); + VERIFY_IS_APPROX(b, a.rowwise().reverse().eval() + (a+a).eval()); + } +}
diff --git a/test/nesting_ops.cpp b/test/nesting_ops.cpp index 2f50253..4b5fc21 100644 --- a/test/nesting_ops.cpp +++ b/test/nesting_ops.cpp
@@ -75,8 +75,8 @@ } else { - VERIFY( verify_eval_type<1>(2*m1, 2*m1) ); - VERIFY( verify_eval_type<2>(2*m1, m1) ); + VERIFY( verify_eval_type<2>(2*m1, 2*m1) ); + VERIFY( verify_eval_type<3>(2*m1, m1) ); } VERIFY( verify_eval_type<2>(m1+m1, m1+m1) ); VERIFY( verify_eval_type<3>(m1+m1, m1) ); @@ -91,7 +91,7 @@ } -void test_nesting_ops() +EIGEN_DECLARE_TEST(nesting_ops) { CALL_SUBTEST_1(run_nesting_ops_1(MatrixXf::Random(25,25))); CALL_SUBTEST_2(run_nesting_ops_1(MatrixXcd::Random(25,25)));
diff --git a/test/nomalloc.cpp b/test/nomalloc.cpp index 50756c2..cb4c073 100644 --- a/test/nomalloc.cpp +++ b/test/nomalloc.cpp
@@ -24,7 +24,6 @@ { /* this test check no dynamic memory allocation are issued with fixed-size matrices */ - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(); @@ -173,7 +172,7 @@ typedef typename MatrixType::Scalar Scalar; enum { Flag = MatrixType::IsRowMajor ? Eigen::RowMajor : Eigen::ColMajor}; enum { TransposeFlag = !MatrixType::IsRowMajor ? Eigen::RowMajor : Eigen::ColMajor}; - typename MatrixType::Index rows = m.rows(), cols=m.cols(); + Index rows = m.rows(), cols=m.cols(); typedef Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Flag > MatrixX; typedef Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, TransposeFlag> MatrixXT; // Dynamic reference: @@ -203,7 +202,7 @@ } -void test_nomalloc() +EIGEN_DECLARE_TEST(nomalloc) { // create some dynamic objects Eigen::MatrixXd M1 = MatrixXd::Random(3,3);
diff --git a/test/nullary.cpp b/test/nullary.cpp index cb87695..9b25ea4 100644 --- a/test/nullary.cpp +++ b/test/nullary.cpp
@@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2010-2011 Jitse Niesen <jitse@maths.leeds.ac.uk> +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -29,20 +30,56 @@ bool diagOK = (A.diagonal().array() == 1).all(); return offDiagOK && diagOK; + +} + +template<typename VectorType> +void check_extremity_accuracy(const VectorType &v, const typename VectorType::Scalar &low, const typename VectorType::Scalar &high) +{ + typedef typename VectorType::Scalar Scalar; + typedef typename VectorType::RealScalar RealScalar; + + RealScalar prec = internal::is_same<RealScalar,float>::value ? NumTraits<RealScalar>::dummy_precision()*10 : NumTraits<RealScalar>::dummy_precision()/10; + Index size = v.size(); + + if(size<20) + return; + + for (int i=0; i<size; ++i) + { + if(i<5 || i>size-6) + { + Scalar ref = (low*RealScalar(size-i-1))/RealScalar(size-1) + (high*RealScalar(i))/RealScalar(size-1); + if(std::abs(ref)>1) + { + if(!internal::isApprox(v(i), ref, prec)) + std::cout << v(i) << " != " << ref << " ; relative error: " << std::abs((v(i)-ref)/ref) << " ; required precision: " << prec << " ; range: " << low << "," << high << " ; i: " << i << "\n"; + VERIFY(internal::isApprox(v(i), (low*RealScalar(size-i-1))/RealScalar(size-1) + (high*RealScalar(i))/RealScalar(size-1), prec)); + } + } + } } template<typename VectorType> void testVectorType(const VectorType& base) { typedef typename VectorType::Scalar Scalar; + typedef typename VectorType::RealScalar RealScalar; const Index size = base.size(); Scalar high = internal::random<Scalar>(-500,500); Scalar low = (size == 1 ? high : internal::random<Scalar>(-500,500)); - if (low>high) std::swap(low,high); + if (numext::real(low)>numext::real(high)) std::swap(low,high); - const Scalar step = ((size == 1) ? 1 : (high-low)/(size-1)); + // check low==high + if(internal::random<float>(0.f,1.f)<0.05f) + low = high; + // check abs(low) >> abs(high) + else if(size>2 && std::numeric_limits<RealScalar>::max_exponent10>0 && internal::random<float>(0.f,1.f)<0.1f) + low = -internal::random<Scalar>(1,2) * RealScalar(std::pow(RealScalar(10),std::numeric_limits<RealScalar>::max_exponent10/2)); + + const Scalar step = ((size == 1) ? 1 : (high-low)/RealScalar(size-1)); // check whether the result yields what we expect it to do VectorType m(base); @@ -52,28 +89,45 @@ { VectorType n(size); for (int i=0; i<size; ++i) - n(i) = low+i*step; + n(i) = low+RealScalar(i)*step; VERIFY_IS_APPROX(m,n); + + CALL_SUBTEST( check_extremity_accuracy(m, low, high) ); } - VectorType n(size); - for (int i=0; i<size; ++i) - n(i) = size==1 ? low : (low + ((high-low)*Scalar(i))/(size-1)); - VERIFY_IS_APPROX(m,n); + RealScalar range_length = numext::real(high-low); + if((!NumTraits<Scalar>::IsInteger) || (range_length>=size && (Index(range_length)%(size-1))==0) || (Index(range_length+1)<size && (size%Index(range_length+1))==0)) + { + VectorType n(size); + if((!NumTraits<Scalar>::IsInteger) || (range_length>=size)) + for (int i=0; i<size; ++i) + n(i) = size==1 ? low : (low + ((high-low)*Scalar(i))/RealScalar(size-1)); + else + for (int i=0; i<size; ++i) + n(i) = size==1 ? low : low + Scalar((double(range_length+1)*double(i))/double(size)); + VERIFY_IS_APPROX(m,n); - // random access version - m = VectorType::LinSpaced(size,low,high); - VERIFY_IS_APPROX(m,n); + // random access version + m = VectorType::LinSpaced(size,low,high); + VERIFY_IS_APPROX(m,n); + VERIFY( internal::isApprox(m(m.size()-1),high) ); + VERIFY( size==1 || internal::isApprox(m(0),low) ); + VERIFY_IS_EQUAL(m(m.size()-1) , high); + if(!NumTraits<Scalar>::IsInteger) + CALL_SUBTEST( check_extremity_accuracy(m, low, high) ); + } - VERIFY( internal::isApprox(m(m.size()-1),high) ); - VERIFY( size==1 || internal::isApprox(m(0),low) ); + VERIFY( numext::real(m(m.size()-1)) <= numext::real(high) ); + VERIFY( (m.array().real() <= numext::real(high)).all() ); + VERIFY( (m.array().real() >= numext::real(low)).all() ); - // sequential access version - m = VectorType::LinSpaced(Sequential,size,low,high); - VERIFY_IS_APPROX(m,n); - VERIFY( internal::isApprox(m(m.size()-1),high) ); - VERIFY( size==1 || internal::isApprox(m(0),low) ); + VERIFY( numext::real(m(m.size()-1)) >= numext::real(low) ); + if(size>=1) + { + VERIFY( internal::isApprox(m(0),low) ); + VERIFY_IS_EQUAL(m(0) , low); + } // check whether everything works with row and col major vectors Matrix<Scalar,Dynamic,1> row_vector(size); @@ -82,7 +136,7 @@ col_vector.setLinSpaced(size,low,high); // when using the extended precision (e.g., FPU) the relative error might exceed 1 bit // when computing the squared sum in isApprox, thus the 2x factor. - VERIFY( row_vector.isApprox(col_vector.transpose(), Scalar(2)*NumTraits<Scalar>::epsilon())); + VERIFY( row_vector.isApprox(col_vector.transpose(), RealScalar(2)*NumTraits<Scalar>::epsilon())); Matrix<Scalar,Dynamic,1> size_changer(size+50); size_changer.setLinSpaced(size,low,high); @@ -95,46 +149,193 @@ VERIFY_IS_APPROX( ScalarMatrix::LinSpaced(1,low,high), ScalarMatrix::Constant(high) ); // regression test for bug 526 (linear vectorized transversal) - if (size > 1) { + if (size > 1 && (!NumTraits<Scalar>::IsInteger)) { m.tail(size-1).setLinSpaced(low, high); VERIFY_IS_APPROX(m(size-1), high); } + + // regression test for bug 1383 (LinSpaced with empty size/range) + { + Index n0 = VectorType::SizeAtCompileTime==Dynamic ? 0 : VectorType::SizeAtCompileTime; + low = internal::random<Scalar>(); + m = VectorType::LinSpaced(n0,low,low-RealScalar(1)); + VERIFY(m.size()==n0); + + if(VectorType::SizeAtCompileTime==Dynamic) + { + VERIFY_IS_EQUAL(VectorType::LinSpaced(n0,0,Scalar(n0-1)).sum(),Scalar(0)); + VERIFY_IS_EQUAL(VectorType::LinSpaced(n0,low,low-RealScalar(1)).sum(),Scalar(0)); + } + + m.setLinSpaced(n0,0,Scalar(n0-1)); + VERIFY(m.size()==n0); + m.setLinSpaced(n0,low,low-RealScalar(1)); + VERIFY(m.size()==n0); + + // empty range only: + VERIFY_IS_APPROX(VectorType::LinSpaced(size,low,low),VectorType::Constant(size,low)); + m.setLinSpaced(size,low,low); + VERIFY_IS_APPROX(m,VectorType::Constant(size,low)); + + if(NumTraits<Scalar>::IsInteger) + { + VERIFY_IS_APPROX( VectorType::LinSpaced(size,low,low+Scalar(size-1)), VectorType::LinSpaced(size,low+Scalar(size-1),low).reverse() ); + + if(VectorType::SizeAtCompileTime==Dynamic) + { + // Check negative multiplicator path: + for(Index k=1; k<5; ++k) + VERIFY_IS_APPROX( VectorType::LinSpaced(size,low,low+Scalar((size-1)*k)), VectorType::LinSpaced(size,low+Scalar((size-1)*k),low).reverse() ); + // Check negative divisor path: + for(Index k=1; k<5; ++k) + VERIFY_IS_APPROX( VectorType::LinSpaced(size*k,low,low+Scalar(size-1)), VectorType::LinSpaced(size*k,low+Scalar(size-1),low).reverse() ); + } + } + } + + // test setUnit() + if(m.size()>0) + { + for(Index k=0; k<10; ++k) + { + Index i = internal::random<Index>(0,m.size()-1); + m.setUnit(i); + VERIFY_IS_APPROX( m, VectorType::Unit(m.size(), i) ); + } + if(VectorType::SizeAtCompileTime==Dynamic) + { + Index i = internal::random<Index>(0,2*m.size()-1); + m.setUnit(2*m.size(),i); + VERIFY_IS_APPROX( m, VectorType::Unit(m.size(),i) ); + } + } + } template<typename MatrixType> void testMatrixType(const MatrixType& m) { + using std::abs; const Index rows = m.rows(); const Index cols = m.cols(); + typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; + + Scalar s1; + do { + s1 = internal::random<Scalar>(); + } while(abs(s1)<RealScalar(1e-5) && (!NumTraits<Scalar>::IsInteger)); MatrixType A; A.setIdentity(rows, cols); VERIFY(equalsIdentity(A)); VERIFY(equalsIdentity(MatrixType::Identity(rows, cols))); + + + A = MatrixType::Constant(rows,cols,s1); + Index i = internal::random<Index>(0,rows-1); + Index j = internal::random<Index>(0,cols-1); + VERIFY_IS_APPROX( MatrixType::Constant(rows,cols,s1)(i,j), s1 ); + VERIFY_IS_APPROX( MatrixType::Constant(rows,cols,s1).coeff(i,j), s1 ); + VERIFY_IS_APPROX( A(i,j), s1 ); } -void test_nullary() +template<int> +void bug79() +{ + // Assignment of a RowVectorXd to a MatrixXd (regression test for bug #79). + VERIFY( (MatrixXd(RowVectorXd::LinSpaced(3, 0, 1)) - RowVector3d(0, 0.5, 1)).norm() < std::numeric_limits<double>::epsilon() ); +} + +template<int> +void bug1630() +{ + Array4d x4 = Array4d::LinSpaced(0.0, 1.0); + Array3d x3(Array4d::LinSpaced(0.0, 1.0).head(3)); + VERIFY_IS_APPROX(x4.head(3), x3); +} + +template<int> +void nullary_overflow() +{ + // Check possible overflow issue + int n = 60000; + ArrayXi a1(n), a2(n); + a1.setLinSpaced(n, 0, n-1); + for(int i=0; i<n; ++i) + a2(i) = i; + VERIFY_IS_APPROX(a1,a2); +} + +template<int> +void nullary_internal_logic() +{ + // check some internal logic + VERIFY(( internal::has_nullary_operator<internal::scalar_constant_op<double> >::value )); + VERIFY(( !internal::has_unary_operator<internal::scalar_constant_op<double> >::value )); + VERIFY(( !internal::has_binary_operator<internal::scalar_constant_op<double> >::value )); + VERIFY(( internal::functor_has_linear_access<internal::scalar_constant_op<double> >::ret )); + + VERIFY(( !internal::has_nullary_operator<internal::scalar_identity_op<double> >::value )); + VERIFY(( !internal::has_unary_operator<internal::scalar_identity_op<double> >::value )); + VERIFY(( internal::has_binary_operator<internal::scalar_identity_op<double> >::value )); + VERIFY(( !internal::functor_has_linear_access<internal::scalar_identity_op<double> >::ret )); + + VERIFY(( !internal::has_nullary_operator<internal::linspaced_op<float> >::value )); + VERIFY(( internal::has_unary_operator<internal::linspaced_op<float> >::value )); + VERIFY(( !internal::has_binary_operator<internal::linspaced_op<float> >::value )); + VERIFY(( internal::functor_has_linear_access<internal::linspaced_op<float> >::ret )); + + // Regression unit test for a weird MSVC bug. + // Search "nullary_wrapper_workaround_msvc" in CoreEvaluators.h for the details. + // See also traits<Ref>::match. + { + MatrixXf A = MatrixXf::Random(3,3); + Ref<const MatrixXf> R = 2.0*A; + VERIFY_IS_APPROX(R, A+A); + + Ref<const MatrixXf> R1 = MatrixXf::Random(3,3)+A; + + VectorXi V = VectorXi::Random(3); + Ref<const VectorXi> R2 = VectorXi::LinSpaced(3,1,3)+V; + VERIFY_IS_APPROX(R2, V+Vector3i(1,2,3)); + + VERIFY(( internal::has_nullary_operator<internal::scalar_constant_op<float> >::value )); + VERIFY(( !internal::has_unary_operator<internal::scalar_constant_op<float> >::value )); + VERIFY(( !internal::has_binary_operator<internal::scalar_constant_op<float> >::value )); + VERIFY(( internal::functor_has_linear_access<internal::scalar_constant_op<float> >::ret )); + + VERIFY(( !internal::has_nullary_operator<internal::linspaced_op<int> >::value )); + VERIFY(( internal::has_unary_operator<internal::linspaced_op<int> >::value )); + VERIFY(( !internal::has_binary_operator<internal::linspaced_op<int> >::value )); + VERIFY(( internal::functor_has_linear_access<internal::linspaced_op<int> >::ret )); + } +} + +EIGEN_DECLARE_TEST(nullary) { CALL_SUBTEST_1( testMatrixType(Matrix2d()) ); CALL_SUBTEST_2( testMatrixType(MatrixXcf(internal::random<int>(1,300),internal::random<int>(1,300))) ); CALL_SUBTEST_3( testMatrixType(MatrixXf(internal::random<int>(1,300),internal::random<int>(1,300))) ); - for(int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_4( testVectorType(VectorXd(internal::random<int>(1,300))) ); + for(int i = 0; i < g_repeat*10; i++) { + CALL_SUBTEST_3( testVectorType(VectorXcd(internal::random<int>(1,30000))) ); + CALL_SUBTEST_4( testVectorType(VectorXd(internal::random<int>(1,30000))) ); CALL_SUBTEST_5( testVectorType(Vector4d()) ); // regression test for bug 232 CALL_SUBTEST_6( testVectorType(Vector3d()) ); - CALL_SUBTEST_7( testVectorType(VectorXf(internal::random<int>(1,300))) ); + CALL_SUBTEST_7( testVectorType(VectorXf(internal::random<int>(1,30000))) ); CALL_SUBTEST_8( testVectorType(Vector3f()) ); CALL_SUBTEST_8( testVectorType(Vector4f()) ); CALL_SUBTEST_8( testVectorType(Matrix<float,8,1>()) ); CALL_SUBTEST_8( testVectorType(Matrix<float,1,1>()) ); - CALL_SUBTEST_9( testVectorType(VectorXi(internal::random<int>(1,300))) ); + CALL_SUBTEST_9( testVectorType(VectorXi(internal::random<int>(1,10))) ); + CALL_SUBTEST_9( testVectorType(VectorXi(internal::random<int>(9,300))) ); CALL_SUBTEST_9( testVectorType(Matrix<int,1,1>()) ); } -#ifdef EIGEN_TEST_PART_6 - // Assignment of a RowVectorXd to a MatrixXd (regression test for bug #79). - VERIFY( (MatrixXd(RowVectorXd::LinSpaced(3, 0, 1)) - RowVector3d(0, 0.5, 1)).norm() < std::numeric_limits<double>::epsilon() ); -#endif + CALL_SUBTEST_6( bug79<0>() ); + CALL_SUBTEST_6( bug1630<0>() ); + CALL_SUBTEST_9( nullary_overflow<0>() ); + CALL_SUBTEST_10( nullary_internal_logic<0>() ); }
diff --git a/test/num_dimensions.cpp b/test/num_dimensions.cpp new file mode 100644 index 0000000..7ad7ef6 --- /dev/null +++ b/test/num_dimensions.cpp
@@ -0,0 +1,90 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" +#include <Eigen/SparseCore> + +template<int ExpectedDim,typename Xpr> +void check_dim(const Xpr& ) { + STATIC_CHECK( Xpr::NumDimensions == ExpectedDim ); +} + +#if EIGEN_HAS_CXX11 +template<template <typename,int,int> class Object> +void map_num_dimensions() +{ + typedef Object<double, 1, 1> ArrayScalarType; + typedef Object<double, 2, 1> ArrayVectorType; + typedef Object<double, 1, 2> TransposeArrayVectorType; + typedef Object<double, 2, 2> ArrayType; + typedef Object<double, Eigen::Dynamic, 1> DynamicArrayVectorType; + typedef Object<double, 1, Eigen::Dynamic> DynamicTransposeArrayVectorType; + typedef Object<double, Eigen::Dynamic, Eigen::Dynamic> DynamicArrayType; + + STATIC_CHECK(ArrayScalarType::NumDimensions == 0); + STATIC_CHECK(ArrayVectorType::NumDimensions == 1); + STATIC_CHECK(TransposeArrayVectorType::NumDimensions == 1); + STATIC_CHECK(ArrayType::NumDimensions == 2); + STATIC_CHECK(DynamicArrayVectorType::NumDimensions == 1); + STATIC_CHECK(DynamicTransposeArrayVectorType::NumDimensions == 1); + STATIC_CHECK(DynamicArrayType::NumDimensions == 2); + + typedef Eigen::Map<ArrayScalarType> ArrayScalarMap; + typedef Eigen::Map<ArrayVectorType> ArrayVectorMap; + typedef Eigen::Map<TransposeArrayVectorType> TransposeArrayVectorMap; + typedef Eigen::Map<ArrayType> ArrayMap; + typedef Eigen::Map<DynamicArrayVectorType> DynamicArrayVectorMap; + typedef Eigen::Map<DynamicTransposeArrayVectorType> DynamicTransposeArrayVectorMap; + typedef Eigen::Map<DynamicArrayType> DynamicArrayMap; + + STATIC_CHECK(ArrayScalarMap::NumDimensions == 0); + STATIC_CHECK(ArrayVectorMap::NumDimensions == 1); + STATIC_CHECK(TransposeArrayVectorMap::NumDimensions == 1); + STATIC_CHECK(ArrayMap::NumDimensions == 2); + STATIC_CHECK(DynamicArrayVectorMap::NumDimensions == 1); + STATIC_CHECK(DynamicTransposeArrayVectorMap::NumDimensions == 1); + STATIC_CHECK(DynamicArrayMap::NumDimensions == 2); +} + +template<typename Scalar, int Rows, int Cols> +using TArray = Array<Scalar,Rows,Cols>; + +template<typename Scalar, int Rows, int Cols> +using TMatrix = Matrix<Scalar,Rows,Cols>; + +#endif + +EIGEN_DECLARE_TEST(num_dimensions) +{ + int n = 10; + ArrayXXd A(n,n); + CALL_SUBTEST( check_dim<2>(A) ); + CALL_SUBTEST( check_dim<2>(A.block(1,1,2,2)) ); + CALL_SUBTEST( check_dim<1>(A.col(1)) ); + CALL_SUBTEST( check_dim<1>(A.row(1)) ); + + MatrixXd M(n,n); + CALL_SUBTEST( check_dim<0>(M.row(1)*M.col(1)) ); + + SparseMatrix<double> S(n,n); + CALL_SUBTEST( check_dim<2>(S) ); + CALL_SUBTEST( check_dim<2>(S.block(1,1,2,2)) ); + CALL_SUBTEST( check_dim<1>(S.col(1)) ); + CALL_SUBTEST( check_dim<1>(S.row(1)) ); + + SparseVector<double> s(n); + CALL_SUBTEST( check_dim<1>(s) ); + CALL_SUBTEST( check_dim<1>(s.head(2)) ); + + + #if EIGEN_HAS_CXX11 + CALL_SUBTEST( map_num_dimensions<TArray>() ); + CALL_SUBTEST( map_num_dimensions<TMatrix>() ); + #endif +}
diff --git a/test/numext.cpp b/test/numext.cpp new file mode 100644 index 0000000..8c6447d --- /dev/null +++ b/test/numext.cpp
@@ -0,0 +1,54 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +template<typename T> +void check_abs() { + typedef typename NumTraits<T>::Real Real; + Real zero(0); + + if(NumTraits<T>::IsSigned) + VERIFY_IS_EQUAL(numext::abs(-T(1)), T(1)); + VERIFY_IS_EQUAL(numext::abs(T(0)), T(0)); + VERIFY_IS_EQUAL(numext::abs(T(1)), T(1)); + + for(int k=0; k<g_repeat*100; ++k) + { + T x = internal::random<T>(); + if(!internal::is_same<T,bool>::value) + x = x/Real(2); + if(NumTraits<T>::IsSigned) + { + VERIFY_IS_EQUAL(numext::abs(x), numext::abs(-x)); + VERIFY( numext::abs(-x) >= zero ); + } + VERIFY( numext::abs(x) >= zero ); + VERIFY_IS_APPROX( numext::abs2(x), numext::abs2(numext::abs(x)) ); + } +} + +EIGEN_DECLARE_TEST(numext) { + CALL_SUBTEST( check_abs<bool>() ); + CALL_SUBTEST( check_abs<signed char>() ); + CALL_SUBTEST( check_abs<unsigned char>() ); + CALL_SUBTEST( check_abs<short>() ); + CALL_SUBTEST( check_abs<unsigned short>() ); + CALL_SUBTEST( check_abs<int>() ); + CALL_SUBTEST( check_abs<unsigned int>() ); + CALL_SUBTEST( check_abs<long>() ); + CALL_SUBTEST( check_abs<unsigned long>() ); + CALL_SUBTEST( check_abs<half>() ); + CALL_SUBTEST( check_abs<float>() ); + CALL_SUBTEST( check_abs<double>() ); + CALL_SUBTEST( check_abs<long double>() ); + + CALL_SUBTEST( check_abs<std::complex<float> >() ); + CALL_SUBTEST( check_abs<std::complex<double> >() ); +}
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 37da6c8..4906f6e 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -9,16 +9,70 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #include "main.h" +#include "unsupported/Eigen/SpecialFunctions" +#include <typeinfo> +#if defined __GNUC__ && __GNUC__>=6 + #pragma GCC diagnostic ignored "-Wignored-attributes" +#endif // using namespace Eigen; +#ifdef EIGEN_VECTORIZE_SSE +const bool g_vectorize_sse = true; +#else +const bool g_vectorize_sse = false; +#endif + +bool g_first_pass = true; + namespace Eigen { namespace internal { + template<typename T> T negate(const T& x) { return -x; } + +template<typename T> +Map<const Array<unsigned char,sizeof(T),1> > +bits(const T& x) { + return Map<const Array<unsigned char,sizeof(T),1> >(reinterpret_cast<const unsigned char *>(&x)); +} + +// The following implement bitwise operations on floating point types +template<typename T,typename Bits,typename Func> +T apply_bit_op(Bits a, Bits b, Func f) { + Array<unsigned char,sizeof(T),1> res; + for(Index i=0; i<res.size();++i) res[i] = f(a[i],b[i]); + return *reinterpret_cast<T*>(&res); +} + +#define EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,T) \ + template<> T EIGEN_CAT(p,OP)(const T& a,const T& b) { \ + return apply_bit_op<T>(bits(a),bits(b),FUNC); \ + } + +#define EIGEN_TEST_MAKE_BITWISE(OP,FUNC) \ + EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,float) \ + EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,double) \ + EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,half) \ + EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,std::complex<float>) \ + EIGEN_TEST_MAKE_BITWISE2(OP,FUNC,std::complex<double>) + +EIGEN_TEST_MAKE_BITWISE(xor,std::bit_xor<unsigned char>()) +EIGEN_TEST_MAKE_BITWISE(and,std::bit_and<unsigned char>()) +EIGEN_TEST_MAKE_BITWISE(or, std::bit_or<unsigned char>()) +struct bit_andnot{ + template<typename T> T + operator()(T a, T b) const { return a & (~b); } +}; +EIGEN_TEST_MAKE_BITWISE(andnot, bit_andnot()) +template<typename T> +bool biteq(T a, T b) { + return (bits(a) == bits(b)).all(); +} + } } -// NOTE: we disbale inlining for this function to workaround a GCC issue when using -O3 and the i387 FPU. +// NOTE: we disable inlining for this function to workaround a GCC issue when using -O3 and the i387 FPU. template<typename Scalar> EIGEN_DONT_INLINE bool isApproxAbs(const Scalar& a, const Scalar& b, const typename NumTraits<Scalar>::Real& refvalue) { @@ -42,7 +96,7 @@ { for (int i=0; i<size; ++i) { - if (a[i]!=b[i] && !internal::isApprox(a[i],b[i])) + if ((!internal::biteq(a[i],b[i])) && a[i]!=b[i] && !internal::isApprox(a[i],b[i])) { std::cout << "ref: [" << Map<const Matrix<Scalar,1,Dynamic> >(a,size) << "]" << " != vec: [" << Map<const Matrix<Scalar,1,Dynamic> >(b,size) << "]\n"; return false; @@ -99,21 +153,25 @@ #define REF_MUL(a,b) ((a)*(b)) #define REF_DIV(a,b) ((a)/(b)) -template<typename Scalar> void packetmath() +template<typename Scalar,typename Packet> void packetmath() { using std::abs; typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; - const int PacketSize = PacketTraits::size; + const int PacketSize = internal::unpacket_traits<Packet>::size; typedef typename NumTraits<Scalar>::Real RealScalar; + if (g_first_pass) + std::cerr << "=== Testing packet of type '" << typeid(Packet).name() + << "' and scalar type '" << typeid(Scalar).name() + << "' and size '" << PacketSize << "' ===\n" ; + const int max_size = PacketSize > 4 ? PacketSize : 4; const int size = PacketSize*max_size; EIGEN_ALIGN_MAX Scalar data1[size]; EIGEN_ALIGN_MAX Scalar data2[size]; EIGEN_ALIGN_MAX Packet packets[PacketSize*2]; EIGEN_ALIGN_MAX Scalar ref[size]; - RealScalar refvalue = 0; + RealScalar refvalue = RealScalar(0); for (int i=0; i<size; ++i) { data1[i] = internal::random<Scalar>()/RealScalar(PacketSize); @@ -138,35 +196,51 @@ for (int offset=0; offset<PacketSize; ++offset) { + #define MIN(A,B) (A<B?A:B) packets[0] = internal::pload<Packet>(data1); packets[1] = internal::pload<Packet>(data1+PacketSize); if (offset==0) internal::palign<0>(packets[0], packets[1]); - else if (offset==1) internal::palign<1>(packets[0], packets[1]); - else if (offset==2) internal::palign<2>(packets[0], packets[1]); - else if (offset==3) internal::palign<3>(packets[0], packets[1]); - else if (offset==4) internal::palign<4>(packets[0], packets[1]); - else if (offset==5) internal::palign<5>(packets[0], packets[1]); - else if (offset==6) internal::palign<6>(packets[0], packets[1]); - else if (offset==7) internal::palign<7>(packets[0], packets[1]); + else if (offset==1) internal::palign<MIN(1,PacketSize-1)>(packets[0], packets[1]); + else if (offset==2) internal::palign<MIN(2,PacketSize-1)>(packets[0], packets[1]); + else if (offset==3) internal::palign<MIN(3,PacketSize-1)>(packets[0], packets[1]); + else if (offset==4) internal::palign<MIN(4,PacketSize-1)>(packets[0], packets[1]); + else if (offset==5) internal::palign<MIN(5,PacketSize-1)>(packets[0], packets[1]); + else if (offset==6) internal::palign<MIN(6,PacketSize-1)>(packets[0], packets[1]); + else if (offset==7) internal::palign<MIN(7,PacketSize-1)>(packets[0], packets[1]); + else if (offset==8) internal::palign<MIN(8,PacketSize-1)>(packets[0], packets[1]); + else if (offset==9) internal::palign<MIN(9,PacketSize-1)>(packets[0], packets[1]); + else if (offset==10) internal::palign<MIN(10,PacketSize-1)>(packets[0], packets[1]); + else if (offset==11) internal::palign<MIN(11,PacketSize-1)>(packets[0], packets[1]); + else if (offset==12) internal::palign<MIN(12,PacketSize-1)>(packets[0], packets[1]); + else if (offset==13) internal::palign<MIN(13,PacketSize-1)>(packets[0], packets[1]); + else if (offset==14) internal::palign<MIN(14,PacketSize-1)>(packets[0], packets[1]); + else if (offset==15) internal::palign<MIN(15,PacketSize-1)>(packets[0], packets[1]); internal::pstore(data2, packets[0]); for (int i=0; i<PacketSize; ++i) ref[i] = data1[i+offset]; + // palign is not used anymore, so let's just put a warning if it fails + ++g_test_level; VERIFY(areApprox(ref, data2, PacketSize) && "internal::palign"); + --g_test_level; } VERIFY((!PacketTraits::Vectorizable) || PacketTraits::HasAdd); VERIFY((!PacketTraits::Vectorizable) || PacketTraits::HasSub); VERIFY((!PacketTraits::Vectorizable) || PacketTraits::HasMul); VERIFY((!PacketTraits::Vectorizable) || PacketTraits::HasNegate); - VERIFY((internal::is_same<Scalar,int>::value) || (!PacketTraits::Vectorizable) || PacketTraits::HasDiv); + // Disabled as it is not clear why it would be mandatory to support division. + //VERIFY((internal::is_same<Scalar,int>::value) || (!PacketTraits::Vectorizable) || PacketTraits::HasDiv); CHECK_CWISE2_IF(PacketTraits::HasAdd, REF_ADD, internal::padd); CHECK_CWISE2_IF(PacketTraits::HasSub, REF_SUB, internal::psub); CHECK_CWISE2_IF(PacketTraits::HasMul, REF_MUL, internal::pmul); CHECK_CWISE2_IF(PacketTraits::HasDiv, REF_DIV, internal::pdiv); + CHECK_CWISE1(internal::pnot, internal::pnot); + CHECK_CWISE1(internal::pzero, internal::pzero); + CHECK_CWISE1(internal::ptrue, internal::ptrue); CHECK_CWISE1(internal::negate, internal::pnegate); CHECK_CWISE1(numext::conj, internal::pconj); @@ -189,7 +263,7 @@ internal::pstore(data2+3*PacketSize, A3); VERIFY(areApprox(ref, data2, 4*PacketSize) && "internal::pbroadcast4"); } - + { for (int i=0; i<PacketSize*2; ++i) ref[i] = data1[i/PacketSize]; @@ -199,11 +273,12 @@ internal::pstore(data2+1*PacketSize, A1); VERIFY(areApprox(ref, data2, 2*PacketSize) && "internal::pbroadcast2"); } - + VERIFY(internal::isApprox(data1[0], internal::pfirst(internal::pload<Packet>(data1))) && "internal::pfirst"); - + if(PacketSize>1) { + // apply different offsets to check that ploaddup is robust to unaligned inputs for(int offset=0;offset<4;++offset) { for(int i=0;i<PacketSize/2;++i) @@ -215,6 +290,7 @@ if(PacketSize>2) { + // apply different offsets to check that ploadquad is robust to unaligned inputs for(int offset=0;offset<4;++offset) { for(int i=0;i<PacketSize/4;++i) @@ -224,34 +300,39 @@ } } - ref[0] = 0; + ref[0] = Scalar(0); for (int i=0; i<PacketSize; ++i) ref[0] += data1[i]; VERIFY(isApproxAbs(ref[0], internal::predux(internal::pload<Packet>(data1)), refvalue) && "internal::predux"); + if(PacketSize==8 && internal::unpacket_traits<typename internal::unpacket_traits<Packet>::half>::size ==4) // so far, predux_half_downto4 is only required in such a case { - for (int i=0; i<4; ++i) - ref[i] = 0; + int HalfPacketSize = PacketSize>4 ? PacketSize/2 : PacketSize; + for (int i=0; i<HalfPacketSize; ++i) + ref[i] = Scalar(0); for (int i=0; i<PacketSize; ++i) - ref[i%4] += data1[i]; - internal::pstore(data2, internal::predux4(internal::pload<Packet>(data1))); - VERIFY(areApprox(ref, data2, PacketSize>4?PacketSize/2:PacketSize) && "internal::predux4"); + ref[i%HalfPacketSize] += data1[i]; + internal::pstore(data2, internal::predux_half_dowto4(internal::pload<Packet>(data1))); + VERIFY(areApprox(ref, data2, HalfPacketSize) && "internal::predux_half_dowto4"); } - ref[0] = 1; + ref[0] = Scalar(1); for (int i=0; i<PacketSize; ++i) ref[0] *= data1[i]; VERIFY(internal::isApprox(ref[0], internal::predux_mul(internal::pload<Packet>(data1))) && "internal::predux_mul"); - for (int j=0; j<PacketSize; ++j) + if (PacketTraits::HasReduxp) { - ref[j] = 0; - for (int i=0; i<PacketSize; ++i) - ref[j] += data1[i+j*PacketSize]; - packets[j] = internal::pload<Packet>(data1+j*PacketSize); + for (int j=0; j<PacketSize; ++j) + { + ref[j] = Scalar(0); + for (int i=0; i<PacketSize; ++i) + ref[j] += data1[i+j*PacketSize]; + packets[j] = internal::pload<Packet>(data1+j*PacketSize); + } + internal::pstore(data2, internal::preduxp(packets)); + VERIFY(areApproxAbs(ref, data2, PacketSize, refvalue) && "internal::preduxp"); } - internal::pstore(data2, internal::preduxp(packets)); - VERIFY(areApproxAbs(ref, data2, PacketSize, refvalue) && "internal::preduxp"); for (int i=0; i<PacketSize; ++i) ref[i] = data1[PacketSize-i-1]; @@ -285,19 +366,61 @@ VERIFY(isApproxAbs(result[i], (selector.select[i] ? data1[i] : data2[i]), refvalue)); } } + + if (PacketTraits::HasBlend || g_vectorize_sse) { + // pinsertfirst + for (int i=0; i<PacketSize; ++i) + ref[i] = data1[i]; + Scalar s = internal::random<Scalar>(); + ref[0] = s; + internal::pstore(data2, internal::pinsertfirst(internal::pload<Packet>(data1),s)); + VERIFY(areApprox(ref, data2, PacketSize) && "internal::pinsertfirst"); + } + + if (PacketTraits::HasBlend || g_vectorize_sse) { + // pinsertlast + for (int i=0; i<PacketSize; ++i) + ref[i] = data1[i]; + Scalar s = internal::random<Scalar>(); + ref[PacketSize-1] = s; + internal::pstore(data2, internal::pinsertlast(internal::pload<Packet>(data1),s)); + VERIFY(areApprox(ref, data2, PacketSize) && "internal::pinsertlast"); + } + + { + for (int i=0; i<PacketSize; ++i) + { + data1[i] = internal::random<Scalar>(); + unsigned char v = internal::random<bool>() ? 0xff : 0; + char* bytes = (char*)(data1+PacketSize+i); + for(int k=0; k<int(sizeof(Scalar)); ++k) + bytes[k] = v; + } + CHECK_CWISE2_IF(true, internal::por, internal::por); + CHECK_CWISE2_IF(true, internal::pxor, internal::pxor); + CHECK_CWISE2_IF(true, internal::pand, internal::pand); + CHECK_CWISE2_IF(true, internal::pandnot, internal::pandnot); + } + + { + for (int i = 0; i < PacketSize; ++i) { + data1[i] = internal::random<Scalar>(); + data2[i] = (i % 2) ? data1[i] : Scalar(0); + } + CHECK_CWISE2_IF(true, internal::pcmp_eq, internal::pcmp_eq); + } } -template<typename Scalar> void packetmath_real() +template<typename Scalar,typename Packet> void packetmath_real() { using std::abs; typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; - const int PacketSize = PacketTraits::size; + const int PacketSize = internal::unpacket_traits<Packet>::size; const int size = PacketSize*4; - EIGEN_ALIGN_MAX Scalar data1[PacketTraits::size*4]; - EIGEN_ALIGN_MAX Scalar data2[PacketTraits::size*4]; - EIGEN_ALIGN_MAX Scalar ref[PacketTraits::size*4]; + EIGEN_ALIGN_MAX Scalar data1[PacketSize*4]; + EIGEN_ALIGN_MAX Scalar data2[PacketSize*4]; + EIGEN_ALIGN_MAX Scalar ref[PacketSize*4]; for (int i=0; i<size; ++i) { @@ -311,7 +434,7 @@ CHECK_CWISE1_IF(PacketTraits::HasRound, numext::round, internal::pround); CHECK_CWISE1_IF(PacketTraits::HasCeil, numext::ceil, internal::pceil); CHECK_CWISE1_IF(PacketTraits::HasFloor, numext::floor, internal::pfloor); - + for (int i=0; i<size; ++i) { data1[i] = internal::random<Scalar>(-1,1); @@ -332,7 +455,7 @@ data2[i] = internal::random<Scalar>(-1,1) * std::pow(Scalar(10), internal::random<Scalar>(-6,6)); } CHECK_CWISE1_IF(PacketTraits::HasTanh, std::tanh, internal::ptanh); - if(PacketTraits::HasExp && PacketTraits::size>=2) + if(PacketTraits::HasExp && PacketSize>=2) { data1[0] = std::numeric_limits<Scalar>::quiet_NaN(); data1[1] = std::numeric_limits<Scalar>::epsilon(); @@ -360,7 +483,15 @@ VERIFY_IS_EQUAL(std::exp(-std::numeric_limits<Scalar>::denorm_min()), data2[1]); } -#ifdef EIGEN_HAS_C99_MATH + if (PacketTraits::HasTanh) { + // NOTE this test migh fail with GCC prior to 6.3, see MathFunctionsImpl.h for details. + data1[0] = std::numeric_limits<Scalar>::quiet_NaN(); + packet_helper<internal::packet_traits<Scalar>::HasTanh,Packet> h; + h.store(data2, internal::ptanh(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + } + +#if EIGEN_HAS_C99_MATH { data1[0] = std::numeric_limits<Scalar>::quiet_NaN(); packet_helper<internal::packet_traits<Scalar>::HasLGamma,Packet> h; @@ -387,67 +518,122 @@ data2[i] = internal::random<Scalar>(0,1) * std::pow(Scalar(10), internal::random<Scalar>(-6,6)); } - if(internal::random<float>(0,1)<0.1) + if(internal::random<float>(0,1)<0.1f) data1[internal::random<int>(0, PacketSize)] = 0; CHECK_CWISE1_IF(PacketTraits::HasSqrt, std::sqrt, internal::psqrt); + CHECK_CWISE1_IF(PacketTraits::HasSqrt, Scalar(1)/std::sqrt, internal::prsqrt); CHECK_CWISE1_IF(PacketTraits::HasLog, std::log, internal::plog); -#if defined(EIGEN_HAS_C99_MATH) && (__cplusplus > 199711L) +#if EIGEN_HAS_C99_MATH && (__cplusplus > 199711L) + CHECK_CWISE1_IF(PacketTraits::HasExpm1, std::expm1, internal::pexpm1); + CHECK_CWISE1_IF(PacketTraits::HasLog1p, std::log1p, internal::plog1p); CHECK_CWISE1_IF(internal::packet_traits<Scalar>::HasLGamma, std::lgamma, internal::plgamma); CHECK_CWISE1_IF(internal::packet_traits<Scalar>::HasErf, std::erf, internal::perf); CHECK_CWISE1_IF(internal::packet_traits<Scalar>::HasErfc, std::erfc, internal::perfc); #endif - if(PacketTraits::HasLog && PacketTraits::size>=2) + if(PacketSize>=2) { data1[0] = std::numeric_limits<Scalar>::quiet_NaN(); data1[1] = std::numeric_limits<Scalar>::epsilon(); - packet_helper<PacketTraits::HasLog,Packet> h; - h.store(data2, internal::plog(h.load(data1))); - VERIFY((numext::isnan)(data2[0])); - VERIFY_IS_EQUAL(std::log(std::numeric_limits<Scalar>::epsilon()), data2[1]); + if(PacketTraits::HasLog) + { + packet_helper<PacketTraits::HasLog,Packet> h; + h.store(data2, internal::plog(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + VERIFY_IS_EQUAL(std::log(std::numeric_limits<Scalar>::epsilon()), data2[1]); - data1[0] = -std::numeric_limits<Scalar>::epsilon(); - data1[1] = 0; - h.store(data2, internal::plog(h.load(data1))); - VERIFY((numext::isnan)(data2[0])); - VERIFY_IS_EQUAL(std::log(Scalar(0)), data2[1]); + data1[0] = -std::numeric_limits<Scalar>::epsilon(); + data1[1] = 0; + h.store(data2, internal::plog(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + VERIFY_IS_EQUAL(std::log(Scalar(0)), data2[1]); - data1[0] = (std::numeric_limits<Scalar>::min)(); - data1[1] = -(std::numeric_limits<Scalar>::min)(); - h.store(data2, internal::plog(h.load(data1))); - VERIFY_IS_EQUAL(std::log((std::numeric_limits<Scalar>::min)()), data2[0]); - VERIFY((numext::isnan)(data2[1])); + data1[0] = (std::numeric_limits<Scalar>::min)(); + data1[1] = -(std::numeric_limits<Scalar>::min)(); + h.store(data2, internal::plog(h.load(data1))); + VERIFY_IS_EQUAL(std::log((std::numeric_limits<Scalar>::min)()), data2[0]); + VERIFY((numext::isnan)(data2[1])); - data1[0] = std::numeric_limits<Scalar>::denorm_min(); - data1[1] = -std::numeric_limits<Scalar>::denorm_min(); - h.store(data2, internal::plog(h.load(data1))); - // VERIFY_IS_EQUAL(std::log(std::numeric_limits<Scalar>::denorm_min()), data2[0]); - VERIFY((numext::isnan)(data2[1])); + data1[0] = std::numeric_limits<Scalar>::denorm_min(); + data1[1] = -std::numeric_limits<Scalar>::denorm_min(); + h.store(data2, internal::plog(h.load(data1))); + // VERIFY_IS_EQUAL(std::log(std::numeric_limits<Scalar>::denorm_min()), data2[0]); + VERIFY((numext::isnan)(data2[1])); - data1[0] = -1.0f; - h.store(data2, internal::plog(h.load(data1))); - VERIFY((numext::isnan)(data2[0])); -#if !EIGEN_FAST_MATH - h.store(data2, internal::psqrt(h.load(data1))); - VERIFY((numext::isnan)(data2[0])); - VERIFY((numext::isnan)(data2[1])); -#endif + data1[0] = Scalar(-1.0f); + h.store(data2, internal::plog(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + data1[0] = std::numeric_limits<Scalar>::infinity(); + h.store(data2, internal::plog(h.load(data1))); + VERIFY((numext::isinf)(data2[0])); + } + if(PacketTraits::HasSqrt) + { + packet_helper<PacketTraits::HasSqrt,Packet> h; + data1[0] = Scalar(-1.0f); + data1[1] = -std::numeric_limits<Scalar>::denorm_min(); + h.store(data2, internal::psqrt(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + VERIFY((numext::isnan)(data2[1])); + } + if(PacketTraits::HasCos) + { + packet_helper<PacketTraits::HasCos,Packet> h; + for(Scalar k = 1; k<Scalar(10000)/std::numeric_limits<Scalar>::epsilon(); k*=2) + { + for(int k1=0;k1<=1; ++k1) + { + data1[0] = (2*k+k1 )*Scalar(EIGEN_PI)/2 * internal::random<Scalar>(0.8,1.2); + data1[1] = (2*k+2+k1)*Scalar(EIGEN_PI)/2 * internal::random<Scalar>(0.8,1.2); + h.store(data2, internal::pcos(h.load(data1))); + h.store(data2+PacketSize, internal::psin(h.load(data1))); + VERIFY(data2[0]<=Scalar(1.) && data2[0]>=Scalar(-1.)); + VERIFY(data2[1]<=Scalar(1.) && data2[1]>=Scalar(-1.)); + VERIFY(data2[PacketSize+0]<=Scalar(1.) && data2[PacketSize+0]>=Scalar(-1.)); + VERIFY(data2[PacketSize+1]<=Scalar(1.) && data2[PacketSize+1]>=Scalar(-1.)); + + VERIFY_IS_APPROX(numext::abs2(data2[0])+numext::abs2(data2[PacketSize+0]), Scalar(1)); + VERIFY_IS_APPROX(numext::abs2(data2[1])+numext::abs2(data2[PacketSize+1]), Scalar(1)); + } + } + + data1[0] = std::numeric_limits<Scalar>::infinity(); + data1[1] = -std::numeric_limits<Scalar>::infinity(); + h.store(data2, internal::psin(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + VERIFY((numext::isnan)(data2[1])); + + h.store(data2, internal::pcos(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + VERIFY((numext::isnan)(data2[1])); + + data1[0] = std::numeric_limits<Scalar>::quiet_NaN(); + h.store(data2, internal::psin(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + h.store(data2, internal::pcos(h.load(data1))); + VERIFY((numext::isnan)(data2[0])); + + data1[0] = -Scalar(0.); + h.store(data2, internal::psin(h.load(data1))); + VERIFY( internal::biteq(data2[0], data1[0]) ); + h.store(data2, internal::pcos(h.load(data1))); + VERIFY_IS_EQUAL(data2[0], Scalar(1)); + } } } -template<typename Scalar> void packetmath_notcomplex() +template<typename Scalar,typename Packet> void packetmath_notcomplex() { using std::abs; typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; - const int PacketSize = PacketTraits::size; + const int PacketSize = internal::unpacket_traits<Packet>::size; - EIGEN_ALIGN_MAX Scalar data1[PacketTraits::size*4]; - EIGEN_ALIGN_MAX Scalar data2[PacketTraits::size*4]; - EIGEN_ALIGN_MAX Scalar ref[PacketTraits::size*4]; - - Array<Scalar,Dynamic,1>::Map(data1, PacketTraits::size*4).setRandom(); + EIGEN_ALIGN_MAX Scalar data1[PacketSize*4]; + EIGEN_ALIGN_MAX Scalar data2[PacketSize*4]; + EIGEN_ALIGN_MAX Scalar ref[PacketSize*4]; + + Array<Scalar,Dynamic,1>::Map(data1, PacketSize*4).setRandom(); ref[0] = data1[0]; for (int i=0; i<PacketSize; ++i) @@ -465,24 +651,45 @@ for (int i=0; i<PacketSize; ++i) ref[0] = (std::max)(ref[0],data1[i]); VERIFY(internal::isApprox(ref[0], internal::predux_max(internal::pload<Packet>(data1))) && "internal::predux_max"); - + for (int i=0; i<PacketSize; ++i) ref[i] = data1[0]+Scalar(i); internal::pstore(data2, internal::plset<Packet>(data1[0])); VERIFY(areApprox(ref, data2, PacketSize) && "internal::plset"); + + { + unsigned char* data1_bits = reinterpret_cast<unsigned char*>(data1); + // predux_all - not needed yet + // for (unsigned int i=0; i<PacketSize*sizeof(Scalar); ++i) data1_bits[i] = 0xff; + // VERIFY(internal::predux_all(internal::pload<Packet>(data1)) && "internal::predux_all(1111)"); + // for(int k=0; k<PacketSize; ++k) + // { + // for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0x0; + // VERIFY( (!internal::predux_all(internal::pload<Packet>(data1))) && "internal::predux_all(0101)"); + // for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0xff; + // } + + // predux_any + for (unsigned int i=0; i<PacketSize*sizeof(Scalar); ++i) data1_bits[i] = 0x0; + VERIFY( (!internal::predux_any(internal::pload<Packet>(data1))) && "internal::predux_any(0000)"); + for(int k=0; k<PacketSize; ++k) + { + for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0xff; + VERIFY( internal::predux_any(internal::pload<Packet>(data1)) && "internal::predux_any(0101)"); + for (unsigned int i=0; i<sizeof(Scalar); ++i) data1_bits[k*sizeof(Scalar)+i] = 0x00; + } + } } -template<typename Scalar,bool ConjLhs,bool ConjRhs> void test_conj_helper(Scalar* data1, Scalar* data2, Scalar* ref, Scalar* pval) +template<typename Scalar,typename Packet,bool ConjLhs,bool ConjRhs> void test_conj_helper(Scalar* data1, Scalar* data2, Scalar* ref, Scalar* pval) { - typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; - const int PacketSize = PacketTraits::size; - + const int PacketSize = internal::unpacket_traits<Packet>::size; + internal::conj_if<ConjLhs> cj0; internal::conj_if<ConjRhs> cj1; internal::conj_helper<Scalar,Scalar,ConjLhs,ConjRhs> cj; internal::conj_helper<Packet,Packet,ConjLhs,ConjRhs> pcj; - + for(int i=0;i<PacketSize;++i) { ref[i] = cj0(data1[i]) * cj1(data2[i]); @@ -490,7 +697,7 @@ } internal::pstore(pval,pcj.pmul(internal::pload<Packet>(data1),internal::pload<Packet>(data2))); VERIFY(areApprox(ref, pval, PacketSize) && "conj_helper pmul"); - + for(int i=0;i<PacketSize;++i) { Scalar tmp = ref[i]; @@ -501,11 +708,9 @@ VERIFY(areApprox(ref, pval, PacketSize) && "conj_helper pmadd"); } -template<typename Scalar> void packetmath_complex() +template<typename Scalar,typename Packet> void packetmath_complex() { - typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; - const int PacketSize = PacketTraits::size; + const int PacketSize = internal::unpacket_traits<Packet>::size; const int size = PacketSize*4; EIGEN_ALIGN_MAX Scalar data1[PacketSize*4]; @@ -518,12 +723,12 @@ data1[i] = internal::random<Scalar>() * Scalar(1e2); data2[i] = internal::random<Scalar>() * Scalar(1e2); } - - test_conj_helper<Scalar,false,false> (data1,data2,ref,pval); - test_conj_helper<Scalar,false,true> (data1,data2,ref,pval); - test_conj_helper<Scalar,true,false> (data1,data2,ref,pval); - test_conj_helper<Scalar,true,true> (data1,data2,ref,pval); - + + test_conj_helper<Scalar,Packet,false,false> (data1,data2,ref,pval); + test_conj_helper<Scalar,Packet,false,true> (data1,data2,ref,pval); + test_conj_helper<Scalar,Packet,true,false> (data1,data2,ref,pval); + test_conj_helper<Scalar,Packet,true,true> (data1,data2,ref,pval); + { for(int i=0;i<PacketSize;++i) ref[i] = Scalar(std::imag(data1[i]),std::real(data1[i])); @@ -532,22 +737,20 @@ } } -template<typename Scalar> void packetmath_scatter_gather() +template<typename Scalar,typename Packet> void packetmath_scatter_gather() { - typedef internal::packet_traits<Scalar> PacketTraits; - typedef typename PacketTraits::type Packet; typedef typename NumTraits<Scalar>::Real RealScalar; - const int PacketSize = PacketTraits::size; + const int PacketSize = internal::unpacket_traits<Packet>::size; EIGEN_ALIGN_MAX Scalar data1[PacketSize]; RealScalar refvalue = 0; for (int i=0; i<PacketSize; ++i) { data1[i] = internal::random<Scalar>()/RealScalar(PacketSize); } - + int stride = internal::random<int>(1,20); - + EIGEN_ALIGN_MAX Scalar buffer[PacketSize*20]; - memset(buffer, 0, 20*sizeof(Packet)); + memset(buffer, 0, 20*PacketSize*sizeof(Scalar)); Packet packet = internal::pload<Packet>(data1); internal::pscatter<Scalar, Packet>(buffer, packet, stride); @@ -569,29 +772,86 @@ } } -void test_packetmath() + +template< + typename Scalar, + typename PacketType, + bool IsComplex = NumTraits<Scalar>::IsComplex, + bool IsInteger = NumTraits<Scalar>::IsInteger> +struct runall; + +template<typename Scalar,typename PacketType> +struct runall<Scalar,PacketType,false,false> { // i.e. float or double + static void run() { + packetmath<Scalar,PacketType>(); + packetmath_scatter_gather<Scalar,PacketType>(); + packetmath_notcomplex<Scalar,PacketType>(); + packetmath_real<Scalar,PacketType>(); + } +}; + +template<typename Scalar,typename PacketType> +struct runall<Scalar,PacketType,false,true> { // i.e. int + static void run() { + packetmath<Scalar,PacketType>(); + packetmath_scatter_gather<Scalar,PacketType>(); + packetmath_notcomplex<Scalar,PacketType>(); + } +}; + +template<typename Scalar,typename PacketType> +struct runall<Scalar,PacketType,true,false> { // i.e. complex + static void run() { + packetmath<Scalar,PacketType>(); + packetmath_scatter_gather<Scalar,PacketType>(); + packetmath_complex<Scalar,PacketType>(); + } +}; + +template< + typename Scalar, + typename PacketType = typename internal::packet_traits<Scalar>::type, + bool Vectorized = internal::packet_traits<Scalar>::Vectorizable, + bool HasHalf = !internal::is_same<typename internal::unpacket_traits<PacketType>::half,PacketType>::value > +struct runner; + +template<typename Scalar,typename PacketType> +struct runner<Scalar,PacketType,true,true> { + static void run() { + runall<Scalar,PacketType>::run(); + runner<Scalar,typename internal::unpacket_traits<PacketType>::half>::run(); + } +}; + +template<typename Scalar,typename PacketType> +struct runner<Scalar,PacketType,true,false> +{ + static void run() { + runall<Scalar,PacketType>::run(); + runall<Scalar,Scalar>::run(); + } +}; + +template<typename Scalar,typename PacketType> +struct runner<Scalar,PacketType,false,false> +{ + static void run() { + runall<Scalar,PacketType>::run(); + } +}; + +EIGEN_DECLARE_TEST(packetmath) +{ + g_first_pass = true; for(int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1( packetmath<float>() ); - CALL_SUBTEST_2( packetmath<double>() ); - CALL_SUBTEST_3( packetmath<int>() ); - CALL_SUBTEST_4( packetmath<std::complex<float> >() ); - CALL_SUBTEST_5( packetmath<std::complex<double> >() ); - - CALL_SUBTEST_1( packetmath_notcomplex<float>() ); - CALL_SUBTEST_2( packetmath_notcomplex<double>() ); - CALL_SUBTEST_3( packetmath_notcomplex<int>() ); - CALL_SUBTEST_1( packetmath_real<float>() ); - CALL_SUBTEST_2( packetmath_real<double>() ); - - CALL_SUBTEST_4( packetmath_complex<std::complex<float> >() ); - CALL_SUBTEST_5( packetmath_complex<std::complex<double> >() ); - - CALL_SUBTEST_1( packetmath_scatter_gather<float>() ); - CALL_SUBTEST_2( packetmath_scatter_gather<double>() ); - CALL_SUBTEST_3( packetmath_scatter_gather<int>() ); - CALL_SUBTEST_4( packetmath_scatter_gather<std::complex<float> >() ); - CALL_SUBTEST_5( packetmath_scatter_gather<std::complex<double> >() ); + CALL_SUBTEST_1( runner<float>::run() ); + CALL_SUBTEST_2( runner<double>::run() ); + CALL_SUBTEST_3( runner<int>::run() ); + CALL_SUBTEST_4( runner<std::complex<float> >::run() ); + CALL_SUBTEST_5( runner<std::complex<double> >::run() ); + CALL_SUBTEST_6(( packetmath<half,internal::packet_traits<half>::type>() )); + g_first_pass = false; } }
diff --git a/test/pardiso_support.cpp b/test/pardiso_support.cpp index 67efad6..9c16ded 100644 --- a/test/pardiso_support.cpp +++ b/test/pardiso_support.cpp
@@ -20,7 +20,7 @@ check_sparse_square_solving(pardiso_lu); } -void test_pardiso_support() +EIGEN_DECLARE_TEST(pardiso_support) { CALL_SUBTEST_1(test_pardiso_T<float>()); CALL_SUBTEST_2(test_pardiso_T<double>());
diff --git a/test/pastix_support.cpp b/test/pastix_support.cpp index b62f857..9b64417 100644 --- a/test/pastix_support.cpp +++ b/test/pastix_support.cpp
@@ -45,7 +45,7 @@ check_sparse_square_solving(pastix_lu); } -void test_pastix_support() +EIGEN_DECLARE_TEST(pastix_support) { CALL_SUBTEST_1(test_pastix_T<float>()); CALL_SUBTEST_2(test_pastix_T<double>());
diff --git a/test/permutationmatrices.cpp b/test/permutationmatrices.cpp index 41aa57d..a26b4ad 100644 --- a/test/permutationmatrices.cpp +++ b/test/permutationmatrices.cpp
@@ -14,14 +14,15 @@ using namespace std; template<typename MatrixType> void permutationmatrices(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; enum { Rows = MatrixType::RowsAtCompileTime, Cols = MatrixType::ColsAtCompileTime, Options = MatrixType::Options }; typedef PermutationMatrix<Rows> LeftPermutationType; + typedef Transpositions<Rows> LeftTranspositionsType; typedef Matrix<int, Rows, 1> LeftPermutationVectorType; typedef Map<LeftPermutationType> MapLeftPerm; typedef PermutationMatrix<Cols> RightPermutationType; + typedef Transpositions<Cols> RightTranspositionsType; typedef Matrix<int, Cols, 1> RightPermutationVectorType; typedef Map<RightPermutationType> MapRightPerm; @@ -35,10 +36,11 @@ RightPermutationVectorType rv; randomPermutationVector(rv, cols); RightPermutationType rp(rv); + LeftTranspositionsType lt(lv); + RightTranspositionsType rt(rv); MatrixType m_permuted = MatrixType::Random(rows,cols); - const int one_if_dynamic = MatrixType::SizeAtCompileTime==Dynamic ? 1 : 0; - VERIFY_EVALUATION_COUNT(m_permuted = lp * m_original * rp, one_if_dynamic); // 1 temp for sub expression "lp * m_original" + VERIFY_EVALUATION_COUNT(m_permuted = lp * m_original * rp, 1); // 1 temp for sub expression "lp * m_original" for (int i=0; i<rows; i++) for (int j=0; j<cols; j++) @@ -50,7 +52,7 @@ VERIFY_IS_APPROX(m_permuted, lm*m_original*rm); m_permuted = m_original; - VERIFY_EVALUATION_COUNT(m_permuted = lp * m_permuted * rp, one_if_dynamic); + VERIFY_EVALUATION_COUNT(m_permuted = lp * m_permuted * rp, 1); VERIFY_IS_APPROX(m_permuted, lm*m_original*rm); VERIFY_IS_APPROX(lp.inverse()*m_permuted*rp.inverse(), m_original); @@ -75,19 +77,19 @@ // check inplace permutations m_permuted = m_original; - VERIFY_EVALUATION_COUNT(m_permuted.noalias()= lp.inverse() * m_permuted, one_if_dynamic); // 1 temp to allocate the mask + VERIFY_EVALUATION_COUNT(m_permuted.noalias()= lp.inverse() * m_permuted, 1); // 1 temp to allocate the mask VERIFY_IS_APPROX(m_permuted, lp.inverse()*m_original); m_permuted = m_original; - VERIFY_EVALUATION_COUNT(m_permuted.noalias() = m_permuted * rp.inverse(), one_if_dynamic); // 1 temp to allocate the mask + VERIFY_EVALUATION_COUNT(m_permuted.noalias() = m_permuted * rp.inverse(), 1); // 1 temp to allocate the mask VERIFY_IS_APPROX(m_permuted, m_original*rp.inverse()); m_permuted = m_original; - VERIFY_EVALUATION_COUNT(m_permuted.noalias() = lp * m_permuted, one_if_dynamic); // 1 temp to allocate the mask + VERIFY_EVALUATION_COUNT(m_permuted.noalias() = lp * m_permuted, 1); // 1 temp to allocate the mask VERIFY_IS_APPROX(m_permuted, lp*m_original); m_permuted = m_original; - VERIFY_EVALUATION_COUNT(m_permuted.noalias() = m_permuted * rp, one_if_dynamic); // 1 temp to allocate the mask + VERIFY_EVALUATION_COUNT(m_permuted.noalias() = m_permuted * rp, 1); // 1 temp to allocate the mask VERIFY_IS_APPROX(m_permuted, m_original*rp); if(rows>1 && cols>1) @@ -108,7 +110,32 @@ rm = rp; rm.col(i).swap(rm.col(j)); VERIFY_IS_APPROX(rm, rp2.toDenseMatrix().template cast<Scalar>()); - } + } + + { + // simple compilation check + Matrix<Scalar, Cols, Cols> A = rp; + Matrix<Scalar, Cols, Cols> B = rp.transpose(); + VERIFY_IS_APPROX(A, B.transpose()); + } + + m_permuted = m_original; + lp = lt; + rp = rt; + VERIFY_EVALUATION_COUNT(m_permuted = lt * m_permuted * rt, 1); + VERIFY_IS_APPROX(m_permuted, lp*m_original*rp.transpose()); + + VERIFY_IS_APPROX(lt.inverse()*m_permuted*rt.inverse(), m_original); + + // Check inplace transpositions + m_permuted = m_original; + VERIFY_IS_APPROX(m_permuted = lt * m_permuted, lp * m_original); + m_permuted = m_original; + VERIFY_IS_APPROX(m_permuted = lt.inverse() * m_permuted, lp.inverse() * m_original); + m_permuted = m_original; + VERIFY_IS_APPROX(m_permuted = m_permuted * rt, m_original * rt); + m_permuted = m_original; + VERIFY_IS_APPROX(m_permuted = m_permuted * rt.inverse(), m_original * rt.inverse()); } template<typename T> @@ -130,12 +157,12 @@ MapType(v1.data(),2,1,S(1,1)) = P * MapType(rhs.data(),2,1,S(1,1)); VERIFY_IS_APPROX(v1, (P * rhs).eval()); - + MapType(v1.data(),2,1,S(1,1)) = P.inverse() * MapType(rhs.data(),2,1,S(1,1)); VERIFY_IS_APPROX(v1, (P.inverse() * rhs).eval()); } -void test_permutationmatrices() +EIGEN_DECLARE_TEST(permutationmatrices) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( permutationmatrices(Matrix<float, 1, 1>()) ); @@ -143,8 +170,8 @@ CALL_SUBTEST_3( permutationmatrices(Matrix<double,3,3,RowMajor>()) ); CALL_SUBTEST_4( permutationmatrices(Matrix4d()) ); CALL_SUBTEST_5( permutationmatrices(Matrix<double,40,60>()) ); - CALL_SUBTEST_6( permutationmatrices(Matrix<double,Dynamic,Dynamic,RowMajor>(20, 30)) ); - CALL_SUBTEST_7( permutationmatrices(MatrixXcf(15, 10)) ); + CALL_SUBTEST_6( permutationmatrices(Matrix<double,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_7( permutationmatrices(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); } CALL_SUBTEST_5( bug890<double>() ); }
diff --git a/test/prec_inverse_4x4.cpp b/test/prec_inverse_4x4.cpp index c4ef2d4..0724664 100644 --- a/test/prec_inverse_4x4.cpp +++ b/test/prec_inverse_4x4.cpp
@@ -53,14 +53,29 @@ // FIXME that 1.25 used to be 1.2 until we tested gcc 4.1 on 30 June 2010 and got 1.21. VERIFY(error_avg < (NumTraits<Scalar>::IsComplex ? 8.0 : 1.25)); VERIFY(error_max < (NumTraits<Scalar>::IsComplex ? 64.0 : 20.0)); + + { + int s = 5;//internal::random<int>(4,10); + int i = 0;//internal::random<int>(0,s-4); + int j = 0;//internal::random<int>(0,s-4); + Matrix<Scalar,5,5> mat(s,s); + mat.setRandom(); + MatrixType submat = mat.template block<4,4>(i,j); + MatrixType mat_inv = mat.template block<4,4>(i,j).inverse(); + VERIFY_IS_APPROX(mat_inv, submat.inverse()); + mat.template block<4,4>(i,j) = submat.inverse(); + VERIFY_IS_APPROX(mat_inv, (mat.template block<4,4>(i,j))); + } } -void test_prec_inverse_4x4() +EIGEN_DECLARE_TEST(prec_inverse_4x4) { CALL_SUBTEST_1((inverse_permutation_4x4<Matrix4f>())); CALL_SUBTEST_1(( inverse_general_4x4<Matrix4f>(200000 * g_repeat) )); + CALL_SUBTEST_1(( inverse_general_4x4<Matrix<float,4,4,RowMajor> >(200000 * g_repeat) )); CALL_SUBTEST_2((inverse_permutation_4x4<Matrix<double,4,4,RowMajor> >())); + CALL_SUBTEST_2(( inverse_general_4x4<Matrix<double,4,4,ColMajor> >(200000 * g_repeat) )); CALL_SUBTEST_2(( inverse_general_4x4<Matrix<double,4,4,RowMajor> >(200000 * g_repeat) )); CALL_SUBTEST_3((inverse_permutation_4x4<Matrix4cf>()));
diff --git a/test/product.h b/test/product.h index 27976a4..d26e806 100644 --- a/test/product.h +++ b/test/product.h
@@ -111,6 +111,17 @@ vcres.noalias() -= m1.transpose() * v1; VERIFY_IS_APPROX(vcres, vc2 - m1.transpose() * v1); + // test scaled products + res = square; + res.noalias() = s1 * m1 * m2.transpose(); + VERIFY_IS_APPROX(res, ((s1*m1).eval() * m2.transpose())); + res = square; + res.noalias() += s1 * m1 * m2.transpose(); + VERIFY_IS_APPROX(res, square + ((s1*m1).eval() * m2.transpose())); + res = square; + res.noalias() -= s1 * m1 * m2.transpose(); + VERIFY_IS_APPROX(res, square - ((s1*m1).eval() * m2.transpose())); + // test d ?= a+b*c rules res.noalias() = square + m1 * m2.transpose(); VERIFY_IS_APPROX(res, square + m1 * m2.transpose()); @@ -119,6 +130,14 @@ res.noalias() -= square + m1 * m2.transpose(); VERIFY_IS_APPROX(res, square + m1 * m2.transpose()); + // test d ?= a-b*c rules + res.noalias() = square - m1 * m2.transpose(); + VERIFY_IS_APPROX(res, square - m1 * m2.transpose()); + res.noalias() += square - m1 * m2.transpose(); + VERIFY_IS_APPROX(res, 2*(square - m1 * m2.transpose())); + res.noalias() -= square - m1 * m2.transpose(); + VERIFY_IS_APPROX(res, square - m1 * m2.transpose()); + tm1 = m1; VERIFY_IS_APPROX(tm1.transpose() * v1, m1.transpose() * v1); @@ -160,6 +179,29 @@ VERIFY_IS_APPROX(res2.block(0,0,1,cols).noalias() = m1.block(0,0,1,cols) * square2, (ref2.row(0) = m1.row(0) * square2)); } + // vector.block() (see bug 1283) + { + RowVectorType w1(rows); + VERIFY_IS_APPROX(square * v1.block(0,0,rows,1), square * v1); + VERIFY_IS_APPROX(w1.noalias() = square * v1.block(0,0,rows,1), square * v1); + VERIFY_IS_APPROX(w1.block(0,0,rows,1).noalias() = square * v1.block(0,0,rows,1), square * v1); + + Matrix<Scalar,1,MatrixType::ColsAtCompileTime> w2(cols); + VERIFY_IS_APPROX(vc2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.noalias() = vc2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.block(0,0,1,cols).noalias() = vc2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + + vc2 = square2.block(0,0,1,cols).transpose(); + VERIFY_IS_APPROX(square2.block(0,0,1,cols) * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.noalias() = square2.block(0,0,1,cols) * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.block(0,0,1,cols).noalias() = square2.block(0,0,1,cols) * square2, vc2.transpose() * square2); + + vc2 = square2.block(0,0,cols,1); + VERIFY_IS_APPROX(square2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.noalias() = square2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + VERIFY_IS_APPROX(w2.block(0,0,1,cols).noalias() = square2.block(0,0,cols,1).transpose() * square2, vc2.transpose() * square2); + } + // inner product { Scalar x = square2.row(c) * square2.col(c2); @@ -185,6 +227,8 @@ // CwiseBinaryOp VERIFY_IS_APPROX(x = y + A*x, A*z); x = z; + VERIFY_IS_APPROX(x = y - A*x, A*(-z)); + x = z; // CwiseUnaryOp VERIFY_IS_APPROX(x = Scalar(1.)*(A*x), A*z); } @@ -196,4 +240,5 @@ VERIFY_IS_APPROX(square * (s1*(square*square)), s1 * square * square * square); VERIFY_IS_APPROX(square * (square*square).conjugate(), square * square.conjugate() * square.conjugate()); } + }
diff --git a/test/product_extra.cpp b/test/product_extra.cpp index d253fd7..bd31df8 100644 --- a/test/product_extra.cpp +++ b/test/product_extra.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void product_extra(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, 1, Dynamic> RowVectorType; typedef Matrix<Scalar, Dynamic, 1> ColVectorType; @@ -98,6 +97,16 @@ // regression test MatrixType tmp = m1 * m1.adjoint() * s1; VERIFY_IS_APPROX(tmp, m1 * m1.adjoint() * s1); + + // regression test for bug 1343, assignment to arrays + Array<Scalar,Dynamic,1> a1 = m1 * vc2; + VERIFY_IS_APPROX(a1.matrix(),m1*vc2); + Array<Scalar,Dynamic,1> a2 = s1 * (m1 * vc2); + VERIFY_IS_APPROX(a2.matrix(),s1*m1*vc2); + Array<Scalar,1,Dynamic> a3 = v1 * m1; + VERIFY_IS_APPROX(a3.matrix(),v1*m1); + Array<Scalar,Dynamic,Dynamic> a4 = m1 * m2.adjoint(); + VERIFY_IS_APPROX(a4.matrix(),m1*m2.adjoint()); } // Regression test for bug reported at http://forum.kde.org/viewtopic.php?f=74&t=96947 @@ -256,7 +265,94 @@ return ret; } -void test_product_extra() +template<typename> +void aliasing_with_resize() +{ + Index m = internal::random<Index>(10,50); + Index n = internal::random<Index>(10,50); + MatrixXd A, B, C(m,n), D(m,m); + VectorXd a, b, c(n); + C.setRandom(); + D.setRandom(); + c.setRandom(); + double s = internal::random<double>(1,10); + + A = C; + B = A * A.transpose(); + A = A * A.transpose(); + VERIFY_IS_APPROX(A,B); + + A = C; + B = (A * A.transpose())/s; + A = (A * A.transpose())/s; + VERIFY_IS_APPROX(A,B); + + A = C; + B = (A * A.transpose()) + D; + A = (A * A.transpose()) + D; + VERIFY_IS_APPROX(A,B); + + A = C; + B = D + (A * A.transpose()); + A = D + (A * A.transpose()); + VERIFY_IS_APPROX(A,B); + + A = C; + B = s * (A * A.transpose()); + A = s * (A * A.transpose()); + VERIFY_IS_APPROX(A,B); + + A = C; + a = c; + b = (A * a)/s; + a = (A * a)/s; + VERIFY_IS_APPROX(a,b); +} + +template<int> +void bug_1308() +{ + int n = 10; + MatrixXd r(n,n); + VectorXd v = VectorXd::Random(n); + r = v * RowVectorXd::Ones(n); + VERIFY_IS_APPROX(r, v.rowwise().replicate(n)); + r = VectorXd::Ones(n) * v.transpose(); + VERIFY_IS_APPROX(r, v.rowwise().replicate(n).transpose()); + + Matrix4d ones44 = Matrix4d::Ones(); + Matrix4d m44 = Matrix4d::Ones() * Matrix4d::Ones(); + VERIFY_IS_APPROX(m44,Matrix4d::Constant(4)); + VERIFY_IS_APPROX(m44.noalias()=ones44*Matrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(m44.noalias()=ones44.transpose()*Matrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(m44.noalias()=Matrix4d::Ones()*ones44, Matrix4d::Constant(4)); + VERIFY_IS_APPROX(m44.noalias()=Matrix4d::Ones()*ones44.transpose(), Matrix4d::Constant(4)); + + typedef Matrix<double,4,4,RowMajor> RMatrix4d; + RMatrix4d r44 = Matrix4d::Ones() * Matrix4d::Ones(); + VERIFY_IS_APPROX(r44,Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=ones44*Matrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=ones44.transpose()*Matrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=Matrix4d::Ones()*ones44, Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=Matrix4d::Ones()*ones44.transpose(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=ones44*RMatrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=ones44.transpose()*RMatrix4d::Ones(), Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=RMatrix4d::Ones()*ones44, Matrix4d::Constant(4)); + VERIFY_IS_APPROX(r44.noalias()=RMatrix4d::Ones()*ones44.transpose(), Matrix4d::Constant(4)); + +// RowVector4d r4; + m44.setOnes(); + r44.setZero(); + VERIFY_IS_APPROX(r44.noalias() += m44.row(0).transpose() * RowVector4d::Ones(), ones44); + r44.setZero(); + VERIFY_IS_APPROX(r44.noalias() += m44.col(0) * RowVector4d::Ones(), ones44); + r44.setZero(); + VERIFY_IS_APPROX(r44.noalias() += Vector4d::Ones() * m44.row(0), ones44); + r44.setZero(); + VERIFY_IS_APPROX(r44.noalias() += Vector4d::Ones() * m44.col(0).transpose(), ones44); +} + +EIGEN_DECLARE_TEST(product_extra) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( product_extra(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); @@ -268,8 +364,11 @@ } CALL_SUBTEST_5( bug_127<0>() ); CALL_SUBTEST_5( bug_817<0>() ); + CALL_SUBTEST_5( bug_1308<0>() ); CALL_SUBTEST_6( unaligned_objects<0>() ); CALL_SUBTEST_7( compute_block_size<float>() ); CALL_SUBTEST_7( compute_block_size<double>() ); CALL_SUBTEST_7( compute_block_size<std::complex<double> >() ); + CALL_SUBTEST_8( aliasing_with_resize<void>() ); + }
diff --git a/test/product_large.cpp b/test/product_large.cpp index 98f84c5..578dfb5 100644 --- a/test/product_large.cpp +++ b/test/product_large.cpp
@@ -8,6 +8,7 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #include "product.h" +#include <Eigen/LU> template<typename T> void test_aliasing() @@ -30,19 +31,9 @@ x = z; } -void test_product_large() +template<int> +void product_large_regressions() { - for(int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1( product(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); - CALL_SUBTEST_2( product(MatrixXd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); - CALL_SUBTEST_3( product(MatrixXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); - CALL_SUBTEST_4( product(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2), internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2))) ); - CALL_SUBTEST_5( product(Matrix<float,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); - - CALL_SUBTEST_1( test_aliasing<float>() ); - } - -#if defined EIGEN_TEST_PART_6 { // test a specific issue in DiagonalProduct int N = 1000000; @@ -71,7 +62,7 @@ std::ptrdiff_t m1 = internal::random<int>(10,100)*16; std::ptrdiff_t n1 = internal::random<int>(10,100)*16; // only makes sure it compiles fine - internal::computeProductBlockingSizes<float,float>(k1,m1,n1,1); + internal::computeProductBlockingSizes<float,float,std::ptrdiff_t>(k1,m1,n1,1); } { @@ -95,7 +86,35 @@ * (((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A)) * ((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))); VERIFY_IS_APPROX(B,C); } -#endif +} + +template<int> +void bug_1622() { + typedef Matrix<double, 2, -1, 0, 2, -1> Mat2X; + Mat2X x(2,2); x.setRandom(); + MatrixXd y(2,2); y.setRandom(); + const Mat2X K1 = x * y.inverse(); + const Matrix2d K2 = x * y.inverse(); + VERIFY_IS_APPROX(K1,K2); +} + +EIGEN_DECLARE_TEST(product_large) +{ + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1( product(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_2( product(MatrixXd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_2( product(MatrixXd(internal::random<int>(1,10), internal::random<int>(1,10))) ); + + CALL_SUBTEST_3( product(MatrixXi(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + CALL_SUBTEST_4( product(MatrixXcf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2), internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2))) ); + CALL_SUBTEST_5( product(Matrix<float,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); + + CALL_SUBTEST_1( test_aliasing<float>() ); + + CALL_SUBTEST_6( bug_1622<1>() ); + } + + CALL_SUBTEST_6( product_large_regressions<0>() ); // Regression test for bug 714: #if defined EIGEN_HAS_OPENMP
diff --git a/test/product_mmtr.cpp b/test/product_mmtr.cpp index b66529a..bb19e6e 100644 --- a/test/product_mmtr.cpp +++ b/test/product_mmtr.cpp
@@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2010-2017 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -10,12 +10,19 @@ #include "main.h" #define CHECK_MMTR(DEST, TRI, OP) { \ + ref3 = DEST; \ ref2 = ref1 = DEST; \ DEST.template triangularView<TRI>() OP; \ ref1 OP; \ ref2.template triangularView<TRI>() \ = ref1.template triangularView<TRI>(); \ VERIFY_IS_APPROX(DEST,ref2); \ + \ + DEST = ref3; \ + ref3 = ref2; \ + ref3.diagonal() = DEST.diagonal(); \ + DEST.template triangularView<TRI|ZeroDiag>() OP; \ + VERIFY_IS_APPROX(DEST,ref3); \ } template<typename Scalar> void mmtr(int size) @@ -27,7 +34,7 @@ MatrixColMaj matc = MatrixColMaj::Zero(size, size); MatrixRowMaj matr = MatrixRowMaj::Zero(size, size); - MatrixColMaj ref1(size, size), ref2(size, size); + MatrixColMaj ref1(size, size), ref2(size, size), ref3(size,size); MatrixColMaj soc(size,othersize); soc.setRandom(); MatrixColMaj osc(othersize,size); osc.setRandom(); @@ -62,9 +69,22 @@ CHECK_MMTR(matc, Upper, -= (s*sqc).template triangularView<Upper>()*sqc); CHECK_MMTR(matc, Lower, = (s*sqr).template triangularView<Lower>()*sqc); CHECK_MMTR(matc, Upper, += (s*sqc).template triangularView<Lower>()*sqc); + + // check aliasing + ref2 = ref1 = matc; + ref1 = sqc.adjoint() * matc * sqc; + ref2.template triangularView<Upper>() = ref1.template triangularView<Upper>(); + matc.template triangularView<Upper>() = sqc.adjoint() * matc * sqc; + VERIFY_IS_APPROX(matc, ref2); + + ref2 = ref1 = matc; + ref1 = sqc * matc * sqc.adjoint(); + ref2.template triangularView<Lower>() = ref1.template triangularView<Lower>(); + matc.template triangularView<Lower>() = sqc * matc * sqc.adjoint(); + VERIFY_IS_APPROX(matc, ref2); } -void test_product_mmtr() +EIGEN_DECLARE_TEST(product_mmtr) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_notemporary.cpp b/test/product_notemporary.cpp index 5a3f3a0..20cb7c0 100644 --- a/test/product_notemporary.cpp +++ b/test/product_notemporary.cpp
@@ -11,11 +11,41 @@ #include "main.h" +template<typename Dst, typename Lhs, typename Rhs> +void check_scalar_multiple3(Dst &dst, const Lhs& A, const Rhs& B) +{ + VERIFY_EVALUATION_COUNT( (dst.noalias() = A * B), 0); + VERIFY_IS_APPROX( dst, (A.eval() * B.eval()).eval() ); + VERIFY_EVALUATION_COUNT( (dst.noalias() += A * B), 0); + VERIFY_IS_APPROX( dst, 2*(A.eval() * B.eval()).eval() ); + VERIFY_EVALUATION_COUNT( (dst.noalias() -= A * B), 0); + VERIFY_IS_APPROX( dst, (A.eval() * B.eval()).eval() ); +} + +template<typename Dst, typename Lhs, typename Rhs, typename S2> +void check_scalar_multiple2(Dst &dst, const Lhs& A, const Rhs& B, S2 s2) +{ + CALL_SUBTEST( check_scalar_multiple3(dst, A, B) ); + CALL_SUBTEST( check_scalar_multiple3(dst, A, -B) ); + CALL_SUBTEST( check_scalar_multiple3(dst, A, s2*B) ); + CALL_SUBTEST( check_scalar_multiple3(dst, A, B*s2) ); + CALL_SUBTEST( check_scalar_multiple3(dst, A, (B*s2).conjugate()) ); +} + +template<typename Dst, typename Lhs, typename Rhs, typename S1, typename S2> +void check_scalar_multiple1(Dst &dst, const Lhs& A, const Rhs& B, S1 s1, S2 s2) +{ + CALL_SUBTEST( check_scalar_multiple2(dst, A, B, s2) ); + CALL_SUBTEST( check_scalar_multiple2(dst, -A, B, s2) ); + CALL_SUBTEST( check_scalar_multiple2(dst, s1*A, B, s2) ); + CALL_SUBTEST( check_scalar_multiple2(dst, A*s1, B, s2) ); + CALL_SUBTEST( check_scalar_multiple2(dst, (A*s1).conjugate(), B, s2) ); +} + template<typename MatrixType> void product_notemporary(const MatrixType& m) { /* This test checks the number of temporaries created * during the evaluation of a complex expression */ - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef Matrix<Scalar, 1, Dynamic> RowVectorType; @@ -51,11 +81,15 @@ VERIFY_EVALUATION_COUNT( m3.noalias() = s1 * (m1 * m2.transpose()), 0); VERIFY_EVALUATION_COUNT( m3 = m3 + (m1 * m2.adjoint()), 1); + VERIFY_EVALUATION_COUNT( m3 = m3 - (m1 * m2.adjoint()), 1); VERIFY_EVALUATION_COUNT( m3 = m3 + (m1 * m2.adjoint()).transpose(), 1); VERIFY_EVALUATION_COUNT( m3.noalias() = m3 + m1 * m2.transpose(), 0); VERIFY_EVALUATION_COUNT( m3.noalias() += m3 + m1 * m2.transpose(), 0); VERIFY_EVALUATION_COUNT( m3.noalias() -= m3 + m1 * m2.transpose(), 0); + VERIFY_EVALUATION_COUNT( m3.noalias() = m3 - m1 * m2.transpose(), 0); + VERIFY_EVALUATION_COUNT( m3.noalias() += m3 - m1 * m2.transpose(), 0); + VERIFY_EVALUATION_COUNT( m3.noalias() -= m3 - m1 * m2.transpose(), 0); VERIFY_EVALUATION_COUNT( m3.noalias() = s1 * m1 * s2 * m2.adjoint(), 0); VERIFY_EVALUATION_COUNT( m3.noalias() = s1 * m1 * s2 * (m1*s3+m2*s2).adjoint(), 1); @@ -102,7 +136,9 @@ VERIFY_EVALUATION_COUNT( m3.noalias() = m1.block(r0,r0,r1,r1).template triangularView<UnitUpper>() * m2.block(r0,c0,r1,c1), 1); // Zero temporaries for lazy products ... + m3.setRandom(rows,cols); VERIFY_EVALUATION_COUNT( Scalar tmp = 0; tmp += Scalar(RealScalar(1)) / (m3.transpose().lazyProduct(m3)).diagonal().sum(), 0 ); + VERIFY_EVALUATION_COUNT( m3.noalias() = m1.conjugate().lazyProduct(m2.conjugate()), 0); // ... and even no temporary for even deeply (>=2) nested products VERIFY_EVALUATION_COUNT( Scalar tmp = 0; tmp += Scalar(RealScalar(1)) / (m3.transpose() * m3).diagonal().sum(), 0 ); @@ -124,18 +160,39 @@ VERIFY_EVALUATION_COUNT( cvres.noalias() = (rm3+rm3) * (m1*cv1), 1 ); // Check outer products + #ifdef EIGEN_ALLOCA + bool temp_via_alloca = m3.rows()*sizeof(Scalar) <= EIGEN_STACK_ALLOCATION_LIMIT; + #else + bool temp_via_alloca = false; + #endif m3 = cv1 * rv1; VERIFY_EVALUATION_COUNT( m3.noalias() = cv1 * rv1, 0 ); - VERIFY_EVALUATION_COUNT( m3.noalias() = (cv1+cv1) * (rv1+rv1), 1 ); + VERIFY_EVALUATION_COUNT( m3.noalias() = (cv1+cv1) * (rv1+rv1), temp_via_alloca ? 0 : 1 ); VERIFY_EVALUATION_COUNT( m3.noalias() = (m1*cv1) * (rv1), 1 ); VERIFY_EVALUATION_COUNT( m3.noalias() += (m1*cv1) * (rv1), 1 ); + rm3 = cv1 * rv1; + VERIFY_EVALUATION_COUNT( rm3.noalias() = cv1 * rv1, 0 ); + VERIFY_EVALUATION_COUNT( rm3.noalias() = (cv1+cv1) * (rv1+rv1), temp_via_alloca ? 0 : 1 ); VERIFY_EVALUATION_COUNT( rm3.noalias() = (cv1) * (rv1 * m1), 1 ); VERIFY_EVALUATION_COUNT( rm3.noalias() -= (cv1) * (rv1 * m1), 1 ); VERIFY_EVALUATION_COUNT( rm3.noalias() = (m1*cv1) * (rv1 * m1), 2 ); VERIFY_EVALUATION_COUNT( rm3.noalias() += (m1*cv1) * (rv1 * m1), 2 ); + + // Check nested products + VERIFY_EVALUATION_COUNT( cvres.noalias() = m1.adjoint() * m1 * cv1, 1 ); + VERIFY_EVALUATION_COUNT( rvres.noalias() = rv1 * (m1 * m2.adjoint()), 1 ); + + // exhaustively check all scalar multiple combinations: + { + // Generic path: + check_scalar_multiple1(m3, m1, m2, s1, s2); + // Force fall back to coeff-based: + typename ColMajorMatrixType::BlockXpr m3_blck = m3.block(r0,r0,1,1); + check_scalar_multiple1(m3_blck, m1.block(r0,c0,1,1), m2.block(c0,r0,1,1), s1, s2); + } } -void test_product_notemporary() +EIGEN_DECLARE_TEST(product_notemporary) { int s; for(int i = 0; i < g_repeat; i++) {
diff --git a/test/product_selfadjoint.cpp b/test/product_selfadjoint.cpp index 3d768aa..bdccd04 100644 --- a/test/product_selfadjoint.cpp +++ b/test/product_selfadjoint.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void product_selfadjoint(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; typedef Matrix<Scalar, 1, MatrixType::RowsAtCompileTime> RowVectorType; @@ -60,7 +59,7 @@ } } -void test_product_selfadjoint() +EIGEN_DECLARE_TEST(product_selfadjoint) { int s = 0; for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_small.cpp b/test/product_small.cpp index c35db6f..b8ce37d 100644 --- a/test/product_small.cpp +++ b/test/product_small.cpp
@@ -12,6 +12,7 @@ #include <Eigen/LU> // regression test for bug 447 +template<int> void product1x1() { Matrix<float,1,3> matAstatic; @@ -177,34 +178,59 @@ CALL_SUBTEST(( test_lazy_all_layout<T,4,-1,-1>(4,cols,depth) )); } -void test_product_small() +template<typename T,int N,int M,int K> +void test_linear_but_not_vectorizable() { - for(int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1( product(Matrix<float, 3, 2>()) ); - CALL_SUBTEST_2( product(Matrix<int, 3, 5>()) ); - CALL_SUBTEST_3( product(Matrix3d()) ); - CALL_SUBTEST_4( product(Matrix4d()) ); - CALL_SUBTEST_5( product(Matrix4f()) ); - CALL_SUBTEST_6( product1x1() ); + // Check tricky cases for which the result of the product is a vector and thus must exhibit the LinearBit flag, + // but is not vectorizable along the linear dimension. + Index n = N==Dynamic ? internal::random<Index>(1,32) : N; + Index m = M==Dynamic ? internal::random<Index>(1,32) : M; + Index k = K==Dynamic ? internal::random<Index>(1,32) : K; - CALL_SUBTEST_11( test_lazy_l1<float>() ); - CALL_SUBTEST_12( test_lazy_l2<float>() ); - CALL_SUBTEST_13( test_lazy_l3<float>() ); + { + Matrix<T,N,M+1> A; A.setRandom(n,m+1); + Matrix<T,M*2,K> B; B.setRandom(m*2,k); + Matrix<T,1,K> C; + Matrix<T,1,K> R; - CALL_SUBTEST_21( test_lazy_l1<double>() ); - CALL_SUBTEST_22( test_lazy_l2<double>() ); - CALL_SUBTEST_23( test_lazy_l3<double>() ); - - CALL_SUBTEST_31( test_lazy_l1<std::complex<float> >() ); - CALL_SUBTEST_32( test_lazy_l2<std::complex<float> >() ); - CALL_SUBTEST_33( test_lazy_l3<std::complex<float> >() ); - - CALL_SUBTEST_41( test_lazy_l1<std::complex<double> >() ); - CALL_SUBTEST_42( test_lazy_l2<std::complex<double> >() ); - CALL_SUBTEST_43( test_lazy_l3<std::complex<double> >() ); + C.noalias() = A.template topLeftCorner<1,M>() * (B.template topRows<M>()+B.template bottomRows<M>()); + R.noalias() = A.template topLeftCorner<1,M>() * (B.template topRows<M>()+B.template bottomRows<M>()).eval(); + VERIFY_IS_APPROX(C,R); } -#ifdef EIGEN_TEST_PART_6 + { + Matrix<T,M+1,N,RowMajor> A; A.setRandom(m+1,n); + Matrix<T,K,M*2,RowMajor> B; B.setRandom(k,m*2); + Matrix<T,K,1> C; + Matrix<T,K,1> R; + + C.noalias() = (B.template leftCols<M>()+B.template rightCols<M>()) * A.template topLeftCorner<M,1>(); + R.noalias() = (B.template leftCols<M>()+B.template rightCols<M>()).eval() * A.template topLeftCorner<M,1>(); + VERIFY_IS_APPROX(C,R); + } +} + +template<int Rows> +void bug_1311() +{ + Matrix< double, Rows, 2 > A; A.setRandom(); + Vector2d b = Vector2d::Random() ; + Matrix<double,Rows,1> res; + res.noalias() = 1. * (A * b); + VERIFY_IS_APPROX(res, A*b); + res.noalias() = 1.*A * b; + VERIFY_IS_APPROX(res, A*b); + res.noalias() = (1.*A).lazyProduct(b); + VERIFY_IS_APPROX(res, A*b); + res.noalias() = (1.*A).lazyProduct(1.*b); + VERIFY_IS_APPROX(res, A*b); + res.noalias() = (A).lazyProduct(1.*b); + VERIFY_IS_APPROX(res, A*b); +} + +template<int> +void product_small_regressions() +{ { // test compilation of (outer_product) * vector Vector3f v = Vector3f::Random(); @@ -230,5 +256,42 @@ * (((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A)) * ((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))*((A*A)*(A*A))); VERIFY_IS_APPROX(B,C); } -#endif +} + +EIGEN_DECLARE_TEST(product_small) +{ + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1( product(Matrix<float, 3, 2>()) ); + CALL_SUBTEST_2( product(Matrix<int, 3, 17>()) ); + CALL_SUBTEST_8( product(Matrix<double, 3, 17>()) ); + CALL_SUBTEST_3( product(Matrix3d()) ); + CALL_SUBTEST_4( product(Matrix4d()) ); + CALL_SUBTEST_5( product(Matrix4f()) ); + CALL_SUBTEST_6( product1x1<0>() ); + + CALL_SUBTEST_11( test_lazy_l1<float>() ); + CALL_SUBTEST_12( test_lazy_l2<float>() ); + CALL_SUBTEST_13( test_lazy_l3<float>() ); + + CALL_SUBTEST_21( test_lazy_l1<double>() ); + CALL_SUBTEST_22( test_lazy_l2<double>() ); + CALL_SUBTEST_23( test_lazy_l3<double>() ); + + CALL_SUBTEST_31( test_lazy_l1<std::complex<float> >() ); + CALL_SUBTEST_32( test_lazy_l2<std::complex<float> >() ); + CALL_SUBTEST_33( test_lazy_l3<std::complex<float> >() ); + + CALL_SUBTEST_41( test_lazy_l1<std::complex<double> >() ); + CALL_SUBTEST_42( test_lazy_l2<std::complex<double> >() ); + CALL_SUBTEST_43( test_lazy_l3<std::complex<double> >() ); + + CALL_SUBTEST_7(( test_linear_but_not_vectorizable<float,2,1,Dynamic>() )); + CALL_SUBTEST_7(( test_linear_but_not_vectorizable<float,3,1,Dynamic>() )); + CALL_SUBTEST_7(( test_linear_but_not_vectorizable<float,2,1,16>() )); + + CALL_SUBTEST_6( bug_1311<3>() ); + CALL_SUBTEST_6( bug_1311<5>() ); + } + + CALL_SUBTEST_6( product_small_regressions<0>() ); }
diff --git a/test/product_symm.cpp b/test/product_symm.cpp index 74d7329..7d78646 100644 --- a/test/product_symm.cpp +++ b/test/product_symm.cpp
@@ -16,7 +16,6 @@ typedef Matrix<Scalar, OtherSize, Size> Rhs2; enum { order = OtherSize==1 ? 0 : RowMajor }; typedef Matrix<Scalar, Size, OtherSize,order> Rhs3; - typedef typename MatrixType::Index Index; Index rows = size; Index cols = size; @@ -39,6 +38,24 @@ VERIFY_IS_APPROX(rhs12 = (s1*m2).template selfadjointView<Lower>() * (s2*rhs1), rhs13 = (s1*m1) * (s2*rhs1)); + VERIFY_IS_APPROX(rhs12 = (s1*m2).transpose().template selfadjointView<Upper>() * (s2*rhs1), + rhs13 = (s1*m1.transpose()) * (s2*rhs1)); + + VERIFY_IS_APPROX(rhs12 = (s1*m2).template selfadjointView<Lower>().transpose() * (s2*rhs1), + rhs13 = (s1*m1.transpose()) * (s2*rhs1)); + + VERIFY_IS_APPROX(rhs12 = (s1*m2).conjugate().template selfadjointView<Lower>() * (s2*rhs1), + rhs13 = (s1*m1).conjugate() * (s2*rhs1)); + + VERIFY_IS_APPROX(rhs12 = (s1*m2).template selfadjointView<Lower>().conjugate() * (s2*rhs1), + rhs13 = (s1*m1).conjugate() * (s2*rhs1)); + + VERIFY_IS_APPROX(rhs12 = (s1*m2).adjoint().template selfadjointView<Upper>() * (s2*rhs1), + rhs13 = (s1*m1).adjoint() * (s2*rhs1)); + + VERIFY_IS_APPROX(rhs12 = (s1*m2).template selfadjointView<Lower>().adjoint() * (s2*rhs1), + rhs13 = (s1*m1).adjoint() * (s2*rhs1)); + m2 = m1.template triangularView<Upper>(); rhs12.setRandom(); rhs13 = rhs12; m3 = m2.template selfadjointView<Upper>(); VERIFY_IS_EQUAL(m1, m3); @@ -77,7 +94,7 @@ } -void test_product_symm() +EIGEN_DECLARE_TEST(product_symm) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_syrk.cpp b/test/product_syrk.cpp index e10f0f2..23406fe 100644 --- a/test/product_syrk.cpp +++ b/test/product_syrk.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void syrk(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime, RowMajor> RMatrixType; typedef Matrix<Scalar, MatrixType::ColsAtCompileTime, Dynamic> Rhs1; @@ -118,7 +117,7 @@ ((s1 * m1.row(c).adjoint() * m1.row(c).adjoint().adjoint()).eval().template triangularView<Upper>().toDenseMatrix())); } -void test_product_syrk() +EIGEN_DECLARE_TEST(product_syrk) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_trmm.cpp b/test/product_trmm.cpp index 12e5544..c7594e5 100644 --- a/test/product_trmm.cpp +++ b/test/product_trmm.cpp
@@ -29,7 +29,7 @@ typedef Matrix<Scalar,Dynamic,OtherCols,OtherCols==1?ColMajor:ResOrder> ResXS; typedef Matrix<Scalar,OtherCols,Dynamic,OtherCols==1?RowMajor:ResOrder> ResSX; - TriMatrix mat(rows,cols), tri(rows,cols), triTr(cols,rows); + TriMatrix mat(rows,cols), tri(rows,cols), triTr(cols,rows), s1tri(rows,cols), s1triTr(cols,rows); OnTheRight ge_right(cols,otherCols); OnTheLeft ge_left(otherCols,rows); @@ -42,6 +42,8 @@ mat.setRandom(); tri = mat.template triangularView<Mode>(); triTr = mat.transpose().template triangularView<Mode>(); + s1tri = (s1*mat).template triangularView<Mode>(); + s1triTr = (s1*mat).transpose().template triangularView<Mode>(); ge_right.setRandom(); ge_left.setRandom(); @@ -51,19 +53,29 @@ VERIFY_IS_APPROX( ge_xs.noalias() = mat.template triangularView<Mode>() * ge_right, tri * ge_right); VERIFY_IS_APPROX( ge_sx.noalias() = ge_left * mat.template triangularView<Mode>(), ge_left * tri); - VERIFY_IS_APPROX( ge_xs.noalias() = (s1*mat.adjoint()).template triangularView<Mode>() * (s2*ge_left.transpose()), s1*triTr.conjugate() * (s2*ge_left.transpose())); - VERIFY_IS_APPROX( ge_sx.noalias() = ge_right.transpose() * mat.adjoint().template triangularView<Mode>(), ge_right.transpose() * triTr.conjugate()); + if((Mode&UnitDiag)==0) + VERIFY_IS_APPROX( ge_xs.noalias() = (s1*mat.adjoint()).template triangularView<Mode>() * (s2*ge_left.transpose()), s1*triTr.conjugate() * (s2*ge_left.transpose())); - VERIFY_IS_APPROX( ge_xs.noalias() = (s1*mat.adjoint()).template triangularView<Mode>() * (s2*ge_left.adjoint()), s1*triTr.conjugate() * (s2*ge_left.adjoint())); - VERIFY_IS_APPROX( ge_sx.noalias() = ge_right.adjoint() * mat.adjoint().template triangularView<Mode>(), ge_right.adjoint() * triTr.conjugate()); + VERIFY_IS_APPROX( ge_xs.noalias() = (s1*mat.transpose()).template triangularView<Mode>() * (s2*ge_left.transpose()), s1triTr * (s2*ge_left.transpose())); + VERIFY_IS_APPROX( ge_sx.noalias() = (s2*ge_left) * (s1*mat).template triangularView<Mode>(), (s2*ge_left)*s1tri); + VERIFY_IS_APPROX( ge_sx.noalias() = ge_right.transpose() * mat.adjoint().template triangularView<Mode>(), ge_right.transpose() * triTr.conjugate()); + VERIFY_IS_APPROX( ge_sx.noalias() = ge_right.adjoint() * mat.adjoint().template triangularView<Mode>(), ge_right.adjoint() * triTr.conjugate()); + ge_xs_save = ge_xs; - VERIFY_IS_APPROX( (ge_xs_save + s1*triTr.conjugate() * (s2*ge_left.adjoint())).eval(), ge_xs.noalias() += (s1*mat.adjoint()).template triangularView<Mode>() * (s2*ge_left.adjoint()) ); + if((Mode&UnitDiag)==0) + VERIFY_IS_APPROX( (ge_xs_save + s1*triTr.conjugate() * (s2*ge_left.adjoint())).eval(), ge_xs.noalias() += (s1*mat.adjoint()).template triangularView<Mode>() * (s2*ge_left.adjoint()) ); + ge_xs_save = ge_xs; + VERIFY_IS_APPROX( (ge_xs_save + s1triTr * (s2*ge_left.adjoint())).eval(), ge_xs.noalias() += (s1*mat.transpose()).template triangularView<Mode>() * (s2*ge_left.adjoint()) ); ge_sx.setRandom(); ge_sx_save = ge_sx; - VERIFY_IS_APPROX( ge_sx_save - (ge_right.adjoint() * (-s1 * triTr).conjugate()).eval(), ge_sx.noalias() -= (ge_right.adjoint() * (-s1 * mat).adjoint().template triangularView<Mode>()).eval()); + if((Mode&UnitDiag)==0) + VERIFY_IS_APPROX( ge_sx_save - (ge_right.adjoint() * (-s1 * triTr).conjugate()).eval(), ge_sx.noalias() -= (ge_right.adjoint() * (-s1 * mat).adjoint().template triangularView<Mode>()).eval()); - VERIFY_IS_APPROX( ge_xs = (s1*mat).adjoint().template triangularView<Mode>() * ge_left.adjoint(), numext::conj(s1) * triTr.conjugate() * ge_left.adjoint()); + if((Mode&UnitDiag)==0) + VERIFY_IS_APPROX( ge_xs = (s1*mat).adjoint().template triangularView<Mode>() * ge_left.adjoint(), numext::conj(s1) * triTr.conjugate() * ge_left.adjoint()); + VERIFY_IS_APPROX( ge_xs = (s1*mat).transpose().template triangularView<Mode>() * ge_left.adjoint(), s1triTr * ge_left.adjoint()); + // TODO check with sub-matrix expressions ? } @@ -103,7 +115,7 @@ CALL_ALL_ORDERS(EIGEN_CAT(3,NB),SCALAR,StrictlyLower) -void test_product_trmm() +EIGEN_DECLARE_TEST(product_trmm) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_trmv.cpp b/test/product_trmv.cpp index 57a202a..5eb1b5a 100644 --- a/test/product_trmv.cpp +++ b/test/product_trmv.cpp
@@ -11,7 +11,6 @@ template<typename MatrixType> void trmv(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; @@ -71,7 +70,7 @@ // TODO check with sub-matrices } -void test_product_trmv() +EIGEN_DECLARE_TEST(product_trmv) { int s = 0; for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/product_trsolve.cpp b/test/product_trsolve.cpp index 4b97fa9..c927cb6 100644 --- a/test/product_trsolve.cpp +++ b/test/product_trsolve.cpp
@@ -71,9 +71,22 @@ int c = internal::random<int>(0,cols-1); VERIFY_TRSM(rmLhs.template triangularView<Lower>(), rmRhs.col(c)); VERIFY_TRSM(cmLhs.template triangularView<Lower>(), rmRhs.col(c)); + + if(Size==Dynamic) + { + cmLhs.resize(0,0); + cmRhs.resize(0,cmRhs.cols()); + Matrix<Scalar,Size,Cols,colmajor> res = cmLhs.template triangularView<Lower>().solve(cmRhs); + VERIFY_IS_EQUAL(res.rows(),0); + VERIFY_IS_EQUAL(res.cols(),cmRhs.cols()); + res = cmRhs; + cmLhs.template triangularView<Lower>().solveInPlace(res); + VERIFY_IS_EQUAL(res.rows(),0); + VERIFY_IS_EQUAL(res.cols(),cmRhs.cols()); + } } -void test_product_trsolve() +EIGEN_DECLARE_TEST(product_trsolve) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/qr.cpp b/test/qr.cpp index 9873877..c38e343 100644 --- a/test/qr.cpp +++ b/test/qr.cpp
@@ -9,11 +9,10 @@ #include "main.h" #include <Eigen/QR> +#include "solverbase.h" template<typename MatrixType> void qr(const MatrixType& m) { - typedef typename MatrixType::Index Index; - Index rows = m.rows(); Index cols = m.cols(); @@ -43,11 +42,7 @@ VERIFY_IS_APPROX(m1, qr.householderQ() * r); - Matrix<Scalar,Cols,Cols2> m2 = Matrix<Scalar,Cols,Cols2>::Random(Cols,Cols2); - Matrix<Scalar,Rows,Cols2> m3 = m1*m2; - m2 = Matrix<Scalar,Cols,Cols2>::Random(Cols,Cols2); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + check_solverbase<Matrix<Scalar,Cols,Cols2>, Matrix<Scalar,Rows,Cols2> >(m1, qr, Rows, Cols, Cols2); } template<typename MatrixType> void qr_invertible() @@ -59,6 +54,8 @@ typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; typedef typename MatrixType::Scalar Scalar; + STATIC_CHECK(( internal::is_same<typename HouseholderQR<MatrixType>::StorageIndex,int>::value )); + int size = internal::random<int>(10,50); MatrixType m1(size, size), m2(size, size), m3(size, size); @@ -72,9 +69,8 @@ } HouseholderQR<MatrixType> qr(m1); - m3 = MatrixType::Random(size,size); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + + check_solverbase<MatrixType, MatrixType>(m1, qr, size, size, size); // now construct a matrix with prescribed determinant m1.setZero(); @@ -85,8 +81,8 @@ qr.compute(m1); VERIFY_IS_APPROX(log(absdet), qr.logAbsDeterminant()); // This test is tricky if the determinant becomes too small. - // Since we generate random numbers with magnitude rrange [0,1], the average determinant is 0.5^size - VERIFY_IS_MUCH_SMALLER_THAN( abs(absdet-qr.absDeterminant()), (max)(RealScalar(pow(0.5,size)),(max)(abs(absdet),abs(qr.absDeterminant()))) ); + // Since we generate random numbers with magnitude range [0,1], the average determinant is 0.5^size + VERIFY_IS_MUCH_SMALLER_THAN( abs(absdet-qr.absDeterminant()), numext::maxi(RealScalar(pow(0.5,size)),numext::maxi<RealScalar>(abs(absdet),abs(qr.absDeterminant()))) ); } @@ -97,12 +93,14 @@ HouseholderQR<MatrixType> qr; VERIFY_RAISES_ASSERT(qr.matrixQR()) VERIFY_RAISES_ASSERT(qr.solve(tmp)) + VERIFY_RAISES_ASSERT(qr.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(qr.adjoint().solve(tmp)) VERIFY_RAISES_ASSERT(qr.householderQ()) VERIFY_RAISES_ASSERT(qr.absDeterminant()) VERIFY_RAISES_ASSERT(qr.logAbsDeterminant()) } -void test_qr() +EIGEN_DECLARE_TEST(qr) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( qr(MatrixXf(internal::random<int>(1,EIGEN_TEST_MAX_SIZE),internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) );
diff --git a/test/qr_colpivoting.cpp b/test/qr_colpivoting.cpp index 46c54b7..a563b54 100644 --- a/test/qr_colpivoting.cpp +++ b/test/qr_colpivoting.cpp
@@ -11,10 +11,11 @@ #include "main.h" #include <Eigen/QR> #include <Eigen/SVD> +#include "solverbase.h" template <typename MatrixType> void cod() { - typedef typename MatrixType::Index Index; + STATIC_CHECK(( internal::is_same<typename CompleteOrthogonalDecomposition<MatrixType>::StorageIndex,int>::value )); Index rows = internal::random<Index>(2, EIGEN_TEST_MAX_SIZE); Index cols = internal::random<Index>(2, EIGEN_TEST_MAX_SIZE); @@ -48,12 +49,12 @@ MatrixType c = q * t * z * cod.colsPermutation().inverse(); VERIFY_IS_APPROX(matrix, c); + check_solverbase<MatrixType, MatrixType>(matrix, cod, rows, cols, cols2); + + // Verify that we get the same minimum-norm solution as the SVD. MatrixType exact_solution = MatrixType::Random(cols, cols2); MatrixType rhs = matrix * exact_solution; MatrixType cod_solution = cod.solve(rhs); - VERIFY_IS_APPROX(rhs, matrix * cod_solution); - - // Verify that we get the same minimum-norm solution as the SVD. JacobiSVD<MatrixType> svd(matrix, ComputeThinU | ComputeThinV); MatrixType svd_solution = svd.solve(rhs); VERIFY_IS_APPROX(cod_solution, svd_solution); @@ -79,13 +80,13 @@ VERIFY(cod.isSurjective() == (rank == Cols)); VERIFY(cod.isInvertible() == (cod.isInjective() && cod.isSurjective())); + check_solverbase<Matrix<Scalar, Cols, Cols2>, Matrix<Scalar, Rows, Cols2> >(matrix, cod, Rows, Cols, Cols2); + + // Verify that we get the same minimum-norm solution as the SVD. Matrix<Scalar, Cols, Cols2> exact_solution; exact_solution.setRandom(Cols, Cols2); Matrix<Scalar, Rows, Cols2> rhs = matrix * exact_solution; Matrix<Scalar, Cols, Cols2> cod_solution = cod.solve(rhs); - VERIFY_IS_APPROX(rhs, matrix * cod_solution); - - // Verify that we get the same minimum-norm solution as the SVD. JacobiSVD<MatrixType> svd(matrix, ComputeFullU | ComputeFullV); Matrix<Scalar, Cols, Cols2> svd_solution = svd.solve(rhs); VERIFY_IS_APPROX(cod_solution, svd_solution); @@ -93,7 +94,9 @@ template<typename MatrixType> void qr() { - typedef typename MatrixType::Index Index; + using std::sqrt; + + STATIC_CHECK(( internal::is_same<typename ColPivHouseholderQR<MatrixType>::StorageIndex,int>::value )); Index rows = internal::random<Index>(2,EIGEN_TEST_MAX_SIZE), cols = internal::random<Index>(2,EIGEN_TEST_MAX_SIZE), cols2 = internal::random<Index>(2,EIGEN_TEST_MAX_SIZE); Index rank = internal::random<Index>(1, (std::min)(rows, cols)-1); @@ -120,14 +123,14 @@ // Verify that the absolute value of the diagonal elements in R are // non-increasing until they reach the singularity threshold. RealScalar threshold = - std::sqrt(RealScalar(rows)) * (std::abs)(r(0, 0)) * NumTraits<Scalar>::epsilon(); + sqrt(RealScalar(rows)) * numext::abs(r(0, 0)) * NumTraits<Scalar>::epsilon(); for (Index i = 0; i < (std::min)(rows, cols) - 1; ++i) { - RealScalar x = (std::abs)(r(i, i)); - RealScalar y = (std::abs)(r(i + 1, i + 1)); + RealScalar x = numext::abs(r(i, i)); + RealScalar y = numext::abs(r(i + 1, i + 1)); if (x < threshold && y < threshold) continue; if (!test_isApproxOrLessThan(y, x)) { for (Index j = 0; j < (std::min)(rows, cols); ++j) { - std::cout << "i = " << j << ", |r_ii| = " << (std::abs)(r(j, j)) << std::endl; + std::cout << "i = " << j << ", |r_ii| = " << numext::abs(r(j, j)) << std::endl; } std::cout << "Failure at i=" << i << ", rank=" << rank << ", threshold=" << threshold << std::endl; @@ -135,15 +138,26 @@ VERIFY_IS_APPROX_OR_LESS_THAN(y, x); } - MatrixType m2 = MatrixType::Random(cols,cols2); - MatrixType m3 = m1*m2; - m2 = MatrixType::Random(cols,cols2); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + check_solverbase<MatrixType, MatrixType>(m1, qr, rows, cols, cols2); + + { + MatrixType m2, m3; + Index size = rows; + do { + m1 = MatrixType::Random(size,size); + qr.compute(m1); + } while(!qr.isInvertible()); + MatrixType m1_inv = qr.inverse(); + m3 = m1 * MatrixType::Random(size,cols2); + m2 = qr.solve(m3); + VERIFY_IS_APPROX(m2, m1_inv*m3); + } } template<typename MatrixType, int Cols2> void qr_fixedsize() { + using std::sqrt; + using std::abs; enum { Rows = MatrixType::RowsAtCompileTime, Cols = MatrixType::ColsAtCompileTime }; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; @@ -161,22 +175,19 @@ Matrix<Scalar,Rows,Cols> c = qr.householderQ() * r * qr.colsPermutation().inverse(); VERIFY_IS_APPROX(m1, c); - Matrix<Scalar,Cols,Cols2> m2 = Matrix<Scalar,Cols,Cols2>::Random(Cols,Cols2); - Matrix<Scalar,Rows,Cols2> m3 = m1*m2; - m2 = Matrix<Scalar,Cols,Cols2>::Random(Cols,Cols2); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + check_solverbase<Matrix<Scalar,Cols,Cols2>, Matrix<Scalar,Rows,Cols2> >(m1, qr, Rows, Cols, Cols2); + // Verify that the absolute value of the diagonal elements in R are // non-increasing until they reache the singularity threshold. RealScalar threshold = - std::sqrt(RealScalar(Rows)) * (std::abs)(r(0, 0)) * NumTraits<Scalar>::epsilon(); + sqrt(RealScalar(Rows)) * (std::abs)(r(0, 0)) * NumTraits<Scalar>::epsilon(); for (Index i = 0; i < (std::min)(int(Rows), int(Cols)) - 1; ++i) { - RealScalar x = (std::abs)(r(i, i)); - RealScalar y = (std::abs)(r(i + 1, i + 1)); + RealScalar x = numext::abs(r(i, i)); + RealScalar y = numext::abs(r(i + 1, i + 1)); if (x < threshold && y < threshold) continue; if (!test_isApproxOrLessThan(y, x)) { for (Index j = 0; j < (std::min)(int(Rows), int(Cols)); ++j) { - std::cout << "i = " << j << ", |r_ii| = " << (std::abs)(r(j, j)) << std::endl; + std::cout << "i = " << j << ", |r_ii| = " << numext::abs(r(j, j)) << std::endl; } std::cout << "Failure at i=" << i << ", rank=" << rank << ", threshold=" << threshold << std::endl; @@ -194,7 +205,8 @@ // page 3 for more detail. template<typename MatrixType> void qr_kahan_matrix() { - typedef typename MatrixType::Index Index; + using std::sqrt; + using std::abs; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; @@ -204,23 +216,25 @@ m1.setZero(rows,cols); RealScalar s = std::pow(NumTraits<RealScalar>::epsilon(), 1.0 / rows); RealScalar c = std::sqrt(1 - s*s); + RealScalar pow_s_i(1.0); // pow(s,i) for (Index i = 0; i < rows; ++i) { - m1(i, i) = pow(s, i); - m1.row(i).tail(rows - i - 1) = -pow(s, i) * c * MatrixType::Ones(1, rows - i - 1); + m1(i, i) = pow_s_i; + m1.row(i).tail(rows - i - 1) = -pow_s_i * c * MatrixType::Ones(1, rows - i - 1); + pow_s_i *= s; } m1 = (m1 + m1.transpose()).eval(); ColPivHouseholderQR<MatrixType> qr(m1); MatrixType r = qr.matrixQR().template triangularView<Upper>(); RealScalar threshold = - std::sqrt(RealScalar(rows)) * (std::abs)(r(0, 0)) * NumTraits<Scalar>::epsilon(); + std::sqrt(RealScalar(rows)) * numext::abs(r(0, 0)) * NumTraits<Scalar>::epsilon(); for (Index i = 0; i < (std::min)(rows, cols) - 1; ++i) { - RealScalar x = (std::abs)(r(i, i)); - RealScalar y = (std::abs)(r(i + 1, i + 1)); + RealScalar x = numext::abs(r(i, i)); + RealScalar y = numext::abs(r(i + 1, i + 1)); if (x < threshold && y < threshold) continue; if (!test_isApproxOrLessThan(y, x)) { for (Index j = 0; j < (std::min)(rows, cols); ++j) { - std::cout << "i = " << j << ", |r_ii| = " << (std::abs)(r(j, j)) << std::endl; + std::cout << "i = " << j << ", |r_ii| = " << numext::abs(r(j, j)) << std::endl; } std::cout << "Failure at i=" << i << ", rank=" << qr.rank() << ", threshold=" << threshold << std::endl; @@ -249,9 +263,8 @@ } ColPivHouseholderQR<MatrixType> qr(m1); - m3 = MatrixType::Random(size,size); - m2 = qr.solve(m3); - //VERIFY_IS_APPROX(m3, m1*m2); + + check_solverbase<MatrixType, MatrixType>(m1, qr, size, size, size); // now construct a matrix with prescribed determinant m1.setZero(); @@ -271,6 +284,8 @@ ColPivHouseholderQR<MatrixType> qr; VERIFY_RAISES_ASSERT(qr.matrixQR()) VERIFY_RAISES_ASSERT(qr.solve(tmp)) + VERIFY_RAISES_ASSERT(qr.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(qr.adjoint().solve(tmp)) VERIFY_RAISES_ASSERT(qr.householderQ()) VERIFY_RAISES_ASSERT(qr.dimensionOfKernel()) VERIFY_RAISES_ASSERT(qr.isInjective()) @@ -281,7 +296,26 @@ VERIFY_RAISES_ASSERT(qr.logAbsDeterminant()) } -void test_qr_colpivoting() +template<typename MatrixType> void cod_verify_assert() +{ + MatrixType tmp; + + CompleteOrthogonalDecomposition<MatrixType> cod; + VERIFY_RAISES_ASSERT(cod.matrixQTZ()) + VERIFY_RAISES_ASSERT(cod.solve(tmp)) + VERIFY_RAISES_ASSERT(cod.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(cod.adjoint().solve(tmp)) + VERIFY_RAISES_ASSERT(cod.householderQ()) + VERIFY_RAISES_ASSERT(cod.dimensionOfKernel()) + VERIFY_RAISES_ASSERT(cod.isInjective()) + VERIFY_RAISES_ASSERT(cod.isSurjective()) + VERIFY_RAISES_ASSERT(cod.isInvertible()) + VERIFY_RAISES_ASSERT(cod.pseudoInverse()) + VERIFY_RAISES_ASSERT(cod.absDeterminant()) + VERIFY_RAISES_ASSERT(cod.logAbsDeterminant()) +} + +EIGEN_DECLARE_TEST(qr_colpivoting) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( qr<MatrixXf>() ); @@ -315,6 +349,13 @@ CALL_SUBTEST_6(qr_verify_assert<MatrixXcf>()); CALL_SUBTEST_3(qr_verify_assert<MatrixXcd>()); + CALL_SUBTEST_7(cod_verify_assert<Matrix3f>()); + CALL_SUBTEST_8(cod_verify_assert<Matrix3d>()); + CALL_SUBTEST_1(cod_verify_assert<MatrixXf>()); + CALL_SUBTEST_2(cod_verify_assert<MatrixXd>()); + CALL_SUBTEST_6(cod_verify_assert<MatrixXcf>()); + CALL_SUBTEST_3(cod_verify_assert<MatrixXcd>()); + // Test problem size constructors CALL_SUBTEST_9(ColPivHouseholderQR<MatrixXf>(10, 20));
diff --git a/test/qr_fullpivoting.cpp b/test/qr_fullpivoting.cpp index d82e123..f2d8cb3 100644 --- a/test/qr_fullpivoting.cpp +++ b/test/qr_fullpivoting.cpp
@@ -10,13 +10,19 @@ #include "main.h" #include <Eigen/QR> +#include "solverbase.h" template<typename MatrixType> void qr() { - typedef typename MatrixType::Index Index; + STATIC_CHECK(( internal::is_same<typename FullPivHouseholderQR<MatrixType>::StorageIndex,int>::value )); - Index rows = internal::random<Index>(20,200), cols = internal::random<int>(20,200), cols2 = internal::random<int>(20,200); - Index rank = internal::random<Index>(1, (std::min)(rows, cols)-1); + static const int Rows = MatrixType::RowsAtCompileTime, Cols = MatrixType::ColsAtCompileTime; + Index max_size = EIGEN_TEST_MAX_SIZE; + Index min_size = numext::maxi(1,EIGEN_TEST_MAX_SIZE/10); + Index rows = Rows == Dynamic ? internal::random<Index>(min_size,max_size) : Rows, + cols = Cols == Dynamic ? internal::random<Index>(min_size,max_size) : Cols, + cols2 = Cols == Dynamic ? internal::random<Index>(min_size,max_size) : Cols, + rank = internal::random<Index>(1, (std::min)(rows, cols)-1); typedef typename MatrixType::Scalar Scalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::RowsAtCompileTime> MatrixQType; @@ -45,11 +51,20 @@ MatrixType tmp; VERIFY_IS_APPROX(tmp.noalias() = qr.matrixQ() * r, (qr.matrixQ() * r).eval()); - MatrixType m2 = MatrixType::Random(cols,cols2); - MatrixType m3 = m1*m2; - m2 = MatrixType::Random(cols,cols2); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + check_solverbase<MatrixType, MatrixType>(m1, qr, rows, cols, cols2); + + { + MatrixType m2, m3; + Index size = rows; + do { + m1 = MatrixType::Random(size,size); + qr.compute(m1); + } while(!qr.isInvertible()); + MatrixType m1_inv = qr.inverse(); + m3 = m1 * MatrixType::Random(size,cols2); + m2 = qr.solve(m3); + VERIFY_IS_APPROX(m2, m1_inv*m3); + } } template<typename MatrixType> void qr_invertible() @@ -59,7 +74,9 @@ typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar; typedef typename MatrixType::Scalar Scalar; - int size = internal::random<int>(10,50); + Index max_size = numext::mini(50,EIGEN_TEST_MAX_SIZE); + Index min_size = numext::maxi(1,EIGEN_TEST_MAX_SIZE/10); + Index size = internal::random<Index>(min_size,max_size); MatrixType m1(size, size), m2(size, size), m3(size, size); m1 = MatrixType::Random(size,size); @@ -76,9 +93,7 @@ VERIFY(qr.isInvertible()); VERIFY(qr.isSurjective()); - m3 = MatrixType::Random(size,size); - m2 = qr.solve(m3); - VERIFY_IS_APPROX(m3, m1*m2); + check_solverbase<MatrixType, MatrixType>(m1, qr, size, size, size); // now construct a matrix with prescribed determinant m1.setZero(); @@ -98,6 +113,8 @@ FullPivHouseholderQR<MatrixType> qr; VERIFY_RAISES_ASSERT(qr.matrixQR()) VERIFY_RAISES_ASSERT(qr.solve(tmp)) + VERIFY_RAISES_ASSERT(qr.transpose().solve(tmp)) + VERIFY_RAISES_ASSERT(qr.adjoint().solve(tmp)) VERIFY_RAISES_ASSERT(qr.matrixQ()) VERIFY_RAISES_ASSERT(qr.dimensionOfKernel()) VERIFY_RAISES_ASSERT(qr.isInjective()) @@ -108,11 +125,12 @@ VERIFY_RAISES_ASSERT(qr.logAbsDeterminant()) } -void test_qr_fullpivoting() +EIGEN_DECLARE_TEST(qr_fullpivoting) { - for(int i = 0; i < 1; i++) { - // FIXME : very weird bug here -// CALL_SUBTEST(qr(Matrix2f()) ); + for(int i = 0; i < 1; i++) { + CALL_SUBTEST_5( qr<Matrix3f>() ); + CALL_SUBTEST_6( qr<Matrix3d>() ); + CALL_SUBTEST_8( qr<Matrix2f>() ); CALL_SUBTEST_1( qr<MatrixXf>() ); CALL_SUBTEST_2( qr<MatrixXd>() ); CALL_SUBTEST_3( qr<MatrixXcd>() );
diff --git a/test/qtvector.cpp b/test/qtvector.cpp index 2be885e..4ec79b1 100644 --- a/test/qtvector.cpp +++ b/test/qtvector.cpp
@@ -18,8 +18,6 @@ template<typename MatrixType> void check_qtvector_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; - Index rows = m.rows(); Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); @@ -127,7 +125,7 @@ } } -void test_qtvector() +EIGEN_DECLARE_TEST(qtvector) { // some non vectorizable fixed sizes CALL_SUBTEST(check_qtvector_matrix(Vector2f()));
diff --git a/test/rand.cpp b/test/rand.cpp index eeec341..1b5c058 100644 --- a/test/rand.cpp +++ b/test/rand.cpp
@@ -9,6 +9,8 @@ #include "main.h" +typedef long long int64; + template<typename Scalar> Scalar check_in_range(Scalar x, Scalar y) { Scalar r = internal::random<Scalar>(x,y); @@ -35,31 +37,49 @@ VERIFY( (mask>0).all() ); } -void test_rand() +template<typename Scalar> void check_histogram(Scalar x, Scalar y, int bins) +{ + Array<int,1,Dynamic> hist(bins); + hist.fill(0); + int f = 100000; + int n = bins*f; + int64 range = int64(y)-int64(x); + int divisor = int((range+1)/bins); + assert(((range+1)%bins)==0); + for(int k=0; k<n; ++k) + { + Scalar r = check_in_range(x,y); + hist( int((int64(r)-int64(x))/divisor) )++; + } + VERIFY( (((hist.cast<double>()/double(f))-1.0).abs()<0.02).all() ); +} + +EIGEN_DECLARE_TEST(rand) { long long_ref = NumTraits<long>::highest()/10; signed char char_offset = (std::min)(g_repeat,64); signed char short_offset = (std::min)(g_repeat,16000); - - for(int i = 0; i < g_repeat*10; i++) { + + for(int i = 0; i < g_repeat*10000; i++) { CALL_SUBTEST(check_in_range<float>(10,11)); CALL_SUBTEST(check_in_range<float>(1.24234523,1.24234523)); CALL_SUBTEST(check_in_range<float>(-1,1)); CALL_SUBTEST(check_in_range<float>(-1432.2352,-1432.2352)); - + CALL_SUBTEST(check_in_range<double>(10,11)); CALL_SUBTEST(check_in_range<double>(1.24234523,1.24234523)); CALL_SUBTEST(check_in_range<double>(-1,1)); CALL_SUBTEST(check_in_range<double>(-1432.2352,-1432.2352)); - + CALL_SUBTEST(check_in_range<int>(0,-1)); CALL_SUBTEST(check_in_range<short>(0,-1)); CALL_SUBTEST(check_in_range<long>(0,-1)); CALL_SUBTEST(check_in_range<int>(-673456,673456)); + CALL_SUBTEST(check_in_range<int>(-RAND_MAX+10,RAND_MAX-10)); CALL_SUBTEST(check_in_range<short>(-24345,24345)); CALL_SUBTEST(check_in_range<long>(-long_ref,long_ref)); } - + CALL_SUBTEST(check_all_in_range<signed char>(11,11)); CALL_SUBTEST(check_all_in_range<signed char>(11,11+char_offset)); CALL_SUBTEST(check_all_in_range<signed char>(-5,5)); @@ -67,25 +87,32 @@ CALL_SUBTEST(check_all_in_range<signed char>(-126,-126+char_offset)); CALL_SUBTEST(check_all_in_range<signed char>(126-char_offset,126)); CALL_SUBTEST(check_all_in_range<signed char>(-126,126)); - + CALL_SUBTEST(check_all_in_range<short>(11,11)); CALL_SUBTEST(check_all_in_range<short>(11,11+short_offset)); CALL_SUBTEST(check_all_in_range<short>(-5,5)); CALL_SUBTEST(check_all_in_range<short>(-11-short_offset,-11)); CALL_SUBTEST(check_all_in_range<short>(-24345,-24345+short_offset)); CALL_SUBTEST(check_all_in_range<short>(24345,24345+short_offset)); - + CALL_SUBTEST(check_all_in_range<int>(11,11)); CALL_SUBTEST(check_all_in_range<int>(11,11+g_repeat)); CALL_SUBTEST(check_all_in_range<int>(-5,5)); CALL_SUBTEST(check_all_in_range<int>(-11-g_repeat,-11)); CALL_SUBTEST(check_all_in_range<int>(-673456,-673456+g_repeat)); CALL_SUBTEST(check_all_in_range<int>(673456,673456+g_repeat)); - + CALL_SUBTEST(check_all_in_range<long>(11,11)); CALL_SUBTEST(check_all_in_range<long>(11,11+g_repeat)); CALL_SUBTEST(check_all_in_range<long>(-5,5)); CALL_SUBTEST(check_all_in_range<long>(-11-g_repeat,-11)); CALL_SUBTEST(check_all_in_range<long>(-long_ref,-long_ref+g_repeat)); CALL_SUBTEST(check_all_in_range<long>( long_ref, long_ref+g_repeat)); + + CALL_SUBTEST(check_histogram<int>(-5,5,11)); + int bins = 100; + CALL_SUBTEST(check_histogram<int>(-3333,-3333+bins*(3333/bins)-1,bins)); + bins = 1000; + CALL_SUBTEST(check_histogram<int>(-RAND_MAX+10,-RAND_MAX+10+bins*(RAND_MAX/bins)-1,bins)); + CALL_SUBTEST(check_histogram<int>(-RAND_MAX+10,-int64(RAND_MAX)+10+bins*(2*int64(RAND_MAX)/bins)-1,bins)); }
diff --git a/test/real_qz.cpp b/test/real_qz.cpp index a1766c6..1cf7aba 100644 --- a/test/real_qz.cpp +++ b/test/real_qz.cpp
@@ -7,6 +7,7 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#define EIGEN_RUNTIME_NO_MALLOC #include "main.h" #include <limits> #include <Eigen/Eigenvalues> @@ -17,7 +18,6 @@ RealQZ.h */ using std::abs; - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index dim = m.cols(); @@ -41,7 +41,11 @@ break; } - RealQZ<MatrixType> qz(A,B); + RealQZ<MatrixType> qz(dim); + // TODO enable full-prealocation of required memory, this probably requires an in-place mode for HessenbergDecomposition + //Eigen::internal::set_is_malloc_allowed(false); + qz.compute(A,B); + //Eigen::internal::set_is_malloc_allowed(true); VERIFY_IS_EQUAL(qz.info(), Success); // check for zeros @@ -49,11 +53,20 @@ for (Index i=0; i<A.cols(); i++) for (Index j=0; j<i; j++) { if (abs(qz.matrixT()(i,j))!=Scalar(0.0)) + { + std::cerr << "Error: T(" << i << "," << j << ") = " << qz.matrixT()(i,j) << std::endl; all_zeros = false; + } if (j<i-1 && abs(qz.matrixS()(i,j))!=Scalar(0.0)) + { + std::cerr << "Error: S(" << i << "," << j << ") = " << qz.matrixS()(i,j) << std::endl; all_zeros = false; + } if (j==i-1 && j>0 && abs(qz.matrixS()(i,j))!=Scalar(0.0) && abs(qz.matrixS()(i-1,j-1))!=Scalar(0.0)) + { + std::cerr << "Error: S(" << i << "," << j << ") = " << qz.matrixS()(i,j) << " && S(" << i-1 << "," << j-1 << ") = " << qz.matrixS()(i-1,j-1) << std::endl; all_zeros = false; + } } VERIFY_IS_EQUAL(all_zeros, true); VERIFY_IS_APPROX(qz.matrixQ()*qz.matrixS()*qz.matrixZ(), A); @@ -62,7 +75,7 @@ VERIFY_IS_APPROX(qz.matrixZ()*qz.matrixZ().adjoint(), MatrixType::Identity(dim,dim)); } -void test_real_qz() +EIGEN_DECLARE_TEST(real_qz) { int s = 0; for(int i = 0; i < g_repeat; i++) {
diff --git a/test/redux.cpp b/test/redux.cpp index 6ddc59c..fdbab77 100644 --- a/test/redux.cpp +++ b/test/redux.cpp
@@ -9,12 +9,13 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #define TEST_ENABLE_TEMPORARY_TRACKING +#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 +// ^^ see bug 1449 #include "main.h" template<typename MatrixType> void matrixRedux(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; @@ -27,6 +28,9 @@ // failures if we underflow into denormals. Thus, we scale so that entries are close to 1. MatrixType m1_for_prod = MatrixType::Ones(rows, cols) + RealScalar(0.2) * m1; + Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::RowsAtCompileTime> m2(rows,rows); + m2.setRandom(); + VERIFY_IS_MUCH_SMALLER_THAN(MatrixType::Zero(rows, cols).sum(), Scalar(1)); VERIFY_IS_APPROX(MatrixType::Ones(rows, cols).sum(), Scalar(float(rows*cols))); // the float() here to shut up excessive MSVC warning about int->complex conversion being lossy Scalar s(0), p(1), minc(numext::real(m1.coeff(0))), maxc(numext::real(m1.coeff(0))); @@ -45,6 +49,10 @@ VERIFY_IS_APPROX(m1_for_prod.prod(), p); VERIFY_IS_APPROX(m1.real().minCoeff(), numext::real(minc)); VERIFY_IS_APPROX(m1.real().maxCoeff(), numext::real(maxc)); + + // test that partial reduction works if nested expressions is forced to evaluate early + VERIFY_IS_APPROX((m1.matrix() * m1.matrix().transpose()) .cwiseProduct(m2.matrix()).rowwise().sum().sum(), + (m1.matrix() * m1.matrix().transpose()).eval().cwiseProduct(m2.matrix()).rowwise().sum().sum()); // test slice vectorization assuming assign is ok Index r0 = internal::random<Index>(0,rows-1); @@ -70,16 +78,13 @@ VERIFY_IS_APPROX(m1.block(r0,c0,0,0).prod(), Scalar(1)); // test nesting complex expression - VERIFY_EVALUATION_COUNT( (m1.matrix()*m1.matrix().transpose()).sum(), (MatrixType::SizeAtCompileTime==Dynamic ? 1 : 0) ); - Matrix<Scalar, MatrixType::RowsAtCompileTime, MatrixType::RowsAtCompileTime> m2(rows,rows); - m2.setRandom(); - VERIFY_EVALUATION_COUNT( ((m1.matrix()*m1.matrix().transpose())+m2).sum(), (MatrixType::SizeAtCompileTime==Dynamic ? 1 : 0) ); + VERIFY_EVALUATION_COUNT( (m1.matrix()*m1.matrix().transpose()).sum(), (MatrixType::IsVectorAtCompileTime && MatrixType::SizeAtCompileTime!=1 ? 0 : 1) ); + VERIFY_EVALUATION_COUNT( ((m1.matrix()*m1.matrix().transpose())+m2).sum(),(MatrixType::IsVectorAtCompileTime && MatrixType::SizeAtCompileTime!=1 ? 0 : 1)); } template<typename VectorType> void vectorRedux(const VectorType& w) { using std::abs; - typedef typename VectorType::Index Index; typedef typename VectorType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; Index size = w.size(); @@ -146,7 +151,7 @@ VERIFY_RAISES_ASSERT(v.head(0).maxCoeff()); } -void test_redux() +EIGEN_DECLARE_TEST(redux) { // the max size cannot be too large, otherwise reduxion operations obviously generate large errors. int maxsize = (std::min)(100,EIGEN_TEST_MAX_SIZE); @@ -156,8 +161,10 @@ CALL_SUBTEST_1( matrixRedux(Array<float, 1, 1>()) ); CALL_SUBTEST_2( matrixRedux(Matrix2f()) ); CALL_SUBTEST_2( matrixRedux(Array2f()) ); + CALL_SUBTEST_2( matrixRedux(Array22f()) ); CALL_SUBTEST_3( matrixRedux(Matrix4d()) ); CALL_SUBTEST_3( matrixRedux(Array4d()) ); + CALL_SUBTEST_3( matrixRedux(Array44d()) ); CALL_SUBTEST_4( matrixRedux(MatrixXcf(internal::random<int>(1,maxsize), internal::random<int>(1,maxsize))) ); CALL_SUBTEST_4( matrixRedux(ArrayXXcf(internal::random<int>(1,maxsize), internal::random<int>(1,maxsize))) ); CALL_SUBTEST_5( matrixRedux(MatrixXd (internal::random<int>(1,maxsize), internal::random<int>(1,maxsize))) );
diff --git a/test/ref.cpp b/test/ref.cpp index 769db04..250135b 100644 --- a/test/ref.cpp +++ b/test/ref.cpp
@@ -13,7 +13,7 @@ #endif #define TEST_ENABLE_TEMPORARY_TRACKING - +#define TEST_CHECK_STATIC_ASSERTIONS #include "main.h" // test Ref.h @@ -32,7 +32,6 @@ template<typename MatrixType> void ref_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef Matrix<Scalar,Dynamic,Dynamic,MatrixType::Options> DynMatrixType; @@ -80,7 +79,6 @@ template<typename VectorType> void ref_vector(const VectorType& m) { - typedef typename VectorType::Index Index; typedef typename VectorType::Scalar Scalar; typedef typename VectorType::RealScalar RealScalar; typedef Matrix<Scalar,Dynamic,1,VectorType::Options> DynMatrixType; @@ -255,7 +253,18 @@ test_ref_ambiguous(A, B); } -void test_ref() +void test_ref_fixed_size_assert() +{ + Vector4f v4 = Vector4f::Random(); + VectorXf vx = VectorXf::Random(10); + VERIFY_RAISES_STATIC_ASSERT( Ref<Vector3f> y = v4; (void)y; ); + VERIFY_RAISES_STATIC_ASSERT( Ref<Vector3f> y = vx.head<4>(); (void)y; ); + VERIFY_RAISES_STATIC_ASSERT( Ref<const Vector3f> y = v4; (void)y; ); + VERIFY_RAISES_STATIC_ASSERT( Ref<const Vector3f> y = vx.head<4>(); (void)y; ); + VERIFY_RAISES_STATIC_ASSERT( Ref<const Vector3f> y = 2*v4; (void)y; ); +} + +EIGEN_DECLARE_TEST(ref) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( ref_vector(Matrix<float, 1, 1>()) ); @@ -277,4 +286,5 @@ } CALL_SUBTEST_7( test_ref_overloads() ); + CALL_SUBTEST_7( test_ref_fixed_size_assert() ); }
diff --git a/test/reshape.cpp b/test/reshape.cpp new file mode 100644 index 0000000..7b16742 --- /dev/null +++ b/test/reshape.cpp
@@ -0,0 +1,216 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2014 yoco <peter.xiau@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +template<typename T1,typename T2> +typename internal::enable_if<internal::is_same<T1,T2>::value,bool>::type +is_same_eq(const T1& a, const T2& b) +{ + return (a.array() == b.array()).all(); +} + +template <int Order,typename MatType> +void check_auto_reshape4x4(MatType m) +{ + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 1> v1( 1); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 2> v2( 2); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 4> v4( 4); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 8> v8( 8); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1:16> v16(16); + + VERIFY(is_same_eq(m.template reshaped<Order>( 1, AutoSize), m.template reshaped<Order>( 1, 16))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, 16 ), m.template reshaped<Order>( 1, 16))); + VERIFY(is_same_eq(m.template reshaped<Order>( 2, AutoSize), m.template reshaped<Order>( 2, 8))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, 8 ), m.template reshaped<Order>( 2, 8))); + VERIFY(is_same_eq(m.template reshaped<Order>( 4, AutoSize), m.template reshaped<Order>( 4, 4))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, 4 ), m.template reshaped<Order>( 4, 4))); + VERIFY(is_same_eq(m.template reshaped<Order>( 8, AutoSize), m.template reshaped<Order>( 8, 2))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, 2 ), m.template reshaped<Order>( 8, 2))); + VERIFY(is_same_eq(m.template reshaped<Order>(16, AutoSize), m.template reshaped<Order>(16, 1))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, 1 ), m.template reshaped<Order>(16, 1))); + + VERIFY(is_same_eq(m.template reshaped<Order>(fix< 1>, AutoSize), m.template reshaped<Order>(fix< 1>, v16 ))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, fix<16> ), m.template reshaped<Order>( v1, fix<16>))); + VERIFY(is_same_eq(m.template reshaped<Order>(fix< 2>, AutoSize), m.template reshaped<Order>(fix< 2>, v8 ))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, fix< 8> ), m.template reshaped<Order>( v2, fix< 8>))); + VERIFY(is_same_eq(m.template reshaped<Order>(fix< 4>, AutoSize), m.template reshaped<Order>(fix< 4>, v4 ))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, fix< 4> ), m.template reshaped<Order>( v4, fix< 4>))); + VERIFY(is_same_eq(m.template reshaped<Order>(fix< 8>, AutoSize), m.template reshaped<Order>(fix< 8>, v2 ))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, fix< 2> ), m.template reshaped<Order>( v8, fix< 2>))); + VERIFY(is_same_eq(m.template reshaped<Order>(fix<16>, AutoSize), m.template reshaped<Order>(fix<16>, v1 ))); + VERIFY(is_same_eq(m.template reshaped<Order>(AutoSize, fix< 1> ), m.template reshaped<Order>(v16, fix< 1>))); +} + +template <typename MatType> +void check_direct_access_reshape4x4(MatType , internal::FixedInt<RowMajorBit>) {} + +template <typename MatType> +void check_direct_access_reshape4x4(MatType m, internal::FixedInt<0>) { + VERIFY_IS_EQUAL(m.reshaped( 1, 16).data(), m.data()); + VERIFY_IS_EQUAL(m.reshaped( 1, 16).innerStride(), 1); + + VERIFY_IS_EQUAL(m.reshaped( 2, 8).data(), m.data()); + VERIFY_IS_EQUAL(m.reshaped( 2, 8).innerStride(), 1); + VERIFY_IS_EQUAL(m.reshaped( 2, 8).outerStride(), 2); +} + +// just test a 4x4 matrix, enumerate all combination manually +template <typename MatType> +void reshape4x4(MatType m) +{ + typedef typename MatType::Scalar Scalar; + + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 1> v1( 1); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 2> v2( 2); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 4> v4( 4); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1: 8> v8( 8); + internal::VariableAndFixedInt<MatType::SizeAtCompileTime==Dynamic?-1:16> v16(16); + + if((MatType::Flags&RowMajorBit)==0) + { + typedef Map<MatrixXi> MapMat; + // dynamic + VERIFY_IS_EQUAL((m.reshaped( 1, 16)), MapMat(m.data(), 1, 16)); + VERIFY_IS_EQUAL((m.reshaped( 2, 8)), MapMat(m.data(), 2, 8)); + VERIFY_IS_EQUAL((m.reshaped( 4, 4)), MapMat(m.data(), 4, 4)); + VERIFY_IS_EQUAL((m.reshaped( 8, 2)), MapMat(m.data(), 8, 2)); + VERIFY_IS_EQUAL((m.reshaped(16, 1)), MapMat(m.data(), 16, 1)); + + // static + VERIFY_IS_EQUAL(m.reshaped(fix< 1>, fix<16>), MapMat(m.data(), 1, 16)); + VERIFY_IS_EQUAL(m.reshaped(fix< 2>, fix< 8>), MapMat(m.data(), 2, 8)); + VERIFY_IS_EQUAL(m.reshaped(fix< 4>, fix< 4>), MapMat(m.data(), 4, 4)); + VERIFY_IS_EQUAL(m.reshaped(fix< 8>, fix< 2>), MapMat(m.data(), 8, 2)); + VERIFY_IS_EQUAL(m.reshaped(fix<16>, fix< 1>), MapMat(m.data(), 16, 1)); + + + // reshape chain + VERIFY_IS_EQUAL( + (m + .reshaped( 1, 16) + .reshaped(fix< 2>,fix< 8>) + .reshaped(16, 1) + .reshaped(fix< 8>,fix< 2>) + .reshaped( 2, 8) + .reshaped(fix< 1>,fix<16>) + .reshaped( 4, 4) + .reshaped(fix<16>,fix< 1>) + .reshaped( 8, 2) + .reshaped(fix< 4>,fix< 4>) + ), + MapMat(m.data(), 4, 4) + ); + } + + VERIFY(is_same_eq(m.reshaped( 1, AutoSize), m.reshaped( 1, 16))); + VERIFY(is_same_eq(m.reshaped(AutoSize, 16), m.reshaped( 1, 16))); + VERIFY(is_same_eq(m.reshaped( 2, AutoSize), m.reshaped( 2, 8))); + VERIFY(is_same_eq(m.reshaped(AutoSize, 8), m.reshaped( 2, 8))); + VERIFY(is_same_eq(m.reshaped( 4, AutoSize), m.reshaped( 4, 4))); + VERIFY(is_same_eq(m.reshaped(AutoSize, 4), m.reshaped( 4, 4))); + VERIFY(is_same_eq(m.reshaped( 8, AutoSize), m.reshaped( 8, 2))); + VERIFY(is_same_eq(m.reshaped(AutoSize, 2), m.reshaped( 8, 2))); + VERIFY(is_same_eq(m.reshaped(16, AutoSize), m.reshaped(16, 1))); + VERIFY(is_same_eq(m.reshaped(AutoSize, 1), m.reshaped(16, 1))); + + VERIFY(is_same_eq(m.reshaped(fix< 1>, AutoSize), m.reshaped(fix< 1>, v16))); + VERIFY(is_same_eq(m.reshaped(AutoSize, fix<16>), m.reshaped( v1, fix<16>))); + VERIFY(is_same_eq(m.reshaped(fix< 2>, AutoSize), m.reshaped(fix< 2>, v8))); + VERIFY(is_same_eq(m.reshaped(AutoSize, fix< 8>), m.reshaped( v2, fix< 8>))); + VERIFY(is_same_eq(m.reshaped(fix< 4>, AutoSize), m.reshaped(fix< 4>, v4))); + VERIFY(is_same_eq(m.reshaped(AutoSize, fix< 4>), m.reshaped( v4, fix< 4>))); + VERIFY(is_same_eq(m.reshaped(fix< 8>, AutoSize), m.reshaped(fix< 8>, v2))); + VERIFY(is_same_eq(m.reshaped(AutoSize, fix< 2>), m.reshaped( v8, fix< 2>))); + VERIFY(is_same_eq(m.reshaped(fix<16>, AutoSize), m.reshaped(fix<16>, v1))); + VERIFY(is_same_eq(m.reshaped(AutoSize, fix< 1>), m.reshaped(v16, fix< 1>))); + + check_auto_reshape4x4<ColMajor> (m); + check_auto_reshape4x4<RowMajor> (m); + check_auto_reshape4x4<AutoOrder>(m); + check_auto_reshape4x4<ColMajor> (m.transpose()); + check_auto_reshape4x4<ColMajor> (m.transpose()); + check_auto_reshape4x4<AutoOrder>(m.transpose()); + + check_direct_access_reshape4x4(m,fix<MatType::Flags&RowMajorBit>); + + if((MatType::Flags&RowMajorBit)==0) + { + VERIFY_IS_EQUAL(m.template reshaped<ColMajor>(2,8),m.reshaped(2,8)); + VERIFY_IS_EQUAL(m.template reshaped<ColMajor>(2,8),m.template reshaped<AutoOrder>(2,8)); + VERIFY_IS_EQUAL(m.transpose().template reshaped<RowMajor>(2,8),m.transpose().template reshaped<AutoOrder>(2,8)); + } + else + { + VERIFY_IS_EQUAL(m.template reshaped<ColMajor>(2,8),m.reshaped(2,8)); + VERIFY_IS_EQUAL(m.template reshaped<RowMajor>(2,8),m.template reshaped<AutoOrder>(2,8)); + VERIFY_IS_EQUAL(m.transpose().template reshaped<ColMajor>(2,8),m.transpose().template reshaped<AutoOrder>(2,8)); + VERIFY_IS_EQUAL(m.transpose().reshaped(2,8),m.transpose().template reshaped<AutoOrder>(2,8)); + } + + MatrixXi m28r1 = m.template reshaped<RowMajor>(2,8); + MatrixXi m28r2 = m.transpose().template reshaped<ColMajor>(8,2).transpose(); + VERIFY_IS_EQUAL( m28r1, m28r2); + + VERIFY(is_same_eq(m.reshaped(v16,fix<1>), m.reshaped())); + VERIFY_IS_EQUAL(m.reshaped(16,1).eval(), m.reshaped().eval()); + VERIFY_IS_EQUAL(m.reshaped(1,16).eval(), m.reshaped().transpose().eval()); + VERIFY_IS_EQUAL(m.reshaped().reshaped(2,8), m.reshaped(2,8)); + VERIFY_IS_EQUAL(m.reshaped().reshaped(4,4), m.reshaped(4,4)); + VERIFY_IS_EQUAL(m.reshaped().reshaped(8,2), m.reshaped(8,2)); + + VERIFY_IS_EQUAL(m.reshaped(), m.template reshaped<ColMajor>()); + VERIFY_IS_EQUAL(m.transpose().reshaped(), m.template reshaped<RowMajor>()); + VERIFY_IS_EQUAL(m.template reshaped<RowMajor>(AutoSize,fix<1>), m.template reshaped<RowMajor>()); + VERIFY_IS_EQUAL(m.template reshaped<AutoOrder>(AutoSize,fix<1>), m.template reshaped<AutoOrder>()); + + VERIFY(is_same_eq(m.reshaped(AutoSize,fix<1>), m.reshaped())); + VERIFY_IS_EQUAL(m.template reshaped<RowMajor>(fix<1>,AutoSize), m.transpose().reshaped().transpose()); + + // check assignment + { + Matrix<Scalar,Dynamic,1> m1x(m.size()); m1x.setRandom(); + VERIFY_IS_APPROX(m.reshaped() = m1x, m1x); + VERIFY_IS_APPROX(m, m1x.reshaped(4,4)); + + Matrix<Scalar,Dynamic,Dynamic> m28(2,8); m28.setRandom(); + VERIFY_IS_APPROX(m.reshaped(2,8) = m28, m28); + VERIFY_IS_APPROX(m, m28.reshaped(4,4)); + VERIFY_IS_APPROX(m.template reshaped<RowMajor>(2,8) = m28, m28); + + Matrix<Scalar,Dynamic,Dynamic> m24(2,4); m24.setRandom(); + VERIFY_IS_APPROX(m(seq(0,last,2),all).reshaped(2,4) = m24, m24); + + // check constness: + m.reshaped(2,8).nestedExpression() = m; + } +} + +EIGEN_DECLARE_TEST(reshape) +{ + typedef Matrix<int,Dynamic,Dynamic,RowMajor> RowMatrixXi; + typedef Matrix<int,4,4,RowMajor> RowMatrix4i; + MatrixXi mx = MatrixXi::Random(4, 4); + Matrix4i m4 = Matrix4i::Random(4, 4); + RowMatrixXi rmx = RowMatrixXi::Random(4, 4); + RowMatrix4i rm4 = RowMatrix4i::Random(4, 4); + + // test dynamic-size matrix + CALL_SUBTEST(reshape4x4(mx)); + // test static-size matrix + CALL_SUBTEST(reshape4x4(m4)); + // test dynamic-size const matrix + CALL_SUBTEST(reshape4x4(static_cast<const MatrixXi>(mx))); + // test static-size const matrix + CALL_SUBTEST(reshape4x4(static_cast<const Matrix4i>(m4))); + + CALL_SUBTEST(reshape4x4(rmx)); + CALL_SUBTEST(reshape4x4(rm4)); +}
diff --git a/test/resize.cpp b/test/resize.cpp index 4adaafe..646a75b 100644 --- a/test/resize.cpp +++ b/test/resize.cpp
@@ -33,7 +33,7 @@ void resizeLikeTest1020() { resizeLikeTest<10,20>(); } void resizeLikeTest31() { resizeLikeTest<3,1>(); } -void test_resize() +EIGEN_DECLARE_TEST(resize) { CALL_SUBTEST(resizeLikeTest12() ); CALL_SUBTEST(resizeLikeTest1020() );
diff --git a/test/rvalue_types.cpp b/test/rvalue_types.cpp index 3eebfc6..5f52fb3 100644 --- a/test/rvalue_types.cpp +++ b/test/rvalue_types.cpp
@@ -11,7 +11,9 @@ #include <Eigen/Core> -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +using internal::UIntPtr; + +#if EIGEN_HAS_RVALUE_REFERENCES template <typename MatrixType> void rvalue_copyassign(const MatrixType& m) { @@ -20,11 +22,11 @@ // create a temporary which we are about to destroy by moving MatrixType tmp = m; - long src_address = reinterpret_cast<long>(tmp.data()); + UIntPtr src_address = reinterpret_cast<UIntPtr>(tmp.data()); // move the temporary to n MatrixType n = std::move(tmp); - long dst_address = reinterpret_cast<long>(n.data()); + UIntPtr dst_address = reinterpret_cast<UIntPtr>(n.data()); if (MatrixType::RowsAtCompileTime==Dynamic|| MatrixType::ColsAtCompileTime==Dynamic) { @@ -41,7 +43,7 @@ void rvalue_copyassign(const MatrixType&) {} #endif -void test_rvalue_types() +EIGEN_DECLARE_TEST(rvalue_types) { CALL_SUBTEST_1(rvalue_copyassign( MatrixXf::Random(50,50).eval() )); CALL_SUBTEST_1(rvalue_copyassign( ArrayXXf::Random(50,50).eval() ));
diff --git a/test/schur_complex.cpp b/test/schur_complex.cpp index deb78e4..03e17e8 100644 --- a/test/schur_complex.cpp +++ b/test/schur_complex.cpp
@@ -79,7 +79,7 @@ } } -void test_schur_complex() +EIGEN_DECLARE_TEST(schur_complex) { CALL_SUBTEST_1(( schur<Matrix4cd>() )); CALL_SUBTEST_2(( schur<MatrixXcf>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE/4)) ));
diff --git a/test/schur_real.cpp b/test/schur_real.cpp index cfe4570..9454610 100644 --- a/test/schur_real.cpp +++ b/test/schur_real.cpp
@@ -13,8 +13,6 @@ template<typename MatrixType> void verifyIsQuasiTriangular(const MatrixType& T) { - typedef typename MatrixType::Index Index; - const Index size = T.cols(); typedef typename MatrixType::Scalar Scalar; @@ -82,7 +80,7 @@ Atriangular.template triangularView<StrictlyLower>().setZero(); rs3.setMaxIterations(1).compute(Atriangular); // triangular matrices do not need any iterations VERIFY_IS_EQUAL(rs3.info(), Success); - VERIFY_IS_EQUAL(rs3.matrixT(), Atriangular); + VERIFY_IS_APPROX(rs3.matrixT(), Atriangular); // approx because of scaling... VERIFY_IS_EQUAL(rs3.matrixU(), MatrixType::Identity(size, size)); // Test computation of only T, not U @@ -100,7 +98,7 @@ } } -void test_schur_real() +EIGEN_DECLARE_TEST(schur_real) { CALL_SUBTEST_1(( schur<Matrix4f>() )); CALL_SUBTEST_2(( schur<MatrixXd>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE/4)) ));
diff --git a/test/selfadjoint.cpp b/test/selfadjoint.cpp index 76dab6d..9ca9cef 100644 --- a/test/selfadjoint.cpp +++ b/test/selfadjoint.cpp
@@ -7,6 +7,7 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#define TEST_CHECK_STATIC_ASSERTIONS #include "main.h" // This file tests the basic selfadjointView API, @@ -14,14 +15,15 @@ template<typename MatrixType> void selfadjoint(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index rows = m.rows(); Index cols = m.cols(); MatrixType m1 = MatrixType::Random(rows, cols), - m3(rows, cols); + m2 = MatrixType::Random(rows, cols), + m3(rows, cols), + m4(rows, cols); m1.diagonal() = m1.diagonal().real().template cast<Scalar>(); @@ -30,10 +32,22 @@ VERIFY_IS_APPROX(MatrixType(m3.template triangularView<Upper>()), MatrixType(m1.template triangularView<Upper>())); VERIFY_IS_APPROX(m3, m3.adjoint()); - m3 = m1.template selfadjointView<Lower>(); VERIFY_IS_APPROX(MatrixType(m3.template triangularView<Lower>()), MatrixType(m1.template triangularView<Lower>())); VERIFY_IS_APPROX(m3, m3.adjoint()); + + m3 = m1.template selfadjointView<Upper>(); + m4 = m2; + m4 += m1.template selfadjointView<Upper>(); + VERIFY_IS_APPROX(m4, m2+m3); + + m3 = m1.template selfadjointView<Lower>(); + m4 = m2; + m4 -= m1.template selfadjointView<Lower>(); + VERIFY_IS_APPROX(m4, m2-m3); + + VERIFY_RAISES_STATIC_ASSERT(m2.template selfadjointView<StrictlyUpper>()); + VERIFY_RAISES_STATIC_ASSERT(m2.template selfadjointView<UnitLower>()); } void bug_159() @@ -42,7 +56,7 @@ EIGEN_UNUSED_VARIABLE(m) } -void test_selfadjoint() +EIGEN_DECLARE_TEST(selfadjoint) { for(int i = 0; i < g_repeat ; i++) {
diff --git a/test/simplicial_cholesky.cpp b/test/simplicial_cholesky.cpp index 649c817..e3c31e3 100644 --- a/test/simplicial_cholesky.cpp +++ b/test/simplicial_cholesky.cpp
@@ -9,17 +9,17 @@ #include "sparse_solver.h" -template<typename T, typename I> void test_simplicial_cholesky_T() +template<typename T, typename I_> void test_simplicial_cholesky_T() { - typedef SparseMatrix<T,0,I> SparseMatrixType; + typedef SparseMatrix<T,0,I_> SparseMatrixType; SimplicialCholesky<SparseMatrixType, Lower> chol_colmajor_lower_amd; SimplicialCholesky<SparseMatrixType, Upper> chol_colmajor_upper_amd; SimplicialLLT< SparseMatrixType, Lower> llt_colmajor_lower_amd; SimplicialLLT< SparseMatrixType, Upper> llt_colmajor_upper_amd; SimplicialLDLT< SparseMatrixType, Lower> ldlt_colmajor_lower_amd; SimplicialLDLT< SparseMatrixType, Upper> ldlt_colmajor_upper_amd; - SimplicialLDLT< SparseMatrixType, Lower, NaturalOrdering<I> > ldlt_colmajor_lower_nat; - SimplicialLDLT< SparseMatrixType, Upper, NaturalOrdering<I> > ldlt_colmajor_upper_nat; + SimplicialLDLT< SparseMatrixType, Lower, NaturalOrdering<I_> > ldlt_colmajor_lower_nat; + SimplicialLDLT< SparseMatrixType, Upper, NaturalOrdering<I_> > ldlt_colmajor_upper_nat; check_sparse_spd_solving(chol_colmajor_lower_amd); check_sparse_spd_solving(chol_colmajor_upper_amd); @@ -35,11 +35,11 @@ check_sparse_spd_determinant(ldlt_colmajor_lower_amd); check_sparse_spd_determinant(ldlt_colmajor_upper_amd); - check_sparse_spd_solving(ldlt_colmajor_lower_nat, 300, 1000); - check_sparse_spd_solving(ldlt_colmajor_upper_nat, 300, 1000); + check_sparse_spd_solving(ldlt_colmajor_lower_nat, (std::min)(300,EIGEN_TEST_MAX_SIZE), 1000); + check_sparse_spd_solving(ldlt_colmajor_upper_nat, (std::min)(300,EIGEN_TEST_MAX_SIZE), 1000); } -void test_simplicial_cholesky() +EIGEN_DECLARE_TEST(simplicial_cholesky) { CALL_SUBTEST_1(( test_simplicial_cholesky_T<double,int>() )); CALL_SUBTEST_2(( test_simplicial_cholesky_T<std::complex<double>, int>() ));
diff --git a/test/sizeof.cpp b/test/sizeof.cpp index 03ad204..af34e97 100644 --- a/test/sizeof.cpp +++ b/test/sizeof.cpp
@@ -15,10 +15,10 @@ if (MatrixType::RowsAtCompileTime!=Dynamic && MatrixType::ColsAtCompileTime!=Dynamic) VERIFY_IS_EQUAL(std::ptrdiff_t(sizeof(MatrixType)),std::ptrdiff_t(sizeof(Scalar))*std::ptrdiff_t(MatrixType::SizeAtCompileTime)); else - VERIFY_IS_EQUAL(sizeof(MatrixType),sizeof(Scalar*) + 2 * sizeof(typename MatrixType::Index)); + VERIFY_IS_EQUAL(sizeof(MatrixType),sizeof(Scalar*) + 2 * sizeof(Index)); } -void test_sizeof() +EIGEN_DECLARE_TEST(sizeof) { CALL_SUBTEST(verifySizeOf(Matrix<float, 1, 1>()) ); CALL_SUBTEST(verifySizeOf(Array<float, 2, 1>()) );
diff --git a/test/sizeoverflow.cpp b/test/sizeoverflow.cpp index 240d222..4213512 100644 --- a/test/sizeoverflow.cpp +++ b/test/sizeoverflow.cpp
@@ -34,7 +34,7 @@ VERIFY_THROWS_BADALLOC( VectorType v; v.conservativeResize(size) ); } -void test_sizeoverflow() +EIGEN_DECLARE_TEST(sizeoverflow) { // there are 2 levels of overflow checking. first in PlainObjectBase.h we check for overflow in rows*cols computations. // this is tested in tests of the form times_itself_gives_0 * times_itself_gives_0
diff --git a/test/smallvectors.cpp b/test/smallvectors.cpp index 7815113..f9803ac 100644 --- a/test/smallvectors.cpp +++ b/test/smallvectors.cpp
@@ -57,7 +57,7 @@ } } -void test_smallvectors() +EIGEN_DECLARE_TEST(smallvectors) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST(smallVectors<int>() );
diff --git a/test/solverbase.h b/test/solverbase.h new file mode 100644 index 0000000..13c0959 --- /dev/null +++ b/test/solverbase.h
@@ -0,0 +1,36 @@ +#ifndef TEST_SOLVERBASE_H +#define TEST_SOLVERBASE_H + +template<typename DstType, typename RhsType, typename MatrixType, typename SolverType> +void check_solverbase(const MatrixType& matrix, const SolverType& solver, Index rows, Index cols, Index cols2) +{ + // solve + DstType m2 = DstType::Random(cols,cols2); + RhsType m3 = matrix*m2; + DstType solver_solution = DstType::Random(cols,cols2); + solver._solve_impl(m3, solver_solution); + VERIFY_IS_APPROX(m3, matrix*solver_solution); + solver_solution = DstType::Random(cols,cols2); + solver_solution = solver.solve(m3); + VERIFY_IS_APPROX(m3, matrix*solver_solution); + // test solve with transposed + m3 = RhsType::Random(rows,cols2); + m2 = matrix.transpose()*m3; + RhsType solver_solution2 = RhsType::Random(rows,cols2); + solver.template _solve_impl_transposed<false>(m2, solver_solution2); + VERIFY_IS_APPROX(m2, matrix.transpose()*solver_solution2); + solver_solution2 = RhsType::Random(rows,cols2); + solver_solution2 = solver.transpose().solve(m2); + VERIFY_IS_APPROX(m2, matrix.transpose()*solver_solution2); + // test solve with conjugate transposed + m3 = RhsType::Random(rows,cols2); + m2 = matrix.adjoint()*m3; + solver_solution2 = RhsType::Random(rows,cols2); + solver.template _solve_impl_transposed<true>(m2, solver_solution2); + VERIFY_IS_APPROX(m2, matrix.adjoint()*solver_solution2); + solver_solution2 = RhsType::Random(rows,cols2); + solver_solution2 = solver.adjoint().solve(m2); + VERIFY_IS_APPROX(m2, matrix.adjoint()*solver_solution2); +} + +#endif // TEST_SOLVERBASE_H
diff --git a/test/sparse.h b/test/sparse.h index 9912e1e..df471b4 100644 --- a/test/sparse.h +++ b/test/sparse.h
@@ -14,7 +14,7 @@ #include "main.h" -#if EIGEN_GNUC_AT_LEAST(4,0) && !defined __ICC && !defined(__clang__) +#if EIGEN_HAS_CXX11 #ifdef min #undef min @@ -24,11 +24,9 @@ #undef max #endif -#include <tr1/unordered_map> +#include <unordered_map> #define EIGEN_UNORDERED_MAP_SUPPORT -namespace std { - using std::tr1::unordered_map; -} + #endif #ifdef EIGEN_GOOGLEHASH_SUPPORT
diff --git a/test/sparseLM.cpp b/test/sparseLM.cpp index 8e148f9..a48fcb6 100644 --- a/test/sparseLM.cpp +++ b/test/sparseLM.cpp
@@ -168,7 +168,7 @@ return ; } -void test_sparseLM() +EIGEN_DECLARE_TEST(sparseLM) { CALL_SUBTEST_1(test_sparseLM_T<double>());
diff --git a/test/sparse_basic.cpp b/test/sparse_basic.cpp index cb8ebae..9e735b3 100644 --- a/test/sparse_basic.cpp +++ b/test/sparse_basic.cpp
@@ -9,9 +9,16 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#ifndef EIGEN_SPARSE_TEST_INCLUDED_FROM_SPARSE_EXTRA static long g_realloc_count = 0; #define EIGEN_SPARSE_COMPRESSED_STORAGE_REALLOCATE_PLUGIN g_realloc_count++; +static long g_dense_op_sparse_count = 0; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_DENSE_OP_SPARSE_PLUGIN g_dense_op_sparse_count++; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_ADD_DENSE_PLUGIN g_dense_op_sparse_count+=10; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_SUB_DENSE_PLUGIN g_dense_op_sparse_count+=20; +#endif + #include "sparse.h" template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& ref) @@ -25,6 +32,7 @@ //const Index outer = ref.outerSize(); typedef typename SparseMatrixType::Scalar Scalar; + typedef typename SparseMatrixType::RealScalar RealScalar; enum { Flags = SparseMatrixType::Flags }; double density = (std::max)(8./(rows*cols), 0.01); @@ -157,23 +165,24 @@ initSparse<Scalar>(density, refM3, m3); initSparse<Scalar>(density, refM4, m4); + if(internal::random<bool>()) + m1.makeCompressed(); + + Index m1_nnz = m1.nonZeros(); + VERIFY_IS_APPROX(m1*s1, refM1*s1); VERIFY_IS_APPROX(m1+m2, refM1+refM2); VERIFY_IS_APPROX(m1+m2+m3, refM1+refM2+refM3); VERIFY_IS_APPROX(m3.cwiseProduct(m1+m2), refM3.cwiseProduct(refM1+refM2)); VERIFY_IS_APPROX(m1*s1-m2, refM1*s1-refM2); - - VERIFY_IS_APPROX(m1*=s1, refM1*=s1); - VERIFY_IS_APPROX(m1/=s1, refM1/=s1); - - VERIFY_IS_APPROX(m1+=m2, refM1+=refM2); - VERIFY_IS_APPROX(m1-=m2, refM1-=refM2); + VERIFY_IS_APPROX(m4=m1/s1, refM1/s1); + VERIFY_IS_EQUAL(m4.nonZeros(), m1_nnz); if(SparseMatrixType::IsRowMajor) VERIFY_IS_APPROX(m1.innerVector(0).dot(refM2.row(0)), refM1.row(0).dot(refM2.row(0))); else VERIFY_IS_APPROX(m1.innerVector(0).dot(refM2.col(0)), refM1.col(0).dot(refM2.col(0))); - + DenseVector rv = DenseVector::Random(m1.cols()); DenseVector cv = DenseVector::Random(m1.rows()); Index r = internal::random<Index>(0,m1.rows()-2); @@ -192,16 +201,132 @@ VERIFY_IS_APPROX(refM4.cwiseProduct(m3), refM4.cwiseProduct(refM3)); // VERIFY_IS_APPROX(m3.cwise()/refM4, refM3.cwise()/refM4); + // mixed sparse-dense VERIFY_IS_APPROX(refM4 + m3, refM4 + refM3); VERIFY_IS_APPROX(m3 + refM4, refM3 + refM4); VERIFY_IS_APPROX(refM4 - m3, refM4 - refM3); VERIFY_IS_APPROX(m3 - refM4, refM3 - refM4); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + RealScalar(0.5)*m3).eval(), RealScalar(0.5)*refM4 + RealScalar(0.5)*refM3); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + m3*RealScalar(0.5)).eval(), RealScalar(0.5)*refM4 + RealScalar(0.5)*refM3); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + m3.cwiseProduct(m3)).eval(), RealScalar(0.5)*refM4 + refM3.cwiseProduct(refM3)); + + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + RealScalar(0.5)*m3).eval(), RealScalar(0.5)*refM4 + RealScalar(0.5)*refM3); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + m3*RealScalar(0.5)).eval(), RealScalar(0.5)*refM4 + RealScalar(0.5)*refM3); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + (m3+m3)).eval(), RealScalar(0.5)*refM4 + (refM3+refM3)); + VERIFY_IS_APPROX(((refM3+m3)+RealScalar(0.5)*m3).eval(), RealScalar(0.5)*refM3 + (refM3+refM3)); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + (refM3+m3)).eval(), RealScalar(0.5)*refM4 + (refM3+refM3)); + VERIFY_IS_APPROX((RealScalar(0.5)*refM4 + (m3+refM3)).eval(), RealScalar(0.5)*refM4 + (refM3+refM3)); + + + VERIFY_IS_APPROX(m1.sum(), refM1.sum()); + + m4 = m1; refM4 = m4; + + VERIFY_IS_APPROX(m1*=s1, refM1*=s1); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + VERIFY_IS_APPROX(m1/=s1, refM1/=s1); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + + VERIFY_IS_APPROX(m1+=m2, refM1+=refM2); + VERIFY_IS_APPROX(m1-=m2, refM1-=refM2); + + refM3 = refM1; + + VERIFY_IS_APPROX(refM1+=m2, refM3+=refM2); + VERIFY_IS_APPROX(refM1-=m2, refM3-=refM2); + + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1 =m2+refM4, refM3 =refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,10); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1+=m2+refM4, refM3+=refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1-=m2+refM4, refM3-=refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1 =refM4+m2, refM3 =refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1+=refM4+m2, refM3+=refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1-=refM4+m2, refM3-=refM2+refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1 =m2-refM4, refM3 =refM2-refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,20); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1+=m2-refM4, refM3+=refM2-refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1-=m2-refM4, refM3-=refM2-refM4); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1 =refM4-m2, refM3 =refM4-refM2); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1+=refM4-m2, refM3+=refM4-refM2); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + g_dense_op_sparse_count=0; VERIFY_IS_APPROX(refM1-=refM4-m2, refM3-=refM4-refM2); VERIFY_IS_EQUAL(g_dense_op_sparse_count,1); + refM3 = m3; + + if (rows>=2 && cols>=2) + { + VERIFY_RAISES_ASSERT( m1 += m1.innerVector(0) ); + VERIFY_RAISES_ASSERT( m1 -= m1.innerVector(0) ); + VERIFY_RAISES_ASSERT( refM1 -= m1.innerVector(0) ); + VERIFY_RAISES_ASSERT( refM1 += m1.innerVector(0) ); + } + m1 = m4; refM1 = refM4; // test aliasing VERIFY_IS_APPROX((m1 = -m1), (refM1 = -refM1)); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + m1 = m4; refM1 = refM4; VERIFY_IS_APPROX((m1 = m1.transpose()), (refM1 = refM1.transpose().eval())); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + m1 = m4; refM1 = refM4; VERIFY_IS_APPROX((m1 = -m1.transpose()), (refM1 = -refM1.transpose().eval())); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + m1 = m4; refM1 = refM4; VERIFY_IS_APPROX((m1 += -m1), (refM1 += -refM1)); + VERIFY_IS_EQUAL(m1.nonZeros(), m1_nnz); + m1 = m4; refM1 = refM4; + + if(m1.isCompressed()) + { + VERIFY_IS_APPROX(m1.coeffs().sum(), m1.sum()); + m1.coeffs() += s1; + for(Index j = 0; j<m1.outerSize(); ++j) + for(typename SparseMatrixType::InnerIterator it(m1,j); it; ++it) + refM1(it.row(), it.col()) += s1; + VERIFY_IS_APPROX(m1, refM1); + } + + // and/or + { + typedef SparseMatrix<bool, SparseMatrixType::Options, typename SparseMatrixType::StorageIndex> SpBool; + SpBool mb1 = m1.real().template cast<bool>(); + SpBool mb2 = m2.real().template cast<bool>(); + VERIFY_IS_EQUAL(mb1.template cast<int>().sum(), refM1.real().template cast<bool>().count()); + VERIFY_IS_EQUAL((mb1 && mb2).template cast<int>().sum(), (refM1.real().template cast<bool>() && refM2.real().template cast<bool>()).count()); + VERIFY_IS_EQUAL((mb1 || mb2).template cast<int>().sum(), (refM1.real().template cast<bool>() || refM2.real().template cast<bool>()).count()); + SpBool mb3 = mb1 && mb2; + if(mb1.coeffs().all() && mb2.coeffs().all()) + { + VERIFY_IS_EQUAL(mb3.nonZeros(), (refM1.real().template cast<bool>() && refM2.real().template cast<bool>()).count()); + } + } + } + + // test reverse iterators + { + DenseMatrix refMat2 = DenseMatrix::Zero(rows, cols); + SparseMatrixType m2(rows, cols); + initSparse<Scalar>(density, refMat2, m2); + std::vector<Scalar> ref_value(m2.innerSize()); + std::vector<Index> ref_index(m2.innerSize()); + if(internal::random<bool>()) + m2.makeCompressed(); + for(Index j = 0; j<m2.outerSize(); ++j) + { + Index count_forward = 0; + + for(typename SparseMatrixType::InnerIterator it(m2,j); it; ++it) + { + ref_value[ref_value.size()-1-count_forward] = it.value(); + ref_index[ref_index.size()-1-count_forward] = it.index(); + count_forward++; + } + Index count_reverse = 0; + for(typename SparseMatrixType::ReverseInnerIterator it(m2,j); it; --it) + { + VERIFY_IS_APPROX( std::abs(ref_value[ref_value.size()-count_forward+count_reverse])+1, std::abs(it.value())+1); + VERIFY_IS_EQUAL( ref_index[ref_index.size()-count_forward+count_reverse] , it.index()); + count_reverse++; + } + VERIFY_IS_EQUAL(count_forward, count_reverse); + } } // test transpose @@ -232,11 +357,11 @@ for (Index i=0; i<m2.rows(); ++i) { float x = internal::random<float>(0,1); - if (x<0.1) + if (x<0.1f) { // do nothing } - else if (x<0.5) + else if (x<0.5f) { countFalseNonZero++; m2.insert(i,j) = Scalar(0); @@ -312,6 +437,17 @@ VERIFY_IS_APPROX(mapMat2+mapMat3, refMat2+refMat3); VERIFY_IS_APPROX(mapMat2+mapMat3, refMat2+refMat3); } + + Index i = internal::random<Index>(0,rows-1); + Index j = internal::random<Index>(0,cols-1); + m2.coeffRef(i,j) = 123; + if(internal::random<bool>()) + m2.makeCompressed(); + Map<SparseMatrixType> mapMat2(rows, cols, m2.nonZeros(), m2.outerIndexPtr(), m2.innerIndexPtr(), m2.valuePtr(), m2.innerNonZeroPtr()); + VERIFY_IS_EQUAL(m2.coeff(i,j),Scalar(123)); + VERIFY_IS_EQUAL(mapMat2.coeff(i,j),Scalar(123)); + mapMat2.coeffRef(i,j) = -123; + VERIFY_IS_EQUAL(m2.coeff(i,j),Scalar(-123)); } // test triangularView @@ -345,7 +481,7 @@ m3 = m2.template triangularView<StrictlyLower>(); VERIFY_IS_APPROX(m3, refMat3); - // check sparse-traingular to dense + // check sparse-triangular to dense refMat3 = m2.template triangularView<StrictlyUpper>(); VERIFY_IS_APPROX(refMat3, DenseMatrix(refMat2.template triangularView<StrictlyUpper>())); } @@ -360,6 +496,14 @@ m3 = m2.template selfadjointView<Lower>(); VERIFY_IS_APPROX(m3, refMat3); + refMat3 += refMat2.template selfadjointView<Lower>(); + m3 += m2.template selfadjointView<Lower>(); + VERIFY_IS_APPROX(m3, refMat3); + + refMat3 -= refMat2.template selfadjointView<Lower>(); + m3 -= m2.template selfadjointView<Lower>(); + VERIFY_IS_APPROX(m3, refMat3); + // selfadjointView only works for square matrices: SparseMatrixType m4(rows, rows+1); VERIFY_RAISES_ASSERT(m4.template selfadjointView<Lower>()); @@ -372,6 +516,12 @@ SparseMatrixType m2(rows, rows); initSparse<Scalar>(density, refMat2, m2); VERIFY_IS_APPROX(m2.eval(), refMat2.sparseView().eval()); + + // sparse view on expressions: + VERIFY_IS_APPROX((s1*m2).eval(), (s1*refMat2).sparseView().eval()); + VERIFY_IS_APPROX((m2+m2).eval(), (refMat2+refMat2).sparseView().eval()); + VERIFY_IS_APPROX((m2*m2).eval(), (refMat2.lazyProduct(refMat2)).sparseView().eval()); + VERIFY_IS_APPROX((m2*m2).eval(), (refMat2*refMat2).sparseView().eval()); } // test diagonal @@ -380,6 +530,10 @@ SparseMatrixType m2(rows, cols); initSparse<Scalar>(density, refMat2, m2); VERIFY_IS_APPROX(m2.diagonal(), refMat2.diagonal().eval()); + DenseVector d = m2.diagonal(); + VERIFY_IS_APPROX(d, refMat2.diagonal().eval()); + d = m2.diagonal().array(); + VERIFY_IS_APPROX(d, refMat2.diagonal().eval()); VERIFY_IS_APPROX(const_cast<const SparseMatrixType&>(m2).diagonal(), refMat2.diagonal().eval()); initSparse<Scalar>(density, refMat2, m2, ForceNonZeroDiag); @@ -392,7 +546,7 @@ { DenseVector d = DenseVector::Random(rows); DenseMatrix refMat2 = d.asDiagonal(); - SparseMatrixType m2(rows, rows); + SparseMatrixType m2; m2 = d.asDiagonal(); VERIFY_IS_APPROX(m2, refMat2); SparseMatrixType m3(d.asDiagonal()); @@ -400,6 +554,28 @@ refMat2 += d.asDiagonal(); m2 += d.asDiagonal(); VERIFY_IS_APPROX(m2, refMat2); + m2.setZero(); m2 += d.asDiagonal(); + refMat2.setZero(); refMat2 += d.asDiagonal(); + VERIFY_IS_APPROX(m2, refMat2); + m2.setZero(); m2 -= d.asDiagonal(); + refMat2.setZero(); refMat2 -= d.asDiagonal(); + VERIFY_IS_APPROX(m2, refMat2); + + initSparse<Scalar>(density, refMat2, m2); + m2.makeCompressed(); + m2 += d.asDiagonal(); + refMat2 += d.asDiagonal(); + VERIFY_IS_APPROX(m2, refMat2); + + initSparse<Scalar>(density, refMat2, m2); + m2.makeCompressed(); + VectorXi res(rows); + for(Index i=0; i<rows; ++i) + res(i) = internal::random<int>(0,3); + m2.reserve(res); + m2 -= d.asDiagonal(); + refMat2 -= d.asDiagonal(); + VERIFY_IS_APPROX(m2, refMat2); } // test conservative resize @@ -504,7 +680,8 @@ { Index r = internal::random<Index>(0,rows-1); Index c = internal::random<Index>(0,cols-1); - Scalar v = internal::random<Scalar>(); + // use positive values to prevent numerical cancellation errors in sum + Scalar v = numext::abs(internal::random<Scalar>()); triplets.push_back(TripletType(r,c,v)); sum += v; } @@ -514,9 +691,26 @@ VERIFY_IS_APPROX(sum, m.sum()); } - -void test_sparse_basic() +template<int> +void bug1105() { + // Regression test for bug 1105 + int n = Eigen::internal::random<int>(200,600); + SparseMatrix<std::complex<double>,0, long> mat(n, n); + std::complex<double> val; + + for(int i=0; i<n; ++i) + { + mat.coeffRef(i, i%(n/10)) = val; + VERIFY(mat.data().allocatedSize()<20*n); + } +} + +#ifndef EIGEN_SPARSE_TEST_INCLUDED_FROM_SPARSE_EXTRA + +EIGEN_DECLARE_TEST(sparse_basic) +{ + g_dense_op_sparse_count = 0; // Suppresses compiler warning. for(int i = 0; i < g_repeat; i++) { int r = Eigen::internal::random<int>(1,200), c = Eigen::internal::random<int>(1,200); if(Eigen::internal::random<int>(0,4) == 0) { @@ -545,18 +739,6 @@ CALL_SUBTEST_3((big_sparse_triplet<SparseMatrix<float, RowMajor, int> >(10000, 10000, 0.125))); CALL_SUBTEST_4((big_sparse_triplet<SparseMatrix<double, ColMajor, long int> >(10000, 10000, 0.125))); - // Regression test for bug 1105 -#ifdef EIGEN_TEST_PART_6 - { - int n = Eigen::internal::random<int>(200,600); - SparseMatrix<std::complex<double>,0, long> mat(n, n); - std::complex<double> val; - - for(int i=0; i<n; ++i) - { - mat.coeffRef(i, i%(n/10)) = val; - VERIFY(mat.data().allocatedSize()<20*n); - } - } -#endif + CALL_SUBTEST_7( bug1105<0>() ); } +#endif
diff --git a/test/sparse_block.cpp b/test/sparse_block.cpp index 8a6e068..f966810 100644 --- a/test/sparse_block.cpp +++ b/test/sparse_block.cpp
@@ -8,6 +8,21 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #include "sparse.h" +#include "AnnoyingScalar.h" + +template<typename T> +typename Eigen::internal::enable_if<(T::Flags&RowMajorBit)==RowMajorBit, typename T::RowXpr>::type +innervec(T& A, Index i) +{ + return A.row(i); +} + +template<typename T> +typename Eigen::internal::enable_if<(T::Flags&RowMajorBit)==0, typename T::ColXpr>::type +innervec(T& A, Index i) +{ + return A.col(i); +} template<typename SparseMatrixType> void sparse_block(const SparseMatrixType& ref) { @@ -17,11 +32,14 @@ const Index outer = ref.outerSize(); typedef typename SparseMatrixType::Scalar Scalar; + typedef typename SparseMatrixType::RealScalar RealScalar; + typedef typename SparseMatrixType::StorageIndex StorageIndex; double density = (std::max)(8./(rows*cols), 0.01); - typedef Matrix<Scalar,Dynamic,Dynamic> DenseMatrix; + typedef Matrix<Scalar,Dynamic,Dynamic,SparseMatrixType::IsRowMajor?RowMajor:ColMajor> DenseMatrix; typedef Matrix<Scalar,Dynamic,1> DenseVector; typedef Matrix<Scalar,1,Dynamic> RowDenseVector; + typedef SparseVector<Scalar> SparseVectorType; Scalar s1 = internal::random<Scalar>(); { @@ -109,33 +127,53 @@ initSparse<Scalar>(density, refMat2, m2); Index j0 = internal::random<Index>(0,outer-1); Index j1 = internal::random<Index>(0,outer-1); - if(SparseMatrixType::IsRowMajor) - VERIFY_IS_APPROX(m2.innerVector(j0), refMat2.row(j0)); - else - VERIFY_IS_APPROX(m2.innerVector(j0), refMat2.col(j0)); + Index r0 = internal::random<Index>(0,rows-1); + Index c0 = internal::random<Index>(0,cols-1); - if(SparseMatrixType::IsRowMajor) - VERIFY_IS_APPROX(m2.innerVector(j0)+m2.innerVector(j1), refMat2.row(j0)+refMat2.row(j1)); - else - VERIFY_IS_APPROX(m2.innerVector(j0)+m2.innerVector(j1), refMat2.col(j0)+refMat2.col(j1)); + VERIFY_IS_APPROX(m2.innerVector(j0), innervec(refMat2,j0)); + VERIFY_IS_APPROX(m2.innerVector(j0)+m2.innerVector(j1), innervec(refMat2,j0)+innervec(refMat2,j1)); + + m2.innerVector(j0) *= Scalar(2); + innervec(refMat2,j0) *= Scalar(2); + VERIFY_IS_APPROX(m2, refMat2); + + m2.row(r0) *= Scalar(3); + refMat2.row(r0) *= Scalar(3); + VERIFY_IS_APPROX(m2, refMat2); + + m2.col(c0) *= Scalar(4); + refMat2.col(c0) *= Scalar(4); + VERIFY_IS_APPROX(m2, refMat2); + + m2.row(r0) /= Scalar(3); + refMat2.row(r0) /= Scalar(3); + VERIFY_IS_APPROX(m2, refMat2); + + m2.col(c0) /= Scalar(4); + refMat2.col(c0) /= Scalar(4); + VERIFY_IS_APPROX(m2, refMat2); + + SparseVectorType v1; + VERIFY_IS_APPROX(v1 = m2.col(c0) * 4, refMat2.col(c0)*4); + VERIFY_IS_APPROX(v1 = m2.row(r0) * 4, refMat2.row(r0).transpose()*4); SparseMatrixType m3(rows,cols); m3.reserve(VectorXi::Constant(outer,int(inner/2))); for(Index j=0; j<outer; ++j) for(Index k=0; k<(std::min)(j,inner); ++k) - m3.insertByOuterInner(j,k) = k+1; + m3.insertByOuterInner(j,k) = internal::convert_index<StorageIndex>(k+1); for(Index j=0; j<(std::min)(outer, inner); ++j) { VERIFY(j==numext::real(m3.innerVector(j).nonZeros())); if(j>0) - VERIFY(j==numext::real(m3.innerVector(j).lastCoeff())); + VERIFY(RealScalar(j)==numext::real(m3.innerVector(j).lastCoeff())); } m3.makeCompressed(); for(Index j=0; j<(std::min)(outer, inner); ++j) { VERIFY(j==numext::real(m3.innerVector(j).nonZeros())); if(j>0) - VERIFY(j==numext::real(m3.innerVector(j).lastCoeff())); + VERIFY(RealScalar(j)==numext::real(m3.innerVector(j).lastCoeff())); } VERIFY(m3.innerVector(j0).nonZeros() == m3.transpose().innerVector(j0).nonZeros()); @@ -150,7 +188,7 @@ DenseMatrix refMat2 = DenseMatrix::Zero(rows, cols); SparseMatrixType m2(rows, cols); initSparse<Scalar>(density, refMat2, m2); - if(internal::random<float>(0,1)>0.5) m2.makeCompressed(); + if(internal::random<float>(0,1)>0.5f) m2.makeCompressed(); Index j0 = internal::random<Index>(0,outer-2); Index j1 = internal::random<Index>(0,outer-2); Index n0 = internal::random<Index>(1,outer-(std::max)(j0,j1)); @@ -222,10 +260,37 @@ VERIFY_IS_APPROX(m2.block(r0,c0,r1,c1), refMat2.block(r0,c0,r1,c1)); VERIFY_IS_APPROX((2*m2).block(r0,c0,r1,c1), (2*refMat2).block(r0,c0,r1,c1)); + + if(m2.nonZeros()>0) + { + VERIFY_IS_APPROX(m2, refMat2); + SparseMatrixType m3(rows, cols); + DenseMatrix refMat3(rows, cols); refMat3.setZero(); + Index n = internal::random<Index>(1,10); + for(Index k=0; k<n; ++k) + { + Index o1 = internal::random<Index>(0,outer-1); + Index o2 = internal::random<Index>(0,outer-1); + if(SparseMatrixType::IsRowMajor) + { + m3.innerVector(o1) = m2.row(o2); + refMat3.row(o1) = refMat2.row(o2); + } + else + { + m3.innerVector(o1) = m2.col(o2); + refMat3.col(o1) = refMat2.col(o2); + } + if(internal::random<bool>()) + m3.makeCompressed(); + } + if(m3.nonZeros()>0) + VERIFY_IS_APPROX(m3, refMat3); + } } } -void test_sparse_block() +EIGEN_DECLARE_TEST(sparse_block) { for(int i = 0; i < g_repeat; i++) { int r = Eigen::internal::random<int>(1,200), c = Eigen::internal::random<int>(1,200); @@ -250,5 +315,8 @@ CALL_SUBTEST_4(( sparse_block(SparseMatrix<double,ColMajor,short int>(short(r), short(c))) )); CALL_SUBTEST_4(( sparse_block(SparseMatrix<double,RowMajor,short int>(short(r), short(c))) )); + + AnnoyingScalar::dont_throw = true; + CALL_SUBTEST_5(( sparse_block(SparseMatrix<AnnoyingScalar>(r,c)) )); } }
diff --git a/test/sparse_permutations.cpp b/test/sparse_permutations.cpp index b82ccef..e93493c 100644 --- a/test/sparse_permutations.cpp +++ b/test/sparse_permutations.cpp
@@ -220,7 +220,7 @@ CALL_SUBTEST(( sparse_permutations<RowMajor>(SparseMatrix<Scalar, RowMajor>(size,size)) )); } -void test_sparse_permutations() +EIGEN_DECLARE_TEST(sparse_permutations) { for(int i = 0; i < g_repeat; i++) { int s = Eigen::internal::random<int>(1,50);
diff --git a/test/sparse_product.cpp b/test/sparse_product.cpp index 7ec5270..db1b0e8 100644 --- a/test/sparse_product.cpp +++ b/test/sparse_product.cpp
@@ -7,8 +7,32 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#if defined(_MSC_VER) && (_MSC_VER==1800) +// This unit test takes forever to compile in Release mode with MSVC 2013, +// multiple hours. So let's switch off optimization for this one. +#pragma optimize("",off) +#endif + +static long int nb_temporaries; + +inline void on_temporary_creation() { + // here's a great place to set a breakpoint when debugging failures in this test! + nb_temporaries++; +} + +#define EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN { on_temporary_creation(); } + #include "sparse.h" +#define VERIFY_EVALUATION_COUNT(XPR,N) {\ + nb_temporaries = 0; \ + CALL_SUBTEST( XPR ); \ + if(nb_temporaries!=N) std::cerr << "nb_temporaries == " << nb_temporaries << "\n"; \ + VERIFY( (#XPR) && nb_temporaries==N ); \ + } + + + template<typename SparseMatrixType> void sparse_product() { typedef typename SparseMatrixType::StorageIndex StorageIndex; @@ -76,6 +100,24 @@ VERIFY_IS_APPROX(m4=(m2t.transpose()*m3t.transpose()).pruned(0), refMat4=refMat2t.transpose()*refMat3t.transpose()); VERIFY_IS_APPROX(m4=(m2*m3t.transpose()).pruned(0), refMat4=refMat2*refMat3t.transpose()); + // make sure the right product implementation is called: + if((!SparseMatrixType::IsRowMajor) && m2.rows()<=m3.cols()) + { + VERIFY_EVALUATION_COUNT(m4 = m2*m3, 3); // 1 temp for the result + 2 for transposing and get a sorted result. + VERIFY_EVALUATION_COUNT(m4 = (m2*m3).pruned(0), 1); + VERIFY_EVALUATION_COUNT(m4 = (m2*m3).eval().pruned(0), 4); + } + + // and that pruning is effective: + { + DenseMatrix Ad(2,2); + Ad << -1, 1, 1, 1; + SparseMatrixType As(Ad.sparseView()), B(2,2); + VERIFY_IS_EQUAL( (As*As.transpose()).eval().nonZeros(), 4); + VERIFY_IS_EQUAL( (Ad*Ad.transpose()).eval().sparseView().eval().nonZeros(), 2); + VERIFY_IS_EQUAL( (As*As.transpose()).pruned(1e-6).eval().nonZeros(), 2); + } + // dense ?= sparse * sparse VERIFY_IS_APPROX(dm4 =m2*m3, refMat4 =refMat2*refMat3); VERIFY_IS_APPROX(dm4+=m2*m3, refMat4+=refMat2*refMat3); @@ -205,12 +247,16 @@ // also check with a SparseWrapper: DenseVector v1 = DenseVector::Random(cols); DenseVector v2 = DenseVector::Random(rows); + DenseVector v3 = DenseVector::Random(rows); VERIFY_IS_APPROX(m3=m2*v1.asDiagonal(), refM3=refM2*v1.asDiagonal()); VERIFY_IS_APPROX(m3=m2.transpose()*v2.asDiagonal(), refM3=refM2.transpose()*v2.asDiagonal()); VERIFY_IS_APPROX(m3=v2.asDiagonal()*m2, refM3=v2.asDiagonal()*refM2); VERIFY_IS_APPROX(m3=v1.asDiagonal()*m2.transpose(), refM3=v1.asDiagonal()*refM2.transpose()); VERIFY_IS_APPROX(m3=v2.asDiagonal()*m2*v1.asDiagonal(), refM3=v2.asDiagonal()*refM2*v1.asDiagonal()); + + VERIFY_IS_APPROX(v2=m2*v1.asDiagonal()*v1, refM2*v1.asDiagonal()*v1); + VERIFY_IS_APPROX(v3=v2.asDiagonal()*m2*v1, v2.asDiagonal()*refM2*v1); // evaluate to a dense matrix to check the .row() and .col() iterator functions VERIFY_IS_APPROX(d3=m2*d1, refM3=refM2*d1); @@ -245,7 +291,7 @@ for (int k=0; k<mS.outerSize(); ++k) for (typename SparseMatrixType::InnerIterator it(mS,k); it; ++it) if (it.index() == k) - it.valueRef() *= 0.5; + it.valueRef() *= Scalar(0.5); VERIFY_IS_APPROX(refS.adjoint(), refS); VERIFY_IS_APPROX(mS.adjoint(), mS); @@ -256,6 +302,14 @@ VERIFY_IS_APPROX(x=mUp.template selfadjointView<Upper>()*b, refX=refS*b); VERIFY_IS_APPROX(x=mLo.template selfadjointView<Lower>()*b, refX=refS*b); VERIFY_IS_APPROX(x=mS.template selfadjointView<Upper|Lower>()*b, refX=refS*b); + + VERIFY_IS_APPROX(x=b * mUp.template selfadjointView<Upper>(), refX=b*refS); + VERIFY_IS_APPROX(x=b * mLo.template selfadjointView<Lower>(), refX=b*refS); + VERIFY_IS_APPROX(x=b * mS.template selfadjointView<Upper|Lower>(), refX=b*refS); + + VERIFY_IS_APPROX(x.noalias()+=mUp.template selfadjointView<Upper>()*b, refX+=refS*b); + VERIFY_IS_APPROX(x.noalias()-=mLo.template selfadjointView<Lower>()*b, refX-=refS*b); + VERIFY_IS_APPROX(x.noalias()+=mS.template selfadjointView<Upper|Lower>()*b, refX+=refS*b); // sparse selfadjointView with sparse matrices SparseMatrixType mSres(rows,rows); @@ -323,7 +377,89 @@ VERIFY_IS_APPROX( ( d.asDiagonal()*cmA ).eval().coeff(0,0), res ); } -void test_sparse_product() +template<typename Real> +void test_mixing_types() +{ + typedef std::complex<Real> Cplx; + typedef SparseMatrix<Real> SpMatReal; + typedef SparseMatrix<Cplx> SpMatCplx; + typedef SparseMatrix<Cplx,RowMajor> SpRowMatCplx; + typedef Matrix<Real,Dynamic,Dynamic> DenseMatReal; + typedef Matrix<Cplx,Dynamic,Dynamic> DenseMatCplx; + + Index n = internal::random<Index>(1,100); + double density = (std::max)(8./(n*n), 0.2); + + SpMatReal sR1(n,n); + SpMatCplx sC1(n,n), sC2(n,n), sC3(n,n); + SpRowMatCplx sCR(n,n); + DenseMatReal dR1(n,n); + DenseMatCplx dC1(n,n), dC2(n,n), dC3(n,n); + + initSparse<Real>(density, dR1, sR1); + initSparse<Cplx>(density, dC1, sC1); + initSparse<Cplx>(density, dC2, sC2); + + VERIFY_IS_APPROX( sC2 = (sR1 * sC1), dC3 = dR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( sC2 = (sC1 * sR1), dC3 = dC1 * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sC2 = (sR1.transpose() * sC1), dC3 = dR1.template cast<Cplx>().transpose() * dC1 ); + VERIFY_IS_APPROX( sC2 = (sC1.transpose() * sR1), dC3 = dC1.transpose() * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sC2 = (sR1 * sC1.transpose()), dC3 = dR1.template cast<Cplx>() * dC1.transpose() ); + VERIFY_IS_APPROX( sC2 = (sC1 * sR1.transpose()), dC3 = dC1 * dR1.template cast<Cplx>().transpose() ); + VERIFY_IS_APPROX( sC2 = (sR1.transpose() * sC1.transpose()), dC3 = dR1.template cast<Cplx>().transpose() * dC1.transpose() ); + VERIFY_IS_APPROX( sC2 = (sC1.transpose() * sR1.transpose()), dC3 = dC1.transpose() * dR1.template cast<Cplx>().transpose() ); + + VERIFY_IS_APPROX( sCR = (sR1 * sC1), dC3 = dR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( sCR = (sC1 * sR1), dC3 = dC1 * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sCR = (sR1.transpose() * sC1), dC3 = dR1.template cast<Cplx>().transpose() * dC1 ); + VERIFY_IS_APPROX( sCR = (sC1.transpose() * sR1), dC3 = dC1.transpose() * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sCR = (sR1 * sC1.transpose()), dC3 = dR1.template cast<Cplx>() * dC1.transpose() ); + VERIFY_IS_APPROX( sCR = (sC1 * sR1.transpose()), dC3 = dC1 * dR1.template cast<Cplx>().transpose() ); + VERIFY_IS_APPROX( sCR = (sR1.transpose() * sC1.transpose()), dC3 = dR1.template cast<Cplx>().transpose() * dC1.transpose() ); + VERIFY_IS_APPROX( sCR = (sC1.transpose() * sR1.transpose()), dC3 = dC1.transpose() * dR1.template cast<Cplx>().transpose() ); + + + VERIFY_IS_APPROX( sC2 = (sR1 * sC1).pruned(), dC3 = dR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( sC2 = (sC1 * sR1).pruned(), dC3 = dC1 * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sC2 = (sR1.transpose() * sC1).pruned(), dC3 = dR1.template cast<Cplx>().transpose() * dC1 ); + VERIFY_IS_APPROX( sC2 = (sC1.transpose() * sR1).pruned(), dC3 = dC1.transpose() * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sC2 = (sR1 * sC1.transpose()).pruned(), dC3 = dR1.template cast<Cplx>() * dC1.transpose() ); + VERIFY_IS_APPROX( sC2 = (sC1 * sR1.transpose()).pruned(), dC3 = dC1 * dR1.template cast<Cplx>().transpose() ); + VERIFY_IS_APPROX( sC2 = (sR1.transpose() * sC1.transpose()).pruned(), dC3 = dR1.template cast<Cplx>().transpose() * dC1.transpose() ); + VERIFY_IS_APPROX( sC2 = (sC1.transpose() * sR1.transpose()).pruned(), dC3 = dC1.transpose() * dR1.template cast<Cplx>().transpose() ); + + VERIFY_IS_APPROX( sCR = (sR1 * sC1).pruned(), dC3 = dR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( sCR = (sC1 * sR1).pruned(), dC3 = dC1 * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sCR = (sR1.transpose() * sC1).pruned(), dC3 = dR1.template cast<Cplx>().transpose() * dC1 ); + VERIFY_IS_APPROX( sCR = (sC1.transpose() * sR1).pruned(), dC3 = dC1.transpose() * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( sCR = (sR1 * sC1.transpose()).pruned(), dC3 = dR1.template cast<Cplx>() * dC1.transpose() ); + VERIFY_IS_APPROX( sCR = (sC1 * sR1.transpose()).pruned(), dC3 = dC1 * dR1.template cast<Cplx>().transpose() ); + VERIFY_IS_APPROX( sCR = (sR1.transpose() * sC1.transpose()).pruned(), dC3 = dR1.template cast<Cplx>().transpose() * dC1.transpose() ); + VERIFY_IS_APPROX( sCR = (sC1.transpose() * sR1.transpose()).pruned(), dC3 = dC1.transpose() * dR1.template cast<Cplx>().transpose() ); + + + VERIFY_IS_APPROX( dC2 = (sR1 * sC1), dC3 = dR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( dC2 = (sC1 * sR1), dC3 = dC1 * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( dC2 = (sR1.transpose() * sC1), dC3 = dR1.template cast<Cplx>().transpose() * dC1 ); + VERIFY_IS_APPROX( dC2 = (sC1.transpose() * sR1), dC3 = dC1.transpose() * dR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( dC2 = (sR1 * sC1.transpose()), dC3 = dR1.template cast<Cplx>() * dC1.transpose() ); + VERIFY_IS_APPROX( dC2 = (sC1 * sR1.transpose()), dC3 = dC1 * dR1.template cast<Cplx>().transpose() ); + VERIFY_IS_APPROX( dC2 = (sR1.transpose() * sC1.transpose()), dC3 = dR1.template cast<Cplx>().transpose() * dC1.transpose() ); + VERIFY_IS_APPROX( dC2 = (sC1.transpose() * sR1.transpose()), dC3 = dC1.transpose() * dR1.template cast<Cplx>().transpose() ); + + + VERIFY_IS_APPROX( dC2 = dR1 * sC1, dC3 = dR1.template cast<Cplx>() * sC1 ); + VERIFY_IS_APPROX( dC2 = sR1 * dC1, dC3 = sR1.template cast<Cplx>() * dC1 ); + VERIFY_IS_APPROX( dC2 = dC1 * sR1, dC3 = dC1 * sR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( dC2 = sC1 * dR1, dC3 = sC1 * dR1.template cast<Cplx>() ); + + VERIFY_IS_APPROX( dC2 = dR1.row(0) * sC1, dC3 = dR1.template cast<Cplx>().row(0) * sC1 ); + VERIFY_IS_APPROX( dC2 = sR1 * dC1.col(0), dC3 = sR1.template cast<Cplx>() * dC1.col(0) ); + VERIFY_IS_APPROX( dC2 = dC1.row(0) * sR1, dC3 = dC1.row(0) * sR1.template cast<Cplx>() ); + VERIFY_IS_APPROX( dC2 = sC1 * dR1.col(0), dC3 = sC1 * dR1.template cast<Cplx>().col(0) ); +} + +EIGEN_DECLARE_TEST(sparse_product) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( (sparse_product<SparseMatrix<double,ColMajor> >()) ); @@ -333,5 +469,7 @@ CALL_SUBTEST_2( (sparse_product<SparseMatrix<std::complex<double>, RowMajor > >()) ); CALL_SUBTEST_3( (sparse_product<SparseMatrix<float,ColMajor,long int> >()) ); CALL_SUBTEST_4( (sparse_product_regression_test<SparseMatrix<double,RowMajor>, Matrix<double, Dynamic, Dynamic, RowMajor> >()) ); + + CALL_SUBTEST_5( (test_mixing_types<float>()) ); } }
diff --git a/test/sparse_ref.cpp b/test/sparse_ref.cpp index f4aefbb..12b6f8a 100644 --- a/test/sparse_ref.cpp +++ b/test/sparse_ref.cpp
@@ -87,8 +87,8 @@ VERIFY_EVALUATION_COUNT( call_ref_3(B, B), 1); VERIFY_EVALUATION_COUNT( call_ref_2(B.transpose(), B.transpose()), 0); VERIFY_EVALUATION_COUNT( call_ref_3(B.transpose(), B.transpose()), 0); - VERIFY_EVALUATION_COUNT( call_ref_2(A*A, AA), 1); - VERIFY_EVALUATION_COUNT( call_ref_3(A*A, AA), 1); + VERIFY_EVALUATION_COUNT( call_ref_2(A*A, AA), 3); + VERIFY_EVALUATION_COUNT( call_ref_3(A*A, AA), 3); VERIFY(!C.isCompressed()); VERIFY_EVALUATION_COUNT( call_ref_3(C, C), 1); @@ -126,7 +126,7 @@ VERIFY_EVALUATION_COUNT( call_ref_5(A.row(2), A.row(2).transpose()), 1); } -void test_sparse_ref() +EIGEN_DECLARE_TEST(sparse_ref) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( check_const_correctness(SparseMatrix<float>()) );
diff --git a/test/sparse_solver.h b/test/sparse_solver.h index b676534..f45d7ef 100644 --- a/test/sparse_solver.h +++ b/test/sparse_solver.h
@@ -11,6 +11,33 @@ #include <Eigen/SparseCore> #include <sstream> +template<typename Solver, typename Rhs, typename Guess,typename Result> +void solve_with_guess(IterativeSolverBase<Solver>& solver, const MatrixBase<Rhs>& b, const Guess& g, Result &x) { + if(internal::random<bool>()) + { + // With a temporary through evaluator<SolveWithGuess> + x = solver.derived().solveWithGuess(b,g) + Result::Zero(x.rows(), x.cols()); + } + else + { + // direct evaluation within x through Assignment<Result,SolveWithGuess> + x = solver.derived().solveWithGuess(b.derived(),g); + } +} + +template<typename Solver, typename Rhs, typename Guess,typename Result> +void solve_with_guess(SparseSolverBase<Solver>& solver, const MatrixBase<Rhs>& b, const Guess& , Result& x) { + if(internal::random<bool>()) + x = solver.derived().solve(b) + Result::Zero(x.rows(), x.cols()); + else + x = solver.derived().solve(b); +} + +template<typename Solver, typename Rhs, typename Guess,typename Result> +void solve_with_guess(SparseSolverBase<Solver>& solver, const SparseMatrixBase<Rhs>& b, const Guess& , Result& x) { + x = solver.derived().solve(b); +} + template<typename Solver, typename Rhs, typename DenseMat, typename DenseRhs> void check_sparse_solving(Solver& solver, const typename Solver::MatrixType& A, const Rhs& b, const DenseMat& dA, const DenseRhs& db) { @@ -32,11 +59,21 @@ x = solver.solve(b); if (solver.info() != Success) { - std::cerr << "WARNING | sparse solver testing: solving failed (" << typeid(Solver).name() << ")\n"; + std::cerr << "WARNING: sparse solver testing: solving failed (" << typeid(Solver).name() << ")\n"; + // dump call stack: + g_test_level++; + VERIFY(solver.info() == Success); + g_test_level--; return; } VERIFY(oldb.isApprox(b) && "sparse solver testing: the rhs should not be modified!"); VERIFY(x.isApprox(refX,test_precision<Scalar>())); + + x.setZero(); + solve_with_guess(solver, b, x, x); + VERIFY(solver.info() == Success && "solving failed when using solve_with_guess API"); + VERIFY(oldb.isApprox(b) && "sparse solver testing: the rhs should not be modified!"); + VERIFY(x.isApprox(refX,test_precision<Scalar>())); x.setZero(); // test the analyze/factorize API @@ -233,12 +270,13 @@ } #endif -template<typename Solver> void check_sparse_spd_solving(Solver& solver, int maxSize = 300, int maxRealWorldSize = 100000) +template<typename Solver> void check_sparse_spd_solving(Solver& solver, int maxSize = (std::min)(300,EIGEN_TEST_MAX_SIZE), int maxRealWorldSize = 100000) { typedef typename Solver::MatrixType Mat; typedef typename Mat::Scalar Scalar; typedef typename Mat::StorageIndex StorageIndex; typedef SparseMatrix<Scalar,ColMajor, StorageIndex> SpMat; + typedef SparseVector<Scalar, 0, StorageIndex> SpVec; typedef Matrix<Scalar,Dynamic,Dynamic> DenseMatrix; typedef Matrix<Scalar,Dynamic,1> DenseVector; @@ -255,6 +293,8 @@ DenseVector b = DenseVector::Random(size); DenseMatrix dB(size,rhsCols); initSparse<Scalar>(density, dB, B, ForceNonZeroDiag); + SpVec c = B.col(0); + DenseVector dc = dB.col(0); CALL_SUBTEST( check_sparse_solving(solver, A, b, dA, b) ); CALL_SUBTEST( check_sparse_solving(solver, halfA, b, dA, b) ); @@ -262,6 +302,8 @@ CALL_SUBTEST( check_sparse_solving(solver, halfA, dB, dA, dB) ); CALL_SUBTEST( check_sparse_solving(solver, A, B, dA, dB) ); CALL_SUBTEST( check_sparse_solving(solver, halfA, B, dA, dB) ); + CALL_SUBTEST( check_sparse_solving(solver, A, c, dA, dc) ); + CALL_SUBTEST( check_sparse_solving(solver, halfA, c, dA, dc) ); // check only once if(i==0) @@ -363,6 +405,7 @@ typedef typename Solver::MatrixType Mat; typedef typename Mat::Scalar Scalar; typedef SparseMatrix<Scalar,ColMajor, typename Mat::StorageIndex> SpMat; + typedef SparseVector<Scalar, 0, typename Mat::StorageIndex> SpVec; typedef Matrix<Scalar,Dynamic,Dynamic> DenseMatrix; typedef Matrix<Scalar,Dynamic,1> DenseVector; @@ -380,15 +423,17 @@ double density = (std::max)(8./(size*rhsCols), 0.1); initSparse<Scalar>(density, dB, B, ForceNonZeroDiag); B.makeCompressed(); + SpVec c = B.col(0); + DenseVector dc = dB.col(0); CALL_SUBTEST(check_sparse_solving(solver, A, b, dA, b)); CALL_SUBTEST(check_sparse_solving(solver, A, dB, dA, dB)); CALL_SUBTEST(check_sparse_solving(solver, A, B, dA, dB)); + CALL_SUBTEST(check_sparse_solving(solver, A, c, dA, dc)); // check only once if(i==0) { - b = DenseVector::Zero(size); - check_sparse_solving(solver, A, b, dA, b); + CALL_SUBTEST(b = DenseVector::Zero(size); check_sparse_solving(solver, A, b, dA, b)); } // regression test for Bug 792 (structurally rank deficient matrices): if(checkDeficient && size>1) {
diff --git a/test/sparse_solvers.cpp b/test/sparse_solvers.cpp index 3a8873d..3b7cd77 100644 --- a/test/sparse_solvers.cpp +++ b/test/sparse_solvers.cpp
@@ -98,10 +98,23 @@ initSparse<Scalar>(density, refMat2, m2, ForceNonZeroDiag|MakeLowerTriangular, &zeroCoords, &nonzeroCoords); VERIFY_IS_APPROX(refMat2.template triangularView<Lower>().solve(vec2), m2.template triangularView<Lower>().solve(vec3)); + + // test empty triangular matrix + { + m2.resize(0,0); + refMatB.resize(0,refMatB.cols()); + DenseMatrix res = m2.template triangularView<Lower>().solve(refMatB); + VERIFY_IS_EQUAL(res.rows(),0); + VERIFY_IS_EQUAL(res.cols(),refMatB.cols()); + res = refMatB; + m2.template triangularView<Lower>().solveInPlace(res); + VERIFY_IS_EQUAL(res.rows(),0); + VERIFY_IS_EQUAL(res.cols(),refMatB.cols()); + } } } -void test_sparse_solvers() +EIGEN_DECLARE_TEST(sparse_solvers) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(sparse_solvers<double>(8, 8) );
diff --git a/test/sparse_vector.cpp b/test/sparse_vector.cpp index d95f301..3512927 100644 --- a/test/sparse_vector.cpp +++ b/test/sparse_vector.cpp
@@ -12,7 +12,7 @@ template<typename Scalar,typename StorageIndex> void sparse_vector(int rows, int cols) { double densityMat = (std::max)(8./(rows*cols), 0.01); - double densityVec = (std::max)(8./float(rows), 0.1); + double densityVec = (std::max)(8./(rows), 0.1); typedef Matrix<Scalar,Dynamic,Dynamic> DenseMatrix; typedef Matrix<Scalar,Dynamic,1> DenseVector; typedef SparseVector<Scalar,0,StorageIndex> SparseVectorType; @@ -145,7 +145,7 @@ } -void test_sparse_vector() +EIGEN_DECLARE_TEST(sparse_vector) { for(int i = 0; i < g_repeat; i++) { int r = Eigen::internal::random<int>(1,500), c = Eigen::internal::random<int>(1,500);
diff --git a/test/sparselu.cpp b/test/sparselu.cpp index bd000ba..84cc6eb 100644 --- a/test/sparselu.cpp +++ b/test/sparselu.cpp
@@ -36,7 +36,7 @@ check_sparse_square_determinant(sparselu_amd); } -void test_sparselu() +EIGEN_DECLARE_TEST(sparselu) { CALL_SUBTEST_1(test_sparselu_T<float>()); CALL_SUBTEST_2(test_sparselu_T<double>());
diff --git a/test/sparseqr.cpp b/test/sparseqr.cpp index 50d1fcd..3576cc6 100644 --- a/test/sparseqr.cpp +++ b/test/sparseqr.cpp
@@ -43,6 +43,7 @@ template<typename Scalar> void test_sparseqr_scalar() { + typedef typename NumTraits<Scalar>::Real RealScalar; typedef SparseMatrix<Scalar,ColMajor> MatrixType; typedef Matrix<Scalar,Dynamic,Dynamic> DenseMat; typedef Matrix<Scalar,Dynamic,1> DenseVector; @@ -54,7 +55,29 @@ b = dA * DenseVector::Random(A.cols()); solver.compute(A); - if(internal::random<float>(0,1)>0.5) + + // Q should be MxM + VERIFY_IS_EQUAL(solver.matrixQ().rows(), A.rows()); + VERIFY_IS_EQUAL(solver.matrixQ().cols(), A.rows()); + + // R should be MxN + VERIFY_IS_EQUAL(solver.matrixR().rows(), A.rows()); + VERIFY_IS_EQUAL(solver.matrixR().cols(), A.cols()); + + // Q and R can be multiplied + DenseMat recoveredA = solver.matrixQ() + * DenseMat(solver.matrixR().template triangularView<Upper>()) + * solver.colsPermutation().transpose(); + VERIFY_IS_EQUAL(recoveredA.rows(), A.rows()); + VERIFY_IS_EQUAL(recoveredA.cols(), A.cols()); + + // and in the full rank case the original matrix is recovered + if (solver.rank() == A.cols()) + { + VERIFY_IS_APPROX(A, recoveredA); + } + + if(internal::random<float>(0,1)>0.5f) solver.factorize(A); // this checks that calling analyzePattern is not needed if the pattern do not change. if (solver.info() != Success) { @@ -69,14 +92,34 @@ exit(0); return; } - - VERIFY_IS_APPROX(A * x, b); - - //Compare with a dense QR solver + + // Compare with a dense QR solver ColPivHouseholderQR<DenseMat> dqr(dA); refX = dqr.solve(b); - VERIFY_IS_EQUAL(dqr.rank(), solver.rank()); + bool rank_deficient = A.cols()>A.rows() || dqr.rank()<A.cols(); + if(rank_deficient) + { + // rank deficient problem -> we might have to increase the threshold + // to get a correct solution. + RealScalar th = RealScalar(20)*dA.colwise().norm().maxCoeff()*(A.rows()+A.cols()) * NumTraits<RealScalar>::epsilon(); + for(Index k=0; (k<16) && !test_isApprox(A*x,b); ++k) + { + th *= RealScalar(10); + solver.setPivotThreshold(th); + solver.compute(A); + x = solver.solve(b); + } + } + + VERIFY_IS_APPROX(A * x, b); + + // For rank deficient problem, the estimated rank might + // be slightly off, so let's only raise a warning in such cases. + if(rank_deficient) ++g_test_level; + VERIFY_IS_EQUAL(solver.rank(), dqr.rank()); + if(rank_deficient) --g_test_level; + if(solver.rank()==A.cols()) // full rank VERIFY_IS_APPROX(x, refX); // else @@ -95,7 +138,7 @@ dQ = solver.matrixQ(); VERIFY_IS_APPROX(Q, dQ); } -void test_sparseqr() +EIGEN_DECLARE_TEST(sparseqr) { for(int i=0; i<g_repeat; ++i) {
diff --git a/test/special_numbers.cpp b/test/special_numbers.cpp index 2f1b704..1e1a636 100644 --- a/test/special_numbers.cpp +++ b/test/special_numbers.cpp
@@ -49,7 +49,7 @@ VERIFY(!mboth.array().allFinite()); } -void test_special_numbers() +EIGEN_DECLARE_TEST(special_numbers) { for(int i = 0; i < 10*g_repeat; i++) { CALL_SUBTEST_1( special_numbers<float>() );
diff --git a/test/split_test_helper.h b/test/split_test_helper.h new file mode 100644 index 0000000..82e82aa --- /dev/null +++ b/test/split_test_helper.h
@@ -0,0 +1,5994 @@ +#if defined(EIGEN_TEST_PART_1) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_1(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_1(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_2) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_2(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_2(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_3) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_3(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_3(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_4) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_4(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_4(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_5) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_5(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_5(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_6) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_6(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_6(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_7) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_7(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_7(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_8) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_8(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_8(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_9) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_9(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_9(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_10) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_10(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_10(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_11) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_11(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_11(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_12) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_12(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_12(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_13) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_13(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_13(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_14) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_14(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_14(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_15) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_15(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_15(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_16) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_16(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_16(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_17) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_17(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_17(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_18) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_18(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_18(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_19) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_19(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_19(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_20) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_20(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_20(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_21) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_21(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_21(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_22) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_22(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_22(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_23) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_23(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_23(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_24) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_24(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_24(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_25) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_25(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_25(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_26) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_26(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_26(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_27) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_27(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_27(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_28) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_28(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_28(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_29) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_29(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_29(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_30) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_30(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_30(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_31) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_31(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_31(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_32) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_32(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_32(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_33) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_33(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_33(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_34) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_34(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_34(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_35) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_35(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_35(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_36) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_36(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_36(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_37) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_37(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_37(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_38) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_38(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_38(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_39) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_39(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_39(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_40) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_40(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_40(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_41) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_41(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_41(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_42) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_42(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_42(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_43) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_43(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_43(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_44) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_44(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_44(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_45) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_45(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_45(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_46) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_46(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_46(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_47) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_47(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_47(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_48) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_48(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_48(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_49) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_49(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_49(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_50) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_50(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_50(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_51) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_51(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_51(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_52) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_52(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_52(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_53) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_53(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_53(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_54) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_54(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_54(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_55) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_55(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_55(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_56) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_56(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_56(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_57) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_57(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_57(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_58) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_58(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_58(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_59) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_59(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_59(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_60) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_60(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_60(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_61) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_61(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_61(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_62) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_62(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_62(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_63) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_63(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_63(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_64) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_64(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_64(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_65) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_65(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_65(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_66) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_66(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_66(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_67) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_67(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_67(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_68) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_68(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_68(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_69) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_69(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_69(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_70) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_70(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_70(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_71) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_71(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_71(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_72) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_72(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_72(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_73) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_73(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_73(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_74) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_74(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_74(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_75) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_75(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_75(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_76) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_76(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_76(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_77) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_77(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_77(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_78) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_78(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_78(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_79) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_79(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_79(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_80) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_80(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_80(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_81) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_81(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_81(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_82) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_82(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_82(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_83) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_83(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_83(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_84) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_84(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_84(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_85) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_85(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_85(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_86) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_86(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_86(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_87) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_87(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_87(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_88) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_88(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_88(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_89) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_89(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_89(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_90) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_90(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_90(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_91) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_91(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_91(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_92) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_92(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_92(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_93) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_93(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_93(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_94) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_94(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_94(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_95) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_95(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_95(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_96) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_96(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_96(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_97) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_97(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_97(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_98) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_98(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_98(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_99) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_99(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_99(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_100) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_100(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_100(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_101) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_101(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_101(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_102) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_102(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_102(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_103) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_103(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_103(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_104) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_104(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_104(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_105) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_105(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_105(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_106) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_106(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_106(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_107) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_107(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_107(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_108) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_108(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_108(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_109) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_109(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_109(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_110) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_110(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_110(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_111) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_111(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_111(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_112) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_112(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_112(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_113) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_113(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_113(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_114) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_114(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_114(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_115) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_115(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_115(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_116) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_116(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_116(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_117) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_117(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_117(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_118) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_118(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_118(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_119) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_119(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_119(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_120) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_120(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_120(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_121) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_121(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_121(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_122) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_122(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_122(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_123) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_123(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_123(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_124) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_124(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_124(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_125) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_125(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_125(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_126) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_126(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_126(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_127) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_127(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_127(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_128) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_128(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_128(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_129) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_129(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_129(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_130) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_130(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_130(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_131) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_131(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_131(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_132) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_132(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_132(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_133) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_133(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_133(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_134) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_134(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_134(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_135) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_135(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_135(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_136) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_136(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_136(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_137) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_137(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_137(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_138) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_138(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_138(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_139) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_139(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_139(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_140) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_140(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_140(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_141) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_141(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_141(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_142) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_142(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_142(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_143) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_143(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_143(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_144) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_144(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_144(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_145) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_145(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_145(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_146) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_146(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_146(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_147) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_147(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_147(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_148) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_148(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_148(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_149) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_149(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_149(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_150) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_150(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_150(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_151) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_151(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_151(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_152) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_152(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_152(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_153) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_153(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_153(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_154) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_154(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_154(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_155) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_155(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_155(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_156) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_156(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_156(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_157) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_157(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_157(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_158) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_158(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_158(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_159) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_159(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_159(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_160) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_160(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_160(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_161) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_161(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_161(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_162) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_162(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_162(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_163) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_163(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_163(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_164) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_164(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_164(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_165) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_165(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_165(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_166) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_166(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_166(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_167) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_167(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_167(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_168) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_168(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_168(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_169) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_169(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_169(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_170) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_170(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_170(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_171) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_171(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_171(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_172) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_172(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_172(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_173) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_173(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_173(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_174) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_174(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_174(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_175) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_175(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_175(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_176) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_176(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_176(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_177) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_177(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_177(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_178) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_178(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_178(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_179) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_179(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_179(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_180) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_180(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_180(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_181) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_181(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_181(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_182) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_182(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_182(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_183) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_183(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_183(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_184) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_184(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_184(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_185) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_185(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_185(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_186) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_186(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_186(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_187) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_187(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_187(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_188) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_188(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_188(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_189) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_189(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_189(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_190) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_190(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_190(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_191) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_191(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_191(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_192) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_192(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_192(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_193) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_193(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_193(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_194) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_194(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_194(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_195) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_195(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_195(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_196) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_196(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_196(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_197) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_197(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_197(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_198) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_198(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_198(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_199) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_199(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_199(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_200) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_200(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_200(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_201) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_201(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_201(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_202) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_202(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_202(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_203) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_203(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_203(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_204) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_204(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_204(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_205) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_205(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_205(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_206) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_206(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_206(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_207) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_207(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_207(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_208) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_208(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_208(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_209) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_209(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_209(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_210) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_210(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_210(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_211) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_211(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_211(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_212) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_212(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_212(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_213) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_213(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_213(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_214) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_214(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_214(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_215) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_215(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_215(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_216) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_216(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_216(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_217) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_217(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_217(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_218) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_218(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_218(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_219) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_219(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_219(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_220) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_220(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_220(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_221) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_221(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_221(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_222) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_222(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_222(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_223) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_223(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_223(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_224) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_224(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_224(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_225) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_225(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_225(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_226) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_226(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_226(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_227) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_227(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_227(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_228) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_228(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_228(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_229) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_229(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_229(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_230) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_230(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_230(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_231) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_231(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_231(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_232) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_232(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_232(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_233) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_233(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_233(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_234) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_234(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_234(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_235) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_235(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_235(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_236) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_236(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_236(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_237) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_237(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_237(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_238) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_238(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_238(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_239) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_239(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_239(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_240) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_240(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_240(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_241) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_241(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_241(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_242) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_242(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_242(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_243) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_243(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_243(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_244) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_244(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_244(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_245) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_245(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_245(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_246) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_246(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_246(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_247) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_247(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_247(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_248) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_248(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_248(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_249) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_249(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_249(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_250) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_250(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_250(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_251) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_251(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_251(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_252) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_252(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_252(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_253) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_253(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_253(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_254) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_254(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_254(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_255) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_255(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_255(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_256) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_256(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_256(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_257) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_257(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_257(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_258) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_258(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_258(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_259) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_259(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_259(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_260) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_260(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_260(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_261) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_261(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_261(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_262) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_262(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_262(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_263) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_263(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_263(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_264) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_264(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_264(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_265) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_265(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_265(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_266) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_266(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_266(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_267) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_267(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_267(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_268) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_268(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_268(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_269) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_269(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_269(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_270) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_270(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_270(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_271) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_271(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_271(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_272) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_272(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_272(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_273) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_273(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_273(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_274) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_274(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_274(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_275) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_275(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_275(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_276) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_276(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_276(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_277) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_277(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_277(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_278) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_278(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_278(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_279) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_279(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_279(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_280) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_280(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_280(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_281) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_281(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_281(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_282) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_282(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_282(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_283) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_283(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_283(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_284) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_284(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_284(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_285) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_285(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_285(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_286) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_286(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_286(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_287) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_287(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_287(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_288) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_288(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_288(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_289) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_289(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_289(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_290) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_290(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_290(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_291) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_291(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_291(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_292) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_292(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_292(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_293) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_293(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_293(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_294) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_294(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_294(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_295) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_295(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_295(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_296) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_296(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_296(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_297) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_297(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_297(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_298) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_298(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_298(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_299) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_299(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_299(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_300) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_300(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_300(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_301) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_301(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_301(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_302) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_302(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_302(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_303) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_303(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_303(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_304) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_304(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_304(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_305) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_305(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_305(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_306) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_306(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_306(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_307) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_307(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_307(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_308) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_308(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_308(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_309) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_309(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_309(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_310) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_310(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_310(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_311) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_311(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_311(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_312) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_312(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_312(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_313) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_313(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_313(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_314) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_314(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_314(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_315) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_315(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_315(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_316) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_316(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_316(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_317) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_317(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_317(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_318) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_318(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_318(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_319) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_319(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_319(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_320) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_320(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_320(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_321) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_321(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_321(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_322) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_322(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_322(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_323) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_323(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_323(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_324) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_324(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_324(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_325) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_325(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_325(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_326) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_326(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_326(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_327) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_327(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_327(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_328) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_328(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_328(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_329) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_329(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_329(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_330) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_330(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_330(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_331) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_331(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_331(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_332) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_332(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_332(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_333) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_333(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_333(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_334) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_334(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_334(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_335) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_335(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_335(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_336) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_336(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_336(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_337) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_337(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_337(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_338) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_338(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_338(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_339) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_339(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_339(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_340) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_340(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_340(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_341) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_341(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_341(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_342) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_342(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_342(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_343) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_343(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_343(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_344) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_344(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_344(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_345) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_345(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_345(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_346) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_346(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_346(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_347) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_347(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_347(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_348) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_348(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_348(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_349) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_349(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_349(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_350) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_350(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_350(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_351) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_351(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_351(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_352) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_352(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_352(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_353) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_353(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_353(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_354) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_354(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_354(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_355) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_355(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_355(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_356) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_356(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_356(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_357) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_357(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_357(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_358) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_358(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_358(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_359) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_359(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_359(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_360) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_360(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_360(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_361) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_361(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_361(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_362) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_362(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_362(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_363) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_363(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_363(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_364) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_364(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_364(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_365) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_365(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_365(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_366) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_366(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_366(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_367) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_367(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_367(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_368) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_368(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_368(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_369) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_369(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_369(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_370) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_370(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_370(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_371) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_371(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_371(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_372) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_372(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_372(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_373) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_373(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_373(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_374) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_374(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_374(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_375) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_375(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_375(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_376) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_376(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_376(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_377) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_377(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_377(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_378) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_378(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_378(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_379) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_379(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_379(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_380) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_380(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_380(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_381) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_381(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_381(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_382) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_382(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_382(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_383) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_383(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_383(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_384) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_384(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_384(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_385) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_385(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_385(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_386) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_386(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_386(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_387) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_387(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_387(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_388) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_388(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_388(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_389) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_389(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_389(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_390) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_390(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_390(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_391) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_391(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_391(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_392) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_392(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_392(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_393) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_393(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_393(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_394) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_394(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_394(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_395) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_395(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_395(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_396) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_396(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_396(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_397) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_397(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_397(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_398) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_398(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_398(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_399) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_399(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_399(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_400) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_400(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_400(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_401) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_401(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_401(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_402) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_402(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_402(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_403) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_403(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_403(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_404) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_404(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_404(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_405) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_405(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_405(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_406) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_406(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_406(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_407) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_407(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_407(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_408) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_408(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_408(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_409) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_409(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_409(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_410) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_410(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_410(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_411) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_411(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_411(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_412) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_412(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_412(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_413) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_413(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_413(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_414) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_414(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_414(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_415) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_415(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_415(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_416) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_416(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_416(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_417) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_417(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_417(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_418) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_418(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_418(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_419) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_419(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_419(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_420) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_420(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_420(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_421) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_421(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_421(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_422) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_422(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_422(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_423) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_423(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_423(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_424) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_424(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_424(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_425) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_425(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_425(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_426) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_426(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_426(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_427) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_427(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_427(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_428) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_428(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_428(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_429) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_429(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_429(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_430) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_430(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_430(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_431) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_431(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_431(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_432) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_432(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_432(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_433) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_433(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_433(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_434) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_434(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_434(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_435) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_435(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_435(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_436) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_436(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_436(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_437) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_437(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_437(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_438) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_438(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_438(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_439) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_439(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_439(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_440) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_440(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_440(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_441) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_441(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_441(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_442) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_442(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_442(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_443) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_443(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_443(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_444) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_444(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_444(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_445) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_445(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_445(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_446) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_446(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_446(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_447) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_447(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_447(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_448) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_448(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_448(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_449) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_449(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_449(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_450) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_450(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_450(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_451) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_451(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_451(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_452) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_452(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_452(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_453) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_453(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_453(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_454) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_454(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_454(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_455) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_455(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_455(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_456) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_456(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_456(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_457) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_457(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_457(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_458) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_458(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_458(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_459) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_459(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_459(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_460) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_460(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_460(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_461) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_461(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_461(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_462) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_462(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_462(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_463) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_463(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_463(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_464) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_464(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_464(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_465) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_465(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_465(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_466) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_466(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_466(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_467) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_467(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_467(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_468) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_468(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_468(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_469) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_469(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_469(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_470) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_470(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_470(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_471) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_471(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_471(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_472) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_472(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_472(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_473) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_473(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_473(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_474) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_474(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_474(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_475) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_475(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_475(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_476) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_476(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_476(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_477) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_477(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_477(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_478) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_478(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_478(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_479) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_479(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_479(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_480) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_480(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_480(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_481) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_481(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_481(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_482) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_482(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_482(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_483) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_483(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_483(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_484) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_484(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_484(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_485) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_485(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_485(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_486) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_486(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_486(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_487) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_487(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_487(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_488) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_488(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_488(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_489) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_489(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_489(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_490) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_490(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_490(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_491) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_491(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_491(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_492) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_492(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_492(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_493) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_493(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_493(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_494) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_494(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_494(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_495) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_495(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_495(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_496) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_496(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_496(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_497) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_497(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_497(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_498) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_498(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_498(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_499) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_499(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_499(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_500) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_500(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_500(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_501) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_501(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_501(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_502) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_502(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_502(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_503) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_503(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_503(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_504) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_504(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_504(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_505) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_505(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_505(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_506) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_506(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_506(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_507) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_507(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_507(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_508) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_508(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_508(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_509) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_509(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_509(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_510) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_510(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_510(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_511) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_511(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_511(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_512) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_512(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_512(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_513) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_513(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_513(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_514) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_514(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_514(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_515) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_515(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_515(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_516) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_516(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_516(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_517) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_517(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_517(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_518) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_518(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_518(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_519) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_519(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_519(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_520) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_520(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_520(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_521) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_521(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_521(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_522) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_522(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_522(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_523) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_523(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_523(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_524) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_524(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_524(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_525) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_525(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_525(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_526) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_526(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_526(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_527) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_527(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_527(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_528) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_528(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_528(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_529) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_529(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_529(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_530) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_530(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_530(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_531) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_531(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_531(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_532) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_532(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_532(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_533) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_533(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_533(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_534) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_534(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_534(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_535) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_535(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_535(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_536) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_536(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_536(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_537) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_537(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_537(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_538) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_538(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_538(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_539) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_539(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_539(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_540) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_540(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_540(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_541) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_541(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_541(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_542) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_542(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_542(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_543) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_543(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_543(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_544) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_544(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_544(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_545) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_545(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_545(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_546) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_546(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_546(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_547) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_547(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_547(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_548) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_548(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_548(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_549) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_549(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_549(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_550) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_550(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_550(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_551) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_551(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_551(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_552) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_552(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_552(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_553) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_553(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_553(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_554) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_554(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_554(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_555) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_555(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_555(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_556) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_556(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_556(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_557) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_557(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_557(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_558) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_558(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_558(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_559) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_559(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_559(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_560) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_560(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_560(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_561) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_561(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_561(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_562) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_562(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_562(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_563) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_563(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_563(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_564) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_564(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_564(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_565) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_565(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_565(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_566) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_566(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_566(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_567) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_567(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_567(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_568) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_568(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_568(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_569) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_569(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_569(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_570) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_570(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_570(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_571) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_571(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_571(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_572) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_572(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_572(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_573) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_573(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_573(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_574) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_574(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_574(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_575) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_575(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_575(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_576) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_576(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_576(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_577) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_577(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_577(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_578) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_578(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_578(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_579) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_579(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_579(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_580) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_580(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_580(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_581) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_581(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_581(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_582) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_582(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_582(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_583) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_583(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_583(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_584) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_584(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_584(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_585) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_585(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_585(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_586) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_586(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_586(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_587) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_587(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_587(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_588) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_588(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_588(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_589) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_589(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_589(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_590) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_590(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_590(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_591) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_591(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_591(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_592) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_592(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_592(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_593) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_593(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_593(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_594) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_594(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_594(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_595) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_595(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_595(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_596) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_596(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_596(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_597) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_597(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_597(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_598) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_598(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_598(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_599) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_599(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_599(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_600) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_600(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_600(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_601) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_601(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_601(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_602) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_602(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_602(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_603) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_603(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_603(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_604) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_604(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_604(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_605) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_605(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_605(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_606) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_606(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_606(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_607) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_607(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_607(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_608) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_608(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_608(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_609) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_609(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_609(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_610) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_610(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_610(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_611) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_611(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_611(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_612) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_612(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_612(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_613) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_613(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_613(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_614) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_614(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_614(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_615) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_615(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_615(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_616) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_616(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_616(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_617) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_617(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_617(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_618) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_618(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_618(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_619) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_619(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_619(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_620) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_620(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_620(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_621) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_621(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_621(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_622) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_622(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_622(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_623) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_623(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_623(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_624) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_624(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_624(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_625) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_625(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_625(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_626) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_626(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_626(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_627) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_627(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_627(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_628) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_628(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_628(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_629) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_629(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_629(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_630) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_630(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_630(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_631) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_631(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_631(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_632) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_632(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_632(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_633) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_633(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_633(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_634) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_634(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_634(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_635) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_635(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_635(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_636) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_636(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_636(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_637) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_637(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_637(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_638) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_638(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_638(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_639) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_639(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_639(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_640) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_640(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_640(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_641) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_641(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_641(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_642) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_642(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_642(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_643) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_643(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_643(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_644) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_644(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_644(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_645) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_645(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_645(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_646) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_646(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_646(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_647) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_647(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_647(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_648) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_648(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_648(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_649) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_649(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_649(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_650) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_650(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_650(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_651) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_651(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_651(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_652) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_652(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_652(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_653) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_653(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_653(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_654) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_654(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_654(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_655) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_655(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_655(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_656) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_656(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_656(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_657) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_657(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_657(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_658) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_658(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_658(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_659) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_659(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_659(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_660) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_660(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_660(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_661) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_661(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_661(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_662) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_662(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_662(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_663) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_663(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_663(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_664) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_664(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_664(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_665) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_665(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_665(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_666) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_666(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_666(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_667) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_667(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_667(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_668) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_668(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_668(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_669) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_669(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_669(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_670) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_670(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_670(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_671) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_671(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_671(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_672) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_672(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_672(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_673) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_673(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_673(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_674) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_674(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_674(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_675) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_675(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_675(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_676) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_676(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_676(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_677) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_677(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_677(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_678) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_678(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_678(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_679) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_679(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_679(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_680) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_680(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_680(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_681) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_681(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_681(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_682) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_682(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_682(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_683) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_683(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_683(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_684) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_684(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_684(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_685) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_685(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_685(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_686) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_686(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_686(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_687) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_687(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_687(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_688) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_688(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_688(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_689) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_689(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_689(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_690) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_690(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_690(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_691) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_691(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_691(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_692) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_692(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_692(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_693) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_693(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_693(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_694) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_694(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_694(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_695) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_695(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_695(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_696) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_696(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_696(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_697) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_697(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_697(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_698) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_698(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_698(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_699) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_699(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_699(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_700) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_700(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_700(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_701) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_701(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_701(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_702) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_702(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_702(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_703) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_703(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_703(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_704) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_704(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_704(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_705) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_705(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_705(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_706) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_706(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_706(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_707) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_707(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_707(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_708) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_708(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_708(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_709) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_709(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_709(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_710) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_710(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_710(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_711) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_711(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_711(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_712) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_712(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_712(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_713) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_713(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_713(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_714) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_714(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_714(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_715) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_715(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_715(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_716) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_716(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_716(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_717) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_717(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_717(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_718) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_718(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_718(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_719) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_719(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_719(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_720) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_720(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_720(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_721) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_721(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_721(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_722) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_722(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_722(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_723) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_723(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_723(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_724) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_724(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_724(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_725) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_725(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_725(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_726) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_726(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_726(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_727) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_727(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_727(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_728) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_728(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_728(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_729) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_729(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_729(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_730) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_730(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_730(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_731) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_731(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_731(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_732) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_732(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_732(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_733) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_733(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_733(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_734) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_734(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_734(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_735) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_735(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_735(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_736) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_736(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_736(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_737) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_737(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_737(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_738) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_738(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_738(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_739) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_739(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_739(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_740) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_740(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_740(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_741) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_741(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_741(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_742) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_742(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_742(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_743) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_743(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_743(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_744) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_744(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_744(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_745) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_745(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_745(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_746) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_746(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_746(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_747) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_747(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_747(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_748) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_748(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_748(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_749) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_749(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_749(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_750) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_750(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_750(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_751) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_751(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_751(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_752) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_752(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_752(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_753) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_753(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_753(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_754) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_754(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_754(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_755) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_755(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_755(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_756) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_756(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_756(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_757) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_757(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_757(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_758) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_758(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_758(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_759) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_759(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_759(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_760) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_760(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_760(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_761) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_761(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_761(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_762) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_762(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_762(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_763) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_763(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_763(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_764) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_764(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_764(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_765) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_765(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_765(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_766) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_766(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_766(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_767) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_767(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_767(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_768) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_768(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_768(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_769) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_769(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_769(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_770) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_770(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_770(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_771) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_771(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_771(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_772) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_772(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_772(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_773) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_773(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_773(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_774) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_774(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_774(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_775) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_775(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_775(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_776) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_776(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_776(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_777) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_777(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_777(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_778) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_778(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_778(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_779) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_779(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_779(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_780) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_780(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_780(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_781) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_781(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_781(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_782) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_782(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_782(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_783) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_783(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_783(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_784) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_784(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_784(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_785) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_785(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_785(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_786) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_786(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_786(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_787) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_787(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_787(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_788) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_788(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_788(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_789) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_789(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_789(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_790) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_790(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_790(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_791) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_791(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_791(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_792) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_792(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_792(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_793) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_793(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_793(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_794) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_794(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_794(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_795) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_795(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_795(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_796) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_796(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_796(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_797) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_797(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_797(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_798) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_798(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_798(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_799) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_799(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_799(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_800) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_800(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_800(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_801) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_801(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_801(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_802) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_802(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_802(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_803) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_803(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_803(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_804) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_804(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_804(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_805) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_805(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_805(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_806) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_806(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_806(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_807) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_807(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_807(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_808) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_808(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_808(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_809) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_809(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_809(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_810) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_810(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_810(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_811) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_811(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_811(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_812) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_812(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_812(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_813) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_813(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_813(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_814) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_814(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_814(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_815) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_815(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_815(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_816) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_816(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_816(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_817) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_817(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_817(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_818) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_818(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_818(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_819) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_819(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_819(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_820) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_820(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_820(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_821) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_821(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_821(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_822) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_822(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_822(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_823) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_823(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_823(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_824) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_824(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_824(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_825) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_825(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_825(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_826) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_826(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_826(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_827) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_827(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_827(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_828) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_828(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_828(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_829) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_829(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_829(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_830) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_830(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_830(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_831) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_831(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_831(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_832) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_832(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_832(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_833) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_833(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_833(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_834) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_834(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_834(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_835) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_835(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_835(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_836) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_836(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_836(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_837) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_837(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_837(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_838) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_838(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_838(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_839) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_839(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_839(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_840) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_840(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_840(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_841) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_841(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_841(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_842) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_842(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_842(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_843) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_843(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_843(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_844) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_844(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_844(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_845) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_845(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_845(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_846) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_846(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_846(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_847) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_847(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_847(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_848) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_848(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_848(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_849) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_849(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_849(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_850) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_850(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_850(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_851) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_851(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_851(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_852) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_852(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_852(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_853) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_853(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_853(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_854) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_854(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_854(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_855) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_855(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_855(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_856) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_856(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_856(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_857) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_857(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_857(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_858) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_858(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_858(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_859) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_859(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_859(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_860) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_860(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_860(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_861) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_861(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_861(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_862) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_862(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_862(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_863) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_863(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_863(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_864) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_864(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_864(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_865) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_865(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_865(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_866) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_866(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_866(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_867) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_867(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_867(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_868) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_868(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_868(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_869) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_869(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_869(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_870) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_870(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_870(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_871) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_871(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_871(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_872) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_872(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_872(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_873) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_873(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_873(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_874) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_874(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_874(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_875) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_875(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_875(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_876) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_876(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_876(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_877) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_877(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_877(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_878) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_878(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_878(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_879) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_879(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_879(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_880) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_880(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_880(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_881) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_881(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_881(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_882) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_882(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_882(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_883) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_883(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_883(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_884) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_884(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_884(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_885) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_885(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_885(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_886) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_886(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_886(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_887) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_887(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_887(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_888) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_888(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_888(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_889) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_889(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_889(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_890) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_890(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_890(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_891) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_891(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_891(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_892) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_892(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_892(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_893) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_893(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_893(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_894) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_894(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_894(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_895) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_895(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_895(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_896) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_896(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_896(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_897) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_897(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_897(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_898) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_898(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_898(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_899) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_899(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_899(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_900) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_900(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_900(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_901) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_901(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_901(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_902) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_902(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_902(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_903) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_903(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_903(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_904) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_904(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_904(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_905) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_905(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_905(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_906) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_906(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_906(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_907) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_907(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_907(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_908) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_908(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_908(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_909) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_909(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_909(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_910) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_910(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_910(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_911) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_911(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_911(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_912) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_912(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_912(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_913) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_913(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_913(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_914) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_914(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_914(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_915) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_915(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_915(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_916) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_916(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_916(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_917) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_917(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_917(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_918) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_918(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_918(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_919) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_919(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_919(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_920) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_920(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_920(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_921) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_921(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_921(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_922) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_922(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_922(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_923) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_923(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_923(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_924) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_924(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_924(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_925) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_925(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_925(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_926) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_926(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_926(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_927) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_927(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_927(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_928) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_928(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_928(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_929) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_929(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_929(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_930) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_930(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_930(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_931) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_931(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_931(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_932) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_932(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_932(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_933) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_933(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_933(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_934) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_934(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_934(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_935) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_935(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_935(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_936) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_936(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_936(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_937) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_937(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_937(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_938) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_938(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_938(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_939) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_939(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_939(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_940) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_940(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_940(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_941) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_941(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_941(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_942) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_942(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_942(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_943) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_943(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_943(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_944) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_944(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_944(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_945) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_945(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_945(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_946) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_946(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_946(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_947) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_947(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_947(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_948) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_948(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_948(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_949) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_949(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_949(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_950) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_950(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_950(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_951) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_951(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_951(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_952) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_952(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_952(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_953) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_953(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_953(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_954) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_954(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_954(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_955) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_955(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_955(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_956) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_956(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_956(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_957) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_957(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_957(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_958) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_958(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_958(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_959) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_959(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_959(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_960) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_960(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_960(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_961) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_961(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_961(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_962) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_962(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_962(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_963) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_963(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_963(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_964) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_964(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_964(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_965) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_965(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_965(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_966) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_966(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_966(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_967) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_967(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_967(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_968) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_968(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_968(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_969) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_969(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_969(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_970) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_970(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_970(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_971) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_971(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_971(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_972) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_972(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_972(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_973) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_973(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_973(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_974) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_974(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_974(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_975) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_975(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_975(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_976) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_976(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_976(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_977) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_977(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_977(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_978) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_978(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_978(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_979) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_979(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_979(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_980) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_980(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_980(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_981) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_981(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_981(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_982) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_982(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_982(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_983) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_983(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_983(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_984) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_984(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_984(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_985) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_985(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_985(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_986) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_986(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_986(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_987) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_987(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_987(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_988) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_988(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_988(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_989) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_989(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_989(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_990) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_990(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_990(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_991) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_991(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_991(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_992) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_992(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_992(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_993) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_993(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_993(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_994) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_994(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_994(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_995) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_995(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_995(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_996) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_996(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_996(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_997) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_997(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_997(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_998) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_998(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_998(FUNC) +#endif + +#if defined(EIGEN_TEST_PART_999) || defined(EIGEN_TEST_PART_ALL) +#define CALL_SUBTEST_999(FUNC) CALL_SUBTEST(FUNC) +#else +#define CALL_SUBTEST_999(FUNC) +#endif +
diff --git a/test/spqr_support.cpp b/test/spqr_support.cpp index baa25a0..79c2c12 100644 --- a/test/spqr_support.cpp +++ b/test/spqr_support.cpp
@@ -20,8 +20,8 @@ int cols = internal::random<int>(1,rows); double density = (std::max)(8./(rows*cols), 0.01); - A.resize(rows,rows); - dA.resize(rows,rows); + A.resize(rows,cols); + dA.resize(rows,cols); initSparse<Scalar>(density, dA, A,ForceNonZeroDiag); A.makeCompressed(); return rows; @@ -57,7 +57,7 @@ refX = dA.colPivHouseholderQr().solve(b); VERIFY(x.isApprox(refX,test_precision<Scalar>())); } -void test_spqr_support() +EIGEN_DECLARE_TEST(spqr_support) { CALL_SUBTEST_1(test_spqr_scalar<double>()); CALL_SUBTEST_2(test_spqr_scalar<std::complex<double> >());
diff --git a/test/stable_norm.cpp b/test/stable_norm.cpp index c3eb5ff..ee5f916 100644 --- a/test/stable_norm.cpp +++ b/test/stable_norm.cpp
@@ -21,7 +21,6 @@ */ using std::sqrt; using std::abs; - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; @@ -65,6 +64,8 @@ factor = internal::random<Scalar>(); Scalar small = factor * ((std::numeric_limits<RealScalar>::min)() * RealScalar(1e4)); + Scalar one(1); + MatrixType vzero = MatrixType::Zero(rows, cols), vrand = MatrixType::Random(rows, cols), vbig(rows, cols), @@ -78,6 +79,14 @@ VERIFY_IS_APPROX(vrand.blueNorm(), vrand.norm()); VERIFY_IS_APPROX(vrand.hypotNorm(), vrand.norm()); + // test with expressions as input + VERIFY_IS_APPROX((one*vrand).stableNorm(), vrand.norm()); + VERIFY_IS_APPROX((one*vrand).blueNorm(), vrand.norm()); + VERIFY_IS_APPROX((one*vrand).hypotNorm(), vrand.norm()); + VERIFY_IS_APPROX((one*vrand+one*vrand-one*vrand).stableNorm(), vrand.norm()); + VERIFY_IS_APPROX((one*vrand+one*vrand-one*vrand).blueNorm(), vrand.norm()); + VERIFY_IS_APPROX((one*vrand+one*vrand-one*vrand).hypotNorm(), vrand.norm()); + RealScalar size = static_cast<RealScalar>(m.size()); // test numext::isfinite @@ -180,13 +189,51 @@ } } -void test_stable_norm() +template<typename Scalar> +void test_hypot() +{ + typedef typename NumTraits<Scalar>::Real RealScalar; + Scalar factor = internal::random<Scalar>(); + while(numext::abs2(factor)<RealScalar(1e-4)) + factor = internal::random<Scalar>(); + Scalar big = factor * ((std::numeric_limits<RealScalar>::max)() * RealScalar(1e-4)); + + factor = internal::random<Scalar>(); + while(numext::abs2(factor)<RealScalar(1e-4)) + factor = internal::random<Scalar>(); + Scalar small = factor * ((std::numeric_limits<RealScalar>::min)() * RealScalar(1e4)); + + Scalar one (1), + zero (0), + sqrt2 (std::sqrt(2)), + nan (std::numeric_limits<RealScalar>::quiet_NaN()); + + Scalar a = internal::random<Scalar>(-1,1); + Scalar b = internal::random<Scalar>(-1,1); + VERIFY_IS_APPROX(numext::hypot(a,b),std::sqrt(numext::abs2(a)+numext::abs2(b))); + VERIFY_IS_EQUAL(numext::hypot(zero,zero), zero); + VERIFY_IS_APPROX(numext::hypot(one, one), sqrt2); + VERIFY_IS_APPROX(numext::hypot(big,big), sqrt2*numext::abs(big)); + VERIFY_IS_APPROX(numext::hypot(small,small), sqrt2*numext::abs(small)); + VERIFY_IS_APPROX(numext::hypot(small,big), numext::abs(big)); + VERIFY((numext::isnan)(numext::hypot(nan,a))); + VERIFY((numext::isnan)(numext::hypot(a,nan))); +} + +EIGEN_DECLARE_TEST(stable_norm) { for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_3( test_hypot<double>() ); + CALL_SUBTEST_4( test_hypot<float>() ); + CALL_SUBTEST_5( test_hypot<std::complex<double> >() ); + CALL_SUBTEST_6( test_hypot<std::complex<float> >() ); + CALL_SUBTEST_1( stable_norm(Matrix<float, 1, 1>()) ); CALL_SUBTEST_2( stable_norm(Vector4d()) ); CALL_SUBTEST_3( stable_norm(VectorXd(internal::random<int>(10,2000))) ); + CALL_SUBTEST_3( stable_norm(MatrixXd(internal::random<int>(10,200), internal::random<int>(10,200))) ); CALL_SUBTEST_4( stable_norm(VectorXf(internal::random<int>(10,2000))) ); CALL_SUBTEST_5( stable_norm(VectorXcd(internal::random<int>(10,2000))) ); + CALL_SUBTEST_6( stable_norm(VectorXcf(internal::random<int>(10,2000))) ); } }
diff --git a/test/stddeque.cpp b/test/stddeque.cpp index bb4b476..ea85ea9 100644 --- a/test/stddeque.cpp +++ b/test/stddeque.cpp
@@ -15,12 +15,10 @@ template<typename MatrixType> void check_stddeque_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; - Index rows = m.rows(); Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::deque<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType(rows,cols)), w(20, y); + std::deque<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType::Zero(rows,cols)), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -35,7 +33,7 @@ ++wi; } - v.resize(21); + v.resize(21,MatrixType::Zero(rows,cols)); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -48,8 +46,8 @@ void check_stddeque_transform(const TransformType&) { typedef typename TransformType::MatrixType MatrixType; - TransformType x(MatrixType::Random()), y(MatrixType::Random()); - std::deque<TransformType,Eigen::aligned_allocator<TransformType> > v(10), w(20, y); + TransformType x(MatrixType::Random()), y(MatrixType::Random()), ti=TransformType::Identity(); + std::deque<TransformType,Eigen::aligned_allocator<TransformType> > v(10,ti), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -64,7 +62,7 @@ ++wi; } - v.resize(21); + v.resize(21,ti); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -77,8 +75,8 @@ void check_stddeque_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::deque<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::deque<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10,qi), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -93,7 +91,7 @@ ++wi; } - v.resize(21); + v.resize(21,qi); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -102,7 +100,7 @@ VERIFY_IS_APPROX(v.back(), x); } -void test_stddeque() +EIGEN_DECLARE_TEST(stddeque) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stddeque_matrix(Vector2f()));
diff --git a/test/stddeque_overload.cpp b/test/stddeque_overload.cpp index 4da618b..0f59f06 100644 --- a/test/stddeque_overload.cpp +++ b/test/stddeque_overload.cpp
@@ -28,10 +28,10 @@ template<typename MatrixType> void check_stddeque_matrix(const MatrixType& m) { - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::deque<MatrixType> v(10, MatrixType(rows,cols)), w(20, y); + std::deque<MatrixType> v(10, MatrixType::Zero(rows,cols)), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -64,8 +64,8 @@ void check_stddeque_transform(const TransformType&) { typedef typename TransformType::MatrixType MatrixType; - TransformType x(MatrixType::Random()), y(MatrixType::Random()); - std::deque<TransformType> v(10), w(20, y); + TransformType x(MatrixType::Random()), y(MatrixType::Random()), ti=TransformType::Identity(); + std::deque<TransformType> v(10,ti), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -75,7 +75,7 @@ VERIFY_IS_APPROX(w[i], v[i]); } - v.resize(21); + v.resize(21,ti); v[20] = x; VERIFY_IS_APPROX(v[20], x); v.resize(22,y); @@ -98,8 +98,8 @@ void check_stddeque_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::deque<QuaternionType> v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::deque<QuaternionType> v(10,qi), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -109,7 +109,7 @@ VERIFY_IS_APPROX(w[i], v[i]); } - v.resize(21); + v.resize(21,qi); v[20] = x; VERIFY_IS_APPROX(v[20], x); v.resize(22,y); @@ -128,7 +128,7 @@ } } -void test_stddeque_overload() +EIGEN_DECLARE_TEST(stddeque_overload) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stddeque_matrix(Vector2f()));
diff --git a/test/stdlist.cpp b/test/stdlist.cpp index 17cce77..1af9e6e 100644 --- a/test/stdlist.cpp +++ b/test/stdlist.cpp
@@ -15,12 +15,10 @@ template<typename MatrixType> void check_stdlist_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; - Index rows = m.rows(); Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::list<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType(rows,cols)), w(20, y); + std::list<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType::Zero(rows,cols)), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -35,7 +33,7 @@ ++wi; } - v.resize(21); + v.resize(21, MatrixType::Zero(rows,cols)); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -48,8 +46,8 @@ void check_stdlist_transform(const TransformType&) { typedef typename TransformType::MatrixType MatrixType; - TransformType x(MatrixType::Random()), y(MatrixType::Random()); - std::list<TransformType,Eigen::aligned_allocator<TransformType> > v(10), w(20, y); + TransformType x(MatrixType::Random()), y(MatrixType::Random()), ti=TransformType::Identity(); + std::list<TransformType,Eigen::aligned_allocator<TransformType> > v(10,ti), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -64,7 +62,7 @@ ++wi; } - v.resize(21); + v.resize(21, ti); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -77,8 +75,8 @@ void check_stdlist_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::list<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::list<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10,qi), w(20, y); v.front() = x; w.front() = w.back(); VERIFY_IS_APPROX(w.front(), w.back()); @@ -93,7 +91,7 @@ ++wi; } - v.resize(21); + v.resize(21,qi); v.back() = x; VERIFY_IS_APPROX(v.back(), x); v.resize(22,y); @@ -102,7 +100,7 @@ VERIFY_IS_APPROX(v.back(), x); } -void test_stdlist() +EIGEN_DECLARE_TEST(stdlist) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stdlist_matrix(Vector2f()));
diff --git a/test/stdlist_overload.cpp b/test/stdlist_overload.cpp index bb910bd..a78516e 100644 --- a/test/stdlist_overload.cpp +++ b/test/stdlist_overload.cpp
@@ -44,10 +44,10 @@ template<typename MatrixType> void check_stdlist_matrix(const MatrixType& m) { - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::list<MatrixType> v(10, MatrixType(rows,cols)), w(20, y); + std::list<MatrixType> v(10, MatrixType::Zero(rows,cols)), w(20, y); typename std::list<MatrixType>::iterator itv = get(v, 5); typename std::list<MatrixType>::iterator itw = get(w, 6); *itv = x; @@ -86,8 +86,8 @@ void check_stdlist_transform(const TransformType&) { typedef typename TransformType::MatrixType MatrixType; - TransformType x(MatrixType::Random()), y(MatrixType::Random()); - std::list<TransformType> v(10), w(20, y); + TransformType x(MatrixType::Random()), y(MatrixType::Random()), ti=TransformType::Identity(); + std::list<TransformType> v(10,ti), w(20, y); typename std::list<TransformType>::iterator itv = get(v, 5); typename std::list<TransformType>::iterator itw = get(w, 6); *itv = x; @@ -103,7 +103,7 @@ ++itw; } - v.resize(21); + v.resize(21, ti); set(v, 20, x); VERIFY_IS_APPROX(*get(v, 20), x); v.resize(22,y); @@ -126,8 +126,8 @@ void check_stdlist_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::list<QuaternionType> v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::list<QuaternionType> v(10,qi), w(20, y); typename std::list<QuaternionType>::iterator itv = get(v, 5); typename std::list<QuaternionType>::iterator itw = get(w, 6); *itv = x; @@ -143,7 +143,7 @@ ++itw; } - v.resize(21); + v.resize(21,qi); set(v, 20, x); VERIFY_IS_APPROX(*get(v, 20), x); v.resize(22,y); @@ -162,7 +162,7 @@ } } -void test_stdlist_overload() +EIGEN_DECLARE_TEST(stdlist_overload) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stdlist_matrix(Vector2f()));
diff --git a/test/stdvector.cpp b/test/stdvector.cpp index 6e173c6..18de240 100644 --- a/test/stdvector.cpp +++ b/test/stdvector.cpp
@@ -14,10 +14,10 @@ template<typename MatrixType> void check_stdvector_matrix(const MatrixType& m) { - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::vector<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType(rows,cols)), w(20, y); + std::vector<MatrixType,Eigen::aligned_allocator<MatrixType> > v(10, MatrixType::Zero(rows,cols)), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -34,7 +34,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(MatrixType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(MatrixType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -69,7 +69,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(TransformType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(TransformType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -86,8 +86,8 @@ void check_stdvector_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::vector<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::vector<QuaternionType,Eigen::aligned_allocator<QuaternionType> > v(10,qi), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -104,7 +104,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(QuaternionType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(QuaternionType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -117,7 +117,17 @@ } } -void test_stdvector() +// the code below triggered an invalid warning with gcc >= 7 +// eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object size 9223372036854775807 +// This has been reported to gcc there: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544 +void std_vector_gcc_warning() +{ + typedef Eigen::Vector3f T; + std::vector<T, Eigen::aligned_allocator<T> > v; + v.push_back(T()); +} + +EIGEN_DECLARE_TEST(stdvector) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stdvector_matrix(Vector2f()));
diff --git a/test/stdvector_overload.cpp b/test/stdvector_overload.cpp index 736ff0e..da04f8a 100644 --- a/test/stdvector_overload.cpp +++ b/test/stdvector_overload.cpp
@@ -28,10 +28,10 @@ template<typename MatrixType> void check_stdvector_matrix(const MatrixType& m) { - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); MatrixType x = MatrixType::Random(rows,cols), y = MatrixType::Random(rows,cols); - std::vector<MatrixType> v(10, MatrixType(rows,cols)), w(20, y); + std::vector<MatrixType> v(10, MatrixType::Zero(rows,cols)), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -48,7 +48,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(MatrixType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(MatrixType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -83,7 +83,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(TransformType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(TransformType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -100,8 +100,8 @@ void check_stdvector_quaternion(const QuaternionType&) { typedef typename QuaternionType::Coefficients Coefficients; - QuaternionType x(Coefficients::Random()), y(Coefficients::Random()); - std::vector<QuaternionType> v(10), w(20, y); + QuaternionType x(Coefficients::Random()), y(Coefficients::Random()), qi=QuaternionType::Identity(); + std::vector<QuaternionType> v(10,qi), w(20, y); v[5] = x; w[6] = v[5]; VERIFY_IS_APPROX(w[6], v[5]); @@ -118,7 +118,7 @@ VERIFY_IS_APPROX(v[21], y); v.push_back(x); VERIFY_IS_APPROX(v[22], x); - VERIFY((size_t)&(v[22]) == (size_t)&(v[21]) + sizeof(QuaternionType)); + VERIFY((internal::UIntPtr)&(v[22]) == (internal::UIntPtr)&(v[21]) + sizeof(QuaternionType)); // do a lot of push_back such that the vector gets internally resized // (with memory reallocation) @@ -131,7 +131,7 @@ } } -void test_stdvector_overload() +EIGEN_DECLARE_TEST(stdvector_overload) { // some non vectorizable fixed sizes CALL_SUBTEST_1(check_stdvector_matrix(Vector2f()));
diff --git a/test/stl_iterators.cpp b/test/stl_iterators.cpp new file mode 100644 index 0000000..5fef34c --- /dev/null +++ b/test/stl_iterators.cpp
@@ -0,0 +1,492 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include <numeric> +#include "main.h" + +template< class Iterator > +std::reverse_iterator<Iterator> +make_reverse_iterator( Iterator i ) +{ + return std::reverse_iterator<Iterator>(i); +} + +#if !EIGEN_HAS_CXX11 +template<class ForwardIt> +ForwardIt is_sorted_until(ForwardIt firstIt, ForwardIt lastIt) +{ + if (firstIt != lastIt) { + ForwardIt next = firstIt; + while (++next != lastIt) { + if (*next < *firstIt) + return next; + firstIt = next; + } + } + return lastIt; +} +template<class ForwardIt> +bool is_sorted(ForwardIt firstIt, ForwardIt lastIt) +{ + return ::is_sorted_until(firstIt, lastIt) == lastIt; +} +#else +using std::is_sorted; +#endif + +template<typename XprType> +bool is_pointer_based_stl_iterator(const internal::pointer_based_stl_iterator<XprType> &) { return true; } + +template<typename XprType> +bool is_generic_randaccess_stl_iterator(const internal::generic_randaccess_stl_iterator<XprType> &) { return true; } + +template<typename Xpr> +void check_begin_end_for_loop(Xpr xpr) +{ + const Xpr& cxpr(xpr); + Index i = 0; + + i = 0; + for(typename Xpr::iterator it = xpr.begin(); it!=xpr.end(); ++it) { VERIFY_IS_EQUAL(*it,xpr[i++]); } + + i = 0; + for(typename Xpr::const_iterator it = xpr.cbegin(); it!=xpr.cend(); ++it) { VERIFY_IS_EQUAL(*it,xpr[i++]); } + + i = 0; + for(typename Xpr::const_iterator it = cxpr.begin(); it!=cxpr.end(); ++it) { VERIFY_IS_EQUAL(*it,xpr[i++]); } + + i = 0; + for(typename Xpr::const_iterator it = xpr.begin(); it!=xpr.end(); ++it) { VERIFY_IS_EQUAL(*it,xpr[i++]); } + + { + // simple API check + typename Xpr::const_iterator cit = xpr.begin(); + cit = xpr.cbegin(); + + #if EIGEN_HAS_CXX11 + auto tmp1 = xpr.begin(); + VERIFY(tmp1==xpr.begin()); + auto tmp2 = xpr.cbegin(); + VERIFY(tmp2==xpr.cbegin()); + #endif + } + + VERIFY( xpr.end() -xpr.begin() == xpr.size() ); + VERIFY( xpr.cend()-xpr.begin() == xpr.size() ); + VERIFY( xpr.end() -xpr.cbegin() == xpr.size() ); + VERIFY( xpr.cend()-xpr.cbegin() == xpr.size() ); + + if(xpr.size()>0) { + VERIFY(xpr.begin() != xpr.end()); + VERIFY(xpr.begin() < xpr.end()); + VERIFY(xpr.begin() <= xpr.end()); + VERIFY(!(xpr.begin() == xpr.end())); + VERIFY(!(xpr.begin() > xpr.end())); + VERIFY(!(xpr.begin() >= xpr.end())); + + VERIFY(xpr.cbegin() != xpr.end()); + VERIFY(xpr.cbegin() < xpr.end()); + VERIFY(xpr.cbegin() <= xpr.end()); + VERIFY(!(xpr.cbegin() == xpr.end())); + VERIFY(!(xpr.cbegin() > xpr.end())); + VERIFY(!(xpr.cbegin() >= xpr.end())); + + VERIFY(xpr.begin() != xpr.cend()); + VERIFY(xpr.begin() < xpr.cend()); + VERIFY(xpr.begin() <= xpr.cend()); + VERIFY(!(xpr.begin() == xpr.cend())); + VERIFY(!(xpr.begin() > xpr.cend())); + VERIFY(!(xpr.begin() >= xpr.cend())); + } +} + +template<typename Scalar, int Rows, int Cols> +void test_stl_iterators(int rows=Rows, int cols=Cols) +{ + typedef Matrix<Scalar,Rows,1> VectorType; + #if EIGEN_HAS_CXX11 + typedef Matrix<Scalar,1,Cols> RowVectorType; + #endif + typedef Matrix<Scalar,Rows,Cols,ColMajor> ColMatrixType; + typedef Matrix<Scalar,Rows,Cols,RowMajor> RowMatrixType; + VectorType v = VectorType::Random(rows); + const VectorType& cv(v); + ColMatrixType A = ColMatrixType::Random(rows,cols); + const ColMatrixType& cA(A); + RowMatrixType B = RowMatrixType::Random(rows,cols); + + Index i, j; + + // Check we got a fast pointer-based iterator when expected + { + VERIFY( is_pointer_based_stl_iterator(v.begin()) ); + VERIFY( is_pointer_based_stl_iterator(v.end()) ); + VERIFY( is_pointer_based_stl_iterator(cv.begin()) ); + VERIFY( is_pointer_based_stl_iterator(cv.end()) ); + + j = internal::random<Index>(0,A.cols()-1); + VERIFY( is_pointer_based_stl_iterator(A.col(j).begin()) ); + VERIFY( is_pointer_based_stl_iterator(A.col(j).end()) ); + VERIFY( is_pointer_based_stl_iterator(cA.col(j).begin()) ); + VERIFY( is_pointer_based_stl_iterator(cA.col(j).end()) ); + + i = internal::random<Index>(0,A.rows()-1); + VERIFY( is_pointer_based_stl_iterator(A.row(i).begin()) ); + VERIFY( is_pointer_based_stl_iterator(A.row(i).end()) ); + VERIFY( is_pointer_based_stl_iterator(cA.row(i).begin()) ); + VERIFY( is_pointer_based_stl_iterator(cA.row(i).end()) ); + + VERIFY( is_pointer_based_stl_iterator(A.reshaped().begin()) ); + VERIFY( is_pointer_based_stl_iterator(A.reshaped().end()) ); + VERIFY( is_pointer_based_stl_iterator(cA.reshaped().begin()) ); + VERIFY( is_pointer_based_stl_iterator(cA.reshaped().end()) ); + + VERIFY( is_pointer_based_stl_iterator(B.template reshaped<AutoOrder>().begin()) ); + VERIFY( is_pointer_based_stl_iterator(B.template reshaped<AutoOrder>().end()) ); + + VERIFY( is_generic_randaccess_stl_iterator(A.template reshaped<RowMajor>().begin()) ); + VERIFY( is_generic_randaccess_stl_iterator(A.template reshaped<RowMajor>().end()) ); + } + + { + check_begin_end_for_loop(v); + check_begin_end_for_loop(A.col(internal::random<Index>(0,A.cols()-1))); + check_begin_end_for_loop(A.row(internal::random<Index>(0,A.rows()-1))); + check_begin_end_for_loop(v+v); + } + +#if EIGEN_HAS_CXX11 + // check swappable + { + using std::swap; + // pointer-based + { + VectorType v_copy = v; + auto a = v.begin(); + auto b = v.end()-1; + swap(a,b); + VERIFY_IS_EQUAL(v,v_copy); + VERIFY_IS_EQUAL(*b,*v.begin()); + VERIFY_IS_EQUAL(*b,v(0)); + VERIFY_IS_EQUAL(*a,v.end()[-1]); + VERIFY_IS_EQUAL(*a,v(last)); + } + + // generic + { + RowMatrixType B_copy = B; + auto Br = B.reshaped(); + auto a = Br.begin(); + auto b = Br.end()-1; + swap(a,b); + VERIFY_IS_EQUAL(B,B_copy); + VERIFY_IS_EQUAL(*b,*Br.begin()); + VERIFY_IS_EQUAL(*b,Br(0)); + VERIFY_IS_EQUAL(*a,Br.end()[-1]); + VERIFY_IS_EQUAL(*a,Br(last)); + } + } + + // check non-const iterator with for-range loops + { + i = 0; + for(auto x : v) { VERIFY_IS_EQUAL(x,v[i++]); } + + j = internal::random<Index>(0,A.cols()-1); + i = 0; + for(auto x : A.col(j)) { VERIFY_IS_EQUAL(x,A(i++,j)); } + + i = 0; + for(auto x : (v+A.col(j))) { VERIFY_IS_APPROX(x,v(i)+A(i,j)); ++i; } + + j = 0; + i = internal::random<Index>(0,A.rows()-1); + for(auto x : A.row(i)) { VERIFY_IS_EQUAL(x,A(i,j++)); } + + i = 0; + for(auto x : A.reshaped()) { VERIFY_IS_EQUAL(x,A(i++)); } + } + + // same for const_iterator + { + i = 0; + for(auto x : cv) { VERIFY_IS_EQUAL(x,v[i++]); } + + i = 0; + for(auto x : cA.reshaped()) { VERIFY_IS_EQUAL(x,A(i++)); } + + j = 0; + i = internal::random<Index>(0,A.rows()-1); + for(auto x : cA.row(i)) { VERIFY_IS_EQUAL(x,A(i,j++)); } + } + + // check reshaped() on row-major + { + i = 0; + Matrix<Scalar,Dynamic,Dynamic,ColMajor> Bc = B; + for(auto x : B.reshaped()) { VERIFY_IS_EQUAL(x,Bc(i++)); } + } + + // check write access + { + VectorType w(v.size()); + i = 0; + for(auto& x : w) { x = v(i++); } + VERIFY_IS_EQUAL(v,w); + } + + // check for dangling pointers + { + // no dangling because pointer-based + { + j = internal::random<Index>(0,A.cols()-1); + auto it = A.col(j).begin(); + for(i=0;i<rows;++i) { + VERIFY_IS_EQUAL(it[i],A(i,j)); + } + } + + // no dangling because pointer-based + { + i = internal::random<Index>(0,A.rows()-1); + auto it = A.row(i).begin(); + for(j=0;j<cols;++j) { VERIFY_IS_EQUAL(it[j],A(i,j)); } + } + + { + j = internal::random<Index>(0,A.cols()-1); + // this would produce a dangling pointer: + // auto it = (A+2*A).col(j).begin(); + // we need to name the temporary expression: + auto tmp = (A+2*A).col(j); + auto it = tmp.begin(); + for(i=0;i<rows;++i) { + VERIFY_IS_APPROX(it[i],3*A(i,j)); + } + } + } +#endif + + if(rows>=3) { + VERIFY_IS_EQUAL((v.begin()+rows/2)[1], v(rows/2+1)); + + VERIFY_IS_EQUAL((A.rowwise().begin()+rows/2)[1], A.row(rows/2+1)); + } + + if(cols>=3) { + VERIFY_IS_EQUAL((A.colwise().begin()+cols/2)[1], A.col(cols/2+1)); + } + + // check std::sort + { + // first check that is_sorted returns false when required + if(rows>=2) + { + v(1) = v(0)-Scalar(1); + #if EIGEN_HAS_CXX11 + VERIFY(!is_sorted(std::begin(v),std::end(v))); + #else + VERIFY(!is_sorted(v.cbegin(),v.cend())); + #endif + } + + // on a vector + { + std::sort(v.begin(),v.end()); + VERIFY(is_sorted(v.begin(),v.end())); + VERIFY(!::is_sorted(make_reverse_iterator(v.end()),make_reverse_iterator(v.begin()))); + } + + // on a column of a column-major matrix -> pointer-based iterator and default increment + { + j = internal::random<Index>(0,A.cols()-1); + // std::sort(begin(A.col(j)),end(A.col(j))); // does not compile because this returns const iterators + typename ColMatrixType::ColXpr Acol = A.col(j); + std::sort(Acol.begin(),Acol.end()); + VERIFY(is_sorted(Acol.cbegin(),Acol.cend())); + A.setRandom(); + + std::sort(A.col(j).begin(),A.col(j).end()); + VERIFY(is_sorted(A.col(j).cbegin(),A.col(j).cend())); + A.setRandom(); + } + + // on a row of a rowmajor matrix -> pointer-based iterator and runtime increment + { + i = internal::random<Index>(0,A.rows()-1); + typename ColMatrixType::RowXpr Arow = A.row(i); + VERIFY_IS_EQUAL( std::distance(Arow.begin(),Arow.end()), cols); + std::sort(Arow.begin(),Arow.end()); + VERIFY(is_sorted(Arow.cbegin(),Arow.cend())); + A.setRandom(); + + std::sort(A.row(i).begin(),A.row(i).end()); + VERIFY(is_sorted(A.row(i).cbegin(),A.row(i).cend())); + A.setRandom(); + } + + // with a generic iterator + { + Reshaped<RowMatrixType,RowMatrixType::SizeAtCompileTime,1> B1 = B.reshaped(); + std::sort(B1.begin(),B1.end()); + VERIFY(is_sorted(B1.cbegin(),B1.cend())); + B.setRandom(); + + // assertion because nested expressions are different + // std::sort(B.reshaped().begin(),B.reshaped().end()); + // VERIFY(is_sorted(B.reshaped().cbegin(),B.reshaped().cend())); + // B.setRandom(); + } + } + + // check with partial_sum + { + j = internal::random<Index>(0,A.cols()-1); + typename ColMatrixType::ColXpr Acol = A.col(j); + std::partial_sum(Acol.begin(), Acol.end(), v.begin()); + VERIFY_IS_APPROX(v(seq(1,last)), v(seq(0,last-1))+Acol(seq(1,last))); + + // inplace + std::partial_sum(Acol.begin(), Acol.end(), Acol.begin()); + VERIFY_IS_APPROX(v, Acol); + } + + // stress random access as required by std::nth_element + if(rows>=3) + { + v.setRandom(); + VectorType v1 = v; + std::sort(v1.begin(),v1.end()); + std::nth_element(v.begin(), v.begin()+rows/2, v.end()); + VERIFY_IS_APPROX(v1(rows/2), v(rows/2)); + + v.setRandom(); + v1 = v; + std::sort(v1.begin()+rows/2,v1.end()); + std::nth_element(v.begin()+rows/2, v.begin()+rows/4, v.end()); + VERIFY_IS_APPROX(v1(rows/4), v(rows/4)); + } + +#if EIGEN_HAS_CXX11 + // check rows/cols iterators with range-for loops + { + j = 0; + for(auto c : A.colwise()) { VERIFY_IS_APPROX(c.sum(), A.col(j).sum()); ++j; } + j = 0; + for(auto c : B.colwise()) { VERIFY_IS_APPROX(c.sum(), B.col(j).sum()); ++j; } + + j = 0; + for(auto c : B.colwise()) { + i = 0; + for(auto& x : c) { + VERIFY_IS_EQUAL(x, B(i,j)); + x = A(i,j); + ++i; + } + ++j; + } + VERIFY_IS_APPROX(A,B); + B.setRandom(); + + i = 0; + for(auto r : A.rowwise()) { VERIFY_IS_APPROX(r.sum(), A.row(i).sum()); ++i; } + i = 0; + for(auto r : B.rowwise()) { VERIFY_IS_APPROX(r.sum(), B.row(i).sum()); ++i; } + } + + + // check rows/cols iterators with STL algorithms + { + RowVectorType row = RowVectorType::Random(cols); + A.rowwise() = row; + VERIFY( std::all_of(A.rowwise().begin(), A.rowwise().end(), [&row](typename ColMatrixType::RowXpr x) { return internal::isApprox(x.norm(),row.norm()); }) ); + + VectorType col = VectorType::Random(rows); + A.colwise() = col; + VERIFY( std::all_of(A.colwise().begin(), A.colwise().end(), [&col](typename ColMatrixType::ColXpr x) { return internal::isApprox(x.norm(),col.norm()); }) ); + + i = internal::random<Index>(0,A.rows()-1); + A.setRandom(); + A.row(i).setZero(); + VERIFY_IS_EQUAL( std::find_if(A.rowwise().begin(), A.rowwise().end(), [](typename ColMatrixType::RowXpr x) { return x.norm() == Scalar(0); })-A.rowwise().begin(), i ); + + j = internal::random<Index>(0,A.cols()-1); + A.setRandom(); + A.col(j).setZero(); + VERIFY_IS_EQUAL( std::find_if(A.colwise().begin(), A.colwise().end(), [](typename ColMatrixType::ColXpr x) { return x.norm() == Scalar(0); })-A.colwise().begin(), j ); + } + +#endif +} + + +#if EIGEN_HAS_CXX11 +// When the compiler sees expression IsContainerTest<C>(0), if C is an +// STL-style container class, the first overload of IsContainerTest +// will be viable (since both C::iterator* and C::const_iterator* are +// valid types and NULL can be implicitly converted to them). It will +// be picked over the second overload as 'int' is a perfect match for +// the type of argument 0. If C::iterator or C::const_iterator is not +// a valid type, the first overload is not viable, and the second +// overload will be picked. +template <class C, + class Iterator = decltype(::std::declval<const C&>().begin()), + class = decltype(::std::declval<const C&>().end()), + class = decltype(++::std::declval<Iterator&>()), + class = decltype(*::std::declval<Iterator>()), + class = typename C::const_iterator> +bool IsContainerType(int /* dummy */) { return true; } + +template <class C> +bool IsContainerType(long /* dummy */) { return false; } + +template <typename Scalar, int Rows, int Cols> +void test_stl_container_detection(int rows=Rows, int cols=Cols) +{ + typedef Matrix<Scalar,Rows,1> VectorType; + typedef Matrix<Scalar,Rows,Cols,ColMajor> ColMatrixType; + typedef Matrix<Scalar,Rows,Cols,RowMajor> RowMatrixType; + + ColMatrixType A = ColMatrixType::Random(rows, cols); + RowMatrixType B = RowMatrixType::Random(rows, cols); + + Index i = 1; + + using ColMatrixColType = decltype(A.col(i)); + using ColMatrixRowType = decltype(A.row(i)); + using RowMatrixColType = decltype(B.col(i)); + using RowMatrixRowType = decltype(B.row(i)); + + // Vector and matrix col/row are valid Stl-style container. + VERIFY_IS_EQUAL(IsContainerType<VectorType>(0), true); + VERIFY_IS_EQUAL(IsContainerType<ColMatrixColType>(0), true); + VERIFY_IS_EQUAL(IsContainerType<ColMatrixRowType>(0), true); + VERIFY_IS_EQUAL(IsContainerType<RowMatrixColType>(0), true); + VERIFY_IS_EQUAL(IsContainerType<RowMatrixRowType>(0), true); + + // But the matrix itself is not a valid Stl-style container. + VERIFY_IS_EQUAL(IsContainerType<ColMatrixType>(0), rows == 1 || cols == 1); + VERIFY_IS_EQUAL(IsContainerType<RowMatrixType>(0), rows == 1 || cols == 1); +} +#endif + +EIGEN_DECLARE_TEST(stl_iterators) +{ + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1(( test_stl_iterators<double,2,3>() )); + CALL_SUBTEST_1(( test_stl_iterators<float,7,5>() )); + CALL_SUBTEST_1(( test_stl_iterators<int,Dynamic,Dynamic>(internal::random<int>(5,10), internal::random<int>(5,10)) )); + CALL_SUBTEST_1(( test_stl_iterators<int,Dynamic,Dynamic>(internal::random<int>(10,200), internal::random<int>(10,200)) )); + } + +#if EIGEN_HAS_CXX11 + CALL_SUBTEST_1(( test_stl_container_detection<float,1,1>() )); + CALL_SUBTEST_1(( test_stl_container_detection<float,5,5>() )); +#endif +}
diff --git a/test/superlu_support.cpp b/test/superlu_support.cpp index 98a7bc5..55450c8 100644 --- a/test/superlu_support.cpp +++ b/test/superlu_support.cpp
@@ -12,7 +12,7 @@ #include <Eigen/SuperLUSupport> -void test_superlu_support() +EIGEN_DECLARE_TEST(superlu_support) { SuperLU<SparseMatrix<double> > superlu_double_colmajor; SuperLU<SparseMatrix<std::complex<double> > > superlu_cplxdouble_colmajor;
diff --git a/test/svd_common.h b/test/svd_common.h index d8611b5..5c0f2a0 100644 --- a/test/svd_common.h +++ b/test/svd_common.h
@@ -17,13 +17,13 @@ #endif #include "svd_fill.h" +#include "solverbase.h" // Check that the matrix m is properly reconstructed and that the U and V factors are unitary // The SVD must have already been computed. template<typename SvdType, typename MatrixType> void svd_check_full(const MatrixType& m, const SvdType& svd) { - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -42,9 +42,14 @@ MatrixUType u = svd.matrixU(); MatrixVType v = svd.matrixV(); RealScalar scaling = m.cwiseAbs().maxCoeff(); - if(scaling<=(std::numeric_limits<RealScalar>::min)()) - scaling = RealScalar(1); - VERIFY_IS_APPROX(m/scaling, u * (sigma/scaling) * v.adjoint()); + if(scaling<(std::numeric_limits<RealScalar>::min)()) + { + VERIFY(sigma.cwiseAbs().maxCoeff() <= (std::numeric_limits<RealScalar>::min)()); + } + else + { + VERIFY_IS_APPROX(m/scaling, u * (sigma/scaling) * v.adjoint()); + } VERIFY_IS_UNITARY(u); VERIFY_IS_UNITARY(v); } @@ -96,7 +101,6 @@ { typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -141,14 +145,14 @@ using std::abs; SolutionType y(x); - y.row(k) = (1.+2*NumTraits<RealScalar>::epsilon())*x.row(k); + y.row(k) = (RealScalar(1)+2*NumTraits<RealScalar>::epsilon())*x.row(k); RealScalar residual_y = (m*y-rhs).norm(); VERIFY( test_isMuchSmallerThan(abs(residual_y-residual), rhs_norm) || residual < residual_y ); if(internal::is_same<RealScalar,float>::value) ++g_test_level; VERIFY( test_isApprox(residual_y,residual) || residual < residual_y ); if(internal::is_same<RealScalar,float>::value) --g_test_level; - y.row(k) = (1.-2*NumTraits<RealScalar>::epsilon())*x.row(k); + y.row(k) = (RealScalar(1)-2*NumTraits<RealScalar>::epsilon())*x.row(k); residual_y = (m*y-rhs).norm(); VERIFY( test_isMuchSmallerThan(abs(residual_y-residual), rhs_norm) || residual < residual_y ); if(internal::is_same<RealScalar,float>::value) ++g_test_level; @@ -163,7 +167,6 @@ void svd_min_norm(const MatrixType& m, unsigned int computationOptions) { typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::Index Index; Index cols = m.cols(); enum { @@ -217,12 +220,33 @@ VERIFY_IS_APPROX(x21, x3); } +template<typename MatrixType, typename SolverType> +void svd_test_solvers(const MatrixType& m, const SolverType& solver) { + Index rows, cols, cols2; + + rows = m.rows(); + cols = m.cols(); + + if(MatrixType::ColsAtCompileTime==Dynamic) + { + cols2 = internal::random<int>(2,EIGEN_TEST_MAX_SIZE); + } + else + { + cols2 = cols; + } + typedef Matrix<typename MatrixType::Scalar, MatrixType::ColsAtCompileTime, MatrixType::ColsAtCompileTime> CMatrixType; + check_solverbase<CMatrixType, MatrixType>(m, solver, rows, cols, cols2); +} + // Check full, compare_to_full, least_square, and min_norm for all possible compute-options template<typename SvdType, typename MatrixType> void svd_test_all_computation_options(const MatrixType& m, bool full_only) { // if (QRPreconditioner == NoQRPreconditioner && m.rows() != m.cols()) // return; + STATIC_CHECK(( internal::is_same<typename SvdType::StorageIndex,int>::value )); + SvdType fullSvd(m, ComputeFullU|ComputeFullV); CALL_SUBTEST(( svd_check_full(m, fullSvd) )); CALL_SUBTEST(( svd_least_square<SvdType>(m, ComputeFullU | ComputeFullV) )); @@ -232,6 +256,9 @@ // remark #111: statement is unreachable #pragma warning disable 111 #endif + + svd_test_solvers(m, fullSvd); + if(full_only) return; @@ -256,7 +283,6 @@ CALL_SUBTEST(( svd_min_norm(m, ComputeThinU | ComputeThinV) )); // test reconstruction - typedef typename MatrixType::Index Index; Index diagSize = (std::min)(m.rows(), m.cols()); SvdType svd(m, ComputeThinU | ComputeThinV); VERIFY_IS_APPROX(m, svd.matrixU().leftCols(diagSize) * svd.singularValues().asDiagonal() * svd.matrixV().leftCols(diagSize).adjoint()); @@ -336,7 +362,7 @@ M << value_set(id(0)), value_set(id(1)), value_set(id(2)), value_set(id(3)); svd.compute(M,ComputeFullU|ComputeFullV); CALL_SUBTEST( svd_check_full(M,svd) ); - + id(k)++; if(id(k)>=value_set.size()) { @@ -344,7 +370,7 @@ id.head(k).setZero(); k=0; } - + } while((id<int(value_set.size())).all()); #if defined __INTEL_COMPILER @@ -432,7 +458,6 @@ void svd_verify_assert(const MatrixType& m) { typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::Index Index; Index rows = m.rows(); Index cols = m.cols(); @@ -448,6 +473,8 @@ VERIFY_RAISES_ASSERT(svd.singularValues()) VERIFY_RAISES_ASSERT(svd.matrixV()) VERIFY_RAISES_ASSERT(svd.solve(rhs)) + VERIFY_RAISES_ASSERT(svd.transpose().solve(rhs)) + VERIFY_RAISES_ASSERT(svd.adjoint().solve(rhs)) MatrixType a = MatrixType::Zero(rows, cols); a.setZero(); svd.compute(a, 0);
diff --git a/test/svd_fill.h b/test/svd_fill.h index 1bbe645..d68647e 100644 --- a/test/svd_fill.h +++ b/test/svd_fill.h
@@ -7,18 +7,28 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +template<typename T> +Array<T,4,1> four_denorms(); + +template<> +Array4f four_denorms() { return Array4f(5.60844e-39f, -5.60844e-39f, 4.94e-44f, -4.94e-44f); } +template<> +Array4d four_denorms() { return Array4d(5.60844e-313, -5.60844e-313, 4.94e-324, -4.94e-324); } +template<typename T> +Array<T,4,1> four_denorms() { return four_denorms<double>().cast<T>(); } + template<typename MatrixType> void svd_fill_random(MatrixType &m, int Option = 0) { + using std::pow; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; - typedef typename MatrixType::Index Index; Index diagSize = (std::min)(m.rows(), m.cols()); RealScalar s = std::numeric_limits<RealScalar>::max_exponent10/4; s = internal::random<RealScalar>(1,s); Matrix<RealScalar,Dynamic,1> d = Matrix<RealScalar,Dynamic,1>::Random(diagSize); for(Index k=0; k<diagSize; ++k) - d(k) = d(k)*std::pow(RealScalar(10),internal::random<RealScalar>(-s,s)); + d(k) = d(k)*pow(RealScalar(10),internal::random<RealScalar>(-s,s)); bool dup = internal::random<int>(0,10) < 3; bool unit_uv = internal::random<int>(0,10) < (dup?7:3); // if we duplicate some diagonal entries, then increase the chance to preserve them using unitary U and V factors @@ -53,8 +63,9 @@ VT.setRandom(); } - Matrix<Scalar,Dynamic,1> samples(7); - samples << 0, 5.60844e-313, -5.60844e-313, 4.94e-324, -4.94e-324, -1./NumTraits<RealScalar>::highest(), 1./NumTraits<RealScalar>::highest(); + Matrix<Scalar,Dynamic,1> samples(9); + samples << 0, four_denorms<RealScalar>(), + -RealScalar(1)/NumTraits<RealScalar>::highest(), RealScalar(1)/NumTraits<RealScalar>::highest(), (std::numeric_limits<RealScalar>::min)(), pow((std::numeric_limits<RealScalar>::min)(),0.8); if(Option==Symmetric) {
diff --git a/test/swap.cpp b/test/swap.cpp index f76e362..5b259d3 100644 --- a/test/swap.cpp +++ b/test/swap.cpp
@@ -28,8 +28,8 @@ typedef typename MatrixType::Scalar Scalar; eigen_assert((!internal::is_same<MatrixType,OtherMatrixType>::value)); - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); // construct 3 matrix guaranteed to be distinct MatrixType m1 = MatrixType::Random(rows,cols); @@ -83,7 +83,7 @@ } } -void test_swap() +EIGEN_DECLARE_TEST(swap) { int s = internal::random<int>(1,EIGEN_TEST_MAX_SIZE); CALL_SUBTEST_1( swap(Matrix3f()) ); // fixed size, no vectorization
diff --git a/test/symbolic_index.cpp b/test/symbolic_index.cpp new file mode 100644 index 0000000..b114cbb --- /dev/null +++ b/test/symbolic_index.cpp
@@ -0,0 +1,84 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifdef EIGEN_TEST_PART_2 +#define EIGEN_MAX_CPP_VER 03 + +// see indexed_view.cpp +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) + #pragma GCC diagnostic ignored "-Wdeprecated" +#endif + +#endif + +#include "main.h" + +template<typename T1,typename T2> +bool is_same_symb(const T1& a, const T2& b, Index size) +{ + return a.eval(last=size-1) == b.eval(last=size-1); +} + +template<typename T> +void check_is_symbolic(const T&) { + STATIC_CHECK(( symbolic::is_symbolic<T>::value )) +} + +template<typename T> +void check_isnot_symbolic(const T&) { + STATIC_CHECK(( !symbolic::is_symbolic<T>::value )) +} + +#define VERIFY_EQ_INT(A,B) VERIFY_IS_APPROX(int(A),int(B)) + +void check_symbolic_index() +{ + check_is_symbolic(last); + check_is_symbolic(lastp1); + check_is_symbolic(last+1); + check_is_symbolic(last-lastp1); + check_is_symbolic(2*last-lastp1/2); + check_isnot_symbolic(fix<3>()); + + Index size=100; + + // First, let's check FixedInt arithmetic: + VERIFY( is_same_type( (fix<5>()-fix<3>())*fix<9>()/(-fix<3>()), fix<-(5-3)*9/3>() ) ); + VERIFY( is_same_type( (fix<5>()-fix<3>())*fix<9>()/fix<2>(), fix<(5-3)*9/2>() ) ); + VERIFY( is_same_type( fix<9>()/fix<2>(), fix<9/2>() ) ); + VERIFY( is_same_type( fix<9>()%fix<2>(), fix<9%2>() ) ); + VERIFY( is_same_type( fix<9>()&fix<2>(), fix<9&2>() ) ); + VERIFY( is_same_type( fix<9>()|fix<2>(), fix<9|2>() ) ); + VERIFY( is_same_type( fix<9>()/2, int(9/2) ) ); + + VERIFY( is_same_symb( lastp1-1, last, size) ); + VERIFY( is_same_symb( lastp1-fix<1>, last, size) ); + + VERIFY_IS_EQUAL( ( (last*5-2)/3 ).eval(last=size-1), ((size-1)*5-2)/3 ); + VERIFY_IS_EQUAL( ( (last*fix<5>-fix<2>)/fix<3> ).eval(last=size-1), ((size-1)*5-2)/3 ); + VERIFY_IS_EQUAL( ( -last*lastp1 ).eval(last=size-1), -(size-1)*size ); + VERIFY_IS_EQUAL( ( lastp1-3*last ).eval(last=size-1), size- 3*(size-1) ); + VERIFY_IS_EQUAL( ( (lastp1-3*last)/lastp1 ).eval(last=size-1), (size- 3*(size-1))/size ); + +#if EIGEN_HAS_CXX14 + { + struct x_tag {}; static const symbolic::SymbolExpr<x_tag> x; + struct y_tag {}; static const symbolic::SymbolExpr<y_tag> y; + struct z_tag {}; static const symbolic::SymbolExpr<z_tag> z; + + VERIFY_IS_APPROX( int(((x+3)/y+z).eval(x=6,y=3,z=-13)), (6+3)/3+(-13) ); + } +#endif +} + +EIGEN_DECLARE_TEST(symbolic_index) +{ + CALL_SUBTEST_1( check_symbolic_index() ); + CALL_SUBTEST_2( check_symbolic_index() ); +}
diff --git a/test/triangular.cpp b/test/triangular.cpp index 936c2ae..0fca5e3 100644 --- a/test/triangular.cpp +++ b/test/triangular.cpp
@@ -19,8 +19,8 @@ RealScalar largerEps = 10*test_precision<RealScalar>(); - typename MatrixType::Index rows = m.rows(); - typename MatrixType::Index cols = m.cols(); + Index rows = m.rows(); + Index cols = m.cols(); MatrixType m1 = MatrixType::Random(rows, cols), m2 = MatrixType::Random(rows, cols), @@ -65,10 +65,10 @@ m1 = MatrixType::Random(rows, cols); for (int i=0; i<rows; ++i) - while (numext::abs2(m1(i,i))<1e-1) m1(i,i) = internal::random<Scalar>(); + while (numext::abs2(m1(i,i))<RealScalar(1e-1)) m1(i,i) = internal::random<Scalar>(); Transpose<MatrixType> trm4(m4); - // test back and forward subsitution with a vector as the rhs + // test back and forward substitution with a vector as the rhs m3 = m1.template triangularView<Upper>(); VERIFY(v2.isApprox(m3.adjoint() * (m1.adjoint().template triangularView<Lower>().solve(v2)), largerEps)); m3 = m1.template triangularView<Lower>(); @@ -78,7 +78,7 @@ m3 = m1.template triangularView<Lower>(); VERIFY(v2.isApprox(m3.conjugate() * (m1.conjugate().template triangularView<Lower>().solve(v2)), largerEps)); - // test back and forward subsitution with a matrix as the rhs + // test back and forward substitution with a matrix as the rhs m3 = m1.template triangularView<Upper>(); VERIFY(m2.isApprox(m3.adjoint() * (m1.adjoint().template triangularView<Lower>().solve(m2)), largerEps)); m3 = m1.template triangularView<Lower>(); @@ -121,12 +121,35 @@ VERIFY_IS_APPROX(m1.template triangularView<Upper>() * m5, m3*m5); VERIFY_IS_APPROX(m6*m1.template triangularView<Upper>(), m6*m3); + m1up = m1.template triangularView<Upper>(); + VERIFY_IS_APPROX(m1.template selfadjointView<Upper>().template triangularView<Upper>().toDenseMatrix(), m1up); + VERIFY_IS_APPROX(m1up.template selfadjointView<Upper>().template triangularView<Upper>().toDenseMatrix(), m1up); + VERIFY_IS_APPROX(m1.template selfadjointView<Upper>().template triangularView<Lower>().toDenseMatrix(), m1up.adjoint()); + VERIFY_IS_APPROX(m1up.template selfadjointView<Upper>().template triangularView<Lower>().toDenseMatrix(), m1up.adjoint()); + + VERIFY_IS_APPROX(m1.template selfadjointView<Upper>().diagonal(), m1.diagonal()); + + m3.setRandom(); + const MatrixType& m3c(m3); + VERIFY( is_same_type(m3c.template triangularView<Lower>(),m3.template triangularView<Lower>().template conjugateIf<false>()) ); + VERIFY( is_same_type(m3c.template triangularView<Lower>().conjugate(),m3.template triangularView<Lower>().template conjugateIf<true>()) ); + VERIFY_IS_APPROX(m3.template triangularView<Lower>().template conjugateIf<true>().toDenseMatrix(), + m3.conjugate().template triangularView<Lower>().toDenseMatrix()); + VERIFY_IS_APPROX(m3.template triangularView<Lower>().template conjugateIf<false>().toDenseMatrix(), + m3.template triangularView<Lower>().toDenseMatrix()); + + VERIFY( is_same_type(m3c.template selfadjointView<Lower>(),m3.template selfadjointView<Lower>().template conjugateIf<false>()) ); + VERIFY( is_same_type(m3c.template selfadjointView<Lower>().conjugate(),m3.template selfadjointView<Lower>().template conjugateIf<true>()) ); + VERIFY_IS_APPROX(m3.template selfadjointView<Lower>().template conjugateIf<true>().toDenseMatrix(), + m3.conjugate().template selfadjointView<Lower>().toDenseMatrix()); + VERIFY_IS_APPROX(m3.template selfadjointView<Lower>().template conjugateIf<false>().toDenseMatrix(), + m3.template selfadjointView<Lower>().toDenseMatrix()); + } template<typename MatrixType> void triangular_rect(const MatrixType& m) { - typedef const typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; enum { Rows = MatrixType::RowsAtCompileTime, Cols = MatrixType::ColsAtCompileTime }; @@ -213,7 +236,7 @@ EIGEN_UNUSED_VARIABLE(m) } -void test_triangular() +EIGEN_DECLARE_TEST(triangular) { int maxsize = (std::min)(EIGEN_TEST_MAX_SIZE,20); for(int i = 0; i < g_repeat ; i++)
diff --git a/test/umeyama.cpp b/test/umeyama.cpp index 2e80924..170c28a 100644 --- a/test/umeyama.cpp +++ b/test/umeyama.cpp
@@ -27,7 +27,7 @@ MatrixType Q; int max_tries = 40; - double is_unitary = false; + bool is_unitary = false; while (!is_unitary && max_tries > 0) { @@ -155,7 +155,7 @@ VERIFY(error < Scalar(16)*std::numeric_limits<Scalar>::epsilon()); } -void test_umeyama() +EIGEN_DECLARE_TEST(umeyama) { for (int i=0; i<g_repeat; ++i) {
diff --git a/test/umfpack_support.cpp b/test/umfpack_support.cpp index 37ab11f..d8f2a6f 100644 --- a/test/umfpack_support.cpp +++ b/test/umfpack_support.cpp
@@ -12,10 +12,10 @@ #include <Eigen/UmfPackSupport> -template<typename T> void test_umfpack_support_T() +template<typename T1, typename T2> void test_umfpack_support_T() { - UmfPackLU<SparseMatrix<T, ColMajor> > umfpack_colmajor; - UmfPackLU<SparseMatrix<T, RowMajor> > umfpack_rowmajor; + UmfPackLU<SparseMatrix<T1, ColMajor, T2> > umfpack_colmajor; + UmfPackLU<SparseMatrix<T1, RowMajor, T2> > umfpack_rowmajor; check_sparse_square_solving(umfpack_colmajor); check_sparse_square_solving(umfpack_rowmajor); @@ -24,9 +24,11 @@ check_sparse_square_determinant(umfpack_rowmajor); } -void test_umfpack_support() +EIGEN_DECLARE_TEST(umfpack_support) { - CALL_SUBTEST_1(test_umfpack_support_T<double>()); - CALL_SUBTEST_2(test_umfpack_support_T<std::complex<double> >()); + CALL_SUBTEST_1((test_umfpack_support_T<double, int>())); + CALL_SUBTEST_2((test_umfpack_support_T<std::complex<double>, int>())); + CALL_SUBTEST_3((test_umfpack_support_T<double, long >())); + CALL_SUBTEST_4((test_umfpack_support_T<std::complex<double>, long>())); }
diff --git a/test/unalignedassert.cpp b/test/unalignedassert.cpp index e2f03ff..120cc42 100644 --- a/test/unalignedassert.cpp +++ b/test/unalignedassert.cpp
@@ -94,7 +94,7 @@ void construct_at_boundary(int boundary) { char buf[sizeof(T)+256]; - size_t _buf = reinterpret_cast<size_t>(buf); + size_t _buf = reinterpret_cast<internal::UIntPtr>(buf); _buf += (EIGEN_MAX_ALIGN_BYTES - (_buf % EIGEN_MAX_ALIGN_BYTES)); // make 16/32/...-byte aligned _buf += boundary; // make exact boundary-aligned T *x = ::new(reinterpret_cast<void*>(_buf)) T; @@ -174,7 +174,7 @@ #endif } -void test_unalignedassert() +EIGEN_DECLARE_TEST(unalignedassert) { CALL_SUBTEST(unalignedassert()); }
diff --git a/test/unalignedcount.cpp b/test/unalignedcount.cpp index d6ffeaf..52cdd9e 100644 --- a/test/unalignedcount.cpp +++ b/test/unalignedcount.cpp
@@ -28,9 +28,16 @@ #include "main.h" -void test_unalignedcount() +EIGEN_DECLARE_TEST(unalignedcount) { - #if defined(EIGEN_VECTORIZE_AVX) + #if defined(EIGEN_VECTORIZE_AVX512) + VectorXf a(48), b(48); + VERIFY_ALIGNED_UNALIGNED_COUNT(a += b, 6, 0, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) += b.segment(0,48), 3, 3, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) -= b.segment(0,48), 3, 3, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) *= 3.5, 3, 0, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) /= 3.5, 3, 0, 3, 0); + #elif defined(EIGEN_VECTORIZE_AVX) VectorXf a(40), b(40); VERIFY_ALIGNED_UNALIGNED_COUNT(a += b, 10, 0, 5, 0); VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,40) += b.segment(0,40), 5, 5, 5, 0);
diff --git a/test/upperbidiagonalization.cpp b/test/upperbidiagonalization.cpp index 847b34b..945c999 100644 --- a/test/upperbidiagonalization.cpp +++ b/test/upperbidiagonalization.cpp
@@ -12,8 +12,8 @@ template<typename MatrixType> void upperbidiag(const MatrixType& m) { - const typename MatrixType::Index rows = m.rows(); - const typename MatrixType::Index cols = m.cols(); + const Index rows = m.rows(); + const Index cols = m.cols(); typedef Matrix<typename MatrixType::RealScalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime> RealMatrixType; typedef Matrix<typename MatrixType::Scalar, MatrixType::ColsAtCompileTime, MatrixType::RowsAtCompileTime> TransposeMatrixType; @@ -29,7 +29,7 @@ VERIFY_IS_APPROX(a.adjoint(),d); } -void test_upperbidiagonalization() +EIGEN_DECLARE_TEST(upperbidiagonalization) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( upperbidiag(MatrixXf(3,3)) );
diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index ee446c3..4bf3b3d 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp
@@ -7,6 +7,14 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#ifdef EIGEN_TEST_PART_1 +#define EIGEN_UNALIGNED_VECTORIZE 1 +#endif + +#ifdef EIGEN_TEST_PART_2 +#define EIGEN_UNALIGNED_VECTORIZE 0 +#endif + #ifdef EIGEN_DEFAULT_TO_ROW_MAJOR #undef EIGEN_DEFAULT_TO_ROW_MAJOR #endif @@ -14,6 +22,14 @@ #include "main.h" #include <typeinfo> +// Disable "ignoring attributes on template argument" +// for packet_traits<Packet*> +// => The only workaround would be to wrap _m128 and the likes +// within wrappers. +#if EIGEN_GNUC_AT_LEAST(6,0) + #pragma GCC diagnostic ignored "-Wignored-attributes" +#endif + using internal::demangle_flags; using internal::demangle_traversal; using internal::demangle_unrolling; @@ -21,7 +37,8 @@ template<typename Dst, typename Src> bool test_assign(const Dst&, const Src&, int traversal, int unrolling) { - typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar> > traits; + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src); + typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar,typename Src::Scalar> > traits; bool res = traits::Traversal==traversal; if(unrolling==InnerUnrolling+CompleteUnrolling) res = res && (int(traits::Unrolling)==InnerUnrolling || int(traits::Unrolling)==CompleteUnrolling); @@ -45,7 +62,8 @@ template<typename Dst, typename Src> bool test_assign(int traversal, int unrolling) { - typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar> > traits; + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src); + typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar,typename Src::Scalar> > traits; bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) { @@ -65,7 +83,8 @@ template<typename Xpr> bool test_redux(const Xpr&, int traversal, int unrolling) { - typedef internal::redux_traits<internal::scalar_sum_op<typename Xpr::Scalar>,internal::redux_evaluator<Xpr> > traits; + typedef typename Xpr::Scalar Scalar; + typedef internal::redux_traits<internal::scalar_sum_op<Scalar,Scalar>,internal::redux_evaluator<Xpr> > traits; bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) @@ -100,26 +119,26 @@ typedef Matrix<Scalar,Dynamic,1> VectorX; typedef Matrix<Scalar,Dynamic,Dynamic> MatrixXX; typedef Matrix<Scalar,PacketSize,PacketSize> Matrix11; - typedef Matrix<Scalar,2*PacketSize,2*PacketSize> Matrix22; + typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?8:2*PacketSize,(Matrix11::Flags&RowMajorBit)?2*PacketSize:8> Matrix22; typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16> Matrix44; typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16,DontAlign|EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION> Matrix44u; typedef Matrix<Scalar,4*PacketSize,4*PacketSize,ColMajor> Matrix44c; typedef Matrix<Scalar,4*PacketSize,4*PacketSize,RowMajor> Matrix44r; typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1) + (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1) > Matrix1; typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1), DontAlign|((Matrix1::Flags&RowMajorBit)?RowMajor:ColMajor)> Matrix1u; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3) + (PacketSize==16 ? 4 : PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 12 : PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3) > Matrix3; #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT @@ -144,10 +163,16 @@ InnerVectorizedTraversal,InnerUnrolling)); VERIFY(test_assign(Matrix44u(),Matrix44()+Matrix44(), - LinearTraversal,NoUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal, + EIGEN_UNALIGNED_VECTORIZE ? InnerUnrolling : NoUnrolling)); + + VERIFY(test_assign(Matrix1(),Matrix1()+Matrix1(), + (Matrix1::InnerSizeAtCompileTime % PacketSize)==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal, + CompleteUnrolling)); VERIFY(test_assign(Matrix1u(),Matrix1()+Matrix1(), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? ((Matrix1::InnerSizeAtCompileTime % PacketSize)==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal) + : LinearTraversal, CompleteUnrolling)); VERIFY(test_assign(Matrix44c().col(1),Matrix44c().col(2)+Matrix44c().col(3), InnerVectorizedTraversal,CompleteUnrolling)); @@ -158,19 +183,30 @@ if(PacketSize>1) { typedef Matrix<Scalar,3,3,ColMajor> Matrix33c; + typedef Matrix<Scalar,3,1,ColMajor> Vector3; VERIFY(test_assign(Matrix33c().row(2),Matrix33c().row(1)+Matrix33c().row(1), LinearTraversal,CompleteUnrolling)); + VERIFY(test_assign(Vector3(),Vector3()+Vector3(), + sizeof(Scalar)==16 ? InnerVectorizedTraversal : (EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal), CompleteUnrolling)); VERIFY(test_assign(Matrix33c().col(0),Matrix33c().col(1)+Matrix33c().col(1), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? (sizeof(Scalar)==16 ? InnerVectorizedTraversal : LinearVectorizedTraversal) + : (sizeof(Scalar)==16 ? SliceVectorizedTraversal : LinearTraversal), + ((!EIGEN_UNALIGNED_VECTORIZE) && (sizeof(Scalar)==16)) ? NoUnrolling : CompleteUnrolling)); VERIFY(test_assign(Matrix3(),Matrix3().cwiseProduct(Matrix3()), LinearVectorizedTraversal,CompleteUnrolling)); VERIFY(test_assign(Matrix<Scalar,17,17>(),Matrix<Scalar,17,17>()+Matrix<Scalar,17,17>(), - HalfPacketSize==1 ? InnerVectorizedTraversal : LinearTraversal,NoUnrolling)); + sizeof(Scalar)==16 ? InnerVectorizedTraversal : + EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : + LinearTraversal, + NoUnrolling)); - VERIFY(test_assign(Matrix11(),Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(2,3)+Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(8,4), - DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling)); + VERIFY(test_assign(Matrix11(), Matrix11()+Matrix11(),InnerVectorizedTraversal,CompleteUnrolling)); + + + VERIFY(test_assign(Matrix11(),Matrix<Scalar,21,21>().template block<PacketSize,PacketSize>(2,3)+Matrix<Scalar,21,21>().template block<PacketSize,PacketSize>(3,2), + (EIGEN_UNALIGNED_VECTORIZE) ? InnerVectorizedTraversal : DefaultTraversal, CompleteUnrolling|InnerUnrolling)); VERIFY(test_assign(Vector1(),Matrix11()*Vector1(), InnerVectorizedTraversal,CompleteUnrolling)); @@ -182,6 +218,12 @@ VERIFY(test_redux(Vector1(), LinearVectorizedTraversal,CompleteUnrolling)); + VERIFY(test_redux(Vector1().array()*Vector1().array(), + LinearVectorizedTraversal,CompleteUnrolling)); + + VERIFY(test_redux((Vector1().array()*Vector1().array()).col(0), + LinearVectorizedTraversal,CompleteUnrolling)); + VERIFY(test_redux(Matrix<Scalar,PacketSize,3>(), LinearVectorizedTraversal,CompleteUnrolling)); @@ -191,8 +233,13 @@ VERIFY(test_redux(Matrix44(), LinearVectorizedTraversal,NoUnrolling)); - VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?4:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:4>(1,2), - DefaultTraversal,CompleteUnrolling)); + if(PacketSize>1) { + VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?4:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:4>(1,2), + SliceVectorizedTraversal,CompleteUnrolling)); + + VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?2:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:2>(1,2), + DefaultTraversal,CompleteUnrolling)); + } VERIFY(test_redux(Matrix44c().template block<2*PacketSize,1>(1,2), LinearVectorizedTraversal,CompleteUnrolling)); @@ -208,7 +255,7 @@ VERIFY((test_assign< Map<Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>, AlignedMax, InnerStride<3*PacketSize> >, Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)> - >(DefaultTraversal,CompleteUnrolling))); + >(DefaultTraversal,PacketSize>=8?InnerUnrolling:CompleteUnrolling))); VERIFY((test_assign(Matrix11(), Matrix<Scalar,PacketSize,EIGEN_PLAIN_ENUM_MIN(2,PacketSize)>()*Matrix<Scalar,EIGEN_PLAIN_ENUM_MIN(2,PacketSize),PacketSize>(), InnerVectorizedTraversal, CompleteUnrolling))); @@ -244,25 +291,21 @@ typedef Matrix<Scalar,5*PacketSize,7,ColMajor> Matrix57; typedef Matrix<Scalar,3*PacketSize,5,ColMajor> Matrix35; typedef Matrix<Scalar,5*PacketSize,7,DontAlign|ColMajor> Matrix57u; -// typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16> Matrix44; -// typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16,DontAlign|EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION> Matrix44u; -// typedef Matrix<Scalar,4*PacketSize,4*PacketSize,ColMajor> Matrix44c; -// typedef Matrix<Scalar,4*PacketSize,4*PacketSize,RowMajor> Matrix44r; typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1) + (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1) > Matrix1; typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1), DontAlign|((Matrix1::Flags&RowMajorBit)?RowMajor:ColMajor)> Matrix1u; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix<Scalar, - (PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1), - (PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3) + (PacketSize==16 ? 4 : PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1), + (PacketSize==16 ? 12 : PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3) > Matrix3; #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT @@ -270,6 +313,12 @@ InnerVectorizedTraversal,CompleteUnrolling)); VERIFY(test_assign(Vector1(),Vector1()+Vector1(), InnerVectorizedTraversal,CompleteUnrolling)); + VERIFY(test_assign(Vector1(),Vector1().template segment<PacketSize>(0).derived(), + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal,CompleteUnrolling)); + VERIFY(test_assign(Vector1(),Scalar(2.1)*Vector1()-Vector1(), + InnerVectorizedTraversal,CompleteUnrolling)); + VERIFY(test_assign(Vector1(),(Scalar(2.1)*Vector1().template segment<PacketSize>(0)-Vector1().template segment<PacketSize>(0)).derived(), + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal,CompleteUnrolling)); VERIFY(test_assign(Vector1(),Vector1().cwiseProduct(Vector1()), InnerVectorizedTraversal,CompleteUnrolling)); VERIFY(test_assign(Vector1(),Vector1().template cast<Scalar>(), @@ -287,10 +336,11 @@ InnerVectorizedTraversal,InnerUnrolling)); VERIFY(test_assign(Matrix57u(),Matrix57()+Matrix57(), - LinearTraversal,NoUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal, + EIGEN_UNALIGNED_VECTORIZE ? InnerUnrolling : NoUnrolling)); VERIFY(test_assign(Matrix1u(),Matrix1()+Matrix1(), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? ((Matrix1::InnerSizeAtCompileTime % PacketSize)==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal) : LinearTraversal,CompleteUnrolling)); if(PacketSize>1) { @@ -298,16 +348,20 @@ VERIFY(test_assign(Matrix33c().row(2),Matrix33c().row(1)+Matrix33c().row(1), LinearTraversal,CompleteUnrolling)); VERIFY(test_assign(Matrix33c().col(0),Matrix33c().col(1)+Matrix33c().col(1), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? (sizeof(Scalar)==16 ? InnerVectorizedTraversal : LinearVectorizedTraversal) + : (sizeof(Scalar)==16 ? SliceVectorizedTraversal : LinearTraversal), + ((!EIGEN_UNALIGNED_VECTORIZE) && (sizeof(Scalar)==16)) ? NoUnrolling : CompleteUnrolling)); VERIFY(test_assign(Matrix3(),Matrix3().cwiseQuotient(Matrix3()), PacketTraits::HasDiv ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling)); VERIFY(test_assign(Matrix<Scalar,17,17>(),Matrix<Scalar,17,17>()+Matrix<Scalar,17,17>(), - LinearTraversal,NoUnrolling)); + sizeof(Scalar)==16 ? InnerVectorizedTraversal : (EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal), + NoUnrolling)); VERIFY(test_assign(Matrix11(),Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(2,3)+Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(8,4), - DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : DefaultTraversal,InnerUnrolling+CompleteUnrolling)); + VERIFY(test_assign(Vector1(),Matrix11()*Vector1(), InnerVectorizedTraversal,CompleteUnrolling)); @@ -328,16 +382,21 @@ VERIFY(test_redux(Matrix35(), LinearVectorizedTraversal,CompleteUnrolling)); - VERIFY(test_redux(Matrix57().template block<PacketSize,3>(1,0), - DefaultTraversal,CompleteUnrolling)); + VERIFY(test_redux(Matrix57().template block<PacketSize==1?2:PacketSize,3>(1,0), + SliceVectorizedTraversal,CompleteUnrolling)); + + if(PacketSize>1) { + VERIFY(test_redux(Matrix57().template block<PacketSize,2>(1,0), + DefaultTraversal,CompleteUnrolling)); + } VERIFY((test_assign< Map<Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>, AlignedMax, InnerStride<3*PacketSize> >, Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)> - >(DefaultTraversal,CompleteUnrolling))); + >(DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling))); VERIFY((test_assign(Matrix57(), Matrix<Scalar,5*PacketSize,3>()*Matrix<Scalar,3,7>(), - InnerVectorizedTraversal, CompleteUnrolling))); + InnerVectorizedTraversal, InnerUnrolling+CompleteUnrolling))); #endif } }; @@ -347,7 +406,7 @@ static void run() {} }; -void test_vectorization_logic() +EIGEN_DECLARE_TEST(vectorization_logic) { #ifdef EIGEN_VECTORIZE @@ -367,19 +426,19 @@ if(internal::packet_traits<float>::Vectorizable) { VERIFY(test_assign(Matrix<float,3,3>(),Matrix<float,3,3>()+Matrix<float,3,3>(), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling)); VERIFY(test_redux(Matrix<float,5,2>(), - DefaultTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : DefaultTraversal,CompleteUnrolling)); } if(internal::packet_traits<double>::Vectorizable) { VERIFY(test_assign(Matrix<double,3,3>(),Matrix<double,3,3>()+Matrix<double,3,3>(), - LinearTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling)); VERIFY(test_redux(Matrix<double,7,3>(), - DefaultTraversal,CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : DefaultTraversal,CompleteUnrolling)); } #endif // EIGEN_VECTORIZE
diff --git a/test/vectorwiseop.cpp b/test/vectorwiseop.cpp index 3cc1987..8ee5884 100644 --- a/test/vectorwiseop.cpp +++ b/test/vectorwiseop.cpp
@@ -15,7 +15,6 @@ template<typename ArrayType> void vectorwiseop_array(const ArrayType& m) { - typedef typename ArrayType::Index Index; typedef typename ArrayType::Scalar Scalar; typedef Array<Scalar, ArrayType::RowsAtCompileTime, 1> ColVectorType; typedef Array<Scalar, 1, ArrayType::ColsAtCompileTime> RowVectorType; @@ -129,13 +128,13 @@ template<typename MatrixType> void vectorwiseop_matrix(const MatrixType& m) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> ColVectorType; typedef Matrix<Scalar, 1, MatrixType::ColsAtCompileTime> RowVectorType; typedef Matrix<RealScalar, MatrixType::RowsAtCompileTime, 1> RealColVectorType; typedef Matrix<RealScalar, 1, MatrixType::ColsAtCompileTime> RealRowVectorType; + typedef Matrix<Scalar,Dynamic,Dynamic> MatrixX; Index rows = m.rows(); Index cols = m.cols(); @@ -151,6 +150,19 @@ RealColVectorType rcres; RealRowVectorType rrres; + // test broadcast assignment + m2 = m1; + m2.colwise() = colvec; + for(Index j=0; j<cols; ++j) + VERIFY_IS_APPROX(m2.col(j), colvec); + m2.rowwise() = rowvec; + for(Index i=0; i<rows; ++i) + VERIFY_IS_APPROX(m2.row(i), rowvec); + if(rows>1) + VERIFY_RAISES_ASSERT(m2.colwise() = colvec.transpose()); + if(cols>1) + VERIFY_RAISES_ASSERT(m2.rowwise() = rowvec.transpose()); + // test addition m2 = m1; @@ -199,11 +211,23 @@ VERIFY_RAISES_ASSERT(m1.rowwise() - rowvec.transpose()); } - // test norm - rrres = m1.colwise().norm(); - VERIFY_IS_APPROX(rrres(c), m1.col(c).norm()); - rcres = m1.rowwise().norm(); - VERIFY_IS_APPROX(rcres(r), m1.row(r).norm()); + // ------ partial reductions ------ + + #define TEST_PARTIAL_REDUX_BASIC(FUNC,ROW,COL,PREPROCESS) { \ + ROW = m1 PREPROCESS .colwise().FUNC ; \ + for(Index k=0; k<cols; ++k) VERIFY_IS_APPROX(ROW(k), m1.col(k) PREPROCESS .FUNC ); \ + COL = m1 PREPROCESS .rowwise().FUNC ; \ + for(Index k=0; k<rows; ++k) VERIFY_IS_APPROX(COL(k), m1.row(k) PREPROCESS .FUNC ); \ + } + + TEST_PARTIAL_REDUX_BASIC(sum(), rowvec,colvec,EIGEN_EMPTY); + TEST_PARTIAL_REDUX_BASIC(prod(), rowvec,colvec,EIGEN_EMPTY); + TEST_PARTIAL_REDUX_BASIC(mean(), rowvec,colvec,EIGEN_EMPTY); + TEST_PARTIAL_REDUX_BASIC(minCoeff(), rrres, rcres, .real()); + TEST_PARTIAL_REDUX_BASIC(maxCoeff(), rrres, rcres, .real()); + TEST_PARTIAL_REDUX_BASIC(norm(), rrres, rcres, EIGEN_EMPTY); + TEST_PARTIAL_REDUX_BASIC(squaredNorm(),rrres, rcres, EIGEN_EMPTY); + TEST_PARTIAL_REDUX_BASIC(redux(internal::scalar_sum_op<Scalar,Scalar>()),rowvec,colvec,EIGEN_EMPTY); VERIFY_IS_APPROX(m1.cwiseAbs().colwise().sum(), m1.colwise().template lpNorm<1>()); VERIFY_IS_APPROX(m1.cwiseAbs().rowwise().sum(), m1.rowwise().template lpNorm<1>()); @@ -231,20 +255,42 @@ Matrix<Scalar,MatrixType::RowsAtCompileTime,MatrixType::RowsAtCompileTime> m1m1 = m1 * m1.transpose(); VERIFY_IS_APPROX( (m1 * m1.transpose()).colwise().sum(), m1m1.colwise().sum()); Matrix<Scalar,1,MatrixType::RowsAtCompileTime> tmp(rows); - VERIFY_EVALUATION_COUNT( tmp = (m1 * m1.transpose()).colwise().sum(), (MatrixType::RowsAtCompileTime==Dynamic ? 1 : 0)); + VERIFY_EVALUATION_COUNT( tmp = (m1 * m1.transpose()).colwise().sum(), 1); - m2 = m1.rowwise() - (m1.colwise().sum()/m1.rows()).eval(); - m1 = m1.rowwise() - (m1.colwise().sum()/m1.rows()); + m2 = m1.rowwise() - (m1.colwise().sum()/RealScalar(m1.rows())).eval(); + m1 = m1.rowwise() - (m1.colwise().sum()/RealScalar(m1.rows())); VERIFY_IS_APPROX( m1, m2 ); - VERIFY_EVALUATION_COUNT( m2 = (m1.rowwise() - m1.colwise().sum()/m1.rows()), (MatrixType::RowsAtCompileTime==Dynamic && MatrixType::ColsAtCompileTime!=1 ? 1 : 0) ); + VERIFY_EVALUATION_COUNT( m2 = (m1.rowwise() - m1.colwise().sum()/RealScalar(m1.rows())), (MatrixType::RowsAtCompileTime!=1 ? 1 : 0) ); + + // test empty expressions + VERIFY_IS_APPROX(m1.matrix().middleCols(0,0).rowwise().sum().eval(), MatrixX::Zero(rows,1)); + VERIFY_IS_APPROX(m1.matrix().middleRows(0,0).colwise().sum().eval(), MatrixX::Zero(1,cols)); + VERIFY_IS_APPROX(m1.matrix().middleCols(0,fix<0>).rowwise().sum().eval(), MatrixX::Zero(rows,1)); + VERIFY_IS_APPROX(m1.matrix().middleRows(0,fix<0>).colwise().sum().eval(), MatrixX::Zero(1,cols)); + + VERIFY_IS_APPROX(m1.matrix().middleCols(0,0).rowwise().prod().eval(), MatrixX::Ones(rows,1)); + VERIFY_IS_APPROX(m1.matrix().middleRows(0,0).colwise().prod().eval(), MatrixX::Ones(1,cols)); + VERIFY_IS_APPROX(m1.matrix().middleCols(0,fix<0>).rowwise().prod().eval(), MatrixX::Ones(rows,1)); + VERIFY_IS_APPROX(m1.matrix().middleRows(0,fix<0>).colwise().prod().eval(), MatrixX::Ones(1,cols)); + + VERIFY_IS_APPROX(m1.matrix().middleCols(0,0).rowwise().squaredNorm().eval(), MatrixX::Zero(rows,1)); + + VERIFY_RAISES_ASSERT(m1.real().middleCols(0,0).rowwise().minCoeff().eval()); + VERIFY_RAISES_ASSERT(m1.real().middleRows(0,0).colwise().maxCoeff().eval()); + VERIFY_IS_EQUAL(m1.real().middleRows(0,0).rowwise().maxCoeff().eval().rows(),0); + VERIFY_IS_EQUAL(m1.real().middleCols(0,0).colwise().maxCoeff().eval().cols(),0); + VERIFY_IS_EQUAL(m1.real().middleRows(0,fix<0>).rowwise().maxCoeff().eval().rows(),0); + VERIFY_IS_EQUAL(m1.real().middleCols(0,fix<0>).colwise().maxCoeff().eval().cols(),0); } -void test_vectorwiseop() +EIGEN_DECLARE_TEST(vectorwiseop) { CALL_SUBTEST_1( vectorwiseop_array(Array22cd()) ); CALL_SUBTEST_2( vectorwiseop_array(Array<double, 3, 2>()) ); CALL_SUBTEST_3( vectorwiseop_array(ArrayXXf(3, 4)) ); CALL_SUBTEST_4( vectorwiseop_matrix(Matrix4cf()) ); + CALL_SUBTEST_5( vectorwiseop_matrix(Matrix4f()) ); + CALL_SUBTEST_5( vectorwiseop_matrix(Vector4f()) ); CALL_SUBTEST_5( vectorwiseop_matrix(Matrix<float,4,5>()) ); CALL_SUBTEST_6( vectorwiseop_matrix(MatrixXd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) ); CALL_SUBTEST_7( vectorwiseop_matrix(VectorXd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) );
diff --git a/test/visitor.cpp b/test/visitor.cpp index 844170e..de938fc 100644 --- a/test/visitor.cpp +++ b/test/visitor.cpp
@@ -12,7 +12,6 @@ template<typename MatrixType> void matrixVisitor(const MatrixType& p) { typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::Index Index; Index rows = p.rows(); Index cols = p.cols(); @@ -65,7 +64,6 @@ template<typename VectorType> void vectorVisitor(const VectorType& w) { typedef typename VectorType::Scalar Scalar; - typedef typename VectorType::Index Index; Index size = w.size(); @@ -115,7 +113,7 @@ VERIFY(eigen_maxidx == (std::min)(idx0,idx2)); } -void test_visitor() +EIGEN_DECLARE_TEST(visitor) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( matrixVisitor(Matrix<float, 1, 1>()) );
diff --git a/test/zerosized.cpp b/test/zerosized.cpp index 477ff00..07afd0f 100644 --- a/test/zerosized.cpp +++ b/test/zerosized.cpp
@@ -16,9 +16,18 @@ VERIFY(!m.any()); VERIFY(m.prod()==1); VERIFY(m.sum()==0); + VERIFY(m.norm()==0); + VERIFY(m.squaredNorm()==0); VERIFY(m.count()==0); VERIFY(m.allFinite()); VERIFY(!m.hasNaN()); + VERIFY_RAISES_ASSERT( m.minCoeff() ); + VERIFY_RAISES_ASSERT( m.maxCoeff() ); + Index i,j; + VERIFY_RAISES_ASSERT( m.minCoeff(&i,&j) ); + VERIFY_RAISES_ASSERT( m.maxCoeff(&i,&j) ); + VERIFY_RAISES_ASSERT( m.reshaped().minCoeff(&i) ); + VERIFY_RAISES_ASSERT( m.reshaped().maxCoeff(&i) ); } @@ -81,7 +90,7 @@ } } -void test_zerosized() +EIGEN_DECLARE_TEST(zerosized) { zeroSizedMatrix<Matrix2d>(); zeroSizedMatrix<Matrix3i>();
diff --git a/unsupported/CMakeLists.txt b/unsupported/CMakeLists.txt index 4fef40a..9a56661 100644 --- a/unsupported/CMakeLists.txt +++ b/unsupported/CMakeLists.txt
@@ -1,7 +1,9 @@ add_subdirectory(Eigen) add_subdirectory(doc EXCLUDE_FROM_ALL) -if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) - add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest -else() - add_subdirectory(test EXCLUDE_FROM_ALL) +if(BUILD_TESTING) + if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) + add_subdirectory(test) # can't do EXCLUDE_FROM_ALL here, breaks CTest + else() + add_subdirectory(test EXCLUDE_FROM_ALL) + endif() endif()
diff --git a/unsupported/Eigen/AdolcForward b/unsupported/Eigen/AdolcForward index 15f5f07..9b8d3cd 100644 --- a/unsupported/Eigen/AdolcForward +++ b/unsupported/Eigen/AdolcForward
@@ -40,7 +40,7 @@ # undef realloc #endif -#include <Eigen/Core> +#include "../../Eigen/Core" namespace Eigen {
diff --git a/unsupported/Eigen/AlignedVector3 b/unsupported/Eigen/AlignedVector3 index 135eec5..a67ee6e 100644 --- a/unsupported/Eigen/AlignedVector3 +++ b/unsupported/Eigen/AlignedVector3
@@ -10,7 +10,9 @@ #ifndef EIGEN_ALIGNED_VECTOR3 #define EIGEN_ALIGNED_VECTOR3 -#include <Eigen/Geometry> +#include "../../Eigen/Geometry" + +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" namespace Eigen { @@ -61,7 +63,7 @@ Scalar* data() { return m_coeffs.data(); } const Scalar* data() const { return m_coeffs.data(); } Index innerStride() const { return 1; } - Index outerStride() const { return m_coeffs.outerStride(); } + Index outerStride() const { return 3; } inline const Scalar& coeff(Index row, Index col) const { return m_coeffs.coeff(row, col); } @@ -221,4 +223,6 @@ } +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + #endif // EIGEN_ALIGNED_VECTOR3
diff --git a/unsupported/Eigen/ArpackSupport b/unsupported/Eigen/ArpackSupport index 37a2799..28c95ff 100644 --- a/unsupported/Eigen/ArpackSupport +++ b/unsupported/Eigen/ArpackSupport
@@ -9,9 +9,7 @@ #ifndef EIGEN_ARPACKSUPPORT_MODULE_H #define EIGEN_ARPACKSUPPORT_MODULE_H -#include <Eigen/Core> - -#include <Eigen/src/Core/util/DisableStupidWarnings.h> +#include "../../Eigen/Core" /** \defgroup ArpackSupport_Module Arpack support module * @@ -22,10 +20,12 @@ * \endcode */ -#include <Eigen/SparseCholesky> +#include "../../Eigen/SparseCholesky" + +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" #include "src/Eigenvalues/ArpackSelfAdjointEigenSolver.h" -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_ARPACKSUPPORT_MODULE_H /* vim: set filetype=cpp et sw=2 ts=2 ai: */
diff --git a/unsupported/Eigen/AutoDiff b/unsupported/Eigen/AutoDiff index abf5b7d..7a4ff46 100644 --- a/unsupported/Eigen/AutoDiff +++ b/unsupported/Eigen/AutoDiff
@@ -28,11 +28,17 @@ //@{ } +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #include "src/AutoDiff/AutoDiffScalar.h" // #include "src/AutoDiff/AutoDiffVector.h" #include "src/AutoDiff/AutoDiffJacobian.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + + + namespace Eigen { //@} }
diff --git a/unsupported/Eigen/BVH b/unsupported/Eigen/BVH index 0161a54..666c983 100644 --- a/unsupported/Eigen/BVH +++ b/unsupported/Eigen/BVH
@@ -10,9 +10,9 @@ #ifndef EIGEN_BVH_MODULE_H #define EIGEN_BVH_MODULE_H -#include <Eigen/Core> -#include <Eigen/Geometry> -#include <Eigen/StdVector> +#include "../../Eigen/Core" +#include "../../Eigen/Geometry" +#include "../../Eigen/StdVector" #include <algorithm> #include <queue>
diff --git a/unsupported/Eigen/CMakeLists.txt b/unsupported/Eigen/CMakeLists.txt index 6d0cf4f..631a060 100644 --- a/unsupported/Eigen/CMakeLists.txt +++ b/unsupported/Eigen/CMakeLists.txt
@@ -4,6 +4,7 @@ ArpackSupport AutoDiff BVH + EulerAngles FFT IterativeSolvers KroneckerProduct @@ -17,6 +18,7 @@ Polynomials Skyline SparseExtra + SpecialFunctions Splines ) @@ -25,5 +27,6 @@ DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen COMPONENT Devel ) -add_subdirectory(src) -add_subdirectory(CXX11) \ No newline at end of file +install(DIRECTORY src DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen COMPONENT Devel FILES_MATCHING PATTERN "*.h") + +add_subdirectory(CXX11)
diff --git a/unsupported/Eigen/CXX11/CMakeLists.txt b/unsupported/Eigen/CXX11/CMakeLists.txt index f1d9f04..385ed24 100644 --- a/unsupported/Eigen/CXX11/CMakeLists.txt +++ b/unsupported/Eigen/CXX11/CMakeLists.txt
@@ -1,8 +1,8 @@ -set(Eigen_CXX11_HEADERS Core Tensor TensorSymmetry) +set(Eigen_CXX11_HEADERS Tensor TensorSymmetry ThreadPool) install(FILES ${Eigen_CXX11_HEADERS} DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11 COMPONENT Devel ) -add_subdirectory(src) +install(DIRECTORY src DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11 COMPONENT Devel FILES_MATCHING PATTERN "*.h")
diff --git a/unsupported/Eigen/CXX11/Core b/unsupported/Eigen/CXX11/Core deleted file mode 100644 index 946145f..0000000 --- a/unsupported/Eigen/CXX11/Core +++ /dev/null
@@ -1,51 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2013 Christian Seiler <christian@iwakd.de> -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_CXX11_CORE_MODULE -#define EIGEN_CXX11_CORE_MODULE - -#include <Eigen/Core> - -#include <Eigen/src/Core/util/DisableStupidWarnings.h> - -/** \defgroup CXX11_Core_Module C++11 Core Module - * - * This module provides common core features for all modules that - * explicitly depend on C++11. Currently, this is only the Tensor - * module. Note that at this stage, you should not need to include - * this module directly. - * - * It also provides a limited fallback for compilers that don't support - * CXX11 yet, such as nvcc. - * - * \code - * #include <Eigen/CXX11/Core> - * \endcode - */ - -#include <vector> - -#include "src/Core/util/EmulateArray.h" -#include "src/Core/util/MaxSizeVector.h" - -// Emulate the cxx11 functionality that we need if the compiler doesn't support it. -// Visual studio 2015 doesn't advertise itself as cxx11 compliant, although it -// supports enough of the standard for our needs -#if __cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900 -#include "src/Core/util/CXX11Workarounds.h" -#include "src/Core/util/CXX11Meta.h" -#else -#include "src/Core/util/EmulateCXX11Meta.h" -#endif - -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> - -#endif // EIGEN_CXX11_CORE_MODULE -
diff --git a/unsupported/Eigen/CXX11/Tensor b/unsupported/Eigen/CXX11/Tensor index c4bcd29..25b6630 100644 --- a/unsupported/Eigen/CXX11/Tensor +++ b/unsupported/Eigen/CXX11/Tensor
@@ -11,10 +11,26 @@ //#ifndef EIGEN_CXX11_TENSOR_MODULE //#define EIGEN_CXX11_TENSOR_MODULE -#include "Core" +#include "../../../Eigen/Core" -#include <Eigen/src/Core/util/DisableStupidWarnings.h> +#if defined(EIGEN_USE_SYCL) +#undef min +#undef max +#undef isnan +#undef isinf +#undef isfinite +#include <CL/sycl.hpp> +#include <iostream> +#include <map> +#include <memory> +#include <utility> +#endif +#include "../SpecialFunctions" + +#include "../../../Eigen/src/Core/util/DisableStupidWarnings.h" +#include "src/util/CXX11Meta.h" +#include "src/util/MaxSizeVector.h" /** \defgroup CXX11_Tensor_Module Tensor Module * @@ -24,6 +40,8 @@ * \code * #include <Eigen/CXX11/Tensor> * \endcode + * + * Much of the documentation can be found \ref eigen_tensors "here". */ #include <cmath> @@ -31,12 +49,16 @@ #include <cstring> #ifdef _WIN32 +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; typedef __int32 int32_t; typedef unsigned __int32 uint32_t; typedef __int64 int64_t; typedef unsigned __int64 uint64_t; +#include <windows.h> #else #include <stdint.h> +#include <unistd.h> #endif #if __cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900 @@ -51,48 +73,63 @@ #include <time.h> #endif +#if defined(EIGEN_USE_LIBXSMM) +#include "libxsmm.h" +#endif + #ifdef EIGEN_USE_THREADS #include "ThreadPool" #endif #ifdef EIGEN_USE_GPU -#include <iostream> -#include <cuda_runtime.h> -#if defined(__CUDACC__) -#include <curand_kernel.h> + #include <iostream> + #if defined(EIGEN_USE_HIP) + #include <hip/hip_runtime.h> + #else + #include <cuda_runtime.h> + #endif + #if __cplusplus >= 201103L + #include <atomic> + #include <unistd.h> + #endif #endif -#endif - #include "src/Tensor/TensorMacros.h" #include "src/Tensor/TensorForwardDeclarations.h" #include "src/Tensor/TensorMeta.h" +#include "src/Tensor/TensorFunctors.h" +#include "src/Tensor/TensorCostModel.h" #include "src/Tensor/TensorDeviceDefault.h" #include "src/Tensor/TensorDeviceThreadPool.h" -#include "src/Tensor/TensorDeviceCuda.h" +#include "src/Tensor/TensorDeviceGpu.h" +#ifndef gpu_assert +#define gpu_assert(x) +#endif +#include "src/Tensor/TensorDeviceSycl.h" #include "src/Tensor/TensorIndexList.h" #include "src/Tensor/TensorDimensionList.h" #include "src/Tensor/TensorDimensions.h" #include "src/Tensor/TensorInitializer.h" #include "src/Tensor/TensorTraits.h" -#include "src/Tensor/TensorFunctors.h" +#include "src/Tensor/TensorRandom.h" #include "src/Tensor/TensorUInt128.h" #include "src/Tensor/TensorIntDiv.h" +#include "src/Tensor/TensorGlobalFunctions.h" #include "src/Tensor/TensorBase.h" +#include "src/Tensor/TensorBlock.h" -#include "src/Tensor/TensorCostModel.h" #include "src/Tensor/TensorEvaluator.h" #include "src/Tensor/TensorExpr.h" #include "src/Tensor/TensorReduction.h" -#include "src/Tensor/TensorReductionCuda.h" +#include "src/Tensor/TensorReductionGpu.h" #include "src/Tensor/TensorArgMax.h" #include "src/Tensor/TensorConcatenation.h" #include "src/Tensor/TensorContractionMapper.h" #include "src/Tensor/TensorContractionBlocking.h" #include "src/Tensor/TensorContraction.h" #include "src/Tensor/TensorContractionThreadPool.h" -#include "src/Tensor/TensorContractionCuda.h" +#include "src/Tensor/TensorContractionGpu.h" #include "src/Tensor/TensorConversion.h" #include "src/Tensor/TensorConvolution.h" #include "src/Tensor/TensorFFT.h" @@ -113,7 +150,10 @@ #include "src/Tensor/TensorForcedEval.h" #include "src/Tensor/TensorGenerator.h" #include "src/Tensor/TensorAssign.h" +#include "src/Tensor/TensorScan.h" +#include "src/Tensor/TensorTrace.h" +#include "src/Tensor/TensorSycl.h" #include "src/Tensor/TensorExecutor.h" #include "src/Tensor/TensorDevice.h" @@ -125,6 +165,6 @@ #include "src/Tensor/TensorIO.h" -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> +#include "../../../Eigen/src/Core/util/ReenableStupidWarnings.h" //#endif // EIGEN_CXX11_TENSOR_MODULE
diff --git a/unsupported/Eigen/CXX11/TensorSymmetry b/unsupported/Eigen/CXX11/TensorSymmetry index f1dc25f..b09c5e4 100644 --- a/unsupported/Eigen/CXX11/TensorSymmetry +++ b/unsupported/Eigen/CXX11/TensorSymmetry
@@ -10,9 +10,11 @@ #ifndef EIGEN_CXX11_TENSORSYMMETRY_MODULE #define EIGEN_CXX11_TENSORSYMMETRY_MODULE -#include <unsupported/Eigen/CXX11/Tensor> +#include "Tensor" -#include <Eigen/src/Core/util/DisableStupidWarnings.h> +#include "../../../Eigen/src/Core/util/DisableStupidWarnings.h" + +#include "src/util/CXX11Meta.h" /** \defgroup CXX11_TensorSymmetry_Module Tensor Symmetry Module * @@ -31,7 +33,7 @@ #include "src/TensorSymmetry/StaticSymmetry.h" #include "src/TensorSymmetry/DynamicSymmetry.h" -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> +#include "../../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_CXX11_TENSORSYMMETRY_MODULE
diff --git a/unsupported/Eigen/CXX11/ThreadPool b/unsupported/Eigen/CXX11/ThreadPool index fe00a0b..d046af9 100644 --- a/unsupported/Eigen/CXX11/ThreadPool +++ b/unsupported/Eigen/CXX11/ThreadPool
@@ -10,9 +10,9 @@ #ifndef EIGEN_CXX11_THREADPOOL_MODULE #define EIGEN_CXX11_THREADPOOL_MODULE -#include "Core" +#include "../../../Eigen/Core" -#include <Eigen/src/Core/util/DisableStupidWarnings.h> +#include "../../../Eigen/src/Core/util/DisableStupidWarnings.h" /** \defgroup CXX11_ThreadPool_Module C++11 ThreadPool Module * @@ -44,19 +44,31 @@ #include <thread> #include <functional> #include <memory> +#include "src/util/CXX11Meta.h" +#include "src/util/MaxSizeVector.h" #include "src/ThreadPool/ThreadLocal.h" +#ifndef EIGEN_THREAD_LOCAL +// There are non-parenthesized calls to "max" in the <unordered_map> header, +// which trigger a check in test/main.h causing compilation to fail. +// We work around the check here by removing the check for max in +// the case where we have to emulate thread_local. +#ifdef max +#undef max +#endif +#include <unordered_map> +#endif #include "src/ThreadPool/ThreadYield.h" +#include "src/ThreadPool/ThreadCancel.h" #include "src/ThreadPool/EventCount.h" #include "src/ThreadPool/RunQueue.h" #include "src/ThreadPool/ThreadPoolInterface.h" #include "src/ThreadPool/ThreadEnvironment.h" -#include "src/ThreadPool/SimpleThreadPool.h" +#include "src/ThreadPool/Barrier.h" #include "src/ThreadPool/NonBlockingThreadPool.h" #endif -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> +#include "../../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_CXX11_THREADPOOL_MODULE -
diff --git a/unsupported/Eigen/CXX11/src/CMakeLists.txt b/unsupported/Eigen/CXX11/src/CMakeLists.txt deleted file mode 100644 index d90ee1b..0000000 --- a/unsupported/Eigen/CXX11/src/CMakeLists.txt +++ /dev/null
@@ -1,3 +0,0 @@ -add_subdirectory(Core) -add_subdirectory(Tensor) -add_subdirectory(TensorSymmetry)
diff --git a/unsupported/Eigen/CXX11/src/Core/CMakeLists.txt b/unsupported/Eigen/CXX11/src/Core/CMakeLists.txt deleted file mode 100644 index 28571dc..0000000 --- a/unsupported/Eigen/CXX11/src/Core/CMakeLists.txt +++ /dev/null
@@ -1 +0,0 @@ -add_subdirectory(util)
diff --git a/unsupported/Eigen/CXX11/src/Core/util/CMakeLists.txt b/unsupported/Eigen/CXX11/src/Core/util/CMakeLists.txt deleted file mode 100644 index 1e3b147..0000000 --- a/unsupported/Eigen/CXX11/src/Core/util/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_CXX11_Core_util_SRCS "*.h") - -INSTALL(FILES - ${Eigen_CXX11_Core_util_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11/src/Core/util COMPONENT Devel - )
diff --git a/unsupported/Eigen/CXX11/src/Tensor/CMakeLists.txt b/unsupported/Eigen/CXX11/src/Tensor/CMakeLists.txt deleted file mode 100644 index 6d4b3ea..0000000 --- a/unsupported/Eigen/CXX11/src/Tensor/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_CXX11_Tensor_SRCS "*.h") - -INSTALL(FILES - ${Eigen_CXX11_Tensor_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11/src/Tensor COMPONENT Devel - )
diff --git a/unsupported/Eigen/CXX11/src/Tensor/README.md b/unsupported/Eigen/CXX11/src/Tensor/README.md index eeca2f6..d84bb85 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/README.md +++ b/unsupported/Eigen/CXX11/src/Tensor/README.md
@@ -1,4 +1,4 @@ -# Eigen Tensors +# Eigen Tensors {#eigen_tensors} Tensors are multidimensional arrays of elements. Elements are typically scalars, but more complex types such as strings are also supported. @@ -8,7 +8,7 @@ ## Tensor Classes You can manipulate a tensor with one of the following classes. They all are in -the namespace ```::Eigen.``` +the namespace `::Eigen.` ### Class Tensor<data_type, rank> @@ -21,10 +21,10 @@ Tensors of this class are resizable. For example, if you assign a tensor of a different size to a Tensor, that tensor is resized to match its new value. -#### Constructor Tensor<data_type, rank>(size0, size1, ...) +#### Constructor `Tensor<data_type, rank>(size0, size1, ...)` -Constructor for a Tensor. The constructor must be passed ```rank``` integers -indicating the sizes of the instance along each of the the ```rank``` +Constructor for a Tensor. The constructor must be passed `rank` integers +indicating the sizes of the instance along each of the the `rank` dimensions. // Create a tensor of rank 3 of sizes 2, 3, 4. This tensor owns @@ -34,18 +34,18 @@ // Resize t_3d by assigning a tensor of different sizes, but same rank. t_3d = Tensor<float, 3>(3, 4, 3); -#### Constructor Tensor<data_type, rank>(size_array) +#### Constructor `Tensor<data_type, rank>(size_array)` Constructor where the sizes for the constructor are specified as an array of values instead of an explicitly list of parameters. The array type to use is -```Eigen::array<Eigen::Index>```. The array can be constructed automatically +`Eigen::array<Eigen::Index>`. The array can be constructed automatically from an initializer list. // Create a tensor of strings of rank 2 with sizes 5, 7. Tensor<string, 2> t_2d({5, 7}); -### Class TensorFixedSize<data_type, Sizes<size0, size1, ...>> +### Class `TensorFixedSize<data_type, Sizes<size0, size1, ...>>` Class to use for tensors of fixed size, where the size is known at compile time. Fixed sized tensors can provide very fast computations because all their @@ -57,7 +57,7 @@ // Create a 4 x 3 tensor of floats. TensorFixedSize<float, Sizes<4, 3>> t_4x3; -### Class TensorMap<Tensor<data_type, rank>> +### Class `TensorMap<Tensor<data_type, rank>>` This is the class to use to create a tensor on top of memory allocated and owned by another part of your code. It allows to view any piece of allocated @@ -67,7 +67,7 @@ A TensorMap is not resizable because it does not own the memory where its data are stored. -#### Constructor TensorMap<Tensor<data_type, rank>>(data, size0, size1, ...) +#### Constructor `TensorMap<Tensor<data_type, rank>>(data, size0, size1, ...)` Constructor for a Tensor. The constructor must be passed a pointer to the storage for the data, and "rank" size attributes. The storage has to be @@ -75,28 +75,28 @@ // Map a tensor of ints on top of stack-allocated storage. int storage[128]; // 2 x 4 x 2 x 8 = 128 - TensorMap<int, 4> t_4d(storage, 2, 4, 2, 8); + TensorMap<Tensor<int, 4>> t_4d(storage, 2, 4, 2, 8); // The same storage can be viewed as a different tensor. // You can also pass the sizes as an array. - TensorMap<int, 2> t_2d(storage, 16, 8); + TensorMap<Tensor<int, 2>> t_2d(storage, 16, 8); // You can also map fixed-size tensors. Here we get a 1d view of // the 2d fixed-size tensor. - Tensor<float, Sizes<4, 5>> t_4x3; - TensorMap<float, 1> t_12(t_4x3, 12); + TensorFixedSize<float, Sizes<4, 5>> t_4x3; + TensorMap<Tensor<float, 1>> t_12(t_4x3.data(), 12); -#### Class TensorRef +#### Class `TensorRef` See Assigning to a TensorRef below. ## Accessing Tensor Elements -#### <data_type> tensor(index0, index1...) +#### `<data_type> tensor(index0, index1...)` -Return the element at position ```(index0, index1...)``` in tensor -```tensor```. You must pass as many parameters as the rank of ```tensor```. +Return the element at position `(index0, index1...)` in tensor +`tensor`. You must pass as many parameters as the rank of `tensor`. The expression can be used as an l-value to set the value of the element at the specified position. The value returned is of the datatype of the tensor. @@ -121,8 +121,8 @@ ## TensorLayout -The tensor library supports 2 layouts: ```ColMajor``` (the default) and -```RowMajor```. Only the default column major layout is currently fully +The tensor library supports 2 layouts: `ColMajor` (the default) and +`RowMajor`. Only the default column major layout is currently fully supported, and it is therefore not recommended to attempt to use the row major layout at the moment. @@ -136,7 +136,7 @@ different layouts will result in a compilation error. It is possible to change the layout of a tensor or an expression using the -```swap_layout()``` method. Note that this will also reverse the order of the +`swap_layout()` method. Note that this will also reverse the order of the dimensions. Tensor<float, 2, ColMajor> col_major(2, 4); @@ -173,35 +173,35 @@ Tensor<float, 3> t3 = t1 + t2; While the code above looks easy enough, it is important to understand that the -expression ```t1 + t2``` is not actually adding the values of the tensors. The +expression `t1 + t2` is not actually adding the values of the tensors. The expression instead constructs a "tensor operator" object of the class TensorCwiseBinaryOp<scalar_sum>, which has references to the tensors -```t1``` and ```t2```. This is a small C++ object that knows how to add -```t1``` and ```t2```. It is only when the value of the expression is assigned -to the tensor ```t3``` that the addition is actually performed. Technically, -this happens through the overloading of ```operator=()``` in the Tensor class. +`t1` and `t2`. This is a small C++ object that knows how to add +`t1` and `t2`. It is only when the value of the expression is assigned +to the tensor `t3` that the addition is actually performed. Technically, +this happens through the overloading of `operator=()` in the Tensor class. This mechanism for computing tensor expressions allows for lazy evaluation and optimizations which are what make the tensor library very fast. -Of course, the tensor operators do nest, and the expression ```t1 + t2 * -0.3f``` is actually represented with the (approximate) tree of operators: +Of course, the tensor operators do nest, and the expression `t1 + t2 * 0.3f` +is actually represented with the (approximate) tree of operators: TensorCwiseBinaryOp<scalar_sum>(t1, TensorCwiseUnaryOp<scalar_mul>(t2, 0.3f)) ### Tensor Operations and C++ "auto" -Because Tensor operations create tensor operators, the C++ ```auto``` keyword +Because Tensor operations create tensor operators, the C++ `auto` keyword does not have its intuitive meaning. Consider these 2 lines of code: Tensor<float, 3> t3 = t1 + t2; auto t4 = t1 + t2; -In the first line we allocate the tensor ```t3``` and it will contain the -result of the addition of ```t1``` and ```t2```. In the second line, ```t4``` +In the first line we allocate the tensor `t3` and it will contain the +result of the addition of `t1` and `t2`. In the second line, `t4` is actually the tree of tensor operators that will compute the addition of -```t1``` and ```t2```. In fact, ```t4``` is *not* a tensor and you cannot get +`t1` and `t2`. In fact, `t4` is *not* a tensor and you cannot get the values of its elements: Tensor<float, 3> t3 = t1 + t2; @@ -210,8 +210,8 @@ auto t4 = t1 + t2; cout << t4(0, 0, 0); // Compilation error! -When you use ```auto``` you do not get a Tensor as a result but instead a -non-evaluated expression. So only use ```auto``` to delay evaluation. +When you use `auto` you do not get a Tensor as a result but instead a +non-evaluated expression. So only use `auto` to delay evaluation. Unfortunately, there is no single underlying concrete type for holding non-evaluated expressions, hence you have to use auto in the case when you do @@ -257,9 +257,9 @@ #### Assigning to a Tensor, TensorFixedSize, or TensorMap. The most common way to evaluate an expression is to assign it to a Tensor. In -the example below, the ```auto``` declarations make the intermediate values +the example below, the `auto` declarations make the intermediate values "Operations", not Tensors, and do not cause the expressions to be evaluated. -The assignment to the Tensor ```result``` causes the evaluation of all the +The assignment to the Tensor `result` causes the evaluation of all the operations. auto t3 = t1 + t2; // t3 is an Operation. @@ -272,17 +272,17 @@ efficient. // We know that the result is a 4x4x2 tensor! - TensorFixedSize<float, 4, 4, 2> result = t5; + TensorFixedSize<float, Sizes<4, 4, 2>> result = t5; Simiarly, assigning an expression to a TensorMap causes its evaluation. Like tensors of type TensorFixedSize, TensorMaps cannot be resized so they have to have the rank and sizes of the expression that are assigned to them. -#### Calling eval(). +#### Calling `eval()`. When you compute large composite expressions, you sometimes want to tell Eigen that an intermediate value in the expression tree is worth evaluating ahead of -time. This is done by inserting a call to the ```eval()``` method of the +time. This is done by inserting a call to the `eval()` method of the expression Operation. // The previous example could have been written: @@ -291,15 +291,15 @@ // If you want to compute (t1 + t2) once ahead of time you can write: Tensor<float, 3> result = ((t1 + t2).eval() * 0.2f).exp(); -Semantically, calling ```eval()``` is equivalent to materializing the value of +Semantically, calling `eval()` is equivalent to materializing the value of the expression in a temporary Tensor of the right size. The code above in effect does: // .eval() knows the size! - TensorFixedSize<float, 4, 4, 2> tmp = t1 + t2; + TensorFixedSize<float, Sizes<4, 4, 2>> tmp = t1 + t2; Tensor<float, 3> result = (tmp * 0.2f).exp(); -Note that the return value of ```eval()``` is itself an Operation, so the +Note that the return value of `eval()` is itself an Operation, so the following code does not do what you may think: // Here t3 is an evaluation Operation. t3 has not been evaluated yet. @@ -312,24 +312,24 @@ // an intermediate tensor to represent t3.x Tensor<float, 3> result = t4; -While in the examples above calling ```eval()``` does not make a difference in +While in the examples above calling `eval()` does not make a difference in performance, in other cases it can make a huge difference. In the expression -below the ```broadcast()``` expression causes the ```X.maximum()``` expression +below the `broadcast()` expression causes the `X.maximum()` expression to be evaluated many times: Tensor<...> X ...; Tensor<...> Y = ((X - X.maximum(depth_dim).reshape(dims2d).broadcast(bcast)) * beta).exp(); -Inserting a call to ```eval()``` between the ```maximum()``` and -```reshape()``` calls guarantees that maximum() is only computed once and +Inserting a call to `eval()` between the `maximum()` and +`reshape()` calls guarantees that maximum() is only computed once and greatly speeds-up execution: Tensor<...> Y = ((X - X.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)) * beta).exp(); -In the other example below, the tensor ```Y``` is both used in the expression +In the other example below, the tensor `Y` is both used in the expression and its assignment. This is an aliasing problem and if the evaluation is not done in the right order Y will be updated incrementally during the evaluation resulting in bogus results: @@ -337,8 +337,8 @@ Tensor<...> Y ...; Y = Y / (Y.sum(depth_dim).reshape(dims2d).broadcast(bcast)); -Inserting a call to ```eval()``` between the ```sum()``` and ```reshape()``` -expressions ensures that the sum is computed before any updates to ```Y``` are +Inserting a call to `eval()` between the `sum()` and `reshape()` +expressions ensures that the sum is computed before any updates to `Y` are done. Y = Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast)); @@ -347,21 +347,21 @@ because the generated has to compute the i-th value of the right hand side before assigning it to the left hand side. -However, if you were assigning the expression value to a shuffle of ```Y``` -then you would need to force an eval for correctness by adding an ```eval()``` +However, if you were assigning the expression value to a shuffle of `Y` +then you would need to force an eval for correctness by adding an `eval()` call for the right hand side: Y.shuffle(...) = (Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast))).eval(); -#### Assigning to a TensorRef. +#### Assigning to a `TensorRef`. If you need to access only a few elements from the value of an expression you can avoid materializing the value in a full tensor by using a TensorRef. A TensorRef is a small wrapper class for any Eigen Operation. It provides -overloads for the ```()``` operator that let you access individual values in +overloads for the `()` operator that let you access individual values in the expression. TensorRef is convenient, because the Operation themselves do not provide a way to access individual elements. @@ -390,7 +390,7 @@ different environments: single threaded on CPU, multi threaded on CPU, or on a GPU using cuda. Additional implementations may be added later. -You can choose which implementation to use with the ```device()``` call. If +You can choose which implementation to use with the `device()` call. If you do not choose an implementation explicitly the default implementation that uses a single thread on the CPU is used. @@ -406,7 +406,7 @@ Tensor<float, 2> b(30, 40); Tensor<float, 2> c = a + b; -To choose a different implementation you have to insert a ```device()``` call +To choose a different implementation you have to insert a `device()` call before the assignment of the result. For technical C++ reasons this requires that the Tensor for the result be declared on its own. This means that you have to know the size of the result. @@ -414,16 +414,16 @@ Eigen::Tensor<float, 2> c(30, 40); c.device(...) = a + b; -The call to ```device()``` must be the last call on the left of the operator=. +The call to `device()` must be the last call on the left of the operator=. -You must pass to the ```device()``` call an Eigen device object. There are +You must pass to the `device()` call an Eigen device object. There are presently three devices you can use: DefaultDevice, ThreadPoolDevice and GpuDevice. #### Evaluating With the DefaultDevice -This is exactly the same as not inserting a ```device()``` call. +This is exactly the same as not inserting a `device()` call. DefaultDevice my_device; c.device(my_device) = a + b; @@ -452,24 +452,24 @@ In the documentation of the tensor methods and Operation we mention datatypes that are tensor-type specific: -#### <Tensor-Type>::Dimensions +#### `<Tensor-Type>::``Dimensions` -Acts like an array of ints. Has an ```int size``` attribute, and can be +Acts like an array of ints. Has an `int size` attribute, and can be indexed like an array to access individual values. Used to represent the -dimensions of a tensor. See ```dimensions()```. +dimensions of a tensor. See `dimensions()`. -#### <Tensor-Type>::Index +#### `<Tensor-Type>::``Index` -Acts like an ```int```. Used for indexing tensors along their dimensions. See -```operator()```, ```dimension()```, and ```size()```. +Acts like an `int`. Used for indexing tensors along their dimensions. See +`operator()`, `dimension()`, and `size()`. -#### <Tensor-Type>::Scalar +#### `<Tensor-Type>::``Scalar` Represents the datatype of individual tensor elements. For example, for a -```Tensor<float>```, ```Scalar``` is the type ```float```. See -```setConstant()```. +`Tensor<float>`, `Scalar` is the type `float`. See +`setConstant()`. -#### <Operation> +#### `<Operation>` We use this pseudo type to indicate that a tensor Operation is returned by a method. We indicate in the text the type and dimensions of the tensor that the @@ -489,7 +489,7 @@ ## Metadata -### int NumDimensions +### `int NumDimensions` Constant value indicating the number of dimensions of a Tensor. This is also known as the tensor "rank". @@ -498,10 +498,10 @@ cout << "Dims " << a.NumDimensions; => Dims 2 -### Dimensions dimensions() +### `Dimensions dimensions()` Returns an array-like object representing the dimensions of the tensor. -The actual type of the dimensions() result is <Tensor-Type>::Dimensions. +The actual type of the `dimensions()` result is `<Tensor-Type>::``Dimensions`. Eigen::Tensor<float, 2> a(3, 4); const Eigen::Tensor<float, 2>::Dimensions& d = a.dimensions(); @@ -509,17 +509,17 @@ << ", dim 1: " << d[1]; => Dim size: 2, dim 0: 3, dim 1: 4 -If you use a C++11 compiler, you can use ```auto``` to simplify the code: +If you use a C++11 compiler, you can use `auto` to simplify the code: const auto& d = a.dimensions(); cout << "Dim size: " << d.size << ", dim 0: " << d[0] << ", dim 1: " << d[1]; => Dim size: 2, dim 0: 3, dim 1: 4 -### Index dimension(Index n) +### `Index dimension(Index n)` Returns the n-th dimension of the tensor. The actual type of the -```dimension()``` result is ```<Tensor-Type>::Index```, but you can +`dimension()` result is `<Tensor-Type>::``Index`, but you can always use it like an int. Eigen::Tensor<float, 2> a(3, 4); @@ -527,11 +527,11 @@ cout << "Dim 1: " << dim1; => Dim 1: 4 -### Index size() +### `Index size()` Returns the total number of elements in the tensor. This is the product of all -the tensor dimensions. The actual type of the ```size()``` result is -```<Tensor-Type>::Index```, but you can always use it like an int. +the tensor dimensions. The actual type of the `size()` result is +`<Tensor-Type>::``Index`, but you can always use it like an int. Eigen::Tensor<float, 2> a(3, 4); cout << "Size: " << a.size(); @@ -540,11 +540,11 @@ ### Getting Dimensions From An Operation -A few operations provide ```dimensions()``` directly, -e.g. ```TensorReslicingOp```. Most operations defer calculating dimensions +A few operations provide `dimensions()` directly, +e.g. `TensorReslicingOp`. Most operations defer calculating dimensions until the operation is being evaluated. If you need access to the dimensions of a deferred operation, you can wrap it in a TensorRef (see Assigning to a -TensorRef above), which provides ```dimensions()``` and ```dimension()``` as +TensorRef above), which provides `dimensions()` and `dimension()` as above. TensorRef can also wrap the plain Tensor types, so this is a useful idiom in @@ -567,11 +567,11 @@ ### TensorFixedSize -Creates a tensor of the specified size. The number of arguments in the Size<> +Creates a tensor of the specified size. The number of arguments in the Sizes<> template parameter determines the rank of the tensor. The content of the tensor is not initialized. - Eigen::TensorFixedSize<float, Size<3, 4>> a; + Eigen::TensorFixedSize<float, Sizes<3, 4>> a; cout << "Rank: " << a.rank() << endl; => Rank: 2 cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; @@ -581,14 +581,14 @@ Creates a tensor mapping an existing array of data. The data must not be freed until the TensorMap is discarded, and the size of the data must be large enough -to accomodate of the coefficients of the tensor. +to accommodate the coefficients of the tensor. float data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; - Eigen::TensorMap<float, 2> a(data, 3, 4); + Eigen::TensorMap<Tensor<float, 2>> a(data, 3, 4); cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; => NumRows: 3 NumCols: 4 cout << "a(1, 2): " << a(1, 2) << endl; - => a(1, 2): 9 + => a(1, 2): 7 ## Contents Initialization @@ -602,9 +602,9 @@ have an immediate effect on the tensor and return the tensor itself as a result. These are not tensor Operations which delay evaluation. -### <Tensor-Type> setConstant(const Scalar& val) +### `<Tensor-Type> setConstant(const Scalar& val)` -Sets all elements of the tensor to the constant value ```val```. ```Scalar``` +Sets all elements of the tensor to the constant value `val`. `Scalar` is the type of data stored in the tensor. You can pass any value that is convertible to that type. @@ -618,8 +618,8 @@ 12.3 12.3 12.3 12.3 12.3 12.3 12.3 12.3 -Note that ```setConstant()``` can be used on any tensor where the element type -has a copy constructor and an ```operator=()```: +Note that `setConstant()` can be used on any tensor where the element type +has a copy constructor and an `operator=()`: Eigen::Tensor<string, 2> a(2, 3); a.setConstant("yolo"); @@ -630,9 +630,9 @@ yolo yolo yolo -### <Tensor-Type> setZero() +### `<Tensor-Type> setZero()` -Fills the tensor with zeros. Equivalent to ```setConstant(Scalar(0))```. +Fills the tensor with zeros. Equivalent to `setConstant(Scalar(0))`. Returns the tensor itself in case you want to chain another call. a.setZero(); @@ -644,7 +644,7 @@ 0 0 0 0 -### <Tensor-Type> setValues({..initializer_list}) +### `<Tensor-Type> setValues({..initializer_list})` Fills the tensor with explicit values specified in a std::initializer_list. The type of the initializer list depends on the type and rank of the tensor. @@ -653,10 +653,10 @@ most deeply nested lists must contains P scalars of the Tensor type where P is the size of the last dimension of the Tensor. -For example, for a ```TensorFixedSize<float, 2, 3>``` the initializer list must +For example, for a `TensorFixedSize<float, 2, 3>` the initializer list must contains 2 lists of 3 floats each. -```setValues()``` returns the tensor itself in case you want to chain another +`setValues()` returns the tensor itself in case you want to chain another call. Eigen::Tensor<float, 2> a(2, 3); @@ -680,7 +680,7 @@ 10 20 30 1000 1000 1000 -### <Tensor-Type> setRandom() +### `<Tensor-Type> setRandom()` Fills the tensor with random values. Returns the tensor itself in case you want to chain another call. @@ -693,16 +693,16 @@ -0.211234 0.823295 0.536459 -0.0452059 0.566198 -0.604897 -0.444451 0.257742 -You can customize ```setRandom()``` by providing your own random number +You can customize `setRandom()` by providing your own random number generator as a template argument: a.setRandom<MyRandomGenerator>(); -Here, ```MyRandomGenerator``` must be a struct with the following member -functions, where Scalar and Index are the same as ```<Tensor-Type>::Scalar``` -and ```<Tensor-Type>::Index```. +Here, `MyRandomGenerator` must be a struct with the following member +functions, where Scalar and Index are the same as `<Tensor-Type>::``Scalar` +and `<Tensor-Type>::``Index`. -See ```struct UniformRandomGenerator``` in TensorFunctors.h for an example. +See `struct UniformRandomGenerator` in TensorFunctors.h for an example. // Custom number generator for use with setRandom(). struct MyRandomGenerator { @@ -747,7 +747,7 @@ wrapped in a TensorRef. -### Scalar* data() and const Scalar* data() const +### `Scalar* data()` and `const Scalar* data() const` Returns a pointer to the storage for the tensor. The pointer is const if the tensor was const. This allows direct access to the data. The layout of the @@ -767,7 +767,7 @@ ## Tensor Operations -All the methods documented below return non evaluated tensor ```Operations```. +All the methods documented below return non evaluated tensor `Operations`. These can be chained: you can apply another Tensor Operation to the value returned by the method. @@ -775,10 +775,10 @@ tensor. See "Controlling when Expression are Evaluated" for more details about their evaluation. -### <Operation> constant(const Scalar& val) +### `<Operation> constant(const Scalar& val)` Returns a tensor of the same type and dimensions as the original tensor but -where all elements have the value ```val```. +where all elements have the value `val`. This is useful, for example, when you want to add or subtract a constant from a tensor, or multiply every element of a tensor by a scalar. @@ -803,14 +803,14 @@ 0.6 0.6 0.6 0.6 0.6 0.6 -### <Operation> random() +### `<Operation> random()` Returns a tensor of the same type and dimensions as the current tensor but where all elements have random values. This is for example useful to add random values to an existing tensor. The generation of random values can be customized in the same manner -as for ```setRandom()```. +as for `setRandom()`. Eigen::Tensor<float, 2> a(2, 3); a.setConstant(1.0f); @@ -833,7 +833,7 @@ of the same type and dimensions as the tensor to which they are applied. The requested operations are applied to each element independently. -### <Operation> operator-() +### `<Operation> operator-()` Returns a tensor of the same type and dimensions as the original tensor containing the opposite values of the original tensor. @@ -852,42 +852,42 @@ -1 -1 -1 -1 -1 -1 -### <Operation> sqrt() +### `<Operation> sqrt()` Returns a tensor of the same type and dimensions as the original tensor containing the square roots of the original tensor. -### <Operation> rsqrt() +### `<Operation> rsqrt()` Returns a tensor of the same type and dimensions as the original tensor containing the inverse square roots of the original tensor. -### <Operation> square() +### `<Operation> square()` Returns a tensor of the same type and dimensions as the original tensor containing the squares of the original tensor values. -### <Operation> inverse() +### `<Operation> inverse()` Returns a tensor of the same type and dimensions as the original tensor containing the inverse of the original tensor values. -### <Operation> exp() +### `<Operation> exp()` Returns a tensor of the same type and dimensions as the original tensor containing the exponential of the original tensor. -### <Operation> log() +### `<Operation> log()` Returns a tensor of the same type and dimensions as the original tensor containing the natural logarithms of the original tensor. -### <Operation> abs() +### `<Operation> abs()` Returns a tensor of the same type and dimensions as the original tensor containing the absolute values of the original tensor. -### <Operation> pow(Scalar exponent) +### `<Operation> pow(Scalar exponent)` Returns a tensor of the same type and dimensions as the original tensor containing the coefficients of the original tensor to the power of the @@ -914,17 +914,17 @@ 0 1 2 3 4 5 -### <Operation> operator * (Scalar scale) +### `<Operation> operator * (Scalar scale)` Multiplies all the coefficients of the input tensor by the provided scale. -### <Operation> cwiseMax(Scalar threshold) +### `<Operation> cwiseMax(Scalar threshold)` TODO -### <Operation> cwiseMin(Scalar threshold) +### `<Operation> cwiseMin(Scalar threshold)` TODO -### <Operation> unaryExpr(const CustomUnaryOp& func) +### `<Operation> unaryExpr(const CustomUnaryOp& func)` TODO @@ -936,39 +936,39 @@ specified it is also of the same type. The requested operations are applied to each pair of elements independently. -### <Operation> operator+(const OtherDerived& other) +### `<Operation> operator+(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise sums of the inputs. -### <Operation> operator-(const OtherDerived& other) +### `<Operation> operator-(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise differences of the inputs. -### <Operation> operator*(const OtherDerived& other) +### `<Operation> operator*(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise products of the inputs. -### <Operation> operator/(const OtherDerived& other) +### `<Operation> operator/(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise quotients of the inputs. This operator is not supported for integer types. -### <Operation> cwiseMax(const OtherDerived& other) +### `<Operation> cwiseMax(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise maximums of the inputs. -### <Operation> cwiseMin(const OtherDerived& other) +### `<Operation> cwiseMin(const OtherDerived& other)` Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise mimimums of the inputs. -### <Operation> Logical operators +### `<Operation> Logical operators` The following logical operators are supported as well: @@ -1013,16 +1013,23 @@ Eigen::Tensor<int, 2> a(2, 3); a.setValues({{1, 2, 3}, {6, 5, 4}}); Eigen::Tensor<int, 2> b(3, 2); - a.setValues({{1, 2}, {4, 5}, {5, 6}}); + b.setValues({{1, 2}, {4, 5}, {5, 6}}); // Compute the traditional matrix product - array<IndexPair<int>, 1> product_dims = { IndexPair(1, 0) }; + Eigen::array<Eigen::IndexPair<int>, 1> product_dims = { Eigen::IndexPair<int>(1, 0) }; Eigen::Tensor<int, 2> AB = a.contract(b, product_dims); // Compute the product of the transpose of the matrices - array<IndexPair<int>, 1> transpose_product_dims = { IndexPair(0, 1) }; + Eigen::array<Eigen::IndexPair<int>, 1> transposed_product_dims = { Eigen::IndexPair<int>(0, 1) }; Eigen::Tensor<int, 2> AtBt = a.contract(b, transposed_product_dims); + // Contraction to scalar value using a double contraction. + // First coordinate of both tensors are contracted as well as both second coordinates, i.e., this computes the sum of the squares of the elements. + Eigen::array<Eigen::IndexPair<int>, 2> double_contraction_product_dims = { Eigen::IndexPair<int>(0, 0), Eigen::IndexPair<int>(1, 1) }; + Eigen::Tensor<int, 0> AdoubleContractedA = a.contract(a, double_contraction_product_dims); + + // Extracting the scalar value of the tensor contraction for further usage + int value = AdoubleContractedA(0); ## Reduction Operations @@ -1032,13 +1039,13 @@ the dimensions along which the slices are made. The Eigen Tensor library provides a set of predefined reduction operators such -as ```maximum()``` and ```sum()``` and lets you define additional operators by +as `maximum()` and `sum()` and lets you define additional operators by implementing a few methods from a reductor template. ### Reduction Dimensions All reduction operations take a single parameter of type -```<TensorType>::Dimensions``` which can always be specified as an array of +`<TensorType>::``Dimensions` which can always be specified as an array of ints. These are called the "reduction dimensions." The values are the indices of the dimensions of the input tensor over which the reduction is done. The parameter can have at most as many element as the rank of the input tensor; @@ -1102,7 +1109,7 @@ As a special case, if you pass no parameter to a reduction operation the original tensor is reduced along *all* its dimensions. The result is a -one-dimension tensor with a single value. +scalar, represented as a zero-dimension tensor. Eigen::Tensor<float, 3> a(2, 3, 4); a.setValues({{{0.0f, 1.0f, 2.0f, 3.0f}, @@ -1112,65 +1119,155 @@ {19.0f, 18.0f, 17.0f, 16.0f}, {20.0f, 21.0f, 22.0f, 23.0f}}}); // Reduce along all dimensions using the sum() operator. - Eigen::Tensor<float, 1> b = a.sum(); + Eigen::Tensor<float, 0> b = a.sum(); cout << "b" << endl << b << endl << endl; => b 276 -### <Operation> sum(const Dimensions& new_dims) -### <Operation> sum() +### `<Operation> sum(const Dimensions& new_dims)` +### `<Operation> sum()` Reduce a tensor using the sum() operator. The resulting values are the sum of the reduced values. -### <Operation> mean(const Dimensions& new_dims) -### <Operation> mean() +### `<Operation> mean(const Dimensions& new_dims)` +### `<Operation> mean()` Reduce a tensor using the mean() operator. The resulting values are the mean of the reduced values. -### <Operation> maximum(const Dimensions& new_dims) -### <Operation> maximum() +### `<Operation> maximum(const Dimensions& new_dims)` +### `<Operation> maximum()` Reduce a tensor using the maximum() operator. The resulting values are the largest of the reduced values. -### <Operation> minimum(const Dimensions& new_dims) -### <Operation> minimum() +### `<Operation> minimum(const Dimensions& new_dims)` +### `<Operation> minimum()` Reduce a tensor using the minimum() operator. The resulting values are the smallest of the reduced values. -### <Operation> prod(const Dimensions& new_dims) -### <Operation> prod() +### `<Operation> prod(const Dimensions& new_dims)` +### `<Operation> prod()` Reduce a tensor using the prod() operator. The resulting values are the product of the reduced values. -### <Operation> all(const Dimensions& new_dims) -### <Operation> all() +### `<Operation> all(const Dimensions& new_dims)` +### `<Operation> all()` Reduce a tensor using the all() operator. Casts tensor to bool and then checks whether all elements are true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### <Operation> any(const Dimensions& new_dims) -### <Operation> any() +### `<Operation> any(const Dimensions& new_dims)` +### `<Operation> any()` Reduce a tensor using the any() operator. Casts tensor to bool and then checks whether any element is true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### <Operation> reduce(const Dimensions& new_dims, const Reducer& reducer) +### `<Operation> reduce(const Dimensions& new_dims, const Reducer& reducer)` -Reduce a tensor using a user-defined reduction operator. See ```SumReducer``` +Reduce a tensor using a user-defined reduction operator. See `SumReducer` in TensorFunctors.h for information on how to implement a reduction operator. +## Trace + +A *Trace* operation returns a tensor with fewer dimensions than the original +tensor. It returns a tensor whose elements are the sum of the elements of the +original tensor along the main diagonal for a list of specified dimensions, the +"trace dimensions". Similar to the `Reduction Dimensions`, the trace dimensions +are passed as an input parameter to the operation, are of type `<TensorType>::``Dimensions` +, and have the same requirements when passed as an input parameter. In addition, +the trace dimensions must have the same size. + +Example: Trace along 2 dimensions. + + // Create a tensor of 3 dimensions + Eigen::Tensor<int, 3> a(2, 2, 3); + a.setValues({{{1, 2, 3}, {4, 5, 6}}, {{7, 8, 9}, {10, 11, 12}}}); + // Specify the dimensions along which the trace will be computed. + // In this example, the trace can only be computed along the dimensions + // with indices 0 and 1 + Eigen::array<int, 2> dims({0, 1}); + // The output tensor contains all but the trace dimensions. + Tensor<int, 1> a_trace = a.trace(dims); + cout << "a_trace:" << endl; + cout << a_trace << endl; + => + a_trace: + 11 + 13 + 15 + + +### `<Operation> trace(const Dimensions& new_dims)` +### `<Operation> trace()` + +As a special case, if no parameter is passed to the operation, trace is computed +along *all* dimensions of the input tensor. + +Example: Trace along all dimensions. + + // Create a tensor of 3 dimensions, with all dimensions having the same size. + Eigen::Tensor<int, 3> a(3, 3, 3); + a.setValues({{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, + {{10, 11, 12}, {13, 14, 15}, {16, 17, 18}}, + {{19, 20, 21}, {22, 23, 24}, {25, 26, 27}}}); + // Result is a zero dimension tensor + Tensor<int, 0> a_trace = a.trace(); + cout<<"a_trace:"<<endl; + cout<<a_trace<<endl; + => + a_trace: + 42 + + +## Scan Operations + +A *Scan* operation returns a tensor with the same dimensions as the original +tensor. The operation performs an inclusive scan along the specified +axis, which means it computes a running total along the axis for a given +reduction operation. +If the reduction operation corresponds to summation, then this computes the +prefix sum of the tensor along the given axis. + +Example: +dd a comment to this line + + // Create a tensor of 2 dimensions + Eigen::Tensor<int, 2> a(2, 3); + a.setValues({{1, 2, 3}, {4, 5, 6}}); + // Scan it along the second dimension (1) using summation + Eigen::Tensor<int, 2> b = a.cumsum(1); + // The result is a tensor with the same size as the input + cout << "a" << endl << a << endl << endl; + cout << "b" << endl << b << endl << endl; + => + a + 1 2 3 + 4 5 6 + + b + 1 3 6 + 4 9 15 + +### `<Operation> cumsum(const Index& axis)` + +Perform a scan by summing consecutive entries. + +### `<Operation> cumprod(const Index& axis)` + +Perform a scan by multiplying consecutive entries. + + ## Convolutions -### <Operation> convolve(const Kernel& kernel, const Dimensions& dims) +### `<Operation> convolve(const Kernel& kernel, const Dimensions& dims)` Returns a tensor that is the output of the convolution of the input tensor with the kernel, along the specified dimensions of the input tensor. The dimension size for dimensions of the output tensor @@ -1213,7 +1310,7 @@ Tensor. They can be used to access slices of tensors, see them with different dimensions, or pad tensors with additional data. -### <Operation> reshape(const Dimensions& new_dims) +### `<Operation> reshape(const Dimensions& new_dims)` Returns a view of the input tensor that has been reshaped to the specified new dimensions. The argument new_dims is an array of Index values. The @@ -1235,7 +1332,7 @@ This operation does not move any data in the input tensor, so the resulting contents of a reshaped Tensor depend on the data layout of the original Tensor. -For example this is what happens when you ```reshape()``` a 2D ColMajor tensor +For example this is what happens when you `reshape()` a 2D ColMajor tensor to one dimension: Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); @@ -1276,7 +1373,7 @@ Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); Eigen::array<Eigen::DenseIndex, 2> two_dim({2, 3}); - Eigen::Tensor<float, 1, Eigen::ColMajor> b; + Eigen::Tensor<float, 1, Eigen::ColMajor> b(6); b.reshape(two_dim) = a; cout << "b" << endl << b << endl; => @@ -1292,7 +1389,7 @@ the reshape view of b. -### <Operation> shuffle(const Shuffle& shuffle) +### `<Operation> shuffle(const Shuffle& shuffle)` Returns a copy of the input tensor whose dimensions have been reordered according to the specified permutation. The argument shuffle @@ -1333,14 +1430,14 @@ output.shuffle({2, 0, 1}) = input; -### <Operation> stride(const Strides& strides) +### `<Operation> stride(const Strides& strides)` Returns a view of the input tensor that strides (skips stride-1 elements) along each of the dimensions. The argument strides is an array of Index values. The dimensions of the resulting tensor are ceil(input_dimensions[i] / strides[i]). -For example this is what happens when you ```stride()``` a 2D tensor: +For example this is what happens when you `stride()` a 2D tensor: Eigen::Tensor<int, 2> a(4, 3); a.setValues({{0, 100, 200}, {300, 400, 500}, {600, 700, 800}, {900, 1000, 1100}}); @@ -1359,7 +1456,7 @@ output.stride({2, 3, 4}) = input; -### <Operation> slice(const StartIndices& offsets, const Sizes& extents) +### `<Operation> slice(const StartIndices& offsets, const Sizes& extents)` Returns a sub-tensor of the given tensor. For each dimension i, the slice is made of the coefficients stored between offset[i] and offset[i] + extents[i] in @@ -1385,7 +1482,7 @@ 600 700 -### <Operation> chip(const Index offset, const Index dim) +### `<Operation> chip(const Index offset, const Index dim)` A chip is a special kind of slice. It is the subtensor at the given offset in the dimension dim. The returned tensor has one fewer dimension than the input @@ -1436,7 +1533,7 @@ 0 0 0 -### <Operation> reverse(const ReverseDimensions& reverse) +### `<Operation> reverse(const ReverseDimensions& reverse)` Returns a view of the input tensor that reverses the order of the coefficients along a subset of the dimensions. The argument reverse is an array of boolean @@ -1444,7 +1541,7 @@ reversed along each of the dimensions. This operation preserves the dimensions of the input tensor. -For example this is what happens when you ```reverse()``` the first dimension +For example this is what happens when you `reverse()` the first dimension of a 2D tensor: Eigen::Tensor<int, 2> a(4, 3); @@ -1466,7 +1563,7 @@ 0 100 200 -### <Operation> broadcast(const Broadcast& broadcast) +### `<Operation> broadcast(const Broadcast& broadcast)` Returns a view of the input tensor in which the input is replicated one to many times. @@ -1490,11 +1587,11 @@ 0 100 200 0 100 200 300 400 500 300 400 500 -### <Operation> concatenate(const OtherDerived& other, Axis axis) +### `<Operation> concatenate(const OtherDerived& other, Axis axis)` TODO -### <Operation> pad(const PaddingDimensions& padding) +### `<Operation> pad(const PaddingDimensions& padding)` Returns a view of the input tensor in which the input is padded with zeros. @@ -1519,7 +1616,7 @@ 0 0 0 0 -### <Operation> extract_patches(const PatchDims& patch_dims) +### `<Operation> extract_patches(const PatchDims& patch_dims)` Returns a tensor of coefficient patches extracted from the input tensor, where each patch is of dimension specified by 'patch_dims'. The returned tensor has @@ -1606,9 +1703,7 @@ 6 7 10 11 -### <Operation> extract_image_patches(const Index patch_rows, const Index patch_cols, - const Index row_stride, const Index col_stride, - const PaddingType padding_type) +### `<Operation> extract_image_patches(const Index patch_rows, const Index patch_cols, const Index row_stride, const Index col_stride, const PaddingType padding_type)` Returns a tensor of coefficient image patches extracted from the input tensor, which is expected to have dimensions ordered as follows (depending on the data @@ -1663,7 +1758,7 @@ ## Special Operations -### <Operation> cast<T>() +### `<Operation> cast<T>()` Returns a tensor of type T with the same dimensions as the original tensor. The returned tensor contains the values of the original tensor converted to @@ -1692,18 +1787,16 @@ 1 2 2 -### <Operation> eval() +### `<Operation> eval()` TODO ## Representation of scalar values -Scalar values are often represented by tensors of size 1 and rank 1. It would be -more logical and user friendly to use tensors of rank 0 instead. For example -Tensor<T, N>::maximum() currently returns a Tensor<T, 1>. Similarly, the inner -product of 2 1d tensors (through contractions) returns a 1d tensor. In the -future these operations might be updated to return 0d tensors instead. +Scalar values are often represented by tensors of size 1 and rank 0.For example +Tensor<T, N>::maximum() currently returns a Tensor<T, 0>. Similarly, the inner +product of 2 1d tensors (through contractions) returns a 0d tensor. ## Limitations
diff --git a/unsupported/Eigen/CXX11/src/Tensor/Tensor.h b/unsupported/Eigen/CXX11/src/Tensor/Tensor.h index 759dede..17cee49 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/Tensor.h +++ b/unsupported/Eigen/CXX11/src/Tensor/Tensor.h
@@ -23,12 +23,12 @@ * The %Tensor class encompasses only dynamic-size objects so far. * * The first two template parameters are required: - * \tparam Scalar_ \anchor tensor_tparam_scalar Numeric type, e.g. float, double, int or std::complex<float>. + * \tparam Scalar_ Numeric type, e.g. float, double, int or `std::complex<float>`. * User defined scalar types are supported as well (see \ref user_defined_scalars "here"). * \tparam NumIndices_ Number of indices (i.e. rank of the tensor) * * The remaining template parameters are optional -- in most cases you don't have to worry about them. - * \tparam Options_ \anchor tensor_tparam_options A combination of either \b #RowMajor or \b #ColMajor, and of either + * \tparam Options_ A combination of either \b #RowMajor or \b #ColMajor, and of either * \b #AutoAlign or \b #DontAlign. * The former controls \ref TopicStorageOrders "storage order", and defaults to column-major. The latter controls alignment, which is required * for vectorization. It defaults to aligning tensors. Note that tensors currently do not support any operations that profit from vectorization. @@ -42,13 +42,13 @@ * \endcode * * This class can be extended with the help of the plugin mechanism described on the page - * \ref TopicCustomizingEigen by defining the preprocessor symbol \c EIGEN_TENSOR_PLUGIN. + * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_TENSOR_PLUGIN. * * <i><b>Some notes:</b></i> * * <dl> * <dt><b>Relation to other parts of Eigen:</b></dt> - * <dd>The midterm developement goal for this class is to have a similar hierarchy as Eigen uses for matrices, so that + * <dd>The midterm development goal for this class is to have a similar hierarchy as Eigen uses for matrices, so that * taking blocks or using tensors in expressions is easily possible, including an interface with the vector/matrix code * by providing .asMatrix() and .asVector() (or similar) methods for rank 2 and 1 tensors. However, currently, the %Tensor * class does not provide any of these features and is only available as a stand-alone class that just allows for @@ -110,9 +110,9 @@ inline Self& base() { return *this; } inline const Self& base() const { return *this; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - EIGEN_DEVICE_FUNC inline const Scalar& coeff(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) const { // The number of indices used to access a tensor coefficient must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -150,7 +150,7 @@ return m_storage.data()[index]; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> inline Scalar& coeffRef(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) { @@ -190,7 +190,7 @@ return m_storage.data()[index]; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> inline const Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) const { @@ -257,7 +257,7 @@ return coeff(index); } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> inline Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) { @@ -336,7 +336,7 @@ { } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index firstDimension, IndexTypes... otherDimensions) : m_storage(firstDimension, otherDimensions...) @@ -350,22 +350,22 @@ { EIGEN_STATIC_ASSERT(1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index dim1, Index dim2) : m_storage(dim1*dim2, array<Index, 2>(dim1, dim2)) { EIGEN_STATIC_ASSERT(2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index dim1, Index dim2, Index dim3) : m_storage(dim1*dim2*dim3, array<Index, 3>(dim1, dim2, dim3)) { EIGEN_STATIC_ASSERT(3 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index dim1, Index dim2, Index dim3, Index dim4) : m_storage(dim1*dim2*dim3*dim4, array<Index, 4>(dim1, dim2, dim3, dim4)) { EIGEN_STATIC_ASSERT(4 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4, Index dim5) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index dim1, Index dim2, Index dim3, Index dim4, Index dim5) : m_storage(dim1*dim2*dim3*dim4*dim5, array<Index, 5>(dim1, dim2, dim3, dim4, dim5)) { EIGEN_STATIC_ASSERT(5 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -398,6 +398,21 @@ internal::TensorExecutor<const Assign, DefaultDevice>::run(assign, DefaultDevice()); } + #if EIGEN_HAS_RVALUE_REFERENCES + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Tensor(Self&& other) + : Tensor() + { + m_storage.swap(other.m_storage); + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Tensor& operator=(Self&& other) + { + m_storage.swap(other.m_storage); + return *this; + } + #endif + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor& operator=(const Tensor& other) { @@ -418,7 +433,7 @@ return *this; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC void resize(Index firstDimension, IndexTypes... otherDimensions) { @@ -462,6 +477,18 @@ // Nothing to do: rank 0 tensors have fixed size } +#ifdef EIGEN_HAS_INDEX_LIST + template <typename FirstType, typename... OtherTypes> + EIGEN_DEVICE_FUNC + void resize(const Eigen::IndexList<FirstType, OtherTypes...>& dimensions) { + array<Index, NumIndices> dims; + for (int i = 0; i < NumIndices; ++i) { + dims[i] = static_cast<Index>(dimensions[i]); + } + resize(dims); + } +#endif + /** Custom Dimension */ #ifdef EIGEN_HAS_SFINAE template<typename CustomDimension,
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h index babafe1..6f7c6d8 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h
@@ -87,6 +87,7 @@ IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/ false, PacketAccess = /*TensorEvaluator<ArgType, Device>::PacketAccess*/ false, BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -119,6 +120,12 @@ EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { + return m_impl; + } +#endif + protected: TensorEvaluator<ArgType, Device> m_impl; }; @@ -172,7 +179,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorTupleReducerOp(const XprType& expr, const ReduceOp& reduce_op, - const int return_dim, + const Index return_dim, const Dims& reduce_dims) : m_xpr(expr), m_reduce_op(reduce_op), m_return_dim(return_dim), m_reduce_dims(reduce_dims) {} @@ -187,12 +194,12 @@ const Dims& reduce_dims() const { return m_reduce_dims; } EIGEN_DEVICE_FUNC - int return_dim() const { return m_return_dim; } + Index return_dim() const { return m_return_dim; } protected: typename XprType::Nested m_xpr; const ReduceOp m_reduce_op; - const int m_return_dim; + const Index m_return_dim; const Dims m_reduce_dims; }; @@ -214,6 +221,7 @@ IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/ false, PacketAccess = /*TensorEvaluator<ArgType, Device>::PacketAccess*/ false, BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<const TensorReductionOp<ReduceOp, Dims, const TensorIndexTupleOp<ArgType> >, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -222,8 +230,11 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_orig_impl(op.expression(), device), m_impl(op.expression().index_tuples().reduce(op.reduce_dims(), op.reduce_op()), device), - m_return_dim(op.return_dim()) { - + m_return_dim(op.return_dim()) +#ifdef EIGEN_USE_SYCL + ,m_device(device) +#endif + { gen_strides(m_orig_impl.dimensions(), m_strides); if (Layout == static_cast<int>(ColMajor)) { const Index total_size = internal::array_prod(m_orig_impl.dimensions()); @@ -232,7 +243,7 @@ const Index total_size = internal::array_prod(m_orig_impl.dimensions()); m_stride_mod = (m_return_dim > 0) ? m_strides[m_return_dim - 1] : total_size; } - m_stride_div = m_strides[m_return_dim]; + m_stride_div = (m_return_dim >= 0) ? m_strides[m_return_dim] : 1; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { @@ -252,7 +263,24 @@ return (m_return_dim < 0) ? v.first : (v.first % m_stride_mod) / m_stride_div; } + #ifndef EIGEN_USE_SYCL EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + #else // following functions are required by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TupleType* data() const { return m_impl.data(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index return_dim() const {return m_return_dim;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const StrideDims& strides() const {return m_strides;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Index& stride_mod() const {return m_stride_mod;} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Index& stride_div() const {return m_stride_div;} + const Device& device() const{return m_device;} + #endif + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost + costPerCoeff(bool vectorized) const { + const double compute_cost = 1.0 + + (m_return_dim < 0 ? 0.0 : (TensorOpCost::ModCost<Index>() + TensorOpCost::DivCost<Index>())); + return m_orig_impl.costPerCoeff(vectorized) + + m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, compute_cost); + } private: EIGEN_DEVICE_FUNC void gen_strides(const InputDimensions& dims, StrideDims& strides) { @@ -280,10 +308,13 @@ protected: TensorEvaluator<const TensorIndexTupleOp<ArgType>, Device> m_orig_impl; TensorEvaluator<const TensorReductionOp<ReduceOp, Dims, const TensorIndexTupleOp<ArgType> >, Device> m_impl; - const int m_return_dim; + const Index m_return_dim; StrideDims m_strides; Index m_stride_mod; Index m_stride_div; +#ifdef EIGEN_USE_SYCL + const Device& m_device; +#endif }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorArgMaxSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMaxSycl.h new file mode 100644 index 0000000..5110e99 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMaxSycl.h
@@ -0,0 +1,148 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorArgMaxSycl.h + * \brief: + * TensorArgMaxSycl + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_ARGMAX_SYCL_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_ARGMAX_SYCL_HPP +namespace Eigen { +namespace internal { + template<typename Dims, typename XprType> + struct eval<TensorTupleReducerDeviceOp<Dims, XprType>, Eigen::Dense> + { + typedef const TensorTupleReducerDeviceOp<Dims, XprType>& type; + }; + + template<typename Dims, typename XprType> + struct nested<TensorTupleReducerDeviceOp<Dims, XprType>, 1, + typename eval<TensorTupleReducerDeviceOp<Dims, XprType> >::type> + { + typedef TensorTupleReducerDeviceOp<Dims, XprType> type; + }; + +template<typename StrideDims, typename XprType> +struct traits<TensorTupleReducerDeviceOp<StrideDims, XprType> > : public traits<XprType> +{ + typedef traits<XprType> XprTraits; + typedef typename XprTraits::StorageKind StorageKind; + typedef typename XprTraits::Index Index; + typedef Index Scalar; + typedef typename XprType::Nested Nested; + typedef typename remove_reference<Nested>::type _Nested; + static const int NumDimensions = XprTraits::NumDimensions; + static const int Layout = XprTraits::Layout; +}; + + +}// end namespace internal +template<typename StrideDims, typename XprType> +class TensorTupleReducerDeviceOp : public TensorBase<TensorTupleReducerDeviceOp<StrideDims, XprType>, ReadOnlyAccessors> +{ + public: + typedef typename Eigen::internal::traits<TensorTupleReducerDeviceOp>::Scalar Scalar; + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; + typedef typename Eigen::internal::nested<TensorTupleReducerDeviceOp>::type Nested; + typedef typename Eigen::internal::traits<TensorTupleReducerDeviceOp>::StorageKind StorageKind; + typedef typename Eigen::internal::traits<TensorTupleReducerDeviceOp>::Index Index; + typedef typename XprType::CoeffReturnType TupleType; + typedef Index CoeffReturnType; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorTupleReducerDeviceOp(XprType expr, + const Index return_dim, + const StrideDims strides, + const Index stride_mod, const Index stride_div) + :m_xpr(expr), m_return_dim(return_dim), m_strides(strides), m_stride_mod(stride_mod), m_stride_div(stride_div) {} + + EIGEN_DEVICE_FUNC + const typename internal::remove_all<typename XprType::Nested>::type& + expression() const { return m_xpr; } + + EIGEN_DEVICE_FUNC + Index return_dim() const { return m_return_dim; } + + EIGEN_DEVICE_FUNC + const StrideDims& strides() const { return m_strides; } + + EIGEN_DEVICE_FUNC + const Index& stride_mod() const { return m_stride_mod; } + + EIGEN_DEVICE_FUNC + const Index& stride_div() const { return m_stride_div; } + + protected: + typename Eigen::internal::remove_all<typename + XprType::Nested + >::type m_xpr; + const Index m_return_dim; + const StrideDims m_strides; + const Index m_stride_mod; + const Index m_stride_div; +}; + + +// Eval as rvalue +template<typename StrideDims, typename ArgType> +struct TensorEvaluator<const TensorTupleReducerDeviceOp<StrideDims, ArgType>, SyclKernelDevice> +{ + typedef TensorTupleReducerDeviceOp<StrideDims, ArgType> XprType; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::TupleType TupleType; + typedef typename TensorEvaluator<ArgType, SyclKernelDevice>::Dimensions Dimensions; + + enum { + IsAligned = false, + PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<ArgType, SyclKernelDevice>::Layout, + CoordAccess = false, + RawAccess = false + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const SyclKernelDevice& device) + : m_impl(op.expression(), device), m_return_dim(op.return_dim()), m_strides(op.strides()), m_stride_mod(op.stride_mod()), + m_stride_div(op.stride_div()){} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { + return m_impl.dimensions(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar*) { + m_impl.evalSubExprsIfNeeded(NULL); + return true; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + m_impl.cleanup(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { + const TupleType v = m_impl.coeff(index); + return (m_return_dim < 0) ? v.first : (v.first % m_stride_mod) / m_stride_div; + } +typedef typename MakeGlobalPointer<typename TensorEvaluator<ArgType , SyclKernelDevice>::CoeffReturnType >::Type ptr_Dev_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptr_Dev_type data() const { return const_cast<ptr_Dev_type>(m_impl.data()); } + +protected: + TensorEvaluator<ArgType , SyclKernelDevice> m_impl; + const Index m_return_dim; + const StrideDims m_strides; + const Index m_stride_mod; + const Index m_stride_div; +}; +} // end namespace Eigen +#endif //UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_ARGMAX_SYCL_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h b/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h index 5abff08..06bf422 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h
@@ -34,9 +34,10 @@ typedef typename remove_reference<RhsNested>::type _RhsNested; static const std::size_t NumDimensions = internal::traits<LhsXprType>::NumDimensions; static const int Layout = internal::traits<LhsXprType>::Layout; + typedef typename traits<LhsXprType>::PointerType PointerType; enum { - Flags = 0, + Flags = 0 }; }; @@ -67,6 +68,8 @@ typedef typename Eigen::internal::traits<TensorAssignOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorAssignOp>::Index Index; + static const int NumDims = Eigen::internal::traits<TensorAssignOp>::NumDimensions; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorAssignOp(LhsXprType& lhs, const RhsXprType& rhs) : m_lhs_xpr(lhs), m_rhs_xpr(rhs) {} @@ -94,20 +97,35 @@ typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<RightArgType, Device>::Dimensions Dimensions; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + static const int NumDims = XprType::NumDims; enum { - IsAligned = TensorEvaluator<LeftArgType, Device>::IsAligned & TensorEvaluator<RightArgType, Device>::IsAligned, - PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, - Layout = TensorEvaluator<LeftArgType, Device>::Layout, - RawAccess = TensorEvaluator<LeftArgType, Device>::RawAccess, + IsAligned = TensorEvaluator<LeftArgType, Device>::IsAligned & + TensorEvaluator<RightArgType, Device>::IsAligned, + PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & + TensorEvaluator<RightArgType, Device>::PacketAccess, + BlockAccess = TensorEvaluator<LeftArgType, Device>::BlockAccess & + TensorEvaluator<RightArgType, Device>::BlockAccess, + PreferBlockAccess = TensorEvaluator<LeftArgType, Device>::PreferBlockAccess | + TensorEvaluator<RightArgType, Device>::PreferBlockAccess, + Layout = TensorEvaluator<LeftArgType, Device>::Layout, + RawAccess = TensorEvaluator<LeftArgType, Device>::RawAccess }; + typedef typename internal::TensorBlock< + typename internal::remove_const<Scalar>::type, Index, NumDims, Layout> + TensorBlock; + EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : m_leftImpl(op.lhsExpression(), device), m_rightImpl(op.rhsExpression(), device) { - EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) == static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT( + (static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) == + static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)), + YOU_MADE_A_PROGRAMMING_MISTAKE); } EIGEN_DEVICE_FUNC const Dimensions& dimensions() const @@ -163,7 +181,31 @@ TensorOpCost(0, sizeof(CoeffReturnType), 0, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_leftImpl.data(); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + m_leftImpl.getResourceRequirements(resources); + m_rightImpl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalBlock(TensorBlock* block) { + if (TensorEvaluator<LeftArgType, Device>::RawAccess && + m_leftImpl.data() != NULL) { + TensorBlock left_block(block->first_coeff_index(), block->block_sizes(), + block->tensor_strides(), block->tensor_strides(), + m_leftImpl.data() + block->first_coeff_index()); + m_rightImpl.block(&left_block); + } else { + m_rightImpl.block(block); + m_leftImpl.writeBlock(*block); + } + } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<LeftArgType, Device>& left_impl() const { return m_leftImpl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<RightArgType, Device>& right_impl() const { return m_rightImpl; } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return m_leftImpl.data(); } private: TensorEvaluator<LeftArgType, Device> m_leftImpl;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h b/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h index 1a34f3c..dd008fe 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h
@@ -20,9 +20,11 @@ * \brief The tensor base class. * * This class is the common parent of the Tensor and TensorMap class, thus - * making it possible to use either class interchangably in expressions. + * making it possible to use either class interchangeably in expressions. */ - +#ifndef EIGEN_PARSED_BY_DOXYGEN +// FIXME Doxygen does not like the inheritance with different template parameters +// Since there is no doxygen documentation inside, we disable it for now template<typename Derived> class TensorBase<Derived, ReadOnlyAccessors> { @@ -133,6 +135,18 @@ return unaryExpr(internal::scalar_digamma_op<Scalar>()); } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_i0e_op<Scalar>, const Derived> + i0e() const { + return unaryExpr(internal::scalar_i0e_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_i1e_op<Scalar>, const Derived> + i1e() const { + return unaryExpr(internal::scalar_i1e_op<Scalar>()); + } + // igamma(a = this, x = other) template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_igamma_op<Scalar>, const Derived, const OtherDerived> @@ -140,6 +154,20 @@ return binaryExpr(other.derived(), internal::scalar_igamma_op<Scalar>()); } + // igamma_der_a(a = this, x = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_igamma_der_a_op<Scalar>, const Derived, const OtherDerived> + igamma_der_a(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_igamma_der_a_op<Scalar>()); + } + + // gamma_sample_der_alpha(alpha = this, sample = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_gamma_sample_der_alpha_op<Scalar>, const Derived, const OtherDerived> + gamma_sample_der_alpha(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_gamma_sample_der_alpha_op<Scalar>()); + } + // igammac(a = this, x = other) template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_igammac_op<Scalar>, const Derived, const OtherDerived> @@ -174,9 +202,9 @@ } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_sigmoid_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_logistic_op<Scalar>, const Derived> sigmoid() const { - return unaryExpr(internal::scalar_sigmoid_op<Scalar>()); + return unaryExpr(internal::scalar_logistic_op<Scalar>()); } EIGEN_DEVICE_FUNC @@ -186,52 +214,112 @@ } EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_expm1_op<Scalar>, const Derived> + expm1() const { + return unaryExpr(internal::scalar_expm1_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_log_op<Scalar>, const Derived> log() const { return unaryExpr(internal::scalar_log_op<Scalar>()); } EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_log1p_op<Scalar>, const Derived> + log1p() const { + return unaryExpr(internal::scalar_log1p_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_abs_op<Scalar>, const Derived> abs() const { return unaryExpr(internal::scalar_abs_op<Scalar>()); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_clamp_op<Scalar>, const Derived> + clip(Scalar min, Scalar max) const { + return unaryExpr(internal::scalar_clamp_op<Scalar>(min, max)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const typename internal::conditional<NumTraits<Scalar>::IsComplex, + TensorCwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const Derived>, + Derived>::type conjugate() const { - return unaryExpr(internal::scalar_conjugate_op<Scalar>()); + return choose(Cond<NumTraits<Scalar>::IsComplex>(), unaryExpr(internal::scalar_conjugate_op<Scalar>()), derived()); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_pow_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::bind2nd_op<internal::scalar_pow_op<Scalar,Scalar> >, const Derived> pow(Scalar exponent) const { - return unaryExpr(internal::scalar_pow_op<Scalar>(exponent)); + return unaryExpr(internal::bind2nd_op<internal::scalar_pow_op<Scalar,Scalar> >(exponent)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_add_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_real_op<Scalar>, const Derived> + real() const { + return unaryExpr(internal::scalar_real_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_imag_op<Scalar>, const Derived> + imag() const { + return unaryExpr(internal::scalar_imag_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::bind2nd_op<internal::scalar_sum_op<Scalar,Scalar> >, const Derived> operator+ (Scalar rhs) const { - return unaryExpr(internal::scalar_add_op<Scalar>(rhs)); + return unaryExpr(internal::bind2nd_op<internal::scalar_sum_op<Scalar,Scalar> >(rhs)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_sub_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE friend + const TensorCwiseUnaryOp<internal::bind1st_op<internal::scalar_sum_op<Scalar> >, const Derived> + operator+ (Scalar lhs, const Derived& rhs) { + return rhs.unaryExpr(internal::bind1st_op<internal::scalar_sum_op<Scalar> >(lhs)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::bind2nd_op<internal::scalar_difference_op<Scalar,Scalar> >, const Derived> operator- (Scalar rhs) const { EIGEN_STATIC_ASSERT((NumTraits<Scalar>::IsSigned || internal::is_same<Scalar, const std::complex<float> >::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - return unaryExpr(internal::scalar_sub_op<Scalar>(rhs)); + return unaryExpr(internal::bind2nd_op<internal::scalar_difference_op<Scalar,Scalar> >(rhs)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE friend + const TensorCwiseUnaryOp<internal::bind1st_op<internal::scalar_difference_op<Scalar> >, const Derived> + operator- (Scalar lhs, const Derived& rhs) { + return rhs.unaryExpr(internal::bind1st_op<internal::scalar_difference_op<Scalar> >(lhs)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::bind2nd_op<internal::scalar_product_op<Scalar,Scalar> >, const Derived> operator* (Scalar rhs) const { - return unaryExpr(internal::scalar_multiple_op<Scalar>(rhs)); + return unaryExpr(internal::bind2nd_op<internal::scalar_product_op<Scalar,Scalar> >(rhs)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_quotient1_op<Scalar>, const Derived> + EIGEN_STRONG_INLINE friend + const TensorCwiseUnaryOp<internal::bind1st_op<internal::scalar_product_op<Scalar> >, const Derived> + operator* (Scalar lhs, const Derived& rhs) { + return rhs.unaryExpr(internal::bind1st_op<internal::scalar_product_op<Scalar> >(lhs)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::bind2nd_op<internal::scalar_quotient_op<Scalar,Scalar> >, const Derived> operator/ (Scalar rhs) const { - return unaryExpr(internal::scalar_quotient1_op<Scalar>(rhs)); + return unaryExpr(internal::bind2nd_op<internal::scalar_quotient_op<Scalar,Scalar> >(rhs)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE friend + const TensorCwiseUnaryOp<internal::bind1st_op<internal::scalar_quotient_op<Scalar> >, const Derived> + operator/ (Scalar lhs, const Derived& rhs) { + return rhs.unaryExpr(internal::bind1st_op<internal::scalar_quotient_op<Scalar> >(lhs)); } EIGEN_DEVICE_FUNC @@ -253,10 +341,13 @@ return cwiseMin(constant(threshold)); } - template <typename NewType> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorConversionOp<NewType, const Derived> + template<typename NewType> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const typename internal::conditional<internal::is_same<NewType, CoeffReturnType>::value, + Derived, + TensorConversionOp<NewType, const Derived> >::type cast() const { - return TensorConversionOp<NewType, const Derived>(derived()); + return choose(Cond<internal::is_same<NewType, CoeffReturnType>::value>(), derived(), TensorConversionOp<NewType, const Derived>(derived())); } EIGEN_DEVICE_FUNC @@ -277,7 +368,6 @@ return unaryExpr(internal::scalar_floor_op<Scalar>()); } - // Generic binary operation support. template <typename CustomBinaryOp, typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<CustomBinaryOp, const Derived, const OtherDerived> @@ -342,66 +432,66 @@ // Comparisons and tests. template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_LT>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LT>, const Derived, const OtherDerived> operator<(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_LT>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LT>()); } template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_LE>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LE>, const Derived, const OtherDerived> operator<=(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_LE>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LE>()); } template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_GT>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GT>, const Derived, const OtherDerived> operator>(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_GT>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GT>()); } template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_GE>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GE>, const Derived, const OtherDerived> operator>=(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_GE>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GE>()); } template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_EQ>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_EQ>, const Derived, const OtherDerived> operator==(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_EQ>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_EQ>()); } template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_NEQ>, const Derived, const OtherDerived> + const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_NEQ>, const Derived, const OtherDerived> operator!=(const OtherDerived& other) const { - return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_NEQ>()); + return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_NEQ>()); } // comparisons and tests for Scalars EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_LT>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LT>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator<(Scalar threshold) const { return operator<(constant(threshold)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_LE>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_LE>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator<=(Scalar threshold) const { return operator<=(constant(threshold)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_GT>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GT>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator>(Scalar threshold) const { return operator>(constant(threshold)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_GE>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_GE>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator>=(Scalar threshold) const { return operator>=(constant(threshold)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_EQ>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_EQ>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator==(Scalar threshold) const { return operator==(constant(threshold)); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_NEQ>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > + EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, Scalar, internal::cmp_NEQ>, const Derived, const TensorCwiseNullaryOp<internal::scalar_constant_op<Scalar>, const Derived> > operator!=(Scalar threshold) const { return operator!=(constant(threshold)); } @@ -434,9 +524,15 @@ typedef Eigen::IndexPair<Index> DimensionPair; template<typename OtherDerived, typename Dimensions> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorContractionOp<const Dimensions, const Derived, const OtherDerived> + const TensorContractionOp<const Dimensions, const Derived, const OtherDerived, const NoOpOutputKernel> contract(const OtherDerived& other, const Dimensions& dims) const { - return TensorContractionOp<const Dimensions, const Derived, const OtherDerived>(derived(), other.derived(), dims); + return TensorContractionOp<const Dimensions, const Derived, const OtherDerived, const NoOpOutputKernel>(derived(), other.derived(), dims); + } + + template<typename OtherDerived, typename Dimensions, typename OutputKernel> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorContractionOp<const Dimensions, const Derived, const OtherDerived, const OutputKernel> + contract(const OtherDerived& other, const Dimensions& dims, const OutputKernel& output_kernel) const { + return TensorContractionOp<const Dimensions, const Derived, const OtherDerived, const OutputKernel>(derived(), other.derived(), dims, output_kernel); } // Convolutions. @@ -449,8 +545,30 @@ // Fourier transforms template <int FFTDataType, int FFTDirection, typename FFT> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorFFTOp<const FFT, const Derived, FFTDataType, FFTDirection> - fft(const FFT& fft) const { - return TensorFFTOp<const FFT, const Derived, FFTDataType, FFTDirection>(derived(), fft); + fft(const FFT& dims) const { + return TensorFFTOp<const FFT, const Derived, FFTDataType, FFTDirection>(derived(), dims); + } + + // Scan. + typedef TensorScanOp<internal::SumReducer<CoeffReturnType>, const Derived> TensorScanSumOp; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorScanSumOp + cumsum(const Index& axis, bool exclusive = false) const { + return TensorScanSumOp(derived(), axis, exclusive); + } + + typedef TensorScanOp<internal::ProdReducer<CoeffReturnType>, const Derived> TensorScanProdOp; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorScanProdOp + cumprod(const Index& axis, bool exclusive = false) const { + return TensorScanProdOp(derived(), axis, exclusive); + } + + template <typename Reducer> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorScanOp<Reducer, const Derived> + scan(const Index& axis, const Reducer& reducer, bool exclusive = false) const { + return TensorScanOp<Reducer, const Derived>(derived(), axis, exclusive, reducer); } // Reductions. @@ -515,26 +633,26 @@ } template <typename Dims> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorReductionOp<internal::AndReducer, const Dims, const TensorConversionOp<bool, const Derived> > + const TensorReductionOp<internal::AndReducer, const Dims, const typename internal::conditional<internal::is_same<bool, CoeffReturnType>::value, Derived, TensorConversionOp<bool, const Derived> >::type > all(const Dims& dims) const { return cast<bool>().reduce(dims, internal::AndReducer()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorReductionOp<internal::AndReducer, const DimensionList<Index, NumDimensions>, const TensorConversionOp<bool, const Derived> > + const TensorReductionOp<internal::AndReducer, const DimensionList<Index, NumDimensions>, const typename internal::conditional<internal::is_same<bool, CoeffReturnType>::value, Derived, TensorConversionOp<bool, const Derived> >::type > all() const { DimensionList<Index, NumDimensions> in_dims; return cast<bool>().reduce(in_dims, internal::AndReducer()); } template <typename Dims> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorReductionOp<internal::OrReducer, const Dims, const TensorConversionOp<bool, const Derived> > + const TensorReductionOp<internal::OrReducer, const Dims, const typename internal::conditional<internal::is_same<bool, CoeffReturnType>::value, Derived, TensorConversionOp<bool, const Derived> >::type > any(const Dims& dims) const { return cast<bool>().reduce(dims, internal::OrReducer()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - const TensorReductionOp<internal::OrReducer, const DimensionList<Index, NumDimensions>, const TensorConversionOp<bool, const Derived> > + const TensorReductionOp<internal::OrReducer, const DimensionList<Index, NumDimensions>, const typename internal::conditional<internal::is_same<bool, CoeffReturnType>::value, Derived, TensorConversionOp<bool, const Derived> >::type > any() const { DimensionList<Index, NumDimensions> in_dims; return cast<bool>().reduce(in_dims, internal::OrReducer()); @@ -546,7 +664,7 @@ const array<Index, NumDimensions>, const Derived> argmax() const { array<Index, NumDimensions> in_dims; - for (int d = 0; d < NumDimensions; ++d) in_dims[d] = d; + for (Index d = 0; d < NumDimensions; ++d) in_dims[d] = d; return TensorTupleReducerOp< internal::ArgMaxTupleReducer<Tuple<Index, CoeffReturnType> >, const array<Index, NumDimensions>, @@ -559,7 +677,7 @@ const array<Index, NumDimensions>, const Derived> argmin() const { array<Index, NumDimensions> in_dims; - for (int d = 0; d < NumDimensions; ++d) in_dims[d] = d; + for (Index d = 0; d < NumDimensions; ++d) in_dims[d] = d; return TensorTupleReducerOp< internal::ArgMinTupleReducer<Tuple<Index, CoeffReturnType> >, const array<Index, NumDimensions>, @@ -570,7 +688,7 @@ const TensorTupleReducerOp< internal::ArgMaxTupleReducer<Tuple<Index, CoeffReturnType> >, const array<Index, 1>, const Derived> - argmax(const int return_dim) const { + argmax(const Index return_dim) const { array<Index, 1> in_dims; in_dims[0] = return_dim; return TensorTupleReducerOp< @@ -583,7 +701,7 @@ const TensorTupleReducerOp< internal::ArgMinTupleReducer<Tuple<Index, CoeffReturnType> >, const array<Index, 1>, const Derived> - argmin(const int return_dim) const { + argmin(const Index return_dim) const { array<Index, 1> in_dims; in_dims[0] = return_dim; return TensorTupleReducerOp< @@ -598,10 +716,22 @@ return TensorReductionOp<Reducer, const Dims, const Derived>(derived(), dims, reducer); } + template <typename Dims> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorTraceOp<const Dims, const Derived> + trace(const Dims& dims) const { + return TensorTraceOp<const Dims, const Derived>(derived(), dims); + } + + const TensorTraceOp<const DimensionList<Index, NumDimensions>, const Derived> + trace() const { + DimensionList<Index, NumDimensions> in_dims; + return TensorTraceOp<const DimensionList<Index, NumDimensions>, const Derived>(derived(), in_dims); + } + template <typename Broadcast> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorBroadcastingOp<const Broadcast, const Derived> - broadcast(const Broadcast& broadcast) const { - return TensorBroadcastingOp<const Broadcast, const Derived>(derived(), broadcast); + broadcast(const Broadcast& bcast) const { + return TensorBroadcastingOp<const Broadcast, const Derived>(derived(), bcast); } template <typename Axis, typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -676,6 +806,12 @@ slice(const StartIndices& startIndices, const Sizes& sizes) const { return TensorSlicingOp<const StartIndices, const Sizes, const Derived>(derived(), startIndices, sizes); } + template <typename StartIndices, typename StopIndices, typename Strides> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, const Derived> + stridedSlice(const StartIndices& startIndices, const StopIndices& stopIndices, const Strides& strides) const { + return TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, + const Derived>(derived(), startIndices, stopIndices, strides); + } template <Index DimId> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorChippingOp<DimId, const Derived> chip(const Index offset) const { @@ -703,8 +839,8 @@ } template <typename Shuffle> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorShufflingOp<const Shuffle, const Derived> - shuffle(const Shuffle& shuffle) const { - return TensorShufflingOp<const Shuffle, const Derived>(derived(), shuffle); + shuffle(const Shuffle& shfl) const { + return TensorShufflingOp<const Shuffle, const Derived>(derived(), shfl); } template <typename Strides> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorStridingOp<const Strides, const Derived> @@ -745,13 +881,14 @@ protected: template <typename Scalar, int NumIndices, int Options, typename IndexType> friend class Tensor; template <typename Scalar, typename Dimensions, int Option, typename IndexTypes> friend class TensorFixedSize; - template <typename OtherDerived, int AccessLevel> friend class TensorBase; + // the Eigen:: prefix is required to workaround a compilation issue with nvcc 9.0 + template <typename OtherDerived, int AccessLevel> friend class Eigen::TensorBase; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Derived& derived() const { return *static_cast<const Derived*>(this); } }; -template<typename Derived> -class TensorBase<Derived, WriteAccessors> : public TensorBase<Derived, ReadOnlyAccessors> { +template<typename Derived, int AccessLevel = internal::accessors_level<Derived>::value> +class TensorBase : public TensorBase<Derived, ReadOnlyAccessors> { public: typedef internal::traits<Derived> DerivedTraits; typedef typename DerivedTraits::Scalar Scalar; @@ -761,7 +898,8 @@ template <typename Scalar, int NumIndices, int Options, typename IndexType> friend class Tensor; template <typename Scalar, typename Dimensions, int Option, typename IndexTypes> friend class TensorFixedSize; - template <typename OtherDerived, int AccessLevel> friend class TensorBase; + // the Eigen:: prefix is required to workaround a compilation issue with nvcc 9.0 + template <typename OtherDerived, int OtherAccessLevel> friend class Eigen::TensorBase; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& setZero() { @@ -780,7 +918,7 @@ return derived() = this->template random<RandomGenerator>(); } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& setValues( const typename internal::Initializer<Derived, NumDimensions>::InitList& vals) { @@ -851,6 +989,19 @@ return TensorSlicingOp<const StartIndices, const Sizes, Derived>(derived(), startIndices, sizes); } + template <typename StartIndices, typename StopIndices, typename Strides> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, const Derived> + stridedSlice(const StartIndices& startIndices, const StopIndices& stopIndices, const Strides& strides) const { + return TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, + const Derived>(derived(), startIndices, stopIndices, strides); + } + template <typename StartIndices, typename StopIndices, typename Strides> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, Derived> + stridedSlice(const StartIndices& startIndices, const StopIndices& stopIndices, const Strides& strides) { + return TensorStridingSlicingOp<const StartIndices, const StopIndices, const Strides, + Derived>(derived(), startIndices, stopIndices, strides); + } + template <DenseIndex DimId> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorChippingOp<DimId, const Derived> chip(const Index offset) const { @@ -886,13 +1037,13 @@ template <typename Shuffle> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorShufflingOp<const Shuffle, const Derived> - shuffle(const Shuffle& shuffle) const { - return TensorShufflingOp<const Shuffle, const Derived>(derived(), shuffle); + shuffle(const Shuffle& shfl) const { + return TensorShufflingOp<const Shuffle, const Derived>(derived(), shfl); } template <typename Shuffle> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorShufflingOp<const Shuffle, Derived> - shuffle(const Shuffle& shuffle) { - return TensorShufflingOp<const Shuffle, Derived>(derived(), shuffle); + shuffle(const Shuffle& shfl) { + return TensorShufflingOp<const Shuffle, Derived>(derived(), shfl); } template <typename Strides> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -908,8 +1059,8 @@ // Select the device on which to evaluate the expression. template <typename DeviceType> - TensorDevice<Derived, DeviceType> device(const DeviceType& device) { - return TensorDevice<Derived, DeviceType>(device, derived()); + TensorDevice<Derived, DeviceType> device(const DeviceType& dev) { + return TensorDevice<Derived, DeviceType>(dev, derived()); } protected: @@ -918,7 +1069,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Derived& derived() const { return *static_cast<const Derived*>(this); } }; - +#endif // EIGEN_PARSED_BY_DOXYGEN } // end namespace Eigen #endif // EIGEN_CXX11_TENSOR_TENSOR_BASE_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorBlock.h b/unsupported/Eigen/CXX11/src/Tensor/TensorBlock.h new file mode 100644 index 0000000..91c77b0 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorBlock.h
@@ -0,0 +1,1169 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Andy Davis <andydavis@google.com> +// Copyright (C) 2018 Eugene Zhulenev <ezhulenev@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_BLOCK_H +#define EIGEN_CXX11_TENSOR_TENSOR_BLOCK_H + +namespace Eigen { +namespace internal { + +namespace { + +// Helper template to choose between ColMajor and RowMajor values. +template <int Layout> +struct cond; + +template <> +struct cond<ColMajor> { + template <typename T> + EIGEN_STRONG_INLINE const T& operator()(const T& col, + const T& /*row*/) const { + return col; + } +}; + +template <> +struct cond<RowMajor> { + template <typename T> + EIGEN_STRONG_INLINE const T& operator()(const T& /*col*/, + const T& row) const { + return row; + } +}; + +} // namespace + +/** + * \class TensorBlockShapeType + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block shape type. + * + * Tensor block shape type defines what are the shape preference for the blocks + * extracted from the larger tensor. + * + * Example: + * + * We want to extract blocks of 100 elements from the large 100x100 tensor: + * - tensor: 100x100 + * - target_block_size: 100 + * + * TensorBlockShapeType: + * - kUniformAllDims: 100 blocks of size 10x10 + * - kSkewedInnerDims: 100 blocks of size 100x1 (or 1x100 depending on a column + * or row major layout) + */ +enum TensorBlockShapeType { + kUniformAllDims, + kSkewedInnerDims +}; + +struct TensorOpResourceRequirements { + TensorBlockShapeType block_shape; + Index block_total_size; + // TODO(andydavis) Add 'target_num_threads' to support communication of + // thread-resource requirements. This will allow ops deep in the + // expression tree (like reductions) to communicate resources + // requirements based on local state (like the total number of reductions + // to be computed). + TensorOpResourceRequirements(TensorBlockShapeType shape, + const Index size) + : block_shape(shape), block_total_size(size) {} +}; + +// Tries to merge multiple resource requirements. +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MergeResourceRequirements( + const std::vector<TensorOpResourceRequirements>& resources, + TensorBlockShapeType* block_shape, Index* block_total_size) { + if (resources.empty()) { + return; + } + // TODO(andydavis) Implement different policies (i.e. revert to a default + // policy if block shapes/sizes conflict). + *block_shape = resources[0].block_shape; + *block_total_size = resources[0].block_total_size; + for (std::vector<TensorOpResourceRequirements>::size_type i = 1; i < resources.size(); ++i) { + if (resources[i].block_shape == kSkewedInnerDims && + *block_shape != kSkewedInnerDims) { + *block_shape = kSkewedInnerDims; + } + *block_total_size = + numext::maxi(*block_total_size, resources[i].block_total_size); + } +} + +/** + * \class TensorBlock + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block class. + * + * This class represents a tensor block specified by the index of the + * first block coefficient, and the size of the block in each dimension. + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout> +class TensorBlock { + public: + typedef DSizes<StorageIndex, NumDims> Dimensions; + + TensorBlock(const StorageIndex first_coeff_index, const Dimensions& block_sizes, + const Dimensions& block_strides, const Dimensions& tensor_strides, + Scalar* data) + : m_first_coeff_index(first_coeff_index), + m_block_sizes(block_sizes), + m_block_strides(block_strides), + m_tensor_strides(tensor_strides), + m_data(data) {} + + StorageIndex first_coeff_index() const { return m_first_coeff_index; } + + const Dimensions& block_sizes() const { return m_block_sizes; } + + const Dimensions& block_strides() const { return m_block_strides; } + + const Dimensions& tensor_strides() const { return m_tensor_strides; } + + Scalar* data() { return m_data; } + + const Scalar* data() const { return m_data; } + + private: + StorageIndex m_first_coeff_index; + Dimensions m_block_sizes; + Dimensions m_block_strides; + Dimensions m_tensor_strides; + Scalar* m_data; // Not owned. +}; + +template <typename Scalar, typename StorageIndex> +struct TensorBlockCopyOp { + + typedef typename packet_traits<Scalar>::type Packet; + enum { + Vectorizable = internal::packet_traits<Scalar>::Vectorizable, + PacketSize = internal::packet_traits<Scalar>::size + }; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const StorageIndex num_coeff_to_copy, const StorageIndex dst_index, + const StorageIndex dst_stride, Scalar* EIGEN_RESTRICT dst_data, + const StorageIndex src_index, const StorageIndex src_stride, + const Scalar* EIGEN_RESTRICT src_data) { + const Scalar* src = &src_data[src_index]; + Scalar* dst = &dst_data[dst_index]; + + if (!Vectorizable) { + for (Index i = 0; i < num_coeff_to_copy; ++i) { + dst[i * dst_stride] = src[i * src_stride]; + } + return; + } + + if (src_stride == 1) { + const StorageIndex vectorized_size = (num_coeff_to_copy / PacketSize) * PacketSize; + if (dst_stride == 1) { + // LINEAR + for (StorageIndex i = 0; i < vectorized_size; i += PacketSize) { + Packet p = internal::ploadu<Packet>(src + i); + internal::pstoreu<Scalar, Packet>(dst + i, p); + } + for (StorageIndex i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst[i] = src[i]; + } + } else { + // SCATTER + for (StorageIndex i = 0; i < vectorized_size; i += PacketSize) { + Packet p = internal::ploadu<Packet>(src + i); + internal::pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride); + } + for (StorageIndex i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst[i * dst_stride] = src[i]; + } + } + } else if (src_stride == 0) { + const StorageIndex vectorized_size = (num_coeff_to_copy / PacketSize) * PacketSize; + if (dst_stride == 1) { + // LINEAR + for (StorageIndex i = 0; i < vectorized_size; i += PacketSize) { + Packet p = internal::pload1<Packet>(src); + internal::pstoreu<Scalar, Packet>(dst + i, p); + } + for (StorageIndex i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst[i] = *src; + } + } else { + // SCATTER + for (StorageIndex i = 0; i < vectorized_size; i += PacketSize) { + Packet p = internal::pload1<Packet>(src); + internal::pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride); + } + for (StorageIndex i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst[i * dst_stride] = *src; + } + } + } else { + if (dst_stride == 1) { + // GATHER + const StorageIndex vectorized_size = (num_coeff_to_copy / PacketSize) * PacketSize; + for (StorageIndex i = 0; i < vectorized_size; i += PacketSize) { + Packet p = internal::pgather<Scalar, Packet>(src + i * src_stride, src_stride); + internal::pstoreu<Scalar, Packet>(dst + i, p); + } + for (StorageIndex i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst[i] = src[i * src_stride]; + } + } else { + // RANDOM + for (StorageIndex i = 0; i < num_coeff_to_copy; ++i) { + dst[i * dst_stride] = src[i * src_stride]; + } + } + } + } +}; + +/** + * \class TensorBlockIO + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block IO class. + * + * This class is responsible for copying data between a tensor and a tensor + * block. + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout, + bool BlockRead> +class TensorBlockIO { + public: + typedef TensorBlock<Scalar, StorageIndex, NumDims, Layout> Block; + typedef TensorBlockCopyOp<Scalar, StorageIndex> BlockCopyOp; + + protected: + typedef array<StorageIndex, NumDims> Dimensions; + + struct BlockIteratorState { + StorageIndex input_stride; + StorageIndex output_stride; + StorageIndex input_span; + StorageIndex output_span; + StorageIndex size; + StorageIndex count; + BlockIteratorState() + : input_stride(0), + output_stride(0), + input_span(0), + output_span(0), + size(0), + count(0) {} + }; + + // Compute how many inner dimensions it's allowed to squeeze when doing IO + // between a tensor and a block. It's safe to squeeze inner dimensions, only + // if they are not reordered. + static int NumSqueezableInnerDims(const Dimensions& tensor_to_block_dim_map) { + int num_squeezable_dims = 0; + if (Layout == ColMajor) { + for (int i = 0; i < NumDims; ++i) { + if (tensor_to_block_dim_map[i] == i) num_squeezable_dims++; + else break; + } + } else { + for (int i = NumDims - 1; i >= 0; --i) { + if (tensor_to_block_dim_map[i] == i) num_squeezable_dims++; + else break; + } + } + return num_squeezable_dims; + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Copy( + const Block& block, StorageIndex first_coeff_index, + const Dimensions& tensor_to_block_dim_map, + const Dimensions& tensor_strides, + const Scalar* src_data, + Scalar* dst_data) { + // Do not squeeze reordered inner dimensions. + int num_squeezable_dims = NumSqueezableInnerDims(tensor_to_block_dim_map); + + // Find the innermost tensor dimension whose size is not 1. This is the + // effective inner dim. If all dimensions are of size 1, then fallback to + // using the actual innermost dim to avoid out-of-bound access. + StorageIndex num_size_one_inner_dims = 0; + for (int i = 0; i < num_squeezable_dims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + if (block.block_sizes()[tensor_to_block_dim_map[dim]] != 1) { + num_size_one_inner_dims = i; + break; + } + } + + // Calculate strides and dimensions. + const StorageIndex tensor_stride1_dim = cond<Layout>()( + num_size_one_inner_dims, NumDims - num_size_one_inner_dims - 1); + const StorageIndex block_dim_for_tensor_stride1_dim = + NumDims == 0 ? 1 : tensor_to_block_dim_map[tensor_stride1_dim]; + StorageIndex block_inner_dim_size = + NumDims == 0 ? 1 + : block.block_sizes()[block_dim_for_tensor_stride1_dim]; + + // Squeeze multiple inner dims into one for larger inner dim size. + for (Index i = num_size_one_inner_dims + 1; i < num_squeezable_dims; ++i) { + const Index dim = cond<Layout>()(i, NumDims - i - 1); + const StorageIndex block_stride = + block.block_strides()[tensor_to_block_dim_map[dim]]; + if (block_inner_dim_size == block_stride && + block_stride == tensor_strides[dim]) { + block_inner_dim_size *= + block.block_sizes()[tensor_to_block_dim_map[dim]]; + ++num_size_one_inner_dims; + } else { + break; + } + } + + StorageIndex inputIndex; + StorageIndex outputIndex; + StorageIndex input_stride; + StorageIndex output_stride; + + // Setup strides to read/write along the tensor's stride1 dimension. + if (BlockRead) { + inputIndex = first_coeff_index; + outputIndex = 0; + input_stride = NumDims == 0 ? 1 : tensor_strides[tensor_stride1_dim]; + output_stride = + NumDims == 0 + ? 1 + : block.block_strides()[block_dim_for_tensor_stride1_dim]; + } else { + inputIndex = 0; + outputIndex = first_coeff_index; + input_stride = + NumDims == 0 + ? 1 + : block.block_strides()[block_dim_for_tensor_stride1_dim]; + output_stride = NumDims == 0 ? 1 : tensor_strides[tensor_stride1_dim]; + } + + const int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1; + array<BlockIteratorState, at_least_1_dim> block_iter_state; + + // Initialize block iterator state. Squeeze away any dimension of size 1. + Index num_squeezed_dims = 0; + for (Index i = num_size_one_inner_dims; i < NumDims - 1; ++i) { + const Index dim = cond<Layout>()(i + 1, NumDims - i - 2); + const StorageIndex size = block.block_sizes()[tensor_to_block_dim_map[dim]]; + if (size == 1) { + continue; + } + block_iter_state[num_squeezed_dims].size = size; + if (BlockRead) { + block_iter_state[num_squeezed_dims].input_stride = tensor_strides[dim]; + block_iter_state[num_squeezed_dims].output_stride = + block.block_strides()[tensor_to_block_dim_map[dim]]; + } else { + block_iter_state[num_squeezed_dims].input_stride = + block.block_strides()[tensor_to_block_dim_map[dim]]; + block_iter_state[num_squeezed_dims].output_stride = tensor_strides[dim]; + } + block_iter_state[num_squeezed_dims].input_span = + block_iter_state[num_squeezed_dims].input_stride * + (block_iter_state[num_squeezed_dims].size - 1); + block_iter_state[num_squeezed_dims].output_span = + block_iter_state[num_squeezed_dims].output_stride * + (block_iter_state[num_squeezed_dims].size - 1); + ++num_squeezed_dims; + } + + // Iterate copying data from src to dst. + const StorageIndex block_total_size = + NumDims == 0 ? 1 : block.block_sizes().TotalSize(); + for (StorageIndex i = 0; i < block_total_size; i += block_inner_dim_size) { + BlockCopyOp::Run(block_inner_dim_size, outputIndex, output_stride, + dst_data, inputIndex, input_stride, src_data); + // Update index. + for (int j = 0; j < num_squeezed_dims; ++j) { + if (++block_iter_state[j].count < block_iter_state[j].size) { + inputIndex += block_iter_state[j].input_stride; + outputIndex += block_iter_state[j].output_stride; + break; + } + block_iter_state[j].count = 0; + inputIndex -= block_iter_state[j].input_span; + outputIndex -= block_iter_state[j].output_span; + } + } + } +}; + +/** + * \class TensorBlockReader + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block reader class. + * + * This class is responsible for reading a tensor block. + * + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout> +class TensorBlockReader : public TensorBlockIO<Scalar, StorageIndex, NumDims, + Layout, /*BlockRead=*/true> { + public: + typedef TensorBlock<Scalar, StorageIndex, NumDims, Layout> Block; + typedef TensorBlockIO<Scalar, StorageIndex, NumDims, Layout, /*BlockRead=*/true> Base; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + Block* block, const Scalar* src_data) { + array<StorageIndex, NumDims> tensor_to_block_dim_map; + for (int i = 0; i < NumDims; ++i) { + tensor_to_block_dim_map[i] = i; + } + Base::Copy(*block, block->first_coeff_index(), tensor_to_block_dim_map, + block->tensor_strides(), src_data, block->data()); + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + Block* block, StorageIndex first_coeff_index, + const array<StorageIndex, NumDims>& tensor_to_block_dim_map, + const array<StorageIndex, NumDims>& tensor_strides, const Scalar* src_data) { + Base::Copy(*block, first_coeff_index, tensor_to_block_dim_map, + tensor_strides, src_data, block->data()); + } +}; + +/** + * \class TensorBlockWriter + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block writer class. + * + * This class is responsible for writing a tensor block. + * + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout> +class TensorBlockWriter : public TensorBlockIO<Scalar, StorageIndex, NumDims, + Layout, /*BlockRead=*/false> { + public: + typedef TensorBlock<Scalar, StorageIndex, NumDims, Layout> Block; + typedef TensorBlockIO<Scalar, StorageIndex, NumDims, Layout, /*BlockRead=*/false> Base; + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const Block& block, Scalar* dst_data) { + array<StorageIndex, NumDims> tensor_to_block_dim_map; + for (int i = 0; i < NumDims; ++i) { + tensor_to_block_dim_map[i] = i; + } + Base::Copy(block, block.first_coeff_index(), tensor_to_block_dim_map, + block.tensor_strides(), block.data(), dst_data); + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const Block& block, StorageIndex first_coeff_index, + const array<StorageIndex, NumDims>& tensor_to_block_dim_map, + const array<StorageIndex, NumDims>& tensor_strides, Scalar* dst_data) { + Base::Copy(block, first_coeff_index, tensor_to_block_dim_map, + tensor_strides, block.data(), dst_data); + } +}; + +/** + * \class TensorBlockCwiseUnaryOp + * \ingroup CXX11_Tensor_Module + * + * \brief Carries out a cwise binary op on a number of coefficients. + * + * This class reads strided input from the argument, and writes the + * result of the cwise unary op to the strided output array. + * + */ +struct TensorBlockCwiseUnaryOp { + template <typename StorageIndex, typename UnaryFunctor, + typename OutputScalar, typename InputScalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const UnaryFunctor& functor, const StorageIndex num_coeff, + const StorageIndex output_index, const StorageIndex output_stride, + OutputScalar* output_data, const StorageIndex input_index, + const StorageIndex input_stride, const InputScalar* input_data) { + typedef const Eigen::Array<InputScalar, Dynamic, 1> Input; + typedef Eigen::Array<OutputScalar, Dynamic, 1> Output; + + typedef Eigen::Map<Input, 0, InnerStride<> > InputMap; + typedef Eigen::Map<Output, 0, InnerStride<> > OutputMap; + + const InputScalar* input_base = &input_data[input_index]; + OutputScalar* output_base = &output_data[output_index]; + + const InputMap input(input_base, num_coeff, InnerStride<>(input_stride)); + OutputMap output(output_base, num_coeff, InnerStride<>(output_stride)); + + output = Eigen::CwiseUnaryOp<UnaryFunctor, InputMap>(input, functor); + } +}; + +/** + * \class TensorBlockCwiseUnaryIO + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block IO class for carrying out cwise unary ops. + * + * This class carries out the unary op on given blocks. + */ +template <typename UnaryFunctor, typename StorageIndex, typename OutputScalar, + int NumDims, int Layout> +struct TensorBlockCwiseUnaryIO { + typedef typename internal::TensorBlock<OutputScalar, StorageIndex, NumDims, + Layout>::Dimensions Dimensions; + + struct BlockIteratorState { + StorageIndex output_stride, output_span; + StorageIndex input_stride, input_span; + StorageIndex size, count; + }; + + template <typename InputScalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const UnaryFunctor& functor, const Dimensions& block_sizes, + const Dimensions& block_strides, OutputScalar* output_data, + const array<StorageIndex, NumDims>& input_strides, + const InputScalar* input_data) { + // Find the innermost dimension whose size is not 1. This is the effective + // inner dim. If all dimensions are of size 1, fallback to using the actual + // innermost dim to avoid out-of-bound access. + int num_size_one_inner_dims = 0; + for (int i = 0; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + if (block_sizes[dim] != 1) { + num_size_one_inner_dims = i; + break; + } + } + // Calculate strides and dimensions. + const int inner_dim = + NumDims == 0 ? 1 + : cond<Layout>()(num_size_one_inner_dims, + NumDims - num_size_one_inner_dims - 1); + StorageIndex inner_dim_size = NumDims == 0 ? 1 : block_sizes[inner_dim]; + for (int i = num_size_one_inner_dims + 1; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + // Merge multiple inner dims into one for larger inner dim size (i.e. + // fewer calls to TensorBlockCwiseUnaryOp::Run()). + if (inner_dim_size == block_strides[dim] && + block_strides[dim] == input_strides[dim]) { + inner_dim_size *= block_sizes[dim]; + ++num_size_one_inner_dims; + } else { + break; + } + } + + StorageIndex output_index = 0, input_index = 0; + + const StorageIndex output_stride = + NumDims == 0 ? 1 : block_strides[inner_dim]; + const StorageIndex input_stride = + NumDims == 0 ? 1 : input_strides[inner_dim]; + + const int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1; + array<BlockIteratorState, at_least_1_dim> block_iter_state; + + // Initialize block iterator state. Squeeze away any dimension of size 1. + int num_squeezed_dims = 0; + for (int i = num_size_one_inner_dims; i < NumDims - 1; ++i) { + const int dim = cond<Layout>()(i + 1, NumDims - i - 2); + const StorageIndex size = block_sizes[dim]; + if (size == 1) { + continue; + } + BlockIteratorState& state = block_iter_state[num_squeezed_dims]; + state.output_stride = block_strides[dim]; + state.input_stride = input_strides[dim]; + state.size = size; + state.output_span = state.output_stride * (size - 1); + state.input_span = state.input_stride * (size - 1); + state.count = 0; + ++num_squeezed_dims; + } + + // Compute cwise unary op. + const StorageIndex block_total_size = + NumDims == 0 ? 1 : block_sizes.TotalSize(); + for (StorageIndex i = 0; i < block_total_size; i += inner_dim_size) { + TensorBlockCwiseUnaryOp::Run(functor, inner_dim_size, output_index, + output_stride, output_data, input_index, + input_stride, input_data); + // Update index. + for (int j = 0; j < num_squeezed_dims; ++j) { + BlockIteratorState& state = block_iter_state[j]; + if (++state.count < state.size) { + output_index += state.output_stride; + input_index += state.input_stride; + break; + } + state.count = 0; + output_index -= state.output_span; + input_index -= state.input_span; + } + } + } +}; + +/** + * \class TensorBlockCwiseBinaryOp + * \ingroup CXX11_Tensor_Module + * + * \brief Carries out a cwise binary op on a number of coefficients. + * + * This class reads strided inputs from left and right operands, and writes the + * result of the cwise binary op to the strided output array. + * + */ +struct TensorBlockCwiseBinaryOp { + template <typename StorageIndex, typename BinaryFunctor, typename OutputScalar, + typename LeftScalar, typename RightScalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const BinaryFunctor& functor, const StorageIndex num_coeff, + const StorageIndex output_index, const StorageIndex output_stride, + OutputScalar* output_data, const StorageIndex left_index, + const StorageIndex left_stride, const LeftScalar* left_data, + const StorageIndex right_index, const StorageIndex right_stride, + const RightScalar* right_data) { + typedef const Array<LeftScalar, Dynamic, 1> Lhs; + typedef const Array<RightScalar, Dynamic, 1> Rhs; + typedef Array<OutputScalar, Dynamic, 1> Out; + + typedef Map<Lhs, 0, InnerStride<> > LhsMap; + typedef Map<Rhs, 0, InnerStride<> > RhsMap; + typedef Map<Out, 0, InnerStride<> > OutMap; + + const LeftScalar* lhs_base = &left_data[left_index]; + const RightScalar* rhs_base = &right_data[right_index]; + OutputScalar* out_base = &output_data[output_index]; + + const LhsMap lhs(lhs_base, num_coeff, InnerStride<>(left_stride)); + const RhsMap rhs(rhs_base, num_coeff, InnerStride<>(right_stride)); + OutMap out(out_base, num_coeff, InnerStride<>(output_stride)); + + out = CwiseBinaryOp<BinaryFunctor, LhsMap, RhsMap>(lhs, rhs, functor); + } +}; + +/** + * \class TensorBlockCwiseBinaryIO + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block IO class for carrying out cwise binary ops. + * + * This class carries out the binary op on given blocks. + * + */ +template <typename BinaryFunctor, typename StorageIndex, typename OutputScalar, + int NumDims, int Layout> +struct TensorBlockCwiseBinaryIO { + typedef typename TensorBlock<OutputScalar, StorageIndex, NumDims, Layout>::Dimensions Dimensions; + + struct BlockIteratorState { + StorageIndex output_stride, output_span; + StorageIndex left_stride, left_span; + StorageIndex right_stride, right_span; + StorageIndex size, count; + }; + + template <typename LeftScalar, typename RightScalar> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const BinaryFunctor& functor, const Dimensions& block_sizes, + const Dimensions& block_strides, OutputScalar* output_data, + const array<StorageIndex, NumDims>& left_strides, + const LeftScalar* left_data, + const array<StorageIndex, NumDims>& right_strides, + const RightScalar* right_data) { + // Find the innermost dimension whose size is not 1. This is the effective + // inner dim. If all dimensions are of size 1, fallback to using the actual + // innermost dim to avoid out-of-bound access. + int num_size_one_inner_dims = 0; + for (int i = 0; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + if (block_sizes[dim] != 1) { + num_size_one_inner_dims = i; + break; + } + } + // Calculate strides and dimensions. + const int inner_dim = + NumDims == 0 ? 1 + : cond<Layout>()(num_size_one_inner_dims, + NumDims - num_size_one_inner_dims - 1); + StorageIndex inner_dim_size = NumDims == 0 ? 1 : block_sizes[inner_dim]; + for (int i = num_size_one_inner_dims + 1; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + // Merge multiple inner dims into one for larger inner dim size (i.e. + // fewer calls to TensorBlockCwiseBinaryOp::Run()). + if (inner_dim_size == block_strides[dim] && + block_strides[dim] == left_strides[dim] && + block_strides[dim] == right_strides[dim]) { + inner_dim_size *= block_sizes[dim]; + ++num_size_one_inner_dims; + } else { + break; + } + } + + StorageIndex output_index = 0, left_index = 0, right_index = 0; + const StorageIndex output_stride = + NumDims == 0 ? 1 : block_strides[inner_dim]; + const StorageIndex left_stride = NumDims == 0 ? 1 : left_strides[inner_dim]; + const StorageIndex right_stride = + NumDims == 0 ? 1 : right_strides[inner_dim]; + + const int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1; + array<BlockIteratorState, at_least_1_dim> block_iter_state; + + // Initialize block iterator state. Squeeze away any dimension of size 1. + int num_squeezed_dims = 0; + for (int i = num_size_one_inner_dims; i < NumDims - 1; ++i) { + const int dim = cond<Layout>()(i + 1, NumDims - i - 2); + const StorageIndex size = block_sizes[dim]; + if (size == 1) { + continue; + } + BlockIteratorState& state = block_iter_state[num_squeezed_dims]; + state.output_stride = block_strides[dim]; + state.left_stride = left_strides[dim]; + state.right_stride = right_strides[dim]; + state.size = size; + state.output_span = state.output_stride * (size - 1); + state.left_span = state.left_stride * (size - 1); + state.right_span = state.right_stride * (size - 1); + state.count = 0; + ++num_squeezed_dims; + } + + // Compute cwise binary op. + const StorageIndex block_total_size = + NumDims == 0 ? 1 : block_sizes.TotalSize(); + for (StorageIndex i = 0; i < block_total_size; i += inner_dim_size) { + TensorBlockCwiseBinaryOp::Run(functor, inner_dim_size, output_index, + output_stride, output_data, left_index, + left_stride, left_data, right_index, + right_stride, right_data); + // Update index. + for (int j = 0; j < num_squeezed_dims; ++j) { + BlockIteratorState& state = block_iter_state[j]; + if (++state.count < state.size) { + output_index += state.output_stride; + left_index += state.left_stride; + right_index += state.right_stride; + break; + } + state.count = 0; + output_index -= state.output_span; + left_index -= state.left_span; + right_index -= state.right_span; + } + } + } +}; + +/** + * \class TensorBlockView + * \ingroup CXX11_Tensor_Module + * + * \brief Read-only view into a block of data. + * + * This class provides read-only access to a block of data in impl. It may need + * to allocate space for holding the intermediate result. + * + */ +template <class ArgType, class Device> +struct TensorBlockView { + typedef TensorEvaluator<ArgType, Device> Impl; + typedef typename Impl::Index StorageIndex; + typedef typename remove_const<typename Impl::Scalar>::type Scalar; + static const int NumDims = array_size<typename Impl::Dimensions>::value; + typedef DSizes<StorageIndex, NumDims> Dimensions; + + // Constructs a TensorBlockView for `impl`. `block` is only used for for + // specifying the start offset, shape, and strides of the block. + template <typename OtherTensorBlock> + TensorBlockView(const Device& device, + const TensorEvaluator<ArgType, Device>& impl, + const OtherTensorBlock& block) + : m_device(device), + m_block_sizes(block.block_sizes()), + m_data(NULL), + m_allocated_data(NULL) { + if (Impl::RawAccess && impl.data() != NULL) { + m_data = impl.data() + block.first_coeff_index(); + m_block_strides = block.tensor_strides(); + } else { + // Actually make a copy. + + // TODO(wuke): This sometimes put a lot pressure on the heap allocator. + // Consider allowing ops to request additional temporary block memory in + // TensorOpResourceRequirements. + m_allocated_data = static_cast<Scalar*>( + m_device.allocate(m_block_sizes.TotalSize() * sizeof(Scalar))); + m_data = m_allocated_data; + if (NumDims > 0) { + if (static_cast<int>(Impl::Layout) == static_cast<int>(ColMajor)) { + m_block_strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + m_block_strides[i] = m_block_strides[i - 1] * m_block_sizes[i - 1]; + } + } else { + m_block_strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + m_block_strides[i] = m_block_strides[i + 1] * m_block_sizes[i + 1]; + } + } + } + TensorBlock<Scalar, StorageIndex, NumDims, Impl::Layout> input_block( + block.first_coeff_index(), m_block_sizes, m_block_strides, + block.tensor_strides(), m_allocated_data); + impl.block(&input_block); + } + } + + ~TensorBlockView() { + if (m_allocated_data != NULL) { + m_device.deallocate(m_allocated_data); + } + } + + const Dimensions& block_sizes() const { return m_block_sizes; } + const Dimensions& block_strides() const { return m_block_strides; } + const Scalar* data() const { return m_data; } + + private: + const Device& m_device; + Dimensions m_block_sizes, m_block_strides; + const Scalar* m_data; // Not owned. + Scalar* m_allocated_data; // Owned. +}; + +/** + * \class TensorBlockMapper + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor block mapper class. + * + * This class is responsible for iterating over the blocks of a tensor. + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout> +class TensorBlockMapper { + public: + typedef TensorBlock<Scalar, StorageIndex, NumDims, Layout> Block; + typedef DSizes<StorageIndex, NumDims> Dimensions; + + TensorBlockMapper(const Dimensions& dims, + const TensorBlockShapeType block_shape, + Index min_target_size) + : m_dimensions(dims), + m_block_dim_sizes(BlockDimensions(dims, block_shape, internal::convert_index<StorageIndex>(min_target_size))) { + // Calculate block counts by dimension and total block count. + DSizes<StorageIndex, NumDims> block_count; + for (Index i = 0; i < block_count.rank(); ++i) { + block_count[i] = divup(m_dimensions[i], m_block_dim_sizes[i]); + } + m_total_block_count = array_prod(block_count); + + // Calculate block strides (used for enumerating blocks). + if (NumDims > 0) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_block_strides[0] = 1; + m_tensor_strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + m_block_strides[i] = m_block_strides[i - 1] * block_count[i - 1]; + m_tensor_strides[i] = m_tensor_strides[i - 1] * m_dimensions[i - 1]; + } + } else { + m_block_strides[NumDims - 1] = 1; + m_tensor_strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + m_block_strides[i] = m_block_strides[i + 1] * block_count[i + 1]; + m_tensor_strides[i] = m_tensor_strides[i + 1] * m_dimensions[i + 1]; + } + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Block + GetBlockForIndex(StorageIndex block_index, Scalar* data) const { + StorageIndex first_coeff_index = 0; + DSizes<StorageIndex, NumDims> coords; + DSizes<StorageIndex, NumDims> sizes; + DSizes<StorageIndex, NumDims> strides; + if (NumDims > 0) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumDims - 1; i > 0; --i) { + const StorageIndex idx = block_index / m_block_strides[i]; + coords[i] = idx * m_block_dim_sizes[i]; + sizes[i] = + numext::mini((m_dimensions[i] - coords[i]), m_block_dim_sizes[i]); + block_index -= idx * m_block_strides[i]; + first_coeff_index += coords[i] * m_tensor_strides[i]; + } + coords[0] = block_index * m_block_dim_sizes[0]; + sizes[0] = + numext::mini((m_dimensions[0] - coords[0]), m_block_dim_sizes[0]); + first_coeff_index += coords[0] * m_tensor_strides[0]; + + strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + strides[i] = strides[i - 1] * sizes[i - 1]; + } + } else { + for (int i = 0; i < NumDims - 1; ++i) { + const StorageIndex idx = block_index / m_block_strides[i]; + coords[i] = idx * m_block_dim_sizes[i]; + sizes[i] = + numext::mini((m_dimensions[i] - coords[i]), m_block_dim_sizes[i]); + block_index -= idx * m_block_strides[i]; + first_coeff_index += coords[i] * m_tensor_strides[i]; + } + coords[NumDims - 1] = block_index * m_block_dim_sizes[NumDims - 1]; + sizes[NumDims - 1] = + numext::mini((m_dimensions[NumDims - 1] - coords[NumDims - 1]), + m_block_dim_sizes[NumDims - 1]); + first_coeff_index += + coords[NumDims - 1] * m_tensor_strides[NumDims - 1]; + + strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + strides[i] = strides[i + 1] * sizes[i + 1]; + } + } + } + + return Block(first_coeff_index, sizes, strides, m_tensor_strides, data); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex total_block_count() const { + return m_total_block_count; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex + block_dims_total_size() const { + return m_block_dim_sizes.TotalSize(); + } + + private: + static Dimensions BlockDimensions(const Dimensions& tensor_dims, + const TensorBlockShapeType block_shape, + StorageIndex min_target_size) { + min_target_size = numext::maxi<StorageIndex>(1, min_target_size); + + // If tensor fully fits into the target size, we'll treat it a single block. + Dimensions block_dim_sizes = tensor_dims; + + if (tensor_dims.TotalSize() == 0) { + // Corner case: one of the dimensions is zero. Logic below is too complex + // to handle this case on a general basis, just use unit block size. + // Note: we must not yield blocks with zero dimensions (recipe for + // overflows/underflows, divisions by zero and NaNs later). + for (int i = 0; i < NumDims; ++i) { + block_dim_sizes[i] = 1; + } + } else if (block_dim_sizes.TotalSize() > min_target_size) { + if (block_shape == kUniformAllDims) { + // Tensor will not fit within 'min_target_size' budget: calculate tensor + // block dimension sizes based on "square" dimension size target. + const StorageIndex dim_size_target = internal::convert_index<StorageIndex>( + std::pow(static_cast<float>(min_target_size), + 1.0f / static_cast<float>(block_dim_sizes.rank()))); + for (Index i = 0; i < block_dim_sizes.rank(); ++i) { + // TODO(andydavis) Adjust the inner most 'block_dim_size' to make it + // a multiple of the packet size. Note that reducing + // 'block_dim_size' in this manner can increase the number of + // blocks, and so will amplify any per-block overhead. + block_dim_sizes[i] = numext::mini(dim_size_target, tensor_dims[i]); + } + // Add any un-allocated coefficients to inner dimension(s). + StorageIndex total_size = block_dim_sizes.TotalSize(); + for (int i = 0; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + if (block_dim_sizes[dim] < tensor_dims[dim]) { + const StorageIndex total_size_other_dims = + total_size / block_dim_sizes[dim]; + const StorageIndex alloc_avail = + divup<StorageIndex>(min_target_size, total_size_other_dims); + if (alloc_avail == block_dim_sizes[dim]) { + // Insufficient excess coefficients to allocate. + break; + } + block_dim_sizes[dim] = numext::mini(tensor_dims[dim], alloc_avail); + total_size = total_size_other_dims * block_dim_sizes[dim]; + } + } + } else if (block_shape == kSkewedInnerDims) { + StorageIndex coeff_to_allocate = min_target_size; + for (int i = 0; i < NumDims; ++i) { + const int dim = cond<Layout>()(i, NumDims - i - 1); + block_dim_sizes[dim] = + numext::mini(coeff_to_allocate, tensor_dims[dim]); + coeff_to_allocate = divup( + coeff_to_allocate, + numext::maxi(static_cast<StorageIndex>(1), block_dim_sizes[dim])); + } + eigen_assert(coeff_to_allocate == 1); + } else { + eigen_assert(false); // someone added new block shape type + } + } + + eigen_assert( + block_dim_sizes.TotalSize() >= + numext::mini<Index>(min_target_size, tensor_dims.TotalSize())); + + return block_dim_sizes; + } + + Dimensions m_dimensions; + Dimensions m_block_dim_sizes; + Dimensions m_block_strides; + Dimensions m_tensor_strides; + StorageIndex m_total_block_count; +}; + +/** + * \class TensorSliceBlockMapper + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor slice block mapper class. + * + * This class is responsible for iterating over the blocks of + * a slice of a tensor. Supports shuffling of the block strides + * for callers that want to reduce strides for dimensions to be + * processed together. + * + */ +template <typename Scalar, typename StorageIndex, int NumDims, int Layout> +class TensorSliceBlockMapper { + public: + typedef TensorBlock<Scalar, StorageIndex, NumDims, Layout> Block; + typedef DSizes<StorageIndex, NumDims> Dimensions; + + TensorSliceBlockMapper(const Dimensions& tensor_dims, + const Dimensions& tensor_slice_offsets, + const Dimensions& tensor_slice_extents, + const Dimensions& block_dim_sizes, + const Dimensions& block_stride_order) + : m_tensor_dimensions(tensor_dims), + m_tensor_slice_offsets(tensor_slice_offsets), + m_tensor_slice_extents(tensor_slice_extents), + m_block_dim_sizes(block_dim_sizes), + m_block_stride_order(block_stride_order), + m_total_block_count(1) { + // Calculate block counts by dimension and total block count. + DSizes<StorageIndex, NumDims> block_count; + for (Index i = 0; i < block_count.rank(); ++i) { + block_count[i] = divup(m_tensor_slice_extents[i], m_block_dim_sizes[i]); + } + m_total_block_count = array_prod(block_count); + + // Calculate block strides (used for enumerating blocks). + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_block_strides[0] = 1; + m_tensor_strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + m_block_strides[i] = m_block_strides[i - 1] * block_count[i - 1]; + m_tensor_strides[i] = + m_tensor_strides[i - 1] * m_tensor_dimensions[i - 1]; + } + } else { + m_block_strides[NumDims - 1] = 1; + m_tensor_strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + m_block_strides[i] = m_block_strides[i + 1] * block_count[i + 1]; + m_tensor_strides[i] = + m_tensor_strides[i + 1] * m_tensor_dimensions[i + 1]; + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Block + GetBlockForIndex(StorageIndex block_index, Scalar* data) const { + StorageIndex first_coeff_index = 0; + DSizes<StorageIndex, NumDims> coords; + DSizes<StorageIndex, NumDims> sizes; + DSizes<StorageIndex, NumDims> strides; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumDims - 1; i > 0; --i) { + const Index idx = block_index / m_block_strides[i]; + coords[i] = m_tensor_slice_offsets[i] + idx * m_block_dim_sizes[i]; + sizes[i] = numext::mini( + m_tensor_slice_offsets[i] + m_tensor_slice_extents[i] - coords[i], + m_block_dim_sizes[i]); + block_index -= idx * m_block_strides[i]; + first_coeff_index += coords[i] * m_tensor_strides[i]; + } + coords[0] = + m_tensor_slice_offsets[0] + block_index * m_block_dim_sizes[0]; + sizes[0] = numext::mini( + m_tensor_slice_offsets[0] + m_tensor_slice_extents[0] - coords[0], + m_block_dim_sizes[0]); + first_coeff_index += coords[0] * m_tensor_strides[0]; + + StorageIndex prev_dim = m_block_stride_order[0]; + strides[prev_dim] = 1; + for (int i = 1; i < NumDims; ++i) { + const StorageIndex curr_dim = m_block_stride_order[i]; + strides[curr_dim] = strides[prev_dim] * sizes[prev_dim]; + prev_dim = curr_dim; + } + } else { + for (int i = 0; i < NumDims - 1; ++i) { + const StorageIndex idx = block_index / m_block_strides[i]; + coords[i] = m_tensor_slice_offsets[i] + idx * m_block_dim_sizes[i]; + sizes[i] = numext::mini( + m_tensor_slice_offsets[i] + m_tensor_slice_extents[i] - coords[i], + m_block_dim_sizes[i]); + block_index -= idx * m_block_strides[i]; + first_coeff_index += coords[i] * m_tensor_strides[i]; + } + coords[NumDims - 1] = m_tensor_slice_offsets[NumDims - 1] + + block_index * m_block_dim_sizes[NumDims - 1]; + sizes[NumDims - 1] = numext::mini( + m_tensor_slice_offsets[NumDims - 1] + + m_tensor_slice_extents[NumDims - 1] - coords[NumDims - 1], + m_block_dim_sizes[NumDims - 1]); + first_coeff_index += coords[NumDims - 1] * m_tensor_strides[NumDims - 1]; + + StorageIndex prev_dim = m_block_stride_order[NumDims - 1]; + strides[prev_dim] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + const StorageIndex curr_dim = m_block_stride_order[i]; + strides[curr_dim] = strides[prev_dim] * sizes[prev_dim]; + prev_dim = curr_dim; + } + } + + return Block(first_coeff_index, sizes, strides, m_tensor_strides, data); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageIndex total_block_count() const { + return m_total_block_count; + } + + private: + Dimensions m_tensor_dimensions; + Dimensions m_tensor_slice_offsets; + Dimensions m_tensor_slice_extents; + Dimensions m_tensor_strides; + Dimensions m_block_dim_sizes; + Dimensions m_block_stride_order; + Dimensions m_block_strides; + StorageIndex m_total_block_count; +}; + +} // namespace internal + +} // namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_BLOCK_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h b/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h index c771496..c102a43 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename Broadcast, typename XprType> @@ -54,7 +55,7 @@ static const bool value = true; }; #ifndef EIGEN_EMULATE_CXX11_META_H -template <typename std::size_t... Indices> +template <typename std::ptrdiff_t... Indices> struct is_input_scalar<Sizes<Indices...> > { static const bool value = (Sizes<Indices...>::total_size == 1); }; @@ -103,27 +104,52 @@ typedef typename TensorEvaluator<ArgType, Device>::Dimensions InputDimensions; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + bool isCopy, nByOne, oneByN; enum { - IsAligned = false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - RawAccess = false + IsAligned = true, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false }; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device) + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + // Block based access to the XprType (input) tensor. + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + TensorBlock; + typedef internal::TensorBlockReader<ScalarNoConst, Index, NumDims, Layout> + TensorBlockReader; + + // We do block based broadcasting using a trick with 2x tensor rank and 0 + // strides. See block method implementation for details. + typedef DSizes<Index, 2 * NumDims> BroadcastDimensions; + typedef internal::TensorBlock<ScalarNoConst, Index, 2 * NumDims, Layout> + BroadcastTensorBlock; + typedef internal::TensorBlockReader<ScalarNoConst, Index, 2 * NumDims, Layout> + BroadcastTensorBlockReader; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, + const Device& device) + : isCopy(false), nByOne(false), oneByN(false), + m_device(device), m_broadcast(op.broadcast()), m_impl(op.expression(), device) { + // The broadcasting op doesn't change the rank of the tensor. One can't broadcast a scalar // and store the result in a scalar. Instead one should reshape the scalar into a a N-D // tensor with N >= 1 of 1 element first and then broadcast. - EIGEN_STATIC_ASSERT(NumDims > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); const InputDimensions& input_dims = m_impl.dimensions(); - const Broadcast& broadcast = op.broadcast(); + isCopy = true; for (int i = 0; i < NumDims; ++i) { eigen_assert(input_dims[i] > 0); - m_dimensions[i] = input_dims[i] * broadcast[i]; + m_dimensions[i] = input_dims[i] * m_broadcast[i]; + if (m_broadcast[i] != 1) { + isCopy = false; + } } if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -141,6 +167,40 @@ m_outputStrides[i] = m_outputStrides[i+1] * m_dimensions[i+1]; } } + + if (input_dims[0] == 1) { + oneByN = true; + for (int i = 1; i < NumDims; ++i) { + if (m_broadcast[i] != 1) { + oneByN = false; + break; + } + } + } else if (input_dims[NumDims-1] == 1) { + nByOne = true; + for (int i = 0; i < NumDims-1; ++i) { + if (m_broadcast[i] != 1) { + nByOne = false; + break; + } + } + } + + // Handle special format like NCHW, its input shape is '[1, N..., 1]' and + // broadcast shape is '[N, 1..., N]' + if (!oneByN && !nByOne) { + if (input_dims[0] == 1 && input_dims[NumDims-1] == 1 && NumDims > 2) { + nByOne = true; + oneByN = true; + for (int i = 1; i < NumDims-1; ++i) { + if (m_broadcast[i] != 1) { + nByOne = false; + oneByN = false; + break; + } + } + } + } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -161,15 +221,22 @@ } if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - return coeffColMajor(index); + if (isCopy) { + return m_impl.coeff(index); + } else { + return coeffColMajor(index); + } } else { - return coeffRowMajor(index); + if (isCopy) { + return m_impl.coeff(index); + } else { + return coeffRowMajor(index); + } } } // TODO: attempt to speed this up. The integer divisions and modulo are slow - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffColMajor(Index index) const - { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index indexColMajor(Index index) const { Index inputIndex = 0; for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -195,11 +262,15 @@ inputIndex += (index % m_impl.dimensions()[0]); } } - return m_impl.coeff(inputIndex); + return inputIndex; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffRowMajor(Index index) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffColMajor(Index index) const { + return m_impl.coeff(indexColMajor(index)); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index indexRowMajor(Index index) const { Index inputIndex = 0; for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; @@ -215,17 +286,22 @@ } index -= idx * m_outputStrides[i]; } - if (internal::index_statically_eq<Broadcast>(NumDims-1, 1)) { - eigen_assert(index < m_impl.dimensions()[NumDims-1]); + if (internal::index_statically_eq<Broadcast>(NumDims - 1, 1)) { + eigen_assert(index < m_impl.dimensions()[NumDims - 1]); inputIndex += index; } else { - if (internal::index_statically_eq<InputDimensions>(NumDims-1, 1)) { - eigen_assert(index % m_impl.dimensions()[NumDims-1] == 0); + if (internal::index_statically_eq<InputDimensions>(NumDims - 1, 1)) { + eigen_assert(index % m_impl.dimensions()[NumDims - 1] == 0); } else { - inputIndex += (index % m_impl.dimensions()[NumDims-1]); + inputIndex += (index % m_impl.dimensions()[NumDims - 1]); } } - return m_impl.coeff(inputIndex); + return inputIndex; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffRowMajor(Index index) const + { + return m_impl.coeff(indexRowMajor(index)); } template<int LoadMode> @@ -236,9 +312,145 @@ } if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - return packetColMajor<LoadMode>(index); + if (isCopy) { + #ifdef EIGEN_GPU_COMPILE_PHASE + // See PR 437: on NVIDIA P100 and K20m we observed a x3-4 speed up by enforcing + // unaligned loads here. The reason is unclear though. + return m_impl.template packet<Unaligned>(index); + #else + return m_impl.template packet<LoadMode>(index); + #endif + } else if (oneByN && !nByOne) { + return packetNByOne<LoadMode>(index); + } else if (!oneByN && nByOne) { + return packetOneByN<LoadMode>(index); + } else if (oneByN && nByOne) { + return packetOneByNByOne<LoadMode>(index); + } else { + return packetColMajor<LoadMode>(index); + } } else { - return packetRowMajor<LoadMode>(index); + if (isCopy) { + #ifdef EIGEN_GPU_COMPILE_PHASE + // See above. + return m_impl.template packet<Unaligned>(index); + #else + return m_impl.template packet<LoadMode>(index); + #endif + } else if (oneByN && !nByOne) { + return packetOneByN<LoadMode>(index); + } else if (!oneByN && nByOne) { + return packetNByOne<LoadMode>(index); + } else if (oneByN && nByOne) { + return packetOneByNByOne<LoadMode>(index); + } else { + return packetRowMajor<LoadMode>(index); + } + } + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetOneByNByOne + (Index index) const + { + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); + + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + Index startDim, endDim; + Index inputIndex, outputOffset, batchedIndex; + + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + startDim = NumDims - 1; + endDim = 1; + } else { + startDim = 0; + endDim = NumDims - 2; + } + + batchedIndex = index % m_outputStrides[startDim]; + inputIndex = batchedIndex / m_outputStrides[endDim]; + outputOffset = batchedIndex % m_outputStrides[endDim]; + + if (outputOffset + PacketSize <= m_outputStrides[endDim]) { + values[0] = m_impl.coeff(inputIndex); + return internal::pload1<PacketReturnType>(values); + } else { + for (int i = 0, cur = 0; i < PacketSize; ++i, ++cur) { + if (outputOffset + cur < m_outputStrides[endDim]) { + values[i] = m_impl.coeff(inputIndex); + } else { + ++inputIndex; + inputIndex = (inputIndex == m_inputStrides[startDim] ? 0 : inputIndex); + values[i] = m_impl.coeff(inputIndex); + outputOffset = 0; + cur = 0; + } + } + return internal::pload<PacketReturnType>(values); + } + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetOneByN(Index index) const + { + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); + + Index dim, inputIndex; + + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + dim = NumDims - 1; + } else { + dim = 0; + } + + inputIndex = index % m_inputStrides[dim]; + if (inputIndex + PacketSize <= m_inputStrides[dim]) { + return m_impl.template packet<Unaligned>(inputIndex); + } else { + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + if (inputIndex > m_inputStrides[dim]-1) { + inputIndex = 0; + } + values[i] = m_impl.coeff(inputIndex++); + } + return internal::pload<PacketReturnType>(values); + } + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetNByOne(Index index) const + { + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); + + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + Index dim, inputIndex, outputOffset; + + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + dim = 1; + } else { + dim = NumDims - 2; + } + + inputIndex = index / m_outputStrides[dim]; + outputOffset = index % m_outputStrides[dim]; + if (outputOffset + PacketSize <= m_outputStrides[dim]) { + values[0] = m_impl.coeff(inputIndex); + return internal::pload1<PacketReturnType>(values); + } else { + for (int i = 0, cur = 0; i < PacketSize; ++i, ++cur) { + if (outputOffset + cur < m_outputStrides[dim]) { + values[i] = m_impl.coeff(inputIndex); + } else { + values[i] = m_impl.coeff(++inputIndex); + outputOffset = 0; + cur = 0; + } + } + return internal::pload<PacketReturnType>(values); } } @@ -247,7 +459,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetColMajor(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); const Index originalIndex = index; @@ -289,7 +501,11 @@ EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; values[0] = m_impl.coeff(inputIndex); for (int i = 1; i < PacketSize; ++i) { - values[i] = coeffColMajor(originalIndex+i); + if (innermostLoc + i < m_impl.dimensions()[0]) { + values[i] = m_impl.coeff(inputIndex+i); + } else { + values[i] = coeffColMajor(originalIndex+i); + } } PacketReturnType rslt = internal::pload<PacketReturnType>(values); return rslt; @@ -299,7 +515,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetRowMajor(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); const Index originalIndex = index; @@ -341,7 +557,11 @@ EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; values[0] = m_impl.coeff(inputIndex); for (int i = 1; i < PacketSize; ++i) { - values[i] = coeffRowMajor(originalIndex+i); + if (innermostLoc + i < m_impl.dimensions()[NumDims-1]) { + values[i] = m_impl.coeff(inputIndex+i); + } else { + values[i] = coeffRowMajor(originalIndex+i); + } } PacketReturnType rslt = internal::pload<PacketReturnType>(values); return rslt; @@ -351,14 +571,14 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { double compute_cost = TensorOpCost::AddCost<Index>(); - if (NumDims > 0) { + if (!isCopy && NumDims > 0) { for (int i = NumDims - 1; i > 0; --i) { compute_cost += TensorOpCost::DivCost<Index>(); - if (internal::index_statically_eq<Broadcast>()(i, 1)) { + if (internal::index_statically_eq<Broadcast>(i, 1)) { compute_cost += TensorOpCost::MulCost<Index>() + TensorOpCost::AddCost<Index>(); } else { - if (!internal::index_statically_eq<InputDimensions>()(i, 1)) { + if (!internal::index_statically_eq<InputDimensions>(i, 1)) { compute_cost += TensorOpCost::MulCost<Index>() + TensorOpCost::ModCost<Index>() + TensorOpCost::AddCost<Index>(); @@ -372,9 +592,291 @@ TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + // TODO(wuke): Targeting L1 size is 30% faster than targeting L{-1} on large + // tensors. But this might need further tuning. + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.firstLevelCacheSize() / sizeof(Scalar)); + + resources->push_back(internal::TensorOpResourceRequirements( + internal::kSkewedInnerDims, block_total_size_max)); + + m_impl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + TensorBlock* output_block) const { + if (NumDims <= 0) { + output_block->data()[0] = m_impl.coeff(0); + return; + } + + // Because we only support kSkewedInnerDims blocking, block size should be + // equal to m_dimensions for inner dims, a smaller than m_dimensions[i] size + // for the first outer dim, and 1 for other outer dims. This is guaranteed + // by MergeResourceRequirements() in TensorBlock.h. + const Dimensions& output_block_sizes = output_block->block_sizes(); + const Dimensions& output_block_strides = output_block->block_strides(); + + // Find where outer dims start. + int outer_dim_start = 0; + Index outer_dim_size = 1, inner_dim_size = 1; + for (int i = 0; i < NumDims; ++i) { + const int dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i + : NumDims - i - 1; + if (i > outer_dim_start) { + eigen_assert(output_block_sizes[dim] == 1); + } else if (output_block_sizes[dim] != m_dimensions[dim]) { + eigen_assert(output_block_sizes[dim] < m_dimensions[dim]); + outer_dim_size = output_block_sizes[dim]; + } else { + inner_dim_size *= output_block_sizes[dim]; + ++outer_dim_start; + } + } + + if (inner_dim_size == 0 || outer_dim_size == 0) { + return; + } + + const Dimensions& input_dims = Dimensions(m_impl.dimensions()); + + // Pre-fill input_block_sizes, broadcast_block_sizes, + // broadcast_block_strides, and broadcast_tensor_strides. Later on we will + // only modify the outer_dim_start-th dimension on these arrays. + + // Calculate the input block size for looking into the input. + Dimensions input_block_sizes; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = 0; i < outer_dim_start; ++i) { + input_block_sizes[i] = input_dims[i]; + } + for (int i = outer_dim_start; i < NumDims; ++i) { + input_block_sizes[i] = 1; + } + } else { + for (int i = 0; i < outer_dim_start; ++i) { + input_block_sizes[NumDims - i - 1] = input_dims[NumDims - i - 1]; + } + for (int i = outer_dim_start; i < NumDims; ++i) { + input_block_sizes[NumDims - i - 1] = 1; + } + } + + // Broadcast with the 0-stride trick: Create 1 extra dim for each + // broadcast, set the input stride to 0. + // + // When ColMajor: + // - broadcast_block_sizes is [d_0, b_0, d_1, b_1, ...]. + // + // - broadcast_block_strides is [output_block_strides[0], + // output_block_strides[0] * d_0, + // output_block_strides[1], + // output_block_strides[1] * d_1, + // ...]. + // + // - broadcast_tensor_strides is [output_block_strides[0], + // 0, + // output_block_strides[1], + // 0, + // ...]. + BroadcastDimensions broadcast_block_sizes, broadcast_block_strides, + broadcast_tensor_strides; + + for (int i = 0; i < outer_dim_start; ++i) { + const int dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i + : NumDims - i - 1; + const int copy_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 2 * i + : 2 * NumDims - 2 * i - 1; + const int broadcast_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) ? copy_dim + 1 + : copy_dim - 1; + broadcast_block_sizes[copy_dim] = input_dims[dim]; + broadcast_block_sizes[broadcast_dim] = m_broadcast[dim]; + broadcast_block_strides[copy_dim] = output_block_strides[dim]; + broadcast_block_strides[broadcast_dim] = + output_block_strides[dim] * input_dims[dim]; + broadcast_tensor_strides[copy_dim] = m_inputStrides[dim]; + broadcast_tensor_strides[broadcast_dim] = 0; + } + for (int i = 2 * outer_dim_start; i < 2 * NumDims; ++i) { + const int dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i + : 2 * NumDims - i - 1; + broadcast_block_sizes[dim] = 1; + broadcast_block_strides[dim] = 0; + broadcast_tensor_strides[dim] = 0; + } + + const int outer_dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? outer_dim_start + : NumDims - outer_dim_start - 1; + + if (outer_dim_size == 1) { + // We just need one block read using the ready-set values above. + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, 0, + output_block); + } else if (input_dims[outer_dim] == 1) { + // Broadcast outer_dim_start-th dimension (< NumDims) by outer_dim_size. + const int broadcast_outer_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 2 * outer_dim_start + 1 + : 2 * NumDims - 2 * outer_dim_start - 2; + broadcast_block_sizes[broadcast_outer_dim] = outer_dim_size; + broadcast_tensor_strides[broadcast_outer_dim] = 0; + broadcast_block_strides[broadcast_outer_dim] = + output_block_strides[outer_dim]; + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, 0, + output_block); + } else { + // The general case. Let's denote the output block as x[..., + // a:a+outer_dim_size, :, ..., :], where a:a+outer_dim_size is a slice on + // the outer_dim_start-th dimension (< NumDims). We need to split the + // a:a+outer_dim_size into possibly 3 sub-blocks: + // + // (1) a:b, where b is the smallest multiple of + // input_dims[outer_dim_start] in [a, a+outer_dim_size]. + // + // (2) b:c, where c is the largest multiple of input_dims[outer_dim_start] + // in [a, a+outer_dim_size]. + // + // (3) c:a+outer_dim_size . + // + // Or, when b and c do not exist, we just need to process the whole block + // together. + + // Find a. + const Index outer_dim_left_index = + output_block->first_coeff_index() / m_outputStrides[outer_dim]; + + // Find b and c. + const Index input_outer_dim_size = input_dims[outer_dim]; + + // First multiple after a. This is b when <= outer_dim_left_index + + // outer_dim_size. + const Index first_multiple = + divup<Index>(outer_dim_left_index, input_outer_dim_size) * + input_outer_dim_size; + + if (first_multiple <= outer_dim_left_index + outer_dim_size) { + // b exists, so does c. Find it. + const Index last_multiple = (outer_dim_left_index + outer_dim_size) / + input_outer_dim_size * input_outer_dim_size; + const int copy_outer_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 2 * outer_dim_start + : 2 * NumDims - 2 * outer_dim_start - 1; + const int broadcast_outer_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 2 * outer_dim_start + 1 + : 2 * NumDims - 2 * outer_dim_start - 2; + if (first_multiple > outer_dim_left_index) { + const Index head_size = first_multiple - outer_dim_left_index; + input_block_sizes[outer_dim] = head_size; + broadcast_block_sizes[copy_outer_dim] = head_size; + broadcast_tensor_strides[copy_outer_dim] = m_inputStrides[outer_dim]; + broadcast_block_strides[copy_outer_dim] = + output_block_strides[outer_dim]; + broadcast_block_sizes[broadcast_outer_dim] = 1; + broadcast_tensor_strides[broadcast_outer_dim] = 0; + broadcast_block_strides[broadcast_outer_dim] = + output_block_strides[outer_dim] * input_dims[outer_dim]; + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, 0, + output_block); + } + if (first_multiple < last_multiple) { + input_block_sizes[outer_dim] = input_outer_dim_size; + broadcast_block_sizes[copy_outer_dim] = input_outer_dim_size; + broadcast_tensor_strides[copy_outer_dim] = m_inputStrides[outer_dim]; + broadcast_block_strides[copy_outer_dim] = + output_block_strides[outer_dim]; + broadcast_block_sizes[broadcast_outer_dim] = + (last_multiple - first_multiple) / input_outer_dim_size; + broadcast_tensor_strides[broadcast_outer_dim] = 0; + broadcast_block_strides[broadcast_outer_dim] = + output_block_strides[outer_dim] * input_dims[outer_dim]; + const Index offset = (first_multiple - outer_dim_left_index) * + m_outputStrides[outer_dim]; + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, + offset, output_block); + } + if (last_multiple < outer_dim_left_index + outer_dim_size) { + const Index tail_size = + outer_dim_left_index + outer_dim_size - last_multiple; + input_block_sizes[outer_dim] = tail_size; + broadcast_block_sizes[copy_outer_dim] = tail_size; + broadcast_tensor_strides[copy_outer_dim] = m_inputStrides[outer_dim]; + broadcast_block_strides[copy_outer_dim] = + output_block_strides[outer_dim]; + broadcast_block_sizes[broadcast_outer_dim] = 1; + broadcast_tensor_strides[broadcast_outer_dim] = 0; + broadcast_block_strides[broadcast_outer_dim] = + output_block_strides[outer_dim] * input_dims[outer_dim]; + const Index offset = (last_multiple - outer_dim_left_index) * + m_outputStrides[outer_dim]; + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, + offset, output_block); + } + } else { + // b and c do not exist. + const int copy_outer_dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 2 * outer_dim_start + : 2 * NumDims - 2 * outer_dim_start - 1; + input_block_sizes[outer_dim] = outer_dim_size; + broadcast_block_sizes[copy_outer_dim] = outer_dim_size; + broadcast_tensor_strides[copy_outer_dim] = m_inputStrides[outer_dim]; + broadcast_block_strides[copy_outer_dim] = + output_block_strides[outer_dim]; + BroadcastBlock(input_block_sizes, broadcast_block_sizes, + broadcast_block_strides, broadcast_tensor_strides, 0, + output_block); + } + } + } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + + Broadcast functor() const { return m_broadcast; } + + private: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void BroadcastBlock( + const Dimensions& input_block_sizes, + const BroadcastDimensions& broadcast_block_sizes, + const BroadcastDimensions& broadcast_block_strides, + const BroadcastDimensions& broadcast_tensor_strides, Index offset, + TensorBlock* output_block) const { + TensorBlock input_view_block( + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? indexColMajor(output_block->first_coeff_index() + offset) + : indexRowMajor(output_block->first_coeff_index() + offset), + input_block_sizes, Dimensions(m_inputStrides), + Dimensions(m_inputStrides), NULL); + + internal::TensorBlockView<ArgType, Device> input_block(m_device, m_impl, + input_view_block); + BroadcastTensorBlock broadcast_block( + 0, broadcast_block_sizes, broadcast_block_strides, + broadcast_tensor_strides, output_block->data() + offset); + + BroadcastTensorBlockReader::Run(&broadcast_block, input_block.data()); + } protected: + const Device& m_device; + const Broadcast m_broadcast; Dimensions m_dimensions; array<Index, NumDims> m_outputStrides; array<Index, NumDims> m_inputStrides;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h b/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h index 2742dbb..8c06449 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h
@@ -32,6 +32,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions - 1; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<DenseIndex DimId, typename XprType> @@ -50,6 +51,7 @@ struct DimensionId { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DimensionId(DenseIndex dim) { + EIGEN_UNUSED_VARIABLE(dim); eigen_assert(dim == DimId); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex actualDim() const { @@ -136,24 +138,32 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { // Alignment can't be guaranteed at compile time since it depends on the // slice offsets. - IsAligned = false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, // to be implemented - RawAccess = false + IsAligned = false, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumInputDims, Layout> + InputTensorBlock; + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + OutputTensorBlock; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device), m_dim(op.dim()), m_device(device) + : m_impl(op.expression(), device), m_dim(op.dim()), m_device(device), m_offset(op.offset()) { - // We could also support the case where NumInputDims==1 if needed. - EIGEN_STATIC_ASSERT(NumInputDims >= 2, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumInputDims >= 1), YOU_MADE_A_PROGRAMMING_MISTAKE); eigen_assert(NumInputDims > m_dim.actualDim()); const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); @@ -182,6 +192,20 @@ } m_inputStride *= input_dims[m_dim.actualDim()]; m_inputOffset = m_stride * op.offset(); + + if (BlockAccess) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_inputStrides[0] = 1; + for (int i = 1; i < NumInputDims; ++i) { + m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; + } + } else { + m_inputStrides[NumInputDims - 1] = 1; + for (int i = NumInputDims - 2; i >= 0; --i) { + m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1]; + } + } + } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -203,11 +227,11 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); if ((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == 0) || - (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == NumInputDims-1)) { + (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == NumInputDims-1)) { // m_stride is equal to 1, so let's avoid the integer division. eigen_assert(m_stride == 1); Index inputIndex = index * m_inputStride + m_inputOffset; @@ -219,8 +243,8 @@ PacketReturnType rslt = internal::pload<PacketReturnType>(values); return rslt; } else if ((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == NumInputDims - 1) || - (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) { - // m_stride is aways greater than index, so let's avoid the integer division. + (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) { + // m_stride is always greater than index, so let's avoid the integer division. eigen_assert(m_stride > index); return m_impl.template packet<LoadMode>(index + m_inputOffset); } else { @@ -264,7 +288,62 @@ TensorOpCost(0, 0, cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType* data() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.lastLevelCacheSize() / sizeof(Scalar)); + resources->push_back(internal::TensorOpResourceRequirements( + internal::kSkewedInnerDims, block_total_size_max)); + m_impl.getResourceRequirements(resources); + } + + // TODO(andydavis) Reduce the overhead of this function (experiment with + // using a fixed block size). + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + OutputTensorBlock* output_block) const { + // Calculate input block sizes. + const DSizes<Index, NumDims>& output_block_sizes = + output_block->block_sizes(); + const DSizes<Index, NumDims>& output_block_strides = + output_block->block_strides(); + const Index chip_dim = m_dim.actualDim(); + DSizes<Index, NumInputDims> input_block_sizes; + DSizes<Index, NumInputDims> input_block_strides; + for (Index i = 0; i < NumInputDims; ++i) { + if (i < chip_dim) { + input_block_sizes[i] = output_block_sizes[i]; + input_block_strides[i] = output_block_strides[i]; + } else if (i > chip_dim) { + input_block_sizes[i] = output_block_sizes[i - 1]; + input_block_strides[i] = output_block_strides[i - 1]; + } else { + input_block_sizes[i] = 1; + } + } + // Fix up input_block_stride for chip dimension. + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + if (chip_dim == 0) { + input_block_strides[chip_dim] = 1; + } else { + input_block_strides[chip_dim] = + input_block_strides[chip_dim - 1] * input_block_sizes[chip_dim - 1]; + } + } else { + if (chip_dim == NumInputDims - 1) { + input_block_strides[chip_dim] = 1; + } else { + input_block_strides[chip_dim] = + input_block_strides[chip_dim + 1] * input_block_sizes[chip_dim + 1]; + } + } + // Instantiate and read input block from input tensor. + InputTensorBlock input_block(srcCoeff(output_block->first_coeff_index()), + input_block_sizes, input_block_strides, + m_inputStrides, output_block->data()); + m_impl.block(&input_block); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { CoeffReturnType* result = const_cast<CoeffReturnType*>(m_impl.data()); if (((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == NumDims) || (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) && @@ -275,18 +354,31 @@ } } + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex dimId() const { + return m_dim.actualDim(); + } + + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const DenseIndex& offset() const { + return m_offset; + } + /// required by sycl in order to extract the accessor + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex; if ((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == 0) || - (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == NumInputDims-1)) { + (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == NumInputDims - 1)) { // m_stride is equal to 1, so let's avoid the integer division. eigen_assert(m_stride == 1); inputIndex = index * m_inputStride + m_inputOffset; - } else if ((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == NumInputDims-1) || - (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) { - // m_stride is aways greater than index, so let's avoid the integer division. + } else if ((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == NumInputDims - 1) || + (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) { + // m_stride is always greater than index, so let's avoid the integer + // division. eigen_assert(m_stride > index); inputIndex = index + m_inputOffset; } else { @@ -302,9 +394,13 @@ Index m_stride; Index m_inputOffset; Index m_inputStride; + DSizes<Index, NumInputDims> m_inputStrides; TensorEvaluator<ArgType, Device> m_impl; const internal::DimensionId<DimId> m_dim; const Device& m_device; +// required by sycl + const DenseIndex m_offset; + }; @@ -322,14 +418,23 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = false, + IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - RawAccess = false + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumInputDims, Layout> + InputTensorBlock; + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + OutputTensorBlock; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) { } @@ -342,10 +447,10 @@ template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) if ((static_cast<int>(this->Layout) == static_cast<int>(ColMajor) && this->m_dim.actualDim() == 0) || - (static_cast<int>(this->Layout) == static_cast<int>(RowMajor) && this->m_dim.actualDim() == NumInputDims-1)) { + (static_cast<int>(this->Layout) == static_cast<int>(RowMajor) && this->m_dim.actualDim() == NumInputDims-1)) { // m_stride is equal to 1, so let's avoid the integer division. eigen_assert(this->m_stride == 1); EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; @@ -356,8 +461,8 @@ inputIndex += this->m_inputStride; } } else if ((static_cast<int>(this->Layout) == static_cast<int>(ColMajor) && this->m_dim.actualDim() == NumInputDims-1) || - (static_cast<int>(this->Layout) == static_cast<int>(RowMajor) && this->m_dim.actualDim() == 0)) { - // m_stride is aways greater than index, so let's avoid the integer division. + (static_cast<int>(this->Layout) == static_cast<int>(RowMajor) && this->m_dim.actualDim() == 0)) { + // m_stride is always greater than index, so let's avoid the integer division. eigen_assert(this->m_stride > index); this->m_impl.template writePacket<StoreMode>(index + this->m_inputOffset, x); } else { @@ -377,6 +482,50 @@ } } } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock( + const OutputTensorBlock& output_block) { + // Calculate input block sizes. + const DSizes<Index, NumDims>& output_block_sizes = + output_block.block_sizes(); + const DSizes<Index, NumDims>& output_block_strides = + output_block.block_strides(); + const Index chip_dim = this->m_dim.actualDim(); + DSizes<Index, NumInputDims> input_block_sizes; + DSizes<Index, NumInputDims> input_block_strides; + for (Index i = 0; i < NumInputDims; ++i) { + if (i < chip_dim) { + input_block_sizes[i] = output_block_sizes[i]; + input_block_strides[i] = output_block_strides[i]; + } else if (i > chip_dim) { + input_block_sizes[i] = output_block_sizes[i - 1]; + input_block_strides[i] = output_block_strides[i - 1]; + } else { + input_block_sizes[i] = 1; + } + } + // Fix up input_block_stride for chip dimension. + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + if (chip_dim == 0) { + input_block_strides[chip_dim] = 1; + } else { + input_block_strides[chip_dim] = + input_block_strides[chip_dim - 1] * input_block_sizes[chip_dim - 1]; + } + } else { + if (chip_dim == NumInputDims - 1) { + input_block_strides[chip_dim] = 1; + } else { + input_block_strides[chip_dim] = + input_block_strides[chip_dim + 1] * input_block_sizes[chip_dim + 1]; + } + } + // Write input block. + this->m_impl.writeBlock(InputTensorBlock( + this->srcCoeff(output_block.first_coeff_index()), input_block_sizes, + input_block_strides, this->m_inputStrides, + const_cast<ScalarNoConst*>(output_block.data()))); + } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h index 839c6e3..3863ee8 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h
@@ -37,6 +37,8 @@ static const int NumDimensions = traits<LhsXprType>::NumDimensions; static const int Layout = traits<LhsXprType>::Layout; enum { Flags = 0 }; + typedef typename conditional<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val, + typename traits<LhsXprType>::PointerType, typename traits<RhsXprType>::PointerType>::type PointerType; }; template<typename Axis, typename LhsXprType, typename RhsXprType> @@ -120,6 +122,8 @@ enum { IsAligned = false, PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<LeftArgType, Device>::Layout, RawAccess = false }; @@ -128,8 +132,8 @@ : m_leftImpl(op.lhsExpression(), device), m_rightImpl(op.rhsExpression(), device), m_axis(op.axis()) { EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) == static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout) || NumDims == 1), YOU_MADE_A_PROGRAMMING_MISTAKE); - EIGEN_STATIC_ASSERT(NumDims == RightNumDims, YOU_MADE_A_PROGRAMMING_MISTAKE); - EIGEN_STATIC_ASSERT(NumDims > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims == RightNumDims), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); eigen_assert(0 <= m_axis && m_axis < NumDims); const Dimensions& lhs_dims = m_leftImpl.dimensions(); @@ -248,8 +252,8 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - static const int packetSize = internal::unpacket_traits<PacketReturnType>::size; - EIGEN_STATIC_ASSERT(packetSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + const int packetSize = PacketType<CoeffReturnType, Device>::size; + EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index + packetSize - 1 < dimensions().TotalSize()); EIGEN_ALIGN_MAX CoeffReturnType values[packetSize]; @@ -275,7 +279,13 @@ TensorOpCost(0, 0, compute_cost); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<LeftArgType, Device>& left_impl() const { return m_leftImpl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<RightArgType, Device>& right_impl() const { return m_rightImpl; } + /// required by sycl in order to extract the accessor + const Axis& axis() const { return m_axis; } protected: Dimensions m_dimensions; @@ -298,6 +308,8 @@ enum { IsAligned = false, PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<LeftArgType, Device>::Layout, RawAccess = false }; @@ -344,8 +356,8 @@ template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - static const int packetSize = internal::unpacket_traits<PacketReturnType>::size; - EIGEN_STATIC_ASSERT(packetSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + const int packetSize = PacketType<CoeffReturnType, Device>::size; + EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index + packetSize - 1 < this->dimensions().TotalSize()); EIGEN_ALIGN_MAX CoeffReturnType values[packetSize];
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h index 9718225..6ca881f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h
@@ -20,13 +20,78 @@ * */ namespace internal { +#if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) +template<typename Scalar, typename Index> +void pack_simple(Scalar * dst, const Scalar * src, Index cols, Index rows, Index lddst, Index ldsrc) { + size_t psize = packet_traits<Scalar>::size; // Packet size + typedef typename packet_traits<Scalar>::type Packet; // Packet type + size_t alignment = psize*sizeof(Scalar); // Needed alignment + if (rows % psize == 0 && (lddst*sizeof(Scalar)) % alignment == 0 && + (ldsrc*sizeof(Scalar)) % alignment == 0 && + reinterpret_cast<uintptr_t>(src) % alignment == 0 && + reinterpret_cast<uintptr_t>(dst) % alignment == 0) { + // Optimized version using packets + size_t num_packets = rows / psize; + for (Index col = 0; col < cols; ++col) { + EIGEN_ASM_COMMENT("begin pack_simple inner copy"); + // Unrolled manually 4 times. + for (size_t i=0; i < num_packets/4; ++i) { + internal::pstore(dst, internal::pload<Packet>(src)); + dst += psize; src += psize; + internal::pstore(dst, internal::pload<Packet>(src)); + dst += psize; src += psize; + internal::pstore(dst, internal::pload<Packet>(src)); + dst += psize; src += psize; + internal::pstore(dst, internal::pload<Packet>(src)); + dst += psize; src += psize; + } + for (size_t i=0; i < num_packets%4; ++i) { + internal::pstore(dst, internal::pload<Packet>(src)); + dst += psize; src += psize; + } + dst += lddst - num_packets*psize; + src += ldsrc - num_packets*psize; + EIGEN_ASM_COMMENT("end pack_simple inner copy"); + } + } else { + // Naive memcpy calls + for (Index col = 0; col < cols; ++col) { + memcpy(dst + col*lddst, src + col*ldsrc, rows*sizeof(Scalar)); + } + } +} -template<typename Dimensions, typename LhsXprType, typename RhsXprType> -struct traits<TensorContractionOp<Dimensions, LhsXprType, RhsXprType> > +template<typename LhsScalar, typename RhsScalar, typename Scalar> + struct libxsmm_wrapper { + libxsmm_wrapper() {} + libxsmm_wrapper(int, int, int, int, int, int, int, float, float, int) {} + void operator()(const LhsScalar*, const RhsScalar*, Scalar*) {} + void operator()(const LhsScalar*, const RhsScalar*, Scalar*, const LhsScalar*, const RhsScalar*, const Scalar*) {} + }; + + template<> + struct libxsmm_wrapper<float, float, float>: public libxsmm_mmfunction<float> { + libxsmm_wrapper(): libxsmm_mmfunction() {} + libxsmm_wrapper(int flags, int m, int n, int k, int lda, int ldb, int ldc, float alpha, float beta, int prefetch) : + libxsmm_mmfunction(flags, m, n, k, lda, ldb, ldc, alpha, beta, prefetch) {} + }; + + template<> + struct libxsmm_wrapper<double, double, double>: public libxsmm_mmfunction<double> { + libxsmm_wrapper(): libxsmm_mmfunction() {} + libxsmm_wrapper(int flags, int m, int n, int k, int lda, int ldb, int ldc, float alpha, float beta, int prefetch) : + libxsmm_mmfunction(flags, m, n, k, lda, ldb, ldc, alpha, beta, prefetch) {} + }; +#endif + + +template<typename Dimensions, typename LhsXprType, typename RhsXprType, typename OutputKernelType> +struct traits<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType> > { // Type promotion to handle the case where the types of the lhs and the rhs are different. - typedef typename internal::promote_storage_type<typename LhsXprType::Scalar, - typename RhsXprType::Scalar>::ret Scalar; + typedef typename gebp_traits<typename remove_const<typename LhsXprType::Scalar>::type, + typename remove_const<typename RhsXprType::Scalar>::type>::ResScalar Scalar; + typedef typename promote_storage_type<typename traits<LhsXprType>::StorageKind, typename traits<RhsXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<LhsXprType>::Index, @@ -37,53 +102,171 @@ typedef typename remove_reference<RhsNested>::type _RhsNested; // From NumDims below. - static const int NumDimensions = max_n_1<traits<RhsXprType>::NumDimensions + traits<RhsXprType>::NumDimensions - 2 * array_size<Dimensions>::value>::size; + static const int NumDimensions = traits<LhsXprType>::NumDimensions + traits<RhsXprType>::NumDimensions - 2 * array_size<Dimensions>::value; static const int Layout = traits<LhsXprType>::Layout; + typedef typename conditional<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val, + typename traits<LhsXprType>::PointerType, typename traits<RhsXprType>::PointerType>::type PointerType; enum { - Flags = 0, + Flags = 0 }; }; -template<typename Dimensions, typename LhsXprType, typename RhsXprType> -struct eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType>, Eigen::Dense> +template<typename Dimensions, typename LhsXprType, typename RhsXprType, typename OutputKernelType> +struct eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>, Eigen::Dense> { - typedef const TensorContractionOp<Dimensions, LhsXprType, RhsXprType>& type; + typedef const TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>& type; }; -template<typename Dimensions, typename LhsXprType, typename RhsXprType> -struct nested<TensorContractionOp<Dimensions, LhsXprType, RhsXprType>, 1, typename eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType> >::type> +template<typename Dimensions, typename LhsXprType, typename RhsXprType, typename OutputKernelType> +struct nested<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>, 1, typename eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType> >::type> { - typedef TensorContractionOp<Dimensions, LhsXprType, RhsXprType> type; + typedef TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType> type; }; -template<typename Indices_, typename LeftArgType_, typename RightArgType_, typename Device_> -struct traits<TensorEvaluator<const TensorContractionOp<Indices_, LeftArgType_, RightArgType_>, Device_> > { +template<typename Indices_, typename LeftArgType_, typename RightArgType_, typename OutputKernelType_, typename Device_> +struct traits<TensorEvaluator<const TensorContractionOp<Indices_, LeftArgType_, RightArgType_, OutputKernelType_>, Device_> > { typedef Indices_ Indices; typedef LeftArgType_ LeftArgType; typedef RightArgType_ RightArgType; + typedef OutputKernelType_ OutputKernelType; typedef Device_ Device; // From NumDims below. - static const int NumDimensions = max_n_1<traits<LeftArgType_>::NumDimensions + traits<RightArgType_>::NumDimensions - 2 * array_size<Indices_>::value>::size; + static const int NumDimensions = traits<LeftArgType_>::NumDimensions + traits<RightArgType_>::NumDimensions - 2 * array_size<Indices_>::value; +}; + +// WARNING: In this code we assume that Lhs and Rhs tensor expressions are in +// ColMajor storage order. This property is guaranteed by the +// TensorContractionOp evaluator. TensorContractionKernel specifies how we pack +// blocks of Lhs and Rhs tensor expressions, and how we invoke matrix +// multiplication for these blocks. Default tensor contraction uses +// gemm_pack_rhs, gemm_pack_lhs and gebp_kernel from Eigen Core (see +// GeneralBlocPanelKernel.h for details). +// +// By specializing contraction kernels we can use other low level libraries to +// perform matrix multiplication, and still rely on Eigen contraction evaluator. +// This also includes full support in TensorContractionThreadPool, assuming that +// underlying gemm do not use it's own threading. +// +// - ResScalar/LhsScalar/RhsScalar - scalar type for the result of +// multiplication, lhs tensor and rhs tensor respectively. +// +// - StorageIndex - index type for the tensor expressions. In practice almost +// always is Eigen::Index. +// +// - OutputMapper provides access to the memory of the output matrix. In +// practice it's always column major blas_data_mapper (it must be of ResScalar +// type). +// +// - LhsMapper/RhsMapper similarly to blas_data_mapper provide a two dimensional +// view into the Lhs/Rhs tensor expressions. In practice it's +// TensorContractionInputMapper, or some specialization of it based on the +// type of tensor expression (e.g. TensorImagePatchOp has optimized input +// mapper). +template<typename ResScalar, typename LhsScalar, typename RhsScalar, + typename StorageIndex, typename OutputMapper, typename LhsMapper, + typename RhsMapper> +struct TensorContractionKernel { + typedef typename internal::gebp_traits<LhsScalar, RhsScalar> Traits; + + typedef internal::gemm_pack_lhs<LhsScalar, StorageIndex, + typename LhsMapper::SubMapper, + Traits::mr, Traits::LhsProgress, + typename Traits::LhsPacket4Packing, ColMajor> + LhsPacker; + + typedef internal::gemm_pack_rhs<RhsScalar, StorageIndex, + typename RhsMapper::SubMapper, Traits::nr, + ColMajor> + RhsPacker; + + typedef internal::gebp_kernel<LhsScalar, RhsScalar, StorageIndex, + OutputMapper, Traits::mr, Traits::nr, + /*ConjugateLhs*/ false, /*ConjugateRhs*/ false> + GebpKernel; + + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE + static void packLhs(LhsScalar* lhsBlock, + const typename LhsMapper::SubMapper& data_mapper, + const StorageIndex depth, const StorageIndex rows) { + LhsPacker()(lhsBlock, data_mapper, depth, rows, /*stride*/ 0, /*offset*/ 0); + } + + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE + static void packRhs(RhsScalar* rhsBlock, + const typename RhsMapper::SubMapper& data_mapper, + const StorageIndex depth, const StorageIndex cols) { + RhsPacker()(rhsBlock, data_mapper, depth, cols); + } + + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE + static void invoke(const OutputMapper& output_mapper, + const LhsScalar* lhsBlock, const RhsScalar* rhsBlock, + const StorageIndex rows, const StorageIndex depth, + const StorageIndex cols, const ResScalar alpha) { + GebpKernel()(output_mapper, lhsBlock, rhsBlock, rows, depth, cols, alpha, + /*strideA*/ -1, /*strideB*/ -1, + /*offsetA*/ 0, /*offsetB*/ 0); + } }; } // end namespace internal -template<typename Indices, typename LhsXprType, typename RhsXprType> -class TensorContractionOp : public TensorBase<TensorContractionOp<Indices, LhsXprType, RhsXprType>, ReadOnlyAccessors> +// Tensor contraction params that should enable to get from output matrix +// 2-dimensional coordinates to the output tensor dimensions. +struct TensorContractionParams { + // TensorContraction evaluator assumes that both tensors are in ColMajor + // layout, if tensors are in RowMajor evaluator swap lhs with rhs. + bool swapped_arguments; +}; + +// Output kernel allows to fuse operations into the tensor contraction. +// +// Examples: +// 1. Elementwise Relu transformation following Conv2D. +// 2. AddBias to the Conv2D output channels dimension. +// +// The NoOpOutputKernel implements an output kernel that does absolutely nothing. +struct NoOpOutputKernel { + /** + * Tensor contraction evaluator calls this kernel after finishing each block + * of output matrix. Output blocks belong to the 2-dimensional output tensor. + * + * TensorContractionParams contains contraction dimensions information + * required to map output 2-d space into the expected output tensor space + * (potentially higher dimensional). + * + * \param[in] output_mapper Access to output tensor memory + * \param[in] params Tensor contraction parameters + * \param[in] i Index of a first row available through output_mapper + * \param[in] j Index of a first column available through output_mapper + * \param[in] num_rows Number of available rows + * \param[in] num_cols Number of available columns + */ + template <typename Index, typename Scalar> + EIGEN_ALWAYS_INLINE void operator()( + const internal::blas_data_mapper<Scalar, Index, ColMajor>& /*output_mapper*/, + const TensorContractionParams& /*params*/, Index /*i*/, + Index /*j*/, Index /*num_rows*/, Index /*num_cols*/) const {} +}; + +template<typename Indices, typename LhsXprType, typename RhsXprType, typename OutputKernelType = const NoOpOutputKernel> +class TensorContractionOp : public TensorBase<TensorContractionOp<Indices, LhsXprType, RhsXprType, OutputKernelType>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorContractionOp>::Scalar Scalar; - typedef typename internal::promote_storage_type<typename LhsXprType::CoeffReturnType, - typename RhsXprType::CoeffReturnType>::ret CoeffReturnType; + typedef typename internal::gebp_traits<typename LhsXprType::CoeffReturnType, + typename RhsXprType::CoeffReturnType>::ResScalar CoeffReturnType; typedef typename Eigen::internal::nested<TensorContractionOp>::type Nested; typedef typename Eigen::internal::traits<TensorContractionOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorContractionOp>::Index Index; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionOp( - const LhsXprType& lhs, const RhsXprType& rhs, const Indices& dims) - : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_indices(dims) {} + const LhsXprType& lhs, const RhsXprType& rhs, const Indices& dims, + const OutputKernelType& output_kernel = OutputKernelType()) + : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_indices(dims), + m_output_kernel(output_kernel) {} EIGEN_DEVICE_FUNC const Indices& indices() const { return m_indices; } @@ -97,10 +280,14 @@ const typename internal::remove_all<typename RhsXprType::Nested>::type& rhsExpression() const { return m_rhs_xpr; } + EIGEN_DEVICE_FUNC + const OutputKernelType& outputKernel() const { return m_output_kernel; } + protected: typename LhsXprType::Nested m_lhs_xpr; typename RhsXprType::Nested m_rhs_xpr; const Indices m_indices; + const OutputKernelType m_output_kernel; }; @@ -110,9 +297,10 @@ typedef typename internal::traits<Derived>::Indices Indices; typedef typename internal::traits<Derived>::LeftArgType LeftArgType; typedef typename internal::traits<Derived>::RightArgType RightArgType; + typedef typename internal::traits<Derived>::OutputKernelType OutputKernelType; typedef typename internal::traits<Derived>::Device Device; - typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; + typedef TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; @@ -120,7 +308,9 @@ enum { IsAligned = true, - PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<LeftArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = true @@ -140,11 +330,11 @@ static const int RDims = internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value; static const int ContractDims = internal::array_size<Indices>::value; - static const int NumDims = max_n_1<LDims + RDims - 2 * ContractDims>::size; + static const int NumDims = LDims + RDims - 2 * ContractDims; typedef array<Index, ContractDims> contract_t; - typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; - typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; + typedef array<Index, LDims - ContractDims> left_nocontract_t; + typedef array<Index, RDims - ContractDims> right_nocontract_t; typedef DSizes<Index, NumDims> Dimensions; @@ -155,9 +345,10 @@ m_rightImpl(choose(Cond<static_cast<int>(Layout) == static_cast<int>(ColMajor)>(), op.rhsExpression(), op.lhsExpression()), device), m_device(device), + m_output_kernel(op.outputKernel()), m_result(NULL) { EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) == - static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)), + static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE); @@ -218,11 +409,9 @@ rhs_strides[i+1] = rhs_strides[i] * eval_right_dims[i]; } - m_i_strides[0] = 1; - m_j_strides[0] = 1; - if(ContractDims) { - m_k_strides[0] = 1; - } + if (m_i_strides.size() > 0) m_i_strides[0] = 1; + if (m_j_strides.size() > 0) m_j_strides[0] = 1; + if (m_k_strides.size() > 0) m_k_strides[0] = 1; m_i_size = 1; m_j_size = 1; @@ -234,7 +423,7 @@ // dimensions and right non-contracting dimensions. m_lhs_inner_dim_contiguous = true; int dim_idx = 0; - unsigned int nocontract_idx = 0; + Index nocontract_idx = 0; for (int i = 0; i < LDims; i++) { // find if we are contracting on index i of left tensor @@ -318,10 +507,7 @@ } } - // Scalar case. We represent the result as a 1d tensor of size 1. - if (LDims + RDims == 2 * ContractDims) { - m_dimensions[0] = 1; - } + EnableXSMMIfPossible(eval_op_indices); // If the layout is RowMajor, we need to reverse the m_dimensions if (static_cast<int>(Layout) == static_cast<int>(RowMajor)) { @@ -329,11 +515,17 @@ numext::swap(m_dimensions[i], m_dimensions[j]); } } + + // A set of parameters that will allow output kernel to get from output + // tensor dimensions (i, j) into the original tensor dimensions. + // TODO(ezhulenev): Add parameters required to infer output tensor index for + // more complex contractions than 2x2 on internal dimension. + m_tensor_contraction_params.swapped_arguments = static_cast<int>(Layout) == RowMajor; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar * data) { m_leftImpl.evalSubExprsIfNeeded(NULL); m_rightImpl.evalSubExprsIfNeeded(NULL); if (data) { @@ -346,47 +538,66 @@ } } - EIGEN_DEVICE_FUNC void evalTo(Scalar* buffer) const { - if (this->m_lhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_reordered) { - static_cast<const Derived*>(this)->template evalProduct<true, true, true, Unaligned>(buffer); - } - else { - static_cast<const Derived*>(this)->template evalProduct<true, true, false, Unaligned>(buffer); - } - } - else { - if (this->m_rhs_inner_dim_reordered) { - static_cast<const Derived*>(this)->template evalProduct<true, false, true, Unaligned>(buffer); - } - else { - static_cast<const Derived*>(this)->template evalProduct<true, false, false, Unaligned>(buffer); - } - } +#define TENSOR_CONTRACTION_DISPATCH(METHOD, ALIGNMENT, ARGS) \ + if (this->m_lhs_inner_dim_contiguous) { \ + if (this->m_rhs_inner_dim_contiguous) { \ + if (this->m_rhs_inner_dim_reordered) { \ + METHOD<true, true, true, ALIGNMENT>ARGS; \ + } \ + else { \ + METHOD<true, true, false, ALIGNMENT>ARGS; \ + } \ + } \ + else { \ + if (this->m_rhs_inner_dim_reordered) { \ + METHOD<true, false, true, ALIGNMENT>ARGS; \ + } \ + else { \ + METHOD<true, false, false, ALIGNMENT>ARGS; \ + } \ + } \ + } \ + else { \ + if (this->m_rhs_inner_dim_contiguous) { \ + if (this->m_rhs_inner_dim_reordered) { \ + METHOD<false, true, true, ALIGNMENT>ARGS; \ + } \ + else { \ + METHOD<false, true, false, ALIGNMENT>ARGS; \ + } \ + } \ + else { \ + if (this->m_rhs_inner_dim_reordered) { \ + METHOD<false, false, true, ALIGNMENT>ARGS; \ + } \ + else { \ + METHOD<false, false, false, ALIGNMENT>ARGS; \ + } \ + } \ } - else { - if (this->m_rhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_reordered) { - static_cast<const Derived*>(this)->template evalProduct<false, true, true, Unaligned>(buffer); - } - else { - static_cast<const Derived*>(this)->template evalProduct<false, true, false, Unaligned>(buffer); - } - } - else { - if (this->m_rhs_inner_dim_reordered) { - static_cast<const Derived*>(this)->template evalProduct<false, false, true, Unaligned>(buffer); - } - else { - static_cast<const Derived*>(this)->template evalProduct<false, false, false, Unaligned>(buffer); - } - } + + EIGEN_DEVICE_FUNC void evalTo(Scalar* buffer) const { + static_cast<const Derived*>(this)->template evalProduct<Unaligned>(buffer); + } + + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, + bool rhs_inner_dim_reordered, int Alignment> + void evalProductSequential(Scalar* buffer) const { + if (this->m_j_size == 1) { + this->template evalGemv<lhs_inner_dim_contiguous, + rhs_inner_dim_contiguous, rhs_inner_dim_reordered, + Alignment>(buffer); + } else { + this->template evalGemm<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, + rhs_inner_dim_reordered, Alignment>(buffer); } } template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - EIGEN_DEVICE_FUNC void evalGemv(Scalar* buffer) const { + #if !defined(EIGEN_HIPCC) + EIGEN_DEVICE_FUNC + #endif + void evalGemv(Scalar* buffer) const { const Index rows = m_i_size; const Index cols = m_k_size; @@ -424,10 +635,25 @@ internal::general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,false,RhsScalar,RhsMapper,false>::run( rows, cols, lhs, rhs, buffer, resIncr, alpha); + + typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; + m_output_kernel(OutputMapper(buffer, rows), m_tensor_contraction_params, + static_cast<Index>(0), static_cast<Index>(0), rows, + static_cast<Index>(1)); } template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - EIGEN_DEVICE_FUNC void evalGemm(Scalar* buffer) const { + #if !defined(EIGEN_HIPCC) + EIGEN_DEVICE_FUNC + #endif + void evalGemm(Scalar* buffer) const { + #if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) + if (m_can_use_xsmm) { + evalGemmXSMM(buffer); + return; + } + #endif + // columns in left side, rows in right side const Index k = this->m_k_size; @@ -439,14 +665,37 @@ // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar) this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); + this->template evalGemmPartial<lhs_inner_dim_contiguous, + rhs_inner_dim_contiguous, + rhs_inner_dim_reordered, + Alignment, true>(buffer, 0, k, 1); + } - // define mr, nr, and all of my data mapper types + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, + bool rhs_inner_dim_reordered, int Alignment> + EIGEN_DEVICE_FUNC void evalGemmPartialWithoutOutputKernel( + Scalar* buffer, Index k_start, Index k_end, int num_threads) const { + evalGemmPartial<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, + rhs_inner_dim_reordered, Alignment, + /*use_output_kernel*/ false>(buffer, k_start, k_end, + num_threads); + } + + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment, bool use_output_kernel> + EIGEN_DEVICE_FUNC void evalGemmPartial(Scalar* buffer, Index k_start, Index k_end, int num_threads) const { + eigen_assert(k_end >= k_start && k_start >= 0 && k_end <= this->m_k_size); + // columns in slice on left side, rows on right side + const Index k_slice = k_end - k_start; + + // rows in left side + const Index m = this->m_i_size; + + // columns in right side + const Index n = this->m_j_size; + + // define data mappers for Lhs and Rhs typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; - typedef typename internal::gebp_traits<LhsScalar, RhsScalar> Traits; - - const Index nr = Traits::nr; - const Index mr = Traits::mr; typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; @@ -468,11 +717,9 @@ typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; - // Declare GEBP packing and kernel structs - internal::gemm_pack_lhs<LhsScalar, Index, typename LhsMapper::SubMapper, mr, Traits::LhsProgress, ColMajor> pack_lhs; - internal::gemm_pack_rhs<RhsScalar, Index, typename RhsMapper::SubMapper, nr, ColMajor> pack_rhs; - - internal::gebp_kernel<LhsScalar, RhsScalar, Index, OutputMapper, mr, nr, false, false> gebp; + typedef internal::TensorContractionKernel< + Scalar, LhsScalar, RhsScalar, Index, OutputMapper, LhsMapper, RhsMapper> + TensorContractionKernel; // initialize data mappers LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides, @@ -484,7 +731,9 @@ OutputMapper output(buffer, m); // Sizes of the blocks to load in cache. See the Goto paper for details. - internal::TensorContractionBlocking<LhsMapper, RhsMapper, Index, internal::ShardByCol> blocking(k, m, n, 1); + internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, + Index, internal::ShardByCol> + blocking(k_slice, m, n, num_threads); const Index kc = blocking.kc(); const Index mc = numext::mini(m, blocking.mc()); const Index nc = numext::mini(n, blocking.nc()); @@ -497,20 +746,31 @@ for(Index i2=0; i2<m; i2+=mc) { const Index actual_mc = numext::mini(i2+mc,m)-i2; - for (Index k2 = 0; k2 < k; k2 += kc) { + for (Index k2 = k_start; k2 < k_end; k2 += kc) { // make sure we don't overshoot right edge of left matrix, then pack vertical panel - const Index actual_kc = numext::mini(k2 + kc, k) - k2; - pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc, 0, 0); + const Index actual_kc = numext::mini(k2 + kc, k_end) - k2; + TensorContractionKernel::packLhs(blockA, lhs.getSubMapper(i2, k2), + actual_kc, actual_mc); // series of horizontal blocks for (Index j2 = 0; j2 < n; j2 += nc) { // make sure we don't overshoot right edge of right matrix, then pack block const Index actual_nc = numext::mini(j2 + nc, n) - j2; - pack_rhs(blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc, 0, 0); + TensorContractionKernel::packRhs(blockB, rhs.getSubMapper(k2, j2), + actual_kc, actual_nc); // call gebp (matrix kernel) // The parameters here are copied from Eigen's GEMM implementation - gebp(output.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, 1.0, -1, -1, 0, 0); + const OutputMapper output_mapper = output.getSubMapper(i2, j2); + TensorContractionKernel::invoke(output_mapper, blockA, blockB, + actual_mc, actual_kc, actual_nc, + Scalar(1)); + + // We are done with this [i2, j2] output block. + if (use_output_kernel && k2 + kc >= k_end) { + m_output_kernel(output_mapper, m_tensor_contraction_params, i2, j2, + actual_mc, actual_nc); + } } } } @@ -542,9 +802,216 @@ return internal::ploadt<PacketReturnType, LoadMode>(m_result + index); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar* data() const { return m_result; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { return m_result; } - protected: +protected: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void EnableXSMMIfPossible(const array<IndexPair<Index>, ContractDims>& eval_op_indices) { + m_can_use_xsmm = false; + +#if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) + typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; + typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; + if (!std::is_same<Scalar, LhsScalar>::value || + !std::is_same<Scalar, RhsScalar>::value || + !(std::is_same<Scalar, float>::value || + std::is_same<Scalar, double>::value) || + m_leftImpl.data() == NULL || + m_rightImpl.data() == NULL) { + return; + } + + // Check if we can use faster matmul algorithms. For contraction to be + // equivalent to matmul, we need both lhs and rhs contracting dims sequences + // to be either a prefix or suffix of all dims. Also, the order of both + // must be the same, so we don't have to do reordering. + // For example: + // * OK: lhs 4D, rhs 4D, contraction: [(0, 2), (1, 3)] + // * BAD: lhs 3D, rhs 3D, contraction: [(1,1)] + // * BAD: lhs 3D, rhs 3D, contraction: [(0, 0), (2, 2)] + // * BAD: lhs 3D, rhs 3D, contraction: [(0, 2), (1, 1)] + // Depending if contraction dims are prefix or suffix of all dims we need to + // pre-transpose matrices in matmul algorithm: + // lhs: prefix -> transpose, suffix -> no transpose + // rhs: prefix -> no transpose, suffix -> transpose + // For example, for lhs 2D, rhs 2D, contraction [(1, 0)] is regular, + // non-transposed matmul. + if (ContractDims == 0) { + // This case is totally uninteresting, filter it out to avoid problems + // with iterations in further tests. + return; + } + + // Check if RHS dims list is increasing. LHS already is, so if not, the + // order is different and we cannot do matmul. + for (int i = 1; i < ContractDims; i++) { + if (eval_op_indices[i].second < eval_op_indices[i-1].second) { + return; + } + } + + // Check if no holes. + int diff; + for (int i = 1; i < ContractDims; i++) { + // LHS contract dims are sorted to form an increasing seq. + diff = eval_op_indices[i].first - eval_op_indices[i-1].first; + if (diff != 1) { + return; + } + // Now we may already assume RHS contract dims seq is increasing too. + diff = eval_op_indices[i].second - eval_op_indices[i-1].second; + if (diff != 1) { + return; + } + } + + // Check if suffix or prefix. + if (eval_op_indices[0].first != 0 && + eval_op_indices[ContractDims-1].first != LDims-1) { + return; + } + if (eval_op_indices[0].second != 0 && + eval_op_indices[ContractDims-1].second != RDims-1) { + return; + } + + m_can_use_xsmm = true; +#else + EIGEN_UNUSED_VARIABLE(eval_op_indices); +#endif + } + +#if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) + EIGEN_DEVICE_FUNC void evalGemmXSMM(Scalar* buffer) const { + // columns in left side, rows in right side + const Index k = this->m_k_size; + + // rows in left side + const Index m = this->m_i_size; + + // columns in right side + const Index n = this->m_j_size; + + const bool transposeA = !m_lhs_inner_dim_contiguous; + const bool transposeB = !m_rhs_inner_dim_contiguous; + + typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; + typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; + + internal::TensorXsmmContractionBlocking<LhsScalar, RhsScalar, Index> blocking( + k, m, n, 1, transposeA, transposeB); + + // Outer blocks sizes + const Index mc_outer = blocking.outer_m(); + const Index nc_outer = blocking.outer_n(); + const Index kc_outer = blocking.outer_k(); + // Inner blocks sizes + const Index mc = blocking.mc(); + const Index nc = blocking.nc(); + const Index kc = blocking.kc(); + // Decisions whether we should copy parts of matrices + const bool copyA = blocking.copyA(); + const bool copyB = blocking.copyB(); + + const LhsScalar* leftData = m_leftImpl.data(); + const RhsScalar* rightData = m_rightImpl.data(); + + const libxsmm_blasint stride_A = static_cast<libxsmm_blasint>(transposeA ? k : m); + const libxsmm_blasint stride_B = static_cast<libxsmm_blasint>(transposeB ? n : k); + const libxsmm_blasint stride_C = static_cast<libxsmm_blasint>(m); + + const libxsmm_blasint stride_blockA = static_cast<libxsmm_blasint>(mc); + // Use bigger stride to avoid hitting same cache line too often. + // This consistently gives +~0.5 Gflops. + const libxsmm_blasint stride_panelB = static_cast<libxsmm_blasint>( + kc % 32 == 0 ? kc + 16 : kc + ); + + // Kernel for the general case (not edges) + internal::libxsmm_wrapper<LhsScalar, RhsScalar, Scalar> kernel; + + LhsScalar* blockA = NULL; + RhsScalar* panelB = NULL; + + if (copyA) { + blockA = static_cast<LhsScalar*>(this->m_device.allocate(mc * kc * sizeof(LhsScalar))); + } + if (copyB) { + panelB = static_cast<RhsScalar*>(this->m_device.allocate(nc_outer * stride_panelB * sizeof(RhsScalar))); + } + + const Index kernel_stride_A = copyA ? stride_blockA : stride_A; + const Index kernel_stride_B = copyB ? stride_panelB : stride_B; + kernel = internal::libxsmm_wrapper<LhsScalar, RhsScalar, Scalar>(0, mc, nc, kc, kernel_stride_A, kernel_stride_B, stride_C, 1, 1, blocking.prefetch()); + + // Outer blocking + for (Index ki_outer = 0; ki_outer < k; ki_outer += kc_outer) { + for (Index mi_outer = 0; mi_outer < m; mi_outer += mc_outer) { + for (Index ni_outer = 0; ni_outer < n; ni_outer += nc_outer) { + using numext::mini; + + Index actual_nc_outer = mini(ni_outer+nc_outer, n) - ni_outer; + + // Inner blocking + for (Index ki = ki_outer; ki < mini(ki_outer+kc_outer, k); ki += kc) { + const Index actual_kc = mini(ki_outer+kc_outer, mini(ki+kc, k)) - ki; + const float beta = ki == 0 ? 0 : 1; + + if (copyB) { + if (transposeB) { + libxsmm_otrans(panelB, rightData + ki*stride_B + ni_outer, sizeof(RhsScalar), actual_nc_outer, actual_kc, stride_B, stride_panelB); + } else { + internal::pack_simple<RhsScalar, Index>(panelB, rightData + ni_outer*stride_B + ki, actual_nc_outer, actual_kc, stride_panelB, stride_B); + } + } + + for (Index mi = mi_outer; mi < mini(mi_outer+mc_outer, m); mi += mc) { + const Index actual_mc = mini(mi_outer+mc_outer, mini(mi+mc, m)) - mi; + + const LhsScalar* a = transposeA ? leftData + mi*stride_A + ki : + leftData + ki*stride_A + mi; + + if (copyA) { + if (transposeA) { + libxsmm_otrans(blockA, a, sizeof(LhsScalar), actual_kc, actual_mc, stride_A, stride_blockA); + } else { + internal::pack_simple<LhsScalar, Index>(blockA, a, actual_kc, actual_mc, stride_blockA, stride_A); + } + } + const LhsScalar* actual_a = copyA ? blockA : a; + + for (Index ni = ni_outer; ni < mini(ni_outer+nc_outer, n); ni += nc) { + const Index actual_nc = mini(ni_outer+nc_outer, mini(ni+nc, n)) - ni; + + const RhsScalar* b = rightData + ni*stride_B + ki; + Scalar* c = buffer + ni*stride_C + mi; + const Scalar* cp = c + nc*stride_C; + + const RhsScalar* actual_b = copyB ? panelB + (ni-ni_outer)*stride_panelB : b; + const RhsScalar* bp = copyB ? panelB + nc*stride_panelB : b + nc*stride_B; + + if (actual_mc == mc && actual_kc == kc && actual_nc == nc && beta == 1) { + // Most used, cached kernel. + kernel(actual_a, actual_b, c, actual_a, bp, cp); + } else { + // Edges - use libxsmm kernel cache. + internal::libxsmm_wrapper<LhsScalar, RhsScalar, Scalar>(0, actual_mc, actual_nc, actual_kc, kernel_stride_A, kernel_stride_B, stride_C, 1, beta, blocking.prefetch())(actual_a, actual_b, c, actual_a, bp, cp); + } + } + } + } + } + } + } + + if (copyA) { + this->m_device.deallocate(blockA); + } + if (copyB) { + this->m_device.deallocate(panelB); + } + } +#endif + // Prevent assignment TensorContractionEvaluatorBase& operator = (const TensorContractionEvaluatorBase&); Dimensions m_dimensions; @@ -566,29 +1033,33 @@ Index m_j_size; Index m_k_size; + TensorContractionParams m_tensor_contraction_params; + TensorEvaluator<EvalLeftArgType, Device> m_leftImpl; TensorEvaluator<EvalRightArgType, Device> m_rightImpl; const Device& m_device; + OutputKernelType m_output_kernel; Scalar* m_result; + bool m_can_use_xsmm; }; // evaluator for default device -template<typename Indices, typename LeftArgType, typename RightArgType, typename Device> -struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> : +template<typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType, typename Device> +struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> : public TensorContractionEvaluatorBase< - TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> > { - typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> Self; + TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> > { + typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> Self; typedef TensorContractionEvaluatorBase<Self> Base; - typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; + typedef TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { - Layout = TensorEvaluator<LeftArgType, Device>::Layout, + Layout = TensorEvaluator<LeftArgType, Device>::Layout }; // Most of the code is assuming that both input tensors are ColMajor. If the @@ -607,26 +1078,20 @@ static const int ContractDims = internal::array_size<Indices>::value; typedef array<Index, ContractDims> contract_t; - typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; - typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; + typedef array<Index, LDims - ContractDims> left_nocontract_t; + typedef array<Index, RDims - ContractDims> right_nocontract_t; - static const int NumDims = max_n_1<LDims + RDims - 2 * ContractDims>::size; + static const int NumDims = LDims + RDims - 2 * ContractDims; // Could we use NumDimensions here? typedef DSizes<Index, NumDims> Dimensions; - EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) { } - template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - EIGEN_DEVICE_FUNC void evalProduct(Scalar* buffer) const { - if (this->m_j_size == 1) { - this->template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer); - return; - } - - this->template evalGemm<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer); + template <int Alignment> + void evalProduct(Scalar* buffer) const { + TENSOR_CONTRACTION_DISPATCH(this->template evalProductSequential, Alignment, (buffer)); } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h index 3d3f690..c51f3f8 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h
@@ -21,37 +21,187 @@ // Default Blocking Strategy -template <typename LhsMapper, typename RhsMapper, typename Index, int ShardingType=ShardByCol> +template<typename ResScalar, typename LhsScalar, typename RhsScalar, typename StorageIndex, int ShardingType = ShardByCol> class TensorContractionBlocking { public: - typedef typename LhsMapper::Scalar LhsScalar; - typedef typename RhsMapper::Scalar RhsScalar; + /* + adding EIGEN_DEVICE_FUNC unconditionally to 'TensorContractionBlocking' constructor in `TensorContractionBlocking.h` + requires adding EIGEN_DEVICE_FUNC to `computeProductBlockingSizes` in `GeneralBlockPanelKernel.h` + which in turn, requires adding EIGEN_DEVICE_FUNC to `evaluateProductBlockingSizesHeuristic` in `GeneralBlockPanelKernel.h` + which in turn, requires adding EIGEN_DEVICE_FUNC to `manage_caching_sizes` in `GeneralBlockPanelKernel.h` + (else HIPCC will error out) - EIGEN_DEVICE_FUNC TensorContractionBlocking(Index k, Index m, Index n, Index num_threads = 1) : + However adding EIGEN_DEVICE_FUNC to `manage_caching_sizes` in `GeneralBlockPanelKernel.h` + results in NVCC erroring out with the following error + + ../Eigen/src/Core/products/GeneralBlockPanelKernel.h(57): error #2901: + dynamic initialization is not supported for function-scope static variables within a __device__/__global__ function + */ + + #if !defined(EIGEN_HIPCC) + EIGEN_DEVICE_FUNC + #endif + TensorContractionBlocking(StorageIndex k, StorageIndex m, StorageIndex n, StorageIndex num_threads = 1) : kc_(k), mc_(m), nc_(n) { if (ShardingType == ShardByCol) { computeProductBlockingSizes<LhsScalar, RhsScalar, 1>(kc_, mc_, nc_, num_threads); } else { - if (kc_ && mc_ && nc_) { - mc_ = (((m / num_threads) + 15) / 16) * 16; - } + computeProductBlockingSizes<LhsScalar, RhsScalar, 1>(kc_, nc_, mc_, num_threads); } + + const int rhs_packet_size = internal::packet_traits<RhsScalar>::size; + kc_ = (rhs_packet_size <= 8 || kc_ <= rhs_packet_size) ? + kc_ : (kc_ / rhs_packet_size) * rhs_packet_size; } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index kc() const { return kc_; } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index mc() const { return mc_; } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index nc() const { return nc_; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE StorageIndex kc() const { return kc_; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE StorageIndex mc() const { return mc_; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE StorageIndex nc() const { return nc_; } private: - Index kc_; - Index mc_; - Index nc_; + StorageIndex kc_; + StorageIndex mc_; + StorageIndex nc_; }; + +#if defined(EIGEN_USE_LIBXSMM) +template <typename LhsScalar, typename RhsScalar, typename StorageIndex> +class TensorXsmmContractionBlocking { + public: + TensorXsmmContractionBlocking(StorageIndex k, StorageIndex m, StorageIndex n, + size_t max_num_threads = 1, bool transposeA = false, + bool transposeB = false): + k_(k), m_(m), n_(n), transposeA_(transposeA), + transposeB_(transposeB), num_threads_(max_num_threads) { +#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES + if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) { + mc_ = EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M; + kc_ = EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K; + nc_ = EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N; + outer_m_ = EIGEN_TEST_SPECIFIC_OUTER_BLOCKING_SIZE_M; + outer_k_ = EIGEN_TEST_SPECIFIC_OUTER_BLOCKING_SIZE_K; + outer_n_ = EIGEN_TEST_SPECIFIC_OUTER_BLOCKING_SIZE_N; + copyA_ = EIGEN_TEST_SPECIFIC_BLOCKING_COPY_A; + copyB_ = EIGEN_TEST_SPECIFIC_BLOCKING_COPY_B; + outer_m_ = outer_m_ != 0 ? outer_m_ : m; + outer_k_ = outer_k_ != 0 ? outer_k_ : k; + outer_n_ = outer_n_ != 0 ? outer_n_ : n; + } +#else + // Defaults, possibly overridden per-platform. + copyA_ = true; + copyB_ = false; + + // If the matrix is small enough, don't do blocking, just call single xsmm + // kernel. + if (static_cast<double>(m)*k*n <= LIBXSMM_THRESHOLD) { + mc_ = m; kc_ = k; nc_ = n; + outer_m_ = m; outer_k_ = k; outer_n_ = n; + copyA_ = false; copyB_ = false; + } else { + int arch = libxsmm_cpuid_x86(); + + if (arch == LIBXSMM_X86_AVX512_CORE) { + // skylake + mc_ = 64; kc_ = 64; nc_ = 24; + outer_m_ = 512; outer_k_ = 512; outer_n_ = 24*22; + // Hack to use this kernel architecture as the other one has performance + // issues (no hardware prefetching). + // TODO(nishantpatil): This should be removed if the issues are fixed, + // or this one becomes the default. + setenv("LIBXSMM_AVX512_CLASSIC_GEMM", "1", 1); + } else if (arch == LIBXSMM_X86_AVX2) { + // haswell + mc_ = 32; kc_ = 192; nc_ = 33; + outer_m_ = 512; outer_k_ = 3*192; outer_n_ = 33*16; + } else if (arch == LIBXSMM_X86_AVX) { + // ivybridge + mc_ = 32; kc_ = 192; nc_ = 48; + outer_m_ = 512; outer_k_ = 3*192; outer_n_ = 48*11; + } else { + // generic kernel size, usually performing well + mc_ = 32; kc_ = 128; nc_ = 32; + outer_m_ = 512; outer_k_ = 512; outer_n_ = 512; + } + + // Only copy if it makes the stride smaller. + copyA_ = copyA_ && (m > mc_); + copyB_ = copyB_ && (k > kc_); + } + + // We need to copy anyway if transposing + copyA_ = copyA_ || transposeA; + copyB_ = copyB_ || transposeB; + + // See libxsmm_gemm_prefetch_type definition in libxsmm_typedefs.h + prefetch_ = LIBXSMM_PREFETCH_AL2CL2BL2_VIA_C; + +#endif + + mc_ = mc_ > m ? m : mc_; + nc_ = nc_ > n ? n : nc_; + kc_ = kc_ > k ? k : kc_; + + size_t compute_parallelism = (m / mc_) * (n / nc_); + size_t pack_parallelism = 0; + if (copyA_) { + pack_parallelism += (m / mc_) * (k / kc_); + } + if (copyB_) { + pack_parallelism += (n / nc_) * (k / kc_); + } + size_t parallelism = numext::maxi(compute_parallelism, pack_parallelism); + + num_threads_ = numext::mini<size_t>(num_threads_, + parallelism / MIN_JOBS_PER_THREAD); + num_threads_ = numext::maxi<size_t>(num_threads_, 1); + + // For optimal performance outer block sizes should be multiplies of kernel + // sizes, or bigger than matrix size (=no outer blocking). + eigen_assert(outer_m_ % mc_ == 0 || outer_m_ >= m); + eigen_assert(outer_k_ % kc_ == 0 || outer_k_ >= k); + eigen_assert(outer_n_ % nc_ == 0 || outer_n_ >= n); + } + + EIGEN_ALWAYS_INLINE StorageIndex kc() const { return kc_; } + EIGEN_ALWAYS_INLINE StorageIndex mc() const { return mc_; } + EIGEN_ALWAYS_INLINE StorageIndex nc() const { return nc_; } + EIGEN_ALWAYS_INLINE StorageIndex outer_k() const { return outer_k_; } + EIGEN_ALWAYS_INLINE StorageIndex outer_m() const { return outer_m_; } + EIGEN_ALWAYS_INLINE StorageIndex outer_n() const { return outer_n_; } + EIGEN_ALWAYS_INLINE bool copyA() const { return copyA_; } + EIGEN_ALWAYS_INLINE bool copyB() const { return copyB_; } + EIGEN_ALWAYS_INLINE bool transposeA() const { return transposeA_; } + EIGEN_ALWAYS_INLINE bool transposeB() const { return transposeB_; } + EIGEN_ALWAYS_INLINE int num_threads() const { return num_threads_; } + EIGEN_ALWAYS_INLINE StorageIndex blocks_m() const { return divup(m_, mc_); } + EIGEN_ALWAYS_INLINE StorageIndex blocks_k() const { return divup(k_, kc_); } + EIGEN_ALWAYS_INLINE StorageIndex blocks_n() const { return divup(n_, nc_); } + EIGEN_ALWAYS_INLINE libxsmm_gemm_prefetch_type prefetch() const { + return prefetch_; + } + + private: + StorageIndex k_, m_, n_; + StorageIndex kc_, mc_, nc_; + StorageIndex outer_k_, outer_m_, outer_n_; + bool copyA_, copyB_, transposeA_, transposeB_; + size_t num_threads_; + + // Threshold for m*k*n to skip blocking and just call libxsmm + const double LIBXSMM_THRESHOLD = 80*80*80; + // For computing optimal number of threads - so that each thread gets at least + // that many jobs. + const double MIN_JOBS_PER_THREAD = 3; + libxsmm_gemm_prefetch_type prefetch_; +}; +#endif // EIGEN_USE_LIBXSMM + } // end namespace internal } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h index dbff660..3f315fe 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h
@@ -1,1391 +1,6 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014-2015 Benoit Steiner <benoit.steiner.goog@gmail.com> -// Copyright (C) 2015 Navdeep Jaitly <ndjaitly@google.com> -// Copyright (C) 2014 Eric Martin <eric@ericmart.in> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_CUDA_H -#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_CUDA_H +#if defined(__clang__) || defined(__GNUC__) +#warning "Deprecated header file, please either include the main Eigen/CXX11/Tensor header or the respective TensorContractionGpu.h file" +#endif -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) - -namespace Eigen { - -template<typename Scalar, typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper, bool needs_edge_check> -__device__ EIGEN_STRONG_INLINE void -EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, Scalar* lhs_shmem, Scalar* rhs_shmem, - const Index m_size, const Index n_size, const Index k_size) { - - const Index m_block_idx = blockIdx.x; - const Index n_block_idx = blockIdx.y; - - const Index base_m = 64 * m_block_idx; - const Index base_n = 64 * n_block_idx; - - // declare and initialize 64 registers for output 8x8 block - - // prefetch registers - Scalar lhs_pf0; - Scalar lhs_pf1; - Scalar lhs_pf2; - Scalar lhs_pf3; - Scalar lhs_pf4; - Scalar lhs_pf5; - Scalar lhs_pf6; - Scalar lhs_pf7; - - Scalar rhs_pf0; - Scalar rhs_pf1; - Scalar rhs_pf2; - Scalar rhs_pf3; - Scalar rhs_pf4; - Scalar rhs_pf5; - Scalar rhs_pf6; - Scalar rhs_pf7; - - // shared memory is formatted - // (contract idx in block, nocontract idx in block, block idx) - // where block idx is column major. This transposition limits the number of - // bank conflicts when reading the LHS. The core idea is that since the contracting - // index is shared by both sides, then the contracting index should be in threadIdx.x. - - // On the LHS, we pad each row inside of each block with an extra element. This makes - // each block 8 rows of 9 elements, which is 72 elements. This gives no bank conflicts - // on writes and very few 2-way conflicts on reads. There is an 8x8 grid of these blocks. - - // On the RHS we just add 8 padding elements to the end of each block. This gives no bank - // conflicts on writes and also none on reads. - - // storage indices - const Index lhs_store_idx_base = threadIdx.y * 72 + threadIdx.x * 9 + threadIdx.z; - const Index rhs_store_idx_base = threadIdx.y * 72 + threadIdx.z * 8 + threadIdx.x; - - const Index lhs_store_idx_0 = lhs_store_idx_base + 576 * 0; - const Index lhs_store_idx_1 = lhs_store_idx_base + 576 * 1; - const Index lhs_store_idx_2 = lhs_store_idx_base + 576 * 2; - const Index lhs_store_idx_3 = lhs_store_idx_base + 576 * 3; - const Index lhs_store_idx_4 = lhs_store_idx_base + 576 * 4; - const Index lhs_store_idx_5 = lhs_store_idx_base + 576 * 5; - const Index lhs_store_idx_6 = lhs_store_idx_base + 576 * 6; - const Index lhs_store_idx_7 = lhs_store_idx_base + 576 * 7; - - const Index rhs_store_idx_0 = rhs_store_idx_base + 576 * 0; - const Index rhs_store_idx_1 = rhs_store_idx_base + 576 * 1; - const Index rhs_store_idx_2 = rhs_store_idx_base + 576 * 2; - const Index rhs_store_idx_3 = rhs_store_idx_base + 576 * 3; - const Index rhs_store_idx_4 = rhs_store_idx_base + 576 * 4; - const Index rhs_store_idx_5 = rhs_store_idx_base + 576 * 5; - const Index rhs_store_idx_6 = rhs_store_idx_base + 576 * 6; - const Index rhs_store_idx_7 = rhs_store_idx_base + 576 * 7; - - // in the loading code, the following variables are important: - // threadIdx.x: the vertical position in an 8x8 block - // threadIdx.y: the vertical index of the 8x8 block in the grid - // threadIdx.z: the horizontal position in an 8x8 block - // k: the horizontal index of the 8x8 block in the grid - // - // The k parameter is implicit (it was the loop counter for a loop that went - // from 0 to <8, but now that loop is unrolled in the below code. - - const Index load_idx_vert = threadIdx.x + 8 * threadIdx.y; - const Index lhs_vert = base_m + load_idx_vert; - -#define prefetchIntoRegisters(base_k) \ - { \ - lhs_pf0 = conv(0); \ - lhs_pf1 = conv(0); \ - lhs_pf2 = conv(0); \ - lhs_pf3 = conv(0); \ - lhs_pf4 = conv(0); \ - lhs_pf5 = conv(0); \ - lhs_pf6 = conv(0); \ - lhs_pf7 = conv(0); \ - \ - rhs_pf0 = conv(0); \ - rhs_pf1 = conv(0); \ - rhs_pf2 = conv(0); \ - rhs_pf3 = conv(0); \ - rhs_pf4 = conv(0); \ - rhs_pf5 = conv(0); \ - rhs_pf6 = conv(0); \ - rhs_pf7 = conv(0); \ - \ - if (!needs_edge_check || lhs_vert < m_size) { \ - const Index lhs_horiz_0 = base_k + threadIdx.z + 0 * 8; \ - const Index lhs_horiz_1 = base_k + threadIdx.z + 1 * 8; \ - const Index lhs_horiz_2 = base_k + threadIdx.z + 2 * 8; \ - const Index lhs_horiz_3 = base_k + threadIdx.z + 3 * 8; \ - const Index lhs_horiz_4 = base_k + threadIdx.z + 4 * 8; \ - const Index lhs_horiz_5 = base_k + threadIdx.z + 5 * 8; \ - const Index lhs_horiz_6 = base_k + threadIdx.z + 6 * 8; \ - const Index lhs_horiz_7 = base_k + threadIdx.z + 7 * 8; \ - \ - if (!needs_edge_check || lhs_horiz_7 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ - lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ - lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ - lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ - lhs_pf7 = lhs(lhs_vert, lhs_horiz_7); \ - } else if (lhs_horiz_6 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ - lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ - lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ - lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ - } else if (lhs_horiz_5 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ - lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ - lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ - } else if (lhs_horiz_4 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ - lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ - } else if (lhs_horiz_3 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ - } else if (lhs_horiz_2 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ - } else if (lhs_horiz_1 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ - } else if (lhs_horiz_0 < k_size) { \ - lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ - } \ - } \ - \ - const Index rhs_vert = base_k + load_idx_vert; \ - if (!needs_edge_check || rhs_vert < k_size) { \ - const Index rhs_horiz_0 = base_n + threadIdx.z + 0 * 8; \ - const Index rhs_horiz_1 = base_n + threadIdx.z + 1 * 8; \ - const Index rhs_horiz_2 = base_n + threadIdx.z + 2 * 8; \ - const Index rhs_horiz_3 = base_n + threadIdx.z + 3 * 8; \ - const Index rhs_horiz_4 = base_n + threadIdx.z + 4 * 8; \ - const Index rhs_horiz_5 = base_n + threadIdx.z + 5 * 8; \ - const Index rhs_horiz_6 = base_n + threadIdx.z + 6 * 8; \ - const Index rhs_horiz_7 = base_n + threadIdx.z + 7 * 8; \ - \ - if (rhs_horiz_7 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ - rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ - rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ - rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ - rhs_pf7 = rhs(rhs_vert, rhs_horiz_7); \ - } else if (rhs_horiz_6 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ - rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ - rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ - rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ - } else if (rhs_horiz_5 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ - rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ - rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ - } else if (rhs_horiz_4 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ - rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ - } else if (rhs_horiz_3 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ - } else if (rhs_horiz_2 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ - } else if (rhs_horiz_1 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ - } else if (rhs_horiz_0 < n_size) { \ - rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ - } \ - } \ - } \ - -#define writeRegToShmem(_) \ - lhs_shmem[lhs_store_idx_0] = lhs_pf0; \ - rhs_shmem[rhs_store_idx_0] = rhs_pf0; \ - \ - lhs_shmem[lhs_store_idx_1] = lhs_pf1; \ - rhs_shmem[rhs_store_idx_1] = rhs_pf1; \ - \ - lhs_shmem[lhs_store_idx_2] = lhs_pf2; \ - rhs_shmem[rhs_store_idx_2] = rhs_pf2; \ - \ - lhs_shmem[lhs_store_idx_3] = lhs_pf3; \ - rhs_shmem[rhs_store_idx_3] = rhs_pf3; \ - \ - lhs_shmem[lhs_store_idx_4] = lhs_pf4; \ - rhs_shmem[rhs_store_idx_4] = rhs_pf4; \ - \ - lhs_shmem[lhs_store_idx_5] = lhs_pf5; \ - rhs_shmem[rhs_store_idx_5] = rhs_pf5; \ - \ - lhs_shmem[lhs_store_idx_6] = lhs_pf6; \ - rhs_shmem[rhs_store_idx_6] = rhs_pf6; \ - \ - lhs_shmem[lhs_store_idx_7] = lhs_pf7; \ - rhs_shmem[rhs_store_idx_7] = rhs_pf7; \ - - // declare and initialize result array -#define res(i, j) _res_##i##j -#define initResultRow(i) \ - Scalar res(i, 0) = conv(0); \ - Scalar res(i, 1) = conv(0); \ - Scalar res(i, 2) = conv(0); \ - Scalar res(i, 3) = conv(0); \ - Scalar res(i, 4) = conv(0); \ - Scalar res(i, 5) = conv(0); \ - Scalar res(i, 6) = conv(0); \ - Scalar res(i, 7) = conv(0); \ - - internal::scalar_cast_op<int, Scalar> conv; - initResultRow(0); - initResultRow(1); - initResultRow(2); - initResultRow(3); - initResultRow(4); - initResultRow(5); - initResultRow(6); - initResultRow(7); -#undef initResultRow - - for (Index base_k = 0; base_k < k_size; base_k += 64) { - // wait for previous iteration to finish with shmem. Despite common sense, - // the code is a bit faster with this here then at bottom of loop - __syncthreads(); - - prefetchIntoRegisters(base_k); - writeRegToShmem(); - - #undef prefetchIntoRegisters - #undef writeRegToShmem - - // wait for shared mem packing to be done before starting computation - __syncthreads(); - - // compute 8x8 matrix product by outer product. This involves packing one column - // of LHS and one row of RHS into registers (takes 16 registers). - -#define lcol(i) _lcol##i - Scalar lcol(0); - Scalar lcol(1); - Scalar lcol(2); - Scalar lcol(3); - Scalar lcol(4); - Scalar lcol(5); - Scalar lcol(6); - Scalar lcol(7); - -#define rrow(j) _rrow##j - Scalar rrow(0); - Scalar rrow(1); - Scalar rrow(2); - Scalar rrow(3); - Scalar rrow(4); - Scalar rrow(5); - Scalar rrow(6); - Scalar rrow(7); - - // Now x corresponds to k, y to m, and z to n - const Scalar* lhs_block = &lhs_shmem[threadIdx.x + 9 * threadIdx.y]; - const Scalar* rhs_block = &rhs_shmem[threadIdx.x + 8 * threadIdx.z]; - -#define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))] -#define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))] - -#define loadData(i, j) \ - lcol(0) = lhs_element(0, j); \ - rrow(0) = rhs_element(i, 0); \ - lcol(1) = lhs_element(1, j); \ - rrow(1) = rhs_element(i, 1); \ - lcol(2) = lhs_element(2, j); \ - rrow(2) = rhs_element(i, 2); \ - lcol(3) = lhs_element(3, j); \ - rrow(3) = rhs_element(i, 3); \ - lcol(4) = lhs_element(4, j); \ - rrow(4) = rhs_element(i, 4); \ - lcol(5) = lhs_element(5, j); \ - rrow(5) = rhs_element(i, 5); \ - lcol(6) = lhs_element(6, j); \ - rrow(6) = rhs_element(i, 6); \ - lcol(7) = lhs_element(7, j); \ - rrow(7) = rhs_element(i, 7); \ - -#define computeCol(j) \ - res(0, j) += lcol(0) * rrow(j); \ - res(1, j) += lcol(1) * rrow(j); \ - res(2, j) += lcol(2) * rrow(j); \ - res(3, j) += lcol(3) * rrow(j); \ - res(4, j) += lcol(4) * rrow(j); \ - res(5, j) += lcol(5) * rrow(j); \ - res(6, j) += lcol(6) * rrow(j); \ - res(7, j) += lcol(7) * rrow(j); \ - -#define computePass(i) \ - loadData(i, i); \ - \ - computeCol(0); \ - computeCol(1); \ - computeCol(2); \ - computeCol(3); \ - computeCol(4); \ - computeCol(5); \ - computeCol(6); \ - computeCol(7); \ - - computePass(0); - computePass(1); - computePass(2); - computePass(3); - computePass(4); - computePass(5); - computePass(6); - computePass(7); - -#undef lcol -#undef rrow -#undef lhs_element -#undef rhs_element -#undef loadData -#undef computeCol -#undef computePass - } // end loop over k - - // we've now iterated over all of the large (ie width 64) k blocks and - // accumulated results in registers. At this point thread (x, y, z) contains - // the sum across all big k blocks of the product of little k block of index (x, y) - // with block of index (y, z). To compute the final output, we need to reduce - // the 8 threads over y by summation. -#define shuffleInc(i, j, mask) res(i, j) += __shfl_xor(res(i, j), mask) - -#define reduceRow(i, mask) \ - shuffleInc(i, 0, mask); \ - shuffleInc(i, 1, mask); \ - shuffleInc(i, 2, mask); \ - shuffleInc(i, 3, mask); \ - shuffleInc(i, 4, mask); \ - shuffleInc(i, 5, mask); \ - shuffleInc(i, 6, mask); \ - shuffleInc(i, 7, mask); \ - -#define reduceMatrix(mask) \ - reduceRow(0, mask); \ - reduceRow(1, mask); \ - reduceRow(2, mask); \ - reduceRow(3, mask); \ - reduceRow(4, mask); \ - reduceRow(5, mask); \ - reduceRow(6, mask); \ - reduceRow(7, mask); \ - - // actually perform the reduction, now each thread of index (_, y, z) - // contains the correct values in its registers that belong in the output - // block - reduceMatrix(1); - reduceMatrix(2); - reduceMatrix(4); - -#undef shuffleInc -#undef reduceRow -#undef reduceMatrix - - // now we need to copy the 64 values into main memory. We can't split work - // among threads because all variables are in registers. There's 2 ways - // to do this: - // (1) have 1 thread do 64 writes from registers into global memory - // (2) have 1 thread do 64 writes into shared memory, and then 8 threads - // each do 8 writes into global memory. We can just overwrite the shared - // memory from the problem we just solved. - // (2) is slightly faster than (1) due to less branching and more ILP - - // TODO: won't yield much gain, but could just use currently unused shared mem - // and then we won't have to sync - // wait for shared mem to be out of use - __syncthreads(); - -#define writeResultShmem(i, j) \ - lhs_shmem[i + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j] = res(i, j); \ - -#define writeRow(i) \ - writeResultShmem(i, 0); \ - writeResultShmem(i, 1); \ - writeResultShmem(i, 2); \ - writeResultShmem(i, 3); \ - writeResultShmem(i, 4); \ - writeResultShmem(i, 5); \ - writeResultShmem(i, 6); \ - writeResultShmem(i, 7); \ - - if (threadIdx.x == 0) { - writeRow(0); - writeRow(1); - writeRow(2); - writeRow(3); - writeRow(4); - writeRow(5); - writeRow(6); - writeRow(7); - } -#undef writeResultShmem -#undef writeRow - - const int max_i_write = (min)((int)((m_size - base_m - threadIdx.y + 7) / 8), 8); - const int max_j_write = (min)((int)((n_size - base_n - threadIdx.z + 7) / 8), 8); - - if (threadIdx.x < max_i_write) { - if (max_j_write == 8) { - // TODO: can i trade bank conflicts for coalesced writes? - Scalar val0 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 0]; - Scalar val1 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 1]; - Scalar val2 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 2]; - Scalar val3 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 3]; - Scalar val4 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 4]; - Scalar val5 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 5]; - Scalar val6 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 6]; - Scalar val7 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 7]; - - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 0) = val0; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 1) = val1; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 2) = val2; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 3) = val3; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 4) = val4; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 5) = val5; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 6) = val6; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 7) = val7; - } else { -#pragma unroll 7 - for (int j = 0; j < max_j_write; j++) { - Scalar val = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j]; - output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * j) = val; - } - } - } -#undef res -} - - -template<typename Scalar, typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper> -__global__ void -__launch_bounds__(512) -EigenContractionKernel(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, - const Index m_size, const Index n_size, const Index k_size) { - __shared__ Scalar lhs_shmem[72 * 64]; - __shared__ Scalar rhs_shmem[72 * 64]; - - const Index m_block_idx = blockIdx.x; - const Index n_block_idx = blockIdx.y; - - const Index base_m = 64 * m_block_idx; - const Index base_n = 64 * n_block_idx; - - if (base_m + 63 < m_size && base_n + 63 < n_size) { - EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size); - } else { - EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size); - } -} - - -template<typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY, - bool CHECK_RHS_BOUNDARY> -__device__ EIGEN_STRONG_INLINE void -EigenFloatContractionKernelInternal16x16(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, float2 lhs_shmem2[][16], - float2 rhs_shmem2[][8], const Index m_size, - const Index n_size, const Index k_size, - const Index base_m, const Index base_n) { - typedef float Scalar; - - // prefetch registers - float4 lhs_pf0, rhs_pf0; - - float4 results[4]; - for (int i=0; i < 4; i++) { - results[i].x = results[i].y = results[i].z = results[i].w = 0; - } - - -#define prefetch_lhs(reg, row, col) \ - if (!CHECK_LHS_BOUNDARY) { \ - if (col < k_size) { \ - reg =lhs.loadPacket(row, col); \ - } \ - } else { \ - if (col < k_size) { \ - if (row + 3 < m_size) { \ - reg =lhs.loadPacket(row, col); \ - } else if (row + 2 < m_size) { \ - reg.x =lhs(row + 0, col); \ - reg.y =lhs(row + 1, col); \ - reg.z =lhs(row + 2, col); \ - } else if (row + 1 < m_size) { \ - reg.x =lhs(row + 0, col); \ - reg.y =lhs(row + 1, col); \ - } else if (row < m_size) { \ - reg.x =lhs(row + 0, col); \ - } \ - } \ - } \ - - - Index lhs_vert = base_m+threadIdx.x*4; - - for (Index k = 0; k < k_size; k += 16) { - lhs_pf0 = internal::pset1<float4>(0); - rhs_pf0 = internal::pset1<float4>(0); - - Index lhs_horiz = threadIdx.y+k; - prefetch_lhs(lhs_pf0, lhs_vert, lhs_horiz) - - Index rhs_vert = k+(threadIdx.x%4)*4; - Index rhs_horiz0 = (threadIdx.x>>2)+threadIdx.y*4+base_n; - - if (!CHECK_RHS_BOUNDARY) { - if ((rhs_vert + 3) < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0 = rhs.loadPacket(rhs_vert, rhs_horiz0); - } else if (rhs_vert + 2 < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); - } else if (rhs_vert + 1 < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - } else if (rhs_vert < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - } - } else { - if (rhs_horiz0 < n_size) { - if ((rhs_vert + 3) < k_size) { - rhs_pf0 = rhs.loadPacket(rhs_vert, rhs_horiz0); - } else if ((rhs_vert + 2) < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); - } else if ((rhs_vert + 1) < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - } else if (rhs_vert < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - } - } - } - float x1, x2 ; - // the following can be a bitwise operation..... some day. - if((threadIdx.x%8) < 4) { - x1 = rhs_pf0.y; - x2 = rhs_pf0.w; - } else { - x1 = rhs_pf0.x; - x2 = rhs_pf0.z; - } - x1 = __shfl_xor(x1, 4); - x2 = __shfl_xor(x2, 4); - if((threadIdx.x%8) < 4) { - rhs_pf0.y = x1; - rhs_pf0.w = x2; - } else { - rhs_pf0.x = x1; - rhs_pf0.z = x2; - } - - // We have 64 features. - // Row 0 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 0, 1. - // Row 1 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 2, 3. - // ... - // Row 31 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 62, 63 - // Row 32 -> times (2, 6, 10, 14, 3, 7, 11, 15) for features 0, 1 - // ... - rhs_shmem2[(threadIdx.x>>3)+ threadIdx.y*2][threadIdx.x%8] = make_float2(rhs_pf0.x, rhs_pf0.y); - rhs_shmem2[(threadIdx.x>>3)+ threadIdx.y*2+32][threadIdx.x%8] = make_float2(rhs_pf0.z, rhs_pf0.w); - - // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) - // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) - // ... - // Row 15 (time 15) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) - // Row 16 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) - // ... - - lhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(lhs_pf0.x, lhs_pf0.y); - lhs_shmem2[threadIdx.y+16][threadIdx.x] = make_float2(lhs_pf0.z, lhs_pf0.w); - - -#define add_vals(fl1, fl2, fr1, fr2)\ - results[0].x += fl1.x * fr1.x;\ - results[0].y += fl1.y * fr1.x;\ - results[0].z += fl2.x * fr1.x;\ - results[0].w += fl2.y * fr1.x;\ -\ - results[1].x += fl1.x * fr1.y;\ - results[1].y += fl1.y * fr1.y;\ - results[1].z += fl2.x * fr1.y;\ - results[1].w += fl2.y * fr1.y;\ -\ - results[2].x += fl1.x * fr2.x;\ - results[2].y += fl1.y * fr2.x;\ - results[2].z += fl2.x * fr2.x;\ - results[2].w += fl2.y * fr2.x;\ -\ - results[3].x += fl1.x * fr2.y;\ - results[3].y += fl1.y * fr2.y;\ - results[3].z += fl2.x * fr2.y;\ - results[3].w += fl2.y * fr2.y;\ - - __syncthreads(); - - // Do the multiplies. - #pragma unroll - for (int koff = 0; koff < 16; koff ++) { - // 32 x threads. - float2 fl1 = lhs_shmem2[koff][threadIdx.x]; - float2 fl2 = lhs_shmem2[koff + 16][threadIdx.x]; - - int start_feature = threadIdx.y * 4; - float2 fr1 = rhs_shmem2[(start_feature>>1) + 32*((koff%4)/2)][koff/4 + (koff%2)*4]; - float2 fr2 = rhs_shmem2[(start_feature>>1) + 1 + 32*((koff%4)/2)][koff/4 + (koff%2)*4]; - - add_vals(fl1, fl2, fr1, fr2) - } - __syncthreads(); - } - -#undef prefetch_lhs -#undef add_vals - - Index horiz_base = threadIdx.y*4+base_n; - if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { - for (int i = 0; i < 4; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } else if (!CHECK_RHS_BOUNDARY) { - // CHECK LHS - if (lhs_vert + 3 < m_size) { - for (int i = 0; i < 4; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } else if (lhs_vert + 2 < m_size) { - for (int i = 0; i < 4; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - } - } else if (lhs_vert + 1 < m_size) { - for (int i = 0; i < 4; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - } - } else if (lhs_vert < m_size) { - for (int i = 0; i < 4; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - } - } - } else if (!CHECK_LHS_BOUNDARY) { - // CHECK RHS - /* - int ncols_rem = fminf(n_size- horiz_base, 4); - for (int i = 0; i < ncols_rem; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - }*/ - for (int i = 0; i < 4; i++) { - if (horiz_base+i < n_size) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } - } else { - // CHECK both boundaries. - for (int i = 0; i < 4; i++) { - if (horiz_base+i < n_size) { - if (lhs_vert < m_size) - output(lhs_vert, horiz_base + i) = results[i].x; - if (lhs_vert + 1 < m_size) - output(lhs_vert + 1, horiz_base + i) = results[i].y; - if (lhs_vert + 2 < m_size) - output(lhs_vert + 2, horiz_base + i) = results[i].z; - if (lhs_vert + 3 < m_size) - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } - } -} - - -template<typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY, - bool CHECK_RHS_BOUNDARY> -__device__ EIGEN_STRONG_INLINE void -EigenFloatContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, float2 lhs_shmem2[][32], - float2 rhs_shmem2[][8], const Index m_size, - const Index n_size, const Index k_size, - const Index base_m, const Index base_n) { - typedef float Scalar; - - // prefetch registers - float4 lhs_pf0, lhs_pf1, lhs_pf2, lhs_pf3; - float4 rhs_pf0, rhs_pf1; - - float4 results[8]; - for (int i=0; i < 8; i++) { - results[i].x = results[i].y = results[i].z = results[i].w = 0; - } - - - Index lhs_vert = base_m+threadIdx.x*4+(threadIdx.y%4)*32; - for (Index k = 0; k < k_size; k += 32) { - lhs_pf0 = internal::pset1<float4>(0); - lhs_pf1 = internal::pset1<float4>(0); - lhs_pf2 = internal::pset1<float4>(0); - lhs_pf3 = internal::pset1<float4>(0); - - rhs_pf0 = internal::pset1<float4>(0); - rhs_pf1 = internal::pset1<float4>(0); - - if (!CHECK_LHS_BOUNDARY) { - if ((threadIdx.y/4+k+24) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - lhs_pf2 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+16)); - lhs_pf3 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+24)); - } else if ((threadIdx.y/4+k+16) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - lhs_pf2 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+16)); - } else if ((threadIdx.y/4+k+8) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - } else if ((threadIdx.y/4+k) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - } - } else { - // just CHECK_LHS_BOUNDARY - if (lhs_vert + 3 < m_size) { - if ((threadIdx.y/4+k+24) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - lhs_pf2 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+16)); - lhs_pf3 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+24)); - } else if ((threadIdx.y/4+k+16) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - lhs_pf2 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+16)); - } else if ((threadIdx.y/4+k+8) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - lhs_pf1 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k+8)); - } else if ((threadIdx.y/4+k) < k_size) { - lhs_pf0 =lhs.loadPacket(lhs_vert, (threadIdx.y/4+k)); - } - } else if (lhs_vert + 2 < m_size) { - if ((threadIdx.y/4+k+24) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); - lhs_pf2.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+16)); - lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); - lhs_pf3.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+24)); - lhs_pf3.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+24)); - } else if ((threadIdx.y/4+k+16) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); - lhs_pf2.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+16)); - } else if ((threadIdx.y/4+k+8) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); - } else if ((threadIdx.y/4+k) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); - } - } else if (lhs_vert + 1 < m_size) { - if ((threadIdx.y/4+k+24) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); - lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); - lhs_pf3.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+24)); - } else if ((threadIdx.y/4+k+16) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); - } else if ((threadIdx.y/4+k+8) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); - } else if ((threadIdx.y/4+k) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); - } - } else if (lhs_vert < m_size) { - if ((threadIdx.y/4+k+24) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); - } else if ((threadIdx.y/4+k+16) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); - } else if ((threadIdx.y/4+k+8) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); - } else if ((threadIdx.y/4+k) < k_size) { - lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); - } - } - } - __syncthreads(); - Index rhs_vert = k+threadIdx.x*4; - Index rhs_horiz0 = threadIdx.y*2+base_n; - Index rhs_horiz1 = threadIdx.y*2+1+base_n; - if (!CHECK_RHS_BOUNDARY) { - if ((rhs_vert + 3) < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0 = rhs.loadPacket(rhs_vert, rhs_horiz0); - rhs_pf1 = rhs.loadPacket(rhs_vert, rhs_horiz1); - } else if (rhs_vert + 2 < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); - rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1); - } else if (rhs_vert + 1 < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); - } else if (rhs_vert < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - } - } else { - if (rhs_horiz1 < n_size) { - if ((rhs_vert + 3) < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0 = rhs.loadPacket(rhs_vert, rhs_horiz0); - rhs_pf1 = rhs.loadPacket(rhs_vert, rhs_horiz1); - } else if (rhs_vert + 2 < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); - rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1); - } else if (k+threadIdx.x*4 + 1 < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); - } else if (k+threadIdx.x*4 < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); - } - } else if (rhs_horiz0 < n_size) { - if ((rhs_vert + 3) < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0 = rhs.loadPacket(rhs_vert, rhs_horiz0); - } else if ((rhs_vert + 2) < k_size) { - // just CHECK_RHS_BOUNDARY - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); - } else if ((rhs_vert + 1) < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); - } else if (rhs_vert < k_size) { - rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); - } - } - } - __syncthreads(); - // Loaded. Do computation - // Row 0 -> times (0, 4, 8, .. 28) for features 0, 1. - // Row 1 -> times (0, 4, 8, .. 28) for features 2, 3. - // .. - // Row 31 -> times (0, 4, 8, .. 28) for features 62, 63 - rhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(rhs_pf0.x, rhs_pf1.x); - // Row 32 -> times (1, 5, 9, .. 29) for features 0, 1. - // Row 33 -> times (1, 5, 9, .. 29) for features 2, 3. - // .. - rhs_shmem2[threadIdx.y+32][threadIdx.x] = make_float2(rhs_pf0.y, rhs_pf1.y); - // Row 64 -> times (2, 6, 10, .. 30) for features 0, 1. - // Row 65 -> times (2, 6, 10, .. 30) for features 2, 3. - rhs_shmem2[threadIdx.y+64][threadIdx.x] = make_float2(rhs_pf0.z, rhs_pf1.z); - // Row 96 -> times (3, 7, 11, .. 31) for features 0, 1. - // Row 97 -> times (3, 7, 11, .. 31) for features 2, 3. - rhs_shmem2[threadIdx.y+96][threadIdx.x] = make_float2(rhs_pf0.w, rhs_pf1.w); - - // LHS. - // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125) - // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125) - // ... - // Row 8 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127) - // Row 15 (time 7) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127) - - -#define add_vals(a_feat1, a_feat2, f1, f2, f3, f4)\ - results[0].x += a_feat1.x * f1.x;\ - results[1].x += a_feat1.x * f1.y;\ - results[2].x += a_feat1.x * f2.x;\ - results[3].x += a_feat1.x * f2.y;\ - results[4].x += a_feat1.x * f3.x;\ - results[5].x += a_feat1.x * f3.y;\ - results[6].x += a_feat1.x * f4.x;\ - results[7].x += a_feat1.x * f4.y;\ -\ - results[0].y += a_feat1.y * f1.x;\ - results[1].y += a_feat1.y * f1.y;\ - results[2].y += a_feat1.y * f2.x;\ - results[3].y += a_feat1.y * f2.y;\ - results[4].y += a_feat1.y * f3.x;\ - results[5].y += a_feat1.y * f3.y;\ - results[6].y += a_feat1.y * f4.x;\ - results[7].y += a_feat1.y * f4.y;\ -\ - results[0].z += a_feat2.x * f1.x;\ - results[1].z += a_feat2.x * f1.y;\ - results[2].z += a_feat2.x * f2.x;\ - results[3].z += a_feat2.x * f2.y;\ - results[4].z += a_feat2.x * f3.x;\ - results[5].z += a_feat2.x * f3.y;\ - results[6].z += a_feat2.x * f4.x;\ - results[7].z += a_feat2.x * f4.y;\ -\ - results[0].w += a_feat2.y * f1.x;\ - results[1].w += a_feat2.y * f1.y;\ - results[2].w += a_feat2.y * f2.x;\ - results[3].w += a_feat2.y * f2.y;\ - results[4].w += a_feat2.y * f3.x;\ - results[5].w += a_feat2.y * f3.y;\ - results[6].w += a_feat2.y * f4.x;\ - results[7].w += a_feat2.y * f4.y;\ - - lhs_shmem2[threadIdx.y/4][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf0.x, lhs_pf0.y); - lhs_shmem2[threadIdx.y/4+8][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf1.x, lhs_pf1.y); - lhs_shmem2[threadIdx.y/4+16][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf2.x, lhs_pf2.y); - lhs_shmem2[threadIdx.y/4+24][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf3.x, lhs_pf3.y); - - lhs_shmem2[threadIdx.y/4 + 32][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf0.z, lhs_pf0.w); - lhs_shmem2[threadIdx.y/4 + 40][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf1.z, lhs_pf1.w); - lhs_shmem2[threadIdx.y/4 + 48][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf2.z, lhs_pf2.w); - lhs_shmem2[threadIdx.y/4 + 56][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf3.z, lhs_pf3.w); - - __syncthreads(); - - // Do the multiplies. - #pragma unroll - for (int koff = 0; koff < 32; koff ++) { - float2 a3 = lhs_shmem2[koff][threadIdx.x + (threadIdx.y % 4) * 8]; - float2 a4 = lhs_shmem2[koff + 32][threadIdx.x + (threadIdx.y % 4) * 8]; - - // first feature is at (threadIdx.y/4) * 8 last is at start + 8. - int start_feature = (threadIdx.y / 4) * 8; - - float2 br1 = rhs_shmem2[start_feature/2 + (koff % 4) * 32][koff/4]; - float2 br2 = rhs_shmem2[start_feature/2 + 1 + (koff % 4) * 32][koff/4]; - float2 br3 = rhs_shmem2[start_feature/2 + 2 + (koff % 4) * 32][koff/4]; - float2 br4 = rhs_shmem2[start_feature/2 + 3 + (koff % 4) * 32][koff/4]; - - add_vals(a3, a4, br1, br2, br3, br4) - } - __syncthreads(); - } // end loop over k - - - __syncthreads(); - Index horiz_base = (threadIdx.y/4)*8+base_n; - if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { - for (int i = 0; i < 8; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } else if (!CHECK_RHS_BOUNDARY) { - if (lhs_vert + 3 < m_size) { - for (int i = 0; i < 8; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } else if (lhs_vert + 2 < m_size) { - for (int i = 0; i < 8; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - } - } else if (lhs_vert + 1 < m_size) { - for (int i = 0; i < 8; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - } - } else if (lhs_vert < m_size) { - for (int i = 0; i < 8; i++) { - output(lhs_vert, horiz_base + i) = results[i].x; - } - } - } else if (!CHECK_LHS_BOUNDARY) { - // CHECK BOUNDARY_B - for (int i = 0; i < 8; i++) { - if (horiz_base + i < n_size) { - output(lhs_vert, horiz_base + i) = results[i].x; - output(lhs_vert + 1, horiz_base + i) = results[i].y; - output(lhs_vert + 2, horiz_base + i) = results[i].z; - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } - } else { - // CHECK both boundaries. - for (int i = 0; i < 8; i++) { - if (horiz_base + i < n_size) { - if (lhs_vert < m_size) - output(lhs_vert, horiz_base + i) = results[i].x; - if (lhs_vert + 1 < m_size) - output(lhs_vert + 1, horiz_base + i) = results[i].y; - if (lhs_vert + 2 < m_size) - output(lhs_vert + 2, horiz_base + i) = results[i].z; - if (lhs_vert + 3 < m_size) - output(lhs_vert + 3, horiz_base + i) = results[i].w; - } - } - } -} - - -template<typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper> -__global__ void -__launch_bounds__(256) -EigenFloatContractionKernel(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, - const Index m_size, const Index n_size, const Index k_size) { - __shared__ float2 lhs_shmem[64*32]; - __shared__ float2 rhs_shmem[128*8]; - - typedef float2 LHS_MEM[64][32]; - typedef float2 RHS_MEM[128][8]; - - typedef float2 LHS_MEM16x16[32][16]; - typedef float2 RHS_MEM16x16[64][8]; - - const Index m_block_idx = blockIdx.x; - const Index n_block_idx = blockIdx.y; - - const Index base_m = 128 * m_block_idx; - const Index base_n = 64 * n_block_idx; - - bool check_rhs = (base_n + 63) >= n_size; - bool check_lhs128 = (base_m + 127) >= m_size; - - if (!check_rhs) { - if (!check_lhs128) { - // >= 128 rows left - EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, false>( - lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); - } else { - EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, false>( - lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); - } - } else { - if (!check_lhs128) { - // >= 128 rows left - EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, true>( - lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); - } else { - EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, true>( - lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); - } - } -} - -template<typename Index, typename LhsMapper, - typename RhsMapper, typename OutputMapper> -__global__ void -__launch_bounds__(256) -EigenFloatContractionKernel16x16(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, - const Index m_size, const Index n_size, const Index k_size) { - __shared__ float2 lhs_shmem[32][16]; - __shared__ float2 rhs_shmem[64][8]; - - const Index m_block_idx = blockIdx.x; - const Index n_block_idx = blockIdx.y; - - const Index base_m = 64 * m_block_idx; - const Index base_n = 64 * n_block_idx; - - if (base_m + 63 < m_size) { - if (base_n + 63 < n_size) { - EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); - } else { - EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); - } - } else { - if (base_n + 63 < n_size) { - EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); - } else { - EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); - } - } -} - - -template<typename Indices, typename LeftArgType, typename RightArgType> -struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, GpuDevice> : - public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, GpuDevice> > { - - typedef GpuDevice Device; - - typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> Self; - typedef TensorContractionEvaluatorBase<Self> Base; - - typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; - typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; - typedef typename XprType::Index Index; - typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType; - - enum { - Layout = TensorEvaluator<LeftArgType, Device>::Layout, - }; - - // Most of the code is assuming that both input tensors are ColMajor. If the - // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS: - // If we want to compute A * B = C, where A is LHS and B is RHS, the code - // will pretend B is LHS and A is RHS. - typedef typename internal::conditional< - static_cast<int>(Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>::type EvalLeftArgType; - typedef typename internal::conditional< - static_cast<int>(Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>::type EvalRightArgType; - - static const int LDims = - internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value; - static const int RDims = - internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value; - static const int ContractDims = internal::array_size<Indices>::value; - - typedef array<Index, LDims> left_dim_mapper_t; - typedef array<Index, RDims> right_dim_mapper_t; - - typedef array<Index, ContractDims> contract_t; - typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; - typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; - - static const int NumDims = max_n_1<LDims + RDims - 2 * ContractDims>::size; - - typedef DSizes<Index, NumDims> Dimensions; - - // typedefs needed in evalTo - typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; - typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; - - typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; - typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; - - typedef typename LeftEvaluator::Dimensions LeftDimensions; - typedef typename RightEvaluator::Dimensions RightDimensions; - - EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : - Base(op, device) {} - - // We need to redefine this method to make nvcc happy - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { - this->m_leftImpl.evalSubExprsIfNeeded(NULL); - this->m_rightImpl.evalSubExprsIfNeeded(NULL); - if (data) { - evalTo(data); - return false; - } else { - this->m_result = static_cast<Scalar *>(this->m_device.allocate(this->dimensions().TotalSize() * sizeof(Scalar))); - evalTo(this->m_result); - return true; - } - } - - void evalTo(Scalar* buffer) const { - if (this->m_lhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_reordered) { - evalTyped<true, true, true, Unaligned>(buffer); - } - else { - evalTyped<true, true, false, Unaligned>(buffer); - } - } - else { - if (this->m_rhs_inner_dim_reordered) { - evalTyped<true, false, true, Unaligned>(buffer); - } - else { - evalTyped<true, false, false, Unaligned>(buffer); - } - } - } - else { - if (this->m_rhs_inner_dim_contiguous) { - if (this->m_rhs_inner_dim_reordered) { - evalTyped<false, true, true, Unaligned>(buffer); - } - else { - evalTyped<false, true, false, Unaligned>(buffer); - } - } - else { - if (this->m_rhs_inner_dim_reordered) { - evalTyped<false, false, true, Unaligned>(buffer); - } - else { - evalTyped<false, false, false, Unaligned>(buffer); - } - } - } - } - - template <typename LhsScalar, typename RhsScalar, typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels { - static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { - const Index m_blocks = (m + 63) / 64; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(8, 8, 8); - LAUNCH_CUDA_KERNEL((EigenContractionKernel<Scalar, Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); - } - }; - - template <typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels<float, float, Index, LhsMapper, RhsMapper, OutputMapper> { - static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { - if (m < 768 || n < 768) { - const Index m_blocks = (m + 63) / 64; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(16, 16, 1); - LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel16x16<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); - } else { - const Index m_blocks = (m + 127) / 128; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(8, 32, 1); - LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); - } - } - }; - - template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - void evalTyped(Scalar* buffer) const { - // columns in left side, rows in right side - const Index k = this->m_k_size; - EIGEN_UNUSED_VARIABLE(k) - - // rows in left side - const Index m = this->m_i_size; - - // columns in right side - const Index n = this->m_j_size; - - // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar) - this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); - - typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, - LeftEvaluator, left_nocontract_t, - contract_t, 4, - lhs_inner_dim_contiguous, - false, Unaligned> LhsMapper; - - typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, - RightEvaluator, right_nocontract_t, - contract_t, 4, - rhs_inner_dim_contiguous, - rhs_inner_dim_reordered, Unaligned> RhsMapper; - - typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; - - - // initialize data mappers - LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides, - this->m_left_contracting_strides, this->m_k_strides); - - RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides, - this->m_right_contracting_strides, this->m_k_strides); - - OutputMapper output(buffer, m); - - setCudaSharedMemConfig(cudaSharedMemBankSizeEightByte); - LaunchKernels<LhsScalar, RhsScalar, Index, LhsMapper, RhsMapper, OutputMapper>::Run(lhs, rhs, output, m, n, k, this->m_device); - } -}; - -} // end namespace Eigen - -#endif // EIGEN_USE_GPU and __CUDACC__ -#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_CUDA_H +#include "TensorContractionGpu.h"
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionGpu.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionGpu.h new file mode 100644 index 0000000..5d19652 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionGpu.h
@@ -0,0 +1,1413 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014-2015 Benoit Steiner <benoit.steiner.goog@gmail.com> +// Copyright (C) 2015 Navdeep Jaitly <ndjaitly@google.com> +// Copyright (C) 2014 Eric Martin <eric@ericmart.in> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_GPU_H +#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_GPU_H + +#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC) + +namespace Eigen { + +template<typename Scalar, typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper, bool needs_edge_check> +__device__ EIGEN_STRONG_INLINE void +EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, Scalar* lhs_shmem, Scalar* rhs_shmem, + const Index m_size, const Index n_size, const Index k_size) { + + const Index m_block_idx = blockIdx.x; + const Index n_block_idx = blockIdx.y; + + const Index base_m = 64 * m_block_idx; + const Index base_n = 64 * n_block_idx; + + // declare and initialize 64 registers for output 8x8 block + + // prefetch registers + Scalar lhs_pf0; + Scalar lhs_pf1; + Scalar lhs_pf2; + Scalar lhs_pf3; + Scalar lhs_pf4; + Scalar lhs_pf5; + Scalar lhs_pf6; + Scalar lhs_pf7; + + Scalar rhs_pf0; + Scalar rhs_pf1; + Scalar rhs_pf2; + Scalar rhs_pf3; + Scalar rhs_pf4; + Scalar rhs_pf5; + Scalar rhs_pf6; + Scalar rhs_pf7; + + // shared memory is formatted + // (contract idx in block, nocontract idx in block, block idx) + // where block idx is column major. This transposition limits the number of + // bank conflicts when reading the LHS. The core idea is that since the contracting + // index is shared by both sides, then the contracting index should be in threadIdx.x. + + // On the LHS, we pad each row inside of each block with an extra element. This makes + // each block 8 rows of 9 elements, which is 72 elements. This gives no bank conflicts + // on writes and very few 2-way conflicts on reads. There is an 8x8 grid of these blocks. + + // On the RHS we just add 8 padding elements to the end of each block. This gives no bank + // conflicts on writes and also none on reads. + + // storage indices + const Index lhs_store_idx_base = threadIdx.y * 72 + threadIdx.x * 9 + threadIdx.z; + const Index rhs_store_idx_base = threadIdx.y * 72 + threadIdx.z * 8 + threadIdx.x; + + const Index lhs_store_idx_0 = lhs_store_idx_base + 576 * 0; + const Index lhs_store_idx_1 = lhs_store_idx_base + 576 * 1; + const Index lhs_store_idx_2 = lhs_store_idx_base + 576 * 2; + const Index lhs_store_idx_3 = lhs_store_idx_base + 576 * 3; + const Index lhs_store_idx_4 = lhs_store_idx_base + 576 * 4; + const Index lhs_store_idx_5 = lhs_store_idx_base + 576 * 5; + const Index lhs_store_idx_6 = lhs_store_idx_base + 576 * 6; + const Index lhs_store_idx_7 = lhs_store_idx_base + 576 * 7; + + const Index rhs_store_idx_0 = rhs_store_idx_base + 576 * 0; + const Index rhs_store_idx_1 = rhs_store_idx_base + 576 * 1; + const Index rhs_store_idx_2 = rhs_store_idx_base + 576 * 2; + const Index rhs_store_idx_3 = rhs_store_idx_base + 576 * 3; + const Index rhs_store_idx_4 = rhs_store_idx_base + 576 * 4; + const Index rhs_store_idx_5 = rhs_store_idx_base + 576 * 5; + const Index rhs_store_idx_6 = rhs_store_idx_base + 576 * 6; + const Index rhs_store_idx_7 = rhs_store_idx_base + 576 * 7; + + // in the loading code, the following variables are important: + // threadIdx.x: the vertical position in an 8x8 block + // threadIdx.y: the vertical index of the 8x8 block in the grid + // threadIdx.z: the horizontal position in an 8x8 block + // k: the horizontal index of the 8x8 block in the grid + // + // The k parameter is implicit (it was the loop counter for a loop that went + // from 0 to <8, but now that loop is unrolled in the below code. + + const Index load_idx_vert = threadIdx.x + 8 * threadIdx.y; + const Index lhs_vert = base_m + load_idx_vert; + +#define prefetchIntoRegisters(base_k) \ + { \ + lhs_pf0 = conv(0); \ + lhs_pf1 = conv(0); \ + lhs_pf2 = conv(0); \ + lhs_pf3 = conv(0); \ + lhs_pf4 = conv(0); \ + lhs_pf5 = conv(0); \ + lhs_pf6 = conv(0); \ + lhs_pf7 = conv(0); \ + \ + rhs_pf0 = conv(0); \ + rhs_pf1 = conv(0); \ + rhs_pf2 = conv(0); \ + rhs_pf3 = conv(0); \ + rhs_pf4 = conv(0); \ + rhs_pf5 = conv(0); \ + rhs_pf6 = conv(0); \ + rhs_pf7 = conv(0); \ + \ + if (!needs_edge_check || lhs_vert < m_size) { \ + const Index lhs_horiz_0 = base_k + threadIdx.z + 0 * 8; \ + const Index lhs_horiz_1 = base_k + threadIdx.z + 1 * 8; \ + const Index lhs_horiz_2 = base_k + threadIdx.z + 2 * 8; \ + const Index lhs_horiz_3 = base_k + threadIdx.z + 3 * 8; \ + const Index lhs_horiz_4 = base_k + threadIdx.z + 4 * 8; \ + const Index lhs_horiz_5 = base_k + threadIdx.z + 5 * 8; \ + const Index lhs_horiz_6 = base_k + threadIdx.z + 6 * 8; \ + const Index lhs_horiz_7 = base_k + threadIdx.z + 7 * 8; \ + \ + if (!needs_edge_check || lhs_horiz_7 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ + lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ + lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ + lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ + lhs_pf7 = lhs(lhs_vert, lhs_horiz_7); \ + } else if (lhs_horiz_6 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ + lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ + lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ + lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \ + } else if (lhs_horiz_5 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ + lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ + lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \ + } else if (lhs_horiz_4 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ + lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \ + } else if (lhs_horiz_3 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \ + } else if (lhs_horiz_2 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \ + } else if (lhs_horiz_1 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \ + } else if (lhs_horiz_0 < k_size) { \ + lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \ + } \ + } \ + \ + const Index rhs_vert = base_k + load_idx_vert; \ + if (!needs_edge_check || rhs_vert < k_size) { \ + const Index rhs_horiz_0 = base_n + threadIdx.z + 0 * 8; \ + const Index rhs_horiz_1 = base_n + threadIdx.z + 1 * 8; \ + const Index rhs_horiz_2 = base_n + threadIdx.z + 2 * 8; \ + const Index rhs_horiz_3 = base_n + threadIdx.z + 3 * 8; \ + const Index rhs_horiz_4 = base_n + threadIdx.z + 4 * 8; \ + const Index rhs_horiz_5 = base_n + threadIdx.z + 5 * 8; \ + const Index rhs_horiz_6 = base_n + threadIdx.z + 6 * 8; \ + const Index rhs_horiz_7 = base_n + threadIdx.z + 7 * 8; \ + \ + if (rhs_horiz_7 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ + rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ + rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ + rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ + rhs_pf7 = rhs(rhs_vert, rhs_horiz_7); \ + } else if (rhs_horiz_6 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ + rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ + rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ + rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \ + } else if (rhs_horiz_5 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ + rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ + rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \ + } else if (rhs_horiz_4 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ + rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \ + } else if (rhs_horiz_3 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \ + } else if (rhs_horiz_2 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \ + } else if (rhs_horiz_1 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \ + } else if (rhs_horiz_0 < n_size) { \ + rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \ + } \ + } \ + } \ + +#define writeRegToShmem(_) \ + lhs_shmem[lhs_store_idx_0] = lhs_pf0; \ + rhs_shmem[rhs_store_idx_0] = rhs_pf0; \ + \ + lhs_shmem[lhs_store_idx_1] = lhs_pf1; \ + rhs_shmem[rhs_store_idx_1] = rhs_pf1; \ + \ + lhs_shmem[lhs_store_idx_2] = lhs_pf2; \ + rhs_shmem[rhs_store_idx_2] = rhs_pf2; \ + \ + lhs_shmem[lhs_store_idx_3] = lhs_pf3; \ + rhs_shmem[rhs_store_idx_3] = rhs_pf3; \ + \ + lhs_shmem[lhs_store_idx_4] = lhs_pf4; \ + rhs_shmem[rhs_store_idx_4] = rhs_pf4; \ + \ + lhs_shmem[lhs_store_idx_5] = lhs_pf5; \ + rhs_shmem[rhs_store_idx_5] = rhs_pf5; \ + \ + lhs_shmem[lhs_store_idx_6] = lhs_pf6; \ + rhs_shmem[rhs_store_idx_6] = rhs_pf6; \ + \ + lhs_shmem[lhs_store_idx_7] = lhs_pf7; \ + rhs_shmem[rhs_store_idx_7] = rhs_pf7; \ + + // declare and initialize result array +#define res(i, j) _res_##i##j +#define initResultRow(i) \ + Scalar res(i, 0) = conv(0); \ + Scalar res(i, 1) = conv(0); \ + Scalar res(i, 2) = conv(0); \ + Scalar res(i, 3) = conv(0); \ + Scalar res(i, 4) = conv(0); \ + Scalar res(i, 5) = conv(0); \ + Scalar res(i, 6) = conv(0); \ + Scalar res(i, 7) = conv(0); \ + + internal::scalar_cast_op<int, Scalar> conv; + initResultRow(0); + initResultRow(1); + initResultRow(2); + initResultRow(3); + initResultRow(4); + initResultRow(5); + initResultRow(6); + initResultRow(7); +#undef initResultRow + + for (Index base_k = 0; base_k < k_size; base_k += 64) { + // wait for previous iteration to finish with shmem. Despite common sense, + // the code is a bit faster with this here then at bottom of loop + __syncthreads(); + + prefetchIntoRegisters(base_k); + writeRegToShmem(); + + #undef prefetchIntoRegisters + #undef writeRegToShmem + + // wait for shared mem packing to be done before starting computation + __syncthreads(); + + // compute 8x8 matrix product by outer product. This involves packing one column + // of LHS and one row of RHS into registers (takes 16 registers). + +#define lcol(i) _lcol##i + Scalar lcol(0); + Scalar lcol(1); + Scalar lcol(2); + Scalar lcol(3); + Scalar lcol(4); + Scalar lcol(5); + Scalar lcol(6); + Scalar lcol(7); + +#define rrow(j) _rrow##j + Scalar rrow(0); + Scalar rrow(1); + Scalar rrow(2); + Scalar rrow(3); + Scalar rrow(4); + Scalar rrow(5); + Scalar rrow(6); + Scalar rrow(7); + + // Now x corresponds to k, y to m, and z to n + const Scalar* lhs_block = &lhs_shmem[threadIdx.x + 9 * threadIdx.y]; + const Scalar* rhs_block = &rhs_shmem[threadIdx.x + 8 * threadIdx.z]; + +#define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))] +#define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))] + +#define loadData(i, j) \ + lcol(0) = lhs_element(0, j); \ + rrow(0) = rhs_element(i, 0); \ + lcol(1) = lhs_element(1, j); \ + rrow(1) = rhs_element(i, 1); \ + lcol(2) = lhs_element(2, j); \ + rrow(2) = rhs_element(i, 2); \ + lcol(3) = lhs_element(3, j); \ + rrow(3) = rhs_element(i, 3); \ + lcol(4) = lhs_element(4, j); \ + rrow(4) = rhs_element(i, 4); \ + lcol(5) = lhs_element(5, j); \ + rrow(5) = rhs_element(i, 5); \ + lcol(6) = lhs_element(6, j); \ + rrow(6) = rhs_element(i, 6); \ + lcol(7) = lhs_element(7, j); \ + rrow(7) = rhs_element(i, 7); \ + +#define computeCol(j) \ + res(0, j) += lcol(0) * rrow(j); \ + res(1, j) += lcol(1) * rrow(j); \ + res(2, j) += lcol(2) * rrow(j); \ + res(3, j) += lcol(3) * rrow(j); \ + res(4, j) += lcol(4) * rrow(j); \ + res(5, j) += lcol(5) * rrow(j); \ + res(6, j) += lcol(6) * rrow(j); \ + res(7, j) += lcol(7) * rrow(j); \ + +#define computePass(i) \ + loadData(i, i); \ + \ + computeCol(0); \ + computeCol(1); \ + computeCol(2); \ + computeCol(3); \ + computeCol(4); \ + computeCol(5); \ + computeCol(6); \ + computeCol(7); \ + + computePass(0); + computePass(1); + computePass(2); + computePass(3); + computePass(4); + computePass(5); + computePass(6); + computePass(7); + +#undef lcol +#undef rrow +#undef lhs_element +#undef rhs_element +#undef loadData +#undef computeCol +#undef computePass + } // end loop over k + + // we've now iterated over all of the large (ie width 64) k blocks and + // accumulated results in registers. At this point thread (x, y, z) contains + // the sum across all big k blocks of the product of little k block of index (x, y) + // with block of index (y, z). To compute the final output, we need to reduce + // the 8 threads over y by summation. +#if defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000) +#define shuffleInc(i, j, mask) res(i, j) += __shfl_xor(res(i, j), mask) +#else +#define shuffleInc(i, j, mask) res(i, j) += __shfl_xor_sync(0xFFFFFFFF, res(i, j), mask) +#endif + +#define reduceRow(i, mask) \ + shuffleInc(i, 0, mask); \ + shuffleInc(i, 1, mask); \ + shuffleInc(i, 2, mask); \ + shuffleInc(i, 3, mask); \ + shuffleInc(i, 4, mask); \ + shuffleInc(i, 5, mask); \ + shuffleInc(i, 6, mask); \ + shuffleInc(i, 7, mask); \ + +#define reduceMatrix(mask) \ + reduceRow(0, mask); \ + reduceRow(1, mask); \ + reduceRow(2, mask); \ + reduceRow(3, mask); \ + reduceRow(4, mask); \ + reduceRow(5, mask); \ + reduceRow(6, mask); \ + reduceRow(7, mask); \ + + // actually perform the reduction, now each thread of index (_, y, z) + // contains the correct values in its registers that belong in the output + // block + reduceMatrix(1); + reduceMatrix(2); + reduceMatrix(4); + +#undef shuffleInc +#undef reduceRow +#undef reduceMatrix + + // now we need to copy the 64 values into main memory. We can't split work + // among threads because all variables are in registers. There's 2 ways + // to do this: + // (1) have 1 thread do 64 writes from registers into global memory + // (2) have 1 thread do 64 writes into shared memory, and then 8 threads + // each do 8 writes into global memory. We can just overwrite the shared + // memory from the problem we just solved. + // (2) is slightly faster than (1) due to less branching and more ILP + + // TODO: won't yield much gain, but could just use currently unused shared mem + // and then we won't have to sync + // wait for shared mem to be out of use + __syncthreads(); + +#define writeResultShmem(i, j) \ + lhs_shmem[i + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j] = res(i, j); \ + +#define writeRow(i) \ + writeResultShmem(i, 0); \ + writeResultShmem(i, 1); \ + writeResultShmem(i, 2); \ + writeResultShmem(i, 3); \ + writeResultShmem(i, 4); \ + writeResultShmem(i, 5); \ + writeResultShmem(i, 6); \ + writeResultShmem(i, 7); \ + + if (threadIdx.x == 0) { + writeRow(0); + writeRow(1); + writeRow(2); + writeRow(3); + writeRow(4); + writeRow(5); + writeRow(6); + writeRow(7); + } +#undef writeResultShmem +#undef writeRow + + const int max_i_write = numext::mini((int)((m_size - base_m - threadIdx.y + 7) / 8), 8); + const int max_j_write = numext::mini((int)((n_size - base_n - threadIdx.z + 7) / 8), 8); + + if (threadIdx.x < max_i_write) { + if (max_j_write == 8) { + // TODO: can i trade bank conflicts for coalesced writes? + Scalar val0 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 0]; + Scalar val1 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 1]; + Scalar val2 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 2]; + Scalar val3 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 3]; + Scalar val4 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 4]; + Scalar val5 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 5]; + Scalar val6 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 6]; + Scalar val7 = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * 7]; + + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 0) = val0; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 1) = val1; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 2) = val2; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 3) = val3; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 4) = val4; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 5) = val5; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 6) = val6; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * 7) = val7; + } else { +#pragma unroll 7 + for (int j = 0; j < max_j_write; j++) { + Scalar val = lhs_shmem[threadIdx.x + 8 * threadIdx.y + 64 * threadIdx.z + 512 * j]; + output(base_m + threadIdx.y + 8 * threadIdx.x, base_n + threadIdx.z + 8 * j) = val; + } + } + } +#undef res +} + + +template<typename Scalar, typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper> +__global__ void +#if defined(EIGEN_HIPCC) +__launch_bounds__(512, 1) +#else +__launch_bounds__(512) +#endif +EigenContractionKernel(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, + const Index m_size, const Index n_size, const Index k_size) { + __shared__ Scalar lhs_shmem[72 * 64]; + __shared__ Scalar rhs_shmem[72 * 64]; + + const Index m_block_idx = blockIdx.x; + const Index n_block_idx = blockIdx.y; + + const Index base_m = 64 * m_block_idx; + const Index base_n = 64 * n_block_idx; + + if (base_m + 63 < m_size && base_n + 63 < n_size) { + EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size); + } else { + EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size); + } +} + + +template<typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY, + bool CHECK_RHS_BOUNDARY> +__device__ EIGEN_STRONG_INLINE void +EigenFloatContractionKernelInternal16x16(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, float2 lhs_shmem2[][16], + float2 rhs_shmem2[][8], const Index m_size, + const Index n_size, const Index k_size, + const Index base_m, const Index base_n) { + + // prefetch registers + float4 lhs_pf0, rhs_pf0; + + float4 results[4]; + for (int i=0; i < 4; i++) { + results[i].x = results[i].y = results[i].z = results[i].w = 0; + } + +#define prefetch_lhs(reg, row, col) \ + if (!CHECK_LHS_BOUNDARY) { \ + if (col < k_size) { \ + reg =lhs.template loadPacket<float4,Unaligned>(row, col); \ + } \ + } else { \ + if (col < k_size) { \ + if (row + 3 < m_size) { \ + reg =lhs.template loadPacket<float4,Unaligned>(row, col); \ + } else if (row + 2 < m_size) { \ + reg.x =lhs(row + 0, col); \ + reg.y =lhs(row + 1, col); \ + reg.z =lhs(row + 2, col); \ + } else if (row + 1 < m_size) { \ + reg.x =lhs(row + 0, col); \ + reg.y =lhs(row + 1, col); \ + } else if (row < m_size) { \ + reg.x =lhs(row + 0, col); \ + } \ + } \ + } \ + + Index lhs_vert = base_m+threadIdx.x*4; + + for (Index k = 0; k < k_size; k += 16) { + + lhs_pf0 = internal::pset1<float4>(0); + rhs_pf0 = internal::pset1<float4>(0); + + Index lhs_horiz = threadIdx.y+k; + prefetch_lhs(lhs_pf0, lhs_vert, lhs_horiz) + + Index rhs_vert = k+(threadIdx.x%4)*4; + Index rhs_horiz0 = (threadIdx.x>>2)+threadIdx.y*4+base_n; + + if (!CHECK_RHS_BOUNDARY) { + if ((rhs_vert + 3) < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz0); + } else if (rhs_vert + 2 < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); + } else if (rhs_vert + 1 < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + } else if (rhs_vert < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + } + } else { + if (rhs_horiz0 < n_size) { + if ((rhs_vert + 3) < k_size) { + rhs_pf0 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz0); + } else if ((rhs_vert + 2) < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); + } else if ((rhs_vert + 1) < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + } else if (rhs_vert < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + } + } + } + float x1, x2 ; + // the following can be a bitwise operation..... some day. + if((threadIdx.x%8) < 4) { + x1 = rhs_pf0.y; + x2 = rhs_pf0.w; + } else { + x1 = rhs_pf0.x; + x2 = rhs_pf0.z; + } + #if defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000) + x1 = __shfl_xor(x1, 4); + x2 = __shfl_xor(x2, 4); + #else + x1 = __shfl_xor_sync(0xFFFFFFFF, x1, 4); + x2 = __shfl_xor_sync(0xFFFFFFFF, x2, 4); + #endif + if((threadIdx.x%8) < 4) { + rhs_pf0.y = x1; + rhs_pf0.w = x2; + } else { + rhs_pf0.x = x1; + rhs_pf0.z = x2; + } + + // We have 64 features. + // Row 0 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 0, 1. + // Row 1 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 2, 3. + // ... + // Row 31 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 62, 63 + // Row 32 -> times (2, 6, 10, 14, 3, 7, 11, 15) for features 0, 1 + // ... + rhs_shmem2[(threadIdx.x>>3)+ threadIdx.y*2][threadIdx.x%8] = make_float2(rhs_pf0.x, rhs_pf0.y); + rhs_shmem2[(threadIdx.x>>3)+ threadIdx.y*2+32][threadIdx.x%8] = make_float2(rhs_pf0.z, rhs_pf0.w); + + // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) + // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) + // ... + // Row 15 (time 15) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) + // Row 16 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) + // ... + + lhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(lhs_pf0.x, lhs_pf0.y); + lhs_shmem2[threadIdx.y+16][threadIdx.x] = make_float2(lhs_pf0.z, lhs_pf0.w); + + +#define add_vals(fl1, fl2, fr1, fr2)\ + results[0].x += fl1.x * fr1.x;\ + results[0].y += fl1.y * fr1.x;\ + results[0].z += fl2.x * fr1.x;\ + results[0].w += fl2.y * fr1.x;\ +\ + results[1].x += fl1.x * fr1.y;\ + results[1].y += fl1.y * fr1.y;\ + results[1].z += fl2.x * fr1.y;\ + results[1].w += fl2.y * fr1.y;\ +\ + results[2].x += fl1.x * fr2.x;\ + results[2].y += fl1.y * fr2.x;\ + results[2].z += fl2.x * fr2.x;\ + results[2].w += fl2.y * fr2.x;\ +\ + results[3].x += fl1.x * fr2.y;\ + results[3].y += fl1.y * fr2.y;\ + results[3].z += fl2.x * fr2.y;\ + results[3].w += fl2.y * fr2.y;\ + + __syncthreads(); + + // Do the multiplies. + #pragma unroll + for (int koff = 0; koff < 16; koff ++) { + // 32 x threads. + float2 fl1 = lhs_shmem2[koff][threadIdx.x]; + float2 fl2 = lhs_shmem2[koff + 16][threadIdx.x]; + + int start_feature = threadIdx.y * 4; + float2 fr1 = rhs_shmem2[(start_feature>>1) + 32*((koff%4)/2)][koff/4 + (koff%2)*4]; + float2 fr2 = rhs_shmem2[(start_feature>>1) + 1 + 32*((koff%4)/2)][koff/4 + (koff%2)*4]; + + add_vals(fl1, fl2, fr1, fr2) + } + __syncthreads(); + } + +#undef prefetch_lhs +#undef add_vals + + Index horiz_base = threadIdx.y*4+base_n; + if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { + for (int i = 0; i < 4; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } else if (!CHECK_RHS_BOUNDARY) { + // CHECK LHS + if (lhs_vert + 3 < m_size) { + for (int i = 0; i < 4; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } else if (lhs_vert + 2 < m_size) { + for (int i = 0; i < 4; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + } + } else if (lhs_vert + 1 < m_size) { + for (int i = 0; i < 4; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + } + } else if (lhs_vert < m_size) { + for (int i = 0; i < 4; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + } + } + } else if (!CHECK_LHS_BOUNDARY) { + // CHECK RHS + /* + int ncols_rem = fminf(n_size- horiz_base, 4); + for (int i = 0; i < ncols_rem; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + }*/ + for (int i = 0; i < 4; i++) { + if (horiz_base+i < n_size) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } + } else { + // CHECK both boundaries. + for (int i = 0; i < 4; i++) { + if (horiz_base+i < n_size) { + if (lhs_vert < m_size) + output(lhs_vert, horiz_base + i) = results[i].x; + if (lhs_vert + 1 < m_size) + output(lhs_vert + 1, horiz_base + i) = results[i].y; + if (lhs_vert + 2 < m_size) + output(lhs_vert + 2, horiz_base + i) = results[i].z; + if (lhs_vert + 3 < m_size) + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } + } +} + + +template<typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY, + bool CHECK_RHS_BOUNDARY> +__device__ EIGEN_STRONG_INLINE void +EigenFloatContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, float2 lhs_shmem2[][32], + float2 rhs_shmem2[][8], const Index m_size, + const Index n_size, const Index k_size, + const Index base_m, const Index base_n) { + + // prefetch registers + float4 lhs_pf0, lhs_pf1, lhs_pf2, lhs_pf3; + float4 rhs_pf0, rhs_pf1; + + float4 results[8]; + for (int i=0; i < 8; i++) { + results[i].x = results[i].y = results[i].z = results[i].w = 0; + } + + Index lhs_vert = base_m+threadIdx.x*4+(threadIdx.y%4)*32; + for (Index k = 0; k < k_size; k += 32) { + lhs_pf0 = internal::pset1<float4>(0); + lhs_pf1 = internal::pset1<float4>(0); + lhs_pf2 = internal::pset1<float4>(0); + lhs_pf3 = internal::pset1<float4>(0); + + rhs_pf0 = internal::pset1<float4>(0); + rhs_pf1 = internal::pset1<float4>(0); + + if (!CHECK_LHS_BOUNDARY) { + if ((threadIdx.y/4+k+24) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + lhs_pf2 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+16)); + lhs_pf3 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+24)); + } else if ((threadIdx.y/4+k+16) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + lhs_pf2 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+16)); + } else if ((threadIdx.y/4+k+8) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + } else if ((threadIdx.y/4+k) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + } + } else { + // just CHECK_LHS_BOUNDARY + if (lhs_vert + 3 < m_size) { + if ((threadIdx.y/4+k+24) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + lhs_pf2 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+16)); + lhs_pf3 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+24)); + } else if ((threadIdx.y/4+k+16) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + lhs_pf2 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+16)); + } else if ((threadIdx.y/4+k+8) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + lhs_pf1 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k+8)); + } else if ((threadIdx.y/4+k) < k_size) { + lhs_pf0 =lhs.template loadPacket<float4,Unaligned>(lhs_vert, (threadIdx.y/4+k)); + } + } else if (lhs_vert + 2 < m_size) { + if ((threadIdx.y/4+k+24) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); + lhs_pf2.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+16)); + lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); + lhs_pf3.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+24)); + lhs_pf3.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+24)); + } else if ((threadIdx.y/4+k+16) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); + lhs_pf2.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+16)); + } else if ((threadIdx.y/4+k+8) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + lhs_pf1.z =lhs(lhs_vert + 2, (threadIdx.y/4+k+8)); + } else if ((threadIdx.y/4+k) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf0.z =lhs(lhs_vert + 2, (threadIdx.y/4+k)); + } + } else if (lhs_vert + 1 < m_size) { + if ((threadIdx.y/4+k+24) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); + lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); + lhs_pf3.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+24)); + } else if ((threadIdx.y/4+k+16) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + lhs_pf2.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+16)); + } else if ((threadIdx.y/4+k+8) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf1.y =lhs(lhs_vert + 1, (threadIdx.y/4+k+8)); + } else if ((threadIdx.y/4+k) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf0.y =lhs(lhs_vert + 1, (threadIdx.y/4+k)); + } + } else if (lhs_vert < m_size) { + if ((threadIdx.y/4+k+24) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + lhs_pf3.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+24)); + } else if ((threadIdx.y/4+k+16) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + lhs_pf2.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+16)); + } else if ((threadIdx.y/4+k+8) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + lhs_pf1.x =lhs(lhs_vert + 0, (threadIdx.y/4+k+8)); + } else if ((threadIdx.y/4+k) < k_size) { + lhs_pf0.x =lhs(lhs_vert + 0, (threadIdx.y/4+k)); + } + } + } + __syncthreads(); + Index rhs_vert = k+threadIdx.x*4; + Index rhs_horiz0 = threadIdx.y*2+base_n; + Index rhs_horiz1 = threadIdx.y*2+1+base_n; + if (!CHECK_RHS_BOUNDARY) { + if ((rhs_vert + 3) < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz0); + rhs_pf1 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz1); + } else if (rhs_vert + 2 < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); + rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1); + } else if (rhs_vert + 1 < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); + } else if (rhs_vert < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + } + } else { + if (rhs_horiz1 < n_size) { + if ((rhs_vert + 3) < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz0); + rhs_pf1 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz1); + } else if (rhs_vert + 2 < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); + rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1); + } else if (k+threadIdx.x*4 + 1 < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1); + } else if (k+threadIdx.x*4 < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf1.x = rhs(rhs_vert, rhs_horiz1); + } + } else if (rhs_horiz0 < n_size) { + if ((rhs_vert + 3) < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0 = rhs.template loadPacket<float4,Unaligned>(rhs_vert, rhs_horiz0); + } else if ((rhs_vert + 2) < k_size) { + // just CHECK_RHS_BOUNDARY + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0); + } else if ((rhs_vert + 1) < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0); + } else if (rhs_vert < k_size) { + rhs_pf0.x = rhs(rhs_vert, rhs_horiz0); + } + } + } + __syncthreads(); + // Loaded. Do computation + // Row 0 -> times (0, 4, 8, .. 28) for features 0, 1. + // Row 1 -> times (0, 4, 8, .. 28) for features 2, 3. + // .. + // Row 31 -> times (0, 4, 8, .. 28) for features 62, 63 + rhs_shmem2[threadIdx.y][threadIdx.x] = make_float2(rhs_pf0.x, rhs_pf1.x); + // Row 32 -> times (1, 5, 9, .. 29) for features 0, 1. + // Row 33 -> times (1, 5, 9, .. 29) for features 2, 3. + // .. + rhs_shmem2[threadIdx.y+32][threadIdx.x] = make_float2(rhs_pf0.y, rhs_pf1.y); + // Row 64 -> times (2, 6, 10, .. 30) for features 0, 1. + // Row 65 -> times (2, 6, 10, .. 30) for features 2, 3. + rhs_shmem2[threadIdx.y+64][threadIdx.x] = make_float2(rhs_pf0.z, rhs_pf1.z); + // Row 96 -> times (3, 7, 11, .. 31) for features 0, 1. + // Row 97 -> times (3, 7, 11, .. 31) for features 2, 3. + rhs_shmem2[threadIdx.y+96][threadIdx.x] = make_float2(rhs_pf0.w, rhs_pf1.w); + + // LHS. + // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125) + // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125) + // ... + // Row 8 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127) + // Row 15 (time 7) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127) + + +#define add_vals(a_feat1, a_feat2, f1, f2, f3, f4)\ + results[0].x += a_feat1.x * f1.x;\ + results[1].x += a_feat1.x * f1.y;\ + results[2].x += a_feat1.x * f2.x;\ + results[3].x += a_feat1.x * f2.y;\ + results[4].x += a_feat1.x * f3.x;\ + results[5].x += a_feat1.x * f3.y;\ + results[6].x += a_feat1.x * f4.x;\ + results[7].x += a_feat1.x * f4.y;\ +\ + results[0].y += a_feat1.y * f1.x;\ + results[1].y += a_feat1.y * f1.y;\ + results[2].y += a_feat1.y * f2.x;\ + results[3].y += a_feat1.y * f2.y;\ + results[4].y += a_feat1.y * f3.x;\ + results[5].y += a_feat1.y * f3.y;\ + results[6].y += a_feat1.y * f4.x;\ + results[7].y += a_feat1.y * f4.y;\ +\ + results[0].z += a_feat2.x * f1.x;\ + results[1].z += a_feat2.x * f1.y;\ + results[2].z += a_feat2.x * f2.x;\ + results[3].z += a_feat2.x * f2.y;\ + results[4].z += a_feat2.x * f3.x;\ + results[5].z += a_feat2.x * f3.y;\ + results[6].z += a_feat2.x * f4.x;\ + results[7].z += a_feat2.x * f4.y;\ +\ + results[0].w += a_feat2.y * f1.x;\ + results[1].w += a_feat2.y * f1.y;\ + results[2].w += a_feat2.y * f2.x;\ + results[3].w += a_feat2.y * f2.y;\ + results[4].w += a_feat2.y * f3.x;\ + results[5].w += a_feat2.y * f3.y;\ + results[6].w += a_feat2.y * f4.x;\ + results[7].w += a_feat2.y * f4.y;\ + + lhs_shmem2[threadIdx.y/4][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf0.x, lhs_pf0.y); + lhs_shmem2[threadIdx.y/4+8][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf1.x, lhs_pf1.y); + lhs_shmem2[threadIdx.y/4+16][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf2.x, lhs_pf2.y); + lhs_shmem2[threadIdx.y/4+24][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf3.x, lhs_pf3.y); + + lhs_shmem2[threadIdx.y/4 + 32][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf0.z, lhs_pf0.w); + lhs_shmem2[threadIdx.y/4 + 40][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf1.z, lhs_pf1.w); + lhs_shmem2[threadIdx.y/4 + 48][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf2.z, lhs_pf2.w); + lhs_shmem2[threadIdx.y/4 + 56][threadIdx.x+(threadIdx.y%4)*8] = make_float2(lhs_pf3.z, lhs_pf3.w); + + __syncthreads(); + + // Do the multiplies. + #pragma unroll + for (int koff = 0; koff < 32; koff ++) { + float2 a3 = lhs_shmem2[koff][threadIdx.x + (threadIdx.y % 4) * 8]; + float2 a4 = lhs_shmem2[koff + 32][threadIdx.x + (threadIdx.y % 4) * 8]; + + // first feature is at (threadIdx.y/4) * 8 last is at start + 8. + int start_feature = (threadIdx.y / 4) * 8; + + float2 br1 = rhs_shmem2[start_feature/2 + (koff % 4) * 32][koff/4]; + float2 br2 = rhs_shmem2[start_feature/2 + 1 + (koff % 4) * 32][koff/4]; + float2 br3 = rhs_shmem2[start_feature/2 + 2 + (koff % 4) * 32][koff/4]; + float2 br4 = rhs_shmem2[start_feature/2 + 3 + (koff % 4) * 32][koff/4]; + + add_vals(a3, a4, br1, br2, br3, br4) + } + __syncthreads(); + } // end loop over k + + __syncthreads(); + Index horiz_base = (threadIdx.y/4)*8+base_n; + if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) { + for (int i = 0; i < 8; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } else if (!CHECK_RHS_BOUNDARY) { + if (lhs_vert + 3 < m_size) { + for (int i = 0; i < 8; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } else if (lhs_vert + 2 < m_size) { + for (int i = 0; i < 8; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + } + } else if (lhs_vert + 1 < m_size) { + for (int i = 0; i < 8; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + } + } else if (lhs_vert < m_size) { + for (int i = 0; i < 8; i++) { + output(lhs_vert, horiz_base + i) = results[i].x; + } + } + } else if (!CHECK_LHS_BOUNDARY) { + // CHECK BOUNDARY_B + for (int i = 0; i < 8; i++) { + if (horiz_base + i < n_size) { + output(lhs_vert, horiz_base + i) = results[i].x; + output(lhs_vert + 1, horiz_base + i) = results[i].y; + output(lhs_vert + 2, horiz_base + i) = results[i].z; + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } + } else { + // CHECK both boundaries. + for (int i = 0; i < 8; i++) { + if (horiz_base + i < n_size) { + if (lhs_vert < m_size) + output(lhs_vert, horiz_base + i) = results[i].x; + if (lhs_vert + 1 < m_size) + output(lhs_vert + 1, horiz_base + i) = results[i].y; + if (lhs_vert + 2 < m_size) + output(lhs_vert + 2, horiz_base + i) = results[i].z; + if (lhs_vert + 3 < m_size) + output(lhs_vert + 3, horiz_base + i) = results[i].w; + } + } + } +} + + +template<typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper> +__global__ void +#if defined(EIGEN_HIPCC) +__launch_bounds__(256, 1) +#else +__launch_bounds__(256) +#endif +EigenFloatContractionKernel(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, + const Index m_size, const Index n_size, const Index k_size) { + __shared__ float2 lhs_shmem[64*32]; + __shared__ float2 rhs_shmem[128*8]; + + typedef float2 LHS_MEM[64][32]; + typedef float2 RHS_MEM[128][8]; + + const Index m_block_idx = blockIdx.x; + const Index n_block_idx = blockIdx.y; + + const Index base_m = 128 * m_block_idx; + const Index base_n = 64 * n_block_idx; + + bool check_rhs = (base_n + 63) >= n_size; + bool check_lhs128 = (base_m + 127) >= m_size; + + if (!check_rhs) { + if (!check_lhs128) { + // >= 128 rows left + EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, false>( + lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); + } else { + EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, false>( + lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); + } + } else { + if (!check_lhs128) { + // >= 128 rows left + EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, true>( + lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); + } else { + EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, true>( + lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n); + } + } +} + +template<typename Index, typename LhsMapper, + typename RhsMapper, typename OutputMapper> +__global__ void +#if defined(EIGEN_HIPCC) +__launch_bounds__(256, 1) +#else +__launch_bounds__(256) +#endif +EigenFloatContractionKernel16x16(const LhsMapper lhs, const RhsMapper rhs, + const OutputMapper output, + const Index m_size, const Index n_size, const Index k_size) { + __shared__ float2 lhs_shmem[32][16]; + __shared__ float2 rhs_shmem[64][8]; + + const Index m_block_idx = blockIdx.x; + const Index n_block_idx = blockIdx.y; + + const Index base_m = 64 * m_block_idx; + const Index base_n = 64 * n_block_idx; + + if (base_m + 63 < m_size) { + if (base_n + 63 < n_size) { + EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); + } else { + EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); + } + } else { + if (base_n + 63 < n_size) { + EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); + } else { + EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n); + } + } +} + + +template<typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType> +struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, GpuDevice> : + public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, GpuDevice> > { + + typedef GpuDevice Device; + + typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> Self; + typedef TensorContractionEvaluatorBase<Self> Base; + + typedef TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType> XprType; + typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType; + + enum { + Layout = TensorEvaluator<LeftArgType, Device>::Layout, + }; + + // Most of the code is assuming that both input tensors are ColMajor. If the + // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS: + // If we want to compute A * B = C, where A is LHS and B is RHS, the code + // will pretend B is LHS and A is RHS. + typedef typename internal::conditional< + static_cast<int>(Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>::type EvalLeftArgType; + typedef typename internal::conditional< + static_cast<int>(Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>::type EvalRightArgType; + + static const int LDims = + internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value; + static const int RDims = + internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value; + static const int ContractDims = internal::array_size<Indices>::value; + + typedef array<Index, LDims> left_dim_mapper_t; + typedef array<Index, RDims> right_dim_mapper_t; + + typedef array<Index, ContractDims> contract_t; + typedef array<Index, LDims - ContractDims> left_nocontract_t; + typedef array<Index, RDims - ContractDims> right_nocontract_t; + + static const int NumDims = LDims + RDims - 2 * ContractDims; + + typedef DSizes<Index, NumDims> Dimensions; + + // typedefs needed in evalTo + typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; + typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; + + typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; + typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; + + typedef typename LeftEvaluator::Dimensions LeftDimensions; + typedef typename RightEvaluator::Dimensions RightDimensions; + + EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : + Base(op, device) + { + EIGEN_STATIC_ASSERT( (internal::is_same<OutputKernelType, const NoOpOutputKernel>::value), + GPU_TENSOR_CONTRACTION_DOES_NOT_SUPPORT_OUTPUT_KERNELS); + } + + // We need to redefine this method to make nvcc happy + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + this->m_leftImpl.evalSubExprsIfNeeded(NULL); + this->m_rightImpl.evalSubExprsIfNeeded(NULL); + if (data) { + evalTo(data); + return false; + } else { + this->m_result = static_cast<Scalar *>(this->m_device.allocate(this->dimensions().TotalSize() * sizeof(Scalar))); + evalTo(this->m_result); + return true; + } + } + + void evalTo(Scalar* buffer) const { + if (this->m_lhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<true, true, true, Unaligned>(buffer); + } + else { + evalTyped<true, true, false, Unaligned>(buffer); + } + } + else { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<true, false, true, Unaligned>(buffer); + } + else { + evalTyped<true, false, false, Unaligned>(buffer); + } + } + } + else { + if (this->m_rhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<false, true, true, Unaligned>(buffer); + } + else { + evalTyped<false, true, false, Unaligned>(buffer); + } + } + else { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<false, false, true, Unaligned>(buffer); + } + else { + evalTyped<false, false, false, Unaligned>(buffer); + } + } + } + } + + template <typename LhsScalar, typename RhsScalar, typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels { + static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { + const Index m_blocks = (m + 63) / 64; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(8, 8, 8); + LAUNCH_GPU_KERNEL((EigenContractionKernel<Scalar, Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } + }; + + template <typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels<float, float, Index, LhsMapper, RhsMapper, OutputMapper> { + static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { + if (m < 768 || n < 768) { + const Index m_blocks = (m + 63) / 64; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(16, 16, 1); + LAUNCH_GPU_KERNEL((EigenFloatContractionKernel16x16<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } else { + const Index m_blocks = (m + 127) / 128; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(8, 32, 1); + LAUNCH_GPU_KERNEL((EigenFloatContractionKernel<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } + } + }; + + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> + void evalTyped(Scalar* buffer) const { + // columns in left side, rows in right side + const Index k = this->m_k_size; + EIGEN_UNUSED_VARIABLE(k) + + // rows in left side + const Index m = this->m_i_size; + + // columns in right side + const Index n = this->m_j_size; + + // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar) + this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); + + typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, + LeftEvaluator, left_nocontract_t, + contract_t, 4, + lhs_inner_dim_contiguous, + false, Unaligned> LhsMapper; + + typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, + RightEvaluator, right_nocontract_t, + contract_t, 4, + rhs_inner_dim_contiguous, + rhs_inner_dim_reordered, Unaligned> RhsMapper; + + typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; + + + // initialize data mappers + LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides, + this->m_left_contracting_strides, this->m_k_strides); + + RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides, + this->m_right_contracting_strides, this->m_k_strides); + + OutputMapper output(buffer, m); + +#if defined(EIGEN_USE_HIP) + setGpuSharedMemConfig(hipSharedMemBankSizeEightByte); +#else + setGpuSharedMemConfig(cudaSharedMemBankSizeEightByte); +#endif + + LaunchKernels<LhsScalar, RhsScalar, Index, LhsMapper, RhsMapper, OutputMapper>::Run(lhs, rhs, output, m, n, k, this->m_device); + } +}; + +} // end namespace Eigen + +#endif // EIGEN_USE_GPU and EIGEN_GPUCC +#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_GPU_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h index 392cb6e..1424926 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h
@@ -16,14 +16,20 @@ enum { Rhs = 0, - Lhs = 1, + Lhs = 1 }; /* * Implementation of the Eigen blas_data_mapper class for tensors. */ +/// The make pointer class is used by sycl in order to build the mapper class on the device. For other platform the default make pointer is used which +/// is scalar * for CoeffLoader. +template <typename Tensor, bool HasRawAccess, template <class> class MakePointer_ = MakePointer> struct CoeffLoader; +template<typename Scalar, typename Index, int side, typename Tensor, typename nocontract_t, typename contract_t, + int packet_size, bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment, + template <class> class MakePointer_ = MakePointer> class BaseTensorContractionMapper; -template <typename Tensor, bool HasRawAccess> struct CoeffLoader { +template <typename Tensor, bool HasRawAccess, template <class> class MakePointer_> struct CoeffLoader { enum { DirectOffsets = false }; @@ -47,7 +53,7 @@ const Tensor m_tensor; }; -template <typename Tensor> struct CoeffLoader<Tensor, true> { +template <typename Tensor, template <class> class MakePointer_> struct CoeffLoader<Tensor, true, MakePointer_> { enum { DirectOffsets = true }; @@ -67,13 +73,14 @@ } private: typedef typename Tensor::Scalar Scalar; - const Scalar* m_data; + + typename MakePointer_<const Scalar>::Type m_data; }; template<typename Scalar, typename Index, int side, typename Tensor, typename nocontract_t, typename contract_t, - int packet_size, bool inner_dim_contiguous, int Alignment> + int packet_size, bool inner_dim_contiguous, int Alignment, template <class> class MakePointer_ = MakePointer> class SimpleTensorContractionMapper { public: EIGEN_DEVICE_FUNC @@ -89,7 +96,7 @@ m_k_strides(k_strides) { } enum { - DirectOffsets = CoeffLoader<Tensor, Tensor::RawAccess>::DirectOffsets + DirectOffsets = CoeffLoader<Tensor, Tensor::RawAccess, MakePointer_>::DirectOffsets }; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void offsetBuffer(typename Tensor::Index offset) { @@ -113,6 +120,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index computeIndex(Index row, Index col) const { const bool left = (side == Lhs); + EIGEN_UNUSED_VARIABLE(left); // annoying bug in g++8.1: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85963 Index nocontract_val = left ? row : col; Index linidx = 0; for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { @@ -130,19 +138,19 @@ } Index contract_val = left ? col : row; - for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { - const Index idx = contract_val / m_k_strides[i]; - linidx += idx * m_contract_strides[i]; - contract_val -= idx * m_k_strides[i]; - } - if(array_size<contract_t>::value > 0) { - if (side == Rhs && inner_dim_contiguous) { - eigen_assert(m_contract_strides[0] == 1); - linidx += contract_val; - } else { - linidx += contract_val * m_contract_strides[0]; - } + for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { + const Index idx = contract_val / m_k_strides[i]; + linidx += idx * m_contract_strides[i]; + contract_val -= idx * m_k_strides[i]; + } + + if (side == Rhs && inner_dim_contiguous) { + eigen_assert(m_contract_strides[0] == 1); + linidx += contract_val; + } else { + linidx += contract_val * m_contract_strides[0]; + } } return linidx; @@ -151,17 +159,18 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE IndexPair<Index> computeIndexPair(Index row, Index col, const Index distance) const { const bool left = (side == Lhs); + EIGEN_UNUSED_VARIABLE(left); // annoying bug in g++8.1: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85963 Index nocontract_val[2] = {left ? row : col, left ? row + distance : col}; Index linidx[2] = {0, 0}; - for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { - const Index idx0 = nocontract_val[0] / m_ij_strides[i]; - const Index idx1 = nocontract_val[1] / m_ij_strides[i]; - linidx[0] += idx0 * m_nocontract_strides[i]; - linidx[1] += idx1 * m_nocontract_strides[i]; - nocontract_val[0] -= idx0 * m_ij_strides[i]; - nocontract_val[1] -= idx1 * m_ij_strides[i]; - } if (array_size<typename Tensor::Dimensions>::value > array_size<contract_t>::value) { + for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { + const Index idx0 = nocontract_val[0] / m_ij_strides[i]; + const Index idx1 = nocontract_val[1] / m_ij_strides[i]; + linidx[0] += idx0 * m_nocontract_strides[i]; + linidx[1] += idx1 * m_nocontract_strides[i]; + nocontract_val[0] -= idx0 * m_ij_strides[i]; + nocontract_val[1] -= idx1 * m_ij_strides[i]; + } if (side == Lhs && inner_dim_contiguous) { eigen_assert(m_nocontract_strides[0] == 1); linidx[0] += nocontract_val[0]; @@ -173,22 +182,24 @@ } Index contract_val[2] = {left ? col : row, left ? col : row + distance}; - for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { - const Index idx0 = contract_val[0] / m_k_strides[i]; - const Index idx1 = contract_val[1] / m_k_strides[i]; - linidx[0] += idx0 * m_contract_strides[i]; - linidx[1] += idx1 * m_contract_strides[i]; - contract_val[0] -= idx0 * m_k_strides[i]; - contract_val[1] -= idx1 * m_k_strides[i]; - } + if (array_size<contract_t>::value> 0) { + for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { + const Index idx0 = contract_val[0] / m_k_strides[i]; + const Index idx1 = contract_val[1] / m_k_strides[i]; + linidx[0] += idx0 * m_contract_strides[i]; + linidx[1] += idx1 * m_contract_strides[i]; + contract_val[0] -= idx0 * m_k_strides[i]; + contract_val[1] -= idx1 * m_k_strides[i]; + } - if (side == Rhs && inner_dim_contiguous) { - eigen_assert(m_contract_strides[0] == 1); - linidx[0] += contract_val[0]; - linidx[1] += contract_val[1]; - } else { - linidx[0] += contract_val[0] * m_contract_strides[0]; - linidx[1] += contract_val[1] * m_contract_strides[0]; + if (side == Rhs && inner_dim_contiguous) { + eigen_assert(m_contract_strides[0] == 1); + linidx[0] += contract_val[0]; + linidx[1] += contract_val[1]; + } else { + linidx[0] += contract_val[0] * m_contract_strides[0]; + linidx[1] += contract_val[1] * m_contract_strides[0]; + } } return IndexPair<Index>(linidx[0], linidx[1]); } @@ -200,27 +211,26 @@ return (Alignment == Aligned) && (side == Lhs) && inner_dim_contiguous ? 0 : size; } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index stride() const { - return ((side == Lhs) && inner_dim_contiguous) ? m_contract_strides[0] : 1; + return ((side == Lhs) && inner_dim_contiguous && array_size<contract_t>::value > 0) ? m_contract_strides[0] : 1; } protected: - CoeffLoader<Tensor, Tensor::RawAccess> m_tensor; + CoeffLoader<Tensor, Tensor::RawAccess, MakePointer_> m_tensor; const nocontract_t m_nocontract_strides; const nocontract_t m_ij_strides; const contract_t m_contract_strides; const contract_t m_k_strides; }; - template<typename Scalar, typename Index, int side, typename Tensor, typename nocontract_t, typename contract_t, int packet_size, bool inner_dim_contiguous, - bool inner_dim_reordered, int Alignment> -class BaseTensorContractionMapper : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment> + bool inner_dim_reordered, int Alignment, template <class> class MakePointer_> +class BaseTensorContractionMapper : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment, MakePointer_> { public: - typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment> ParentMapper; + typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment, MakePointer_> ParentMapper; EIGEN_DEVICE_FUNC BaseTensorContractionMapper(const Tensor& tensor, @@ -230,12 +240,11 @@ const contract_t& k_strides) : ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } - typedef typename Tensor::PacketReturnType Packet; - typedef typename unpacket_traits<Packet>::half HalfPacket; - - template <int AlignmentType = Alignment> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { + template <typename PacketT,int AlignmentType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::enable_if<internal::unpacket_traits<PacketT>::size==packet_size,PacketT>::type + load(Index i, Index j) const + { // whole method makes column major assumption // don't need to add offsets for now (because operator handles that) @@ -250,7 +259,7 @@ const IndexPair<Index> indexPair = this->computeIndexPair(i, j, packet_size - 1); const Index first = indexPair.first; - const Index last = indexPair.second; + const Index lastIdx = indexPair.second; // We can always do optimized packet reads from left hand side right now, because // the vertical matrix dimension on the left hand side is never contracting. @@ -258,7 +267,7 @@ // been shuffled first. if (Tensor::PacketAccess && (side == Lhs || internal::array_size<contract_t>::value <= 1 || !inner_dim_reordered) && - (last - first) == (packet_size - 1)) { + (lastIdx - first) == (packet_size - 1)) { return this->m_tensor.template packet<AlignmentType>(first); } @@ -271,26 +280,38 @@ data[k] = this->m_tensor.coeff(internal_pair.first); data[k + 1] = this->m_tensor.coeff(internal_pair.second); } - data[packet_size - 1] = this->m_tensor.coeff(last); + data[packet_size - 1] = this->m_tensor.coeff(lastIdx); - return pload<Packet>(data); + return pload<PacketT>(data); } - template <int AlignmentType = Alignment> - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE HalfPacket loadHalfPacket(Index i, Index j) const { - // whole method makes column major assumption + template <typename PacketT,int AlignmentType> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::enable_if<internal::unpacket_traits<PacketT>::size!=packet_size,PacketT>::type + load(Index i, Index j) const + { + const Index requested_packet_size = internal::unpacket_traits<PacketT>::size; + EIGEN_ALIGN_MAX Scalar data[requested_packet_size]; - // don't need to add offsets for now (because operator handles that) - const Index half_packet_size = unpacket_traits<HalfPacket>::size; - if (half_packet_size == packet_size) { - return loadPacket<AlignmentType>(i, j); + const IndexPair<Index> indexPair = this->computeIndexPair(i, j, requested_packet_size - 1); + const Index first = indexPair.first; + const Index lastIdx = indexPair.second; + + data[0] = this->m_tensor.coeff(first); + for (Index k = 1; k < requested_packet_size - 1; k += 2) { + const IndexPair<Index> internal_pair = this->computeIndexPair(i + k, j, 1); + data[k] = this->m_tensor.coeff(internal_pair.first); + data[k + 1] = this->m_tensor.coeff(internal_pair.second); } - EIGEN_ALIGN_MAX Scalar data[half_packet_size]; - for (Index k = 0; k < half_packet_size; k++) { - data[k] = operator()(i + k, j); - } - return pload<HalfPacket>(data); + data[requested_packet_size - 1] = this->m_tensor.coeff(lastIdx); + + return pload<PacketT>(data); + } + + template <typename PacketT,int AlignmentType> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE PacketT loadPacket(Index i, Index j) const { + return this->load<PacketT,AlignmentType>(i,j); } }; @@ -299,11 +320,12 @@ typename Tensor, typename nocontract_t, typename contract_t, bool inner_dim_contiguous, - bool inner_dim_reordered, int Alignment> -class BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, inner_dim_reordered, Alignment> : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment> + bool inner_dim_reordered, int Alignment, template <class> class MakePointer_> +class BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> + : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment, MakePointer_> { public: - typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment> ParentMapper; + typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment, MakePointer_> ParentMapper; EIGEN_DEVICE_FUNC BaseTensorContractionMapper(const Tensor& tensor, @@ -313,16 +335,17 @@ const contract_t& k_strides) : ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } - typedef typename Tensor::PacketReturnType Packet; - template <int> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { + template <typename PacketT,int> EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE PacketT loadPacket(Index i, Index j) const { EIGEN_ALIGN_MAX Scalar data[1]; data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); - return pload<typename Tensor::PacketReturnType>(data); + return pload<PacketT>(data); } - template <int> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadHalfPacket(Index i, Index j) const { - return loadPacket(i, j); + template <typename PacketT,int> EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE PacketT load(Index i, Index j) const { + EIGEN_ALIGN_MAX Scalar data[1]; + data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); + return pload<PacketT>(data); } }; @@ -331,14 +354,12 @@ typename Tensor, typename nocontract_t, typename contract_t, int packet_size, - bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> + bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment, template <class> class MakePointer_=MakePointer> class TensorContractionSubMapper { public: - typedef typename Tensor::PacketReturnType Packet; - typedef typename unpacket_traits<Packet>::half HalfPacket; - typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> ParentMapper; - typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Self; + typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> ParentMapper; + typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> Self; typedef Self LinearMapper; enum { @@ -370,27 +391,32 @@ return m_base_mapper(i + m_vert_offset, j + m_horiz_offset); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i) const { + template <typename PacketT> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT loadPacket(Index i) const { if (UseDirectOffsets) { - return m_base_mapper.template loadPacket<Alignment>(i, 0); + return m_base_mapper.template loadPacket<PacketT,Alignment>(i, 0); } - return m_base_mapper.template loadPacket<Alignment>(i + m_vert_offset, m_horiz_offset); - } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i, Index j) const { - if (UseDirectOffsets) { - return m_base_mapper.template loadPacket<Alignment>(i, j); - } - return m_base_mapper.template loadPacket<Alignment>(i + m_vert_offset, j + m_horiz_offset); + return m_base_mapper.template loadPacket<PacketT,Alignment>(i + m_vert_offset, m_horiz_offset); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE HalfPacket loadHalfPacket(Index i) const { + template <typename PacketT> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT loadPacket(Index i, Index j) const { if (UseDirectOffsets) { - return m_base_mapper.template loadHalfPacket<Alignment>(i, 0); + return m_base_mapper.template loadPacket<PacketT,Alignment>(i, j); } - return m_base_mapper.template loadHalfPacket<Alignment>(i + m_vert_offset, m_horiz_offset); + return m_base_mapper.template loadPacket<PacketT,Alignment>(i + m_vert_offset, j + m_horiz_offset); } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Packet p) const { + template <typename PacketT, int AlignmentType> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT loadPacket(Index i, Index j) const { + if (UseDirectOffsets) { + return m_base_mapper.template load<PacketT,AlignmentType>(i, j); + } + return m_base_mapper.template loadPacket<PacketT,AlignmentType>(i + m_vert_offset, j + m_horiz_offset); + } + + template <typename PacketT> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketT& p) const { if (UseDirectOffsets) { m_base_mapper.storePacket(i, 0, p); } @@ -406,15 +432,15 @@ template <typename PacketT, int AlignmentType> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i) const { - EIGEN_STATIC_ASSERT((internal::is_same<PacketT, Packet>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((internal::is_same<PacketT, PacketT>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); const int ActualAlignment = (AlignmentType == Aligned) && (Alignment == Aligned) ? Aligned : Unaligned; if (UseDirectOffsets) { - return m_base_mapper.template loadPacket<ActualAlignment>(i, 0); + return m_base_mapper.template loadPacket<PacketT,ActualAlignment>(i, 0); } - return m_base_mapper.template loadPacket<ActualAlignment>(i + m_vert_offset, m_horiz_offset); + return m_base_mapper.template loadPacket<PacketT,ActualAlignment>(i + m_vert_offset, m_horiz_offset); } - template <typename Packet> + template <typename PacketT> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool aligned(Index) const { return false; } @@ -430,14 +456,14 @@ typename Tensor, typename nocontract_t, typename contract_t, int packet_size, - bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> + bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment, template <class> class MakePointer_=MakePointer> class TensorContractionInputMapper - : public BaseTensorContractionMapper<Scalar_, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> { + : public BaseTensorContractionMapper<Scalar_, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> { public: typedef Scalar_ Scalar; - typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Base; - typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> SubMapper; + typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> Base; + typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_> SubMapper; typedef SubMapper VectorMapper; EIGEN_DEVICE_FUNC TensorContractionInputMapper(const Tensor& tensor,
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionSycl.h new file mode 100644 index 0000000..35f931c --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionSycl.h
@@ -0,0 +1,404 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorTensorContractionsycl.h + * + * \brief: + * TensorContractionsycl + * +*****************************************************************/ + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_SYCL_H +#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_SYCL_H +namespace Eigen { + +template <typename Index, typename LhsScalar, typename RhsScalar,bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered> struct LaunchSyclKernels; +template<typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType> +struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, const Eigen::SyclDevice> : + public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, const Eigen::SyclDevice> > { + + static_assert(std::is_same<OutputKernelType, const NoOpOutputKernel>::value, + "SYCL tensor contraction does not support output kernels."); + + typedef const Eigen::SyclDevice Device; + + typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> Self; + typedef TensorContractionEvaluatorBase<Self> Base; + typedef TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType> XprType; + typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + + enum { + Layout = TensorEvaluator<LeftArgType, Device>::Layout, + }; + + // Most of the code is assuming that both input tensors are ColMajor. If the + // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS: + // If we want to compute A * B = C, where A is LHS and B is RHS, the code + // will pretend B is LHS and A is RHS. + typedef typename internal::conditional< + static_cast<int>(Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>::type EvalLeftArgType; + typedef typename internal::conditional< + static_cast<int>(Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>::type EvalRightArgType; + + static const int LDims = + internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value; + static const int RDims = + internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value; + static const int ContractDims = internal::array_size<Indices>::value; + + typedef array<Index, LDims> left_dim_mapper_t; + typedef array<Index, RDims> right_dim_mapper_t; + + typedef array<Index, ContractDims> contract_t; + typedef array<Index, LDims - ContractDims> left_nocontract_t; + typedef array<Index, RDims - ContractDims> right_nocontract_t; + + static const int NumDims = LDims + RDims - 2 * ContractDims; + + typedef DSizes<Index, NumDims> Dimensions; + + // typedefs needed in evalTo + typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar; + typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; + + typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; + typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; + + typedef typename LeftEvaluator::Dimensions LeftDimensions; + typedef typename RightEvaluator::Dimensions RightDimensions; + + EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : + Base(op, device) {} + + // We need to redefine this method to make nvcc happy + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + this->m_leftImpl.evalSubExprsIfNeeded(NULL); + this->m_rightImpl.evalSubExprsIfNeeded(NULL); + if (data) { + evalTo(data); + return false; + } else { + this->m_result = static_cast<Scalar*>(this->m_device.allocate(this->dimensions().TotalSize() * sizeof(Scalar))); + evalTo(this->m_result); + return true; + } + } + const Eigen::SyclDevice& device() const {return this->m_device;} + void evalTo(Scalar* buffer) const { + // Here is the result + if (this->m_lhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<true, true, true, Unaligned>(buffer); + } + else { + evalTyped<true, true, false, Unaligned>(buffer); + } + } + else { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<true, false, true, Unaligned>(buffer); + } + else { + evalTyped<true, false, false, Unaligned>(buffer); + } + } + } + else { + if (this->m_rhs_inner_dim_contiguous) { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<false, true, true, Unaligned>(buffer); + } + else { + evalTyped<false, true, false, Unaligned>(buffer); + } + } + else { + if (this->m_rhs_inner_dim_reordered) { + evalTyped<false, false, true, Unaligned>(buffer); + } + else { + evalTyped<false, false, false, Unaligned>(buffer); + } + } + } + } + + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> + void evalTyped(Scalar* buffer) const { + // columns in left side, rows in right side + const Index k = this->m_k_size; + EIGEN_UNUSED_VARIABLE(k) + // rows in left side + const Index m = this->m_i_size; + // columns in right side + const Index n = this->m_j_size; + + // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar) + this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); + LaunchSyclKernels<Index, LhsScalar, RhsScalar,lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered>::Run(*this, buffer, m, n, k, + this->m_k_strides, this->m_left_contracting_strides, this->m_right_contracting_strides, + this->m_i_strides, this->m_j_strides, this->m_left_nocontract_strides, this->m_right_nocontract_strides); + } + // required by sycl to construct the expr on the device. Returns original left_impl + const TensorEvaluator<LeftArgType, Device>& left_impl() const { + return choose(Cond<static_cast<int>(Layout) == static_cast<int>(ColMajor)>(), this->m_leftImpl, this->m_rightImpl); + } + // required by sycl to construct the expr on the device. Returns original right_impl + const TensorEvaluator<RightArgType, Device>& right_impl() const { + return choose(Cond<static_cast<int>(Layout) == static_cast<int>(ColMajor)>(), this->m_rightImpl, this->m_leftImpl); + } +}; + +template <typename HostExpr, typename OutScalar, typename LhsScalar, typename RhsScalar, typename LHSFunctorExpr, typename RHSFunctorExpr, typename LhsLocalAcc, typename RhsLocalAcc, typename OutAccessor, typename Index, typename ContractT, typename LeftNocontractT, +typename RightNocontractT, bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, +typename HostExpr::Index TileSizeDimM, typename HostExpr::Index TileSizeDimN,typename HostExpr::Index TileSizeDimK, typename HostExpr::Index WorkLoadPerThreadM,typename HostExpr::Index WorkLoadPerThreadN, +typename HostExpr::Index LocalThreadSizeM, typename HostExpr::Index LocalThreadSizeN, typename HostExpr::Index LoadPerThreadLhs, typename HostExpr::Index LoadPerThreadRhs, typename LHSTupleType, typename RHSTupleType, typename Device> struct KernelConstructor{ + typedef typename Eigen::internal::traits<HostExpr>::_LhsNested LHSHostExpr; + typedef typename Eigen::internal::traits<HostExpr>::_RhsNested RHSHostExpr; + typedef typename Eigen::TensorSycl::internal::createPlaceHolderExpression<LHSHostExpr>::Type LHSPlaceHolderExpr; + typedef typename Eigen::TensorSycl::internal::createPlaceHolderExpression<RHSHostExpr>::Type RHSPlaceHolderExpr; + LHSFunctorExpr lhs_functors; + RHSFunctorExpr rhs_functors; + LhsLocalAcc localLhs; + RhsLocalAcc localRhs; + OutAccessor out_res; + size_t out_offset; + Index roundUpK, M, N, K; + ContractT m_k_strides, m_left_contracting_strides, m_right_contracting_strides; + LeftNocontractT m_i_strides, m_left_nocontract_strides; + RightNocontractT m_j_strides, m_right_nocontract_strides; + LHSTupleType left_tuple_of_accessors; + RHSTupleType right_tuple_of_accessors; + Device dev; + + + KernelConstructor(LHSFunctorExpr lhs_functors_, RHSFunctorExpr rhs_functors_, LhsLocalAcc localLhs_, RhsLocalAcc localRhs_, OutAccessor out_res_, size_t out_offset_, + Index roundUpK_, Index M_, Index N_, Index K_, ContractT m_k_strides_, ContractT m_left_contracting_strides_, + ContractT m_right_contracting_strides_, LeftNocontractT m_i_strides_, RightNocontractT m_j_strides_, + LeftNocontractT m_left_nocontract_strides_, RightNocontractT m_right_nocontract_strides_, LHSTupleType left_tuple_of_accessors_, RHSTupleType right_tuple_of_accessors_, Device dev_) + :lhs_functors(lhs_functors_), rhs_functors(rhs_functors_), localLhs(localLhs_), localRhs(localRhs_), out_res(out_res_), + out_offset(out_offset_), roundUpK(roundUpK_), M(M_), N(N_), K(K_), + m_k_strides(m_k_strides_), m_left_contracting_strides(m_left_contracting_strides_), + m_right_contracting_strides(m_right_contracting_strides_), + m_i_strides(m_i_strides_), m_left_nocontract_strides(m_left_nocontract_strides_), + m_j_strides(m_j_strides_), m_right_nocontract_strides(m_right_nocontract_strides_), + left_tuple_of_accessors(left_tuple_of_accessors_), right_tuple_of_accessors(right_tuple_of_accessors_), dev(dev_){} + + void operator()(cl::sycl::nd_item<2> itemID) { + typedef typename Eigen::TensorSycl::internal::ConvertToDeviceExpression<HostExpr>::Type DevExpr; + typedef typename Eigen::TensorSycl::internal::ConvertToDeviceExpression<LHSHostExpr>::Type LHSDevExpr; + typedef typename Eigen::TensorSycl::internal::ConvertToDeviceExpression<RHSHostExpr>::Type RHSDevExpr; + auto lhs_dev_expr = Eigen::TensorSycl::internal::createDeviceExpression<LHSDevExpr, LHSPlaceHolderExpr>(lhs_functors, left_tuple_of_accessors); + auto rhs_dev_expr = Eigen::TensorSycl::internal::createDeviceExpression<RHSDevExpr, RHSPlaceHolderExpr>(rhs_functors, right_tuple_of_accessors); + typedef decltype(lhs_dev_expr.expr) LeftArgType; + typedef decltype(rhs_dev_expr.expr) RightArgType; + typedef typename internal::conditional<static_cast<int>(Eigen::internal::traits<DevExpr>::Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>::type EvalLeftArgType; + typedef typename internal::conditional<static_cast<int>(Eigen::internal::traits<DevExpr>::Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>::type EvalRightArgType; + typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; + typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; + typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, + LeftEvaluator, LeftNocontractT, + ContractT, 1, + lhs_inner_dim_contiguous, + false, Unaligned, MakeGlobalPointer> LhsMapper; + + typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, + RightEvaluator, RightNocontractT, + ContractT, 1, + rhs_inner_dim_contiguous, + rhs_inner_dim_reordered, Unaligned, MakeGlobalPointer> RhsMapper; + // initialize data mappers must happen inside the kernel for device eval + LhsMapper lhs(LeftEvaluator(choose(Cond<static_cast<int>(Eigen::internal::traits<DevExpr>::Layout) == static_cast<int>(ColMajor)>(), + lhs_dev_expr.expr, rhs_dev_expr.expr), dev), m_left_nocontract_strides, m_i_strides, m_left_contracting_strides, m_k_strides); + RhsMapper rhs(RightEvaluator(choose(Cond<static_cast<int>(Eigen::internal::traits<DevExpr>::Layout) == static_cast<int>(ColMajor)>(), + rhs_dev_expr.expr, lhs_dev_expr.expr),dev), m_right_nocontract_strides, m_j_strides, m_right_contracting_strides, m_k_strides); + auto out_ptr = ConvertToActualTypeSycl(OutScalar, out_res); + // Matmul Kernel + // Thread identifiers + const Index mLocalThreadId = itemID.get_local(0); // Local ID row + const Index nLocalThreadId = itemID.get_local(1); // Local ID col + const Index mGroupId = itemID.get_group(0); // Work-group ID row + const Index nGroupId = itemID.get_group(1); // Work-group ID localCol + const Index linearLocalThreadId = nLocalThreadId*LocalThreadSizeM + mLocalThreadId; // linear local thread ID + // Allocate register space + LhsScalar privateLhs; + RhsScalar privateRhs[WorkLoadPerThreadN]; + OutScalar privateRes[WorkLoadPerThreadM][WorkLoadPerThreadN]; + // Initialise the privateResumulation registers + for (Index wLPTM=0; wLPTM<WorkLoadPerThreadM; wLPTM++) { + for (Index wLPTN=0; wLPTN<WorkLoadPerThreadN; wLPTN++) { + privateRes[wLPTM][wLPTN] = static_cast<OutScalar>(0); + } + } + + // Tile Lhs + for (Index lPTL=0; lPTL<LoadPerThreadLhs; lPTL++) { + Index localLhsLinearId = lPTL*LocalThreadSizeN*LocalThreadSizeM + linearLocalThreadId; + Index localLhsRow = localLhsLinearId% TileSizeDimM; + Index localLhsCol = localLhsLinearId/TileSizeDimM; + // Load the value (wide vector load) + Index GlobalLhsColId = TileSizeDimK*0 + localLhsCol; + localLhs[0 + ((localLhsCol*TileSizeDimM + localLhsRow)*2)] =((GlobalLhsColId < K)&& (mGroupId*(TileSizeDimM)+ localLhsRow <M))? lhs(mGroupId*(TileSizeDimM) + localLhsRow, GlobalLhsColId):static_cast<OutScalar>(0); + } + // Tile Rhs + for (Index lPTR=0; lPTR<LoadPerThreadRhs; lPTR++) { + Index localRhsLinearId = lPTR*LocalThreadSizeN*LocalThreadSizeM + linearLocalThreadId; + Index localRhsRow = localRhsLinearId% TileSizeDimN; + Index localRhsCol = localRhsLinearId/TileSizeDimN; + // Load the value (wide vector load) + Index GlobalRhsRowId = TileSizeDimK*0 + localRhsCol; + localRhs[0 + ((localRhsCol*TileSizeDimN + localRhsRow) *2)] = ((GlobalRhsRowId < K)&& ((nGroupId*(TileSizeDimN) + localRhsRow)< N))? rhs(GlobalRhsRowId, nGroupId*(TileSizeDimN) + localRhsRow): static_cast<OutScalar>(0); + + } + // Loop over all tiles + const Index numTiles = roundUpK/TileSizeDimK; + Index firstHalf=0; + do { + // Synchronise + itemID.barrier(cl::sycl::access::fence_space::local_space); + // Load the next tile of Lhs and Rhs into local memory + Index nextHalf = firstHalf + 1; + if (nextHalf < numTiles) { + // Tile A + for (Index lPTL=0; lPTL<LoadPerThreadLhs; lPTL++) { + Index localLhsLinearId = lPTL*LocalThreadSizeN*LocalThreadSizeM + linearLocalThreadId; + Index localLhsRow = localLhsLinearId% TileSizeDimM; + Index localLhsCol = localLhsLinearId/TileSizeDimM; + // global K id + Index GlobalLhsColId = TileSizeDimK*nextHalf + localLhsCol; + // Store the loaded value into local memory + localLhs[(nextHalf%2) + ((localLhsCol*TileSizeDimM + localLhsRow) *2)] = ((GlobalLhsColId < K)&& (mGroupId*(TileSizeDimM)+ localLhsRow <M))? lhs(mGroupId*(TileSizeDimM) + localLhsRow, GlobalLhsColId): static_cast<OutScalar>(0); + } + // Tile B + for (Index lPTR=0; lPTR<LoadPerThreadRhs; lPTR++) { + Index localRhsLinearId = lPTR*LocalThreadSizeN*LocalThreadSizeM + linearLocalThreadId; + Index localRhsRow = localRhsLinearId% TileSizeDimN; + Index localRhsCol = localRhsLinearId/TileSizeDimN; + // Load the value (wide vector load) + Index GlobalRhsRowId = TileSizeDimK*nextHalf + localRhsCol; + // Store the loaded vector into local memory + localRhs[(nextHalf%2) +((localRhsCol*TileSizeDimN + localRhsRow)*2)] = ((GlobalRhsRowId < K)&& ((nGroupId*(TileSizeDimN) + localRhsRow)< N))? rhs(GlobalRhsRowId, nGroupId*(TileSizeDimN) + localRhsRow):static_cast<OutScalar>(0); + } + } + // Loop over the values of a single tile + for (Index k=0; k<TileSizeDimK; k++) { + // Cache the values of localRhs in registers + for (Index wLPTN=0; wLPTN<WorkLoadPerThreadN; wLPTN++) { + Index localRhsCol = nLocalThreadId + wLPTN*LocalThreadSizeN; + privateRhs[wLPTN] = localRhs[(firstHalf%2) +((k*TileSizeDimN + localRhsCol)*2)]; + } + // Perform the computation + for (Index wLPTM=0; wLPTM<WorkLoadPerThreadM; wLPTM++) { + Index localLhsRow = mLocalThreadId + wLPTM*LocalThreadSizeM; + privateLhs = localLhs[(firstHalf%2)+ ((k*TileSizeDimM + localLhsRow)*2)]; + for (Index wLPTN=0; wLPTN<WorkLoadPerThreadN; wLPTN++) { + privateRes[wLPTM][wLPTN] += privateLhs * privateRhs[wLPTN]; + } + } + } + // Next tile + firstHalf++; + } while (firstHalf<numTiles); + + // Store the final results in C + for (Index wLPTM=0; wLPTM<WorkLoadPerThreadM; wLPTM++) { + Index globalRow = mGroupId*TileSizeDimM + mLocalThreadId + wLPTM*LocalThreadSizeM; + if (globalRow< M){ + for (Index wLPTN=0; wLPTN<WorkLoadPerThreadN; wLPTN++) { + Index globalCol = nGroupId*TileSizeDimN + nLocalThreadId + wLPTN*LocalThreadSizeN; + if(globalCol<N) + out_ptr[globalCol*M + globalRow +ConvertToActualSyclOffset(OutScalar, out_offset)] = privateRes[wLPTM][wLPTN]; + } + } + } + + } + +}; +template <typename Index, typename LhsScalar, typename RhsScalar, bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered> struct LaunchSyclKernels { + +static const Index TileSizeDimM = 32ul; // Tile size for dimension M +static const Index TileSizeDimN = 32ul; // Tile size for dimension N +static const Index TileSizeDimK = 16ul; // Tile size for dimension K +static const Index WorkLoadPerThreadM = 4ul; // Work load per thread in dimension M +static const Index WorkLoadPerThreadN = 4ul; // work load per thread in dimension N +static const Index LocalThreadSizeM = (TileSizeDimM/WorkLoadPerThreadM); // Local thread size for the first dimension (M here) +static const Index LocalThreadSizeN = (TileSizeDimN/WorkLoadPerThreadN); // Local thread size for the second dimension (N here) +static const Index LoadPerThreadLhs = ((TileSizeDimK*WorkLoadPerThreadM*WorkLoadPerThreadN)/(TileSizeDimN)); // workload per thread for Lhs expression +static const Index LoadPerThreadRhs = ((TileSizeDimK*WorkLoadPerThreadM*WorkLoadPerThreadN)/(TileSizeDimM)); // workload per thread for Rhs expression + +// RoundUp function to make sure that the global threadId is divisable by local threadId +static Index RoundUp(Index x, Index y) { + return ((((x) + (y) - 1) / (y))*(y)); +} + +template< typename Self, typename OutScalar, typename ContractT, typename LeftNocontractT, typename RightNocontractT> + static void Run(const Self& self, OutScalar* buffer, Index M, Index N, Index K, + ContractT m_k_strides, ContractT m_left_contracting_strides, ContractT m_right_contracting_strides, + LeftNocontractT m_i_strides, RightNocontractT m_j_strides, LeftNocontractT m_left_nocontract_strides, RightNocontractT m_right_nocontract_strides){ + + typedef typename Self::XprType HostExpr; + typedef typename Eigen::internal::traits<HostExpr>::_LhsNested LHSHostExpr; + typedef typename Eigen::internal::traits<HostExpr>::_RhsNested RHSHostExpr; + typedef TensorEvaluator<LHSHostExpr, const Eigen::SyclDevice> OrigLHSExpr; + typedef TensorEvaluator<RHSHostExpr, const Eigen::SyclDevice> OrigRHSExpr; + typedef Eigen::TensorSycl::internal::FunctorExtractor<OrigLHSExpr> LHSFunctorExpr; + typedef Eigen::TensorSycl::internal::FunctorExtractor<OrigRHSExpr> RHSFunctorExpr; + // extract lhs functor list + LHSFunctorExpr lhs_functors = Eigen::TensorSycl::internal::extractFunctors(self.left_impl()); + // extract rhs functor list + RHSFunctorExpr rhs_functors = Eigen::TensorSycl::internal::extractFunctors(self.right_impl()); + + Index roundUpK = RoundUp(K, TileSizeDimK); + Index roundUpM = RoundUp(M, TileSizeDimM); + Index roundUpN = RoundUp(N, TileSizeDimN); + ptrdiff_t out_offset = self.device().get_offset(buffer); + self.device().sycl_queue().submit([&](cl::sycl::handler &cgh) { + /// work-around for gcc bug + typedef decltype(Eigen::TensorSycl::internal::createTupleOfAccessors<OrigLHSExpr>(cgh, self.left_impl())) LHSTupleType; + /// work-around for gcc bug + typedef decltype(Eigen::TensorSycl::internal::createTupleOfAccessors<OrigRHSExpr>(cgh, self.right_impl())) RHSTupleType; + // create lhs tuple of accessors + LHSTupleType left_tuple_of_accessors = Eigen::TensorSycl::internal::createTupleOfAccessors<OrigLHSExpr>(cgh, self.left_impl()); + // create rhs tuple of accessors + RHSTupleType right_tuple_of_accessors = Eigen::TensorSycl::internal::createTupleOfAccessors<OrigRHSExpr>(cgh, self.right_impl()); + + // Local memory for elements of Lhs + typedef cl::sycl::accessor<LhsScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local> LhsLocalAcc; + LhsLocalAcc localLhs(cl::sycl::range<1>(2* TileSizeDimM * TileSizeDimK), cgh); + // Local memory for elements of Rhs + typedef cl::sycl::accessor<RhsScalar, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local> RhsLocalAcc; + RhsLocalAcc localRhs(cl::sycl::range<1>(2* TileSizeDimK * TileSizeDimN), cgh); + + typedef cl::sycl::accessor<uint8_t, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer> OutAccessor; + //OutScalar memory + OutAccessor out_res= self.device(). template get_sycl_accessor<cl::sycl::access::mode::read_write>(cgh, buffer); + // sycl parallel for + cgh.parallel_for(cl::sycl::nd_range<2>(cl::sycl::range<2>(roundUpM/WorkLoadPerThreadM, roundUpN/WorkLoadPerThreadN), + cl::sycl::range<2>(LocalThreadSizeM, LocalThreadSizeN)), + KernelConstructor<HostExpr, OutScalar, LhsScalar, RhsScalar, LHSFunctorExpr, RHSFunctorExpr, LhsLocalAcc, RhsLocalAcc, OutAccessor, Index, ContractT, LeftNocontractT, + RightNocontractT, lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, TileSizeDimM, TileSizeDimN, TileSizeDimK, + WorkLoadPerThreadM, WorkLoadPerThreadN, LocalThreadSizeM, LocalThreadSizeN, LoadPerThreadLhs, LoadPerThreadRhs, LHSTupleType, RHSTupleType, Eigen::SyclKernelDevice>(lhs_functors, rhs_functors, + localLhs, localRhs, out_res, out_offset, roundUpK, M, N, K, m_k_strides, m_left_contracting_strides, m_right_contracting_strides,m_i_strides, m_j_strides, + m_left_nocontract_strides,m_right_nocontract_strides, left_tuple_of_accessors, right_tuple_of_accessors, Eigen::SyclKernelDevice())); + }); + self.device().asynchronousExec(); + } +}; + +} // end namespace Eigen +#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_SYCL_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h index 9044454..d7cd995 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h
@@ -14,56 +14,17 @@ #ifdef EIGEN_USE_THREADS namespace Eigen { -namespace internal { -template<typename LhsScalar, typename LhsMapper, typename Index> -struct packLhsArg { - LhsScalar* blockA; - const LhsMapper& lhs; - const Index m_start; - const Index k_start; - const Index mc; - const Index kc; -}; - -template<typename LhsScalar, typename RhsScalar, typename RhsMapper, typename OutputMapper, typename Index> -struct packRhsAndKernelArg { - const MaxSizeVector<LhsScalar*>* blockAs; - RhsScalar* blockB; - const RhsMapper& rhs; - OutputMapper& output; - const Index m; - const Index k; - const Index n; - const Index mc; - const Index kc; - const Index nc; - const Index num_threads; - const Index num_blockAs; - const Index max_m; - const Index k_block_idx; - const Index m_block_idx; - const Index n_block_idx; - const Index m_blocks; - const Index n_blocks; - MaxSizeVector<Notification*>* kernel_notifications; - const MaxSizeVector<Notification*>* lhs_notifications; - const bool need_to_pack; -}; - -} // end namespace internal - - -template<typename Indices, typename LeftArgType, typename RightArgType> -struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, ThreadPoolDevice> : - public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, ThreadPoolDevice> > { +template<typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType> +struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, ThreadPoolDevice> : + public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, ThreadPoolDevice> > { typedef ThreadPoolDevice Device; - typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> Self; + typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device> Self; typedef TensorContractionEvaluatorBase<Self> Base; - typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; + typedef TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; @@ -92,10 +53,10 @@ typedef array<Index, RDims> right_dim_mapper_t; typedef array<Index, ContractDims> contract_t; - typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; - typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; + typedef array<Index, LDims - ContractDims> left_nocontract_t; + typedef array<Index, RDims - ContractDims> right_nocontract_t; - static const int NumDims = max_n_1<LDims + RDims - 2 * ContractDims>::size; + static const int NumDims = LDims + RDims - 2 * ContractDims; typedef DSizes<Index, NumDims> Dimensions; @@ -110,280 +71,1353 @@ TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) {} - template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> + template <int Alignment> void evalProduct(Scalar* buffer) const { - if (this->m_j_size == 1) { - this->template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer); + const Index m = this->m_i_size; + const Index n = this->m_j_size; + const Index k = this->m_k_size; + if (m == 0 || n == 0 || k == 0) return; + +#if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) + if (this->m_can_use_xsmm) { + bool transposeA = !this->m_lhs_inner_dim_contiguous; + bool transposeB = !this->m_rhs_inner_dim_contiguous; + internal::TensorXsmmContractionBlocking<LhsScalar, RhsScalar, Index> + blocking(k, m, n, this->m_device.numThreads(), transposeA, + transposeB); + + if (blocking.num_threads() == 1) { + this->evalGemmXSMM(buffer); + } else { + ContextXsmm<Alignment>(this, buffer, m, n, k, blocking).run(); + } + return; + } +#endif + + // Compute a set of algorithm parameters: + // - kernel block sizes (bm, bn, bk) + // - task grain sizes (number of kernels executed per task: gm, gn) + // - number of threads + // - sharding by row/column + // - parallel packing or first lhs then rhs + // and some derived parameters: + // - number of tasks (nm, nn, nk) + // - number of kernels (nm0, nn0) + // Unfortunately, all these parameters are tightly interdependent. + // So in some cases we first compute approximate values, then compute other + // values based on these approximations and then refine the approximations. + + // There are lots of heuristics here. There is some reasoning behind them, + // but ultimately they are just tuned on contraction benchmarks for + // different input configurations, thread counts and instruction sets. + // So feel free to question any of them. + + // Compute whether we want to shard by row or by column. + // This is a first approximation, it will be refined later. Since we don't + // know number of threads yet we use 2, because what's we are most + // interested in at this point is whether it makes sense to use + // parallelization at all or not. + bool shard_by_col = shardByCol(m, n, 2); + + // First approximation of kernel blocking sizes. + // Again, we don't know number of threads yet, so we use 2. + Index bm, bn, bk; + if (shard_by_col) { + internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, + internal::ShardByCol> + blocking(k, m, n, 2); + bm = blocking.mc(); + bn = blocking.nc(); + bk = blocking.kc(); + } else { + internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, + internal::ShardByRow> + blocking(k, m, n, 2); + bm = blocking.mc(); + bn = blocking.nc(); + bk = blocking.kc(); + } + + // Compute optimal number of threads. + // Note: we use bk instead of k here because we are interested in amount of + // _parallelizable_ computations, and computations are not parallelizable + // across k dimension. + const TensorOpCost cost = + contractionCost(m, n, bm, bn, bk, shard_by_col, false); + int num_threads = TensorCostModel<ThreadPoolDevice>::numThreads( + static_cast<double>(n) * m, cost, this->m_device.numThreads()); + int num_threads_by_k = numThreadsInnerDim(m, n, k); + if (shardByInnerDim(m, n, k, num_threads, num_threads_by_k)) { + // We are in the scenario where it is more effective to shard by the + // inner dimension. + this->template evalShardedByInnerDim<Alignment>(num_threads_by_k, + buffer); return; } - evalGemm<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer); + // TODO(dvyukov): this is a stop-gap to prevent regressions while the cost + // model is not tuned. Remove this when the cost model is tuned. + if (n == 1) num_threads = 1; + + if (num_threads == 1) { + TENSOR_CONTRACTION_DISPATCH(this->template evalProductSequential, + Unaligned, (buffer)); + return; + } + + // Now that we know number of threads, recalculate sharding and blocking. + shard_by_col = shardByCol(m, n, num_threads); + if (shard_by_col) { + internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, + internal::ShardByCol> + blocking(k, m, n, num_threads); + bm = blocking.mc(); + bn = blocking.nc(); + bk = blocking.kc(); + } else { + internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, + internal::ShardByRow> + blocking(k, m, n, num_threads); + bm = blocking.mc(); + bn = blocking.nc(); + bk = blocking.kc(); + } + + // Number of kernels for each dimension. + Index nm0 = divup(m, bm); + Index nn0 = divup(n, bn); + Index nk = divup(k, bk); + + // Calculate task grain size (number of kernels executed per task). + // This task size coarsening serves two purposes: + // 1. It reduces per-task overheads including synchronization overheads. + // 2. It allows to use caches better (reuse the same packed rhs in several + // consecutive kernels). + Index gm = 1; + Index gn = 1; + // If we are sharding by column, then we prefer to reduce rows first. + if (shard_by_col) { + gm = coarsenM(m, n, bm, bn, bk, gn, num_threads, shard_by_col); + gn = coarsenN(m, n, bm, bn, bk, gm, num_threads, shard_by_col); + } else { + gn = coarsenN(m, n, bm, bn, bk, gm, num_threads, shard_by_col); + gm = coarsenM(m, n, bm, bn, bk, gn, num_threads, shard_by_col); + } + // Number of tasks in each dimension. + Index nm = divup(nm0, gm); + Index nn = divup(nn0, gn); + + // If there is enough concurrency in the sharding dimension, we choose not + // to paralellize by the other dimension, and execute all kernels in sync + // mode. This reduces parallelism from the nm x nn down to nn + // (shard_by_col==true) or nm (shard_by_col==false). + const Index sharding_dim_tasks = shard_by_col ? nn : nm; + const int num_worker_threads = this->m_device.numThreadsInPool(); + + // With small number of threads we want to make sure that we do not reduce + // parallelism too much. + const int oversharding_factor = + num_worker_threads <= 4 ? 8 : + num_worker_threads <= 8 ? 4 : + num_worker_threads <= 16 ? 2 : 1; + + const bool parallelize_by_sharding_dim_only = + sharding_dim_tasks >= oversharding_factor * num_worker_threads; + + // Last by not least, decide whether we want to issue both lhs and rhs + // packing in parallel; or issue lhs packing first, and then issue rhs + // packing when lhs packing completes (for !shard_by_col lhs and rhs are + // swapped). Parallel packing allows more parallelism (for both packing and + // kernels), while sequential packing provides better locality (once + // a thread finishes rhs packing it proceed to kernels with that rhs). + // First, we are interested in parallel packing if there are few tasks. + bool parallel_pack = num_threads >= nm * nn; + // Also do parallel packing if all data fits into L2$. + if (m * bk * Index(sizeof(LhsScalar)) + n * bk * Index(sizeof(RhsScalar)) <= + l2CacheSize() * num_threads) + parallel_pack = true; + // But don't do it if we will use each rhs only once. Locality seems to be + // more important in this case. + if ((shard_by_col ? nm : nn) == 1) parallel_pack = false; + // Also don't get in the way of parallelize_by_sharding_dim_only + // optimization. + if (parallelize_by_sharding_dim_only) parallel_pack = false; + +#define CONTEXT_ARGS \ + (this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, \ + nn0, shard_by_col, parallel_pack, parallelize_by_sharding_dim_only) \ + .run() + + TENSOR_CONTRACTION_DISPATCH(Context, Alignment, CONTEXT_ARGS); + +#undef CONTEXT_ARGS + } - template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - void evalGemm(Scalar* buffer) const { - // columns in left side, rows in right side - const Index k = this->m_k_size; - - // rows in left side - const Index m = this->m_i_size; - - // columns in right side - const Index n = this->m_j_size; - - // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar) - this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); - - - const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size; - const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size; - - typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, - LeftEvaluator, left_nocontract_t, - contract_t, lhs_packet_size, - lhs_inner_dim_contiguous, - false, Unaligned> LhsMapper; - - typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, - RightEvaluator, right_nocontract_t, - contract_t, rhs_packet_size, - rhs_inner_dim_contiguous, - rhs_inner_dim_reordered, Unaligned> RhsMapper; + // Context coordinates a single parallel gemm operation. + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, + bool rhs_inner_dim_reordered, int Alignment> + class Context { + public: + typedef internal::TensorContractionInputMapper< + LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t, + contract_t, internal::packet_traits<LhsScalar>::size, + lhs_inner_dim_contiguous, false, Unaligned> + LhsMapper; + typedef internal::TensorContractionInputMapper< + RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t, + contract_t, internal::packet_traits<RhsScalar>::size, + rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Unaligned> + RhsMapper; typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; - // TODO: packing could be faster sometimes if we supported row major tensor mappers - typedef internal::gemm_pack_lhs<LhsScalar, Index, typename LhsMapper::SubMapper, Traits::mr, - Traits::LhsProgress, ColMajor> LhsPacker; - typedef internal::gemm_pack_rhs<RhsScalar, Index, typename RhsMapper::SubMapper, Traits::nr, ColMajor> RhsPacker; + typedef internal::TensorContractionKernel< + Scalar, LhsScalar, RhsScalar, Index, OutputMapper, LhsMapper, RhsMapper> + TensorContractionKernel; - // TODO: replace false, false with conjugate values? - typedef internal::gebp_kernel<LhsScalar, RhsScalar, Index, OutputMapper, - Traits::mr, Traits::nr, false, false> GebpKernel; + Context(const Self* self, int num_threads, Scalar* buffer, Index tm, Index tn, + Index tk, Index bm, Index bn, Index bk, Index nm, Index nn, Index nk, + Index gm, Index gn, Index nm0, Index nn0, bool shard_by_col, + bool parallel_pack, bool parallelize_by_sharding_dim_only) + : device_(self->m_device), + lhs_(self->m_leftImpl, self->m_left_nocontract_strides, + self->m_i_strides, self->m_left_contracting_strides, + self->m_k_strides), + rhs_(self->m_rightImpl, self->m_right_nocontract_strides, + self->m_j_strides, self->m_right_contracting_strides, + self->m_k_strides), + buffer_(buffer), + output_(buffer, tm), + output_kernel_(self->m_output_kernel), + tensor_contraction_params_(self->m_tensor_contraction_params), + num_threads_(num_threads), + shard_by_col_(shard_by_col), + parallel_pack_(parallel_pack), + parallelize_by_sharding_dim_only_(parallelize_by_sharding_dim_only), + m_(tm), + n_(tn), + k_(tk), + bm_(bm), + bn_(bn), + bk_(bk), + nm_(nm), + nn_(nn), + nk_(nk), + gm_(gm), + gn_(gn), + nm0_(nm0), + nn0_(nn0) + { + // These two options are mutually exclusive. + eigen_assert(!(parallel_pack && parallelize_by_sharding_dim_only)); - typedef internal::packLhsArg<LhsScalar, LhsMapper, Index> packLArg; - typedef internal::packRhsAndKernelArg<LhsScalar, RhsScalar, RhsMapper, OutputMapper, Index> packRKArg; - - // initialize data mappers - LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides, - this->m_left_contracting_strides, this->m_k_strides); - - RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides, - this->m_right_contracting_strides, this->m_k_strides); - - OutputMapper output(buffer, m); - - // compute block sizes (which depend on number of threads) - const Index num_threads = this->m_device.numThreads(); - internal::TensorContractionBlocking<LhsMapper, RhsMapper, Index, internal::ShardByCol> blocking(k, m, n, num_threads); - Index mc = blocking.mc(); - Index nc = blocking.nc(); - Index kc = blocking.kc(); - eigen_assert(mc <= m); - eigen_assert(nc <= n); - eigen_assert(kc <= k); - -#define CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) - const Index k_blocks = CEIL_DIV(k, kc); - const Index n_blocks = CEIL_DIV(n, nc); - const Index m_blocks = CEIL_DIV(m, mc); - const Index sizeA = mc * kc; - const Index sizeB = kc * nc; - - /* cout << "m: " << m << " n: " << n << " k: " << k << endl; - cout << "mc: " << mc << " nc: " << nc << " kc: " << kc << endl; - cout << "m_blocks: " << m_blocks << " n_blocks: " << n_blocks << " k_blocks: " << k_blocks << endl; - cout << "num threads: " << num_threads << endl; - */ - - // note: m_device.allocate should return 16 byte aligned pointers, but if blockA and blockB - // aren't 16 byte aligned segfaults will happen due to SIMD instructions - // note: You can get away with allocating just a single blockA and offsets and meet the - // the alignment requirements with the assumption that - // (Traits::mr * sizeof(ResScalar)) % 16 == 0 - const Index numBlockAs = numext::mini(num_threads, m_blocks); - MaxSizeVector<LhsScalar *> blockAs(num_threads); - for (int i = 0; i < num_threads; i++) { - blockAs.push_back(static_cast<LhsScalar *>(this->m_device.allocate(sizeA * sizeof(LhsScalar)))); - } - - // To circumvent alignment issues, I'm just going to separately allocate the memory for each thread - // TODO: is this too much memory to allocate? This simplifies coding a lot, but is wasteful. - // Other options: (1) reuse memory when a thread finishes. con: tricky - // (2) allocate block B memory in each thread. con: overhead - MaxSizeVector<RhsScalar *> blockBs(n_blocks); - for (int i = 0; i < n_blocks; i++) { - blockBs.push_back(static_cast<RhsScalar *>(this->m_device.allocate(sizeB * sizeof(RhsScalar)))); - } - - // lhs_notifications starts with all null Notifications - MaxSizeVector<Notification*> lhs_notifications(num_threads, nullptr); - - // this should really be numBlockAs * n_blocks; - const Index num_kernel_notifications = num_threads * n_blocks; - MaxSizeVector<Notification*> kernel_notifications(num_kernel_notifications, - nullptr); - - for (Index k_block_idx = 0; k_block_idx < k_blocks; k_block_idx++) { - const Index k_start = k_block_idx * kc; - // make sure we don't overshoot right edge of left matrix - const Index actual_kc = numext::mini(k_start + kc, k) - k_start; - - for (Index m_block_idx = 0; m_block_idx < m_blocks; m_block_idx += numBlockAs) { - const Index num_blocks = numext::mini(m_blocks-m_block_idx, numBlockAs); - - for (Index mt_block_idx = m_block_idx; mt_block_idx < m_block_idx+num_blocks; mt_block_idx++) { - const Index m_start = mt_block_idx * mc; - const Index actual_mc = numext::mini(m_start + mc, m) - m_start; - eigen_assert(actual_mc > 0); - - Index blockAId = (k_block_idx * m_blocks + mt_block_idx) % num_threads; - - for (int i = 0; i < n_blocks; ++i) { - Index notification_id = (blockAId * n_blocks + i); - // Wait for any current kernels using this slot to complete - // before using it. - if (kernel_notifications[notification_id]) { - wait_until_ready(kernel_notifications[notification_id]); - delete kernel_notifications[notification_id]; - } - kernel_notifications[notification_id] = new Notification(); - } - const packLArg arg = { - blockAs[blockAId], // blockA - lhs, // lhs - m_start, // m - k_start, // k - actual_mc, // mc - actual_kc, // kc - }; - - // Delete any existing notification since we may be - // replacing it. The algorithm should ensure that there are - // no existing waiters on this notification. - delete lhs_notifications[blockAId]; - lhs_notifications[blockAId] = - this->m_device.enqueue(&Self::packLhs<packLArg, LhsPacker>, arg); + for (Index x = 0; x < P; x++) { + // Normal number of notifications for k slice switch is + // nm_ + nn_ + nm_ * nn_. However, first P - 1 slices will receive only + // nm_ + nn_ notifications, because they will not receive notifications + // from preceding kernels. + state_switch_[x] = + x == 0 + ? 1 + : (parallel_pack_ ? nn_ + nm_ : (shard_by_col_ ? nn_ : nm_)) + + (x == P - 1 ? nm_ * nn_ : 0); + state_packing_ready_[x] = + parallel_pack_ ? 0 : (shard_by_col_ ? nm_ : nn_); + state_kernel_[x] = new std::atomic<uint8_t>*[nm_]; + for (Index m = 0; m < nm_; m++) { + state_kernel_[x][m] = new std::atomic<uint8_t>[nn_]; + // Kernels generally receive 3 notifications (previous kernel + 2 + // packing), but the first slice won't get notifications from previous + // kernels. + for (Index n = 0; n < nn_; n++) + state_kernel_[x][m][n].store( + (x == 0 ? 0 : 1) + (parallel_pack_ ? 2 : 1), + std::memory_order_relaxed); } + } - // now start kernels. - const Index m_base_start = m_block_idx * mc; - const bool need_to_pack = m_block_idx == 0; + // Allocate memory for packed rhs/lhs matrices. + size_t align = numext::maxi(EIGEN_MAX_ALIGN_BYTES, 1); + size_t lhs_size = + divup<size_t>(bm_ * bk_ * sizeof(LhsScalar), align) * align; + size_t rhs_size = + divup<size_t>(bn_ * bk_ * sizeof(RhsScalar), align) * align; + packed_mem_ = static_cast<char*>(device_.allocate( + (nm0_ * lhs_size + nn0_ * rhs_size) * std::min<size_t>(nk_, P - 1))); + char* mem = static_cast<char*>(packed_mem_); + for (Index x = 0; x < numext::mini<Index>(nk_, P - 1); x++) { + packed_lhs_[x].resize(nm0_); + for (Index m = 0; m < nm0_; m++) { + packed_lhs_[x][m] = reinterpret_cast<LhsScalar*>(mem); + mem += lhs_size; + } + packed_rhs_[x].resize(nn0_); + for (Index n = 0; n < nn0_; n++) { + packed_rhs_[x][n] = reinterpret_cast<RhsScalar*>(mem); + mem += rhs_size; + } + } - for (Index n_block_idx = 0; n_block_idx < n_blocks; n_block_idx++) { - const Index n_start = n_block_idx * nc; - const Index actual_nc = numext::mini(n_start + nc, n) - n_start; + if (parallelize_by_sharding_dim_only_) { + const int num_worker_threads = device_.numThreadsInPool(); - // first make sure the previous kernels are all done before overwriting rhs. Also wait if - // we're going to start new k. In both cases need_to_pack is true. - if (need_to_pack) { - for (Index i = num_blocks; i < num_threads; ++i) { - Index blockAId = (k_block_idx * m_blocks + i + m_block_idx) % num_threads; - Index future_id = (blockAId * n_blocks + n_block_idx); - wait_until_ready(kernel_notifications[future_id]); - } + if (shard_by_col) { + can_use_thread_local_packed_ = new std::atomic<bool>[nn_]; + for (int i = 0; i < nn_; ++i) + can_use_thread_local_packed_[i].store(true, + std::memory_order_relaxed); + + Index num_blocks = num_worker_threads * gn_; + thread_local_packed_mem_ = device_.allocate(num_blocks * rhs_size); + mem = static_cast<char*>(thread_local_packed_mem_); + + thread_local_packed_rhs_.resize(num_blocks, nullptr); + for (Index i = 0; i < num_blocks; ++i) { + thread_local_packed_rhs_[i] = reinterpret_cast<RhsScalar*>(mem); + mem += rhs_size; } + } else { + can_use_thread_local_packed_ = new std::atomic<bool>[nm_]; + for (int i = 0; i < nm_; ++i) + can_use_thread_local_packed_[i].store(true, + std::memory_order_relaxed); - packRKArg arg = { - &blockAs, // blockA - blockBs[n_block_idx], // blockB - rhs, // rhs - output, // output - m_base_start, // m - k_start, // k - n_start, // n - mc, // mc - actual_kc, // kc - actual_nc, // nc - num_threads, - numBlockAs, - m, - k_block_idx, - m_block_idx, - n_block_idx, // n_block_idx - m_blocks, // m_blocks - n_blocks, // n_blocks - &kernel_notifications, // kernel notifications - &lhs_notifications, // lhs notifications - need_to_pack, // need_to_pack - }; + Index num_blocks = num_worker_threads * gm_; + thread_local_packed_mem_ = device_.allocate(num_blocks * lhs_size); + mem = static_cast<char*>(thread_local_packed_mem_); - // We asynchronously kick off this function, which ends up - // notifying the appropriate kernel_notifications objects, - // which this thread waits on before exiting. - this->m_device.enqueueNoNotification(&Self::packRhsAndKernel<packRKArg, RhsPacker, GebpKernel>, arg); + thread_local_packed_lhs_.resize(num_blocks, nullptr); + for (Index i = 0; i < num_blocks; ++i) { + thread_local_packed_lhs_[i] = reinterpret_cast<LhsScalar*>(mem); + mem += lhs_size; + } } } } - // Make sure all the kernels are done. - for (size_t i = 0; i < kernel_notifications.size(); ++i) { - wait_until_ready(kernel_notifications[i]); - delete kernel_notifications[i]; - } - - // No need to wait for lhs notifications since they should have - // already been waited on. Just clean them up. - for (size_t i = 0; i < lhs_notifications.size(); ++i) { - delete lhs_notifications[i]; - } - - // deallocate all of the memory for both A and B's - for (size_t i = 0; i < blockAs.size(); i++) { - this->m_device.deallocate(blockAs[i]); - } - for (size_t i = 0; i < blockBs.size(); i++) { - this->m_device.deallocate(blockBs[i]); - } - -#undef CEIL_DIV - } - - /* - * Packs a LHS block of size (mt, kc) starting at lhs(m, k). Before packing - * the LHS block, check that all of the kernels that worked on the same - * mt_block_idx in the previous m_block are done. - */ - template <typename packLArg, typename LhsPacker> - static void packLhs(const packLArg arg) { - // perform actual packing - LhsPacker pack_lhs; - pack_lhs(arg.blockA, arg.lhs.getSubMapper(arg.m_start, arg.k_start), arg.kc, arg.mc); - } - - /* - * Packs a RHS block of size (kc, nc) starting at (k, n) after checking that - * all kernels in the previous block are done. - * Then for each LHS future, we wait on the future and then call GEBP - * on the area packed by the future (which starts at - * blockA + future_idx * mt * kc) on the LHS and with the full packed - * RHS block. - * The output of this GEBP is written to output(m + i * mt, n). - */ - template <typename packRKArg, typename RhsPacker, typename GebpKernel> - static void packRhsAndKernel(packRKArg arg) { - if (arg.need_to_pack) { - RhsPacker pack_rhs; - pack_rhs(arg.blockB, arg.rhs.getSubMapper(arg.k, arg.n), arg.kc, arg.nc); - } - - GebpKernel gebp; - for (Index mt_block_idx = 0; mt_block_idx < arg.num_blockAs; mt_block_idx++) { - const Index m_base_start = arg.m + arg.mc*mt_block_idx; - if (m_base_start < arg.max_m) { - Index blockAId = (arg.k_block_idx * arg.m_blocks + mt_block_idx + arg.m_block_idx) % arg.num_threads; - wait_until_ready((*arg.lhs_notifications)[blockAId]); - const Index actual_mc = numext::mini(m_base_start + arg.mc, arg.max_m) - m_base_start; - gebp(arg.output.getSubMapper(m_base_start, arg.n), - (*arg.blockAs)[blockAId], arg.blockB, - actual_mc, arg.kc, arg.nc, 1.0, -1, -1, 0, 0); - - // Notify that the kernel is done. - const Index set_idx = blockAId * arg.n_blocks + arg.n_block_idx; - (*arg.kernel_notifications)[set_idx]->Notify(); + ~Context() { + for (Index x = 0; x < P; x++) { + for (Index m = 0; m < nm_; m++) delete[] state_kernel_[x][m]; + delete[] state_kernel_[x]; + } + device_.deallocate(packed_mem_); + if (parallelize_by_sharding_dim_only_) { + device_.deallocate(thread_local_packed_mem_); + delete[] can_use_thread_local_packed_; } } + + void run() { + // Kick off packing of the first slice. + signal_switch(0, 1); + // Wait for overall completion. + // TODO(dvyukov): this wait can lead to deadlock. + // If nthreads contractions are concurrently submitted from worker + // threads, this wait will block all worker threads and the system will + // deadlock. + done_.Wait(); + } + + private: + Notification done_; + const Device& device_; + LhsMapper lhs_; + RhsMapper rhs_; + Scalar* const buffer_; + OutputMapper output_; + OutputKernelType output_kernel_; + TensorContractionParams tensor_contraction_params_; + const int num_threads_; + const bool shard_by_col_; + const bool parallel_pack_; + const bool parallelize_by_sharding_dim_only_; + // Matrix sizes. + const Index m_; + const Index n_; + const Index k_; + // Block sizes. + const Index bm_; + const Index bn_; + const Index bk_; + // Number of tasks. + const Index nm_; + const Index nn_; + const Index nk_; + // Task grain sizes (number of kernels executed per task). + const Index gm_; + const Index gn_; + // Number of blocks (this is different from ni_/nn_ because of task size + // coarsening). + const Index nm0_; + const Index nn0_; + + // Parallelization strategy. + // + // Blocks related to the same k block can run in parallel because they write + // to different output blocks. So we parallelize within k slices, this + // gives us parallelism level of m x n. Before we can start any kernels + // related to k-th slice, we need to issue m lhs packing tasks and n rhs + // packing tasks. + // + // However, there is a bottleneck when we are finishing kernels for k-th + // slice (at the very end there is only 1 runnable kernel). To mitigate this + // bottleneck we allow kernels from k-th and k+1-th slices to run in + // parallel. Note that (m, n, k) and (m, n, k+1) kernels write to the same + // output block, so they must not run in parallel. + // + // This gives us the following dependency graph. + // On each k slice we have m x n kernel tasks, m lhs paking tasks and n rhs + // packing tasks. + // Kernel (m, n, k) can start when: + // - kernel (m, n, k-1) has finished + // - lhs packing (m, k) has finished + // - rhs packing (n, k) has finished + // Lhs/rhs packing can start when: + // - all k-1 packing has finished (artificially imposed to limit amount of + // parallel packing) + // + // On top of that we limit runnable tasks to two consecutive k slices. + // This is done to limit amount of memory we need for packed lhs/rhs + // (for each k slice we need m*bk + n*bk memory in packed_lhs_/packed_rhs_). + // + // state_switch_ tracks when we are ready to switch to the next k slice. + // state_kernel_[m][n] tracks when we are ready to kick off kernel (m, n). + // These variable are rolling over 3 consecutive k slices: first two we are + // actively executing + one to track completion of kernels in the second + // slice. + static const Index P = 3; + void* packed_mem_; + std::vector<LhsScalar*> packed_lhs_[P - 1]; + std::vector<RhsScalar*> packed_rhs_[P - 1]; + + // If we choose to parallelize only by the sharding dimension, each thread + // will have it's own "thead local" (not a c++ thread local storage) memory + // for packed_lhs or packed_rhs (shard_by_col = false of true). This memory + // can't be passed to a kernel that might execute on a different thread. + // + // In practice when we are ready to pack memory for the sharding dimension + // (rhs if shard_by_col==true) of the K-th slice, all kernels for K-1 slice + // already computed (99% of the time), and we can pack data into the thread + // local storage, and guarantee that all the kernels will be executed + // immediately in the same thread. This significantly increases L1 cache hit + // ratio and reduces pressure on the memory bus. + // + // It's still possible that kernel for the K-th slice will be ready before + // completion of the K-1 kernel, so we have to allocate "global" packed_lhs_ + // and packed_rhs_ to allow kernels to be executed later on a thread + // different from the thread that was used for packing. + void* thread_local_packed_mem_; + + // Only one of these will beinitialized depending on shard_by_col value. + std::vector<LhsScalar*> thread_local_packed_lhs_; + std::vector<RhsScalar*> thread_local_packed_rhs_; + + // After a particular shard for Kth slice missed thread local execution + // opportunity (K-1 slice didn't complete kernels execution), we can no + // longer schedule K+1 and following slices in thread local mode, because + // there is no more guarantee that previous kernels were executed + // sequentially in the same thread (size is nn_ or nm_). + std::atomic<bool>* can_use_thread_local_packed_; + + std::atomic<uint8_t>** state_kernel_[P]; + // state_switch_ is frequently modified by worker threads, while other + // fields are read-only after constructor. Let's move it to a separate cache + // line to reduce cache-coherency traffic. + char pad_[128]; + std::atomic<Index> state_packing_ready_[P]; + std::atomic<Index> state_switch_[P]; + + LhsScalar* packed_lhs(Index m, Index k, Index m1, bool use_thread_local) { + if (use_thread_local) { + eigen_assert(!shard_by_col_); + + Index base_idx = gm_ * device_.currentThreadId(); + Index grain_idx = m1 - m * gm_; + Index block_idx = base_idx + grain_idx; + + return thread_local_packed_lhs_[block_idx]; + } else { + return packed_lhs_[k % (P - 1)][m1]; + } + } + + RhsScalar* packed_rhs(Index n, Index k, Index n1, bool use_thread_local) { + if (use_thread_local) { + eigen_assert(shard_by_col_); + + Index base_idx = gn_ * device_.currentThreadId(); + Index grain_idx = n1 - n * gn_; + Index block_idx = base_idx + grain_idx; + + return thread_local_packed_rhs_[block_idx]; + } else { + return packed_rhs_[k % (P - 1)][n1]; + } + } + + // In following two methods (pack_lhs and pack_rhs), if we know for sure + // that we'll be able to immediately call a kernel with packed data, and do + // not submit it to the thread pool, we can use thread local memory for + // packed data. + // + // We can only reliably check it if we are running all kernels in sync mode + // (parallelize only by sharding dim). If kernel for m==0 (n==0) is ready to + // run, it's guaranteed that all kernels with larger values of m (n) are + // also ready, because we execute them in the same order for all K slices. + + void pack_lhs(Index m, Index k) { + bool use_thread_local = false; + + if (parallelize_by_sharding_dim_only_ && !shard_by_col_ && + can_use_thread_local_packed_[m].load(std::memory_order_relaxed)) { + if (state_kernel_[k % P][m][0].load(std::memory_order_relaxed) == 1) { + use_thread_local = true; + } else { + // If we can't guarantee that all kernels in `k` slice will be + // executed sequentially in current thread, it's no longer safe to use + // thread local memory in followig slices along the k dimensions. + eigen_assert(k > 0); + can_use_thread_local_packed_[m].store(false, + std::memory_order_relaxed); + } + } + + const Index mend = m * gm_ + gm(m); + for (Index m1 = m * gm_; m1 < mend; m1++) + TensorContractionKernel::packLhs(packed_lhs(m, k, m1, use_thread_local), + lhs_.getSubMapper(m1 * bm_, k * bk_), + bk(k), bm(m1)); + + if (!parallel_pack_ && shard_by_col_) { + assert(!use_thread_local); + signal_packing(k); + } else { + signal_switch(k + 1); + for (Index n = nn_ - 1; n >= 0; n--) { + bool sync = parallelize_by_sharding_dim_only_ || n == 0; + signal_kernel(m, n, k, sync, use_thread_local); + } + } + } + + void pack_rhs(Index n, Index k) { + bool use_thread_local = false; + + if (parallelize_by_sharding_dim_only_ && shard_by_col_ && + can_use_thread_local_packed_[n].load(std::memory_order_relaxed)) { + if (state_kernel_[k % P][0][n].load(std::memory_order_relaxed) == 1) { + use_thread_local = true; + } else { + // If we can't guarantee that all kernels in `k` slice will be + // executed sequentially in current thread, it's no longer safe to use + // thread local memory in followig slices along the k dimensions. + eigen_assert(k > 0); + can_use_thread_local_packed_[n].store(false, + std::memory_order_relaxed); + } + } + + const Index nend = n * gn_ + gn(n); + for (Index n1 = n * gn_; n1 < nend; n1++) { + if (k == 0) { + // Zero the output memory in parallel. + // On 10000x2x10000 mm zeroing can easily take half of time. + // Zero (bn x m) row. Safe to do here because all kernels that will + // write to this memory depend on completion of this task. + // Note: don't call device_.memset() here. device_.memset() blocks on + // thread pool worker thread, which can lead to underutilization and + // deadlocks. + memset(buffer_ + n1 * bn_ * m_, 0, bn(n1) * m_ * sizeof(Scalar)); + } + TensorContractionKernel::packRhs(packed_rhs(n, k, n1, use_thread_local), + rhs_.getSubMapper(k * bk_, n1 * bn_), + bk(k), bn(n1)); + } + + if (parallel_pack_ || shard_by_col_) { + signal_switch(k + 1); + for (Index m = nm_ - 1; m >= 0; m--) { + bool sync = parallelize_by_sharding_dim_only_ || m == 0; + signal_kernel(m, n, k, sync, use_thread_local); + } + } else { + assert(!use_thread_local); + signal_packing(k); + } + } + + void kernel(Index m, Index n, Index k, bool use_thread_local) { + // Note: order of iteration matters here. Iteration over m is innermost + // because we want to reuse the same packed rhs in consecutive tasks + // (rhs fits into L2$ while lhs only into L3$). + const Index nend = n * gn_ + gn(n); + const Index mend = m * gm_ + gm(m); + if (shard_by_col_) { + for (Index n1 = n * gn_; n1 < nend; n1++) { + for (Index m1 = m * gm_; m1 < mend; m1++) { + const auto output_mapper = output_.getSubMapper(m1 * bm_, n1 * bn_); + TensorContractionKernel::invoke( + output_mapper, + packed_lhs(m, k, m1, !shard_by_col_ && use_thread_local), + packed_rhs(n, k, n1, shard_by_col_ && use_thread_local), bm(m1), + bk(k), bn(n1), Scalar(1)); + + // We are done with the last task for the [m1, n1] block. + if (k + 1 == nk_) { + output_kernel_(output_mapper, tensor_contraction_params_, + m1 * bm_, n1 * bn_, bm(m1), bn(n1)); + } + } + } + } else { + for (Index m1 = m * gm_; m1 < mend; m1++) + for (Index n1 = n * gn_; n1 < nend; n1++) { + const auto output_mapper = output_.getSubMapper(m1 * bm_, n1 * bn_); + TensorContractionKernel::invoke( + output_mapper, + packed_lhs(m, k, m1, !shard_by_col_ && use_thread_local), + packed_rhs(n, k, n1, shard_by_col_ && use_thread_local), bm(m1), + bk(k), bn(n1), Scalar(1)); + + // We are done with the last task for the [m1, n1] block. + if (k + 1 == nk_) { + output_kernel_(output_mapper, tensor_contraction_params_, + m1 * bm_, n1 * bn_, bm(m1), bn(n1)); + } + } + } + signal_kernel(m, n, k + 1, /*sync=*/false, /*use_thread_local=*/false); + signal_switch(k + 2); + } + + void signal_packing(Index k) { + eigen_assert(!parallel_pack_); + Index s = state_packing_ready_[k % P].fetch_sub(1); + eigen_assert(s > 0); + if (s != 1) return; + state_packing_ready_[k % P] = shard_by_col_ ? nm_ : nn_; + enqueue_packing(k, shard_by_col_); + } + + void signal_kernel(Index m, Index n, Index k, bool sync, + bool use_thread_local) { + std::atomic<uint8_t>* state = &state_kernel_[k % P][m][n]; + Index s = state->load(); + eigen_assert(s > 0); + if (s != 1 && state->fetch_sub(1) != 1) { + eigen_assert(!use_thread_local); + return; + } + state->store(parallel_pack_ ? 3 : 2, std::memory_order_relaxed); + if (sync) { + kernel(m, n, k, use_thread_local); + } else { + eigen_assert(!use_thread_local); + device_.enqueueNoNotification( + [=]() { kernel(m, n, k, use_thread_local); }); + } + } + + void signal_switch(Index k, Index v = 1) { + Index s = state_switch_[k % P].fetch_sub(v); + eigen_assert(s >= v); + if (s != v) return; + + // Ready to switch to the next k slice. + // Reset counter for the next iteration. + state_switch_[k % P] = + (parallel_pack_ ? nm_ + nn_ : (shard_by_col_ ? nn_ : nm_)) + + nm_ * nn_; + if (k < nk_) { + // Issue lhs/rhs packing. Their completion will in turn kick off + // kernels. + if (parallel_pack_) { + enqueue_packing(k, !shard_by_col_); + enqueue_packing(k, shard_by_col_); + } else if (shard_by_col_) { + enqueue_packing(k, false); + } else { + enqueue_packing(k, true); + } + + // Termination handling. + // Because kernel completion signals k + 2 switch, we need to finish nk + // + 2 slices without issuing any tasks on nk + 1 slice. So here we + // pretend that all nk + 1 packing tasks just finish instantly; so that + // nk + 2 switch only waits for completion of nk kernels. + } else if (k == nk_) { + signal_switch(k + 1, + parallel_pack_ ? nm_ + nn_ : (shard_by_col_ ? nn_ : nm_)); + } else { + done_.Notify(); + } + } + + // Enqueue all rhs/lhs packing for k-th slice. + void enqueue_packing(Index k, bool rhs) { + enqueue_packing_helper(0, rhs ? nn_ : nm_, k, rhs); + } + + void enqueue_packing_helper(Index start, Index end, Index k, bool rhs) { + if (end - start == 1) { + if (rhs) + pack_rhs(start, k); + else + pack_lhs(start, k); + } else { + while (end - start > 1) { + Index mid = (start + end) / 2; + device_.enqueueNoNotification( + [=]() { enqueue_packing_helper(mid, end, k, rhs); }); + end = mid; + } + + // Decide if we want to run first packing task (start == 0) in + // async mode if we parallelize only by sharding dim: + // (1) pack_lhs and pack_rhs call signal_switch before completing + // all calls to signal_kernel, which in sync mode might lead + // to the execution of the first kernel of the k+1 slice, before + // completing a call to the last kernel of the k slice. + // (2) all pack tasks for sharded dim must be executed in a thread + // pool. + bool pack_async = + (start == 0) && + (parallelize_by_sharding_dim_only_&& shard_by_col_ == rhs) && + (k > 0 || device_.currentThreadId() < 0); + + if (pack_async) { + device_.enqueueNoNotification( + [=]() { enqueue_packing_helper(start, end, k, rhs); }); + } else { + enqueue_packing_helper(start, end, k, rhs); + } + } + } + + // Block sizes with accounting for potentially incomplete last block. + Index bm(Index m) const { return m + 1 < nm0_ ? bm_ : m_ + bm_ - bm_ * nm0_; } + Index bn(Index n) const { return n + 1 < nn0_ ? bn_ : n_ + bn_ - bn_ * nn0_; } + Index bk(Index k) const { return k + 1 < nk_ ? bk_ : k_ + bk_ - bk_ * nk_; } + // Task grain sizes accounting for potentially incomplete last task. + Index gm(Index m) const { return m + 1 < nm_ ? gm_ : nm0_ + gm_ - gm_ * nm_; } + Index gn(Index n) const { return n + 1 < nn_ ? gn_ : nn0_ + gn_ - gn_ * nn_; } + + Context(const Context&) = delete; + void operator=(const Context&) = delete; + }; + + // Decide whether we want to shard m x n contraction by columns or by rows. + static bool shardByCol(Index m, Index n, Index num_threads) { + // Note: we are comparing both n and m against Traits::nr, it is not + // a mistake. We are trying to figure out how both n and m will fit into + // the main sharding dimension. + + // Sharding by column is the default + // ... unless there is enough data for vectorization over rows + if (m / num_threads >= Traits::nr && + // and not enough data for vectorization over columns + (n / num_threads < Traits::nr || + // ... or barely enough data for vectorization over columns, + // but it is not evenly dividable across threads + (n / num_threads < 4 * Traits::nr && + (n % (num_threads * Traits::nr)) != 0 && + // ... and it is evenly dividable across threads for rows + ((m % (num_threads * Traits::nr)) == 0 || + // .. or it is not evenly dividable for both dimensions but + // there is much more data over rows so that corner effects are + // mitigated. + (m / n >= 6))))) + return false; + // Wait, or if matrices are just substantially prolonged over the other + // dimension. + if (n / num_threads < 16 * Traits::nr && m > n * 32) return false; + return true; } + + Index coarsenM(Index m, Index n, Index bm, Index bn, Index bk, Index gn, + int num_threads, bool shard_by_col) const { + Index gm = 1; + Index gm1 = 1; + Index nm0 = divup(m, bm); + Index nm1 = nm0; + for (;;) { + // Find the next candidate for m grain size. It needs to result in + // different number of blocks. E.g. if we have 10 kernels, we want to try + // 5 and 10, but not 6, 7, 8 and 9. + while (gm1 <= nm0 && nm1 == divup(nm0, gm1)) gm1++; + if (gm1 > nm0) break; + // Check the candidate. + int res = checkGrain(m, n, bm, bn, bk, gm1, gn, gm, gn, num_threads, + shard_by_col); + if (res < 0) break; + nm1 = divup(nm0, gm1); + if (res == 0) continue; + // Commit new grain size. + gm = gm1; + } + return gm; + } + + Index coarsenN(Index m, Index n, Index bm, Index bn, Index bk, Index gm, + int num_threads, bool shard_by_col) const { + Index gn = 1; + Index gn1 = 1; + Index nn0 = divup(n, bn); + Index nn1 = nn0; + for (;;) { + while (gn1 <= nn0 && nn1 == divup(nn0, gn1)) gn1++; + if (gn1 > nn0) break; + int res = checkGrain(m, n, bm, bn, bk, gm, gn1, gm, gn, num_threads, + shard_by_col); + if (res < 0) break; + nn1 = divup(nn0, gn1); + if (res == 0) continue; + gn = gn1; + } + return gn; + } + + // checkGrain checks whether grain (gm, gn) is suitable and is better than + // (oldgm, oldgn). + int checkGrain(Index m, Index n, Index bm, Index bn, Index bk, Index gm, + Index gn, Index oldgm, Index oldgn, int num_threads, + bool shard_by_col) const { + const TensorOpCost cost = + contractionCost(bm * gm, bn * gn, bm, bn, bk, shard_by_col, true); + double taskSize = TensorCostModel<ThreadPoolDevice>::taskSize( + static_cast<double>(bm) * gm * bn * gn, cost); + // If the task is too small, then we agree on it regardless of anything + // else. Otherwise synchronization overheads will dominate. + if (taskSize < 1) return 1; + // If it is too large, then we reject it and all larger tasks. + if (taskSize > 2) return -1; + // Now we are in presumably good task size range. + // The main deciding factor here is parallelism. Consider that we have 12 + // kernels and 4 threads. Grains of 2, 3 and 4 all yield good task sizes. + // But 2/4 yield 6/3 tasks, which gives us parallelism of 0.75 (at most 3/4 + // of cores will be busy). While grain size 3 gives us 4 tasks, which gives + // us parallelism of 1 (we can load all cores). + Index nm0 = divup(m, bm); + Index nn0 = divup(n, bn); + Index new_tasks = divup(nm0, gm) * divup(nn0, gn); + double new_parallelism = static_cast<double>(new_tasks) / + (divup<int>(new_tasks, num_threads) * num_threads); + Index old_tasks = divup(nm0, oldgm) * divup(nn0, oldgn); + double old_parallelism = static_cast<double>(old_tasks) / + (divup<int>(old_tasks, num_threads) * num_threads); + if (new_parallelism > old_parallelism || new_parallelism == 1) return 1; + return 0; + } + + TensorOpCost contractionCost(Index m, Index n, Index bm, Index bn, Index bk, + bool shard_by_col, bool prepacked) const { + const int packed_size = std::min<int>(PacketType<LhsScalar, Device>::size, + PacketType<RhsScalar, Device>::size); + const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size; + const double kd = static_cast<double>(bk); + double compute_bandwidth = computeBandwidth(false, bm, bn, bk); + // Computations. + TensorOpCost cost = TensorOpCost(0, 0, kd * compute_bandwidth, true, packed_size); + // Output stores. + cost += TensorOpCost(0, sizeof(CoeffReturnType), 0, true, output_packet_size); + if (prepacked) { + // Packing and kernels are executed in different tasks. When we calculate + // task grain size we look only at kernel cost assuming that kernel + // is more expensive than packing. + return cost; + } + // Lhs/rhs loads + computations. + TensorOpCost lhsCost = this->m_leftImpl.costPerCoeff(true) * (kd / n); + TensorOpCost rhsCost = this->m_rightImpl.costPerCoeff(true) * (kd / m); + // Lhs packing memory cost does not contribute considerably to overall + // execution time because lhs is prefetched early and accessed sequentially. + if (shard_by_col) + lhsCost.dropMemoryCost(); + else + rhsCost.dropMemoryCost(); + return cost + lhsCost + rhsCost; + } + + template <int Alignment> + EIGEN_STRONG_INLINE void addToBuffer(size_t n, const Scalar* src_buf, + Scalar* tgt_buf) const { + const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size; + size_t i = 0; + const size_t num_packets = n / output_packet_size; + for (; i < output_packet_size * num_packets; i += output_packet_size) { + const PacketReturnType src_val = + internal::pload<PacketReturnType>(src_buf + i); + const PacketReturnType tgt_val = + internal::ploadt<PacketReturnType, Alignment>(tgt_buf + i); + const PacketReturnType sum = internal::padd(src_val, tgt_val); + internal::pstoret<Scalar, PacketReturnType, Alignment>(tgt_buf + i, sum); + } + for (; i < n; ++i) { + tgt_buf[i] += src_buf[i]; + } + } + + template <int Alignment> + EIGEN_STRONG_INLINE void addAllToBuffer(size_t n, const Scalar* src_buf0, + const Scalar* src_buf1, + const Scalar* src_buf2, + Scalar* dst_buf) const { + using ::Eigen::internal::padd; + using ::Eigen::internal::pload; + using ::Eigen::internal::ploadt; + using ::Eigen::internal::pstoret; + + const int output_packet_size = + internal::unpacket_traits<PacketReturnType>::size; + + size_t i = 0; + const size_t num_packets = n / output_packet_size; + for (; i < output_packet_size * num_packets; i += output_packet_size) { + const auto src_val0 = pload<PacketReturnType>(src_buf0 + i); + const auto src_val1 = pload<PacketReturnType>(src_buf1 + i); + const auto src_val2 = pload<PacketReturnType>(src_buf2 + i); + + const auto dst_val = ploadt<PacketReturnType, Alignment>(dst_buf + i); + const auto sum = padd(padd(dst_val, src_val0), padd(src_val1, src_val2)); + + pstoret<Scalar, PacketReturnType, Alignment>(dst_buf + i, sum); + } + for (; i < n; ++i) { + dst_buf[i] += src_buf0[i] + src_buf1[i] + src_buf2[i]; + } + } + + // Decide whether we want to shard m x k x n contraction over the inner + // (contraction) dimension (k). + static bool shardByInnerDim(Index m, Index n, Index k, int num_threads, + int num_threads_by_k) { + std::ptrdiff_t bufsize = m * n * sizeof(Scalar); + bool shard_by_k = false; + if (n == 1 || // If mat*vec or... + num_threads_by_k < 2 || // running single threaded or... + num_threads_by_k < + num_threads || // sharding by k gives less parallelism or... + bufsize > l3CacheSize() / num_threads_by_k || // need more buffer space + // than L3 cache or... + k / num_threads_by_k < 2 * Traits::nr) { // k per thread is tiny. + shard_by_k = false; + } else if (numext::maxi(m, n) / num_threads < + Traits::nr || // both other dimensions are tiny or... + // k per thread is not small and... + (k / num_threads_by_k > 8 * Traits::nr && + // one of the outer dimensions is tiny or sharding by k offers + // more parallelism. + (numext::mini(m, n) < 2 * Traits::nr || + num_threads_by_k > num_threads))) { + shard_by_k = true; + } + return shard_by_k; + } + + template <int Alignment> + void evalShardedByInnerDim(int num_threads, Scalar* result) const { + const Index m = this->m_i_size; + const Index n = this->m_j_size; + const Index k = this->m_k_size; + + // We will compute partial results into the buffers of this size. + const Index buffer_size_bytes = m * n * sizeof(Scalar); + + // The underlying GEMM kernel assumes that k is a multiple of + // the packet size and subtle breakage occurs if this is violated. + const Index packet_size = internal::packet_traits<RhsScalar>::size; + + const auto round_up = [=](Index index) -> Index { + const Index kmultiple = packet_size <= 8 ? 8 : packet_size; + return divup<Index>(index, kmultiple) * kmultiple; + }; + + // Cost model doesn't capture well the cost associated with constructing + // tensor contraction mappers and computing loop bounds in gemm_pack_lhs and + // gemm_pack_rhs, so we specify minimum desired block size. + const Index target_block_size = round_up(divup<Index>(k, num_threads)); + const Index desired_min_block_size = 12 * packet_size; + + const Index block_size = numext::mini<Index>( + k, numext::maxi<Index>(desired_min_block_size, target_block_size)); + const Index num_blocks = divup<Index>(k, block_size); + + // Compute block size with accounting for potentially incomplete last block. + const auto actual_block_size = [=](Index block_idx) -> Index { + return block_idx + 1 < num_blocks + ? block_size + : k + block_size - block_size * num_blocks; + }; + + // We compute partial gemm results in parallel, and to get the final result + // we need to add them all together. For the large number of threads (>= 48) + // this adds a very expensive sequential step at the end. + // + // We split the [0, num_blocks) into small ranges, and when a task for the + // block finishes its partial gemm computation, it checks if it was the last + // gemm in the range, and if so, it will add all blocks of the range. + // + // After all tasks finihes, we need to add only these pre-aggregated blocks. + + // Compute range size with accounting for potentially incomplete last range. + const auto actual_range_size = [=](Index num_ranges, Index range_size, + Index range_idx) -> Index { + eigen_assert(range_idx < num_ranges); + return range_idx + 1 < num_ranges + ? range_size + : num_blocks + range_size - range_size * num_ranges; + }; + + // For now we use just a single level of ranges to compute pre-aggregated + // partial sums, but in general we can use more layers to compute tree + // aggregation in parallel and reduce the size of the sequential step. + // + // TODO(ezhulenev): Add multilevel tree aggregation? Probably will make + // sense only if number of threads >= ~128? + static const Index l0_size = 4; + const Index l0_ranges = divup<Index>(num_blocks, l0_size); + + // Keep count of pending gemm tasks for each l0 range. + MaxSizeVector<std::atomic<int>> l0_state(l0_ranges); + for (int i = 0; i < l0_ranges; ++i) { + const Index num_pending_tasks = actual_range_size(l0_ranges, l0_size, i); + l0_state.emplace_back(internal::convert_index<int>(num_pending_tasks)); + } + + MaxSizeVector<Scalar*> block_buffers(num_blocks); + + auto process_block = [&, this](Index block_idx, Index begin, Index end) { + Scalar* buf = block_buffers[block_idx]; + ::memset(buf, 0, buffer_size_bytes); + + TENSOR_CONTRACTION_DISPATCH( + this->template evalGemmPartialWithoutOutputKernel, Alignment, + (buf, begin, end, + /*num_threads=*/internal::convert_index<int>(num_blocks))); + + // Check if it was the last task in l0 range. + const Index l0_index = block_idx / l0_size; + const int v = l0_state[l0_index].fetch_sub(1); + eigen_assert(v >= 1); + + // If we processed the last block of the range, we can aggregate all + // partial results into the first block of the range. + if (v == 1) { + const Index rng_size = actual_range_size(l0_ranges, l0_size, l0_index); + const Index dst_block_idx = l0_index * l0_size; + + if (rng_size == l0_size) { + addAllToBuffer<Alignment>( + m * n, + /*src_buf0=*/block_buffers[dst_block_idx + 1], + /*src_buf1=*/block_buffers[dst_block_idx + 2], + /*src_buf2=*/block_buffers[dst_block_idx + 3], + /*dst_buf= */ block_buffers[dst_block_idx]); + } else { + // Aggregate blocks of potentially incomplete last range. + for (int i = 1; i < rng_size; ++i) { + addToBuffer<Alignment>(m * n, + /*src_buf=*/block_buffers[dst_block_idx + i], + /*dst_buf=*/block_buffers[dst_block_idx]); + } + } + } + }; + + Barrier barrier(internal::convert_index<int>(num_blocks)); + for (Index block_idx = 0; block_idx < num_blocks; ++block_idx) { + Scalar* buf = block_idx == 0 + ? result + : static_cast<Scalar*>( + this->m_device.allocate(buffer_size_bytes)); + block_buffers.push_back(buf); + + Index block_start = block_idx * block_size; + Index block_end = block_start + actual_block_size(block_idx); + + this->m_device.enqueueNoNotification([=, &barrier, &process_block]() { + process_block(block_idx, block_start, block_end); + barrier.Notify(); + }); + } + barrier.Wait(); + + // Aggregate partial sums from l0 ranges. + Index l0_index = 1; + for (; l0_index + 2 < l0_ranges; l0_index += 3) { + addAllToBuffer<Alignment>( + m * n, + /*src_buf0=*/block_buffers[(l0_index + 0) * l0_size], + /*src_buf1=*/block_buffers[(l0_index + 1) * l0_size], + /*src_buf2=*/block_buffers[(l0_index + 2) * l0_size], + /*dst_buf= */block_buffers[0]); + } + for (; l0_index < l0_ranges; ++l0_index) { + addToBuffer<Alignment>(m * n, block_buffers[l0_index * l0_size], + block_buffers[0]); + } + + // Don't forget to deallocate ALL temporary buffers. + for (Index i = 1; i < num_blocks; ++i) { + this->m_device.deallocate(block_buffers[i]); + } + + // Finally call output kernel with finalized output buffer. + typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper; + this->m_output_kernel(OutputMapper(result, m), + this->m_tensor_contraction_params, + static_cast<Eigen::Index>(0), + static_cast<Eigen::Index>(0), + m, n); + } + + TensorOpCost contractionCostPerInnerDim(Index m, Index n, Index k) const { + // Compute cost. + const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size; + TensorOpCost cost(0, 0, (computeBandwidth(true, m, n, k) * m) * n); + // Output stores. + cost += TensorOpCost(0, sizeof(CoeffReturnType), 0, true, output_packet_size); + TensorOpCost lhsCost = this->m_leftImpl.costPerCoeff(true) * m; + TensorOpCost rhsCost = this->m_rightImpl.costPerCoeff(true) * n; + // Since the inner gemm kernel is always sharded by column, the lhs + // load cost is negligible. + lhsCost.dropMemoryCost(); + return cost + lhsCost + rhsCost; + } + + int numThreadsInnerDim(Index m, Index n, Index k) const { + const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size; + TensorOpCost cost = contractionCostPerInnerDim(m, n, k); + double total_parallel_cost = + TensorCostModel<ThreadPoolDevice>::totalCost(k, cost); + // Cost of reduction step accumulating the m*n per-thread buffers into the + // result. + double reduction_cost = TensorCostModel<ThreadPoolDevice>::totalCost( + m * n, TensorOpCost(2, 1, 1, true, output_packet_size)); + int num_threads = 1; + double min_cost = total_parallel_cost; + double kPerThreadOverHead = 4000; + double kFixedOverHead = 100000; + for (int nt = 2; nt <= this->m_device.numThreads(); nt++) { + double sequential_cost = + kFixedOverHead + nt * (reduction_cost + kPerThreadOverHead); + double parallel_cost = total_parallel_cost / nt + sequential_cost; + if (parallel_cost < min_cost) { + num_threads = nt; + min_cost = parallel_cost; + } + } + return num_threads; + } + + + double computeBandwidth(bool shard_by_col, Index bm, Index bn, + Index bk) const { + // Peak VFMA bandwidth is 0.5. However if we have not enough data for + // vectorization bandwidth drops. The 4.0 and 2.0 bandwidth is determined + // experimentally. + double computeBandwidth = + bk == 1 ? 4.0 + : (shard_by_col ? bn : bm) < Traits::nr || + (shard_by_col ? bm : bn) < Traits::mr + ? 2.0 + : 0.5; +#ifndef EIGEN_VECTORIZE_FMA + // Bandwidth of all of VFMA/MULPS/ADDPS is 0.5 on latest Intel processors. + // However for MULPS/ADDPS we have dependent sequence of 2 such + // instructions, + // so overall bandwidth is 1.0. + if (computeBandwidth == 0.5) computeBandwidth = 1.0; +#endif + return computeBandwidth; + } + +#if defined(EIGEN_VECTORIZE_AVX) && defined(EIGEN_USE_LIBXSMM) + // TODO(ezhulenev): Add support for output kernels and LIBXSMM. + static_assert(std::is_same<OutputKernelType, const NoOpOutputKernel>::value, + "XSMM does not support contraction output kernels."); + + template<int Alignment> + class ContextXsmm { + public: + ContextXsmm(const Self* self, Scalar* buffer, Index m, Index n, Index k, + const internal::TensorXsmmContractionBlocking<LhsScalar, + RhsScalar, Index>& blocking): + device(self->m_device), + m(m), k(k), n(n), + stride_a(blocking.transposeA() ? k : m), + stride_b(blocking.transposeB() ? n : k), + stride_c(m), + bm(blocking.mc()), bk(blocking.kc()), bn(blocking.nc()), + blocks_m(blocking.blocks_m()), blocks_k(blocking.blocks_k()), + blocks_n(blocking.blocks_n()), + copyA(blocking.copyA()), copyB(blocking.copyB()), + transposeA(blocking.transposeA()), transposeB(blocking.transposeB()), + num_threads(blocking.num_threads()), + buffer(buffer), + leftData(self->m_leftImpl.data()), rightData(self->m_rightImpl.data()), + workers_done(blocking.num_threads()), + + packingA_jobs(0), packingB_jobs(0), compute_jobs(0), + packingA_done(blocking.blocks_m()), packingB_done(blocking.blocks_n()) {} + + void worker() { + // Pack + + if (copyA) { + while (true) { + uint32_t mk = packingA_jobs++; + Index mi = mk / blocks_k; + Index ki = mk % blocks_k; + if (mi >= blocks_m) break; + + LhsScalar * blockA = blocksA + (bk*bm) * (mi*blocks_k+ki); + if (transposeA) { + const LhsScalar * current_a = leftData + (bm*mi)*stride_a + (bk*ki); + libxsmm_otrans(blockA, current_a, sizeof(LhsScalar), actual_bk(ki), + actual_bm(mi), stride_a, bm); + } else { + const LhsScalar * current_a = leftData + (bk*ki)*stride_a + (bm*mi); + internal::pack_simple<LhsScalar, Index>(blockA, current_a, + actual_bk(ki), actual_bm(mi), bm, stride_a); + } + packingA_done.at(mi)++; + } + } + + if (copyB) { + while (true) { + uint32_t nk = packingB_jobs++; + Index ni = nk / blocks_k; + Index ki = nk % blocks_k; + if (ni >= blocks_n) break; + + RhsScalar * blockB = blocksB + (bk*bn) * (ni*blocks_k+ki); + if (transposeB) { + const RhsScalar * current_b = rightData + (ki*bk)*stride_b + + (ni*bn); + libxsmm_otrans(blockB, current_b, sizeof(RhsScalar), actual_bn(ni), + actual_bk(ki), stride_b, bk); + } else { + const RhsScalar * current_b = rightData + (ni*bn)*stride_b + + (ki*bk); + internal::pack_simple<RhsScalar, Index>(blockB, current_b, + actual_bn(ni), actual_bk(ki), bk, stride_b); + } + packingB_done.at(ni)++; + } + } + + // Compute + + while (true) { + uint32_t mn = compute_jobs++; + Index mi = mn / blocks_n; + Index ni = mn % blocks_n; + if (mi >= blocks_m) break; + + // Wait for mi, ni packings to be done. This is more fine-grained than + // waiting for all workers to finish packing. + while ((copyA && (packingA_done.at(mi) < blocks_k)) || + (copyB && (packingB_done.at(ni) < blocks_k))) + {} + + for (Index ki=0; ki < blocks_k; ++ki) { + const LhsScalar * current_a = copyA ? + blocksA + (bk*bm) * (mi*blocks_k+ki) : + leftData + (bk*ki)*stride_a + (bm*mi); + const RhsScalar * current_b = copyB ? + blocksB + (bk*bn) * (ni*blocks_k+ki) : + rightData + (ni*bn)*stride_b + (bk*ki); + + Index current_stride_a = copyA ? bm : stride_a; + Index current_stride_b = copyB ? bk : stride_b; + + // Memory may not be zeroed, overwrite instead of adding in first + // iteration. + float beta = ki == 0 ? 0 : 1; + + Scalar * current_c = buffer + (mi*bm) + (ni*bn)*stride_c; + internal::libxsmm_wrapper<LhsScalar, RhsScalar, Scalar>( + 0, actual_bm(mi), actual_bn(ni), actual_bk(ki), + current_stride_a, current_stride_b, stride_c, 1, beta, 0) + (current_a, current_b, current_c); + } + } + + workers_done.Notify(); + } + + void run() { + // Parallelization strategy. + // + // First pack A into blocks (sharding by m, k) and B (sharding by n,k), + // then shard by m, n. + // + // Do not use advanced ThreadPool queuing, just run a single long-standing + // function in each thread. + if (copyA) { + blocksA = static_cast<LhsScalar*>(device.allocate( + (blocks_m*bm)*(blocks_k*bk)*sizeof(LhsScalar))); + } + if (copyB) { + blocksB = static_cast<RhsScalar*>(device.allocate( + (blocks_n*bn)*(blocks_k*bk)*sizeof(RhsScalar))); + } + + for (Index i = 0; i < num_threads; ++i) { + device.enqueueNoNotification([=]() { worker(); }); + } + + workers_done.Wait(); + + if (copyA) { + device.deallocate(blocksA); + } + if (copyB) { + device.deallocate(blocksB); + } + } + + private: + // real block size for block index in [0, ..., blocks - 1]. + Index actual_bm(Index mi) const { + return mi != blocks_m - 1 ? bm : m + bm - bm * blocks_m; + } + Index actual_bk(Index ki) const { + return ki != blocks_k - 1 ? bk : k + bk - bk * blocks_k; + } + Index actual_bn(Index ni) const { + return ni != blocks_n - 1 ? bn : n + bn - bn * blocks_n; + } + + const Device& device; + Index m, k, n; + Index stride_a, stride_b, stride_c; + Index bm, bk, bn; // Block sizes. + Index blocks_m, blocks_k, blocks_n; // Number of blocks in each dimension. + bool copyA, copyB, transposeA, transposeB; + Index num_threads; + Scalar *buffer; + const LhsScalar *leftData; + const RhsScalar *rightData; + + LhsScalar *blocksA; + RhsScalar *blocksB; + // barrier for joining all threads after all done. + Barrier workers_done; + // "queues" of (mi,ki), (ki,ni), (mi,ni) jobs packed [0,p)x[0,q) -> [0, p*q) + std::atomic<uint32_t> packingA_jobs; + std::atomic<uint32_t> packingB_jobs; + std::atomic<uint32_t> compute_jobs; + // already packed blocks for each mi-panel in A and ni-panel in B. + std::vector<std::atomic<uint8_t>> packingA_done; + std::vector<std::atomic<uint8_t>> packingB_done; + }; +#endif + }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h index a2f1f71..938fd0f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h
@@ -32,6 +32,7 @@ static const int NumDimensions = traits<XprType>::NumDimensions; static const int Layout = traits<XprType>::Layout; enum { Flags = 0 }; + typedef typename TypeConversion<Scalar, typename traits<XprType>::PointerType>::type PointerType; }; template<typename TargetType, typename XprType> @@ -164,18 +165,93 @@ }; template <bool SameType, typename Eval, typename Scalar> struct ConversionSubExprEval { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool run(Eval& impl, Scalar*) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(Eval& impl, Scalar*) { impl.evalSubExprsIfNeeded(NULL); return true; } }; template <typename Eval, typename Scalar> struct ConversionSubExprEval<true, Eval, Scalar> { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool run(Eval& impl, Scalar* data) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(Eval& impl, Scalar* data) { return impl.evalSubExprsIfNeeded(data); } }; +namespace internal { + +template <typename SrcType, typename TargetType, bool IsSameT> +struct CoeffConv { + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetType run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + internal::scalar_cast_op<SrcType, TargetType> converter; + return converter(impl.coeff(index)); + } +}; + +template <typename SrcType, typename TargetType> +struct CoeffConv<SrcType, TargetType, true> { + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetType run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + return impl.coeff(index); + } +}; + +template <typename SrcPacket, typename TargetPacket, int LoadMode, bool ActuallyVectorize, bool IsSameT> +struct PacketConv { + typedef typename internal::unpacket_traits<SrcPacket>::type SrcType; + typedef typename internal::unpacket_traits<TargetPacket>::type TargetType; + + static const int PacketSize = internal::unpacket_traits<TargetPacket>::size; + + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + internal::scalar_cast_op<SrcType, TargetType> converter; + EIGEN_ALIGN_MAX typename internal::remove_const<TargetType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + values[i] = converter(impl.coeff(index+i)); + } + TargetPacket rslt = internal::pload<TargetPacket>(values); + return rslt; + } +}; + +template <typename SrcPacket, typename TargetPacket, int LoadMode, bool IsSameT> +struct PacketConv<SrcPacket, TargetPacket, LoadMode, true, IsSameT> { + typedef typename internal::unpacket_traits<SrcPacket>::type SrcType; + typedef typename internal::unpacket_traits<TargetPacket>::type TargetType; + + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + const int SrcCoeffRatio = internal::type_casting_traits<SrcType, TargetType>::SrcCoeffRatio; + const int TgtCoeffRatio = internal::type_casting_traits<SrcType, TargetType>::TgtCoeffRatio; + PacketConverter<TensorEvaluator<ArgType, Device>, SrcPacket, TargetPacket, + SrcCoeffRatio, TgtCoeffRatio> converter(impl); + return converter.template packet<LoadMode>(index); + } +}; + +template <typename SrcPacket, typename TargetPacket, int LoadMode> +struct PacketConv<SrcPacket, TargetPacket, LoadMode, /*ActuallyVectorize=*/false, /*IsSameT=*/true> { + typedef typename internal::unpacket_traits<TargetPacket>::type TargetType; + static const int PacketSize = internal::unpacket_traits<TargetPacket>::size; + + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + EIGEN_ALIGN_MAX typename internal::remove_const<TargetType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) values[i] = impl.coeff(index+i); + return internal::pload<TargetPacket>(values); + } +}; + +template <typename SrcPacket, typename TargetPacket, int LoadMode> +struct PacketConv<SrcPacket, TargetPacket, LoadMode, /*ActuallyVectorize=*/true, /*IsSameT=*/true> { + template <typename ArgType, typename Device> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator<ArgType, Device>& impl, Index index) { + return impl.template packet<LoadMode>(index); + } +}; + +} // namespace internal // Eval as rvalue template<typename TargetType, typename ArgType, typename Device> @@ -189,11 +265,14 @@ typedef typename internal::remove_all<typename internal::traits<ArgType>::Scalar>::type SrcType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename PacketType<SrcType, Device>::type PacketSourceType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + static const bool IsSameType = internal::is_same<TargetType, SrcType>::value; enum { IsAligned = false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess && internal::type_casting_traits<SrcType, TargetType>::VectorizedCast, + PacketAccess = true, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, RawAccess = false }; @@ -207,7 +286,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { - return ConversionSubExprEval<internal::is_same<TargetType, SrcType>::value, TensorEvaluator<ArgType, Device>, Scalar>::run(m_impl, data); + return ConversionSubExprEval<IsSameType, TensorEvaluator<ArgType, Device>, Scalar>::run(m_impl, data); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() @@ -217,18 +296,23 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - internal::scalar_cast_op<SrcType, TargetType> converter; - return converter(m_impl.coeff(index)); + return internal::CoeffConv<SrcType, TargetType, IsSameType>::run(m_impl,index); } template<int LoadMode> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const - { - const int SrcCoeffRatio = internal::type_casting_traits<SrcType, TargetType>::SrcCoeffRatio; - const int TgtCoeffRatio = internal::type_casting_traits<SrcType, TargetType>::TgtCoeffRatio; - PacketConverter<TensorEvaluator<ArgType, Device>, PacketSourceType, PacketReturnType, - SrcCoeffRatio, TgtCoeffRatio> converter(m_impl); - return converter.template packet<LoadMode>(index); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType + packet(Index index) const { + // If we are not going to do the cast, we just need to check that base + // TensorEvaluator has packet access. Otherwise we also need to make sure, + // that we have an implementation of vectorized cast. + const bool Vectorizable = + IsSameType + ? TensorEvaluator<ArgType, Device>::PacketAccess + : TensorEvaluator<ArgType, Device>::PacketAccess & + internal::type_casting_traits<SrcType, TargetType>::VectorizedCast; + + return internal::PacketConv<PacketSourceType, PacketReturnType, LoadMode, + Vectorizable, IsSameType>::run(m_impl, index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost @@ -246,10 +330,13 @@ } } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } - protected: - TensorEvaluator<ArgType, Device> m_impl; + /// required by sycl in order to extract the sycl accessor + const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + + protected: + TensorEvaluator<ArgType, Device> m_impl; }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h index ff3c566..2d0e659 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h
@@ -54,8 +54,8 @@ } } - array<Index, NumDims> cudaInputDimensions; - array<Index, NumDims> cudaOutputDimensions; + array<Index, NumDims> gpuInputDimensions; + array<Index, NumDims> gpuOutputDimensions; array<Index, NumDims> tmp = dimensions; array<Index, NumDims> ordering; const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) @@ -65,8 +65,8 @@ const Index index = i + offset; ordering[index] = indices[i]; tmp[indices[i]] = -1; - cudaInputDimensions[index] = input_dims[indices[i]]; - cudaOutputDimensions[index] = dimensions[indices[i]]; + gpuInputDimensions[index] = input_dims[indices[i]]; + gpuOutputDimensions[index] = dimensions[indices[i]]; } int written = static_cast<int>(Layout) == static_cast<int>(ColMajor) @@ -75,8 +75,8 @@ for (int i = 0; i < NumDims; ++i) { if (tmp[i] >= 0) { ordering[written] = i; - cudaInputDimensions[written] = input_dims[i]; - cudaOutputDimensions[written] = dimensions[i]; + gpuInputDimensions[written] = input_dims[i]; + gpuOutputDimensions[written] = dimensions[i]; ++written; } } @@ -89,37 +89,37 @@ if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { for (int i = 0; i < NumDims; ++i) { if (i > NumKernelDims) { - m_cudaInputStrides[i] = - m_cudaInputStrides[i - 1] * cudaInputDimensions[i - 1]; - m_cudaOutputStrides[i] = - m_cudaOutputStrides[i - 1] * cudaOutputDimensions[i - 1]; + m_gpuInputStrides[i] = + m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1]; + m_gpuOutputStrides[i] = + m_gpuOutputStrides[i - 1] * gpuOutputDimensions[i - 1]; } else { - m_cudaInputStrides[i] = 1; - m_cudaOutputStrides[i] = 1; + m_gpuInputStrides[i] = 1; + m_gpuOutputStrides[i] = 1; } } } else { for (int i = NumDims - 1; i >= 0; --i) { - if (i + 1 < offset) { - m_cudaInputStrides[i] = - m_cudaInputStrides[i + 1] * cudaInputDimensions[i + 1]; - m_cudaOutputStrides[i] = - m_cudaOutputStrides[i + 1] * cudaOutputDimensions[i + 1]; + if (static_cast<size_t>(i + 1) < offset) { + m_gpuInputStrides[i] = + m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1]; + m_gpuOutputStrides[i] = + m_gpuOutputStrides[i + 1] * gpuOutputDimensions[i + 1]; } else { - m_cudaInputStrides[i] = 1; - m_cudaOutputStrides[i] = 1; + m_gpuInputStrides[i] = 1; + m_gpuOutputStrides[i] = 1; } } } } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaInputPlaneToTensorInputOffset(Index p) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const { Index inputIndex = 0; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { - const Index idx = p / m_cudaInputStrides[d]; + const Index idx = p / m_gpuInputStrides[d]; inputIndex += idx * m_inputStrides[d]; - p -= idx * m_cudaInputStrides[d]; + p -= idx * m_gpuInputStrides[d]; } inputIndex += p * m_inputStrides[NumKernelDims]; } else { @@ -128,22 +128,22 @@ limit = NumDims - NumKernelDims - 1; } for (int d = 0; d < limit; ++d) { - const Index idx = p / m_cudaInputStrides[d]; + const Index idx = p / m_gpuInputStrides[d]; inputIndex += idx * m_inputStrides[d]; - p -= idx * m_cudaInputStrides[d]; + p -= idx * m_gpuInputStrides[d]; } inputIndex += p * m_inputStrides[limit]; } return inputIndex; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaOutputPlaneToTensorOutputOffset(Index p) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const { Index outputIndex = 0; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { - const Index idx = p / m_cudaOutputStrides[d]; + const Index idx = p / m_gpuOutputStrides[d]; outputIndex += idx * m_outputStrides[d]; - p -= idx * m_cudaOutputStrides[d]; + p -= idx * m_gpuOutputStrides[d]; } outputIndex += p * m_outputStrides[NumKernelDims]; } else { @@ -152,44 +152,44 @@ limit = NumDims - NumKernelDims - 1; } for (int d = 0; d < limit; ++d) { - const Index idx = p / m_cudaOutputStrides[d]; + const Index idx = p / m_gpuOutputStrides[d]; outputIndex += idx * m_outputStrides[d]; - p -= idx * m_cudaOutputStrides[d]; + p -= idx * m_gpuOutputStrides[d]; } outputIndex += p * m_outputStrides[limit]; } return outputIndex; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaInputKernelToTensorInputOffset(Index i) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_inputStrides[offset]; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaOutputKernelToTensorOutputOffset(Index i) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_outputStrides[offset]; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaInputKernelToTensorInputOffset(Index i, Index j) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1]; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaOutputKernelToTensorOutputOffset(Index i, Index j) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1]; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaInputKernelToTensorInputOffset(Index i, Index j, Index k) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j, Index k) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; @@ -197,7 +197,7 @@ k * m_inputStrides[offset + 2]; } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapCudaOutputKernelToTensorOutputOffset(Index i, Index j, Index k) const { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j, Index k) const { const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims; @@ -209,8 +209,8 @@ static const int NumDims = internal::array_size<InputDims>::value; array<Index, NumDims> m_inputStrides; array<Index, NumDims> m_outputStrides; - array<Index, NumDims> m_cudaInputStrides; - array<Index, NumDims> m_cudaOutputStrides; + array<Index, NumDims> m_gpuInputStrides; + array<Index, NumDims> m_gpuOutputStrides; }; @@ -231,9 +231,11 @@ typedef typename remove_reference<RhsNested>::type _RhsNested; static const int NumDimensions = traits<InputXprType>::NumDimensions; static const int Layout = traits<InputXprType>::Layout; + typedef typename conditional<Pointer_type_promotion<typename InputXprType::Scalar, Scalar>::val, + typename traits<InputXprType>::PointerType, typename traits<KernelXprType>::PointerType>::type PointerType; enum { - Flags = 0, + Flags = 0 }; }; @@ -254,7 +256,7 @@ template<typename Indices, typename InputXprType, typename KernelXprType> -class TensorConvolutionOp : public TensorBase<TensorConvolutionOp<Indices, InputXprType, KernelXprType> > +class TensorConvolutionOp : public TensorBase<TensorConvolutionOp<Indices, InputXprType, KernelXprType>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorConvolutionOp>::Scalar Scalar; @@ -300,11 +302,13 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = TensorEvaluator<InputArgType, Device>::IsAligned & TensorEvaluator<KernelArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<InputArgType, Device>::PacketAccess & TensorEvaluator<KernelArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<InputArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -465,7 +469,7 @@ PacketSize)); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } private: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const { @@ -524,8 +528,8 @@ Scalar* local = (Scalar*)m_device.allocate(kernel_sz); typedef TensorEvalToOp<const KernelArgType> EvalTo; EvalTo evalToTmp(local, m_kernelArg); - const bool PacketAccess = internal::IsVectorizable<Device, KernelArgType>::value; - internal::TensorExecutor<const EvalTo, Device, PacketAccess>::run(evalToTmp, m_device); + const bool Vectorize = internal::IsVectorizable<Device, KernelArgType>::value; + internal::TensorExecutor<const EvalTo, Device, Vectorize>::run(evalToTmp, m_device); m_kernel = local; m_local_kernel = true; @@ -551,7 +555,7 @@ // Use an optimized implementation of the evaluation code for GPUs whenever possible. -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) +#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC) template <int StaticKernelSize> struct GetKernelSize { @@ -574,7 +578,11 @@ indexMapper, const float* __restrict kernel, const int numPlanes, const int numX, const int maxX, const int kernelSize, float* buffer) { +#if defined(EIGEN_HIPCC) + HIP_DYNAMIC_SHARED(float, s) +#else extern __shared__ float s[]; +#endif const int first_x = blockIdx.x * maxX; const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1; @@ -586,18 +594,18 @@ for (int p = first_plane + threadIdx.y; p < numPlanes; p += plane_stride) { // Load inputs to shared memory - const int plane_input_offset = indexMapper.mapCudaInputPlaneToTensorInputOffset(p); + const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p); const int plane_kernel_offset = threadIdx.y * num_x_input; #pragma unroll for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) { - const int tensor_index = plane_input_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i+first_x); + const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i+first_x); s[i + plane_kernel_offset] = eval.coeff(tensor_index); } __syncthreads(); // Compute the convolution - const int plane_output_offset = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(p); + const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p); #pragma unroll for (int i = threadIdx.x; i < num_x_output; i += blockDim.x) { @@ -607,7 +615,7 @@ for (int k = 0; k < GetKernelSize<StaticKernelSize>()(kernelSize); ++k) { result += s[k + kernel_offset] * kernel[k]; } - const int tensor_index = plane_output_offset + indexMapper.mapCudaOutputKernelToTensorOutputOffset(i+first_x); + const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i+first_x); buffer[tensor_index] = result; } __syncthreads(); @@ -623,7 +631,11 @@ const float* __restrict kernel, const int numPlanes, const int numX, const int maxX, const int numY, const int maxY, const int kernelSizeX, const int kernelSizeY, float* buffer) { +#if defined(EIGEN_HIPCC) + HIP_DYNAMIC_SHARED(float, s) +#else extern __shared__ float s[]; +#endif const int first_x = blockIdx.x * maxX; const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1; @@ -640,7 +652,7 @@ for (int p = first_plane + threadIdx.z; p < numPlanes; p += plane_stride) { - const int plane_input_offset = indexMapper.mapCudaInputPlaneToTensorInputOffset(p); + const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p); const int plane_kernel_offset = threadIdx.z * num_y_input; // Load inputs to shared memory @@ -649,7 +661,7 @@ const int input_offset = num_x_input * (j + plane_kernel_offset); #pragma unroll for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) { - const int tensor_index = plane_input_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i+first_x, j+first_y); + const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i+first_x, j+first_y); s[i + input_offset] = eval.coeff(tensor_index); } } @@ -657,7 +669,7 @@ __syncthreads(); // Convolution - const int plane_output_offset = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(p); + const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p); #pragma unroll for (int j = threadIdx.y; j < num_y_output; j += blockDim.y) { @@ -673,7 +685,7 @@ result += s[k + input_offset] * kernel[k + kernel_offset]; } } - const int tensor_index = plane_output_offset + indexMapper.mapCudaOutputKernelToTensorOutputOffset(i+first_x, j+first_y); + const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i+first_x, j+first_y); buffer[tensor_index] = result; } } @@ -691,7 +703,11 @@ const size_t maxX, const size_t numY, const size_t maxY, const size_t numZ, const size_t maxZ, const size_t kernelSizeX, const size_t kernelSizeY, const size_t kernelSizeZ, float* buffer) { +#if defined(EIGEN_HIPCC) + HIP_DYNAMIC_SHARED(float, s) +#else extern __shared__ float s[]; +#endif // Load inputs to shared memory const int first_x = blockIdx.x * maxX; @@ -708,13 +724,13 @@ for (int p = 0; p < numPlanes; ++p) { - const int plane_input_offset = indexMapper.mapCudaInputPlaneToTensorInputOffset(p); + const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p); const int plane_kernel_offset = 0; for (int k = threadIdx.z; k < num_z_input; k += blockDim.z) { for (int j = threadIdx.y; j < num_y_input; j += blockDim.y) { for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) { - const int tensor_index = plane_input_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i+first_x, j+first_y, k+first_z); + const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i+first_x, j+first_y, k+first_z); s[i + num_x_input * (j + num_y_input * (k + plane_kernel_offset))] = eval.coeff(tensor_index); } } @@ -726,7 +742,7 @@ const int num_z_output = last_z - first_z + 1; const int num_y_output = last_y - first_y + 1; const int num_x_output = last_x - first_x + 1; - const int plane_output_offset = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(p); + const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p); for (int k = threadIdx.z; k < num_z_output; k += blockDim.z) { for (int j = threadIdx.y; j < num_y_output; j += blockDim.y) { @@ -739,7 +755,7 @@ } } } - const int tensor_index = plane_output_offset + indexMapper.mapCudaOutputKernelToTensorOutputOffset(i+first_x, j+first_y, k+first_z); + const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i+first_x, j+first_y, k+first_z); buffer[tensor_index] = result; } } @@ -764,13 +780,15 @@ enum { IsAligned = TensorEvaluator<InputArgType, GpuDevice>::IsAligned & TensorEvaluator<KernelArgType, GpuDevice>::IsAligned, PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<InputArgType, GpuDevice>::Layout, CoordAccess = false, // to be implemented RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const GpuDevice& device) - : m_inputImpl(op.inputExpression(), device), m_kernelArg(op.kernelExpression()), m_kernelImpl(op.kernelExpression(), device), m_indices(op.indices()), m_buf(NULL), m_kernel(NULL), m_local_kernel(false), m_device(device) + : m_inputImpl(op.inputExpression(), device), m_kernelImpl(op.kernelExpression(), device), m_kernelArg(op.kernelExpression()), m_indices(op.indices()), m_buf(NULL), m_kernel(NULL), m_local_kernel(false), m_device(device) { EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, GpuDevice>::Layout) == static_cast<int>(TensorEvaluator<KernelArgType, GpuDevice>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE); @@ -852,9 +870,9 @@ typedef typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions InputDims; const int maxSharedMem = m_device.sharedMemPerBlock(); - const int maxThreadsPerBlock = m_device.maxCudaThreadsPerBlock(); - const int maxBlocksPerProcessor = m_device.maxCudaThreadsPerMultiProcessor() / maxThreadsPerBlock; - const int numMultiProcessors = m_device.getNumCudaMultiProcessors(); + const int maxThreadsPerBlock = m_device.maxGpuThreadsPerBlock(); + const int maxBlocksPerProcessor = m_device.maxGpuThreadsPerMultiProcessor() / maxThreadsPerBlock; + const int numMultiProcessors = m_device.getNumGpuMultiProcessors(); const int warpSize = 32; switch (NumKernelDims) { @@ -889,7 +907,7 @@ } const int shared_mem = block_size.y * (maxX + kernel_size - 1) * sizeof(Scalar); - assert(shared_mem <= maxSharedMem); + gpu_assert(shared_mem <= maxSharedMem); const int num_x_blocks = ceil(numX, maxX); const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem); @@ -906,15 +924,15 @@ m_inputImpl.dimensions(), kernel_dims, indices); switch(kernel_size) { case 4: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 4, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 4, data); break; } case 7: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 7, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 7, data); break; } default: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, kernel_size, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, kernel_size, data); } } break; @@ -946,7 +964,7 @@ block_size.z = numext::mini<int>(1024/(block_size.x*block_size.y), maxP); const int shared_mem = block_size.z * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * sizeof(Scalar); - assert(shared_mem <= maxSharedMem); + gpu_assert(shared_mem <= maxSharedMem); const int num_x_blocks = ceil(numX, maxX); const int num_y_blocks = ceil(numY, maxY); @@ -967,11 +985,11 @@ case 4: { switch (kernel_size_y) { case 7: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, 7>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, 7, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, 7>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, 7, data); break; } default: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, kernel_size_y, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, kernel_size_y, data); break; } } @@ -980,18 +998,18 @@ case 7: { switch (kernel_size_y) { case 4: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, 4>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, 4, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, 4>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, 4, data); break; } default: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, kernel_size_y, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, kernel_size_y, data); break; } } break; } default: { - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, kernel_size_x, kernel_size_y, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic, Dynamic>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, kernel_size_x, kernel_size_y, data); break; } } @@ -1026,7 +1044,7 @@ dim3 num_blocks(ceil(numX, maxX), ceil(numY, maxY), ceil(numZ, maxZ)); const int shared_mem = (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * (maxZ + kernel_size_z - 1) * sizeof(Scalar); - assert(shared_mem <= maxSharedMem); + gpu_assert(shared_mem <= maxSharedMem); //cout << "launching 3D kernel with block_size.x: " << block_size.x << " block_size.y: " << block_size.y << " block_size.z: " << block_size.z << " num_blocks.x: " << num_blocks.x << " num_blocks.y: " << num_blocks.y << " num_blocks.z: " << num_blocks.z << " shared_mem: " << shared_mem << " in stream " << m_device.stream() << endl; const array<Index, 3> indices(m_indices[idxX], m_indices[idxY], @@ -1037,7 +1055,7 @@ internal::IndexMapper<Index, InputDims, 3, Layout> indexMapper( m_inputImpl.dimensions(), kernel_dims, indices); - LAUNCH_CUDA_KERNEL((EigenConvolutionKernel3D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, numZ, maxZ, kernel_size_x, kernel_size_y, kernel_size_z, data); + LAUNCH_GPU_KERNEL((EigenConvolutionKernel3D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>), num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, numZ, maxZ, kernel_size_x, kernel_size_y, kernel_size_z, data); break; }
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionSycl.h new file mode 100644 index 0000000..e79958f --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionSycl.h
@@ -0,0 +1,483 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> + +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_SYCL_H +#define EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_SYCL_H + +namespace Eigen { + +/** \class TensorConvolution + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor convolution class. + * + * + */ +template <typename CoeffReturnType, typename KernelType, typename HostExpr, typename FunctorExpr, typename Index, +typename InputDims, typename Kernel_accessor, typename Buffer_accessor, typename Local_accessor, typename TupleType> +struct EigenConvolutionKernel1D{ +typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; +internal::IndexMapper<Index, InputDims, 1, Eigen::internal::traits<HostExpr>::Layout> indexMapper; +Kernel_accessor kernel_filter; +const size_t kernelSize, range_x, range_y; +Buffer_accessor buffer_acc; +ptrdiff_t out_offset; +Local_accessor local_acc; +FunctorExpr functors; +TupleType tuple_of_accessors; +EigenConvolutionKernel1D(internal::IndexMapper<Index, InputDims, 1, Eigen::internal::traits<HostExpr>::Layout> indexMapper_, + Kernel_accessor kernel_filter_, const size_t kernelSize_, const size_t range_x_, const size_t range_y_, + Buffer_accessor buffer_acc_, ptrdiff_t out_offset_, Local_accessor local_acc_, FunctorExpr functors_, TupleType tuple_of_accessors_) + :indexMapper(indexMapper_), kernel_filter(kernel_filter_), kernelSize(kernelSize_), range_x(range_x_), range_y(range_y_), + buffer_acc(buffer_acc_), out_offset(out_offset_),local_acc(local_acc_), functors(functors_), tuple_of_accessors(tuple_of_accessors_) {} + + void operator()(cl::sycl::nd_item<2> itemID) { + typedef typename TensorSycl::internal::ConvertToDeviceExpression<HostExpr>::Type DevExpr; + auto device_expr =TensorSycl::internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + auto device_evaluator = Eigen::TensorEvaluator<DevExpr, Eigen::SyclKernelDevice>(device_expr.expr, Eigen::SyclKernelDevice()); + + auto buffer_ptr = ConvertToActualTypeSycl(CoeffReturnType, buffer_acc); + auto kernel_ptr = ConvertToActualTypeSycl(KernelType, kernel_filter); + + const size_t num_x_input = (itemID.get_local_range()[0] +kernelSize -1); //the required row to be calculated for the for each plane in shered memory + const size_t plane_kernel_offset = itemID.get_local(1) * num_x_input; + const size_t first_input_start = itemID.get_group(0)*itemID.get_local_range()[0]; + const size_t plane_tensor_offset =indexMapper.mapCudaInputPlaneToTensorInputOffset(itemID.get_global(1)); + /// fill the shared memory + for (size_t i = itemID.get_local(0); i < num_x_input ; i += itemID.get_local_range()[0]) { + const size_t local_index = i + plane_kernel_offset ; + const size_t tensor_index = plane_tensor_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i + first_input_start); + if(((i + first_input_start) < (range_x +kernelSize-1)) && itemID.get_global(1)< range_y){ + local_acc[local_index] = device_evaluator.coeff(tensor_index); + } + else local_acc[local_index]=0.0f; + } + + itemID.barrier(cl::sycl::access::fence_space::local_space); + + // calculate the convolution + const size_t first_output_start =itemID.get_group(0)*(itemID.get_local_range()[0]); // output start x + if(itemID.get_global(0)< range_x && itemID.get_global(1)< range_y){ + CoeffReturnType result = static_cast<CoeffReturnType>(0); + const size_t index = plane_kernel_offset+ itemID.get_local(0); + for (size_t k = 0; k < kernelSize; ++k) { + result += (local_acc[k + index] * kernel_ptr[k]); + } + const size_t tensor_index = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(itemID.get_global(1)) + +indexMapper.mapCudaOutputKernelToTensorOutputOffset(itemID.get_local(0) + first_output_start); + buffer_ptr[tensor_index+ConvertToActualSyclOffset(CoeffReturnType, out_offset)] = result; + } + } +}; + + +template <typename CoeffReturnType, typename KernelType, typename HostExpr, typename FunctorExpr, typename Index, +typename InputDims, typename Kernel_accessor, typename Buffer_accessor, typename Local_accessor, typename TupleType> +struct EigenConvolutionKernel2D{ +typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; +internal::IndexMapper<Index, InputDims, 2, Eigen::internal::traits<HostExpr>::Layout> indexMapper; +Kernel_accessor kernel_filter; +const size_t kernelSize_x, kernelSize_y, range_x, range_y , range_z; +Buffer_accessor buffer_acc; +ptrdiff_t out_offset; +Local_accessor local_acc; +FunctorExpr functors; +TupleType tuple_of_accessors; +EigenConvolutionKernel2D(internal::IndexMapper<Index, InputDims, 2, Eigen::internal::traits<HostExpr>::Layout> indexMapper_, + Kernel_accessor kernel_filter_, const size_t kernelSize_x_, const size_t kernelSize_y_ ,const size_t range_x_, const size_t range_y_, const size_t range_z_, + Buffer_accessor buffer_acc_, ptrdiff_t out_offset_, Local_accessor local_acc_, FunctorExpr functors_, TupleType tuple_of_accessors_) + :indexMapper(indexMapper_), kernel_filter(kernel_filter_), kernelSize_x(kernelSize_x_), kernelSize_y(kernelSize_y_), range_x(range_x_), range_y(range_y_), range_z(range_z_), + buffer_acc(buffer_acc_), out_offset(out_offset_), local_acc(local_acc_), functors(functors_), tuple_of_accessors(tuple_of_accessors_) {} + + void operator()(cl::sycl::nd_item<3> itemID) { + typedef typename TensorSycl::internal::ConvertToDeviceExpression<HostExpr>::Type DevExpr; + auto device_expr =TensorSycl::internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + auto device_evaluator = Eigen::TensorEvaluator<DevExpr, Eigen::SyclKernelDevice>(device_expr.expr, Eigen::SyclKernelDevice()); + + auto buffer_ptr = ConvertToActualTypeSycl(CoeffReturnType, buffer_acc); + auto kernel_ptr = ConvertToActualTypeSycl(KernelType, kernel_filter); + const size_t num_x_input = (itemID.get_local_range()[0] +kernelSize_x -1); //the required row to be calculated for the for each plane in shered memory + const size_t num_y_input = (itemID.get_local_range()[1] +kernelSize_y -1); //the required row to be calculated for the for each plane in shered memory + const size_t plane_input_offset = indexMapper.mapCudaInputPlaneToTensorInputOffset(itemID.get_global(2)); + const size_t plane_kernel_offset = itemID.get_local(2) * num_y_input; + + /// fill the shared memory + const size_t first_x_input_start = itemID.get_group(0)*itemID.get_local_range()[0]; + const size_t first_y_input_start = itemID.get_group(1)*itemID.get_local_range()[1]; + for (size_t j = itemID.get_local(1); j < num_y_input; j += itemID.get_local_range()[1]) { + const size_t local_input_offset = num_x_input * (j + plane_kernel_offset); + for (size_t i = itemID.get_local(0); i < num_x_input ; i += itemID.get_local_range()[0]) { + const size_t local_index = i + local_input_offset; + const size_t tensor_index = plane_input_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i + first_x_input_start, j+ first_y_input_start ); + if(((i + first_x_input_start) < (range_x +kernelSize_x-1)) &&((j + first_y_input_start) < (range_y +kernelSize_y-1)) && itemID.get_global(2)< range_z){ + local_acc[local_index] = device_evaluator.coeff(tensor_index); + } + else local_acc[local_index]=0.0f; + } + } + + itemID.barrier(cl::sycl::access::fence_space::local_space); + + // calculate the convolution + const size_t fitst_x_output_start =itemID.get_group(0)*(itemID.get_local_range()[0]); // output start x + const size_t fitst_y_output_start =itemID.get_group(1)*(itemID.get_local_range()[1]); // output start y + if(itemID.get_global(0)< range_x && itemID.get_global(1)< range_y && itemID.get_global(2)< range_z){ + CoeffReturnType result = static_cast<CoeffReturnType>(0); + for (size_t j = 0; j < kernelSize_y; j++) { + size_t kernel_offset =kernelSize_x * j; + const size_t index = (num_x_input*(plane_kernel_offset + j+ itemID.get_local(1))) + itemID.get_local(0); + for (size_t i = 0; i < kernelSize_x; i++) { + result += (local_acc[i + index] * kernel_ptr[i+kernel_offset]); + } + } + const size_t tensor_index = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(itemID.get_global(2)) + +indexMapper.mapCudaOutputKernelToTensorOutputOffset(itemID.get_local(0) + fitst_x_output_start, itemID.get_local(1) + fitst_y_output_start); + buffer_ptr[tensor_index +ConvertToActualSyclOffset(CoeffReturnType, out_offset)] = result; + } + } +}; + + + +template <typename CoeffReturnType, typename KernelType, typename HostExpr, typename FunctorExpr, typename Index, +typename InputDims, typename Kernel_accessor, typename Buffer_accessor, typename Local_accessor, typename TupleType> +struct EigenConvolutionKernel3D{ +typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; +internal::IndexMapper<Index, InputDims, 3, Eigen::internal::traits<HostExpr>::Layout> indexMapper; +Kernel_accessor kernel_filter; +const size_t kernelSize_x, kernelSize_y, kernelSize_z, range_x, range_y , range_z, numP; +Buffer_accessor buffer_acc; +ptrdiff_t out_offset; +Local_accessor local_acc; +FunctorExpr functors; +TupleType tuple_of_accessors; +EigenConvolutionKernel3D(internal::IndexMapper<Index, InputDims, 3, Eigen::internal::traits<HostExpr>::Layout> indexMapper_, + Kernel_accessor kernel_filter_, const size_t kernelSize_x_, const size_t kernelSize_y_ , const size_t kernelSize_z_ , + const size_t range_x_, const size_t range_y_, const size_t range_z_, const size_t numP_, + Buffer_accessor buffer_acc_, ptrdiff_t out_offset_, Local_accessor local_acc_, FunctorExpr functors_, TupleType tuple_of_accessors_) + :indexMapper(indexMapper_), kernel_filter(kernel_filter_), kernelSize_x(kernelSize_x_), kernelSize_y(kernelSize_y_), + kernelSize_z(kernelSize_z_), range_x(range_x_), range_y(range_y_), range_z(range_z_), numP(numP_), + buffer_acc(buffer_acc_), out_offset(out_offset_), local_acc(local_acc_), functors(functors_), tuple_of_accessors(tuple_of_accessors_) {} + + void operator()(cl::sycl::nd_item<3> itemID) { + typedef typename TensorSycl::internal::ConvertToDeviceExpression<HostExpr>::Type DevExpr; + auto device_expr =TensorSycl::internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + auto device_evaluator = Eigen::TensorEvaluator<DevExpr, Eigen::SyclKernelDevice>(device_expr.expr, Eigen::SyclKernelDevice()); + + auto buffer_ptr = ConvertToActualTypeSycl(CoeffReturnType, buffer_acc); + auto kernel_ptr = ConvertToActualTypeSycl(KernelType, kernel_filter); + const size_t num_x_input = (itemID.get_local_range()[0] +kernelSize_x -1); //the required row to be calculated for the for each plane in shered memory + const size_t num_y_input = (itemID.get_local_range()[1] +kernelSize_y -1); //the required row to be calculated for the for each plane in shered memory + const size_t num_z_input = (itemID.get_local_range()[2] +kernelSize_z -1); //the required row to be calculated for the for each plane in shered memory + const size_t first_x_input_start = itemID.get_group(0)*itemID.get_local_range()[0]; + const size_t first_y_input_start = itemID.get_group(1)*itemID.get_local_range()[1]; + const size_t first_z_input_start = itemID.get_group(2)*itemID.get_local_range()[2]; + for(size_t p=0; p<numP; p++){ + /// fill the shared memory + const size_t plane_input_offset = indexMapper.mapCudaInputPlaneToTensorInputOffset(p); + for (size_t k = itemID.get_local(2); k < num_z_input; k += itemID.get_local_range()[2]) { + for (size_t j = itemID.get_local(1); j < num_y_input; j += itemID.get_local_range()[1]) { + for (size_t i = itemID.get_local(0); i < num_x_input ; i += itemID.get_local_range()[0]) { + const size_t local_index = i + (num_x_input * (j + (num_y_input * k))); + const size_t tensor_index = plane_input_offset + indexMapper.mapCudaInputKernelToTensorInputOffset(i + first_x_input_start, j+ first_y_input_start , k+ first_z_input_start ); + if(((i + first_x_input_start) < (range_x +kernelSize_x-1)) && ((j + first_y_input_start) < (range_y +kernelSize_y-1)) && ((k + first_z_input_start) < (range_z +kernelSize_z-1)) ){ + local_acc[local_index] = device_evaluator.coeff(tensor_index); + } + else local_acc[local_index]=0.0f; + } + } + } + itemID.barrier(cl::sycl::access::fence_space::local_space); + + // calculate the convolution + const size_t fitst_x_output_start =itemID.get_group(0)*(itemID.get_local_range()[0]); // x + const size_t fitst_y_output_start =itemID.get_group(1)*(itemID.get_local_range()[1]); // y + const size_t fitst_z_output_start =itemID.get_group(2)*(itemID.get_local_range()[2]); // z + + if(itemID.get_global(0)< range_x && itemID.get_global(1)< range_y && itemID.get_global(2)< range_z){ + CoeffReturnType result = static_cast<CoeffReturnType>(0); + for (size_t k = 0; k < kernelSize_z; k++) { + for (size_t j = 0; j < kernelSize_y; j++) { + for (size_t i = 0; i < kernelSize_x; i++) { + const size_t kernel_index =i + kernelSize_x * (j + kernelSize_y * k); + const size_t local_index = ((i+ itemID.get_local(0))+ num_x_input*((j+ itemID.get_local(1)) + num_y_input * (k+ itemID.get_local(2)))); + result += (local_acc[local_index] * kernel_ptr[kernel_index]); + } + } + } + const size_t tensor_index = indexMapper.mapCudaOutputPlaneToTensorOutputOffset(p) + +indexMapper.mapCudaOutputKernelToTensorOutputOffset(itemID.get_local(0) + fitst_x_output_start, itemID.get_local(1) + fitst_y_output_start, itemID.get_local(2) + fitst_z_output_start ); + buffer_ptr[tensor_index+ConvertToActualSyclOffset(CoeffReturnType, out_offset)] = result; + } + + itemID.barrier(cl::sycl::access::fence_space::local_space); + } + } +}; + + +template<typename Indices, typename InputArgType, typename KernelArgType> +struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelArgType>, const Eigen::SyclDevice> +{ + typedef TensorConvolutionOp<Indices, InputArgType, KernelArgType> XprType; + + static const int NumDims = internal::array_size<typename TensorEvaluator<InputArgType, const Eigen::SyclDevice>::Dimensions>::value; + static const int NumKernelDims = internal::array_size<Indices>::value; + typedef typename XprType::Index Index; + typedef DSizes<Index, NumDims> Dimensions; + typedef typename TensorEvaluator<KernelArgType, const Eigen::SyclDevice>::Dimensions KernelDimensions; + typedef const Eigen::SyclDevice Device; + + enum { + IsAligned = TensorEvaluator<InputArgType, const Eigen::SyclDevice>::IsAligned & TensorEvaluator<KernelArgType, const Eigen::SyclDevice>::IsAligned, + PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<InputArgType, const Eigen::SyclDevice>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false + }; + + EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Eigen::SyclDevice& device) + : m_inputImpl(op.inputExpression(), device), m_kernelArg(op.kernelExpression()), m_kernelImpl(op.kernelExpression(), device), m_indices(op.indices()), m_buf(NULL), m_kernel(NULL), m_local_kernel(false), m_device(device) + { + EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, const Eigen::SyclDevice>::Layout) == static_cast<int>(TensorEvaluator<KernelArgType, const Eigen::SyclDevice>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE); + + const typename TensorEvaluator<InputArgType, const Eigen::SyclDevice>::Dimensions& input_dims = m_inputImpl.dimensions(); + const typename TensorEvaluator<KernelArgType, const Eigen::SyclDevice>::Dimensions& kernel_dims = m_kernelImpl.dimensions(); + + m_dimensions = m_inputImpl.dimensions(); + for (int i = 0; i < NumKernelDims; ++i) { + const Index index = op.indices()[i]; + const Index input_dim = input_dims[index]; + const Index kernel_dim = kernel_dims[i]; + const Index result_dim = input_dim - kernel_dim + 1; + m_dimensions[index] = result_dim; + } + } + + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, const Eigen::SyclDevice>::type PacketReturnType; + typedef typename InputArgType::Scalar Scalar; + static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + + EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_dimensions; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + preloadKernel(); + m_inputImpl.evalSubExprsIfNeeded(NULL); + if (data) { + executeEval(data); + return false; + } else { + m_buf = (Scalar*)m_device.allocate(dimensions().TotalSize() * sizeof(Scalar)); + executeEval(m_buf); + return true; + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + m_inputImpl.cleanup(); + if (m_buf) { + m_device.deallocate(m_buf); + m_buf = NULL; + } + if (m_local_kernel) { + m_device.deallocate((void*)m_kernel); + m_local_kernel = false; + } + m_kernel = NULL; + } + /// used by sycl in order to build the sycl buffer + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Device& device() const{return m_device;} + /// used by sycl in order to build the sycl buffer + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { return m_buf; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void preloadKernel() { + // Don't make a local copy of the kernel unless we have to (i.e. it's an + // expression that needs to be evaluated) + const Scalar* in_place = m_kernelImpl.data(); + if (in_place) { + m_kernel = in_place; + m_local_kernel = false; + } else { + ptrdiff_t kernel_sz = m_kernelImpl.dimensions().TotalSize() * sizeof(Scalar); + Scalar* local = (Scalar*)m_device.allocate(kernel_sz); + typedef TensorEvalToOp<const KernelArgType> EvalTo; + EvalTo evalToTmp(local, m_kernelArg); + const bool PacketAccess = internal::IsVectorizable<const Eigen::SyclDevice, KernelArgType>::value; + internal::TensorExecutor<const EvalTo, const Eigen::SyclDevice, PacketAccess>::run(evalToTmp, m_device); + m_kernel = local; + m_local_kernel = true; + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void executeEval(Scalar* data) const { + typedef TensorEvaluator<InputArgType, const Eigen::SyclDevice> InputEvaluator; + typedef typename InputEvaluator::Dimensions InputDims; + + typedef Eigen::TensorSycl::internal::FunctorExtractor<InputEvaluator> InputFunctorExpr; + // extract input functor list + InputFunctorExpr input_functors = Eigen::TensorSycl::internal::extractFunctors(m_inputImpl); + ptrdiff_t out_offset = m_device.get_offset(data); + + + m_device.sycl_queue().submit([&](cl::sycl::handler &cgh) { + + typedef cl::sycl::accessor<CoeffReturnType, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::local> InputLocalAcc; + /// work-around for gcc 4.8 auto bug + typedef decltype(Eigen::TensorSycl::internal::createTupleOfAccessors<InputEvaluator>(cgh, m_inputImpl)) InputTupleType; + // create input tuple of accessors + InputTupleType tuple_of_accessors = Eigen::TensorSycl::internal::createTupleOfAccessors<InputEvaluator>(cgh, m_inputImpl); + + typedef cl::sycl::accessor<uint8_t, 1, cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer> OutputAccessorType; + OutputAccessorType out_res= m_device. template get_sycl_accessor<cl::sycl::access::mode::write>(cgh, data); + typedef cl::sycl::accessor<uint8_t, 1, cl::sycl::access::mode::read, cl::sycl::access::target::global_buffer> KernelAccessorType; + KernelAccessorType kernel_acc= m_device. template get_sycl_accessor<cl::sycl::access::mode::read>(cgh, m_kernel); + + switch (NumKernelDims) { + case 1: { + const size_t numX = dimensions()[m_indices[0]]; + const size_t numP = dimensions().TotalSize() / numX; + const size_t kernel_size = m_kernelImpl.dimensions().TotalSize(); + size_t range_x, GRange_x, tileSize_x, range_y, GRange_y, tileSize_y; + m_device.parallel_for_setup(numX, numP, tileSize_x,tileSize_y,range_x,range_y, GRange_x, GRange_y ); + const size_t shared_mem =(tileSize_x +kernel_size -1)*(tileSize_y); + gpu_assert(static_cast<unsigned long>(shared_mem) <= m_device.sharedMemPerBlock()); + auto global_range=cl::sycl::range<2>(GRange_x, GRange_y); // global range + auto local_range=cl::sycl::range<2>(tileSize_x, tileSize_y); // local range + InputLocalAcc local_acc(cl::sycl::range<1>(shared_mem), cgh); + const array<Index, 1> indices{{m_indices[0]}}; + const array<Index, 1> kernel_dims{{m_kernelImpl.dimensions()[0]}}; + internal::IndexMapper<Index, InputDims, 1, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices); + cgh.parallel_for(cl::sycl::nd_range<2>(global_range, local_range), + EigenConvolutionKernel1D<CoeffReturnType, Scalar, InputArgType, InputFunctorExpr, Index, + InputDims, KernelAccessorType, OutputAccessorType, InputLocalAcc, InputTupleType>( + indexMapper,kernel_acc, kernel_size, numX, numP, out_res, out_offset, local_acc, input_functors, tuple_of_accessors)); + break; + } + + case 2: { + const size_t idxX =static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 1; + const size_t idxY =static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 0; + const size_t kernel_size_x = m_kernelImpl.dimensions()[idxX]; + const size_t kernel_size_y = m_kernelImpl.dimensions()[idxY]; + const size_t numX = dimensions()[m_indices[idxX]]; + const size_t numY = dimensions()[m_indices[idxY]]; + const size_t numP = dimensions().TotalSize() / (numX*numY); + size_t range_x, GRange_x, tileSize_x, range_y, GRange_y, tileSize_y, range_z, GRange_z, tileSize_z; + m_device.parallel_for_setup(numX, numY, numP, tileSize_x, tileSize_y, tileSize_z, range_x, range_y, range_z, GRange_x, GRange_y, GRange_z ); + const size_t shared_mem =(tileSize_x +kernel_size_x -1)*(tileSize_y +kernel_size_y -1) * tileSize_z; + gpu_assert(static_cast<unsigned long>(shared_mem) <= m_device.sharedMemPerBlock()); + auto global_range=cl::sycl::range<3>(GRange_x, GRange_y, GRange_z); // global range + auto local_range=cl::sycl::range<3>(tileSize_x, tileSize_y, tileSize_z); // local range + InputLocalAcc local_acc(cl::sycl::range<1>(shared_mem), cgh); + const array<Index, 2> indices {{m_indices[idxX], m_indices[idxY]}}; + const array<Index, 2> kernel_dims{{m_kernelImpl.dimensions()[idxX], m_kernelImpl.dimensions()[idxY]}}; + internal::IndexMapper<Index, InputDims, 2, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices); + cgh.parallel_for(cl::sycl::nd_range<3>(global_range, local_range), + EigenConvolutionKernel2D<CoeffReturnType, Scalar, InputArgType, InputFunctorExpr, Index, + InputDims, KernelAccessorType, OutputAccessorType, InputLocalAcc, InputTupleType>( + indexMapper,kernel_acc, kernel_size_x, kernel_size_y, numX, numY, numP, out_res, out_offset, local_acc, input_functors, tuple_of_accessors)); + break; + } + + case 3: { + const size_t idxX =static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 2; + const size_t idxY =static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 1; + const size_t idxZ =static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 0; + const size_t kernel_size_x = m_kernelImpl.dimensions()[idxX]; + const size_t kernel_size_y = m_kernelImpl.dimensions()[idxY]; + const size_t kernel_size_z = m_kernelImpl.dimensions()[idxZ]; + const size_t numX = dimensions()[m_indices[idxX]]; + const size_t numY = dimensions()[m_indices[idxY]]; + const size_t numZ = dimensions()[m_indices[idxZ]]; + const size_t numP = dimensions().TotalSize() / (numX*numY*numZ); + const array<Index, 3> indices{{m_indices[idxX], m_indices[idxY], m_indices[idxZ]}}; + const array<Index, 3> kernel_dims{{m_kernelImpl.dimensions()[idxX],m_kernelImpl.dimensions()[idxY], m_kernelImpl.dimensions()[idxZ]}}; + internal::IndexMapper<Index, InputDims, 3, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices); + size_t range_x, GRange_x, tileSize_x, range_y, GRange_y, tileSize_y, range_z, GRange_z, tileSize_z; + m_device.parallel_for_setup(numX, numY, numZ, tileSize_x, tileSize_y, tileSize_z, range_x, range_y, range_z, GRange_x, GRange_y, GRange_z ); + const size_t shared_mem =(tileSize_x +kernel_size_x -1)*(tileSize_y +kernel_size_y -1) * (tileSize_z +kernel_size_y -1); + gpu_assert(static_cast<unsigned long>(shared_mem) <= m_device.sharedMemPerBlock()); + auto global_range=cl::sycl::range<3>(GRange_x, GRange_y, GRange_z); // global range + auto local_range=cl::sycl::range<3>(tileSize_x, tileSize_y, tileSize_z); // local range + InputLocalAcc local_acc(cl::sycl::range<1>(shared_mem), cgh); + cgh.parallel_for(cl::sycl::nd_range<3>(global_range, local_range), + EigenConvolutionKernel3D<CoeffReturnType, Scalar, InputArgType, InputFunctorExpr, Index, + InputDims, KernelAccessorType, OutputAccessorType, InputLocalAcc, InputTupleType>( + indexMapper,kernel_acc, kernel_size_x, kernel_size_y, kernel_size_z, numX, numY, + numZ, numP, out_res, out_offset, local_acc, input_functors, tuple_of_accessors)); + break; + } + + default: { + EIGEN_STATIC_ASSERT((NumKernelDims >= 1 && NumKernelDims <= 3), THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE); + } + } + }); + m_device.asynchronousExec(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const + { + eigen_assert(m_buf); + eigen_assert(index < m_dimensions.TotalSize()); + return m_buf[index]; + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(const Index index) const + { + eigen_assert(m_buf); + eigen_assert(index < m_dimensions.TotalSize()); + return internal::ploadt<PacketReturnType, LoadMode>(m_buf+index); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost + costPerCoeff(bool vectorized) const { + // TODO(rmlarsen): FIXME: For now, this is just a copy of the CPU cost + // model. + const double kernel_size = m_kernelImpl.dimensions().TotalSize(); + // We ignore the use of fused multiply-add. + const double convolve_compute_cost = + TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>(); + const double firstIndex_compute_cost = + NumDims * + (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + + TensorOpCost::DivCost<Index>()); + return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) + + kernel_size * (m_inputImpl.costPerCoeff(vectorized) + + m_kernelImpl.costPerCoeff(vectorized) + + TensorOpCost(0, 0, convolve_compute_cost, vectorized, + PacketSize)); + } + + private: + // No assignment (copies are needed by the kernels) + TensorEvaluator& operator = (const TensorEvaluator&); + TensorEvaluator<InputArgType, const Eigen::SyclDevice> m_inputImpl; + KernelArgType m_kernelArg; + TensorEvaluator<KernelArgType, const Eigen::SyclDevice> m_kernelImpl; + Indices m_indices; + Dimensions m_dimensions; + Scalar* m_buf; + const Scalar* m_kernel; + bool m_local_kernel; + const Eigen::SyclDevice& m_device; +}; + +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorCostModel.h b/unsupported/Eigen/CXX11/src/Tensor/TensorCostModel.h index 4e8f866..7f79ac3 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorCostModel.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorCostModel.h
@@ -10,10 +10,6 @@ #ifndef EIGEN_CXX11_TENSOR_TENSOR_COST_MODEL_H #define EIGEN_CXX11_TENSOR_TENSOR_COST_MODEL_H -//#if !defined(EIGEN_USE_GPU) -//#define EIGEN_USE_COST_MODEL -//#endif - namespace Eigen { /** \class TensorEvaluator @@ -32,45 +28,47 @@ // model based on minimal reciprocal throughput numbers from Intel or // Agner Fog's tables would be better than what is there now. template <typename ArgType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static int MulCost() { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int MulCost() { return internal::functor_traits< - internal::scalar_product_op<ArgType, ArgType>>::Cost; + internal::scalar_product_op<ArgType, ArgType> >::Cost; } template <typename ArgType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static int AddCost() { - return internal::functor_traits<internal::scalar_sum_op<ArgType>>::Cost; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int AddCost() { + return internal::functor_traits<internal::scalar_sum_op<ArgType> >::Cost; } template <typename ArgType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static int DivCost() { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int DivCost() { return internal::functor_traits< - internal::scalar_quotient_op<ArgType, ArgType>>::Cost; + internal::scalar_quotient_op<ArgType, ArgType> >::Cost; } template <typename ArgType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static int ModCost() { - return internal::functor_traits<internal::scalar_mod_op<ArgType>>::Cost; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int ModCost() { + return internal::functor_traits<internal::scalar_mod_op<ArgType> >::Cost; } template <typename SrcType, typename TargetType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static int CastCost() { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int CastCost() { return internal::functor_traits< - internal::scalar_cast_op<SrcType, TargetType>>::Cost; + internal::scalar_cast_op<SrcType, TargetType> >::Cost; } + EIGEN_DEVICE_FUNC TensorOpCost() : bytes_loaded_(0), bytes_stored_(0), compute_cycles_(0) {} + EIGEN_DEVICE_FUNC TensorOpCost(double bytes_loaded, double bytes_stored, double compute_cycles) : bytes_loaded_(bytes_loaded), bytes_stored_(bytes_stored), compute_cycles_(compute_cycles) {} + EIGEN_DEVICE_FUNC TensorOpCost(double bytes_loaded, double bytes_stored, double compute_cycles, bool vectorized, double packet_size) : bytes_loaded_(bytes_loaded), bytes_stored_(bytes_stored), compute_cycles_(vectorized ? compute_cycles / packet_size : compute_cycles) { - using std::isfinite; - eigen_assert(bytes_loaded >= 0 && (isfinite)(bytes_loaded)); - eigen_assert(bytes_stored >= 0 && (isfinite)(bytes_stored)); - eigen_assert(compute_cycles >= 0 && (isfinite)(compute_cycles)); + eigen_assert(bytes_loaded >= 0 && (numext::isfinite)(bytes_loaded)); + eigen_assert(bytes_stored >= 0 && (numext::isfinite)(bytes_stored)); + eigen_assert(compute_cycles >= 0 && (numext::isfinite)(compute_cycles)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bytes_loaded() const { @@ -96,21 +94,21 @@ } // TODO(rmlarsen): Define min in terms of total cost, not elementwise. - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost& cwiseMin( - const TensorOpCost& rhs) { - bytes_loaded_ = numext::mini(bytes_loaded_, rhs.bytes_loaded()); - bytes_stored_ = numext::mini(bytes_stored_, rhs.bytes_stored()); - compute_cycles_ = numext::mini(compute_cycles_, rhs.compute_cycles()); - return *this; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost cwiseMin( + const TensorOpCost& rhs) const { + double bytes_loaded = numext::mini(bytes_loaded_, rhs.bytes_loaded()); + double bytes_stored = numext::mini(bytes_stored_, rhs.bytes_stored()); + double compute_cycles = numext::mini(compute_cycles_, rhs.compute_cycles()); + return TensorOpCost(bytes_loaded, bytes_stored, compute_cycles); } // TODO(rmlarsen): Define max in terms of total cost, not elementwise. - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost& cwiseMax( - const TensorOpCost& rhs) { - bytes_loaded_ = numext::maxi(bytes_loaded_, rhs.bytes_loaded()); - bytes_stored_ = numext::maxi(bytes_stored_, rhs.bytes_stored()); - compute_cycles_ = numext::maxi(compute_cycles_, rhs.compute_cycles()); - return *this; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost cwiseMax( + const TensorOpCost& rhs) const { + double bytes_loaded = numext::maxi(bytes_loaded_, rhs.bytes_loaded()); + double bytes_stored = numext::maxi(bytes_stored_, rhs.bytes_stored()); + double compute_cycles = numext::maxi(compute_cycles_, rhs.compute_cycles()); + return TensorOpCost(bytes_loaded, bytes_stored, compute_cycles); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost& operator+=( @@ -176,8 +174,10 @@ static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int numThreads( double output_size, const TensorOpCost& cost_per_coeff, int max_threads) { double cost = totalCost(output_size, cost_per_coeff); - int threads = (cost - kStartupCycles) / kPerThreadCycles + 0.9; - return numext::mini(max_threads, numext::maxi(1, threads)); + double threads = (cost - kStartupCycles) / kPerThreadCycles + 0.9; + // Make sure we don't invoke undefined behavior when we convert to an int. + threads = numext::mini<double>(threads, GenericNumTraits<int>::highest()); + return numext::mini(max_threads, numext::maxi<int>(1, threads)); } // taskSize assesses parallel task size. @@ -188,14 +188,13 @@ return totalCost(output_size, cost_per_coeff) / kTaskSize; } - private: static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double totalCost( double output_size, const TensorOpCost& cost_per_coeff) { // Cost of memory fetches from L2 cache. 64 is typical cache line size. // 11 is L2 cache latency on Haswell. // We don't know whether data is in L1, L2 or L3. But we are most interested // in single-threaded computational time around 100us-10ms (smaller time - // is too small for parallelization, larger time is not intersting + // is too small for parallelization, larger time is not interesting // either because we are probably using all available threads already). // And for the target time range, L2 seems to be what matters. Data set // fitting into L1 is too small to take noticeable time. Data set fitting
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h b/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h index e020d07..6ee3827 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h
@@ -30,6 +30,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = traits<XprType>::NumDimensions; static const int Layout = traits<XprType>::Layout; + typedef typename traits<XprType>::PointerType PointerType; }; template<typename CustomUnaryFunc, typename XprType> @@ -86,12 +87,14 @@ typedef typename internal::remove_const<typename ArgType::Scalar>::type Scalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + typedef typename PointerType<CoeffReturnType, Device>::Type PointerT; enum { IsAligned = false, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<XprType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -105,13 +108,13 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* data) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(PointerT data) { if (data) { evalTo(data); return false; } else { - m_result = static_cast<CoeffReturnType*>( - m_device.allocate(dimensions().TotalSize() * sizeof(Scalar))); + m_result = static_cast<PointerT>( + m_device.allocate_temp(dimensions().TotalSize() * sizeof(Scalar))); evalTo(m_result); return true; } @@ -119,7 +122,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { if (m_result != NULL) { - m_device.deallocate(m_result); + m_device.deallocate_temp(m_result); m_result = NULL; } } @@ -138,19 +141,22 @@ return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_result; } + EIGEN_DEVICE_FUNC PointerT data() const { return m_result; } + +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Device& device() const { return m_device; } +#endif protected: - EIGEN_DEVICE_FUNC void evalTo(Scalar* data) { - TensorMap<Tensor<CoeffReturnType, NumDims, Layout, Index> > result( - data, m_dimensions); + EIGEN_DEVICE_FUNC void evalTo(PointerT data) { + TensorMap<Tensor<CoeffReturnType, NumDims, Layout, Index> > result(data, m_dimensions); m_op.func().eval(m_op.expression(), result, m_device); } Dimensions m_dimensions; const ArgType m_op; const Device& m_device; - CoeffReturnType* m_result; + PointerT m_result; }; @@ -180,6 +186,8 @@ typedef typename remove_reference<RhsNested>::type _RhsNested; static const int NumDimensions = traits<LhsXprType>::NumDimensions; static const int Layout = traits<LhsXprType>::Layout; + typedef typename conditional<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val, + typename traits<LhsXprType>::PointerType, typename traits<RhsXprType>::PointerType>::type PointerType; }; template<typename CustomBinaryFunc, typename LhsXprType, typename RhsXprType> @@ -242,12 +250,14 @@ typedef typename XprType::Scalar Scalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + typedef typename PointerType<CoeffReturnType, Device>::Type PointerT; enum { IsAligned = false, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<LhsXprType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -261,12 +271,12 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* data) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(PointerT data) { if (data) { evalTo(data); return false; } else { - m_result = static_cast<Scalar *>(m_device.allocate(dimensions().TotalSize() * sizeof(Scalar))); + m_result = static_cast<PointerT>(m_device.allocate_temp(dimensions().TotalSize() * sizeof(CoeffReturnType))); evalTo(m_result); return true; } @@ -274,7 +284,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { if (m_result != NULL) { - m_device.deallocate(m_result); + m_device.deallocate_temp(m_result); m_result = NULL; } } @@ -293,18 +303,22 @@ return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_result; } + EIGEN_DEVICE_FUNC PointerT data() const { return m_result; } + +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Device& device() const { return m_device; } +#endif protected: - EIGEN_DEVICE_FUNC void evalTo(Scalar* data) { - TensorMap<Tensor<Scalar, NumDims, Layout> > result(data, m_dimensions); + EIGEN_DEVICE_FUNC void evalTo(PointerT data) { + TensorMap<Tensor<CoeffReturnType, NumDims, Layout> > result(data, m_dimensions); m_op.func().eval(m_op.lhsExpression(), m_op.rhsExpression(), result, m_device); } Dimensions m_dimensions; const XprType m_op; const Device& m_device; - CoeffReturnType* m_result; + PointerT m_result; };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h index 1d2d162..f779239 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h
@@ -1,314 +1,6 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H) -#define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H - -namespace Eigen { - -// This defines an interface that GPUDevice can take to use -// CUDA streams underneath. -class StreamInterface { - public: - virtual ~StreamInterface() {} - - virtual const cudaStream_t& stream() const = 0; - virtual const cudaDeviceProp& deviceProperties() const = 0; - - // Allocate memory on the actual device where the computation will run - virtual void* allocate(size_t num_bytes) const = 0; - virtual void deallocate(void* buffer) const = 0; -}; - -static cudaDeviceProp* m_deviceProperties; -static bool m_devicePropInitialized = false; - -static void initializeDeviceProp() { - if (!m_devicePropInitialized) { - if (!m_devicePropInitialized) { - int num_devices; - cudaError_t status = cudaGetDeviceCount(&num_devices); - if (status != cudaSuccess) { - std::cerr << "Failed to get the number of CUDA devices: " - << cudaGetErrorString(status) - << std::endl; - assert(status == cudaSuccess); - } - m_deviceProperties = new cudaDeviceProp[num_devices]; - for (int i = 0; i < num_devices; ++i) { - status = cudaGetDeviceProperties(&m_deviceProperties[i], i); - if (status != cudaSuccess) { - std::cerr << "Failed to initialize CUDA device #" - << i - << ": " - << cudaGetErrorString(status) - << std::endl; - assert(status == cudaSuccess); - } - } - m_devicePropInitialized = true; - } - } -} - -static const cudaStream_t default_stream = cudaStreamDefault; - -class CudaStreamDevice : public StreamInterface { - public: - // Use the default stream on the current device - CudaStreamDevice() : stream_(&default_stream) { - cudaGetDevice(&device_); - initializeDeviceProp(); - } - // Use the default stream on the specified device - CudaStreamDevice(int device) : stream_(&default_stream), device_(device) { - initializeDeviceProp(); - } - // Use the specified stream. Note that it's the - // caller responsibility to ensure that the stream can run on - // the specified device. If no device is specified the code - // assumes that the stream is associated to the current gpu device. - CudaStreamDevice(const cudaStream_t* stream, int device = -1) - : stream_(stream), device_(device) { - if (device < 0) { - cudaGetDevice(&device_); - } else { - int num_devices; - cudaError_t err = cudaGetDeviceCount(&num_devices); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); - assert(device < num_devices); - device_ = device; - } - initializeDeviceProp(); - } - - const cudaStream_t& stream() const { return *stream_; } - const cudaDeviceProp& deviceProperties() const { - return m_deviceProperties[device_]; - } - virtual void* allocate(size_t num_bytes) const { - cudaError_t err = cudaSetDevice(device_); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); - void* result; - err = cudaMalloc(&result, num_bytes); - assert(err == cudaSuccess); - assert(result != NULL); - return result; - } - virtual void deallocate(void* buffer) const { - cudaError_t err = cudaSetDevice(device_); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); - assert(buffer != NULL); - err = cudaFree(buffer); - assert(err == cudaSuccess); - } - - private: - const cudaStream_t* stream_; - int device_; -}; - -struct GpuDevice { - // The StreamInterface is not owned: the caller is - // responsible for its initialization and eventual destruction. - explicit GpuDevice(const StreamInterface* stream) : stream_(stream), max_blocks_(INT_MAX) { - eigen_assert(stream); - } - explicit GpuDevice(const StreamInterface* stream, int num_blocks) : stream_(stream), max_blocks_(num_blocks) { - eigen_assert(stream); - } - // TODO(bsteiner): This is an internal API, we should not expose it. - EIGEN_STRONG_INLINE const cudaStream_t& stream() const { - return stream_->stream(); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void* allocate(size_t num_bytes) const { -#ifndef __CUDA_ARCH__ - return stream_->allocate(num_bytes); -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return NULL; -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void deallocate(void* buffer) const { -#ifndef __CUDA_ARCH__ - stream_->deallocate(buffer); - -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpy(void* dst, const void* src, size_t n) const { -#ifndef __CUDA_ARCH__ - cudaError_t err = cudaMemcpyAsync(dst, src, n, cudaMemcpyDeviceToDevice, - stream_->stream()); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpyHostToDevice(void* dst, const void* src, size_t n) const { -#ifndef __CUDA_ARCH__ - cudaError_t err = - cudaMemcpyAsync(dst, src, n, cudaMemcpyHostToDevice, stream_->stream()); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpyDeviceToHost(void* dst, const void* src, size_t n) const { -#ifndef __CUDA_ARCH__ - cudaError_t err = - cudaMemcpyAsync(dst, src, n, cudaMemcpyDeviceToHost, stream_->stream()); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memset(void* buffer, int c, size_t n) const { -#ifndef __CUDA_ARCH__ - cudaError_t err = cudaMemsetAsync(buffer, c, n, stream_->stream()); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t numThreads() const { - // FIXME - return 32; - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const { - // FIXME - return 48*1024; - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t lastLevelCacheSize() const { - // We won't try to take advantage of the l2 cache for the time being, and - // there is no l3 cache on cuda devices. - return firstLevelCacheSize(); - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void synchronize() const { -#if defined(__CUDACC__) && !defined(__CUDA_ARCH__) - cudaError_t err = cudaStreamSynchronize(stream_->stream()); - if (err != cudaSuccess) { - std::cerr << "Error detected in CUDA stream: " - << cudaGetErrorString(err) - << std::endl; - assert(err == cudaSuccess); - } -#else - assert(false && "The default device should be used instead to generate kernel code"); -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int getNumCudaMultiProcessors() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().multiProcessorCount; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxCudaThreadsPerBlock() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().maxThreadsPerBlock; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxCudaThreadsPerMultiProcessor() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().maxThreadsPerMultiProcessor; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int sharedMemPerBlock() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().sharedMemPerBlock; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int majorDeviceVersion() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().major; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int minorDeviceVersion() const { -#ifndef __CUDA_ARCH__ - return stream_->deviceProperties().minor; -#else - eigen_assert(false && "The default device should be used instead to generate kernel code"); - return 0; -#endif - } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxBlocks() const { - return max_blocks_; - } - - // This function checks if the CUDA runtime recorded an error for the - // underlying stream device. - inline bool ok() const { -#ifdef __CUDACC__ - cudaError_t error = cudaStreamQuery(stream_->stream()); - return (error == cudaSuccess) || (error == cudaErrorNotReady); -#else - return false; -#endif - } - - private: - const StreamInterface* stream_; - int max_blocks_; -}; - -#define LAUNCH_CUDA_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \ - (kernel) <<< (gridsize), (blocksize), (sharedmem), (device).stream() >>> (__VA_ARGS__); \ - assert(cudaGetLastError() == cudaSuccess); - - -// FIXME: Should be device and kernel specific. -#ifdef __CUDACC__ -static EIGEN_DEVICE_FUNC inline void setCudaSharedMemConfig(cudaSharedMemConfig config) { -#ifndef __CUDA_ARCH__ - cudaError_t status = cudaDeviceSetSharedMemConfig(config); - EIGEN_UNUSED_VARIABLE(status) - assert(status == cudaSuccess); -#else - EIGEN_UNUSED_VARIABLE(config) -#endif -} +#if defined(__clang__) || defined(__GNUC__) +#warning "Deprecated header file, please either include the main Eigen/CXX11/Tensor header or the respective TensorDeviceGpu.h file" #endif -} // end namespace Eigen - -#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H +#include "TensorDeviceGpu.h"
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h index 9d14139..8cb95f7 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h
@@ -21,6 +21,12 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void deallocate(void* buffer) const { internal::aligned_free(buffer); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void* allocate_temp(size_t num_bytes) const { + return allocate(num_bytes); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void deallocate_temp(void* buffer) const { + deallocate(buffer); + } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpy(void* dst, const void* src, size_t n) const { ::memcpy(dst, src, n); } @@ -35,9 +41,12 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t numThreads() const { -#ifndef __CUDA_ARCH__ +#if !defined(EIGEN_GPU_COMPILE_PHASE) // Running on the host CPU return 1; +#elif defined(EIGEN_HIP_DEVICE_COMPILE) + // Running on a HIP device + return 64; #else // Running on a CUDA device return 32; @@ -45,9 +54,12 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const { -#ifndef __CUDA_ARCH__ +#if !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__) // Running on the host CPU return l1CacheSize(); +#elif defined(EIGEN_HIP_DEVICE_COMPILE) + // Running on a HIP device + return 48*1024; // FIXME : update this number for HIP #else // Running on a CUDA device, return the amount of shared memory available. return 48*1024; @@ -55,9 +67,12 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t lastLevelCacheSize() const { -#ifndef __CUDA_ARCH__ +#if !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__) // Running single threaded on the host CPU return l3CacheSize(); +#elif defined(EIGEN_HIP_DEVICE_COMPILE) + // Running on a HIP device + return firstLevelCacheSize(); // FIXME : update this number for HIP #else // Running on a CUDA device return firstLevelCacheSize(); @@ -65,13 +80,17 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int majorDeviceVersion() const { -#ifndef __CUDA_ARCH__ +#if !defined(EIGEN_GPU_COMPILE_PHASE) // Running single threaded on the host CPU // Should return an enum that encodes the ISA supported by the CPU return 1; +#elif defined(EIGEN_HIP_DEVICE_COMPILE) + // Running on a HIP device + // return 1 as major for HIP + return 1; #else // Running on a CUDA device - return __CUDA_ARCH__ / 100; + return EIGEN_CUDA_ARCH / 100; #endif } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h new file mode 100644 index 0000000..83cde6a --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h
@@ -0,0 +1,366 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H) +#define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H + +// This header file container defines fo gpu* macros which will resolve to +// their equivalent hip* or cuda* versions depending on the compiler in use +// A separate header (included at the end of this file) will undefine all +#include "TensorGpuHipCudaDefines.h" + +namespace Eigen { + +static const int kGpuScratchSize = 1024; + +// This defines an interface that GPUDevice can take to use +// HIP / CUDA streams underneath. +class StreamInterface { + public: + virtual ~StreamInterface() {} + + virtual const gpuStream_t& stream() const = 0; + virtual const gpuDeviceProp_t& deviceProperties() const = 0; + + // Allocate memory on the actual device where the computation will run + virtual void* allocate(size_t num_bytes) const = 0; + virtual void deallocate(void* buffer) const = 0; + + // Return a scratchpad buffer of size 1k + virtual void* scratchpad() const = 0; + + // Return a semaphore. The semaphore is initially initialized to 0, and + // each kernel using it is responsible for resetting to 0 upon completion + // to maintain the invariant that the semaphore is always equal to 0 upon + // each kernel start. + virtual unsigned int* semaphore() const = 0; +}; + +static gpuDeviceProp_t* m_deviceProperties; +static bool m_devicePropInitialized = false; + +static void initializeDeviceProp() { + if (!m_devicePropInitialized) { + // Attempts to ensure proper behavior in the case of multiple threads + // calling this function simultaneously. This would be trivial to + // implement if we could use std::mutex, but unfortunately mutex don't + // compile with nvcc, so we resort to atomics and thread fences instead. + // Note that if the caller uses a compiler that doesn't support c++11 we + // can't ensure that the initialization is thread safe. +#if __cplusplus >= 201103L + static std::atomic<bool> first(true); + if (first.exchange(false)) { +#else + static bool first = true; + if (first) { + first = false; +#endif + // We're the first thread to reach this point. + int num_devices; + gpuError_t status = gpuGetDeviceCount(&num_devices); + if (status != gpuSuccess) { + std::cerr << "Failed to get the number of GPU devices: " + << gpuGetErrorString(status) + << std::endl; + gpu_assert(status == gpuSuccess); + } + m_deviceProperties = new gpuDeviceProp_t[num_devices]; + for (int i = 0; i < num_devices; ++i) { + status = gpuGetDeviceProperties(&m_deviceProperties[i], i); + if (status != gpuSuccess) { + std::cerr << "Failed to initialize GPU device #" + << i + << ": " + << gpuGetErrorString(status) + << std::endl; + gpu_assert(status == gpuSuccess); + } + } + +#if __cplusplus >= 201103L + std::atomic_thread_fence(std::memory_order_release); +#endif + m_devicePropInitialized = true; + } else { + // Wait for the other thread to inititialize the properties. + while (!m_devicePropInitialized) { +#if __cplusplus >= 201103L + std::atomic_thread_fence(std::memory_order_acquire); +#endif + EIGEN_SLEEP(1000); + } + } + } +} + +static const gpuStream_t default_stream = gpuStreamDefault; + +class GpuStreamDevice : public StreamInterface { + public: + // Use the default stream on the current device + GpuStreamDevice() : stream_(&default_stream), scratch_(NULL), semaphore_(NULL) { + gpuGetDevice(&device_); + initializeDeviceProp(); + } + // Use the default stream on the specified device + GpuStreamDevice(int device) : stream_(&default_stream), device_(device), scratch_(NULL), semaphore_(NULL) { + initializeDeviceProp(); + } + // Use the specified stream. Note that it's the + // caller responsibility to ensure that the stream can run on + // the specified device. If no device is specified the code + // assumes that the stream is associated to the current gpu device. + GpuStreamDevice(const gpuStream_t* stream, int device = -1) + : stream_(stream), device_(device), scratch_(NULL), semaphore_(NULL) { + if (device < 0) { + gpuGetDevice(&device_); + } else { + int num_devices; + gpuError_t err = gpuGetDeviceCount(&num_devices); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + gpu_assert(device < num_devices); + device_ = device; + } + initializeDeviceProp(); + } + + virtual ~GpuStreamDevice() { + if (scratch_) { + deallocate(scratch_); + } + } + + const gpuStream_t& stream() const { return *stream_; } + const gpuDeviceProp_t& deviceProperties() const { + return m_deviceProperties[device_]; + } + virtual void* allocate(size_t num_bytes) const { + gpuError_t err = gpuSetDevice(device_); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + void* result; + err = gpuMalloc(&result, num_bytes); + gpu_assert(err == gpuSuccess); + gpu_assert(result != NULL); + return result; + } + virtual void deallocate(void* buffer) const { + gpuError_t err = gpuSetDevice(device_); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + gpu_assert(buffer != NULL); + err = gpuFree(buffer); + gpu_assert(err == gpuSuccess); + } + + virtual void* scratchpad() const { + if (scratch_ == NULL) { + scratch_ = allocate(kGpuScratchSize + sizeof(unsigned int)); + } + return scratch_; + } + + virtual unsigned int* semaphore() const { + if (semaphore_ == NULL) { + char* scratch = static_cast<char*>(scratchpad()) + kGpuScratchSize; + semaphore_ = reinterpret_cast<unsigned int*>(scratch); + gpuError_t err = gpuMemsetAsync(semaphore_, 0, sizeof(unsigned int), *stream_); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + } + return semaphore_; + } + + private: + const gpuStream_t* stream_; + int device_; + mutable void* scratch_; + mutable unsigned int* semaphore_; +}; + +struct GpuDevice { + // The StreamInterface is not owned: the caller is + // responsible for its initialization and eventual destruction. + explicit GpuDevice(const StreamInterface* stream) : stream_(stream), max_blocks_(INT_MAX) { + eigen_assert(stream); + } + explicit GpuDevice(const StreamInterface* stream, int num_blocks) : stream_(stream), max_blocks_(num_blocks) { + eigen_assert(stream); + } + // TODO(bsteiner): This is an internal API, we should not expose it. + EIGEN_STRONG_INLINE const gpuStream_t& stream() const { + return stream_->stream(); + } + + EIGEN_STRONG_INLINE void* allocate(size_t num_bytes) const { + return stream_->allocate(num_bytes); + } + + EIGEN_STRONG_INLINE void deallocate(void* buffer) const { + stream_->deallocate(buffer); + } + + EIGEN_STRONG_INLINE void* allocate_temp(size_t num_bytes) const { + return stream_->allocate(num_bytes); + } + + EIGEN_STRONG_INLINE void deallocate_temp(void* buffer) const { + stream_->deallocate(buffer); + } + + + EIGEN_STRONG_INLINE void* scratchpad() const { + return stream_->scratchpad(); + } + + EIGEN_STRONG_INLINE unsigned int* semaphore() const { + return stream_->semaphore(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpy(void* dst, const void* src, size_t n) const { +#ifndef EIGEN_GPU_COMPILE_PHASE + gpuError_t err = gpuMemcpyAsync(dst, src, n, gpuMemcpyDeviceToDevice, + stream_->stream()); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); +#else + EIGEN_UNUSED_VARIABLE(dst); + EIGEN_UNUSED_VARIABLE(src); + EIGEN_UNUSED_VARIABLE(n); + eigen_assert(false && "The default device should be used instead to generate kernel code"); +#endif + } + + EIGEN_STRONG_INLINE void memcpyHostToDevice(void* dst, const void* src, size_t n) const { + gpuError_t err = + gpuMemcpyAsync(dst, src, n, gpuMemcpyHostToDevice, stream_->stream()); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + } + + EIGEN_STRONG_INLINE void memcpyDeviceToHost(void* dst, const void* src, size_t n) const { + gpuError_t err = + gpuMemcpyAsync(dst, src, n, gpuMemcpyDeviceToHost, stream_->stream()); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memset(void* buffer, int c, size_t n) const { +#ifndef EIGEN_GPU_COMPILE_PHASE + gpuError_t err = gpuMemsetAsync(buffer, c, n, stream_->stream()); + EIGEN_UNUSED_VARIABLE(err) + gpu_assert(err == gpuSuccess); +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); +#endif + } + + EIGEN_STRONG_INLINE size_t numThreads() const { + // FIXME + return 32; + } + + EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const { + // FIXME + return 48*1024; + } + + EIGEN_STRONG_INLINE size_t lastLevelCacheSize() const { + // We won't try to take advantage of the l2 cache for the time being, and + // there is no l3 cache on hip/cuda devices. + return firstLevelCacheSize(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void synchronize() const { +#if defined(EIGEN_GPUCC) && !defined(EIGEN_GPU_COMPILE_PHASE) + gpuError_t err = gpuStreamSynchronize(stream_->stream()); + if (err != gpuSuccess) { + std::cerr << "Error detected in GPU stream: " + << gpuGetErrorString(err) + << std::endl; + gpu_assert(err == gpuSuccess); + } +#else + gpu_assert(false && "The default device should be used instead to generate kernel code"); +#endif + } + + EIGEN_STRONG_INLINE int getNumGpuMultiProcessors() const { + return stream_->deviceProperties().multiProcessorCount; + } + EIGEN_STRONG_INLINE int maxGpuThreadsPerBlock() const { + return stream_->deviceProperties().maxThreadsPerBlock; + } + EIGEN_STRONG_INLINE int maxGpuThreadsPerMultiProcessor() const { + return stream_->deviceProperties().maxThreadsPerMultiProcessor; + } + EIGEN_STRONG_INLINE int sharedMemPerBlock() const { + return stream_->deviceProperties().sharedMemPerBlock; + } + EIGEN_STRONG_INLINE int majorDeviceVersion() const { + return stream_->deviceProperties().major; + } + EIGEN_STRONG_INLINE int minorDeviceVersion() const { + return stream_->deviceProperties().minor; + } + + EIGEN_STRONG_INLINE int maxBlocks() const { + return max_blocks_; + } + + // This function checks if the GPU runtime recorded an error for the + // underlying stream device. + inline bool ok() const { +#ifdef EIGEN_GPUCC + gpuError_t error = gpuStreamQuery(stream_->stream()); + return (error == gpuSuccess) || (error == gpuErrorNotReady); +#else + return false; +#endif + } + + private: + const StreamInterface* stream_; + int max_blocks_; +}; + +#if defined(EIGEN_HIPCC) + +#define LAUNCH_GPU_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \ + hipLaunchKernelGGL(kernel, dim3(gridsize), dim3(blocksize), (sharedmem), (device).stream(), __VA_ARGS__); \ + gpu_assert(hipGetLastError() == hipSuccess); + +#else + +#define LAUNCH_GPU_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \ + (kernel) <<< (gridsize), (blocksize), (sharedmem), (device).stream() >>> (__VA_ARGS__); \ + gpu_assert(cudaGetLastError() == cudaSuccess); + +#endif + +// FIXME: Should be device and kernel specific. +#ifdef EIGEN_GPUCC +static EIGEN_DEVICE_FUNC inline void setGpuSharedMemConfig(gpuSharedMemConfig config) { +#ifndef EIGEN_GPU_COMPILE_PHASE + gpuError_t status = gpuDeviceSetSharedMemConfig(config); + EIGEN_UNUSED_VARIABLE(status) + gpu_assert(status == gpuSuccess); +#else + EIGEN_UNUSED_VARIABLE(config) +#endif +} +#endif + +} // end namespace Eigen + +// undefine all the gpu* macros we defined at the beginning of the file +#include "TensorGpuHipCudaUndefines.h" + +#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h new file mode 100644 index 0000000..e7beb2c --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h
@@ -0,0 +1,552 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> + +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_USE_SYCL) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_SYCL_H) +#define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_SYCL_H +template<size_t Align> struct CheckAlignStatically { + static const bool Val= (((Align&(Align-1))==0) && (Align >= sizeof(void *))); +}; +template <bool IsAligned, size_t Align> +struct Conditional_Allocate { + + EIGEN_ALWAYS_INLINE static void* conditional_allocate(std::size_t elements) { + return aligned_alloc(Align, elements); + } +}; +template <size_t Align> +struct Conditional_Allocate<false, Align> { + + EIGEN_ALWAYS_INLINE static void* conditional_allocate(std::size_t elements){ + return malloc(elements); + } +}; +template <typename Scalar, size_t Align = EIGEN_MAX_ALIGN_BYTES, class Allocator = std::allocator<Scalar>> +struct SyclAllocator { + typedef Scalar value_type; + typedef typename std::allocator_traits<Allocator>::pointer pointer; + typedef typename std::allocator_traits<Allocator>::size_type size_type; + + SyclAllocator( ){}; + Scalar* allocate(std::size_t elements) { + return static_cast<Scalar*>(Conditional_Allocate<CheckAlignStatically<Align>::Val, Align>::conditional_allocate(elements)); + } + void deallocate(Scalar * p, std::size_t size) { EIGEN_UNUSED_VARIABLE(size); free(p); } +}; + +namespace Eigen { + +#define ConvertToActualTypeSycl(Scalar, buf_acc) static_cast<Scalar*>(static_cast<void*>(((buf_acc.get_pointer().get())))) +#define ConvertToActualSyclOffset(Scalar, offset) offset/sizeof(Scalar) + + + template <typename Scalar, typename read_accessor, typename write_accessor> class MemCopyFunctor { + public: + MemCopyFunctor(read_accessor src_acc, write_accessor dst_acc, size_t rng, size_t i, size_t offset) : m_src_acc(src_acc), m_dst_acc(dst_acc), m_rng(rng), m_i(i), m_offset(offset) {} + + void operator()(cl::sycl::nd_item<1> itemID) { + auto src_ptr = ConvertToActualTypeSycl(Scalar, m_src_acc); + auto dst_ptr = ConvertToActualTypeSycl(Scalar, m_dst_acc); + auto globalid = itemID.get_global_linear_id(); + if (globalid < m_rng) { + dst_ptr[globalid + m_i] = src_ptr[globalid + m_offset]; + } + } + + private: + read_accessor m_src_acc; + write_accessor m_dst_acc; + size_t m_rng; + size_t m_i; + size_t m_offset; + }; + +template<typename AccType> + struct memsetkernelFunctor{ + AccType m_acc; + const ptrdiff_t buff_offset; + const size_t m_rng, m_c; + memsetkernelFunctor(AccType acc, const ptrdiff_t buff_offset_, const size_t rng, const size_t c):m_acc(acc), buff_offset(buff_offset_), m_rng(rng), m_c(c){} + void operator()(cl::sycl::nd_item<1> itemID) { + auto globalid=itemID.get_global_linear_id(); + if (globalid< m_rng) m_acc[globalid + buff_offset] = m_c; + } + + }; + +struct memsetCghFunctor{ + cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> >& m_buf; + const ptrdiff_t& buff_offset; + const size_t& rng , GRange, tileSize; + const int &c; + memsetCghFunctor(cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> >& buff, const ptrdiff_t& buff_offset_, const size_t& rng_, const size_t& GRange_, const size_t& tileSize_, const int& c_) + :m_buf(buff), buff_offset(buff_offset_), rng(rng_), GRange(GRange_), tileSize(tileSize_), c(c_){} + + void operator()(cl::sycl::handler &cgh) const { + auto buf_acc = m_buf.template get_access<cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer>(cgh); + typedef decltype(buf_acc) AccType; + cgh.parallel_for(cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), memsetkernelFunctor<AccType>(buf_acc, buff_offset, rng, c)); + } +}; + +//get_devices returns all the available opencl devices. Either use device_selector or exclude devices that computecpp does not support (AMD OpenCL for CPU and intel GPU) +EIGEN_STRONG_INLINE auto get_sycl_supported_devices()->decltype(cl::sycl::device::get_devices()){ +std::vector<cl::sycl::device> supported_devices; +auto plafrom_list =cl::sycl::platform::get_platforms(); +for(const auto& platform : plafrom_list){ + auto device_list = platform.get_devices(); + auto platform_name =platform.template get_info<cl::sycl::info::platform::name>(); + std::transform(platform_name.begin(), platform_name.end(), platform_name.begin(), ::tolower); + for(const auto& device : device_list){ + auto vendor = device.template get_info<cl::sycl::info::device::vendor>(); + std::transform(vendor.begin(), vendor.end(), vendor.begin(), ::tolower); + bool unsuported_condition = (device.is_cpu() && platform_name.find("amd")!=std::string::npos && vendor.find("apu") == std::string::npos) || + (device.is_gpu() && platform_name.find("intel")!=std::string::npos); + if(!unsuported_condition){ + std::cout << "Platform name "<< platform_name << std::endl; + supported_devices.push_back(device); + } + } +} +return supported_devices; +} + +class QueueInterface { +public: + /// creating device by using cl::sycl::selector or cl::sycl::device both are the same and can be captured through dev_Selector typename + /// SyclStreamDevice is not owned. it is the caller's responsibility to destroy it. + template<typename dev_Selector> explicit QueueInterface(const dev_Selector& s): +#ifdef EIGEN_EXCEPTIONS + m_queue(cl::sycl::queue(s, [&](cl::sycl::exception_list l) { + for (const auto& e : l) { + try { + if (e) { + exception_caught_ = true; + std::rethrow_exception(e); + } + } catch (cl::sycl::exception e) { + std::cerr << e.what() << std::endl; + } + } + })) +#else +m_queue(cl::sycl::queue(s, [&](cl::sycl::exception_list l) { + for (const auto& e : l) { + if (e) { + exception_caught_ = true; + std::cerr << "Error detected Inside Sycl Device."<< std::endl; + + } + } +})) +#endif + {} + + /// Allocating device pointer. This pointer is actually an 8 bytes host pointer used as key to access the sycl device buffer. + /// The reason is that we cannot use device buffer as a pointer as a m_data in Eigen leafNode expressions. So we create a key + /// pointer to be used in Eigen expression construction. When we convert the Eigen construction into the sycl construction we + /// use this pointer as a key in our buffer_map and we make sure that we dedicate only one buffer only for this pointer. + /// The device pointer would be deleted by calling deallocate function. + EIGEN_STRONG_INLINE void* allocate(size_t num_bytes) const { + std::lock_guard<std::mutex> lock(mutex_); + auto buf = cl::sycl::buffer<uint8_t,1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> >(cl::sycl::range<1>(num_bytes)); + auto ptr =buf.get_access<cl::sycl::access::mode::discard_write, cl::sycl::access::target::host_buffer>().get_pointer(); + buf.set_final_data(nullptr); + buffer_map.insert(std::pair<const uint8_t *, cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> > >(static_cast<const uint8_t*>(ptr),buf)); + return static_cast<void*>(ptr); + } + + /// This is used to deallocate the device pointer. p is used as a key inside + /// the map to find the device buffer and delete it. + EIGEN_STRONG_INLINE void deallocate(void *p) const { + std::lock_guard<std::mutex> lock(mutex_); + auto it = buffer_map.find(static_cast<const uint8_t*>(p)); + if (it != buffer_map.end()) { + buffer_map.erase(it); + } + } + + EIGEN_STRONG_INLINE void deallocate_all() const { + std::lock_guard<std::mutex> lock(mutex_); + buffer_map.clear(); + } + /// The memcpyHostToDevice is used to copy the device only pointer to a host pointer. Using the device + /// pointer created as a key we find the sycl buffer and get the host accessor with write mode + /// on it. Then we use the memcpy to copy the data to the host accessor. The first time that + /// this buffer is accessed, the data will be copied to the device. + /// In this case we can separate the kernel actual execution from data transfer which is required for benchmark + /// Also, this is faster as it uses the map_allocator instead of memcpy + template<typename Index> EIGEN_STRONG_INLINE void memcpyHostToDevice(Index *dst, const Index *src, size_t n) const { + auto it =find_buffer(dst); + auto offset =static_cast<const uint8_t*>(static_cast<const void*>(dst))- it->first; + offset/=sizeof(Index); + size_t rng, GRange, tileSize; + parallel_for_setup(n/sizeof(Index), tileSize, rng, GRange); + auto src_buf = cl::sycl::buffer<uint8_t, 1, cl::sycl::map_allocator<uint8_t> >(static_cast<uint8_t*>(static_cast<void*>(const_cast<Index*>(src))), cl::sycl::range<1>(n)); + m_queue.submit([&](cl::sycl::handler &cgh) { + auto dst_acc= it->second.template get_access<cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer>(cgh); + auto src_acc =src_buf.template get_access<cl::sycl::access::mode::read, cl::sycl::access::target::global_buffer>(cgh); + typedef decltype(src_acc) read_accessor; + typedef decltype(dst_acc) write_accessor; + cgh.parallel_for( cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), MemCopyFunctor<Index, read_accessor, write_accessor>(src_acc, dst_acc, rng, offset, 0)); + }); + synchronize(); + } + /// The memcpyDeviceToHost is used to copy the data from host to device. Here, in order to avoid double copying the data. We create a sycl + /// buffer with map_allocator for the destination pointer with a discard_write accessor on it. The lifespan of the buffer is bound to the + /// lifespan of the memcpyDeviceToHost function. We create a kernel to copy the data, from the device- only source buffer to the destination + /// buffer with map_allocator on the gpu in parallel. At the end of the function call the destination buffer would be destroyed and the data + /// would be available on the dst pointer using fast copy technique (map_allocator). In this case we can make sure that we copy the data back + /// to the cpu only once per function call. + template<typename Index> EIGEN_STRONG_INLINE void memcpyDeviceToHost(void *dst, const Index *src, size_t n) const { + auto it =find_buffer(src); + auto offset =static_cast<const uint8_t*>(static_cast<const void*>(src))- it->first; + offset/=sizeof(Index); + size_t rng, GRange, tileSize; + parallel_for_setup(n/sizeof(Index), tileSize, rng, GRange); + auto dest_buf = cl::sycl::buffer<uint8_t, 1, cl::sycl::map_allocator<uint8_t> >(static_cast<uint8_t*>(dst), cl::sycl::range<1>(n)); + m_queue.submit([&](cl::sycl::handler &cgh) { + auto src_acc= it->second.template get_access<cl::sycl::access::mode::read, cl::sycl::access::target::global_buffer>(cgh); + auto dst_acc =dest_buf.template get_access<cl::sycl::access::mode::discard_write, cl::sycl::access::target::global_buffer>(cgh); + typedef decltype(src_acc) read_accessor; + typedef decltype(dst_acc) write_accessor; + cgh.parallel_for( cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), MemCopyFunctor<Index, read_accessor, write_accessor>(src_acc, dst_acc, rng, 0, offset)); + }); + synchronize(); + } + + /// the memcpy function + template<typename Index> EIGEN_STRONG_INLINE void memcpy(void *dst, const Index *src, size_t n) const { + auto it1 = find_buffer(static_cast<const void*>(src)); + auto it2 = find_buffer(dst); + auto offset= (static_cast<const uint8_t*>(static_cast<const void*>(src))) - it1->first; + auto i= (static_cast<const uint8_t*>(dst)) - it2->first; + offset/=sizeof(Index); + i/=sizeof(Index); + size_t rng, GRange, tileSize; + parallel_for_setup(n/sizeof(Index), tileSize, rng, GRange); + m_queue.submit([&](cl::sycl::handler &cgh) { + auto src_acc =it1->second.template get_access<cl::sycl::access::mode::read, cl::sycl::access::target::global_buffer>(cgh); + auto dst_acc =it2->second.template get_access<cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer>(cgh); + typedef decltype(src_acc) read_accessor; + typedef decltype(dst_acc) write_accessor; + cgh.parallel_for(cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), MemCopyFunctor<Index, read_accessor, write_accessor>(src_acc, dst_acc, rng, i, offset)); + }); + synchronize(); + } + + EIGEN_STRONG_INLINE void memset(void *data, int c, size_t n) const { + size_t rng, GRange, tileSize; + parallel_for_setup(n, tileSize, rng, GRange); + auto it1 = find_buffer(static_cast<const void*>(data)); + ptrdiff_t buff_offset= (static_cast<const uint8_t*>(data)) - it1->first; + m_queue.submit(memsetCghFunctor(it1->second, buff_offset, rng, GRange, tileSize, c )); + synchronize(); + } + + /// Creation of sycl accessor for a buffer. This function first tries to find + /// the buffer in the buffer_map. If found it gets the accessor from it, if not, + /// the function then adds an entry by creating a sycl buffer for that particular pointer. + template <cl::sycl::access::mode AcMd> EIGEN_STRONG_INLINE cl::sycl::accessor<uint8_t, 1, AcMd, cl::sycl::access::target::global_buffer> + get_sycl_accessor(cl::sycl::handler &cgh, const void* ptr) const { + return (find_buffer(ptr)->second.template get_access<AcMd, cl::sycl::access::target::global_buffer>(cgh)); + } + + /// Accessing the created sycl device buffer for the device pointer + EIGEN_STRONG_INLINE cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> >& get_sycl_buffer(const void * ptr) const { + return find_buffer(ptr)->second; + } + + EIGEN_STRONG_INLINE ptrdiff_t get_offset(const void *ptr) const { + return (static_cast<const uint8_t*>(ptr))-(find_buffer(ptr)->first); + } + + EIGEN_STRONG_INLINE void synchronize() const { + m_queue.wait_and_throw(); //pass + } + + EIGEN_STRONG_INLINE void asynchronousExec() const { + ///FIXEDME:: currently there is a race condition regarding the asynch scheduler. + //sycl_queue().throw_asynchronous();// FIXME::does not pass. Temporarily disabled + m_queue.wait_and_throw(); //pass + } + + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index n, Index &tileSize, Index &rng, Index &GRange) const { + tileSize =static_cast<Index>(m_queue.get_device(). template get_info<cl::sycl::info::device::max_work_group_size>()); + auto s= m_queue.get_device().template get_info<cl::sycl::info::device::vendor>(); + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + if(m_queue.get_device().is_cpu()){ // intel doesn't allow to use max workgroup size + tileSize=std::min(static_cast<Index>(256), static_cast<Index>(tileSize)); + } + rng = n; + if (rng==0) rng=static_cast<Index>(1); + GRange=rng; + if (tileSize>GRange) tileSize=GRange; + else if(GRange>tileSize){ + Index xMode = static_cast<Index>(GRange % tileSize); + if (xMode != 0) GRange += static_cast<Index>(tileSize - xMode); + } + } + + /// This is used to prepare the number of threads and also the number of threads per block for sycl kernels + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index dim0, Index dim1, Index &tileSize0, Index &tileSize1, Index &rng0, Index &rng1, Index &GRange0, Index &GRange1) const { + Index max_workgroup_Size = static_cast<Index>(maxSyclThreadsPerBlock()); + if(m_queue.get_device().is_cpu()){ // intel doesn't allow to use max workgroup size + max_workgroup_Size=std::min(static_cast<Index>(256), static_cast<Index>(max_workgroup_Size)); + } + Index pow_of_2 = static_cast<Index>(std::log2(max_workgroup_Size)); + tileSize1 =static_cast<Index>(std::pow(2, static_cast<Index>(pow_of_2/2))); + rng1=dim1; + if (rng1==0 ) rng1=static_cast<Index>(1); + GRange1=rng1; + if (tileSize1>GRange1) tileSize1=GRange1; + else if(GRange1>tileSize1){ + Index xMode = static_cast<Index>(GRange1 % tileSize1); + if (xMode != 0) GRange1 += static_cast<Index>(tileSize1 - xMode); + } + tileSize0 = static_cast<Index>(max_workgroup_Size/tileSize1); + rng0 = dim0; + if (rng0==0 ) rng0=static_cast<Index>(1); + GRange0=rng0; + if (tileSize0>GRange0) tileSize0=GRange0; + else if(GRange0>tileSize0){ + Index xMode = static_cast<Index>(GRange0 % tileSize0); + if (xMode != 0) GRange0 += static_cast<Index>(tileSize0 - xMode); + } + } + + /// This is used to prepare the number of threads and also the number of threads per block for sycl kernels + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index dim0, Index dim1,Index dim2, Index &tileSize0, Index &tileSize1, Index &tileSize2, Index &rng0, Index &rng1, Index &rng2, Index &GRange0, Index &GRange1, Index &GRange2) const { + Index max_workgroup_Size = static_cast<Index>(maxSyclThreadsPerBlock()); + if(m_queue.get_device().is_cpu()){ // intel doesn't allow to use max workgroup size + max_workgroup_Size=std::min(static_cast<Index>(256), static_cast<Index>(max_workgroup_Size)); + } + Index pow_of_2 = static_cast<Index>(std::log2(max_workgroup_Size)); + tileSize2 =static_cast<Index>(std::pow(2, static_cast<Index>(pow_of_2/3))); + rng2=dim2; + if (rng2==0 ) rng1=static_cast<Index>(1); + GRange2=rng2; + if (tileSize2>GRange2) tileSize2=GRange2; + else if(GRange2>tileSize2){ + Index xMode = static_cast<Index>(GRange2 % tileSize2); + if (xMode != 0) GRange2 += static_cast<Index>(tileSize2 - xMode); + } + pow_of_2 = static_cast<Index>(std::log2(static_cast<Index>(max_workgroup_Size/tileSize2))); + tileSize1 =static_cast<Index>(std::pow(2, static_cast<Index>(pow_of_2/2))); + rng1=dim1; + if (rng1==0 ) rng1=static_cast<Index>(1); + GRange1=rng1; + if (tileSize1>GRange1) tileSize1=GRange1; + else if(GRange1>tileSize1){ + Index xMode = static_cast<Index>(GRange1 % tileSize1); + if (xMode != 0) GRange1 += static_cast<Index>(tileSize1 - xMode); + } + tileSize0 = static_cast<Index>(max_workgroup_Size/(tileSize1*tileSize2)); + rng0 = dim0; + if (rng0==0 ) rng0=static_cast<Index>(1); + GRange0=rng0; + if (tileSize0>GRange0) tileSize0=GRange0; + else if(GRange0>tileSize0){ + Index xMode = static_cast<Index>(GRange0 % tileSize0); + if (xMode != 0) GRange0 += static_cast<Index>(tileSize0 - xMode); + } + } + + EIGEN_STRONG_INLINE unsigned long getNumSyclMultiProcessors() const { + return m_queue.get_device(). template get_info<cl::sycl::info::device::max_compute_units>(); + } + + EIGEN_STRONG_INLINE unsigned long maxSyclThreadsPerBlock() const { + return m_queue.get_device(). template get_info<cl::sycl::info::device::max_work_group_size>(); + } + + /// No need for sycl it should act the same as CPU version + EIGEN_STRONG_INLINE int majorDeviceVersion() const { return 1; } + + EIGEN_STRONG_INLINE unsigned long maxSyclThreadsPerMultiProcessor() const { + // OpenCL doesn't have such concept + return 2; + } + + EIGEN_STRONG_INLINE size_t sharedMemPerBlock() const { + return m_queue.get_device(). template get_info<cl::sycl::info::device::local_mem_size>(); + } + + EIGEN_STRONG_INLINE cl::sycl::queue& sycl_queue() const { return m_queue;} + + // This function checks if the runtime recorded an error for the + // underlying stream device. + EIGEN_STRONG_INLINE bool ok() const { + if (!exception_caught_) { + m_queue.wait_and_throw(); + } + return !exception_caught_; + } + + // destructor + ~QueueInterface() { buffer_map.clear(); } + +private: + /// class members: + bool exception_caught_ = false; + + mutable std::mutex mutex_; + + /// std::map is the container used to make sure that we create only one buffer + /// per pointer. The lifespan of the buffer now depends on the lifespan of SyclDevice. + /// If a non-read-only pointer is needed to be accessed on the host we should manually deallocate it. + mutable std::map<const uint8_t *, cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> > > buffer_map; + /// sycl queue + mutable cl::sycl::queue m_queue; + + EIGEN_STRONG_INLINE std::map<const uint8_t *, cl::sycl::buffer<uint8_t,1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> > >::iterator find_buffer(const void* ptr) const { + std::lock_guard<std::mutex> lock(mutex_); + auto it1 = buffer_map.find(static_cast<const uint8_t*>(ptr)); + if (it1 != buffer_map.end()){ + return it1; + } + else{ + for(std::map<const uint8_t *, cl::sycl::buffer<uint8_t,1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> > >::iterator it=buffer_map.begin(); it!=buffer_map.end(); ++it){ + auto size = it->second.get_size(); + if((it->first < (static_cast<const uint8_t*>(ptr))) && ((static_cast<const uint8_t*>(ptr)) < (it->first + size)) ) return it; + } + } + std::cerr << "No sycl buffer found. Make sure that you have allocated memory for your buffer by calling malloc-ed function."<< std::endl; + abort(); + } +}; + +// Here is a sycl deviuce struct which accept the sycl queue interface +// as an input +struct SyclDevice { + // class member. + QueueInterface* m_queue_stream; + /// QueueInterface is not owned. it is the caller's responsibility to destroy it. + explicit SyclDevice(QueueInterface* queue_stream) : m_queue_stream(queue_stream){} + + // get sycl accessor + template <cl::sycl::access::mode AcMd> EIGEN_STRONG_INLINE cl::sycl::accessor<uint8_t, 1, AcMd, cl::sycl::access::target::global_buffer> + get_sycl_accessor(cl::sycl::handler &cgh, const void* ptr) const { + return m_queue_stream->template get_sycl_accessor<AcMd>(cgh, ptr); + } + + /// Accessing the created sycl device buffer for the device pointer + EIGEN_STRONG_INLINE cl::sycl::buffer<uint8_t, 1, SyclAllocator<uint8_t, EIGEN_MAX_ALIGN_BYTES> >& get_sycl_buffer(const void * ptr) const { + return m_queue_stream->get_sycl_buffer(ptr); + } + + /// This is used to prepare the number of threads and also the number of threads per block for sycl kernels + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index n, Index &tileSize, Index &rng, Index &GRange) const { + m_queue_stream->parallel_for_setup(n, tileSize, rng, GRange); + } + + /// This is used to prepare the number of threads and also the number of threads per block for sycl kernels + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index dim0, Index dim1, Index &tileSize0, Index &tileSize1, Index &rng0, Index &rng1, Index &GRange0, Index &GRange1) const { + m_queue_stream->parallel_for_setup(dim0, dim1, tileSize0, tileSize1, rng0, rng1, GRange0, GRange1); + } + + /// This is used to prepare the number of threads and also the number of threads per block for sycl kernels + template<typename Index> + EIGEN_STRONG_INLINE void parallel_for_setup(Index dim0, Index dim1,Index dim2, Index &tileSize0, Index &tileSize1, Index &tileSize2, Index &rng0, Index &rng1, Index &rng2, Index &GRange0, Index &GRange1, Index &GRange2) const { + m_queue_stream->parallel_for_setup(dim0, dim1, dim2, tileSize0, tileSize1, tileSize2, rng0, rng1, rng2, GRange0, GRange1, GRange2); + + } + /// allocate device memory + EIGEN_STRONG_INLINE void *allocate(size_t num_bytes) const { + return m_queue_stream->allocate(num_bytes); + } + /// deallocate device memory + EIGEN_STRONG_INLINE void deallocate(void *p) const { + m_queue_stream->deallocate(p); + } + + // some runtime conditions that can be applied here + EIGEN_STRONG_INLINE bool isDeviceSuitable() const { return true; } + + /// the memcpy function + template<typename Index> EIGEN_STRONG_INLINE void memcpy(void *dst, const Index *src, size_t n) const { + m_queue_stream->memcpy(dst,src,n); + } + + EIGEN_STRONG_INLINE ptrdiff_t get_offset(const void *ptr) const { + return m_queue_stream->get_offset(ptr); + + } +// memcpyHostToDevice + template<typename Index> EIGEN_STRONG_INLINE void memcpyHostToDevice(Index *dst, const Index *src, size_t n) const { + m_queue_stream->memcpyHostToDevice(dst,src,n); + } +/// here is the memcpyDeviceToHost + template<typename Index> EIGEN_STRONG_INLINE void memcpyDeviceToHost(void *dst, const Index *src, size_t n) const { + m_queue_stream->memcpyDeviceToHost(dst,src,n); + } + /// Here is the implementation of memset function on sycl. + EIGEN_STRONG_INLINE void memset(void *data, int c, size_t n) const { + m_queue_stream->memset(data,c,n); + } + /// returning the sycl queue + EIGEN_STRONG_INLINE cl::sycl::queue& sycl_queue() const { return m_queue_stream->sycl_queue();} + + EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const { + // FIXME + return 48*1024; + } + + EIGEN_STRONG_INLINE size_t lastLevelCacheSize() const { + // We won't try to take advantage of the l2 cache for the time being, and + // there is no l3 cache on sycl devices. + return firstLevelCacheSize(); + } + EIGEN_STRONG_INLINE unsigned long getNumSyclMultiProcessors() const { + return m_queue_stream->getNumSyclMultiProcessors(); + } + EIGEN_STRONG_INLINE unsigned long maxSyclThreadsPerBlock() const { + return m_queue_stream->maxSyclThreadsPerBlock(); + } + EIGEN_STRONG_INLINE unsigned long maxSyclThreadsPerMultiProcessor() const { + // OpenCL doesn't have such concept + return m_queue_stream->maxSyclThreadsPerMultiProcessor(); + // return stream_->deviceProperties().maxThreadsPerMultiProcessor; + } + EIGEN_STRONG_INLINE size_t sharedMemPerBlock() const { + return m_queue_stream->sharedMemPerBlock(); + } + /// No need for sycl it should act the same as CPU version + EIGEN_STRONG_INLINE int majorDeviceVersion() const { return m_queue_stream->majorDeviceVersion(); } + + EIGEN_STRONG_INLINE void synchronize() const { + m_queue_stream->synchronize(); //pass + } + + EIGEN_STRONG_INLINE void asynchronousExec() const { + m_queue_stream->asynchronousExec(); + } + // This function checks if the runtime recorded an error for the + // underlying stream device. + EIGEN_STRONG_INLINE bool ok() const { + return m_queue_stream->ok(); + } +}; +// This is used as a distingushable device inside the kernel as the sycl device class is not Standard layout. +// This is internal and must not be used by user. This dummy device allow us to specialise the tensor evaluator +// inside the kernel. So we can have two types of eval for host and device. This is required for TensorArgMax operation +struct SyclKernelDevice:DefaultDevice{}; + +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_SYCL_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h index c028914..fb34cd7 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h
@@ -12,67 +12,6 @@ namespace Eigen { -// Use the SimpleThreadPool by default. We'll switch to the new non blocking -// thread pool later. -#ifdef EIGEN_USE_NONBLOCKING_THREAD_POOL -template <typename Env> using ThreadPoolTempl = NonBlockingThreadPoolTempl<Env>; -typedef NonBlockingThreadPool ThreadPool; -#else -template <typename Env> using ThreadPoolTempl = SimpleThreadPoolTempl<Env>; -typedef SimpleThreadPool ThreadPool; -#endif - - -// Barrier is an object that allows one or more threads to wait until -// Notify has been called a specified number of times. -class Barrier { - public: - Barrier(unsigned int count) : state_(count << 1), notified_(false) { - eigen_assert(((count << 1) >> 1) == count); - } - ~Barrier() { - eigen_assert((state_>>1) == 0); - } - - void Notify() { - unsigned int v = state_.fetch_sub(2, std::memory_order_acq_rel) - 2; - if (v != 1) { - eigen_assert(((v + 2) & ~1) != 0); - return; // either count has not dropped to 0, or waiter is not waiting - } - std::unique_lock<std::mutex> l(mu_); - eigen_assert(!notified_); - notified_ = true; - cv_.notify_all(); - } - - void Wait() { - unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel); - if ((v >> 1) == 0) return; - std::unique_lock<std::mutex> l(mu_); - while (!notified_) { - cv_.wait(l); - } - } - - private: - std::mutex mu_; - std::condition_variable cv_; - std::atomic<unsigned int> state_; // low bit is waiter flag - bool notified_; -}; - - -// Notification is an object that allows a user to to wait for another -// thread to signal a notification that an event has occurred. -// -// Multiple threads can wait on the same Notification object, -// but only one caller must call Notify() on the object. -struct Notification : Barrier { - Notification() : Barrier(1) {}; -}; - - // Runs an arbitrary function and then calls Notify() on the passed in // Notification. template <typename Function, typename... Args> struct FunctionWrapperWithNotification @@ -102,22 +41,71 @@ } } +// An abstract interface to a device specific memory allocator. +class Allocator { + public: + virtual ~Allocator() {} + EIGEN_DEVICE_FUNC virtual void* allocate(size_t num_bytes) const = 0; + EIGEN_DEVICE_FUNC virtual void deallocate(void* buffer) const = 0; +}; // Build a thread pool device on top the an existing pool of threads. struct ThreadPoolDevice { // The ownership of the thread pool remains with the caller. - ThreadPoolDevice(ThreadPoolInterface* pool, size_t num_cores) : pool_(pool), num_threads_(num_cores) { } + ThreadPoolDevice(ThreadPoolInterface* pool, int num_cores, Allocator* allocator = NULL) + : pool_(pool), num_threads_(num_cores), allocator_(allocator) { } EIGEN_STRONG_INLINE void* allocate(size_t num_bytes) const { - return internal::aligned_malloc(num_bytes); + return allocator_ ? allocator_->allocate(num_bytes) + : internal::aligned_malloc(num_bytes); } EIGEN_STRONG_INLINE void deallocate(void* buffer) const { - internal::aligned_free(buffer); + if (allocator_) { + allocator_->deallocate(buffer); + } else { + internal::aligned_free(buffer); + } + } + + EIGEN_STRONG_INLINE void* allocate_temp(size_t num_bytes) const { + return allocate(num_bytes); + } + + EIGEN_STRONG_INLINE void deallocate_temp(void* buffer) const { + deallocate(buffer); } EIGEN_STRONG_INLINE void memcpy(void* dst, const void* src, size_t n) const { +#ifdef __ANDROID__ ::memcpy(dst, src, n); +#else + // TODO(rmlarsen): Align blocks on cache lines. + // We have observed that going beyond 4 threads usually just wastes + // CPU cycles due to the threads competing for memory bandwidth, so we + // statically schedule at most 4 block copies here. + const size_t kMinBlockSize = 32768; + typedef TensorCostModel<ThreadPoolDevice> CostModel; + const size_t num_threads = CostModel::numThreads(n, TensorOpCost(1.0, 1.0, 0), 4); + if (n <= kMinBlockSize || num_threads < 2) { + ::memcpy(dst, src, n); + } else { + const char* src_ptr = static_cast<const char*>(src); + char* dst_ptr = static_cast<char*>(dst); + const size_t blocksize = (n + (num_threads - 1)) / num_threads; + Barrier barrier(static_cast<int>(num_threads - 1)); + // Launch the last 3 blocks on worker threads. + for (size_t i = 1; i < num_threads; ++i) { + enqueue_with_barrier(&barrier, [n, i, src_ptr, dst_ptr, blocksize] { + ::memcpy(dst_ptr + i * blocksize, src_ptr + i * blocksize, + numext::mini(blocksize, n - (i * blocksize))); + }); + } + // Launch the first block on the main thread. + ::memcpy(dst_ptr, src_ptr, blocksize); + barrier.Wait(); + } +#endif } EIGEN_STRONG_INLINE void memcpyHostToDevice(void* dst, const void* src, size_t n) const { memcpy(dst, src, n); @@ -130,10 +118,16 @@ ::memset(buffer, c, n); } - EIGEN_STRONG_INLINE size_t numThreads() const { + EIGEN_STRONG_INLINE int numThreads() const { return num_threads_; } + // Number of theads available in the underlying thread pool. This number can + // be different from the value returned by numThreads(). + EIGEN_STRONG_INLINE int numThreadsInPool() const { + return pool_->NumThreads(); + } + EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const { return l1CacheSize(); } @@ -149,32 +143,153 @@ } template <class Function, class... Args> - EIGEN_STRONG_INLINE Notification* enqueue(Function&& f, Args&&... args) const { + EIGEN_STRONG_INLINE Notification* enqueue(Function&& f, + Args&&... args) const { Notification* n = new Notification(); - std::function<void()> func = - std::bind(&FunctionWrapperWithNotification<Function, Args...>::run, n, f, args...); - pool_->Schedule(func); + pool_->Schedule( + std::bind(&FunctionWrapperWithNotification<Function, Args...>::run, n, + std::move(f), args...)); return n; } template <class Function, class... Args> - EIGEN_STRONG_INLINE void enqueue_with_barrier(Barrier* b, - Function&& f, + EIGEN_STRONG_INLINE void enqueue_with_barrier(Barrier* b, Function&& f, Args&&... args) const { - std::function<void()> func = std::bind( - &FunctionWrapperWithBarrier<Function, Args...>::run, b, f, args...); - pool_->Schedule(func); + pool_->Schedule( + std::bind(&FunctionWrapperWithBarrier<Function, Args...>::run, b, + std::move(f), args...)); } template <class Function, class... Args> - EIGEN_STRONG_INLINE void enqueueNoNotification(Function&& f, Args&&... args) const { - std::function<void()> func = std::bind(f, args...); - pool_->Schedule(func); + EIGEN_STRONG_INLINE void enqueueNoNotification(Function&& f, + Args&&... args) const { + if (sizeof...(args) > 0) { + pool_->Schedule(std::bind(std::move(f), args...)); + } else { + pool_->Schedule(std::move(f)); + } } + // Returns a logical thread index between 0 and pool_->NumThreads() - 1 if + // called from one of the threads in pool_. Returns -1 otherwise. + EIGEN_STRONG_INLINE int currentThreadId() const { + return pool_->CurrentThreadId(); + } + + // parallelFor executes f with [0, n) arguments in parallel and waits for + // completion. F accepts a half-open interval [first, last). + // Block size is chosen based on the iteration cost and resulting parallel + // efficiency. If block_align is not nullptr, it is called to round up the + // block size. + void parallelFor(Index n, const TensorOpCost& cost, + std::function<Index(Index)> block_align, + std::function<void(Index, Index)> f) const { + typedef TensorCostModel<ThreadPoolDevice> CostModel; + if (n <= 1 || numThreads() == 1 || + CostModel::numThreads(n, cost, static_cast<int>(numThreads())) == 1) { + f(0, n); + return; + } + + // Calculate block size based on (1) the iteration cost and (2) parallel + // efficiency. We want blocks to be not too small to mitigate + // parallelization overheads; not too large to mitigate tail + // effect and potential load imbalance and we also want number + // of blocks to be evenly dividable across threads. + + double block_size_f = 1.0 / CostModel::taskSize(1, cost); + const Index max_oversharding_factor = 4; + Index block_size = numext::mini( + n, numext::maxi<Index>(divup<Index>(n, max_oversharding_factor * numThreads()), + block_size_f)); + const Index max_block_size = numext::mini(n, 2 * block_size); + if (block_align) { + Index new_block_size = block_align(block_size); + eigen_assert(new_block_size >= block_size); + block_size = numext::mini(n, new_block_size); + } + Index block_count = divup(n, block_size); + // Calculate parallel efficiency as fraction of total CPU time used for + // computations: + double max_efficiency = + static_cast<double>(block_count) / + (divup<int>(block_count, numThreads()) * numThreads()); + // Now try to increase block size up to max_block_size as long as it + // doesn't decrease parallel efficiency. + for (Index prev_block_count = block_count; + max_efficiency < 1.0 && prev_block_count > 1;) { + // This is the next block size that divides size into a smaller number + // of blocks than the current block_size. + Index coarser_block_size = divup(n, prev_block_count - 1); + if (block_align) { + Index new_block_size = block_align(coarser_block_size); + eigen_assert(new_block_size >= coarser_block_size); + coarser_block_size = numext::mini(n, new_block_size); + } + if (coarser_block_size > max_block_size) { + break; // Reached max block size. Stop. + } + // Recalculate parallel efficiency. + const Index coarser_block_count = divup(n, coarser_block_size); + eigen_assert(coarser_block_count < prev_block_count); + prev_block_count = coarser_block_count; + const double coarser_efficiency = + static_cast<double>(coarser_block_count) / + (divup<int>(coarser_block_count, numThreads()) * numThreads()); + if (coarser_efficiency + 0.01 >= max_efficiency) { + // Taking it. + block_size = coarser_block_size; + block_count = coarser_block_count; + if (max_efficiency < coarser_efficiency) { + max_efficiency = coarser_efficiency; + } + } + } + + // Recursively divide size into halves until we reach block_size. + // Division code rounds mid to block_size, so we are guaranteed to get + // block_count leaves that do actual computations. + Barrier barrier(static_cast<unsigned int>(block_count)); + std::function<void(Index, Index)> handleRange; + handleRange = [=, &handleRange, &barrier, &f](Index firstIdx, Index lastIdx) { + while (lastIdx - firstIdx > block_size) { + // Split into halves and schedule the second half on a different thread. + const Index midIdx = firstIdx + divup((lastIdx - firstIdx) / 2, block_size) * block_size; + pool_->Schedule([=, &handleRange]() { handleRange(midIdx, lastIdx); }); + lastIdx = midIdx; + } + // Single block or less, execute directly. + f(firstIdx, lastIdx); + barrier.Notify(); + }; + if (block_count <= numThreads()) { + // Avoid a thread hop by running the root of the tree and one block on the + // main thread. + handleRange(0, n); + } else { + // Execute the root in the thread pool to avoid running work on more than + // numThreads() threads. + pool_->Schedule([=, &handleRange]() { handleRange(0, n); }); + } + barrier.Wait(); + } + + // Convenience wrapper for parallelFor that does not align blocks. + void parallelFor(Index n, const TensorOpCost& cost, + std::function<void(Index, Index)> f) const { + parallelFor(n, cost, NULL, std::move(f)); + } + + // Thread pool accessor. + ThreadPoolInterface* getPool() const { return pool_; } + + // Allocator accessor. + Allocator* allocator() const { return allocator_; } + private: ThreadPoolInterface* pool_; - size_t num_threads_; + int num_threads_; + Allocator* allocator_; };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h index ca9ac79..1a30e45 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h
@@ -44,7 +44,7 @@ } -#if defined(EIGEN_HAS_CONSTEXPR) +#if EIGEN_HAS_CONSTEXPR template <typename Index, std::size_t Rank> struct index_known_statically_impl<DimensionList<Index, Rank> > { EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex) {
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h index 977dcaf..dbf5af0 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h
@@ -29,27 +29,19 @@ * \sa Tensor */ -// Can't use std::pair on cuda devices -template <typename Index> struct IndexPair { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE IndexPair() : first(0), second(0) { } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE IndexPair(Index f, Index s) : first(f), second(s) { } - Index first; - Index second; -}; - // Boilerplate code namespace internal { -template<std::size_t n, typename Dimension> struct dget { - static const std::size_t value = get<n, Dimension>::value; +template<std::ptrdiff_t n, typename Dimension> struct dget { + static const std::ptrdiff_t value = get<n, Dimension>::value; }; -template<typename Index, std::size_t NumIndices, std::size_t n, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, std::ptrdiff_t n, bool RowMajor> struct fixed_size_tensor_index_linearization_helper { template <typename Dimensions> EIGEN_DEVICE_FUNC - static inline Index run(array<Index, NumIndices> const& indices, + static EIGEN_STRONG_INLINE Index run(array<Index, NumIndices> const& indices, const Dimensions& dimensions) { return array_get<RowMajor ? n - 1 : (NumIndices - n)>(indices) + @@ -58,21 +50,21 @@ } }; -template<typename Index, std::size_t NumIndices, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, bool RowMajor> struct fixed_size_tensor_index_linearization_helper<Index, NumIndices, 0, RowMajor> { template <typename Dimensions> EIGEN_DEVICE_FUNC - static inline Index run(array<Index, NumIndices> const&, const Dimensions&) + static EIGEN_STRONG_INLINE Index run(array<Index, NumIndices> const&, const Dimensions&) { return 0; } }; -template<typename Index, std::size_t n> +template<typename Index, std::ptrdiff_t n> struct fixed_size_tensor_index_extraction_helper { template <typename Dimensions> EIGEN_DEVICE_FUNC - static inline Index run(const Index index, + static EIGEN_STRONG_INLINE Index run(const Index index, const Dimensions& dimensions) { const Index mult = (index == n-1) ? 1 : 0; @@ -85,7 +77,7 @@ struct fixed_size_tensor_index_extraction_helper<Index, 0> { template <typename Dimensions> EIGEN_DEVICE_FUNC - static inline Index run(const Index, + static EIGEN_STRONG_INLINE Index run(const Index, const Dimensions&) { return 0; @@ -98,9 +90,11 @@ // Fixed size #ifndef EIGEN_EMULATE_CXX11_META_H template <typename std::ptrdiff_t... Indices> -struct Sizes : internal::numeric_list<std::ptrdiff_t, Indices...> { +struct Sizes { typedef internal::numeric_list<std::ptrdiff_t, Indices...> Base; + const Base t = Base(); static const std::ptrdiff_t total_size = internal::arg_prod(Indices...); + static const ptrdiff_t count = Base::count; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t rank() const { return Base::count; @@ -115,7 +109,7 @@ explicit EIGEN_DEVICE_FUNC Sizes(const array<DenseIndex, Base::count>& /*indices*/) { // todo: add assertion } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template <typename... DenseIndex> EIGEN_DEVICE_FUNC Sizes(DenseIndex...) { } explicit EIGEN_DEVICE_FUNC Sizes(std::initializer_list<std::ptrdiff_t> /*l*/) { // todo: add assertion @@ -127,17 +121,17 @@ return *this; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t operator[] (const std::size_t index) const { - return internal::fixed_size_tensor_index_extraction_helper<std::ptrdiff_t, Base::count>::run(index, *this); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t operator[] (const std::ptrdiff_t index) const { + return internal::fixed_size_tensor_index_extraction_helper<std::ptrdiff_t, Base::count>::run(index, t); } template <typename DenseIndex> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - size_t IndexOfColMajor(const array<DenseIndex, Base::count>& indices) const { - return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, false>::run(indices, *static_cast<const Base*>(this)); + ptrdiff_t IndexOfColMajor(const array<DenseIndex, Base::count>& indices) const { + return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, false>::run(indices, t); } template <typename DenseIndex> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - size_t IndexOfRowMajor(const array<DenseIndex, Base::count>& indices) const { - return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, true>::run(indices, *static_cast<const Base*>(this)); + ptrdiff_t IndexOfRowMajor(const array<DenseIndex, Base::count>& indices) const { + return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, true>::run(indices, t); } }; @@ -150,25 +144,25 @@ #else -template <std::size_t n> +template <std::ptrdiff_t n> struct non_zero_size { - typedef internal::type2val<std::size_t, n> type; + typedef internal::type2val<std::ptrdiff_t, n> type; }; template <> struct non_zero_size<0> { typedef internal::null_type type; }; -template <std::size_t V1=0, std::size_t V2=0, std::size_t V3=0, std::size_t V4=0, std::size_t V5=0> struct Sizes { +template <std::ptrdiff_t V1=0, std::ptrdiff_t V2=0, std::ptrdiff_t V3=0, std::ptrdiff_t V4=0, std::ptrdiff_t V5=0> struct Sizes { typedef typename internal::make_type_list<typename non_zero_size<V1>::type, typename non_zero_size<V2>::type, typename non_zero_size<V3>::type, typename non_zero_size<V4>::type, typename non_zero_size<V5>::type >::type Base; - static const size_t count = Base::count; - static const std::size_t total_size = internal::arg_prod<Base>::value; + static const std::ptrdiff_t count = Base::count; + static const std::ptrdiff_t total_size = internal::arg_prod<Base>::value; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t rank() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptrdiff_t rank() const { return count; } - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t TotalSize() { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptrdiff_t TotalSize() { return internal::arg_prod<Base>::value; } @@ -182,25 +176,25 @@ return *this; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template <typename... DenseIndex> Sizes(DenseIndex... /*indices*/) { } - explicit Sizes(std::initializer_list<std::size_t>) { + explicit Sizes(std::initializer_list<std::ptrdiff_t>) { // todo: add assertion } #else EIGEN_DEVICE_FUNC explicit Sizes(const DenseIndex) { } - EIGEN_DEVICE_FUNC explicit Sizes(const DenseIndex, const DenseIndex) { + EIGEN_DEVICE_FUNC Sizes(const DenseIndex, const DenseIndex) { } - EIGEN_DEVICE_FUNC explicit Sizes(const DenseIndex, const DenseIndex, const DenseIndex) { + EIGEN_DEVICE_FUNC Sizes(const DenseIndex, const DenseIndex, const DenseIndex) { } - EIGEN_DEVICE_FUNC explicit Sizes(const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex) { + EIGEN_DEVICE_FUNC Sizes(const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex) { } - EIGEN_DEVICE_FUNC explicit Sizes(const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex) { + EIGEN_DEVICE_FUNC Sizes(const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex, const DenseIndex) { } #endif - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex operator[] (const int index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index operator[] (const Index index) const { switch (index) { case 0: return internal::get<0, Base>::value; @@ -214,23 +208,23 @@ return internal::get<4, Base>::value; default: eigen_assert(false && "index overflow"); - return static_cast<DenseIndex>(-1); + return static_cast<Index>(-1); } } template <typename DenseIndex> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - size_t IndexOfColMajor(const array<DenseIndex, Base::count>& indices) const { + ptrdiff_t IndexOfColMajor(const array<DenseIndex, Base::count>& indices) const { return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, false>::run(indices, *reinterpret_cast<const Base*>(this)); } template <typename DenseIndex> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - size_t IndexOfRowMajor(const array<DenseIndex, Base::count>& indices) const { + ptrdiff_t IndexOfRowMajor(const array<DenseIndex, Base::count>& indices) const { return internal::fixed_size_tensor_index_linearization_helper<DenseIndex, Base::count, Base::count, true>::run(indices, *reinterpret_cast<const Base*>(this)); } }; namespace internal { -template <std::size_t V1, std::size_t V2, std::size_t V3, std::size_t V4, std::size_t V5> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t array_prod(const Sizes<V1, V2, V3, V4, V5>&) { +template <std::ptrdiff_t V1, std::ptrdiff_t V2, std::ptrdiff_t V3, std::ptrdiff_t V4, std::ptrdiff_t V5> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t array_prod(const Sizes<V1, V2, V3, V4, V5>&) { return Sizes<V1, V2, V3, V4, V5>::total_size; } } @@ -239,7 +233,7 @@ // Boilerplate namespace internal { -template<typename Index, std::size_t NumIndices, std::size_t n, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, std::ptrdiff_t n, bool RowMajor> struct tensor_index_linearization_helper { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -251,7 +245,7 @@ } }; -template<typename Index, std::size_t NumIndices, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, bool RowMajor> struct tensor_index_linearization_helper<Index, NumIndices, 0, RowMajor> { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -270,12 +264,12 @@ typedef array<DenseIndex, NumDims> Base; static const int count = NumDims; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t rank() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rank() const { return NumDims; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex TotalSize() const { - return internal::array_prod(*static_cast<const Base*>(this)); + return (NumDims == 0) ? 1 : internal::array_prod(*static_cast<const Base*>(this)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DSizes() { @@ -290,31 +284,82 @@ (*this)[0] = i0; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES + EIGEN_DEVICE_FUNC DSizes(const DimensionList<DenseIndex, NumDims>& a) { + for (int i = 0 ; i < NumDims; ++i) { + (*this)[i] = a[i]; + } + } + + // Enable DSizes index type promotion only if we are promoting to the + // larger type, e.g. allow to promote dimensions of type int to long. + template<typename OtherIndex> + EIGEN_DEVICE_FUNC + explicit DSizes(const array<OtherIndex, NumDims>& other, + // Default template parameters require c++11. + typename internal::enable_if< + internal::is_same< + DenseIndex, + typename internal::promote_index_type< + DenseIndex, + OtherIndex + >::type + >::value, void*>::type = 0) { + for (int i = 0; i < NumDims; ++i) { + (*this)[i] = static_cast<DenseIndex>(other[i]); + } + } + +#ifdef EIGEN_HAS_INDEX_LIST + template <typename FirstType, typename... OtherTypes> + EIGEN_DEVICE_FUNC + explicit DSizes(const Eigen::IndexList<FirstType, OtherTypes...>& dimensions) { + for (int i = 0; i < dimensions.count; ++i) { + (*this)[i] = dimensions[i]; + } + } +#endif + +#ifndef EIGEN_EMULATE_CXX11_META_H + template <typename std::ptrdiff_t... Indices> + EIGEN_DEVICE_FUNC DSizes(const Sizes<Indices...>& a) { + for (int i = 0 ; i < NumDims; ++i) { + (*this)[i] = a[i]; + } + } +#else + template <std::ptrdiff_t V1, std::ptrdiff_t V2, std::ptrdiff_t V3, std::ptrdiff_t V4, std::ptrdiff_t V5> + EIGEN_DEVICE_FUNC DSizes(const Sizes<V1, V2, V3, V4, V5>& a) { + for (int i = 0 ; i < NumDims; ++i) { + (*this)[i] = a[i]; + } + } +#endif + +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit DSizes(DenseIndex firstDimension, DenseIndex secondDimension, IndexTypes... otherDimensions) : Base({{firstDimension, secondDimension, otherDimensions...}}) { EIGEN_STATIC_ASSERT(sizeof...(otherDimensions) + 2 == NumDims, YOU_MADE_A_PROGRAMMING_MISTAKE) } #else - EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0, const DenseIndex i1) { + EIGEN_DEVICE_FUNC DSizes(const DenseIndex i0, const DenseIndex i1) { eigen_assert(NumDims == 2); (*this)[0] = i0; (*this)[1] = i1; } - EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2) { + EIGEN_DEVICE_FUNC DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2) { eigen_assert(NumDims == 3); (*this)[0] = i0; (*this)[1] = i1; (*this)[2] = i2; } - EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2, const DenseIndex i3) { + EIGEN_DEVICE_FUNC DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2, const DenseIndex i3) { eigen_assert(NumDims == 4); (*this)[0] = i0; (*this)[1] = i1; (*this)[2] = i2; (*this)[3] = i3; } - EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2, const DenseIndex i3, const DenseIndex i4) { + EIGEN_DEVICE_FUNC DSizes(const DenseIndex i0, const DenseIndex i1, const DenseIndex i2, const DenseIndex i3, const DenseIndex i4) { eigen_assert(NumDims == 5); (*this)[0] = i0; (*this)[1] = i1; @@ -343,7 +388,7 @@ // Boilerplate namespace internal { -template<typename Index, std::size_t NumIndices, std::size_t n, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, std::ptrdiff_t n, bool RowMajor> struct tensor_vsize_index_linearization_helper { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -355,7 +400,7 @@ } }; -template<typename Index, std::size_t NumIndices, bool RowMajor> +template<typename Index, std::ptrdiff_t NumIndices, bool RowMajor> struct tensor_vsize_index_linearization_helper<Index, NumIndices, 0, RowMajor> { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -370,10 +415,10 @@ namespace internal { template <typename DenseIndex, int NumDims> struct array_size<const DSizes<DenseIndex, NumDims> > { - static const size_t value = NumDims; + static const ptrdiff_t value = NumDims; }; template <typename DenseIndex, int NumDims> struct array_size<DSizes<DenseIndex, NumDims> > { - static const size_t value = NumDims; + static const ptrdiff_t value = NumDims; }; #ifndef EIGEN_EMULATE_CXX11_META_H template <typename std::ptrdiff_t... Indices> struct array_size<const Sizes<Indices...> > { @@ -383,42 +428,42 @@ static const std::ptrdiff_t value = Sizes<Indices...>::count; }; template <std::ptrdiff_t n, typename std::ptrdiff_t... Indices> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t array_get(const Sizes<Indices...>&) { - return get<n, internal::numeric_list<std::size_t, Indices...> >::value; + return get<n, internal::numeric_list<std::ptrdiff_t, Indices...> >::value; } template <std::ptrdiff_t n> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t array_get(const Sizes<>&) { eigen_assert(false && "should never be called"); return -1; } #else -template <std::size_t V1, std::size_t V2, std::size_t V3, std::size_t V4, std::size_t V5> struct array_size<const Sizes<V1,V2,V3,V4,V5> > { - static const size_t value = Sizes<V1,V2,V3,V4,V5>::count; +template <std::ptrdiff_t V1, std::ptrdiff_t V2, std::ptrdiff_t V3, std::ptrdiff_t V4, std::ptrdiff_t V5> struct array_size<const Sizes<V1,V2,V3,V4,V5> > { + static const ptrdiff_t value = Sizes<V1,V2,V3,V4,V5>::count; }; -template <std::size_t V1, std::size_t V2, std::size_t V3, std::size_t V4, std::size_t V5> struct array_size<Sizes<V1,V2,V3,V4,V5> > { - static const size_t value = Sizes<V1,V2,V3,V4,V5>::count; +template <std::ptrdiff_t V1, std::ptrdiff_t V2, std::ptrdiff_t V3, std::ptrdiff_t V4, std::ptrdiff_t V5> struct array_size<Sizes<V1,V2,V3,V4,V5> > { + static const ptrdiff_t value = Sizes<V1,V2,V3,V4,V5>::count; }; -template <std::size_t n, std::size_t V1, std::size_t V2, std::size_t V3, std::size_t V4, std::size_t V5> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t array_get(const Sizes<V1,V2,V3,V4,V5>&) { +template <std::ptrdiff_t n, std::ptrdiff_t V1, std::ptrdiff_t V2, std::ptrdiff_t V3, std::ptrdiff_t V4, std::ptrdiff_t V5> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t array_get(const Sizes<V1,V2,V3,V4,V5>&) { return get<n, typename Sizes<V1,V2,V3,V4,V5>::Base>::value; } #endif -template <typename Dims1, typename Dims2, size_t n, size_t m> +template <typename Dims1, typename Dims2, ptrdiff_t n, ptrdiff_t m> struct sizes_match_below_dim { - static EIGEN_DEVICE_FUNC inline bool run(Dims1&, Dims2&) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(Dims1&, Dims2&) { return false; } }; -template <typename Dims1, typename Dims2, size_t n> +template <typename Dims1, typename Dims2, ptrdiff_t n> struct sizes_match_below_dim<Dims1, Dims2, n, n> { - static EIGEN_DEVICE_FUNC inline bool run(Dims1& dims1, Dims2& dims2) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(Dims1& dims1, Dims2& dims2) { return (array_get<n-1>(dims1) == array_get<n-1>(dims2)) & sizes_match_below_dim<Dims1, Dims2, n-1, n-1>::run(dims1, dims2); } }; template <typename Dims1, typename Dims2> struct sizes_match_below_dim<Dims1, Dims2, 0, 0> { - static EIGEN_DEVICE_FUNC inline bool run(Dims1&, Dims2&) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(Dims1&, Dims2&) { return true; } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h b/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h index 893542d..554ee5f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h
@@ -20,8 +20,8 @@ * */ namespace internal { -template<typename XprType> -struct traits<TensorEvalToOp<XprType> > +template<typename XprType, template <class> class MakePointer_> +struct traits<TensorEvalToOp<XprType, MakePointer_> > { // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename XprType::Scalar Scalar; @@ -32,22 +32,32 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename MakePointer_<Scalar>::Type PointerType; enum { - Flags = 0, + Flags = 0 + }; + template <class T> + struct MakePointer { + // Intermediate typedef to workaround MSVC issue. + typedef MakePointer_<T> MakePointerT; + typedef typename MakePointerT::Type Type; + typedef typename MakePointerT::RefType RefType; + + }; }; -template<typename XprType> -struct eval<TensorEvalToOp<XprType>, Eigen::Dense> +template<typename XprType, template <class> class MakePointer_> +struct eval<TensorEvalToOp<XprType, MakePointer_>, Eigen::Dense> { - typedef const TensorEvalToOp<XprType>& type; + typedef const TensorEvalToOp<XprType, MakePointer_>& type; }; -template<typename XprType> -struct nested<TensorEvalToOp<XprType>, 1, typename eval<TensorEvalToOp<XprType> >::type> +template<typename XprType, template <class> class MakePointer_> +struct nested<TensorEvalToOp<XprType, MakePointer_>, 1, typename eval<TensorEvalToOp<XprType, MakePointer_> >::type> { - typedef TensorEvalToOp<XprType> type; + typedef TensorEvalToOp<XprType, MakePointer_> type; }; } // end namespace internal @@ -55,62 +65,72 @@ -template<typename XprType> -class TensorEvalToOp : public TensorBase<TensorEvalToOp<XprType> > +template<typename XprType, template <class> class MakePointer_> +class TensorEvalToOp : public TensorBase<TensorEvalToOp<XprType, MakePointer_>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorEvalToOp>::Scalar Scalar; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; + typedef typename MakePointer_<CoeffReturnType>::Type PointerType; typedef typename Eigen::internal::nested<TensorEvalToOp>::type Nested; typedef typename Eigen::internal::traits<TensorEvalToOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorEvalToOp>::Index Index; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvalToOp(CoeffReturnType* buffer, const XprType& expr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvalToOp(PointerType buffer, const XprType& expr) : m_xpr(expr), m_buffer(buffer) {} EIGEN_DEVICE_FUNC const typename internal::remove_all<typename XprType::Nested>::type& expression() const { return m_xpr; } - EIGEN_DEVICE_FUNC CoeffReturnType* buffer() const { return m_buffer; } + EIGEN_DEVICE_FUNC PointerType buffer() const { return m_buffer; } protected: typename XprType::Nested m_xpr; - CoeffReturnType* m_buffer; + PointerType m_buffer; }; -template<typename ArgType, typename Device> -struct TensorEvaluator<const TensorEvalToOp<ArgType>, Device> +template<typename ArgType, typename Device, template <class> class MakePointer_> +struct TensorEvaluator<const TensorEvalToOp<ArgType, MakePointer_>, Device> { - typedef TensorEvalToOp<ArgType> XprType; + typedef TensorEvalToOp<ArgType, MakePointer_> XprType; typedef typename ArgType::Scalar Scalar; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; typedef typename XprType::Index Index; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = true, - PacketAccess = true, + IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = true }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device), m_device(device), m_buffer(op.buffer()) + : m_impl(op.expression(), device), m_device(device), + m_buffer(op.buffer()), m_op(op), m_expression(op.expression()) { } + // Used for accessor extraction in SYCL Managed TensorMap: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const XprType& op() const { + return m_op; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~TensorEvaluator() { } + typedef typename internal::traits<const TensorEvalToOp<ArgType, MakePointer_> >::template MakePointer<CoeffReturnType>::Type DevicePointer; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_impl.dimensions(); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* scalar) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(DevicePointer scalar) { EIGEN_UNUSED_VARIABLE(scalar); eigen_assert(scalar == NULL); return m_impl.evalSubExprsIfNeeded(m_buffer); @@ -145,12 +165,20 @@ TensorOpCost(0, sizeof(CoeffReturnType), 0, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_buffer; } + EIGEN_DEVICE_FUNC DevicePointer data() const { return m_buffer; } + ArgType expression() const { return m_expression; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + /// added for sycl in order to construct the buffer from the sycl device + const Device& device() const{return m_device;} private: TensorEvaluator<ArgType, Device> m_impl; const Device& m_device; - CoeffReturnType* m_buffer; + DevicePointer m_buffer; + const XprType& m_op; + const ArgType m_expression; };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h b/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h index ae4ce3c..4ca6b3d 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h
@@ -32,6 +32,8 @@ typedef typename Derived::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; + typedef Derived XprType; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; // NumDimensions is -1 for variable dim tensors static const int NumCoords = internal::traits<Derived>::NumDimensions > 0 ? @@ -39,20 +41,34 @@ enum { IsAligned = Derived::IsAligned, - PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = internal::is_arithmetic<typename internal::remove_const<Scalar>::type>::value, + PreferBlockAccess = false, Layout = Derived::Layout, CoordAccess = NumCoords > 0, RawAccess = true }; + typedef typename internal::TensorBlock< + typename internal::remove_const<Scalar>::type, Index, NumCoords, Layout> + TensorBlock; + typedef typename internal::TensorBlockReader< + typename internal::remove_const<Scalar>::type, Index, NumCoords, Layout> + TensorBlockReader; + typedef typename internal::TensorBlockWriter< + typename internal::remove_const<Scalar>::type, Index, NumCoords, Layout> + TensorBlockWriter; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const Derived& m, const Device& device) - : m_data(const_cast<Scalar*>(m.data())), m_dims(m.dimensions()), m_device(device) + : m_data(const_cast<typename internal::traits<Derived>::template MakePointer<Scalar>::Type>(m.data())), m_dims(m.dimensions()), m_device(device), m_impl(m) { } + // Used for accessor extraction in SYCL Managed TensorMap: + const Derived& derived() const { return m_impl; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dims; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* dest) { - if (dest) { + if (!NumTraits<typename internal::remove_const<Scalar>::type>::RequireInitialization && dest) { m_device.memcpy((void*)dest, m_data, sizeof(Scalar) * m_dims.TotalSize()); return false; } @@ -66,7 +82,9 @@ return m_data[index]; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::traits<Derived>::template MakePointer<Scalar>::RefType + coeffRef(Index index) { eigen_assert(m_data); return m_data[index]; } @@ -92,7 +110,9 @@ } } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(const array<DenseIndex, NumCoords>& coords) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename internal::traits<Derived>::template MakePointer<Scalar>::RefType + coeffRef(const array<DenseIndex, NumCoords>& coords) { eigen_assert(m_data); if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { return m_data[m_dims.IndexOfColMajor(coords)]; @@ -103,15 +123,33 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, - internal::unpacket_traits<PacketReturnType>::size); + PacketType<CoeffReturnType, Device>::size); } - EIGEN_DEVICE_FUNC Scalar* data() const { return m_data; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>*) const {} + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block(TensorBlock* block) const { + assert(m_data != NULL); + TensorBlockReader::Run(block, m_data); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock( + const TensorBlock& block) { + assert(m_data != NULL); + TensorBlockWriter::Run(block, m_data); + } + + EIGEN_DEVICE_FUNC typename internal::traits<Derived>::template MakePointer<Scalar>::Type data() const { return m_data; } + + /// required by sycl in order to construct sycl buffer from raw pointer + const Device& device() const{return m_device;} protected: - Scalar* m_data; + typename internal::traits<Derived>::template MakePointer<Scalar>::Type m_data; Dimensions m_dims; const Device& m_device; + const Derived& m_impl; }; namespace { @@ -120,7 +158,7 @@ return *address; } // Use the texture cache on CUDA devices whenever possible -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350 +#if defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350 template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float loadConstant(const float* address) { return __ldg(address); @@ -129,6 +167,10 @@ double loadConstant(const double* address) { return __ldg(address); } +template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +Eigen::half loadConstant(const Eigen::half* address) { + return Eigen::half(half_impl::raw_uint16_to_half(__ldg(&address->x))); +} #endif } @@ -142,21 +184,36 @@ typedef typename Derived::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; + typedef const Derived XprType; + // NumDimensions is -1 for variable dim tensors static const int NumCoords = internal::traits<Derived>::NumDimensions > 0 ? internal::traits<Derived>::NumDimensions : 0; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = Derived::IsAligned, - PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = internal::is_arithmetic<typename internal::remove_const<Scalar>::type>::value, + PreferBlockAccess = false, Layout = Derived::Layout, CoordAccess = NumCoords > 0, RawAccess = true }; + typedef typename internal::TensorBlock< + typename internal::remove_const<Scalar>::type, Index, NumCoords, Layout> + TensorBlock; + typedef typename internal::TensorBlockReader< + typename internal::remove_const<Scalar>::type, Index, NumCoords, Layout> + TensorBlockReader; + + // Used for accessor extraction in SYCL Managed TensorMap: + const Derived& derived() const { return m_impl; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const Derived& m, const Device& device) - : m_data(m.data()), m_dims(m.dimensions()), m_device(device) + : m_data(m.data()), m_dims(m.dimensions()), m_device(device), m_impl(m) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dims; } @@ -173,7 +230,12 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { eigen_assert(m_data); +#ifndef __SYCL_DEVICE_ONLY__ return loadConstant(m_data+index); +#else + CoeffReturnType tmp = m_data[index]; + return tmp; +#endif } template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -191,15 +253,27 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, - internal::unpacket_traits<PacketReturnType>::size); + PacketType<CoeffReturnType, Device>::size); } - EIGEN_DEVICE_FUNC const Scalar* data() const { return m_data; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>*) const {} + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block(TensorBlock* block) const { + assert(m_data != NULL); + TensorBlockReader::Run(block, m_data); + } + + EIGEN_DEVICE_FUNC typename internal::traits<Derived>::template MakePointer<const Scalar>::Type data() const { return m_data; } + + /// added for sycl in order to construct the buffer from the sycl device + const Device& device() const{return m_device;} protected: - const Scalar* m_data; + typename internal::traits<Derived>::template MakePointer<const Scalar>::Type m_data; Dimensions m_dims; const Device& m_device; + const Derived& m_impl; }; @@ -215,6 +289,8 @@ enum { IsAligned = true, PacketAccess = internal::functor_traits<NullaryOp>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -222,14 +298,14 @@ EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) - : m_functor(op.functor()), m_argImpl(op.nestedExpression(), device) + : m_functor(op.functor()), m_argImpl(op.nestedExpression(), device), m_wrapper() { } typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); } @@ -239,26 +315,33 @@ EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const { - return m_functor(index); + return m_wrapper(m_functor, index); } template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return m_functor.template packetOp<Index, PacketReturnType>(index); + return m_wrapper.template packetOp<PacketReturnType, Index>(m_functor, index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, - internal::unpacket_traits<PacketReturnType>::size); + PacketType<CoeffReturnType, Device>::size); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<ArgType, Device>& impl() const { return m_argImpl; } + /// required by sycl in order to extract the accessor + NullaryOp functor() const { return m_functor; } + private: const NullaryOp m_functor; TensorEvaluator<ArgType, Device> m_argImpl; + const internal::nullary_wrapper<CoeffReturnType,NullaryOp> m_wrapper; }; @@ -271,25 +354,34 @@ typedef TensorCwiseUnaryOp<UnaryOp, ArgType> XprType; enum { - IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess & internal::functor_traits<UnaryOp>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, // to be implemented - RawAccess = false + IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess & + internal::functor_traits<UnaryOp>::PacketAccess, + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = TensorEvaluator<ArgType, Device>::PreferBlockAccess, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) - : m_functor(op.functor()), + : m_device(device), + m_functor(op.functor()), m_argImpl(op.nestedExpression(), device) { } typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; + static const int NumDims = internal::array_size<Dimensions>::value; + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + TensorBlock; + EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar*) { @@ -317,9 +409,39 @@ TensorOpCost(0, 0, functor_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + m_argImpl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + TensorBlock* output_block) const { + if (NumDims <= 0) { + output_block->data()[0] = coeff(0); + return; + } + internal::TensorBlockView<ArgType, Device> arg_block(m_device, m_argImpl, + *output_block); + internal::TensorBlockCwiseUnaryIO<UnaryOp, Index, ScalarNoConst, NumDims, + Layout>::Run(m_functor, + output_block->block_sizes(), + output_block + ->block_strides(), + output_block->data(), + arg_block.block_strides(), + arg_block.data()); + } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<ArgType, Device> & impl() const { return m_argImpl; } + /// added for sycl in order to construct the buffer from sycl device + UnaryOp functor() const { return m_functor; } + private: + const Device& m_device; const UnaryOp m_functor; TensorEvaluator<ArgType, Device> m_argImpl; }; @@ -333,16 +455,23 @@ typedef TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArgType> XprType; enum { - IsAligned = TensorEvaluator<LeftArgType, Device>::IsAligned & TensorEvaluator<RightArgType, Device>::IsAligned, - PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess & - internal::functor_traits<BinaryOp>::PacketAccess, - Layout = TensorEvaluator<LeftArgType, Device>::Layout, - CoordAccess = false, // to be implemented - RawAccess = false + IsAligned = TensorEvaluator<LeftArgType, Device>::IsAligned & + TensorEvaluator<RightArgType, Device>::IsAligned, + PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & + TensorEvaluator<RightArgType, Device>::PacketAccess & + internal::functor_traits<BinaryOp>::PacketAccess, + BlockAccess = TensorEvaluator<LeftArgType, Device>::BlockAccess & + TensorEvaluator<RightArgType, Device>::BlockAccess, + PreferBlockAccess = TensorEvaluator<LeftArgType, Device>::PreferBlockAccess | + TensorEvaluator<RightArgType, Device>::PreferBlockAccess, + Layout = TensorEvaluator<LeftArgType, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) - : m_functor(op.functor()), + : m_device(device), + m_functor(op.functor()), m_leftImpl(op.lhsExpression(), device), m_rightImpl(op.rhsExpression(), device) { @@ -354,9 +483,17 @@ typedef typename XprType::Scalar Scalar; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; typedef typename TensorEvaluator<LeftArgType, Device>::Dimensions Dimensions; + static const int NumDims = internal::array_size< + typename TensorEvaluator<LeftArgType, Device>::Dimensions>::value; + + typedef internal::TensorBlock< + typename internal::remove_const<Scalar>::type, Index, NumDims, + TensorEvaluator<LeftArgType, Device>::Layout> + TensorBlock; + EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { // TODO: use right impl instead if right impl dimensions are known at compile time. @@ -391,14 +528,149 @@ TensorOpCost(0, 0, functor_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + m_leftImpl.getResourceRequirements(resources); + m_rightImpl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + TensorBlock* output_block) const { + if (NumDims <= 0) { + output_block->data()[0] = coeff(Index(0)); + return; + } + internal::TensorBlockView<LeftArgType, Device> left_block( + m_device, m_leftImpl, *output_block); + internal::TensorBlockView<RightArgType, Device> right_block( + m_device, m_rightImpl, *output_block); + internal::TensorBlockCwiseBinaryIO< + BinaryOp, Index, typename internal::remove_const<Scalar>::type, NumDims, + Layout>::Run(m_functor, output_block->block_sizes(), + output_block->block_strides(), output_block->data(), + left_block.block_strides(), left_block.data(), + right_block.block_strides(), right_block.data()); + } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<LeftArgType, Device>& left_impl() const { return m_leftImpl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<RightArgType, Device>& right_impl() const { return m_rightImpl; } + /// required by sycl in order to extract the accessor + BinaryOp functor() const { return m_functor; } private: + const Device& m_device; const BinaryOp m_functor; TensorEvaluator<LeftArgType, Device> m_leftImpl; TensorEvaluator<RightArgType, Device> m_rightImpl; }; +// -------------------- CwiseTernaryOp -------------------- + +template<typename TernaryOp, typename Arg1Type, typename Arg2Type, typename Arg3Type, typename Device> +struct TensorEvaluator<const TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type>, Device> +{ + typedef TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type> XprType; + + enum { + IsAligned = TensorEvaluator<Arg1Type, Device>::IsAligned & TensorEvaluator<Arg2Type, Device>::IsAligned & TensorEvaluator<Arg3Type, Device>::IsAligned, + PacketAccess = TensorEvaluator<Arg1Type, Device>::PacketAccess & TensorEvaluator<Arg2Type, Device>::PacketAccess & TensorEvaluator<Arg3Type, Device>::PacketAccess & + internal::functor_traits<TernaryOp>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<Arg1Type, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false + }; + + EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) + : m_functor(op.functor()), + m_arg1Impl(op.arg1Expression(), device), + m_arg2Impl(op.arg2Expression(), device), + m_arg3Impl(op.arg3Expression(), device) + { + EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<Arg1Type, Device>::Layout) == static_cast<int>(TensorEvaluator<Arg3Type, Device>::Layout) || internal::traits<XprType>::NumDimensions <= 1), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::StorageKind, + typename internal::traits<Arg2Type>::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) + EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::StorageKind, + typename internal::traits<Arg3Type>::StorageKind>::value), + STORAGE_KIND_MUST_MATCH) + EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::Index, + typename internal::traits<Arg2Type>::Index>::value), + STORAGE_INDEX_MUST_MATCH) + EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::Index, + typename internal::traits<Arg3Type>::Index>::value), + STORAGE_INDEX_MUST_MATCH) + + eigen_assert(dimensions_match(m_arg1Impl.dimensions(), m_arg2Impl.dimensions()) && dimensions_match(m_arg1Impl.dimensions(), m_arg3Impl.dimensions())); + } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename internal::traits<XprType>::Scalar CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; + typedef typename TensorEvaluator<Arg1Type, Device>::Dimensions Dimensions; + + EIGEN_DEVICE_FUNC const Dimensions& dimensions() const + { + // TODO: use arg2 or arg3 dimensions if they are known at compile time. + return m_arg1Impl.dimensions(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType*) { + m_arg1Impl.evalSubExprsIfNeeded(NULL); + m_arg2Impl.evalSubExprsIfNeeded(NULL); + m_arg3Impl.evalSubExprsIfNeeded(NULL); + return true; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + m_arg1Impl.cleanup(); + m_arg2Impl.cleanup(); + m_arg3Impl.cleanup(); + } + + EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const + { + return m_functor(m_arg1Impl.coeff(index), m_arg2Impl.coeff(index), m_arg3Impl.coeff(index)); + } + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const + { + return m_functor.packetOp(m_arg1Impl.template packet<LoadMode>(index), + m_arg2Impl.template packet<LoadMode>(index), + m_arg3Impl.template packet<LoadMode>(index)); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost + costPerCoeff(bool vectorized) const { + const double functor_cost = internal::functor_traits<TernaryOp>::Cost; + return m_arg1Impl.costPerCoeff(vectorized) + + m_arg2Impl.costPerCoeff(vectorized) + + m_arg3Impl.costPerCoeff(vectorized) + + TensorOpCost(0, 0, functor_cost, vectorized, PacketSize); + } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<Arg1Type, Device> & arg1Impl() const { return m_arg1Impl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<Arg2Type, Device>& arg2Impl() const { return m_arg2Impl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<Arg3Type, Device>& arg3Impl() const { return m_arg3Impl; } + + private: + const TernaryOp m_functor; + TensorEvaluator<Arg1Type, Device> m_arg1Impl; + TensorEvaluator<Arg2Type, Device> m_arg2Impl; + TensorEvaluator<Arg3Type, Device> m_arg3Impl; +}; + // -------------------- SelectOp -------------------- @@ -411,7 +683,9 @@ enum { IsAligned = TensorEvaluator<ThenArgType, Device>::IsAligned & TensorEvaluator<ElseArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<ThenArgType, Device>::PacketAccess & TensorEvaluator<ElseArgType, Device>::PacketAccess & - internal::packet_traits<Scalar>::HasBlend, + PacketType<Scalar, Device>::HasBlend, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<IfArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -431,7 +705,7 @@ typedef typename XprType::Index Index; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; typedef typename TensorEvaluator<IfArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const @@ -475,7 +749,13 @@ .cwiseMax(m_elseImpl.costPerCoeff(vectorized)); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<IfArgType, Device> & cond_impl() const { return m_condImpl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<ThenArgType, Device>& then_impl() const { return m_thenImpl; } + /// required by sycl in order to extract the accessor + const TensorEvaluator<ElseArgType, Device>& else_impl() const { return m_elseImpl; } private: TensorEvaluator<IfArgType, Device> m_condImpl;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h index 5c3d4d6..1c44541 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h
@@ -12,31 +12,40 @@ namespace Eigen { -/** \class TensorExecutor - * \ingroup CXX11_Tensor_Module - * - * \brief The tensor executor class. - * - * This class is responsible for launch the evaluation of the expression on - * the specified computing device. - */ +/** + * \class TensorExecutor + * \ingroup CXX11_Tensor_Module + * + * \brief The tensor executor class. + * + * This class is responsible for launch the evaluation of the expression on + * the specified computing device. + * + * @tparam Vectorizable can use packet math (SSE/AVX/etc... registers and + * instructions) + * @tparam Tileable can use block based tensor evaluation + * (see TensorBlock.h) + */ namespace internal { -// Default strategy: the expression is evaluated with a single cpu thread. -template<typename Expression, typename Device, bool Vectorizable> -class TensorExecutor -{ +/** + * Default strategy: the expression is evaluated sequentially with a single cpu + * thread, without vectorization and block evaluation. + */ +template <typename Expression, typename Device, bool Vectorizable, + bool Tileable> +class TensorExecutor { public: - typedef typename Expression::Index Index; + typedef typename Expression::Index StorageIndex; + EIGEN_DEVICE_FUNC - static inline void run(const Expression& expr, const Device& device = Device()) - { + static EIGEN_STRONG_INLINE void run(const Expression& expr, + const Device& device = Device()) { TensorEvaluator<Expression, Device> evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); - if (needs_assign) - { - const Index size = array_prod(evaluator.dimensions()); - for (Index i = 0; i < size; ++i) { + if (needs_assign) { + const StorageIndex size = array_prod(evaluator.dimensions()); + for (StorageIndex i = 0; i < size; ++i) { evaluator.evalScalar(i); } } @@ -44,34 +53,40 @@ } }; - -template<typename Expression> -class TensorExecutor<Expression, DefaultDevice, true> -{ +/** + * Process all the data with a single cpu thread, using vectorized instructions. + */ +template <typename Expression> +class TensorExecutor<Expression, DefaultDevice, /*Vectorizable*/ true, + /*Tileable*/ false> { public: - typedef typename Expression::Index Index; + typedef typename Expression::Index StorageIndex; + EIGEN_DEVICE_FUNC - static inline void run(const Expression& expr, const DefaultDevice& device = DefaultDevice()) - { + static EIGEN_STRONG_INLINE void run(const Expression& expr, + const DefaultDevice& device = DefaultDevice()) { TensorEvaluator<Expression, DefaultDevice> evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); - if (needs_assign) - { - const Index size = array_prod(evaluator.dimensions()); - const int PacketSize = unpacket_traits<typename TensorEvaluator<Expression, DefaultDevice>::PacketReturnType>::size; - // Manually unroll this loop since compilers don't do it. - const Index UnrolledSize = (size / (4 * PacketSize)) * 4 * PacketSize; - for (Index i = 0; i < UnrolledSize; i += 4*PacketSize) { - evaluator.evalPacket(i); - evaluator.evalPacket(i+PacketSize); - evaluator.evalPacket(i+2*PacketSize); - evaluator.evalPacket(i+3*PacketSize); + if (needs_assign) { + const StorageIndex size = array_prod(evaluator.dimensions()); + const int PacketSize = unpacket_traits<typename TensorEvaluator< + Expression, DefaultDevice>::PacketReturnType>::size; + + // Give compiler a strong possibility to unroll the loop. But don't insist + // on unrolling, because if the function is expensive compiler should not + // unroll the loop at the expense of inlining. + const StorageIndex UnrolledSize = + (size / (4 * PacketSize)) * 4 * PacketSize; + for (StorageIndex i = 0; i < UnrolledSize; i += 4 * PacketSize) { + for (StorageIndex j = 0; j < 4; j++) { + evaluator.evalPacket(i + j * PacketSize); + } } - const Index VectorizedSize = (size / PacketSize) * PacketSize; - for (Index i = UnrolledSize; i < VectorizedSize; i += PacketSize) { + const StorageIndex VectorizedSize = (size / PacketSize) * PacketSize; + for (StorageIndex i = UnrolledSize; i < VectorizedSize; i += PacketSize) { evaluator.evalPacket(i); } - for (Index i = VectorizedSize; i < size; ++i) { + for (StorageIndex i = VectorizedSize; i < size; ++i) { evaluator.evalScalar(i); } } @@ -79,177 +94,319 @@ } }; +/** + * Process all the data with a single cpu thread, using blocks of data. By + * sizing a block to fit L1 cache we get better cache performance. + */ +template <typename Expression, bool Vectorizable> +class TensorExecutor<Expression, DefaultDevice, Vectorizable, + /*Tileable*/ true> { + public: + typedef typename traits<Expression>::Scalar Scalar; + typedef typename remove_const<Scalar>::type ScalarNoConst; + typedef TensorEvaluator<Expression, DefaultDevice> Evaluator; + typedef typename traits<Expression>::Index StorageIndex; -// Multicore strategy: the index space is partitioned and each partition is executed on a single core + static const int NumDims = traits<Expression>::NumDimensions; + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE void run(const Expression& expr, + const DefaultDevice& device = DefaultDevice()) { + typedef TensorBlock<ScalarNoConst, StorageIndex, NumDims, Evaluator::Layout> TensorBlock; + typedef TensorBlockMapper<ScalarNoConst, StorageIndex, NumDims, Evaluator::Layout> TensorBlockMapper; + typedef typename TensorBlock::Dimensions TensorBlockDimensions; + + Evaluator evaluator(expr, device); + Index total_size = array_prod(evaluator.dimensions()); + Index cache_size = device.firstLevelCacheSize() / sizeof(Scalar); + + if (total_size < cache_size) { + // TODO(andydavis) Reduce block management overhead for small tensors. + // TODO(wuke) Do not do this when evaluating TensorBroadcastingOp. + internal::TensorExecutor<Expression, DefaultDevice, Vectorizable, + /*Tileable*/ false>::run(expr, device); + return; + } + + const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); + if (needs_assign) { + // Size tensor blocks to fit in cache (or requested target block size). + Index block_total_size = numext::mini(cache_size, total_size); + TensorBlockShapeType block_shape = kSkewedInnerDims; + // Query expression tree for desired block size/shape. + std::vector<TensorOpResourceRequirements> resources; + evaluator.getResourceRequirements(&resources); + MergeResourceRequirements(resources, &block_shape, &block_total_size); + + TensorBlockMapper block_mapper( + TensorBlockDimensions(evaluator.dimensions()), block_shape, + block_total_size); + block_total_size = block_mapper.block_dims_total_size(); + + Scalar* data = static_cast<Scalar*>( + device.allocate(block_total_size * sizeof(Scalar))); + + const StorageIndex total_block_count = block_mapper.total_block_count(); + for (StorageIndex i = 0; i < total_block_count; ++i) { + TensorBlock block = block_mapper.GetBlockForIndex(i, data); + evaluator.evalBlock(&block); + } + device.deallocate(data); + } + evaluator.cleanup(); + } +}; + +/** + * Multicore strategy: the index space is partitioned and each partition is + * executed on a single core. + */ #ifdef EIGEN_USE_THREADS -template <typename Evaluator, typename Index, bool Vectorizable> +template <typename Evaluator, typename StorageIndex, bool Vectorizable> struct EvalRange { - static void run(Evaluator* evaluator_in, const Index first, const Index last) { + static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, + const StorageIndex lastIdx) { Evaluator evaluator = *evaluator_in; - eigen_assert(last >= first); - for (Index i = first; i < last; ++i) { + eigen_assert(lastIdx >= firstIdx); + for (StorageIndex i = firstIdx; i < lastIdx; ++i) { evaluator.evalScalar(i); } } + + static StorageIndex alignBlockSize(StorageIndex size) { return size; } }; -template <typename Evaluator, typename Index> -struct EvalRange<Evaluator, Index, true> { - static void run(Evaluator* evaluator_in, const Index first, const Index last) { +template <typename Evaluator, typename StorageIndex> +struct EvalRange<Evaluator, StorageIndex, /*Vectorizable*/ true> { + static const int PacketSize = + unpacket_traits<typename Evaluator::PacketReturnType>::size; + + static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, + const StorageIndex lastIdx) { Evaluator evaluator = *evaluator_in; - eigen_assert(last >= first); - Index i = first; - const int PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size; - if (last - first >= PacketSize) { - eigen_assert(first % PacketSize == 0); - Index last_chunk_offset = last - 4 * PacketSize; - // Manually unroll this loop since compilers don't do it. - for (; i <= last_chunk_offset; i += 4*PacketSize) { - evaluator.evalPacket(i); - evaluator.evalPacket(i+PacketSize); - evaluator.evalPacket(i+2*PacketSize); - evaluator.evalPacket(i+3*PacketSize); + eigen_assert(lastIdx >= firstIdx); + StorageIndex i = firstIdx; + if (lastIdx - firstIdx >= PacketSize) { + eigen_assert(firstIdx % PacketSize == 0); + StorageIndex last_chunk_offset = lastIdx - 4 * PacketSize; + // Give compiler a strong possibility to unroll the loop. But don't insist + // on unrolling, because if the function is expensive compiler should not + // unroll the loop at the expense of inlining. + for (; i <= last_chunk_offset; i += 4 * PacketSize) { + for (StorageIndex j = 0; j < 4; j++) { + evaluator.evalPacket(i + j * PacketSize); + } } - last_chunk_offset = last - PacketSize; + last_chunk_offset = lastIdx - PacketSize; for (; i <= last_chunk_offset; i += PacketSize) { evaluator.evalPacket(i); } } - for (; i < last; ++i) { + for (; i < lastIdx; ++i) { evaluator.evalScalar(i); } } + + static StorageIndex alignBlockSize(StorageIndex size) { + // Align block size to packet size and account for unrolling in run above. + if (size >= 16 * PacketSize) { + return (size + 4 * PacketSize - 1) & ~(4 * PacketSize - 1); + } + // Aligning to 4 * PacketSize would increase block size by more than 25%. + return (size + PacketSize - 1) & ~(PacketSize - 1); + } }; -template <typename Expression, bool Vectorizable> -class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable> { +template <typename Expression, bool Vectorizable, bool Tileable> +class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, Tileable> { public: - typedef typename Expression::Index Index; - static inline void run(const Expression& expr, const ThreadPoolDevice& device) - { + typedef typename Expression::Index StorageIndex; + + static EIGEN_STRONG_INLINE void run(const Expression& expr, + const ThreadPoolDevice& device) { typedef TensorEvaluator<Expression, ThreadPoolDevice> Evaluator; + typedef EvalRange<Evaluator, StorageIndex, Vectorizable> EvalRange; + Evaluator evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); - if (needs_assign) - { - const Index PacketSize = Vectorizable ? unpacket_traits<typename Evaluator::PacketReturnType>::size : 1; - const Index size = array_prod(evaluator.dimensions()); - size_t num_threads = device.numThreads(); -#ifdef EIGEN_USE_COST_MODEL - if (num_threads > 1) { - num_threads = TensorCostModel<ThreadPoolDevice>::numThreads( - size, evaluator.costPerCoeff(Vectorizable), num_threads); - } -#endif - if (num_threads == 1) { - EvalRange<Evaluator, Index, Vectorizable>::run(&evaluator, 0, size); - } else { - Index blocksz = std::ceil<Index>(static_cast<float>(size)/num_threads) + PacketSize - 1; - const Index blocksize = numext::maxi<Index>(PacketSize, (blocksz - (blocksz % PacketSize))); - const Index numblocks = size / blocksize; - - Barrier barrier(numblocks); - for (int i = 0; i < numblocks; ++i) { - device.enqueue_with_barrier( - &barrier, &EvalRange<Evaluator, Index, Vectorizable>::run, - &evaluator, i * blocksize, (i + 1) * blocksize); - } - if (numblocks * blocksize < size) { - EvalRange<Evaluator, Index, Vectorizable>::run( - &evaluator, numblocks * blocksize, size); - } - barrier.Wait(); - } + if (needs_assign) { + const StorageIndex size = array_prod(evaluator.dimensions()); + device.parallelFor(size, evaluator.costPerCoeff(Vectorizable), + EvalRange::alignBlockSize, + [&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { + EvalRange::run(&evaluator, firstIdx, lastIdx); + }); } evaluator.cleanup(); } }; -#endif + +template <typename Expression, bool Vectorizable> +class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, /*Tileable*/ true> { + public: + typedef typename traits<Expression>::Scalar Scalar; + typedef typename remove_const<Scalar>::type ScalarNoConst; + + typedef TensorEvaluator<Expression, ThreadPoolDevice> Evaluator; + typedef typename traits<Expression>::Index StorageIndex; + + static const int NumDims = traits<Expression>::NumDimensions; + + static EIGEN_STRONG_INLINE void run(const Expression& expr, + const ThreadPoolDevice& device) { + typedef TensorBlockMapper<ScalarNoConst, StorageIndex, NumDims, Evaluator::Layout> TensorBlockMapper; + + Evaluator evaluator(expr, device); + Index total_size = array_prod(evaluator.dimensions()); + Index cache_size = device.firstLevelCacheSize() / sizeof(Scalar); + if (total_size < cache_size) { + // TODO(andydavis) Reduce block management overhead for small tensors. + internal::TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, + false>::run(expr, device); + evaluator.cleanup(); + return; + } + + const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); + if (needs_assign) { + TensorBlockShapeType block_shape = kSkewedInnerDims; + Index block_total_size = 0; + // Query expression tree for desired block size/shape. + std::vector<internal::TensorOpResourceRequirements> resources; + evaluator.getResourceRequirements(&resources); + MergeResourceRequirements(resources, &block_shape, &block_total_size); + int num_threads = device.numThreads(); + + // Estimate minimum block size based on cost. + TensorOpCost cost = evaluator.costPerCoeff(Vectorizable); + double taskSize = TensorCostModel<ThreadPoolDevice>::taskSize(1, cost); + size_t block_size = static_cast<size_t>(1.0 / taskSize); + TensorBlockMapper block_mapper( + typename TensorBlockMapper::Dimensions(evaluator.dimensions()), + block_shape, block_size); + block_size = block_mapper.block_dims_total_size(); + const size_t aligned_blocksize = + EIGEN_MAX_ALIGN_BYTES * + divup<size_t>(block_size * sizeof(Scalar), EIGEN_MAX_ALIGN_BYTES); + void* buf = device.allocate((num_threads + 1) * aligned_blocksize); + device.parallelFor( + block_mapper.total_block_count(), cost * block_size, + [=, &device, &evaluator, &block_mapper](StorageIndex firstIdx, + StorageIndex lastIdx) { + // currentThreadId() returns -1 if called from a thread not in the + // thread pool, such as the main thread dispatching Eigen + // expressions. + const int thread_idx = device.currentThreadId(); + eigen_assert(thread_idx >= -1 && thread_idx < num_threads); + Scalar* thread_buf = reinterpret_cast<Scalar*>( + static_cast<char*>(buf) + aligned_blocksize * (thread_idx + 1)); + for (StorageIndex i = firstIdx; i < lastIdx; ++i) { + auto block = block_mapper.GetBlockForIndex(i, thread_buf); + evaluator.evalBlock(&block); + } + }); + device.deallocate(buf); + } + evaluator.cleanup(); + } +}; + +#endif // EIGEN_USE_THREADS // GPU: the evaluation of the expression is offloaded to a GPU. #if defined(EIGEN_USE_GPU) -template <typename Expression, bool Vectorizable> -class TensorExecutor<Expression, GpuDevice, Vectorizable> { +template <typename Expression, bool Vectorizable, bool Tileable> +class TensorExecutor<Expression, GpuDevice, Vectorizable, Tileable> { public: - typedef typename Expression::Index Index; + typedef typename Expression::Index StorageIndex; static void run(const Expression& expr, const GpuDevice& device); }; -#if defined(__CUDACC__) -template <typename Evaluator, typename Index, bool Vectorizable> +#if defined(EIGEN_GPUCC) +template <typename Evaluator, typename StorageIndex, bool Vectorizable> struct EigenMetaKernelEval { static __device__ EIGEN_ALWAYS_INLINE - void run(Evaluator& eval, Index first, Index last, Index step_size) { - for (Index i = first; i < last; i += step_size) { + void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx, StorageIndex step_size) { + for (StorageIndex i = firstIdx; i < lastIdx; i += step_size) { eval.evalScalar(i); } } }; -template <typename Evaluator, typename Index> -struct EigenMetaKernelEval<Evaluator, Index, true> { +template <typename Evaluator, typename StorageIndex> +struct EigenMetaKernelEval<Evaluator, StorageIndex, true> { static __device__ EIGEN_ALWAYS_INLINE - void run(Evaluator& eval, Index first, Index last, Index step_size) { - const Index PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size; - const Index vectorized_size = (last / PacketSize) * PacketSize; - const Index vectorized_step_size = step_size * PacketSize; + void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx, StorageIndex step_size) { + const StorageIndex PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size; + const StorageIndex vectorized_size = (lastIdx / PacketSize) * PacketSize; + const StorageIndex vectorized_step_size = step_size * PacketSize; // Use the vector path - for (Index i = first * PacketSize; i < vectorized_size; + for (StorageIndex i = firstIdx * PacketSize; i < vectorized_size; i += vectorized_step_size) { eval.evalPacket(i); } - for (Index i = vectorized_size + first; i < last; i += step_size) { + for (StorageIndex i = vectorized_size + firstIdx; i < lastIdx; i += step_size) { eval.evalScalar(i); } } }; -template <typename Evaluator, typename Index> +template <typename Evaluator, typename StorageIndex> __global__ void __launch_bounds__(1024) -EigenMetaKernel(Evaluator memcopied_eval, Index size) { +EigenMetaKernel(Evaluator eval, StorageIndex size) { - const Index first_index = blockIdx.x * blockDim.x + threadIdx.x; - const Index step_size = blockDim.x * gridDim.x; - - // Cuda memcopies the kernel arguments. That's fine for POD, but for more - // complex types such as evaluators we should really conform to the C++ - // standard and call a proper copy constructor. - Evaluator eval(memcopied_eval); + const StorageIndex first_index = blockIdx.x * blockDim.x + threadIdx.x; + const StorageIndex step_size = blockDim.x * gridDim.x; const bool vectorizable = Evaluator::PacketAccess & Evaluator::IsAligned; - EigenMetaKernelEval<Evaluator, Index, vectorizable>::run(eval, first_index, size, step_size); + EigenMetaKernelEval<Evaluator, StorageIndex, vectorizable>::run(eval, first_index, size, step_size); } /*static*/ -template <typename Expression, bool Vectorizable> -inline void TensorExecutor<Expression, GpuDevice, Vectorizable>::run( +template <typename Expression, bool Vectorizable, bool Tileable> +EIGEN_STRONG_INLINE void TensorExecutor<Expression, GpuDevice, Vectorizable, Tileable>::run( const Expression& expr, const GpuDevice& device) { TensorEvaluator<Expression, GpuDevice> evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); if (needs_assign) { - const int block_size = device.maxCudaThreadsPerBlock(); - const int max_blocks = device.getNumCudaMultiProcessors() * - device.maxCudaThreadsPerMultiProcessor() / block_size; - const Index size = array_prod(evaluator.dimensions()); + + const int block_size = device.maxGpuThreadsPerBlock(); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / block_size; + const StorageIndex size = array_prod(evaluator.dimensions()); // Create a least one block to ensure we won't crash when tensorflow calls with tensors of size 0. const int num_blocks = numext::maxi<int>(numext::mini<int>(max_blocks, divup<int>(size, block_size)), 1); - LAUNCH_CUDA_KERNEL( - (EigenMetaKernel<TensorEvaluator<Expression, GpuDevice>, Index>), + LAUNCH_GPU_KERNEL( + (EigenMetaKernel<TensorEvaluator<Expression, GpuDevice>, StorageIndex>), num_blocks, block_size, 0, device, evaluator, size); } evaluator.cleanup(); } -#endif // __CUDACC__ +#endif // EIGEN_GPUCC #endif // EIGEN_USE_GPU +// SYCL Executor policy +#ifdef EIGEN_USE_SYCL + +template <typename Expression, bool Vectorizable> +class TensorExecutor<Expression, SyclDevice, Vectorizable> { +public: + static EIGEN_STRONG_INLINE void run(const Expression &expr, const SyclDevice &device) { + // call TensorSYCL module + TensorSycl::run(expr, device); + } +}; + +#endif + } // end namespace internal } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h b/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h index 49d849e..4b6540c 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h
@@ -38,9 +38,9 @@ typedef typename remove_reference<XprTypeNested>::type _XprTypeNested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; - + typedef typename XprTraits::PointerType PointerType; enum { - Flags = 0, + Flags = 0 }; }; @@ -89,6 +89,7 @@ typedef typename remove_reference<XprTypeNested>::type _XprTypeNested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename TypeConversion<Scalar, typename XprTraits::PointerType>::type PointerType; }; template<typename UnaryOp, typename XprType> @@ -161,9 +162,13 @@ typedef typename remove_reference<RhsNested>::type _RhsNested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; - + typedef typename TypeConversion<Scalar, + typename conditional<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val, + typename traits<LhsXprType>::PointerType, + typename traits<RhsXprType>::PointerType>::type + >::type PointerType; enum { - Flags = 0, + Flags = 0 }; }; @@ -219,6 +224,90 @@ namespace internal { +template<typename TernaryOp, typename Arg1XprType, typename Arg2XprType, typename Arg3XprType> +struct traits<TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType> > +{ + // Type promotion to handle the case where the types of the args are different. + typedef typename result_of< + TernaryOp(typename Arg1XprType::Scalar, + typename Arg2XprType::Scalar, + typename Arg3XprType::Scalar)>::type Scalar; + typedef traits<Arg1XprType> XprTraits; + typedef typename traits<Arg1XprType>::StorageKind StorageKind; + typedef typename traits<Arg1XprType>::Index Index; + typedef typename Arg1XprType::Nested Arg1Nested; + typedef typename Arg2XprType::Nested Arg2Nested; + typedef typename Arg3XprType::Nested Arg3Nested; + typedef typename remove_reference<Arg1Nested>::type _Arg1Nested; + typedef typename remove_reference<Arg2Nested>::type _Arg2Nested; + typedef typename remove_reference<Arg3Nested>::type _Arg3Nested; + static const int NumDimensions = XprTraits::NumDimensions; + static const int Layout = XprTraits::Layout; + typedef typename TypeConversion<Scalar, + typename conditional<Pointer_type_promotion<typename Arg2XprType::Scalar, Scalar>::val, + typename traits<Arg2XprType>::PointerType, + typename traits<Arg3XprType>::PointerType>::type + >::type PointerType; + enum { + Flags = 0 + }; +}; + +template<typename TernaryOp, typename Arg1XprType, typename Arg2XprType, typename Arg3XprType> +struct eval<TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType>, Eigen::Dense> +{ + typedef const TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType>& type; +}; + +template<typename TernaryOp, typename Arg1XprType, typename Arg2XprType, typename Arg3XprType> +struct nested<TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType>, 1, typename eval<TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType> >::type> +{ + typedef TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType> type; +}; + +} // end namespace internal + + + +template<typename TernaryOp, typename Arg1XprType, typename Arg2XprType, typename Arg3XprType> +class TensorCwiseTernaryOp : public TensorBase<TensorCwiseTernaryOp<TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType>, ReadOnlyAccessors> +{ + public: + typedef typename Eigen::internal::traits<TensorCwiseTernaryOp>::Scalar Scalar; + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; + typedef Scalar CoeffReturnType; + typedef typename Eigen::internal::nested<TensorCwiseTernaryOp>::type Nested; + typedef typename Eigen::internal::traits<TensorCwiseTernaryOp>::StorageKind StorageKind; + typedef typename Eigen::internal::traits<TensorCwiseTernaryOp>::Index Index; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorCwiseTernaryOp(const Arg1XprType& arg1, const Arg2XprType& arg2, const Arg3XprType& arg3, const TernaryOp& func = TernaryOp()) + : m_arg1_xpr(arg1), m_arg2_xpr(arg2), m_arg3_xpr(arg3), m_functor(func) {} + + EIGEN_DEVICE_FUNC + const TernaryOp& functor() const { return m_functor; } + + /** \returns the nested expressions */ + EIGEN_DEVICE_FUNC + const typename internal::remove_all<typename Arg1XprType::Nested>::type& + arg1Expression() const { return m_arg1_xpr; } + + EIGEN_DEVICE_FUNC + const typename internal::remove_all<typename Arg2XprType::Nested>::type& + arg2Expression() const { return m_arg2_xpr; } + + EIGEN_DEVICE_FUNC + const typename internal::remove_all<typename Arg3XprType::Nested>::type& + arg3Expression() const { return m_arg3_xpr; } + + protected: + typename Arg1XprType::Nested m_arg1_xpr; + typename Arg2XprType::Nested m_arg2_xpr; + typename Arg3XprType::Nested m_arg3_xpr; + const TernaryOp m_functor; +}; + + +namespace internal { template<typename IfXprType, typename ThenXprType, typename ElseXprType> struct traits<TensorSelectOp<IfXprType, ThenXprType, ElseXprType> > : traits<ThenXprType> @@ -234,6 +323,9 @@ typedef typename ElseXprType::Nested ElseNested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename conditional<Pointer_type_promotion<typename ThenXprType::Scalar, Scalar>::val, + typename traits<ThenXprType>::PointerType, + typename traits<ElseXprType>::PointerType>::type PointerType; }; template<typename IfXprType, typename ThenXprType, typename ElseXprType> @@ -252,7 +344,7 @@ template<typename IfXprType, typename ThenXprType, typename ElseXprType> -class TensorSelectOp : public TensorBase<TensorSelectOp<IfXprType, ThenXprType, ElseXprType> > +class TensorSelectOp : public TensorBase<TensorSelectOp<IfXprType, ThenXprType, ElseXprType>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorSelectOp>::Scalar Scalar;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h index ece2ed9..480cf1f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h
@@ -71,6 +71,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename traits<XprType>::PointerType PointerType; }; template <typename FFT, typename XprType, int FFTResultType, int FFTDirection> @@ -135,6 +136,7 @@ IsAligned = false, PacketAccess = true, BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, RawAccess = false @@ -230,20 +232,32 @@ // t_n = exp(sqrt(-1) * pi * n^2 / line_len) // for n = 0, 1,..., line_len-1. // For n > 2 we use the recurrence t_n = t_{n-1}^2 / t_{n-2} * t_1^2 - pos_j_base_powered[0] = ComplexScalar(1, 0); - if (line_len > 1) { - const RealScalar pi_over_len(EIGEN_PI / line_len); - const ComplexScalar pos_j_base = ComplexScalar( - std::cos(pi_over_len), std::sin(pi_over_len)); - pos_j_base_powered[1] = pos_j_base; - if (line_len > 2) { - const ComplexScalar pos_j_base_sq = pos_j_base * pos_j_base; - for (int j = 2; j < line_len + 1; ++j) { - pos_j_base_powered[j] = pos_j_base_powered[j - 1] * - pos_j_base_powered[j - 1] / - pos_j_base_powered[j - 2] * pos_j_base_sq; - } - } + + // The recurrence is correct in exact arithmetic, but causes + // numerical issues for large transforms, especially in + // single-precision floating point. + // + // pos_j_base_powered[0] = ComplexScalar(1, 0); + // if (line_len > 1) { + // const ComplexScalar pos_j_base = ComplexScalar( + // numext::cos(M_PI / line_len), numext::sin(M_PI / line_len)); + // pos_j_base_powered[1] = pos_j_base; + // if (line_len > 2) { + // const ComplexScalar pos_j_base_sq = pos_j_base * pos_j_base; + // for (int i = 2; i < line_len + 1; ++i) { + // pos_j_base_powered[i] = pos_j_base_powered[i - 1] * + // pos_j_base_powered[i - 1] / + // pos_j_base_powered[i - 2] * + // pos_j_base_sq; + // } + // } + // } + // TODO(rmlarsen): Find a way to use Eigen's vectorized sin + // and cosine functions here. + for (int j = 0; j < line_len + 1; ++j) { + double arg = ((EIGEN_PI * j) * j) / line_len; + std::complex<double> tmp(numext::cos(arg), numext::sin(arg)); + pos_j_base_powered[j] = static_cast<ComplexScalar>(tmp); } } @@ -253,7 +267,7 @@ // get data into line_buf const Index stride = m_strides[dim]; if (stride == 1) { - memcpy(line_buf, &buf[base_offset], line_len*sizeof(ComplexScalar)); + m_device.memcpy(line_buf, &buf[base_offset], line_len*sizeof(ComplexScalar)); } else { Index offset = base_offset; for (int j = 0; j < line_len; ++j, offset += stride) { @@ -261,7 +275,7 @@ } } - // processs the line + // process the line if (is_power_of_two) { processDataLineCooleyTukey(line_buf, line_len, log_len); } @@ -271,7 +285,7 @@ // write back if (FFTDir == FFT_FORWARD && stride == 1) { - memcpy(&buf[base_offset], line_buf, line_len*sizeof(ComplexScalar)); + m_device.memcpy(&buf[base_offset], line_buf, line_len*sizeof(ComplexScalar)); } else { Index offset = base_offset; const ComplexScalar div_factor = ComplexScalar(1.0 / line_len, 0); @@ -329,7 +343,7 @@ for (Index i = 0; i < n; ++i) { if(FFTDir == FFT_FORWARD) { - a[i] = data[i] * std::conj(pos_j_base_powered[i]); + a[i] = data[i] * numext::conj(pos_j_base_powered[i]); } else { a[i] = data[i] * pos_j_base_powered[i]; @@ -344,7 +358,7 @@ b[i] = pos_j_base_powered[i]; } else { - b[i] = std::conj(pos_j_base_powered[i]); + b[i] = numext::conj(pos_j_base_powered[i]); } } for (Index i = n; i < m - n; ++i) { @@ -355,7 +369,7 @@ b[i] = pos_j_base_powered[m-i]; } else { - b[i] = std::conj(pos_j_base_powered[m-i]); + b[i] = numext::conj(pos_j_base_powered[m-i]); } } @@ -379,7 +393,7 @@ for (Index i = 0; i < n; ++i) { if(FFTDir == FFT_FORWARD) { - data[i] = a[i] * std::conj(pos_j_base_powered[i]); + data[i] = a[i] * numext::conj(pos_j_base_powered[i]); } else { data[i] = a[i] * pos_j_base_powered[i];
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h index 9c0ed43..71ba567 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h
@@ -20,7 +20,7 @@ * The fixed sized equivalent of * Eigen::Tensor<float, 3> t(3, 5, 7); * is - * Eigen::TensorFixedSize<float, Size<3,5,7>> t; + * Eigen::TensorFixedSize<float, Sizes<3,5,7>> t; */ template<typename Scalar_, typename Dimensions_, int Options_, typename IndexType> @@ -40,6 +40,9 @@ enum { IsAligned = bool(EIGEN_MAX_ALIGN_BYTES>0), + PacketAccess = (internal::packet_traits<Scalar>::size > 1), + BlockAccess = false, + PreferBlockAccess = false, Layout = Options_ & RowMajor ? RowMajor : ColMajor, CoordAccess = true, RawAccess = true @@ -65,7 +68,7 @@ inline Self& base() { return *this; } inline const Self& base() const { return *this; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index firstIndex, IndexTypes... otherIndices) const { @@ -97,7 +100,7 @@ } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index firstIndex, IndexTypes... otherIndices) { @@ -128,8 +131,7 @@ return m_storage.data()[0]; } - -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) const { @@ -137,8 +139,54 @@ EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) return this->operator()(array<Index, NumIndices>{{firstIndex, otherIndices...}}); } +#else + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1) const + { + if (Options&RowMajor) { + const Index index = i1 + i0 * m_storage.dimensions()[1]; + return m_storage.data()[index]; + } else { + const Index index = i0 + i1 * m_storage.dimensions()[0]; + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1, Index i2) const + { + if (Options&RowMajor) { + const Index index = i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * i2); + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1, Index i2, Index i3) const + { + if (Options&RowMajor) { + const Index index = i3 + m_storage.dimensions()[3] * (i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0)); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * (i2 + m_storage.dimensions()[2] * i3)); + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1, Index i2, Index i3, Index i4) const + { + if (Options&RowMajor) { + const Index index = i4 + m_storage.dimensions()[4] * (i3 + m_storage.dimensions()[3] * (i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0))); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * (i2 + m_storage.dimensions()[2] * (i3 + m_storage.dimensions()[3] * i4))); + return m_storage.data()[index]; + } + } #endif + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(const array<Index, NumIndices>& indices) const { @@ -168,7 +216,7 @@ return coeff(index); } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) { @@ -176,6 +224,51 @@ EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) return operator()(array<Index, NumIndices>{{firstIndex, otherIndices...}}); } +#else + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1) + { + if (Options&RowMajor) { + const Index index = i1 + i0 * m_storage.dimensions()[1]; + return m_storage.data()[index]; + } else { + const Index index = i0 + i1 * m_storage.dimensions()[0]; + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1, Index i2) + { + if (Options&RowMajor) { + const Index index = i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * i2); + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1, Index i2, Index i3) + { + if (Options&RowMajor) { + const Index index = i3 + m_storage.dimensions()[3] * (i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0)); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * (i2 + m_storage.dimensions()[2] * i3)); + return m_storage.data()[index]; + } + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1, Index i2, Index i3, Index i4) + { + if (Options&RowMajor) { + const Index index = i4 + m_storage.dimensions()[4] * (i3 + m_storage.dimensions()[3] * (i2 + m_storage.dimensions()[2] * (i1 + m_storage.dimensions()[1] * i0))); + return m_storage.data()[index]; + } else { + const Index index = i0 + m_storage.dimensions()[0] * (i1 + m_storage.dimensions()[1] * (i2 + m_storage.dimensions()[2] * (i3 + m_storage.dimensions()[3] * i4))); + return m_storage.data()[index]; + } + } #endif EIGEN_DEVICE_FUNC @@ -219,7 +312,7 @@ { } -#ifdef EIGEN_HAVE_RVALUE_REFERENCES +#if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorFixedSize(Self&& other) : m_storage(other.m_storage) {
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h b/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h index d2b0b30..78068be 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h
@@ -32,9 +32,10 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; enum { - Flags = 0, + Flags = 0 }; }; @@ -55,7 +56,7 @@ template<typename XprType> -class TensorForcedEvalOp : public TensorBase<TensorForcedEvalOp<XprType> > +class TensorForcedEvalOp : public TensorBase<TensorForcedEvalOp<XprType>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorForcedEvalOp>::Scalar Scalar; @@ -86,38 +87,44 @@ typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = true, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, RawAccess = true }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) + /// op_ is used for sycl : m_impl(op.expression(), device), m_op(op.expression()), m_device(device), m_buffer(NULL) { } EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_impl.dimensions(); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType*) { - const Index numValues = m_impl.dimensions().TotalSize(); - m_buffer = (CoeffReturnType*)m_device.allocate(numValues * sizeof(CoeffReturnType)); + #if !defined(EIGEN_HIPCC) + EIGEN_DEVICE_FUNC + #endif + EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType*) { + const Index numValues = internal::array_prod(m_impl.dimensions()); + m_buffer = (CoeffReturnType*)m_device.allocate_temp(numValues * sizeof(CoeffReturnType)); // Should initialize the memory in case we're dealing with non POD types. if (NumTraits<CoeffReturnType>::RequireInitialization) { for (Index i = 0; i < numValues; ++i) { new(m_buffer+i) CoeffReturnType(); } } - typedef TensorEvalToOp<const ArgType> EvalTo; + typedef TensorEvalToOp< const typename internal::remove_const<ArgType>::type > EvalTo; EvalTo evalToTmp(m_buffer, m_op); - const bool PacketAccess = internal::IsVectorizable<Device, const ArgType>::value; - internal::TensorExecutor<const EvalTo, Device, PacketAccess>::run(evalToTmp, m_device); + const bool Vectorize = internal::IsVectorizable<Device, const ArgType>::value; + internal::TensorExecutor<const EvalTo, typename internal::remove_const<Device>::type, Vectorize>::run(evalToTmp, m_device); return true; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { - m_device.deallocate(m_buffer); + m_device.deallocate_temp(m_buffer); m_buffer = NULL; } @@ -136,8 +143,13 @@ return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return m_buffer; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename Eigen::internal::traits<XprType>::PointerType data() const { return m_buffer; } + /// required by sycl in order to extract the sycl accessor + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() { return m_impl; } + /// used by sycl in order to build the sycl buffer + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Device& device() const{return m_device;} private: TensorEvaluator<ArgType, Device> m_impl; const ArgType m_op;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorForwardDeclarations.h b/unsupported/Eigen/CXX11/src/Tensor/TensorForwardDeclarations.h index a8bd8b8..09b7c99 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorForwardDeclarations.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorForwardDeclarations.h
@@ -12,21 +12,69 @@ namespace Eigen { +// MakePointer class is used as a container of the address space of the pointer +// on the host and on the device. From the host side it generates the T* pointer +// and when EIGEN_USE_SYCL is used it construct a buffer with a map_allocator to +// T* m_data on the host. It is always called on the device. +// Specialisation of MakePointer class for creating the sycl buffer with +// map_allocator. +template<typename T> struct MakePointer { + typedef T* Type; + typedef T& RefType; + typedef T ScalarType; +}; + +// The PointerType class is a container of the device specefic pointer +// used for referring to a Pointer on TensorEvaluator class. While the TensorExpression +// is a device-agnostic type and need MakePointer class for type conversion, +// the TensorEvaluator calls can be specialized for a device, hence it is possible +// to construct different types of temproray storage memory in TensorEvaluator +// for different devices by specializing the following PointerType class. +template<typename T, typename Device> struct PointerType : MakePointer<T>{}; + +namespace internal{ +template<typename A, typename B> struct Pointer_type_promotion { + static const bool val=false; +}; +template<typename A> struct Pointer_type_promotion<A, A> { + static const bool val = true; +}; +template<typename A, typename B> struct TypeConversion; +#ifndef __SYCL_DEVICE_ONLY__ +template<typename A, typename B> struct TypeConversion{ + typedef A* type; +}; +#endif +} + +#if defined(EIGEN_USE_SYCL) +namespace TensorSycl { +namespace internal{ +template <typename HostExpr, typename FunctorExpr, typename Tuple_of_Acc, typename Dims, typename Op, typename Index> class ReductionFunctor; +template<typename CoeffReturnType ,typename OutAccessor, typename HostExpr, typename FunctorExpr, typename Op, typename Dims, typename Index, typename TupleType> +class FullReductionKernelFunctor; +} +} +#endif + + + +template<typename PlainObjectType, int Options_ = Unaligned, template <class> class MakePointer_ = MakePointer> class TensorMap; template<typename Scalar_, int NumIndices_, int Options_ = 0, typename IndexType = DenseIndex> class Tensor; template<typename Scalar_, typename Dimensions, int Options_ = 0, typename IndexType = DenseIndex> class TensorFixedSize; -template<typename PlainObjectType, int Options_ = Unaligned> class TensorMap; template<typename PlainObjectType> class TensorRef; -template<typename Derived, int AccessLevel = internal::accessors_level<Derived>::value> class TensorBase; +template<typename Derived, int AccessLevel> class TensorBase; template<typename NullaryOp, typename PlainObjectType> class TensorCwiseNullaryOp; template<typename UnaryOp, typename XprType> class TensorCwiseUnaryOp; template<typename BinaryOp, typename LeftXprType, typename RightXprType> class TensorCwiseBinaryOp; +template<typename TernaryOp, typename Arg1XprType, typename Arg2XprType, typename Arg3XprType> class TensorCwiseTernaryOp; template<typename IfXprType, typename ThenXprType, typename ElseXprType> class TensorSelectOp; -template<typename Op, typename Dims, typename XprType> class TensorReductionOp; +template<typename Op, typename Dims, typename XprType, template <class> class MakePointer_ = MakePointer > class TensorReductionOp; template<typename XprType> class TensorIndexTupleOp; template<typename ReduceOp, typename Dims, typename XprType> class TensorTupleReducerOp; template<typename Axis, typename LeftXprType, typename RightXprType> class TensorConcatenationOp; -template<typename Dimensions, typename LeftXprType, typename RightXprType> class TensorContractionOp; +template<typename Dimensions, typename LeftXprType, typename RightXprType, typename OutputKernelType> class TensorContractionOp; template<typename TargetType, typename XprType> class TensorConversionOp; template<typename Dimensions, typename InputXprType, typename KernelXprType> class TensorConvolutionOp; template<typename FFT, typename XprType, int FFTDataType, int FFTDirection> class TensorFFTOp; @@ -42,22 +90,28 @@ template<typename PaddingDimensions, typename XprType> class TensorPaddingOp; template<typename Shuffle, typename XprType> class TensorShufflingOp; template<typename Strides, typename XprType> class TensorStridingOp; +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType> class TensorStridingSlicingOp; template<typename Strides, typename XprType> class TensorInflationOp; template<typename Generator, typename XprType> class TensorGeneratorOp; template<typename LeftXprType, typename RightXprType> class TensorAssignOp; +template<typename Op, typename XprType> class TensorScanOp; +template<typename Dims, typename XprType> class TensorTraceOp; template<typename CustomUnaryFunc, typename XprType> class TensorCustomUnaryOp; template<typename CustomBinaryFunc, typename LhsXprType, typename RhsXprType> class TensorCustomBinaryOp; -template<typename XprType> class TensorEvalToOp; +template<typename XprType, template <class> class MakePointer_ = MakePointer> class TensorEvalToOp; template<typename XprType> class TensorForcedEvalOp; template<typename ExpressionType, typename DeviceType> class TensorDevice; template<typename Derived, typename Device> struct TensorEvaluator; +struct NoOpOutputKernel; + struct DefaultDevice; struct ThreadPoolDevice; struct GpuDevice; +struct SyclDevice; enum FFTResultType { RealPart = 0, @@ -84,8 +138,18 @@ TensorEvaluator<Expression, GpuDevice>::IsAligned; }; +template <typename Device, typename Expression> +struct IsTileable { + // Check that block evaluation is supported and it's a preferred option (at + // least one sub-expression has much faster block evaluation, e.g. + // broadcasting). + static const bool value = TensorEvaluator<Expression, Device>::BlockAccess && + TensorEvaluator<Expression, Device>::PreferBlockAccess; +}; + template <typename Expression, typename Device, - bool Vectorizable = IsVectorizable<Device, Expression>::value> + bool Vectorizable = IsVectorizable<Device, Expression>::value, + bool Tileable = IsTileable<Device, Expression>::value> class TensorExecutor; } // end namespace internal
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h index 33cd003..51572f9 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h
@@ -20,12 +20,12 @@ template <typename Scalar> struct scalar_mod_op { EIGEN_DEVICE_FUNC scalar_mod_op(const Scalar& divisor) : m_divisor(divisor) {} - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a % m_divisor; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a % m_divisor; } const Scalar m_divisor; }; template <typename Scalar> struct functor_traits<scalar_mod_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::template Div<false>::Cost, PacketAccess = false }; }; +{ enum { Cost = scalar_div_cost<Scalar,false>::value, PacketAccess = false }; }; /** \internal @@ -33,16 +33,16 @@ */ template <typename Scalar> struct scalar_mod2_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_mod2_op); - EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a, const Scalar& b) const { return a % b; } + EIGEN_EMPTY_STRUCT_CTOR(scalar_mod2_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a, const Scalar& b) const { return a % b; } }; template <typename Scalar> struct functor_traits<scalar_mod2_op<Scalar> > -{ enum { Cost = NumTraits<Scalar>::template Div<false>::Cost, PacketAccess = false }; }; +{ enum { Cost = scalar_div_cost<Scalar,false>::value, PacketAccess = false }; }; template <typename Scalar> struct scalar_fmod_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_fmod_op); + EIGEN_EMPTY_STRUCT_CTOR(scalar_fmod_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a, const Scalar& b) const { return numext::fmod(a, b); @@ -54,44 +54,22 @@ PacketAccess = false }; }; - -/** \internal - * \brief Template functor to compute the sigmoid of a scalar - * \sa class CwiseUnaryOp, ArrayBase::sigmoid() - */ -template <typename T> -struct scalar_sigmoid_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { - const T one = T(1); - return one / (one + numext::exp(-x)); - } - - template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - Packet packetOp(const Packet& x) const { - const Packet one = pset1<Packet>(1); - return pdiv(one, padd(one, pexp(pnegate(x)))); - } -}; - -template <typename T> -struct functor_traits<scalar_sigmoid_op<T> > { +template<typename Reducer, typename Device> +struct reducer_traits { enum { - Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost * 6, - PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasDiv && - packet_traits<T>::HasNegate && packet_traits<T>::HasExp + Cost = 1, + PacketAccess = false, + IsStateful = false, + IsExactlyAssociative = true }; }; - // Standard reduction functors template <typename T> struct SumReducer { - static const bool PacketAccess = true; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) const { - (*accum) += t; + internal::scalar_sum_op<T> sum_op; + *accum = sum_op(*accum, t); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p, Packet* accum) const { @@ -115,20 +93,29 @@ } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalizeBoth(const T saccum, const Packet& vaccum) const { - return saccum + predux(vaccum); + internal::scalar_sum_op<T> sum_op; + return sum_op(saccum, predux(vaccum)); } }; +template <typename T, typename Device> +struct reducer_traits<SumReducer<T>, Device> { + enum { + Cost = NumTraits<T>::AddCost, + PacketAccess = PacketType<T, Device>::HasAdd, + IsStateful = false, + IsExactlyAssociative = NumTraits<T>::IsInteger + }; +}; + template <typename T> struct MeanReducer { - static const bool PacketAccess = !NumTraits<T>::IsInteger; - static const bool IsStateful = true; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE MeanReducer() : scalarCount_(0), packetCount_(0) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) { - (*accum) += t; + internal::scalar_sum_op<T> sum_op; + *accum = sum_op(*accum, t); scalarCount_++; } template <typename Packet> @@ -146,15 +133,20 @@ return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { - return accum / scalarCount_; + internal::scalar_quotient_op<T> quotient_op; + return quotient_op(accum, T(scalarCount_)); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet finalizePacket(const Packet& vaccum) const { - return pdiv(vaccum, pset1<Packet>(packetCount_)); + return pdiv(vaccum, pset1<Packet>(T(packetCount_))); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalizeBoth(const T saccum, const Packet& vaccum) const { - return (saccum + predux(vaccum)) / (scalarCount_ + packetCount_ * unpacket_traits<Packet>::size); + internal::scalar_sum_op<T> sum_op; + internal::scalar_quotient_op<T> quotient_op; + return quotient_op( + sum_op(saccum, predux(vaccum)), + T(scalarCount_ + packetCount_ * unpacket_traits<Packet>::size)); } protected: @@ -162,11 +154,46 @@ DenseIndex packetCount_; }; +template <typename T, typename Device> +struct reducer_traits<MeanReducer<T>, Device> { + enum { + Cost = NumTraits<T>::AddCost, + PacketAccess = PacketType<T, Device>::HasAdd && + PacketType<T, Device>::HasDiv && !NumTraits<T>::IsInteger, + IsStateful = true, + IsExactlyAssociative = NumTraits<T>::IsInteger + }; +}; + + +template <typename T, bool IsMax = true, bool IsInteger = true> +struct MinMaxBottomValue { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T bottom_value() { + return Eigen::NumTraits<T>::lowest(); + } +}; +template <typename T> +struct MinMaxBottomValue<T, true, false> { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T bottom_value() { + return -Eigen::NumTraits<T>::infinity(); + } +}; +template <typename T> +struct MinMaxBottomValue<T, false, true> { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T bottom_value() { + return Eigen::NumTraits<T>::highest(); + } +}; +template <typename T> +struct MinMaxBottomValue<T, false, false> { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T bottom_value() { + return Eigen::NumTraits<T>::infinity(); + } +}; + + template <typename T> struct MaxReducer { - static const bool PacketAccess = true; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) const { if (t > *accum) { *accum = t; } } @@ -174,9 +201,8 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p, Packet* accum) const { (*accum) = pmax<Packet>(*accum, p); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return Eigen::NumTraits<T>::lowest(); + return MinMaxBottomValue<T, true, Eigen::NumTraits<T>::IsInteger>::bottom_value(); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { @@ -195,11 +221,19 @@ } }; +template <typename T, typename Device> +struct reducer_traits<MaxReducer<T>, Device> { + enum { + Cost = NumTraits<T>::AddCost, + PacketAccess = PacketType<T, Device>::HasMax, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + + template <typename T> struct MinReducer { - static const bool PacketAccess = true; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) const { if (t < *accum) { *accum = t; } } @@ -207,9 +241,8 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p, Packet* accum) const { (*accum) = pmin<Packet>(*accum, p); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return Eigen::NumTraits<T>::highest(); + return MinMaxBottomValue<T, false, Eigen::NumTraits<T>::IsInteger>::bottom_value(); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { @@ -228,14 +261,22 @@ } }; +template <typename T, typename Device> +struct reducer_traits<MinReducer<T>, Device> { + enum { + Cost = NumTraits<T>::AddCost, + PacketAccess = PacketType<T, Device>::HasMin, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + template <typename T> struct ProdReducer { - static const bool PacketAccess = true; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) const { - (*accum) *= t; + internal::scalar_product_op<T> prod_op; + (*accum) = prod_op(*accum, t); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p, Packet* accum) const { @@ -259,16 +300,24 @@ } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalizeBoth(const T saccum, const Packet& vaccum) const { - return saccum * predux_mul(vaccum); + internal::scalar_product_op<T> prod_op; + return prod_op(saccum, predux_mul(vaccum)); } }; +template <typename T, typename Device> +struct reducer_traits<ProdReducer<T>, Device> { + enum { + Cost = NumTraits<T>::MulCost, + PacketAccess = PacketType<T, Device>::HasMul, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + struct AndReducer { - static const bool PacketAccess = false; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum && t; } @@ -280,10 +329,18 @@ } }; -struct OrReducer { - static const bool PacketAccess = false; - static const bool IsStateful = false; +template <typename Device> +struct reducer_traits<AndReducer, Device> { + enum { + Cost = 1, + PacketAccess = false, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + +struct OrReducer { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum || t; } @@ -295,12 +352,20 @@ } }; +template <typename Device> +struct reducer_traits<OrReducer, Device> { + enum { + Cost = 1, + PacketAccess = false, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + + // Argmin/Argmax reducers template <typename T> struct ArgMaxTupleReducer { - static const bool PacketAccess = false; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) const { if (t.second > accum->second) { *accum = t; } } @@ -312,11 +377,19 @@ } }; +template <typename T, typename Device> +struct reducer_traits<ArgMaxTupleReducer<T>, Device> { + enum { + Cost = NumTraits<T>::AddCost, + PacketAccess = false, + IsStateful = false, + IsExactlyAssociative = true + }; +}; + + template <typename T> struct ArgMinTupleReducer { - static const bool PacketAccess = false; - static const bool IsStateful = false; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T& t, T* accum) const { if (t.second < accum->second) { *accum = t; } } @@ -328,457 +401,13 @@ } }; - -// Random number generation -namespace { -#ifdef __CUDA_ARCH__ -__device__ int get_random_seed() { - return clock(); -} -#else -int get_random_seed() { -#ifdef _WIN32 - SYSTEMTIME st; - GetSystemTime(&st); - return st.wSecond + 1000 * st.wMilliseconds; -#elif defined __APPLE__ - return static_cast<int>(mach_absolute_time()); -#else - timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return static_cast<int>(ts.tv_nsec); -#endif -} -#endif -} - -#if !defined (EIGEN_USE_GPU) || !defined(__CUDACC__) || !defined(__CUDA_ARCH__) -// We're not compiling a cuda kernel -template <typename T> class UniformRandomGenerator { - - public: - static const bool PacketAccess = true; - - UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - if (!deterministic) { - srand(get_random_seed()); - } - } - UniformRandomGenerator(const UniformRandomGenerator& other) { - m_deterministic = other.m_deterministic; - } - - template<typename Index> - T operator()(Index) const { - return random<T>(); - } - template<typename Index, typename PacketType> - PacketType packetOp(Index) const { - const int packetSize = internal::unpacket_traits<PacketType>::size; - EIGEN_ALIGN_MAX T values[packetSize]; - for (int i = 0; i < packetSize; ++i) { - values[i] = random<T>(); - } - return internal::pload<PacketType>(values); - } - - private: - bool m_deterministic; -}; - -#if __cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900 -template <> class UniformRandomGenerator<float> { - public: - static const bool PacketAccess = true; - - UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_generator(new std::mt19937()) { - if (!deterministic) { - m_generator->seed(get_random_seed()); - } - } - UniformRandomGenerator(const UniformRandomGenerator<float>& other) { - m_generator = new std::mt19937(); - m_generator->seed(other(0) * UINT_MAX); - m_deterministic = other.m_deterministic; - } - ~UniformRandomGenerator() { - delete m_generator; - } - - template<typename Index> - float operator()(Index) const { - return m_distribution(*m_generator); - } - template<typename Index, typename PacketType> - PacketType packetOp(Index i) const { - const int packetSize = internal::unpacket_traits<PacketType>::size; - EIGEN_ALIGN_MAX float values[packetSize]; - for (int k = 0; k < packetSize; ++k) { - values[k] = this->operator()(i); - } - return internal::pload<PacketType>(values); - } - - private: - UniformRandomGenerator& operator = (const UniformRandomGenerator&); - // Make sure m_deterministic comes first to match the layout of the cpu - // version of the code. - bool m_deterministic; - std::mt19937* m_generator; - mutable std::uniform_real_distribution<float> m_distribution; -}; - -template <> class UniformRandomGenerator<double> { - public: - static const bool PacketAccess = true; - - UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_generator(new std::mt19937()) { - if (!deterministic) { - m_generator->seed(get_random_seed()); - } - } - UniformRandomGenerator(const UniformRandomGenerator<double>& other) { - m_generator = new std::mt19937(); - m_generator->seed(other(0) * UINT_MAX); - m_deterministic = other.m_deterministic; - } - ~UniformRandomGenerator() { - delete m_generator; - } - - template<typename Index> - double operator()(Index) const { - return m_distribution(*m_generator); - } - template<typename Index, typename PacketType> - PacketType packetOp(Index i) const { - const int packetSize = internal::unpacket_traits<PacketType>::size; - EIGEN_ALIGN_MAX double values[packetSize]; - for (int k = 0; k < packetSize; ++k) { - values[k] = this->operator()(i); - } - return internal::pload<PacketType>(values); - } - - private: - UniformRandomGenerator& operator = (const UniformRandomGenerator&); - // Make sure m_deterministic comes first to match the layout of the cpu - // version of the code. - bool m_deterministic; - std::mt19937* m_generator; - mutable std::uniform_real_distribution<double> m_distribution; -}; -#endif - -#else - -// We're compiling a cuda kernel -template <typename T> class UniformRandomGenerator; - -template <> class UniformRandomGenerator<float> { - public: - static const bool PacketAccess = true; - - __device__ UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - - __device__ UniformRandomGenerator(const UniformRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - - template<typename Index> - __device__ float operator()(Index) const { - return curand_uniform(&m_state); - } - template<typename Index, typename PacketType> - __device__ float4 packetOp(Index) const { - EIGEN_STATIC_ASSERT((is_same<PacketType, float4>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - return curand_uniform4(&m_state); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class UniformRandomGenerator<double> { - public: - static const bool PacketAccess = true; - - __device__ UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ UniformRandomGenerator(const UniformRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ double operator()(Index) const { - return curand_uniform_double(&m_state); - } - template<typename Index, typename PacketType> - __device__ double2 packetOp(Index) const { - EIGEN_STATIC_ASSERT((is_same<PacketType, double2>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - return curand_uniform2_double(&m_state); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class UniformRandomGenerator<std::complex<float> > { - public: - static const bool PacketAccess = false; - - __device__ UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ UniformRandomGenerator(const UniformRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ std::complex<float> operator()(Index) const { - float4 vals = curand_uniform4(&m_state); - return std::complex<float>(vals.x, vals.y); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class UniformRandomGenerator<std::complex<double> > { - public: - static const bool PacketAccess = false; - - __device__ UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ UniformRandomGenerator(const UniformRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ std::complex<double> operator()(Index) const { - double2 vals = curand_uniform2_double(&m_state); - return std::complex<double>(vals.x, vals.y); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -#endif - -template <typename Scalar> -struct functor_traits<UniformRandomGenerator<Scalar> > { +template <typename T, typename Device> +struct reducer_traits<ArgMinTupleReducer<T>, Device> { enum { - // Rough estimate. - Cost = 100 * NumTraits<Scalar>::MulCost, - PacketAccess = UniformRandomGenerator<Scalar>::PacketAccess - }; -}; - - - -#if (!defined (EIGEN_USE_GPU) || !defined(__CUDACC__) || !defined(__CUDA_ARCH__)) && (__cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900) -// We're not compiling a cuda kernel -template <typename T> class NormalRandomGenerator { - public: - static const bool PacketAccess = true; - - NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_distribution(0, 1), m_generator(new std::mt19937()) { - if (!deterministic) { - m_generator->seed(get_random_seed()); - } - } - NormalRandomGenerator(const NormalRandomGenerator& other) - : m_deterministic(other.m_deterministic), m_distribution(other.m_distribution), m_generator(new std::mt19937()) { - m_generator->seed(other(0) * UINT_MAX); - } - ~NormalRandomGenerator() { - delete m_generator; - } - template<typename Index> - T operator()(Index) const { - return m_distribution(*m_generator); - } - template<typename Index, typename PacketType> - PacketType packetOp(Index) const { - const int packetSize = internal::unpacket_traits<PacketType>::size; - EIGEN_ALIGN_MAX T values[packetSize]; - for (int i = 0; i < packetSize; ++i) { - values[i] = m_distribution(*m_generator); - } - return internal::pload<PacketType>(values); - } - - private: - // No assignment - NormalRandomGenerator& operator = (const NormalRandomGenerator&); - - bool m_deterministic; - mutable std::normal_distribution<T> m_distribution; - std::mt19937* m_generator; -}; - -#elif defined (EIGEN_USE_GPU) && defined(__CUDACC__) && defined(__CUDA_ARCH__) - -// We're compiling a cuda kernel -template <typename T> class NormalRandomGenerator; - -template <> class NormalRandomGenerator<float> { - public: - static const bool PacketAccess = true; - - __device__ NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ NormalRandomGenerator(const NormalRandomGenerator<float>& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ float operator()(Index) const { - return curand_normal(&m_state); - } - template<typename Index, typename PacketType> - __device__ float4 packetOp(Index) const { - EIGEN_STATIC_ASSERT((is_same<PacketType, float4>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - return curand_normal4(&m_state); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class NormalRandomGenerator<double> { - public: - static const bool PacketAccess = true; - - __device__ NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ NormalRandomGenerator(const NormalRandomGenerator<double>& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ double operator()(Index) const { - return curand_normal_double(&m_state); - } - template<typename Index, typename PacketType> - __device__ double2 packetOp(Index) const { - EIGEN_STATIC_ASSERT((is_same<PacketType, double2>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - return curand_normal2_double(&m_state); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class NormalRandomGenerator<std::complex<float> > { - public: - static const bool PacketAccess = false; - - __device__ NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ NormalRandomGenerator(const NormalRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ std::complex<float> operator()(Index) const { - float4 vals = curand_normal4(&m_state); - return std::complex<float>(vals.x, vals.y); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -template <> class NormalRandomGenerator<std::complex<double> > { - public: - static const bool PacketAccess = false; - - __device__ NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - __device__ NormalRandomGenerator(const NormalRandomGenerator& other) { - m_deterministic = other.m_deterministic; - const int tid = blockIdx.x * blockDim.x + threadIdx.x; - const int seed = m_deterministic ? 0 : get_random_seed(); - curand_init(seed, tid, 0, &m_state); - } - template<typename Index> - __device__ std::complex<double> operator()(Index) const { - double2 vals = curand_normal2_double(&m_state); - return std::complex<double>(vals.x, vals.y); - } - - private: - bool m_deterministic; - mutable curandStatePhilox4_32_10_t m_state; -}; - -#else - -template <typename T> class NormalRandomGenerator { - public: - static const bool PacketAccess = false; - NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) {} - - private: - bool m_deterministic; -}; - -#endif - -template <typename Scalar> -struct functor_traits<NormalRandomGenerator<Scalar> > { - enum { - // Rough estimate. - Cost = 100 * NumTraits<Scalar>::MulCost, - PacketAccess = NormalRandomGenerator<Scalar>::PacketAccess + Cost = NumTraits<T>::AddCost, + PacketAccess = false, + IsStateful = false, + IsExactlyAssociative = true }; }; @@ -797,7 +426,7 @@ } } - T operator()(const array<Index, NumDims>& coordinates) const { + EIGEN_DEVICE_FUNC T operator()(const array<Index, NumDims>& coordinates) const { T tmp = T(0); for (size_t i = 0; i < NumDims; ++i) { T offset = coordinates[i] - m_means[i]; @@ -821,6 +450,25 @@ }; }; +template <typename Scalar> +struct scalar_clamp_op { + EIGEN_DEVICE_FUNC inline scalar_clamp_op(const Scalar& _min, const Scalar& _max) : m_min(_min), m_max(_max) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar + operator()(const Scalar& x) const { + return numext::mini(numext::maxi(x, m_min), m_max); + } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet + packetOp(const Packet& x) const { + return internal::pmin(internal::pmax(x, pset1<Packet>(m_min)), pset1<Packet>(m_max)); + } + const Scalar m_min; + const Scalar m_max; +}; +template<typename Scalar> +struct functor_traits<scalar_clamp_op<Scalar> > +{ enum { Cost = 2 * NumTraits<Scalar>::AddCost, PacketAccess = (packet_traits<Scalar>::HasMin && packet_traits<Scalar>::HasMax)}; }; + } // end namespace internal } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h b/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h index 8ff7d58..ac66f9c 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h
@@ -12,7 +12,7 @@ namespace Eigen { -/** \class TensorGenerator +/** \class TensorGeneratorOp * \ingroup CXX11_Tensor_Module * * \brief Tensor generator class. @@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename Generator, typename XprType> @@ -89,8 +90,9 @@ typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { IsAligned = false, - PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -98,9 +100,12 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_generator(op.generator()) +#ifdef EIGEN_USE_SYCL + , m_argImpl(op.expression(), device) +#endif { - TensorEvaluator<ArgType, Device> impl(op.expression(), device); - m_dimensions = impl.dimensions(); + TensorEvaluator<ArgType, Device> argImpl(op.expression(), device); + m_dimensions = argImpl.dimensions(); if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { m_strides[0] = 1; @@ -133,8 +138,8 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - const int packetSize = internal::unpacket_traits<PacketReturnType>::size; - EIGEN_STATIC_ASSERT(packetSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + const int packetSize = PacketType<CoeffReturnType, Device>::size; + EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+packetSize-1 < dimensions().TotalSize()); EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[packetSize]; @@ -153,7 +158,12 @@ TensorOpCost::MulCost<Scalar>()); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return m_argImpl; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Generator& functor() const { return m_generator; } +#endif protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -178,6 +188,9 @@ Dimensions m_dimensions; array<Index, NumDims> m_strides; Generator m_generator; +#ifdef EIGEN_USE_SYCL + TensorEvaluator<ArgType, Device> m_argImpl; +#endif }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGlobalFunctions.h b/unsupported/Eigen/CXX11/src/Tensor/TensorGlobalFunctions.h new file mode 100644 index 0000000..665b861 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorGlobalFunctions.h
@@ -0,0 +1,33 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Eugene Brevdo <ebrevdo@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_GLOBAL_FUNCTIONS_H +#define EIGEN_CXX11_TENSOR_TENSOR_GLOBAL_FUNCTIONS_H + +namespace Eigen { + +/** \cpp11 \returns an expression of the coefficient-wise betainc(\a x, \a a, \a b) to the given tensors. + * + * This function computes the regularized incomplete beta function (integral). + * + */ +template <typename ADerived, typename BDerived, typename XDerived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const + TensorCwiseTernaryOp<internal::scalar_betainc_op<typename XDerived::Scalar>, + const ADerived, const BDerived, const XDerived> + betainc(const ADerived& a, const BDerived& b, const XDerived& x) { + return TensorCwiseTernaryOp< + internal::scalar_betainc_op<typename XDerived::Scalar>, const ADerived, + const BDerived, const XDerived>( + a, b, x, internal::scalar_betainc_op<typename XDerived::Scalar>()); +} + +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_GLOBAL_FUNCTIONS_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h b/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h new file mode 100644 index 0000000..40f58f6 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h
@@ -0,0 +1,88 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// Copyright (C) 2018 Deven Desai <deven.desai.amd@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H) +#define EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H + +// Note that we are using EIGEN_USE_HIP here instead of EIGEN_HIPCC...this is by design +// There is code in the Tensorflow codebase that will define EIGEN_USE_GPU, but +// for some reason gets sent to the gcc/host compiler instead of the gpu/nvcc/hipcc compiler +// When compiling such files, gcc will end up trying to pick up the CUDA headers by +// default (see the code within "unsupported/Eigen/CXX11/Tensor" that is guarded by EIGEN_USE_GPU) +// This will obsviously not work when trying to compile tensorflow on a system with no CUDA +// To work around this issue for HIP systems (and leave the default behaviour intact), the +// HIP tensorflow build defines EIGEN_USE_HIP when compiling all source files, and +// "unsupported/Eigen/CXX11/Tensor" has been updated to use HIP header when EIGEN_USE_HIP is +// defined. In continuation of that requirement, the guard here needs to be EIGEN_USE_HIP as well + +#if defined(EIGEN_USE_HIP) + +#define gpuStream_t hipStream_t +#define gpuDeviceProp_t hipDeviceProp_t +#define gpuError_t hipError_t +#define gpuSuccess hipSuccess +#define gpuErrorNotReady hipErrorNotReady +#define gpuGetDeviceCount hipGetDeviceCount +#define gpuGetErrorString hipGetErrorString +#define gpuGetDeviceProperties hipGetDeviceProperties +#define gpuStreamDefault hipStreamDefault +#define gpuGetDevice hipGetDevice +#define gpuSetDevice hipSetDevice +#define gpuMalloc hipMalloc +#define gpuFree hipFree +#define gpuMemsetAsync hipMemsetAsync +#define gpuMemcpyAsync hipMemcpyAsync +#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost +#define gpuMemcpyHostToDevice hipMemcpyHostToDevice +#define gpuStreamQuery hipStreamQuery +#define gpuSharedMemConfig hipSharedMemConfig +#define gpuDeviceSetSharedMemConfig hipDeviceSetSharedMemConfig +#define gpuStreamSynchronize hipStreamSynchronize +#define gpuDeviceSynchronize hipDeviceSynchronize +#define gpuMemcpy hipMemcpy + +#else + +#define gpuStream_t cudaStream_t +#define gpuDeviceProp_t cudaDeviceProp +#define gpuError_t cudaError_t +#define gpuSuccess cudaSuccess +#define gpuErrorNotReady cudaErrorNotReady +#define gpuGetDeviceCount cudaGetDeviceCount +#define gpuGetErrorString cudaGetErrorString +#define gpuGetDeviceProperties cudaGetDeviceProperties +#define gpuStreamDefault cudaStreamDefault +#define gpuGetDevice cudaGetDevice +#define gpuSetDevice cudaSetDevice +#define gpuMalloc cudaMalloc +#define gpuFree cudaFree +#define gpuMemsetAsync cudaMemsetAsync +#define gpuMemcpyAsync cudaMemcpyAsync +#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice +#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost +#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice +#define gpuStreamQuery cudaStreamQuery +#define gpuSharedMemConfig cudaSharedMemConfig +#define gpuDeviceSetSharedMemConfig cudaDeviceSetSharedMemConfig +#define gpuStreamSynchronize cudaStreamSynchronize +#define gpuDeviceSynchronize cudaDeviceSynchronize +#define gpuMemcpy cudaMemcpy + +#endif + +#if defined(EIGEN_HIP_DEVICE_COMPILE) || (defined(EIGEN_CUDACC) && (EIGEN_CUDACC_VER==0)) +// clang-cuda and HIPCC do not support the use of assert on the GPU side. +#define gpu_assert(COND) +#else +#define gpu_assert(COND) assert(COND) +#endif + +#endif // EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h b/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h new file mode 100644 index 0000000..db394bc --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h
@@ -0,0 +1,40 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// Copyright (C) 2018 Deven Desai <deven.desai.amd@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H) + +#undef gpuStream_t +#undef gpuDeviceProp_t +#undef gpuError_t +#undef gpuSuccess +#undef gpuErrorNotReady +#undef gpuGetDeviceCount +#undef gpuGetErrorString +#undef gpuGetDeviceProperties +#undef gpuStreamDefault +#undef gpuGetDevice +#undef gpuSetDevice +#undef gpuMalloc +#undef gpuFree +#undef gpuMemsetAsync +#undef gpuMemcpyAsync +#undef gpuMemcpyDeviceToDevice +#undef gpuMemcpyDeviceToHost +#undef gpuMemcpyHostToDevice +#undef gpuStreamQuery +#undef gpuSharedMemConfig +#undef gpuDeviceSetSharedMemConfig +#undef gpuStreamSynchronize +#undef gpuDeviceSynchronize +#undef gpuMemcpy + +#undef EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H + +#endif // EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorIO.h b/unsupported/Eigen/CXX11/src/Tensor/TensorIO.h index 38a833f..a901c5d 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorIO.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorIO.h
@@ -13,38 +13,61 @@ namespace Eigen { namespace internal { -template<> -struct significant_decimals_impl<std::string> - : significant_decimals_default_impl<std::string, true> -{}; -} +// Print the tensor as a 2d matrix +template <typename Tensor, int Rank> +struct TensorPrinter { + static void run (std::ostream& os, const Tensor& tensor) { + typedef typename internal::remove_const<typename Tensor::Scalar>::type Scalar; + typedef typename Tensor::Index Index; + const Index total_size = internal::array_prod(tensor.dimensions()); + if (total_size > 0) { + const Index first_dim = Eigen::internal::array_get<0>(tensor.dimensions()); + static const int layout = Tensor::Layout; + Map<const Array<Scalar, Dynamic, Dynamic, layout> > matrix(const_cast<Scalar*>(tensor.data()), first_dim, total_size/first_dim); + os << matrix; + } + } +}; + + +// Print the tensor as a vector +template <typename Tensor> +struct TensorPrinter<Tensor, 1> { + static void run (std::ostream& os, const Tensor& tensor) { + typedef typename internal::remove_const<typename Tensor::Scalar>::type Scalar; + typedef typename Tensor::Index Index; + const Index total_size = internal::array_prod(tensor.dimensions()); + if (total_size > 0) { + Map<const Array<Scalar, Dynamic, 1> > array(const_cast<Scalar*>(tensor.data()), total_size); + os << array; + } + } +}; + + +// Print the tensor as a scalar +template <typename Tensor> +struct TensorPrinter<Tensor, 0> { + static void run (std::ostream& os, const Tensor& tensor) { + os << tensor.coeff(0); + } +}; +} template <typename T> std::ostream& operator << (std::ostream& os, const TensorBase<T, ReadOnlyAccessors>& expr) { + typedef TensorEvaluator<const TensorForcedEvalOp<const T>, DefaultDevice> Evaluator; + typedef typename Evaluator::Dimensions Dimensions; + // Evaluate the expression if needed TensorForcedEvalOp<const T> eval = expr.eval(); - TensorEvaluator<const TensorForcedEvalOp<const T>, DefaultDevice> tensor(eval, DefaultDevice()); + Evaluator tensor(eval, DefaultDevice()); tensor.evalSubExprsIfNeeded(NULL); - typedef typename internal::remove_const<typename T::Scalar>::type Scalar; - typedef typename T::Index Index; - typedef typename TensorEvaluator<const TensorForcedEvalOp<const T>, DefaultDevice>::Dimensions Dimensions; - const Index total_size = internal::array_prod(tensor.dimensions()); - - // Print the tensor as a 1d vector or a 2d matrix. + // Print the result static const int rank = internal::array_size<Dimensions>::value; - if (rank == 0) { - os << tensor.coeff(0); - } else if (rank == 1) { - Map<const Array<Scalar, Dynamic, 1> > array(const_cast<Scalar*>(tensor.data()), total_size); - os << array; - } else { - const Index first_dim = Eigen::internal::array_get<0>(tensor.dimensions()); - static const int layout = TensorEvaluator<const TensorForcedEvalOp<const T>, DefaultDevice>::Layout; - Map<const Array<Scalar, Dynamic, Dynamic, layout> > matrix(const_cast<Scalar*>(tensor.data()), first_dim, total_size/first_dim); - os << matrix; - } + internal::TensorPrinter<Evaluator, rank>::run(os, tensor); // Cleanup. tensor.cleanup();
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h index bafcc67..965bd8f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h
@@ -27,6 +27,7 @@ * patch_cols, and 1 for all the additional dimensions. */ namespace internal { + template<DenseIndex Rows, DenseIndex Cols, typename XprType> struct traits<TensorImagePatchOp<Rows, Cols, XprType> > : public traits<XprType> { @@ -38,6 +39,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions + 1; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<DenseIndex Rows, DenseIndex Cols, typename XprType> @@ -52,6 +54,66 @@ typedef TensorImagePatchOp<Rows, Cols, XprType> type; }; +template <typename Self, bool Vectorizable> +struct ImagePatchCopyOp { + typedef typename Self::Index Index; + typedef typename Self::Scalar Scalar; + typedef typename Self::Impl Impl; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const Self& self, const Index num_coeff_to_copy, const Index dst_index, + Scalar* dst_data, const Index src_index) { + const Impl& impl = self.impl(); + for (Index i = 0; i < num_coeff_to_copy; ++i) { + dst_data[dst_index + i] = impl.coeff(src_index + i); + } + } +}; + +template <typename Self> +struct ImagePatchCopyOp<Self, true> { + typedef typename Self::Index Index; + typedef typename Self::Scalar Scalar; + typedef typename Self::Impl Impl; + typedef typename packet_traits<Scalar>::type Packet; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const Self& self, const Index num_coeff_to_copy, const Index dst_index, + Scalar* dst_data, const Index src_index) { + const Impl& impl = self.impl(); + const Index packet_size = internal::unpacket_traits<Packet>::size; + const Index vectorized_size = + (num_coeff_to_copy / packet_size) * packet_size; + for (Index i = 0; i < vectorized_size; i += packet_size) { + Packet p = impl.template packet<Unaligned>(src_index + i); + internal::pstoret<Scalar, Packet, Unaligned>(dst_data + dst_index + i, p); + } + for (Index i = vectorized_size; i < num_coeff_to_copy; ++i) { + dst_data[dst_index + i] = impl.coeff(src_index + i); + } + } +}; + +template <typename Self> +struct ImagePatchPaddingOp { + typedef typename Self::Index Index; + typedef typename Self::Scalar Scalar; + typedef typename packet_traits<Scalar>::type Packet; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run( + const Index num_coeff_to_pad, const Scalar padding_value, + const Index dst_index, Scalar* dst_data) { + const Index packet_size = internal::unpacket_traits<Packet>::size; + const Packet padded_packet = internal::pset1<Packet>(padding_value); + const Index vectorized_size = + (num_coeff_to_pad / packet_size) * packet_size; + for (Index i = 0; i < vectorized_size; i += packet_size) { + internal::pstoret<Scalar, Packet, Unaligned>(dst_data + dst_index + i, + padded_packet); + } + for (Index i = vectorized_size; i < num_coeff_to_pad; ++i) { + dst_data[dst_index + i] = padding_value; + } + } +}; + } // end namespace internal template<DenseIndex Rows, DenseIndex Cols, typename XprType> @@ -70,12 +132,12 @@ DenseIndex in_row_strides, DenseIndex in_col_strides, DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, PaddingType padding_type, Scalar padding_value) - : m_xpr(expr), m_patch_rows(patch_rows), m_patch_cols(patch_cols), - m_row_strides(row_strides), m_col_strides(col_strides), - m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), - m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), - m_padding_explicit(false), m_padding_top(0), m_padding_bottom(0), m_padding_left(0), m_padding_right(0), - m_padding_type(padding_type), m_padding_value(padding_value) {} + : m_xpr(expr), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_row_strides(row_strides), m_col_strides(col_strides), + m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), + m_padding_explicit(false), m_padding_top(0), m_padding_bottom(0), m_padding_left(0), m_padding_right(0), + m_padding_type(padding_type), m_padding_value(padding_value) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorImagePatchOp(const XprType& expr, DenseIndex patch_rows, DenseIndex patch_cols, DenseIndex row_strides, DenseIndex col_strides, @@ -84,13 +146,31 @@ DenseIndex padding_top, DenseIndex padding_bottom, DenseIndex padding_left, DenseIndex padding_right, Scalar padding_value) - : m_xpr(expr), m_patch_rows(patch_rows), m_patch_cols(patch_cols), - m_row_strides(row_strides), m_col_strides(col_strides), - m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), - m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), - m_padding_explicit(true), m_padding_top(padding_top), m_padding_bottom(padding_bottom), - m_padding_left(padding_left), m_padding_right(padding_right), - m_padding_type(PADDING_VALID), m_padding_value(padding_value) {} + : m_xpr(expr), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_row_strides(row_strides), m_col_strides(col_strides), + m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), + m_padding_explicit(true), m_padding_top(padding_top), m_padding_bottom(padding_bottom), + m_padding_left(padding_left), m_padding_right(padding_right), + m_padding_type(PADDING_VALID), m_padding_value(padding_value) {} + +#ifdef EIGEN_USE_SYCL // this is work around for sycl as Eigen could not use c++11 deligate constructor feature +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorImagePatchOp(const XprType& expr, DenseIndex patch_rows, DenseIndex patch_cols, + DenseIndex row_strides, DenseIndex col_strides, + DenseIndex in_row_strides, DenseIndex in_col_strides, + DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, + bool padding_explicit, DenseIndex padding_top, DenseIndex padding_bottom, + DenseIndex padding_left, DenseIndex padding_right, PaddingType padding_type, + Scalar padding_value) + : m_xpr(expr), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_row_strides(row_strides), m_col_strides(col_strides), + m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), + m_padding_explicit(padding_explicit), m_padding_top(padding_top), m_padding_bottom(padding_bottom), + m_padding_left(padding_left), m_padding_right(padding_right), + m_padding_type(padding_type), m_padding_value(padding_value) {} + +#endif EIGEN_DEVICE_FUNC DenseIndex patch_rows() const { return m_patch_rows; } @@ -161,20 +241,32 @@ typedef TensorEvaluator<ArgType, Device> Impl; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, - RawAccess = false + IsAligned = false, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = true, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, + RawAccess = false }; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device) + typedef internal::TensorBlock<Scalar, Index, NumDims, Layout> + OutputTensorBlock; + +#ifdef __SYCL_DEVICE_ONLY__ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( const XprType op, const Device& device) + #else + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( const XprType& op, const Device& device) + #endif + : m_device(device), m_impl(op.expression(), device) +#ifdef EIGEN_USE_SYCL + , m_op(op) +#endif { - EIGEN_STATIC_ASSERT(NumDims >= 4, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims >= 4), YOU_MADE_A_PROGRAMMING_MISTAKE); m_paddingValue = op.padding_value(); @@ -238,9 +330,15 @@ // Calculate the padding m_rowPaddingTop = ((m_outputRows - 1) * m_row_strides + m_patch_rows_eff - m_input_rows_eff) / 2; m_colPaddingLeft = ((m_outputCols - 1) * m_col_strides + m_patch_cols_eff - m_input_cols_eff) / 2; + // The padding size calculation for PADDING_SAME has been updated to + // be consistent with how TensorFlow extracts its paddings. + m_rowPaddingTop = numext::maxi<Index>(0, m_rowPaddingTop); + m_colPaddingLeft = numext::maxi<Index>(0, m_colPaddingLeft); break; default: eigen_assert(false && "unexpected padding"); + m_outputCols=0; // silence the uninitialised warning; + m_outputRows=0; //// silence the uninitialised warning; } } eigen_assert(m_outputRows > 0); @@ -362,7 +460,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); if (m_in_row_strides != 1 || m_in_col_strides != 1 || m_row_inflate_strides != 1 || m_col_inflate_strides != 1) { @@ -418,9 +516,14 @@ return packetWithPossibleZero(index); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } - const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + +#ifdef EIGEN_USE_SYCL + // Required by SYCL in order to construct the expression tree on the device + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const XprType& xpr() const { return m_op; } +#endif Index rowPaddingTop() const { return m_rowPaddingTop; } Index colPaddingLeft() const { return m_colPaddingLeft; } @@ -445,6 +548,147 @@ TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.lastLevelCacheSize() / sizeof(Scalar)); + resources->push_back(internal::TensorOpResourceRequirements( + internal::kSkewedInnerDims, block_total_size_max)); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + OutputTensorBlock* output_block) const { + typedef internal::ImagePatchCopyOp<Self, PacketAccess> ImagePatchCopyOp; + typedef internal::ImagePatchPaddingOp<Self> ImagePatchPaddingOp; + + // Calculate loop limits and various input/output dim sizes. + const DSizes<Index, NumDims>& block_sizes = output_block->block_sizes(); + const bool col_major = + static_cast<int>(Layout) == static_cast<int>(ColMajor); + const Index depth_dim_size = block_sizes[col_major ? 0 : NumDims - 1]; + const Index output_depth_dim_size = + m_dimensions[col_major ? 0 : NumDims - 1]; + const Index row_dim_size = block_sizes[col_major ? 1 : NumDims - 2]; + const Index output_row_dim_size = m_dimensions[col_major ? 1 : NumDims - 2]; + const Index col_dim_size = block_sizes[col_major ? 2 : NumDims - 3]; + const Index block_col_stride = row_dim_size * depth_dim_size; + const Index patch_index_dim_size = block_sizes[col_major ? 3 : NumDims - 4]; + const Index outer_dim_size = + block_sizes.TotalSize() / + (depth_dim_size * row_dim_size * col_dim_size * patch_index_dim_size); + + const Index patch_size = row_dim_size * col_dim_size * depth_dim_size; + const Index batch_size = patch_size * patch_index_dim_size; + + Index output_index = output_block->first_coeff_index(); + + // Loop through outer dimensions. + for (Index outer_dim_index = 0; outer_dim_index < outer_dim_size; + ++outer_dim_index) { + const Index outer_output_base_index = outer_dim_index * batch_size; + // Find the offset of the element wrt the location of the first element. + const Index patchIndexStart = output_index / m_fastPatchStride; + const Index patchOffset = + (output_index - patchIndexStart * m_patchStride) / m_fastOutputDepth; + const Index colOffsetStart = patchOffset / m_fastColStride; + // Other ways to index this element. + const Index otherIndex = + (NumDims == 4) ? 0 : output_index / m_fastOtherStride; + const Index patch2DIndexStart = + (NumDims == 4) + ? 0 + : (output_index - otherIndex * m_otherStride) / m_fastPatchStride; + // Calculate starting depth index. + const Index depth = output_index - (output_index / m_fastOutputDepth) * + output_depth_dim_size; + const Index patch_input_base_index = + depth + otherIndex * m_patchInputStride; + + // Loop through patches. + for (Index patch_index_dim_index = 0; + patch_index_dim_index < patch_index_dim_size; + ++patch_index_dim_index) { + const Index patch_output_base_index = + outer_output_base_index + patch_index_dim_index * patch_size; + // Patch index corresponding to the passed in index. + const Index patchIndex = patchIndexStart + patch_index_dim_index; + const Index patch2DIndex = + (NumDims == 4) ? patchIndex + : patch2DIndexStart + patch_index_dim_index; + const Index colIndex = patch2DIndex / m_fastOutputRows; + const Index input_col_base = colIndex * m_col_strides; + const Index row_offset_base = + (patch2DIndex - colIndex * m_outputRows) * m_row_strides - + m_rowPaddingTop; + + // Loop through columns. + for (Index col_dim_index = 0; col_dim_index < col_dim_size; + ++col_dim_index) { + const Index col_output_base_index = + patch_output_base_index + col_dim_index * block_col_stride; + + // Calculate col index in the input original tensor. + Index colOffset = colOffsetStart + col_dim_index; + Index inputCol = + input_col_base + colOffset * m_in_col_strides - m_colPaddingLeft; + Index origInputCol = + (m_col_inflate_strides == 1) + ? inputCol + : ((inputCol >= 0) ? (inputCol / m_fastInflateColStride) : 0); + + bool pad_column = false; + if (inputCol < 0 || inputCol >= m_input_cols_eff || + ((m_col_inflate_strides != 1) && + (inputCol != origInputCol * m_col_inflate_strides))) { + pad_column = true; + } + + const Index col_input_base_index = + patch_input_base_index + origInputCol * m_colInputStride; + const Index input_row_base = + row_offset_base + + ((patchOffset + col_dim_index * output_row_dim_size) - + colOffset * m_colStride) * + m_in_row_strides; + // Loop through rows. + for (Index row_dim_index = 0; row_dim_index < row_dim_size; + ++row_dim_index) { + const Index output_base_index = + col_output_base_index + row_dim_index * depth_dim_size; + bool pad_row = false; + Index inputIndex; + if (!pad_column) { + Index inputRow = + input_row_base + row_dim_index * m_in_row_strides; + Index origInputRow = + (m_row_inflate_strides == 1) + ? inputRow + : ((inputRow >= 0) ? (inputRow / m_fastInflateRowStride) + : 0); + if (inputRow < 0 || inputRow >= m_input_rows_eff || + ((m_row_inflate_strides != 1) && + (inputRow != origInputRow * m_row_inflate_strides))) { + pad_row = true; + } else { + inputIndex = + col_input_base_index + origInputRow * m_rowInputStride; + } + } + // Copy (or pad) along depth dimension. + if (pad_column || pad_row) { + ImagePatchPaddingOp::Run(depth_dim_size, Scalar(m_paddingValue), + output_base_index, output_block->data()); + } else { + ImagePatchCopyOp::Run(*this, depth_dim_size, output_base_index, + output_block->data(), inputIndex); + } + } + } + } + output_index += m_otherStride; + } + } + protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { @@ -500,7 +744,12 @@ Scalar m_paddingValue; + const Device& m_device; TensorEvaluator<ArgType, Device> m_impl; + #ifdef EIGEN_USE_SYCL + // Required for SYCL in order to construct the expression tree on the device + XprType m_op; + #endif };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h b/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h index 985594b..32caccf 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h
@@ -10,7 +10,8 @@ #ifndef EIGEN_CXX11_TENSOR_TENSOR_INDEX_LIST_H #define EIGEN_CXX11_TENSOR_TENSOR_INDEX_LIST_H -#if defined(EIGEN_HAS_CONSTEXPR) && defined(EIGEN_HAS_VARIADIC_TEMPLATES) + +#if EIGEN_HAS_CONSTEXPR && EIGEN_HAS_VARIADIC_TEMPLATES #define EIGEN_HAS_INDEX_LIST @@ -36,18 +37,36 @@ * \sa Tensor */ -template <DenseIndex n> +template <Index n> struct type2index { - static const DenseIndex value = n; - EIGEN_DEVICE_FUNC constexpr operator DenseIndex() const { return n; } - EIGEN_DEVICE_FUNC void set(DenseIndex val) { + static const Index value = n; + EIGEN_DEVICE_FUNC constexpr operator Index() const { return n; } + EIGEN_DEVICE_FUNC void set(Index val) { eigen_assert(val == n); } }; -template<DenseIndex n> struct NumTraits<type2index<n> > +// This can be used with IndexPairList to get compile-time constant pairs, +// such as IndexPairList<type2indexpair<1,2>, type2indexpair<3,4>>(). +template <Index f, Index s> +struct type2indexpair { + static const Index first = f; + static const Index second = s; + + constexpr EIGEN_DEVICE_FUNC operator IndexPair<Index>() const { + return IndexPair<Index>(f, s); + } + + EIGEN_DEVICE_FUNC void set(const IndexPair<Index>& val) { + eigen_assert(val.first == f); + eigen_assert(val.second == s); + } +}; + + +template<Index n> struct NumTraits<type2index<n> > { - typedef DenseIndex Real; + typedef Index Real; enum { IsComplex = 0, RequireInitialization = false, @@ -56,45 +75,70 @@ MulCost = 1 }; - EIGEN_DEVICE_FUNC static inline Real epsilon() { return 0; } - EIGEN_DEVICE_FUNC static inline Real dummy_precision() { return 0; } - EIGEN_DEVICE_FUNC static inline Real highest() { return n; } - EIGEN_DEVICE_FUNC static inline Real lowest() { return n; } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Real epsilon() { return 0; } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Real dummy_precision() { return 0; } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Real highest() { return n; } + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Real lowest() { return n; } }; namespace internal { template <typename T> -EIGEN_DEVICE_FUNC void update_value(T& val, DenseIndex new_val) { - val = new_val; +EIGEN_DEVICE_FUNC void update_value(T& val, Index new_val) { + val = internal::convert_index<T>(new_val); } -template <DenseIndex n> -EIGEN_DEVICE_FUNC void update_value(type2index<n>& val, DenseIndex new_val) { +template <Index n> +EIGEN_DEVICE_FUNC void update_value(type2index<n>& val, Index new_val) { val.set(new_val); } template <typename T> +EIGEN_DEVICE_FUNC void update_value(T& val, IndexPair<Index> new_val) { + val = new_val; +} +template <Index f, Index s> +EIGEN_DEVICE_FUNC void update_value(type2indexpair<f, s>& val, IndexPair<Index> new_val) { + val.set(new_val); +} + + +template <typename T> struct is_compile_time_constant { static constexpr bool value = false; }; -template <DenseIndex idx> +template <Index idx> struct is_compile_time_constant<type2index<idx> > { static constexpr bool value = true; }; -template <DenseIndex idx> +template <Index idx> struct is_compile_time_constant<const type2index<idx> > { static constexpr bool value = true; }; -template <DenseIndex idx> +template <Index idx> struct is_compile_time_constant<type2index<idx>& > { static constexpr bool value = true; }; -template <DenseIndex idx> +template <Index idx> struct is_compile_time_constant<const type2index<idx>& > { static constexpr bool value = true; }; - +template <Index f, Index s> +struct is_compile_time_constant<type2indexpair<f, s> > { + static constexpr bool value = true; +}; +template <Index f, Index s> +struct is_compile_time_constant<const type2indexpair<f, s> > { + static constexpr bool value = true; +}; +template <Index f, Index s> +struct is_compile_time_constant<type2indexpair<f, s>& > { + static constexpr bool value = true; +}; +template <Index f, Index s> +struct is_compile_time_constant<const type2indexpair<f, s>& > { + static constexpr bool value = true; +}; template<typename... T> @@ -184,31 +228,32 @@ -template <DenseIndex Idx> +template <Index Idx, typename ValueT> struct tuple_coeff { template <typename... T> - EIGEN_DEVICE_FUNC static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { - return array_get<Idx>(t) * (i == Idx) + tuple_coeff<Idx-1>::get(i, t) * (i != Idx); + EIGEN_DEVICE_FUNC static constexpr ValueT get(const Index i, const IndexTuple<T...>& t) { + // return array_get<Idx>(t) * (i == Idx) + tuple_coeff<Idx-1>::get(i, t) * (i != Idx); + return (i == Idx ? array_get<Idx>(t) : tuple_coeff<Idx-1, ValueT>::get(i, t)); } template <typename... T> - EIGEN_DEVICE_FUNC static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { + EIGEN_DEVICE_FUNC static void set(const Index i, IndexTuple<T...>& t, const ValueT& value) { if (i == Idx) { update_value(array_get<Idx>(t), value); } else { - tuple_coeff<Idx-1>::set(i, t, value); + tuple_coeff<Idx-1, ValueT>::set(i, t, value); } } template <typename... T> - EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const Index i, const IndexTuple<T...>& t) { return ((i == Idx) & is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value) || - tuple_coeff<Idx-1>::value_known_statically(i, t); + tuple_coeff<Idx-1, ValueT>::value_known_statically(i, t); } template <typename... T> EIGEN_DEVICE_FUNC static constexpr bool values_up_to_known_statically(const IndexTuple<T...>& t) { return is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && - tuple_coeff<Idx-1>::values_up_to_known_statically(t); + tuple_coeff<Idx-1, ValueT>::values_up_to_known_statically(t); } template <typename... T> @@ -216,24 +261,24 @@ return is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && array_get<Idx>(t) > array_get<Idx-1>(t) && - tuple_coeff<Idx-1>::values_up_to_statically_known_to_increase(t); + tuple_coeff<Idx-1, ValueT>::values_up_to_statically_known_to_increase(t); } }; -template <> -struct tuple_coeff<0> { +template <typename ValueT> +struct tuple_coeff<0, ValueT> { template <typename... T> - EIGEN_DEVICE_FUNC static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr ValueT get(const Index /*i*/, const IndexTuple<T...>& t) { // eigen_assert (i == 0); // gcc fails to compile assertions in constexpr - return array_get<0>(t) * (i == 0); + return array_get<0>(t)/* * (i == 0)*/; } template <typename... T> - EIGEN_DEVICE_FUNC static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { + EIGEN_DEVICE_FUNC static void set(const Index i, IndexTuple<T...>& t, const ValueT value) { eigen_assert (i == 0); update_value(array_get<0>(t), value); } template <typename... T> - EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>&) { + EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const Index i, const IndexTuple<T...>&) { return is_compile_time_constant<typename IndexTupleExtractor<0, T...>::ValType>::value & (i == 0); } @@ -253,29 +298,29 @@ template<typename FirstType, typename... OtherTypes> struct IndexList : internal::IndexTuple<FirstType, OtherTypes...> { - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr DenseIndex operator[] (const DenseIndex i) const { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::get(i, *this); + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr Index operator[] (const Index i) const { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::get(i, *this); } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr DenseIndex get(const DenseIndex i) const { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::get(i, *this); + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr Index get(const Index i) const { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::get(i, *this); } - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC void set(const DenseIndex i, const DenseIndex value) { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::set(i, *this, value); + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC void set(const Index i, const Index value) { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::set(i, *this, value); } EIGEN_DEVICE_FUNC constexpr IndexList(const internal::IndexTuple<FirstType, OtherTypes...>& other) : internal::IndexTuple<FirstType, OtherTypes...>(other) { } EIGEN_DEVICE_FUNC constexpr IndexList(FirstType& first, OtherTypes... other) : internal::IndexTuple<FirstType, OtherTypes...>(first, other...) { } EIGEN_DEVICE_FUNC constexpr IndexList() : internal::IndexTuple<FirstType, OtherTypes...>() { } - EIGEN_DEVICE_FUNC constexpr bool value_known_statically(const DenseIndex i) const { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::value_known_statically(i, *this); + EIGEN_DEVICE_FUNC constexpr bool value_known_statically(const Index i) const { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::value_known_statically(i, *this); } EIGEN_DEVICE_FUNC constexpr bool all_values_known_statically() const { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::values_up_to_known_statically(*this); + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::values_up_to_known_statically(*this); } EIGEN_DEVICE_FUNC constexpr bool values_statically_known_to_increase() const { - return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::values_up_to_statically_known_to_increase(*this); + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::values_up_to_statically_known_to_increase(*this); } }; @@ -286,11 +331,29 @@ } +template<typename FirstType, typename... OtherTypes> +struct IndexPairList : internal::IndexTuple<FirstType, OtherTypes...> { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr IndexPair<Index> operator[] (const Index i) const { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, IndexPair<Index>>::get(i, *this); + } + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC void set(const Index i, const IndexPair<Index> value) { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...>>::value-1, IndexPair<Index> >::set(i, *this, value); + } + + EIGEN_DEVICE_FUNC constexpr IndexPairList(const internal::IndexTuple<FirstType, OtherTypes...>& other) : internal::IndexTuple<FirstType, OtherTypes...>(other) { } + EIGEN_DEVICE_FUNC constexpr IndexPairList() : internal::IndexTuple<FirstType, OtherTypes...>() { } + + EIGEN_DEVICE_FUNC constexpr bool value_known_statically(const Index i) const { + return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1, Index>::value_known_statically(i, *this); + } +}; + namespace internal { -template<typename FirstType, typename... OtherTypes> size_t array_prod(const IndexList<FirstType, OtherTypes...>& sizes) { - size_t result = 1; - for (int i = 0; i < array_size<IndexList<FirstType, OtherTypes...> >::value; ++i) { +template<typename FirstType, typename... OtherTypes> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index array_prod(const IndexList<FirstType, OtherTypes...>& sizes) { + Index result = 1; + for (size_t i = 0; i < array_size<IndexList<FirstType, OtherTypes...> >::value; ++i) { result *= sizes[i]; } return result; @@ -303,30 +366,37 @@ static const size_t value = array_size<IndexTuple<FirstType, OtherTypes...> >::value; }; -template<DenseIndex N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr DenseIndex array_get(IndexList<FirstType, OtherTypes...>& a) { +template<typename FirstType, typename... OtherTypes> struct array_size<IndexPairList<FirstType, OtherTypes...> > { + static const size_t value = std::tuple_size<std::tuple<FirstType, OtherTypes...> >::value; +}; +template<typename FirstType, typename... OtherTypes> struct array_size<const IndexPairList<FirstType, OtherTypes...> > { + static const size_t value = std::tuple_size<std::tuple<FirstType, OtherTypes...> >::value; +}; + +template<Index N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr Index array_get(IndexList<FirstType, OtherTypes...>& a) { return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a); } -template<DenseIndex N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr DenseIndex array_get(const IndexList<FirstType, OtherTypes...>& a) { +template<Index N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr Index array_get(const IndexList<FirstType, OtherTypes...>& a) { return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a); } template <typename T> struct index_known_statically_impl { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_known_statically_impl<IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i); } }; template <typename FirstType, typename... OtherTypes> struct index_known_statically_impl<const IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i); } }; @@ -378,14 +448,14 @@ template <typename Tx> struct index_statically_eq_impl { - EIGEN_DEVICE_FUNC static constexpr bool run(DenseIndex, DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_statically_eq_impl<IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) == value); } @@ -393,7 +463,7 @@ template <typename FirstType, typename... OtherTypes> struct index_statically_eq_impl<const IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) == value); } @@ -402,14 +472,14 @@ template <typename T> struct index_statically_ne_impl { - EIGEN_DEVICE_FUNC static constexpr bool run(DenseIndex, DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_statically_ne_impl<IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) != value); } @@ -417,7 +487,7 @@ template <typename FirstType, typename... OtherTypes> struct index_statically_ne_impl<const IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) != value); } @@ -426,14 +496,14 @@ template <typename T> struct index_statically_gt_impl { - EIGEN_DEVICE_FUNC static constexpr bool run(DenseIndex, DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_statically_gt_impl<IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) > value); } @@ -441,7 +511,7 @@ template <typename FirstType, typename... OtherTypes> struct index_statically_gt_impl<const IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) > value); } @@ -451,14 +521,14 @@ template <typename T> struct index_statically_lt_impl { - EIGEN_DEVICE_FUNC static constexpr bool run(DenseIndex, DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_statically_lt_impl<IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) < value); } @@ -466,12 +536,63 @@ template <typename FirstType, typename... OtherTypes> struct index_statically_lt_impl<const IndexList<FirstType, OtherTypes...> > { - EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i, const DenseIndex value) { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i) & (IndexList<FirstType, OtherTypes...>().get(i) < value); } }; + + +template <typename Tx> +struct index_pair_first_statically_eq_impl { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { + return false; + } +}; + +template <typename FirstType, typename... OtherTypes> +struct index_pair_first_statically_eq_impl<IndexPairList<FirstType, OtherTypes...> > { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { + return IndexPairList<FirstType, OtherTypes...>().value_known_statically(i) & + (IndexPairList<FirstType, OtherTypes...>().operator[](i).first == value); + } +}; + +template <typename FirstType, typename... OtherTypes> +struct index_pair_first_statically_eq_impl<const IndexPairList<FirstType, OtherTypes...> > { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { + return IndexPairList<FirstType, OtherTypes...>().value_known_statically(i) & + (IndexPairList<FirstType, OtherTypes...>().operator[](i).first == value); + } +}; + + + +template <typename Tx> +struct index_pair_second_statically_eq_impl { + EIGEN_DEVICE_FUNC static constexpr bool run(Index, Index) { + return false; + } +}; + +template <typename FirstType, typename... OtherTypes> +struct index_pair_second_statically_eq_impl<IndexPairList<FirstType, OtherTypes...> > { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { + return IndexPairList<FirstType, OtherTypes...>().value_known_statically(i) & + (IndexPairList<FirstType, OtherTypes...>().operator[](i).second == value); + } +}; + +template <typename FirstType, typename... OtherTypes> +struct index_pair_second_statically_eq_impl<const IndexPairList<FirstType, OtherTypes...> > { + EIGEN_DEVICE_FUNC static constexpr bool run(const Index i, const Index value) { + return IndexPairList<FirstType, OtherTypes...>().value_known_statically(i) & + (IndexPairList<FirstType, OtherTypes...>().operator[](i).second == value); + } +}; + + } // end namespace internal } // end namespace Eigen @@ -482,53 +603,69 @@ template <typename T> struct index_known_statically_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(const DenseIndex) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(const Index) { return false; } }; template <typename T> struct all_indices_known_statically_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run() { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run() { return false; } }; template <typename T> struct indices_statically_known_to_increase_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run() { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run() { return false; } }; template <typename T> struct index_statically_eq_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(DenseIndex, DenseIndex) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { return false; } }; template <typename T> struct index_statically_ne_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(DenseIndex, DenseIndex) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { return false; } }; template <typename T> struct index_statically_gt_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(DenseIndex, DenseIndex) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { return false; } }; template <typename T> struct index_statically_lt_impl { - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(DenseIndex, DenseIndex) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { return false; } }; +template <typename Tx> +struct index_pair_first_statically_eq_impl { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { + return false; + } +}; + +template <typename Tx> +struct index_pair_second_statically_eq_impl { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(Index, Index) { + return false; + } +}; + + + } // end namespace internal } // end namespace Eigen @@ -538,7 +675,7 @@ namespace Eigen { namespace internal { template <typename T> -static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_known_statically(DenseIndex i) { +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_known_statically(Index i) { return index_known_statically_impl<T>::run(i); } @@ -553,25 +690,35 @@ } template <typename T> -static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_eq(DenseIndex i, DenseIndex value) { +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_eq(Index i, Index value) { return index_statically_eq_impl<T>::run(i, value); } template <typename T> -static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_ne(DenseIndex i, DenseIndex value) { +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_ne(Index i, Index value) { return index_statically_ne_impl<T>::run(i, value); } template <typename T> -static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_gt(DenseIndex i, DenseIndex value) { +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_gt(Index i, Index value) { return index_statically_gt_impl<T>::run(i, value); } template <typename T> -static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_lt(DenseIndex i, DenseIndex value) { +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_statically_lt(Index i, Index value) { return index_statically_lt_impl<T>::run(i, value); } +template <typename T> +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_pair_first_statically_eq(Index i, Index value) { + return index_pair_first_statically_eq_impl<T>::run(i, value); +} + +template <typename T> +static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool index_pair_second_statically_eq(Index i, Index value) { + return index_pair_second_statically_eq_impl<T>::run(i, value); +} + } // end namespace internal } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h b/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h index de2f67d..e285650 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename Strides, typename XprType> @@ -84,12 +85,13 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/ false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -189,7 +191,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; @@ -213,7 +215,12 @@ compute_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Strides& functor() const { return m_strides; } +#endif protected: Dimensions m_dimensions;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h b/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h index 2d22314..33edc49 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h
@@ -10,7 +10,7 @@ #ifndef EIGEN_CXX11_TENSOR_TENSOR_INITIALIZER_H #define EIGEN_CXX11_TENSOR_TENSOR_INITIALIZER_H -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES #include <initializer_list>
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h b/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h index 33c6c1b..b6d445c 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h
@@ -21,7 +21,7 @@ * \brief Fast integer division by a constant. * * See the paper from Granlund and Montgomery for explanation. - * (at http://dx.doi.org/10.1145/773473.178249) + * (at https://doi.org/10.1145/773473.178249) * * \sa Tensor */ @@ -29,25 +29,51 @@ namespace internal { namespace { + // Note: result is undefined if val == 0 template <typename T> - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int count_leading_zeros(const T val) + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + typename internal::enable_if<sizeof(T)==4,int>::type count_leading_zeros(const T val) { -#ifdef __CUDA_ARCH__ - return (sizeof(T) == 8) ? __clzll(val) : __clz(val); +#ifdef EIGEN_GPU_COMPILE_PHASE + return __clz(val); +#elif defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::clz(val); #elif EIGEN_COMP_MSVC - unsigned long index; - if (sizeof(T) == 8) { - _BitScanReverse64(&index, val); - } else { - _BitScanReverse(&index, val); - } - return (sizeof(T) == 8) ? 63 - index : 31 - index; + unsigned long index; + _BitScanReverse(&index, val); + return 31 - index; #else EIGEN_STATIC_ASSERT(sizeof(unsigned long long) == 8, YOU_MADE_A_PROGRAMMING_MISTAKE); - return (sizeof(T) == 8) ? - __builtin_clzll(static_cast<uint64_t>(val)) : - __builtin_clz(static_cast<uint32_t>(val)); + return __builtin_clz(static_cast<uint32_t>(val)); +#endif + } + + template <typename T> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE + typename internal::enable_if<sizeof(T)==8,int>::type count_leading_zeros(const T val) + { +#ifdef EIGEN_GPU_COMPILE_PHASE + return __clzll(val); +#elif defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::clz(val); +#elif EIGEN_COMP_MSVC && EIGEN_ARCH_x86_64 + unsigned long index; + _BitScanReverse64(&index, val); + return 63 - index; +#elif EIGEN_COMP_MSVC + // MSVC's _BitScanReverse64 is not available for 32bits builds. + unsigned int lo = (unsigned int)(val&0xffffffff); + unsigned int hi = (unsigned int)((val>>32)&0xffffffff); + int n; + if(hi==0) + n = 32 + count_leading_zeros<unsigned int>(lo); + else + n = count_leading_zeros<unsigned int>(hi); + return n; +#else + EIGEN_STATIC_ASSERT(sizeof(unsigned long long) == 8, YOU_MADE_A_PROGRAMMING_MISTAKE); + return __builtin_clzll(static_cast<uint64_t>(val)); #endif } @@ -64,8 +90,10 @@ template <typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint32_t muluh(const uint32_t a, const T b) { -#if defined(__CUDA_ARCH__) +#if defined(EIGEN_GPU_COMPILE_PHASE) return __umulhi(a, b); +#elif defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::mul_hi(a, static_cast<uint32_t>(b)); #else return (static_cast<uint64_t>(a) * b) >> 32; #endif @@ -73,8 +101,10 @@ template <typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint64_t muluh(const uint64_t a, const T b) { -#if defined(__CUDA_ARCH__) +#if defined(EIGEN_GPU_COMPILE_PHASE) return __umul64hi(a, b); +#elif defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::mul_hi(a, static_cast<uint64_t>(b)); #elif defined(__SIZEOF_INT128__) __uint128_t v = static_cast<__uint128_t>(a) * static_cast<__uint128_t>(b); return static_cast<uint64_t>(v >> 64); @@ -94,11 +124,13 @@ template <typename T> struct DividerHelper<64, T> { static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint64_t computeMultiplier(const int log_div, const T divider) { -#if defined(__SIZEOF_INT128__) && !defined(__CUDA_ARCH__) +#if defined(__SIZEOF_INT128__) && !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__) return static_cast<uint64_t>((static_cast<__uint128_t>(1) << (64+log_div)) / static_cast<__uint128_t>(divider) - (static_cast<__uint128_t>(1) << 64) + 1); #else const uint64_t shift = 1ULL << log_div; - TensorUInt128<uint64_t, uint64_t> result = (TensorUInt128<uint64_t, static_val<0> >(shift, 0) / TensorUInt128<static_val<0>, uint64_t>(divider) - TensorUInt128<static_val<1>, static_val<0> >(1, 0) + TensorUInt128<static_val<0>, static_val<1> >(1)); + TensorUInt128<uint64_t, uint64_t> result = TensorUInt128<uint64_t, static_val<0> >(shift, 0) / TensorUInt128<static_val<0>, uint64_t>(divider) + - TensorUInt128<static_val<1>, static_val<0> >(1, 0) + + TensorUInt128<static_val<0>, static_val<1> >(1); return static_cast<uint64_t>(result); #endif } @@ -135,7 +167,7 @@ shift2 = log_div > 1 ? log_div-1 : 0; } - // Must have 0 <= numerator. On platforms that dont support the __uint128_t + // Must have 0 <= numerator. On platforms that don't support the __uint128_t // type numerator should also be less than 2^32-1. EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T divide(const T numerator) const { eigen_assert(static_cast<typename UnsignedTraits<T>::type>(numerator) < NumTraits<UnsignedType>::highest()/2); @@ -171,8 +203,10 @@ } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int divide(const int32_t n) const { -#ifdef __CUDA_ARCH__ +#ifdef EIGEN_GPU_COMPILE_PHASE return (__umulhi(magic, n) >> shift); +#elif defined(__SYCL_DEVICE_ONLY__) + return (cl::sycl::mul_hi(static_cast<uint64_t>(magic), static_cast<uint64_t>(n)) >> shift); #else uint64_t v = static_cast<uint64_t>(magic) * static_cast<uint64_t>(n); return (static_cast<uint32_t>(v >> 32) >> shift);
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h b/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h index 63a8476..998757d 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h
@@ -46,6 +46,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = traits<XprType>::NumDimensions; static const int Layout = (traits<XprType>::Layout == ColMajor) ? RowMajor : ColMajor; + typedef typename XprTraits::PointerType PointerType; }; template<typename XprType> @@ -118,6 +119,8 @@ enum { IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = (static_cast<int>(TensorEvaluator<ArgType, Device>::Layout) == static_cast<int>(ColMajor)) ? RowMajor : ColMajor, CoordAccess = false, // to be implemented RawAccess = TensorEvaluator<ArgType, Device>::RawAccess @@ -159,7 +162,7 @@ return m_impl.costPerCoeff(vectorized); } - EIGEN_DEVICE_FUNC Scalar* data() const { return m_impl.data(); } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return m_impl.data(); } const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } @@ -180,8 +183,10 @@ enum { IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = (static_cast<int>(TensorEvaluator<ArgType, Device>::Layout) == static_cast<int>(ColMajor)) ? RowMajor : ColMajor, - CoordAccess = false, // to be implemented + CoordAccess = false // to be implemented }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMacros.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMacros.h index 8ed71f8..c6ca396 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMacros.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMacros.h
@@ -27,8 +27,8 @@ */ // SFINAE requires variadic templates -#ifndef __CUDACC__ -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if !defined(EIGEN_GPUCC) +#if EIGEN_HAS_VARIADIC_TEMPLATES // SFINAE doesn't work for gcc <= 4.7 #ifdef EIGEN_COMP_GNUC #if EIGEN_GNUC_AT_LEAST(4,8) @@ -44,11 +44,19 @@ typename internal::enable_if< ( __condition__ ) , int >::type = 0 -#if defined(EIGEN_HAS_CONSTEXPR) +#if EIGEN_HAS_CONSTEXPR #define EIGEN_CONSTEXPR constexpr #else #define EIGEN_CONSTEXPR #endif +#if EIGEN_OS_WIN || EIGEN_OS_WIN64 +#define EIGEN_SLEEP(n) Sleep(n) +#elif EIGEN_OS_GNULINUX +#define EIGEN_SLEEP(n) usleep(n * 1000); +#else +#define EIGEN_SLEEP(n) sleep(std::max<unsigned>(1, n/1000)) +#endif + #endif
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h index 9ebd917..28f6290 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h
@@ -12,17 +12,24 @@ namespace Eigen { +// FIXME use proper doxygen documentation (e.g. \tparam MakePointer_) + /** \class TensorMap * \ingroup CXX11_Tensor_Module * * \brief A tensor expression mapping an existing array of data. * */ - -template<typename PlainObjectType, int Options_> class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_> > +/// `template <class> class MakePointer_` is added to convert the host pointer to the device pointer. +/// It is added due to the fact that for our device compiler `T*` is not allowed. +/// If we wanted to use the same Evaluator functions we have to convert that type to our pointer `T`. +/// This is done through our `MakePointer_` class. By default the Type in the `MakePointer_<T>` is `T*` . +/// Therefore, by adding the default value, we managed to convert the type and it does not break any +/// existing code as its default value is `T*`. +template<typename PlainObjectType, int Options_, template <class> class MakePointer_> class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePointer_> > { public: - typedef TensorMap<PlainObjectType, Options_> Self; + typedef TensorMap<PlainObjectType, Options_, MakePointer_> Self; typedef typename PlainObjectType::Base Base; typedef typename Eigen::internal::nested<Self>::type Nested; typedef typename internal::traits<PlainObjectType>::StorageKind StorageKind; @@ -36,7 +43,7 @@ Scalar *, const Scalar *>::type PointerType;*/ - typedef Scalar* PointerType; + typedef typename MakePointer_<Scalar>::Type PointerType; typedef PointerType PointerArgType; static const int Options = Options_; @@ -57,7 +64,7 @@ EIGEN_STATIC_ASSERT((0 == NumIndices || NumIndices == Dynamic), YOU_MADE_A_PROGRAMMING_MISTAKE) } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMap(PointerArgType dataPtr, Index firstDimension, IndexTypes... otherDimensions) : m_data(dataPtr), m_dimensions(firstDimension, otherDimensions...) { // The number of dimensions used to construct a tensor must be equal to the rank of the tensor. @@ -109,9 +116,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_dimensions.TotalSize(); } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar* data() { return m_data; } + EIGEN_STRONG_INLINE PointerType data() { return m_data; } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar* data() const { return m_data; } + EIGEN_STRONG_INLINE const PointerType data() const { return m_data; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(const array<Index, NumIndices>& indices) const @@ -140,11 +147,12 @@ return m_data[index]; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) const { EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...)); if (PlainObjectType::Options&RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; @@ -227,11 +235,12 @@ return m_data[index]; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) { static_assert(sizeof...(otherIndices) + 2 == NumIndices || NumIndices == Dynamic, "Number of indices used to access a tensor coefficient must be equal to the rank of the tensor."); + eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...)); const std::size_t NumDims = sizeof...(otherIndices) + 2; if (PlainObjectType::Options&RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}}); @@ -307,7 +316,7 @@ } private: - Scalar* m_data; + typename MakePointer_<Scalar>::Type m_data; Dimensions m_dimensions; };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h index cd04716..87be090 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h
@@ -47,25 +47,64 @@ // Default packet types template <typename Scalar, typename Device> -struct PacketType { +struct PacketType : internal::packet_traits<Scalar> { typedef typename internal::packet_traits<Scalar>::type type; - enum { size = internal::unpacket_traits<type>::size }; }; // For CUDA packet types when using a GpuDevice -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) +#if defined(EIGEN_USE_GPU) && defined(EIGEN_HAS_GPU_FP16) template <> -struct PacketType<float, GpuDevice> { - typedef float4 type; - static const int size = 4; -}; -template <> -struct PacketType<double, GpuDevice> { - typedef double2 type; +struct PacketType<half, GpuDevice> { + typedef half2 type; static const int size = 2; + enum { + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasNegate = 1, + HasAbs = 1, + HasArg = 0, + HasAbs2 = 0, + HasMin = 1, + HasMax = 1, + HasConj = 0, + HasSetLinear = 0, + HasBlend = 0, + + HasDiv = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasExp = 1, + HasExpm1 = 0, + HasLog = 1, + HasLog1p = 0, + HasLog10 = 0, + HasPow = 1, + }; }; #endif +#if defined(EIGEN_USE_SYCL) +template <typename T> + struct PacketType<T, SyclDevice> { + typedef T type; + static const int size = 1; + enum { + HasAdd = 0, + HasSub = 0, + HasMul = 0, + HasNegate = 0, + HasAbs = 0, + HasArg = 0, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasConj = 0, + HasSetLinear = 0, + HasBlend = 0 + }; +}; +#endif // Tuple mimics std::pair but works on e.g. nvcc. @@ -85,7 +124,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tuple& operator= (const Tuple& rhs) { + #ifndef __SYCL_DEVICE_ONLY__ if (&rhs == this) return *this; + #endif first = rhs.first; second = rhs.second; return *this; @@ -112,16 +153,30 @@ } +// Can't use std::pairs on cuda devices +template <typename Idx> struct IndexPair { + EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE IndexPair() : first(0), second(0) {} + EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE IndexPair(Idx f, Idx s) : first(f), second(s) {} + + EIGEN_DEVICE_FUNC void set(IndexPair<Idx> val) { + first = val.first; + second = val.second; + } + + Idx first; + Idx second; +}; + #ifdef EIGEN_HAS_SFINAE namespace internal { - template<typename IndexType, Index... Is> + template<typename IndexType, typename Index, Index... Is> EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, sizeof...(Is)> customIndices2Array(IndexType& idx, numeric_list<Index, Is...>) { return { idx[Is]... }; } - template<typename IndexType> + template<typename IndexType, typename Index> EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, 0> customIndices2Array(IndexType&, numeric_list<Index>) { return array<Index, 0>();
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h index bfa65a6..69a5990 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = array_size<NewDimensions>::value; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename NewDimensions, typename XprType> @@ -101,26 +102,69 @@ typedef TensorReshapingOp<NewDimensions, ArgType> XprType; typedef NewDimensions Dimensions; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + + static const int NumOutputDims = internal::array_size<Dimensions>::value; + static const int NumInputDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value; + enum { - IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, // to be implemented - RawAccess = TensorEvaluator<ArgType, Device>::RawAccess + IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + // TODO(andydavis, wuke) Enable BlockAccess for the general case when the + // performance issue with block-based reshape is resolved. + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess && + TensorEvaluator<ArgType, Device>::RawAccess && + NumInputDims > 0 && NumOutputDims > 0, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = TensorEvaluator<ArgType, Device>::RawAccess }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumInputDims, Layout> + InputTensorBlock; + typedef internal::TensorBlock<ScalarNoConst, Index, NumOutputDims, Layout> + OutputTensorBlock; + typedef internal::TensorBlockReader<ScalarNoConst, Index, NumOutputDims, + Layout> + OutputTensorBlockReader; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_impl(op.expression(), device), m_dimensions(op.dimensions()) { // The total size of the reshaped tensor must be equal to the total size // of the input tensor. eigen_assert(internal::array_prod(m_impl.dimensions()) == internal::array_prod(op.dimensions())); - } - typedef typename XprType::Index Index; - typedef typename XprType::Scalar Scalar; - typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + if (BlockAccess) { + const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = + m_impl.dimensions(); + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_outputStrides[0] = 1; + for (int i = 1; i < NumOutputDims; ++i) { + m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; + } + m_inputStrides[0] = 1; + for (int i = 1; i < NumInputDims; ++i) { + m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; + } + } else { + m_outputStrides[NumOutputDims - 1] = 1; + for (int i = NumOutputDims - 2; i >= 0; --i) { + m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; + } + m_inputStrides[NumInputDims - 1] = 1; + for (int i = NumInputDims - 2; i >= 0; --i) { + m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1]; + } + } + } + } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -146,13 +190,151 @@ return m_impl.costPerCoeff(vectorized); } - EIGEN_DEVICE_FUNC Scalar* data() const { return const_cast<Scalar*>(m_impl.data()); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + m_impl.getResourceRequirements(resources); + } - const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + // required in block(OutputTensorBlock* output_block) const + // For C++03 compatibility this must be defined outside the method + struct BlockIteratorState { + Index stride; + Index span; + Index size; + Index count; + }; + // TODO(andydavis) Reduce the overhead of this function. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + OutputTensorBlock* output_block) const { + if (m_impl.data() != NULL) { + OutputTensorBlockReader::Run(output_block, m_impl.data()); + return; + } + + // Calculate output block unit-stride inner dimension length. + const DSizes<Index, NumOutputDims>& output_block_sizes = + output_block->block_sizes(); + Index output_inner_dim_size = 1; + Index output_outer_dim_start = NumOutputDims; + for (Index i = 0; i < NumOutputDims; ++i) { + const Index dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i : NumOutputDims - i - 1; + output_inner_dim_size *= output_block_sizes[dim]; + if (output_block_sizes[dim] < m_dimensions[dim]) { + output_outer_dim_start = i + 1; + break; + } + } + + // Initialize output block iterator state. + array<BlockIteratorState, NumOutputDims> block_iter_state; + + for (Index i = 0; i < NumOutputDims; ++i) { + const Index dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i : NumOutputDims - i - 1; + block_iter_state[i].size = output_block_sizes[dim]; + block_iter_state[i].stride = m_outputStrides[dim]; + block_iter_state[i].span = + block_iter_state[i].stride * (block_iter_state[i].size - 1); + block_iter_state[i].count = 0; + } + + const Index output_outer_dim_size = output_block_sizes.TotalSize() / + output_inner_dim_size; + const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = + m_impl.dimensions(); + + Index index = output_block->first_coeff_index(); + for (Index outer_idx = 0; outer_idx < output_outer_dim_size; ++outer_idx) { + Index inner_idx = 0; + while (inner_idx < output_inner_dim_size) { + // Calculate input coords based on 'index'. + array<Index, NumInputDims> input_coords; + Index idx = index; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumInputDims - 1; i > 0; --i) { + input_coords[i] = idx / m_inputStrides[i]; + idx -= input_coords[i] * m_inputStrides[i]; + } + input_coords[0] = idx; + } else { + for (int i = 0; i < NumInputDims - 1; ++i) { + input_coords[i] = idx / m_inputStrides[i]; + idx -= input_coords[i] * m_inputStrides[i]; + } + input_coords[NumInputDims - 1] = idx; + } + + // Calculate target input block shape, using at most + // 'output_inner_dim_size' coefficients along the input block's inner + // dimensions. + DSizes<Index, NumInputDims> input_block_sizes; + Index num_to_allocate = output_inner_dim_size - inner_idx; + for (Index i = 0; i < NumInputDims; ++i) { + const Index dim = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i : NumInputDims - i - 1; + input_block_sizes[dim] = numext::mini( + num_to_allocate, (static_cast<Index>(input_dims[dim]) - + input_coords[dim])); + if (input_coords[dim] == 0) { + num_to_allocate /= input_block_sizes[dim]; + } else { + num_to_allocate = 1; + } + } + + // Calculate input block strides. + DSizes<Index, NumInputDims> input_block_strides; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + input_block_strides[0] = 1; + for (int i = 1; i < NumInputDims; ++i) { + input_block_strides[i] = input_block_strides[i - 1] * + input_block_sizes[i - 1]; + } + } else { + input_block_strides[NumInputDims - 1] = 1; + for (int i = NumInputDims - 2; i >= 0; --i) { + input_block_strides[i] = input_block_strides[i + 1] * + input_block_sizes[i + 1]; + } + } + + // Instantiate and read input block from input tensor. + InputTensorBlock input_block(index, input_block_sizes, + input_block_strides, m_inputStrides, + output_block->data() + outer_idx * + output_inner_dim_size + inner_idx); + + m_impl.block(&input_block); + + const Index input_block_total_size = input_block_sizes.TotalSize(); + index += input_block_total_size; + inner_idx += input_block_total_size; + } + eigen_assert(inner_idx == output_inner_dim_size); + index -= output_inner_dim_size; + // Update index. + for (Index i = output_outer_dim_start; i < NumOutputDims; ++i) { + if (++block_iter_state[i].count < block_iter_state[i].size) { + index += block_iter_state[i].stride; + break; + } + block_iter_state[i].count = 0; + index -= block_iter_state[i].span; + } + } + } + + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return const_cast<Scalar*>(m_impl.data()); } + + EIGEN_DEVICE_FUNC const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } protected: TensorEvaluator<ArgType, Device> m_impl; NewDimensions m_dimensions; + DSizes<Index, NumOutputDims> m_outputStrides; + DSizes<Index, NumInputDims> m_inputStrides; }; @@ -169,6 +351,8 @@ enum { IsAligned = TensorEvaluator<ArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = TensorEvaluator<ArgType, Device>::RawAccess @@ -214,6 +398,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = array_size<StartIndices>::value; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename StartIndices, typename Sizes, typename XprType> @@ -299,6 +484,16 @@ EIGEN_DEVICE_FUNC bool operator ()(Index val) const { return val > 4*1024*1024; } }; #endif + +// It is very expensive to start the memcpy kernel on GPU: we therefore only +// use it for large copies. +#ifdef EIGEN_USE_SYCL +template <typename Index> struct MemcpyTriggerForSlicing<Index, const Eigen::SyclDevice> { + EIGEN_DEVICE_FUNC MemcpyTriggerForSlicing(const SyclDevice&) { } + EIGEN_DEVICE_FUNC bool operator ()(Index val) const { return val > 4*1024*1024; } +}; +#endif + } // Eval as rvalue @@ -308,23 +503,46 @@ typedef TensorSlicingOp<StartIndices, Sizes, ArgType> XprType; static const int NumDims = internal::array_size<Sizes>::value; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + typedef Sizes Dimensions; + enum { // Alignment can't be guaranteed at compile time since it depends on the // slice offsets and sizes. - IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, - RawAccess = false + IsAligned = false, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, + RawAccess = false }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> TensorBlock; + typedef typename TensorBlock::Dimensions TensorBlockDimensions; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_impl(op.expression(), device), m_device(device), m_dimensions(op.sizes()), m_offsets(op.startIndices()) { - for (std::size_t i = 0; i < internal::array_size<Dimensions>::value; ++i) { + for (Index i = 0; i < internal::array_size<Dimensions>::value; ++i) { eigen_assert(m_impl.dimensions()[i] >= op.sizes()[i] + op.startIndices()[i]); } + m_is_identity = true; + for (int i = 0; i < internal::array_size<Dimensions>::value; ++i) { + eigen_assert(m_impl.dimensions()[i] >= + op.sizes()[i] + op.startIndices()[i]); + if (m_impl.dimensions()[i] != op.sizes()[i] || + op.startIndices()[i] != 0) { + m_is_identity = false; + } + } + const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); const Sizes& output_dims = op.sizes(); if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -354,12 +572,6 @@ } } - typedef typename XprType::Index Index; - typedef typename XprType::Scalar Scalar; - typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - typedef Sizes Dimensions; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -386,7 +598,7 @@ const MemcpyTriggerForSlicing<Index, Device> trigger(m_device); if (trigger(contiguous_values)) { Scalar* src = (Scalar*)m_impl.data(); - for (int i = 0; i < internal::array_prod(dimensions()); i += contiguous_values) { + for (Index i = 0; i < internal::array_prod(dimensions()); i += contiguous_values) { Index offset = srcCoeff(i); m_device.memcpy((void*)(data+i), src+offset, contiguous_values * sizeof(Scalar)); } @@ -402,16 +614,24 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_impl.coeff(srcCoeff(index)); + if (m_is_identity) { + return m_impl.coeff(index); + } else { + return m_impl.coeff(srcCoeff(index)); + } } template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - const int packetSize = internal::unpacket_traits<PacketReturnType>::size; - EIGEN_STATIC_ASSERT(packetSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + const int packetSize = PacketType<CoeffReturnType, Device>::size; + EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+packetSize-1 < internal::array_prod(dimensions())); + if (m_is_identity) { + return m_impl.template packet<LoadMode>(index); + } + Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -454,12 +674,30 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { - return m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, NumDims); + return m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, m_is_identity ? 1 : NumDims); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.lastLevelCacheSize() / sizeof(Scalar)); + resources->push_back(internal::TensorOpResourceRequirements( + internal::kSkewedInnerDims, block_total_size_max)); + m_impl.getResourceRequirements(resources); + } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar* data() const { - Scalar* result = m_impl.data(); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + TensorBlock* output_block) const { + TensorBlock input_block(srcCoeff(output_block->first_coeff_index()), + output_block->block_sizes(), + output_block->block_strides(), + TensorBlockDimensions(m_inputStrides), + output_block->data()); + m_impl.block(&input_block); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { + Scalar* result = const_cast<Scalar*>(m_impl.data()); if (result) { Index offset = 0; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -493,7 +731,14 @@ } return NULL; } - + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const{ + return m_impl; + } + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const StartIndices& startIndices() const{ + return m_offsets; + } protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { @@ -522,6 +767,7 @@ TensorEvaluator<ArgType, Device> m_impl; const Device& m_device; Dimensions m_dimensions; + bool m_is_identity; const StartIndices m_offsets; }; @@ -535,33 +781,49 @@ typedef TensorSlicingOp<StartIndices, Sizes, ArgType> XprType; static const int NumDims = internal::array_size<Sizes>::value; - enum { - IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, - PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, - RawAccess = false - }; - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : Base(op, device) - { } - typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef Sizes Dimensions; + enum { + IsAligned = false, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, + RawAccess = (NumDims == 1) & TensorEvaluator<ArgType, Device>::RawAccess + }; + + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> TensorBlock; + typedef typename TensorBlock::Dimensions TensorBlockDimensions; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) + : Base(op, device) + { } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { - return this->m_impl.coeffRef(this->srcCoeff(index)); + if (this->m_is_identity) { + return this->m_impl.coeffRef(index); + } else { + return this->m_impl.coeffRef(this->srcCoeff(index)); + } } template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - const int packetSize = internal::unpacket_traits<PacketReturnType>::size; + if (this->m_is_identity) { + this->m_impl.template writePacket<StoreMode>(index, x); + return; + } + + const int packetSize = PacketType<CoeffReturnType, Device>::size; Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -600,6 +862,327 @@ } } } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock( + const TensorBlock& block) { + this->m_impl.writeBlock(TensorBlock( + this->srcCoeff(block.first_coeff_index()), block.block_sizes(), + block.block_strides(), TensorBlockDimensions(this->m_inputStrides), + const_cast<ScalarNoConst*>(block.data()))); + } +}; + +namespace internal { +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType> +struct traits<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> > : public traits<XprType> +{ + typedef typename XprType::Scalar Scalar; + typedef traits<XprType> XprTraits; + typedef typename XprTraits::StorageKind StorageKind; + typedef typename XprTraits::Index Index; + typedef typename XprType::Nested Nested; + typedef typename remove_reference<Nested>::type _Nested; + static const int NumDimensions = array_size<StartIndices>::value; + static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; +}; + +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType> +struct eval<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, Eigen::Dense> +{ + typedef const TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>& type; +}; + +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType> +struct nested<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, 1, typename eval<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> >::type> +{ + typedef TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> type; +}; + +} // end namespace internal + + +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType> +class TensorStridingSlicingOp : public TensorBase<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> > +{ + public: + typedef typename internal::traits<TensorStridingSlicingOp>::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename internal::nested<TensorStridingSlicingOp>::type Nested; + typedef typename internal::traits<TensorStridingSlicingOp>::StorageKind StorageKind; + typedef typename internal::traits<TensorStridingSlicingOp>::Index Index; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorStridingSlicingOp( + const XprType& expr, const StartIndices& startIndices, + const StopIndices& stopIndices, const Strides& strides) + : m_xpr(expr), m_startIndices(startIndices), m_stopIndices(stopIndices), + m_strides(strides) {} + + EIGEN_DEVICE_FUNC + const StartIndices& startIndices() const { return m_startIndices; } + EIGEN_DEVICE_FUNC + const StartIndices& stopIndices() const { return m_stopIndices; } + EIGEN_DEVICE_FUNC + const StartIndices& strides() const { return m_strides; } + + EIGEN_DEVICE_FUNC + const typename internal::remove_all<typename XprType::Nested>::type& + expression() const { return m_xpr; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE TensorStridingSlicingOp& operator = (const TensorStridingSlicingOp& other) + { + typedef TensorAssignOp<TensorStridingSlicingOp, const TensorStridingSlicingOp> Assign; + Assign assign(*this, other); + internal::TensorExecutor<const Assign, DefaultDevice>::run( + assign, DefaultDevice()); + return *this; + } + + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE TensorStridingSlicingOp& operator = (const OtherDerived& other) + { + typedef TensorAssignOp<TensorStridingSlicingOp, const OtherDerived> Assign; + Assign assign(*this, other); + internal::TensorExecutor<const Assign, DefaultDevice>::run( + assign, DefaultDevice()); + return *this; + } + + protected: + typename XprType::Nested m_xpr; + const StartIndices m_startIndices; + const StopIndices m_stopIndices; + const Strides m_strides; +}; + +// Eval as rvalue +template<typename StartIndices, typename StopIndices, typename Strides, typename ArgType, typename Device> +struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType>, Device> +{ + typedef TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType> XprType; + static const int NumDims = internal::array_size<Strides>::value; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + typedef Strides Dimensions; + + enum { + // Alignment can't be guaranteed at compile time since it depends on the + // slice offsets and sizes. + IsAligned = false, + PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) + : m_impl(op.expression(), device), + m_device(device), + m_strides(op.strides()), m_exprStartIndices(op.startIndices()), + m_exprStopIndices(op.stopIndices()) + { + // Handle degenerate intervals by gracefully clamping and allowing m_dimensions to be zero + DSizes<Index, NumDims> startIndicesClamped, stopIndicesClamped; + for (ptrdiff_t i = 0; i < internal::array_size<Dimensions>::value; ++i) { + eigen_assert(m_strides[i] != 0 && "0 stride is invalid"); + if (m_strides[i] > 0) { + startIndicesClamped[i] = + clamp(op.startIndices()[i], 0, m_impl.dimensions()[i]); + stopIndicesClamped[i] = + clamp(op.stopIndices()[i], 0, m_impl.dimensions()[i]); + } else { + /* implies m_strides[i] < 0 by assert */ + startIndicesClamped[i] = + clamp(op.startIndices()[i], -1, m_impl.dimensions()[i] - 1); + stopIndicesClamped[i] = + clamp(op.stopIndices()[i], -1, m_impl.dimensions()[i] - 1); + } + m_startIndices[i] = startIndicesClamped[i]; + } + + typedef typename TensorEvaluator<ArgType, Device>::Dimensions InputDimensions; + const InputDimensions& input_dims = m_impl.dimensions(); + + // check for degenerate intervals and compute output tensor shape + bool degenerate = false; + m_is_identity = true; + for (int i = 0; i < NumDims; i++) { + Index interval = stopIndicesClamped[i] - startIndicesClamped[i]; + if (interval == 0 || ((interval < 0) != (m_strides[i] < 0))) { + m_dimensions[i] = 0; + degenerate = true; + } else { + m_dimensions[i] = + (interval / m_strides[i]) + (interval % m_strides[i] != 0 ? 1 : 0); + eigen_assert(m_dimensions[i] >= 0); + } + if (m_strides[i] != 1 || interval != m_impl.dimensions()[i]) { + m_is_identity = false; + } + } + + Strides output_dims = m_dimensions; + + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_inputStrides[0] = m_strides[0]; + m_offsets[0] = startIndicesClamped[0]; + Index previousDimProduct = 1; + for (int i = 1; i < NumDims; ++i) { + previousDimProduct *= input_dims[i-1]; + m_inputStrides[i] = previousDimProduct * m_strides[i]; + m_offsets[i] = startIndicesClamped[i] * previousDimProduct; + } + + // Don't initialize m_fastOutputStrides[0] since it won't ever be accessed. + m_outputStrides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + m_outputStrides[i] = m_outputStrides[i-1] * output_dims[i-1]; + // NOTE: if tensor is degenerate, we send 1 to prevent TensorIntDivisor constructor crash + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(degenerate ? 1 : m_outputStrides[i]); + } + } else { + m_inputStrides[NumDims-1] = m_strides[NumDims-1]; + m_offsets[NumDims-1] = startIndicesClamped[NumDims-1]; + Index previousDimProduct = 1; + for (int i = NumDims - 2; i >= 0; --i) { + previousDimProduct *= input_dims[i+1]; + m_inputStrides[i] = previousDimProduct * m_strides[i]; + m_offsets[i] = startIndicesClamped[i] * previousDimProduct; + } + + m_outputStrides[NumDims-1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + m_outputStrides[i] = m_outputStrides[i+1] * output_dims[i+1]; + // NOTE: if tensor is degenerate, we send 1 to prevent TensorIntDivisor constructor crash + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(degenerate ? 1 : m_outputStrides[i]); + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } + + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType*) { + m_impl.evalSubExprsIfNeeded(NULL); + return true; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + m_impl.cleanup(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const + { + if (m_is_identity) { + return m_impl.coeff(index); + } else { + return m_impl.coeff(srcCoeff(index)); + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { + return m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, m_is_identity ? 1 : NumDims); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { + return NULL; + } + + //use by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const StartIndices& exprStartIndices() const { return m_exprStartIndices; } + //use by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const StartIndices& exprStopIndices() const { return m_exprStopIndices; } + //use by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const StartIndices& strides() const { return m_strides; } + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const{return m_impl;} + + protected: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const + { + Index inputIndex = 0; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumDims - 1; i >= 0; --i) { + const Index idx = index / m_fastOutputStrides[i]; + inputIndex += idx * m_inputStrides[i] + m_offsets[i]; + index -= idx * m_outputStrides[i]; + } + } else { + for (int i = 0; i < NumDims; ++i) { + const Index idx = index / m_fastOutputStrides[i]; + inputIndex += idx * m_inputStrides[i] + m_offsets[i]; + index -= idx * m_outputStrides[i]; + } + } + return inputIndex; + } + + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index clamp(Index value, Index min, Index max) { +#ifndef __SYCL_DEVICE_ONLY__ + return numext::maxi(min, numext::mini(max,value)); +#else + return cl::sycl::clamp(value, min, max); +#endif + } + + array<Index, NumDims> m_outputStrides; + array<internal::TensorIntDivisor<Index>, NumDims> m_fastOutputStrides; + array<Index, NumDims> m_inputStrides; + bool m_is_identity; + TensorEvaluator<ArgType, Device> m_impl; + const Device& m_device; + DSizes<Index, NumDims> m_startIndices; // clamped startIndices + DSizes<Index, NumDims> m_dimensions; + DSizes<Index, NumDims> m_offsets; // offset in a flattened shape + const Strides m_strides; + //use by sycl + const StartIndices m_exprStartIndices; + //use by sycl + const StopIndices m_exprStopIndices; +}; + +// Eval as lvalue +template<typename StartIndices, typename StopIndices, typename Strides, typename ArgType, typename Device> +struct TensorEvaluator<TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType>, Device> + : public TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType>, Device> +{ + typedef TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType>, Device> Base; + typedef TensorStridingSlicingOp<StartIndices, StopIndices, Strides, ArgType> XprType; + static const int NumDims = internal::array_size<Strides>::value; + + enum { + IsAligned = false, + PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = TensorEvaluator<ArgType, Device>::CoordAccess, + RawAccess = false + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) + : Base(op, device) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + typedef Strides Dimensions; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) + { + if (this->m_is_identity) { + return this->m_impl.coeffRef(index); + } else { + return this->m_impl.coeffRef(this->srcCoeff(index)); + } + } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h index 88b838b..4837f22 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename PaddingDimensions, typename XprType> @@ -90,11 +91,13 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = false, + IsAligned = true, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = true, RawAccess = false @@ -106,7 +109,7 @@ // The padding op doesn't change the rank of the tensor. Directly padding a scalar would lead // to a vector, which doesn't make sense. Instead one should reshape the scalar into a vector // of 1 element first and then pad. - EIGEN_STATIC_ASSERT(NumDims > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); // Compute dimensions m_dimensions = m_impl.dimensions(); @@ -150,27 +153,26 @@ if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; - if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { + if (isPaddingAtIndexForDim(idx, i)) { return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } - if (index < m_padding[0].first || index >= m_dimensions[0] - m_padding[0].second) { + if (isPaddingAtIndexForDim(index, 0)) { return m_paddingValue; } inputIndex += (index - m_padding[0].first); } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i+1]; - if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { + if (isPaddingAtIndexForDim(idx, i)) { return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; index -= idx * m_outputStrides[i+1]; } - if (index < m_padding[NumDims-1].first || - index >= m_dimensions[NumDims-1] - m_padding[NumDims-1].second) { + if (isPaddingAtIndexForDim(index, NumDims-1)) { return m_paddingValue; } inputIndex += (index - m_padding[NumDims-1].first); @@ -187,43 +189,6 @@ return packetRowMajor(index); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<Index, NumDims>& coords) const - { - Index inputIndex; - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - { - const Index idx = coords[0]; - if (idx < m_padding[0].first || idx >= m_dimensions[0] - m_padding[0].second) { - return m_paddingValue; - } - inputIndex = idx - m_padding[0].first; - } - for (int i = 1; i < NumDims; ++i) { - const Index idx = coords[i]; - if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return m_paddingValue; - } - inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; - } - } else { - { - const Index idx = coords[NumDims-1]; - if (idx < m_padding[NumDims-1].first || idx >= m_dimensions[NumDims-1] - m_padding[NumDims-1].second) { - return m_paddingValue; - } - inputIndex = idx - m_padding[NumDims-1].first; - } - for (int i = NumDims - 2; i >= 0; --i) { - const Index idx = coords[i]; - if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return m_paddingValue; - } - inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; - } - } - return m_impl.coeff(inputIndex); - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { TensorOpCost cost = m_impl.costPerCoeff(vectorized); if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -236,9 +201,50 @@ return cost; } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PaddingDimensions& padding() const { return m_padding; } + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& padding_value() const { return m_paddingValue; } + /// used by sycl + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const{return m_impl;} private: + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isPaddingAtIndexForDim( + Index index, int dim_index) const { +#if defined(EIGEN_HAS_INDEX_LIST) + return (!internal::index_pair_first_statically_eq<PaddingDimensions>(dim_index, 0) && + index < m_padding[dim_index].first) || + (!internal::index_pair_second_statically_eq<PaddingDimensions>(dim_index, 0) && + index >= m_dimensions[dim_index] - m_padding[dim_index].second); +#else + return (index < m_padding[dim_index].first) || + (index >= m_dimensions[dim_index] - m_padding[dim_index].second); +#endif + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isLeftPaddingCompileTimeZero( + int dim_index) const { +#if defined(EIGEN_HAS_INDEX_LIST) + return internal::index_pair_first_statically_eq<PaddingDimensions>(dim_index, 0); +#else + EIGEN_UNUSED_VARIABLE(dim_index); + return false; +#endif + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool isRightPaddingCompileTimeZero( + int dim_index) const { +#if defined(EIGEN_HAS_INDEX_LIST) + return internal::index_pair_second_statically_eq<PaddingDimensions>(dim_index, 0); +#else + EIGEN_UNUSED_VARIABLE(dim_index); + return false; +#endif + } + + void updateCostPerDimension(TensorOpCost& cost, int i, bool first) const { const double in = static_cast<double>(m_impl.dimensions()[i]); const double out = in + m_padding[i].first + m_padding[i].second; @@ -261,27 +267,27 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetColMajor(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); const Index initialIndex = index; Index inputIndex = 0; for (int i = NumDims - 1; i > 0; --i) { - const Index first = index; - const Index last = index + PacketSize - 1; + const Index firstIdx = index; + const Index lastIdx = index + PacketSize - 1; const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i]; const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i]; const Index lastPaddedRight = m_outputStrides[i+1]; - if (last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= lastPaddedLeft && last < firstPaddedRight) { + else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. const Index idx = index / m_outputStrides[i]; inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; @@ -293,21 +299,21 @@ } } - const Index last = index + PacketSize - 1; - const Index first = index; + const Index lastIdx = index + PacketSize - 1; + const Index firstIdx = index; const Index lastPaddedLeft = m_padding[0].first; const Index firstPaddedRight = (m_dimensions[0] - m_padding[0].second); const Index lastPaddedRight = m_outputStrides[1]; - if (last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(0) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(0) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= lastPaddedLeft && last < firstPaddedRight) { + else if ((isLeftPaddingCompileTimeZero(0) && isRightPaddingCompileTimeZero(0)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. inputIndex += (index - m_padding[0].first); return m_impl.template packet<Unaligned>(inputIndex); @@ -318,28 +324,28 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetRowMajor(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); const Index initialIndex = index; Index inputIndex = 0; for (int i = 0; i < NumDims - 1; ++i) { - const Index first = index; - const Index last = index + PacketSize - 1; + const Index firstIdx = index; + const Index lastIdx = index + PacketSize - 1; const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i+1]; const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i+1]; const Index lastPaddedRight = m_outputStrides[i]; - if (last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= lastPaddedLeft && last < firstPaddedRight) { + else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. const Index idx = index / m_outputStrides[i+1]; inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; @@ -351,21 +357,21 @@ } } - const Index last = index + PacketSize - 1; - const Index first = index; + const Index lastIdx = index + PacketSize - 1; + const Index firstIdx = index; const Index lastPaddedLeft = m_padding[NumDims-1].first; const Index firstPaddedRight = (m_dimensions[NumDims-1] - m_padding[NumDims-1].second); const Index lastPaddedRight = m_outputStrides[NumDims-1]; - if (last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(NumDims-1) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(NumDims-1) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1<PacketReturnType>(m_paddingValue); } - else if (first >= lastPaddedLeft && last < firstPaddedRight) { + else if ((isLeftPaddingCompileTimeZero(NumDims-1) && isRightPaddingCompileTimeZero(NumDims-1)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. inputIndex += (index - m_padding[NumDims-1].first); return m_impl.template packet<Unaligned>(inputIndex);
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h index a87e453..4292fe0 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions + 1; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename PatchDim, typename XprType> @@ -87,12 +88,14 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, RawAccess = false @@ -100,6 +103,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_impl(op.expression(), device) +#ifdef EIGEN_USE_SYCL + , m_patch_dims(op.patch_dims()) +#endif { Index num_patches = 1; const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); @@ -184,7 +190,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); Index output_stride_index = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? NumDims - 1 : 0; @@ -253,7 +259,12 @@ TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + +#ifdef EIGEN_USE_SYCL + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PatchDim& functor() const { return m_patch_dims; } +#endif protected: Dimensions m_dimensions; @@ -262,6 +273,10 @@ array<Index, NumDims-1> m_patchStrides; TensorEvaluator<ArgType, Device> m_impl; + +#ifdef EIGEN_USE_SYCL + const PatchDim m_patch_dims; +#endif }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorRandom.h b/unsupported/Eigen/CXX11/src/Tensor/TensorRandom.h new file mode 100644 index 0000000..787cbd0 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorRandom.h
@@ -0,0 +1,268 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_RANDOM_H +#define EIGEN_CXX11_TENSOR_TENSOR_RANDOM_H + +namespace Eigen { +namespace internal { + +namespace { + +EIGEN_DEVICE_FUNC uint64_t get_random_seed() { +#if defined(EIGEN_GPU_COMPILE_PHASE) + // We don't support 3d kernels since we currently only use 1 and + // 2d kernels. + gpu_assert(threadIdx.z == 0); + return clock64() + + blockIdx.x * blockDim.x + threadIdx.x + + gridDim.x * blockDim.x * (blockIdx.y * blockDim.y + threadIdx.y); + +#elif defined _WIN32 + // Use the current time as a baseline. + SYSTEMTIME st; + GetSystemTime(&st); + int time = st.wSecond + 1000 * st.wMilliseconds; + // Mix in a random number to make sure that we get different seeds if + // we try to generate seeds faster than the clock resolution. + // We need 2 random values since the generator only generate 16 bits at + // a time (https://msdn.microsoft.com/en-us/library/398ax69y.aspx) + int rnd1 = ::rand(); + int rnd2 = ::rand(); + uint64_t rnd = (rnd1 | rnd2 << 16) ^ time; + return rnd; + +#elif defined __APPLE__ + // Same approach as for win32, except that the random number generator + // is better (// https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man3/random.3.html#//apple_ref/doc/man/3/random). + uint64_t rnd = ::random() ^ mach_absolute_time(); + return rnd; + +#else + // Augment the current time with pseudo random number generation + // to ensure that we get different seeds if we try to generate seeds + // faster than the clock resolution. + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + uint64_t rnd = ::random() ^ ts.tv_nsec; + return rnd; +#endif +} + +static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE unsigned PCG_XSH_RS_generator(uint64_t* state, uint64_t stream) { + // TODO: Unify with the implementation in the non blocking thread pool. + uint64_t current = *state; + // Update the internal state + *state = current * 6364136223846793005ULL + (stream << 1 | 1); + // Generate the random output (using the PCG-XSH-RS scheme) + return static_cast<unsigned>((current ^ (current >> 22)) >> (22 + (current >> 61))); +} + +static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE uint64_t PCG_XSH_RS_state(uint64_t seed) { + seed = seed ? seed : get_random_seed(); + return seed * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL; +} + +} // namespace + + +template <typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +T RandomToTypeUniform(uint64_t* state, uint64_t stream) { + unsigned rnd = PCG_XSH_RS_generator(state, stream); + return static_cast<T>(rnd); +} + + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Eigen::half RandomToTypeUniform<Eigen::half>(uint64_t* state, uint64_t stream) { + Eigen::half result; + // Generate 10 random bits for the mantissa + unsigned rnd = PCG_XSH_RS_generator(state, stream); + result.x = static_cast<uint16_t>(rnd & 0x3ffu); + // Set the exponent + result.x |= (static_cast<uint16_t>(15) << 10); + // Return the final result + return result - Eigen::half(1.0f); +} + + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float RandomToTypeUniform<float>(uint64_t* state, uint64_t stream) { + typedef union { + uint32_t raw; + float fp; + } internal; + internal result; + // Generate 23 random bits for the mantissa mantissa + const unsigned rnd = PCG_XSH_RS_generator(state, stream); + result.raw = rnd & 0x7fffffu; + // Set the exponent + result.raw |= (static_cast<uint32_t>(127) << 23); + // Return the final result + return result.fp - 1.0f; +} + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double RandomToTypeUniform<double>(uint64_t* state, uint64_t stream) { + typedef union { + uint64_t raw; + double dp; + } internal; + internal result; + result.raw = 0; + // Generate 52 random bits for the mantissa + // First generate the upper 20 bits + unsigned rnd1 = PCG_XSH_RS_generator(state, stream) & 0xfffffu; + // The generate the lower 32 bits + unsigned rnd2 = PCG_XSH_RS_generator(state, stream); + result.raw = (static_cast<uint64_t>(rnd1) << 32) | rnd2; + // Set the exponent + result.raw |= (static_cast<uint64_t>(1023) << 52); + // Return the final result + return result.dp - 1.0; +} + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +std::complex<float> RandomToTypeUniform<std::complex<float> >(uint64_t* state, uint64_t stream) { + return std::complex<float>(RandomToTypeUniform<float>(state, stream), + RandomToTypeUniform<float>(state, stream)); +} +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +std::complex<double> RandomToTypeUniform<std::complex<double> >(uint64_t* state, uint64_t stream) { + return std::complex<double>(RandomToTypeUniform<double>(state, stream), + RandomToTypeUniform<double>(state, stream)); +} + +template <typename T> class UniformRandomGenerator { + public: + static const bool PacketAccess = true; + + // Uses the given "seed" if non-zero, otherwise uses a random seed. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE UniformRandomGenerator( + uint64_t seed = 0) { + m_state = PCG_XSH_RS_state(seed); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE UniformRandomGenerator( + const UniformRandomGenerator& other) { + m_state = other.m_state; + } + + template<typename Index> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T operator()(Index i) const { + T result = RandomToTypeUniform<T>(&m_state, i); + return result; + } + + template<typename Packet, typename Index> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(Index i) const { + const int packetSize = internal::unpacket_traits<Packet>::size; + EIGEN_ALIGN_MAX T values[packetSize]; + for (int j = 0; j < packetSize; ++j) { + values[j] = RandomToTypeUniform<T>(&m_state, i); + } + return internal::pload<Packet>(values); + } + + private: + mutable uint64_t m_state; +}; + +template <typename Scalar> +struct functor_traits<UniformRandomGenerator<Scalar> > { + enum { + // Rough estimate for floating point, multiplied by ceil(sizeof(T) / sizeof(float)). + Cost = 12 * NumTraits<Scalar>::AddCost * + ((sizeof(Scalar) + sizeof(float) - 1) / sizeof(float)), + PacketAccess = UniformRandomGenerator<Scalar>::PacketAccess + }; +}; + + + +template <typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +T RandomToTypeNormal(uint64_t* state, uint64_t stream) { + // Use the ratio of uniform method to generate numbers following a normal + // distribution. See for example Numerical Recipes chapter 7.3.9 for the + // details. + T u, v, q; + do { + u = RandomToTypeUniform<T>(state, stream); + v = T(1.7156) * (RandomToTypeUniform<T>(state, stream) - T(0.5)); + const T x = u - T(0.449871); + const T y = numext::abs(v) + T(0.386595); + q = x*x + y * (T(0.196)*y - T(0.25472)*x); + } while (q > T(0.27597) && + (q > T(0.27846) || v*v > T(-4) * numext::log(u) * u*u)); + + return v/u; +} + +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +std::complex<float> RandomToTypeNormal<std::complex<float> >(uint64_t* state, uint64_t stream) { + return std::complex<float>(RandomToTypeNormal<float>(state, stream), + RandomToTypeNormal<float>(state, stream)); +} +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +std::complex<double> RandomToTypeNormal<std::complex<double> >(uint64_t* state, uint64_t stream) { + return std::complex<double>(RandomToTypeNormal<double>(state, stream), + RandomToTypeNormal<double>(state, stream)); +} + + +template <typename T> class NormalRandomGenerator { + public: + static const bool PacketAccess = true; + + // Uses the given "seed" if non-zero, otherwise uses a random seed. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE NormalRandomGenerator(uint64_t seed = 0) { + m_state = PCG_XSH_RS_state(seed); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE NormalRandomGenerator( + const NormalRandomGenerator& other) { + m_state = other.m_state; + } + + template<typename Index> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T operator()(Index i) const { + T result = RandomToTypeNormal<T>(&m_state, i); + return result; + } + + template<typename Packet, typename Index> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(Index i) const { + const int packetSize = internal::unpacket_traits<Packet>::size; + EIGEN_ALIGN_MAX T values[packetSize]; + for (int j = 0; j < packetSize; ++j) { + values[j] = RandomToTypeNormal<T>(&m_state, i); + } + return internal::pload<Packet>(values); + } + + private: + mutable uint64_t m_state; +}; + + +template <typename Scalar> +struct functor_traits<NormalRandomGenerator<Scalar> > { + enum { + // On average, we need to generate about 3 random numbers + // 15 mul, 8 add, 1.5 logs + Cost = 3 * functor_traits<UniformRandomGenerator<Scalar> >::Cost + + 15 * NumTraits<Scalar>::AddCost + 8 * NumTraits<Scalar>::AddCost + + 3 * functor_traits<scalar_log_op<Scalar> >::Cost / 2, + PacketAccess = NormalRandomGenerator<Scalar>::PacketAccess + }; +}; + + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_RANDOM_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h index 885295f..6706b44 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h
@@ -2,6 +2,7 @@ // for linear algebra. // // Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// Copyright (C) 2016 Mehdi Goli, Codeplay Software Ltd <eigen@codeplay.com> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -10,8 +11,20 @@ #ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_H #define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_H +// clang is incompatible with the CUDA syntax wrt making a kernel a class friend, +// so we'll use a macro to make clang happy. +#ifndef KERNEL_FRIEND +#if defined(__clang__) && (defined(__CUDA__) || defined(__HIP__)) +#define KERNEL_FRIEND friend __global__ +#else +#define KERNEL_FRIEND friend +#endif +#endif + + namespace Eigen { + /** \class TensorReduction * \ingroup CXX11_Tensor_Module * @@ -20,8 +33,8 @@ */ namespace internal { -template<typename Op, typename Dims, typename XprType> -struct traits<TensorReductionOp<Op, Dims, XprType> > + template<typename Op, typename Dims, typename XprType,template <class> class MakePointer_ > + struct traits<TensorReductionOp<Op, Dims, XprType, MakePointer_> > : traits<XprType> { typedef traits<XprType> XprTraits; @@ -31,18 +44,25 @@ typedef typename XprType::Nested Nested; static const int NumDimensions = XprTraits::NumDimensions - array_size<Dims>::value; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; + + template <class T> struct MakePointer { + // Intermediate typedef to workaround MSVC issue. + typedef MakePointer_<T> MakePointerT; + typedef typename MakePointerT::Type Type; + }; }; -template<typename Op, typename Dims, typename XprType> -struct eval<TensorReductionOp<Op, Dims, XprType>, Eigen::Dense> +template<typename Op, typename Dims, typename XprType, template <class> class MakePointer_> +struct eval<TensorReductionOp<Op, Dims, XprType, MakePointer_>, Eigen::Dense> { - typedef const TensorReductionOp<Op, Dims, XprType>& type; + typedef const TensorReductionOp<Op, Dims, XprType, MakePointer_>& type; }; -template<typename Op, typename Dims, typename XprType> -struct nested<TensorReductionOp<Op, Dims, XprType>, 1, typename eval<TensorReductionOp<Op, Dims, XprType> >::type> +template<typename Op, typename Dims, typename XprType, template <class> class MakePointer_> +struct nested<TensorReductionOp<Op, Dims, XprType, MakePointer_>, 1, typename eval<TensorReductionOp<Op, Dims, XprType, MakePointer_> >::type> { - typedef TensorReductionOp<Op, Dims, XprType> type; + typedef TensorReductionOp<Op, Dims, XprType, MakePointer_> type; }; @@ -87,7 +107,7 @@ static const bool value = false; }; -#if defined(EIGEN_HAS_CONSTEXPR) && defined(EIGEN_HAS_VARIADIC_TEMPLATES) +#if EIGEN_HAS_CONSTEXPR && EIGEN_HAS_VARIADIC_TEMPLATES template <typename ReducedDims, int NumTensorDims> struct are_inner_most_dims<ReducedDims, NumTensorDims, ColMajor>{ static const bool tmp1 = indices_statically_known_to_increase<ReducedDims>(); @@ -122,7 +142,7 @@ template <int DimIndex, typename Self, typename Op> struct GenericDimReducer { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const Self& self, typename Self::Index firstIndex, Op& reducer, typename Self::CoeffReturnType* accum) { - EIGEN_STATIC_ASSERT(DimIndex > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((DimIndex > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); for (int j = 0; j < self.m_reducedDims[DimIndex]; ++j) { const typename Self::Index input = firstIndex + j * self.m_reducedStrides[DimIndex]; GenericDimReducer<DimIndex-1, Self, Op>::reduce(self, input, reducer, accum); @@ -145,7 +165,9 @@ } }; -template <typename Self, typename Op, bool Vectorizable = (Self::InputPacketAccess & Op::PacketAccess)> +template <typename Self, typename Op, bool Vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess), + bool UseTreeReduction = (!Self::ReducerTraits::IsStateful && + !Self::ReducerTraits::IsExactlyAssociative)> struct InnerMostDimReducer { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Self::CoeffReturnType reduce(const Self& self, typename Self::Index firstIndex, typename Self::Index numValuesToReduce, Op& reducer) { typename Self::CoeffReturnType accum = reducer.initialize(); @@ -157,23 +179,88 @@ }; template <typename Self, typename Op> -struct InnerMostDimReducer<Self, Op, true> { +struct InnerMostDimReducer<Self, Op, true, false> { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Self::CoeffReturnType reduce(const Self& self, typename Self::Index firstIndex, typename Self::Index numValuesToReduce, Op& reducer) { - const int packetSize = internal::unpacket_traits<typename Self::PacketReturnType>::size; + const typename Self::Index packetSize = internal::unpacket_traits<typename Self::PacketReturnType>::size; const typename Self::Index VectorizedSize = (numValuesToReduce / packetSize) * packetSize; - typename Self::PacketReturnType p = reducer.template initializePacket<typename Self::PacketReturnType>(); + typename Self::PacketReturnType paccum = reducer.template initializePacket<typename Self::PacketReturnType>(); for (typename Self::Index j = 0; j < VectorizedSize; j += packetSize) { - reducer.reducePacket(self.m_impl.template packet<Unaligned>(firstIndex + j), &p); + reducer.reducePacket(self.m_impl.template packet<Unaligned>(firstIndex + j), &paccum); } typename Self::CoeffReturnType accum = reducer.initialize(); for (typename Self::Index j = VectorizedSize; j < numValuesToReduce; ++j) { reducer.reduce(self.m_impl.coeff(firstIndex + j), &accum); } - return reducer.finalizeBoth(accum, p); + return reducer.finalizeBoth(accum, paccum); } }; -template <int DimIndex, typename Self, typename Op, bool vectorizable = (Self::InputPacketAccess & Op::PacketAccess)> +static const int kLeafSize = 1024; + +template <typename Self, typename Op> +struct InnerMostDimReducer<Self, Op, false, true> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Self::CoeffReturnType + reduce(const Self& self, typename Self::Index firstIndex, + typename Self::Index numValuesToReduce, Op& reducer) { + typename Self::CoeffReturnType accum = reducer.initialize(); + if (numValuesToReduce > kLeafSize) { + const typename Self::Index half = numValuesToReduce / 2; + reducer.reduce(reduce(self, firstIndex, half, reducer), &accum); + reducer.reduce( + reduce(self, firstIndex + half, numValuesToReduce - half, reducer), + &accum); + } else { + for (typename Self::Index j = 0; j < numValuesToReduce; ++j) { + reducer.reduce(self.m_impl.coeff(firstIndex + j), &accum); + } + } + return reducer.finalize(accum); + } +}; + +#if !defined(EIGEN_HIPCC) +template <typename Self, typename Op> +struct InnerMostDimReducer<Self, Op, true, true> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Self::CoeffReturnType + reduce(const Self& self, typename Self::Index firstIndex, + typename Self::Index numValuesToReduce, Op& reducer) { + const typename Self::Index packetSize = + internal::unpacket_traits<typename Self::PacketReturnType>::size; + typename Self::CoeffReturnType accum = reducer.initialize(); + if (numValuesToReduce > packetSize * kLeafSize) { + // Make sure the split point is aligned on a packet boundary. + const typename Self::Index split = + packetSize * + divup(firstIndex + divup(numValuesToReduce, typename Self::Index(2)), + packetSize); + const typename Self::Index num_left = + numext::mini(split - firstIndex, numValuesToReduce); + reducer.reduce(reduce(self, firstIndex, num_left, reducer), &accum); + if (num_left < numValuesToReduce) { + reducer.reduce( + reduce(self, split, numValuesToReduce - num_left, reducer), &accum); + } + return reducer.finalize(accum); + } else { + const typename Self::Index VectorizedSize = + (numValuesToReduce / packetSize) * packetSize; + typename Self::PacketReturnType paccum = + reducer.template initializePacket<typename Self::PacketReturnType>(); + for (typename Self::Index j = 0; j < VectorizedSize; j += packetSize) { + reducer.reducePacket( + self.m_impl.template packet<Unaligned>(firstIndex + j), &paccum); + } + for (typename Self::Index j = VectorizedSize; j < numValuesToReduce; + ++j) { + reducer.reduce(self.m_impl.coeff(firstIndex + j), &accum); + } + return reducer.finalizeBoth(accum, paccum); + } + } +}; +#endif + +template <int DimIndex, typename Self, typename Op, bool vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess)> struct InnerMostDimPreserver { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const Self&, typename Self::Index, Op&, typename Self::PacketReturnType*) { eigen_assert(false && "should never be called"); @@ -183,7 +270,7 @@ template <int DimIndex, typename Self, typename Op> struct InnerMostDimPreserver<DimIndex, Self, Op, true> { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const Self& self, typename Self::Index firstIndex, Op& reducer, typename Self::PacketReturnType* accum) { - EIGEN_STATIC_ASSERT(DimIndex > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((DimIndex > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); for (typename Self::Index j = 0; j < self.m_reducedDims[DimIndex]; ++j) { const typename Self::Index input = firstIndex + j * self.m_reducedStrides[DimIndex]; InnerMostDimPreserver<DimIndex-1, Self, Op>::reduce(self, input, reducer, accum); @@ -208,7 +295,7 @@ }; // Default full reducer -template <typename Self, typename Op, typename Device, bool Vectorizable = (Self::InputPacketAccess & Op::PacketAccess)> +template <typename Self, typename Op, typename Device, bool Vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess)> struct FullReducer { static const bool HasOptimizedImplementation = false; @@ -222,7 +309,7 @@ #ifdef EIGEN_USE_THREADS // Multithreaded full reducers template <typename Self, typename Op, - bool Vectorizable = (Self::InputPacketAccess & Op::PacketAccess)> + bool Vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess)> struct FullReducerShard { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Self& self, typename Self::Index firstIndex, typename Self::Index numValuesToReduce, Op& reducer, @@ -235,8 +322,8 @@ // Multithreaded full reducer template <typename Self, typename Op, bool Vectorizable> struct FullReducer<Self, Op, ThreadPoolDevice, Vectorizable> { - static const bool HasOptimizedImplementation = !Op::IsStateful; - static const int PacketSize = + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful; + static const Index PacketSize = unpacket_traits<typename Self::PacketReturnType>::size; // launch one reducer per thread and accumulate the result. @@ -248,16 +335,12 @@ *output = reducer.finalize(reducer.initialize()); return; } -#ifdef EIGEN_USE_COST_MODEL const TensorOpCost cost = self.m_impl.costPerCoeff(Vectorizable) + TensorOpCost(0, 0, internal::functor_traits<Op>::Cost, Vectorizable, PacketSize); const int num_threads = TensorCostModel<ThreadPoolDevice>::numThreads( num_coeffs, cost, device.numThreads()); -#else - const int num_threads = device.numThreads(); -#endif if (num_threads == 1) { *output = InnerMostDimReducer<Self, Op, Vectorizable>::reduce(self, 0, num_coeffs, reducer); @@ -268,7 +351,7 @@ const Index numblocks = blocksize > 0 ? num_coeffs / blocksize : 0; eigen_assert(num_coeffs >= numblocks * blocksize); - Barrier barrier(numblocks); + Barrier barrier(internal::convert_index<unsigned int>(numblocks)); MaxSizeVector<typename Self::CoeffReturnType> shards(numblocks, reducer.initialize()); for (Index i = 0; i < numblocks; ++i) { device.enqueue_with_barrier(&barrier, &FullReducerShard<Self, Op, Vectorizable>::run, @@ -318,22 +401,97 @@ }; -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) -template <int B, int N, typename S, typename R, typename I> -__global__ void FullReductionKernel(R, const S, I, typename S::CoeffReturnType*); +#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC)) +template <int B, int N, typename S, typename R, typename I_> +__global__ void FullReductionKernel(R, const S, I_, typename S::CoeffReturnType*, unsigned int*); -template <int NPT, typename S, typename R, typename I> -__global__ void InnerReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); -template <int NPT, typename S, typename R, typename I> -__global__ void OuterReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); +#if defined(EIGEN_HAS_GPU_FP16) +template <typename S, typename R, typename I_> +__global__ void ReductionInitFullReduxKernelHalfFloat(R, const S, I_, half2*); +template <int B, int N, typename S, typename R, typename I_> +__global__ void FullReductionKernelHalfFloat(R, const S, I_, half*, half2*); +template <int NPT, typename S, typename R, typename I_> +__global__ void InnerReductionKernelHalfFloat(R, const S, I_, I_, half*); + #endif +template <int NPT, typename S, typename R, typename I_> +__global__ void InnerReductionKernel(R, const S, I_, I_, typename S::CoeffReturnType*); + +template <int NPT, typename S, typename R, typename I_> +__global__ void OuterReductionKernel(R, const S, I_, I_, typename S::CoeffReturnType*); +#endif + +template <typename Self, typename Op, + bool Vectorizable = + (Self::InputPacketAccess & Self::ReducerTraits::PacketAccess)> +class BlockReducer { + public: + typedef typename Self::Index Index; + typedef typename Self::Scalar Scalar; + typedef typename Self::CoeffReturnType CoeffReturnType; + typedef typename Self::PacketReturnType PacketReturnType; + explicit BlockReducer(const Op& reducer) : op_(reducer) { + accum_ = op_.initialize(); + } + void Reduce(Index index, Index num_values_to_reduce, Scalar* data) { + for (Index i = 0; i < num_values_to_reduce; ++i) { + op_.reduce(data[index + i], &accum_); + } + } + CoeffReturnType Finalize() { return op_.finalize(accum_); } + PacketReturnType FinalizePacket() { + // TODO(andydavis) This function should not be called for Scalar + // reductions: clean this up or add an assert here. + return PacketReturnType(); + } + + private: + CoeffReturnType accum_; + Op op_; +}; + +template <typename Self, typename Op> +class BlockReducer<Self, Op, true> { + public: + typedef typename Self::Index Index; + typedef typename Self::Scalar Scalar; + typedef typename Self::CoeffReturnType CoeffReturnType; + typedef typename Self::PacketReturnType PacketReturnType; + static const Index PacketSize = + internal::unpacket_traits<PacketReturnType>::size; + + explicit BlockReducer(const Op& reducer) : op_(reducer) { + vaccum_ = op_.template initializePacket<PacketReturnType>(); + accum_ = op_.initialize(); + } + void Reduce(Index index, Index num_values_to_reduce, Scalar* data) { + const Index vectorized_size = + (num_values_to_reduce / PacketSize) * PacketSize; + for (Index i = 0; i < vectorized_size; i += PacketSize) { + op_.reducePacket( + internal::ploadt<PacketReturnType, Unaligned>(&data[index + i]), + &vaccum_); + } + for (Index i = vectorized_size; i < num_values_to_reduce; ++i) { + op_.reduce(data[index + i], &accum_); + } + } + CoeffReturnType Finalize() { return op_.finalizeBoth(accum_, vaccum_); } + PacketReturnType FinalizePacket() { return op_.finalizePacket(vaccum_); } + + private: + PacketReturnType vaccum_; + CoeffReturnType accum_; + Op op_; +}; + } // end namespace internal -template <typename Op, typename Dims, typename XprType> -class TensorReductionOp : public TensorBase<TensorReductionOp<Op, Dims, XprType>, ReadOnlyAccessors> { +template <typename Op, typename Dims, typename XprType, template <class> class MakePointer_> +class TensorReductionOp : public TensorBase<TensorReductionOp<Op, Dims, XprType, MakePointer_>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorReductionOp>::Scalar Scalar; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; @@ -364,39 +522,53 @@ // Eval as rvalue -template<typename Op, typename Dims, typename ArgType, typename Device> -struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> +template<typename Op, typename Dims, typename ArgType, template <class> class MakePointer_, typename Device> +struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> { - typedef TensorReductionOp<Op, Dims, ArgType> XprType; + typedef internal::reducer_traits<Op, Device> ReducerTraits; + typedef TensorReductionOp<Op, Dims, ArgType, MakePointer_> XprType; typedef typename XprType::Index Index; + typedef ArgType ChildType; typedef typename TensorEvaluator<ArgType, Device>::Dimensions InputDimensions; static const int NumInputDims = internal::array_size<InputDimensions>::value; static const int NumReducedDims = internal::array_size<Dims>::value; static const int NumOutputDims = NumInputDims - NumReducedDims; typedef typename internal::conditional<NumOutputDims==0, Sizes<>, DSizes<Index, NumOutputDims> >::type Dimensions; typedef typename XprType::Scalar Scalar; - typedef TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> Self; + typedef TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> Self; static const bool InputPacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const Index PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = false, - PacketAccess = Self::InputPacketAccess && Op::PacketAccess, + PacketAccess = Self::InputPacketAccess && ReducerTraits::PacketAccess, + BlockAccess = false, + PreferBlockAccess = true, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumOutputDims, Layout> + OutputTensorBlock; + typedef internal::TensorBlock<ScalarNoConst, Index, NumInputDims, Layout> + InputTensorBlock; + static const bool ReducingInnerMostDims = internal::are_inner_most_dims<Dims, NumInputDims, Layout>::value; static const bool PreservingInnerMostDims = internal::preserve_inner_most_dims<Dims, NumInputDims, Layout>::value; static const bool RunningFullReduction = (NumOutputDims==0); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_impl(op.expression(), device), m_reducer(op.reducer()), m_result(NULL), m_device(device) +#if defined(EIGEN_USE_SYCL) + , m_xpr_dims(op.dims()) +#endif { - EIGEN_STATIC_ASSERT(NumInputDims >= NumReducedDims, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumInputDims >= NumReducedDims), YOU_MADE_A_PROGRAMMING_MISTAKE); EIGEN_STATIC_ASSERT((!ReducingInnerMostDims | !PreservingInnerMostDims | (NumReducedDims == NumInputDims)), YOU_MADE_A_PROGRAMMING_MISTAKE); @@ -416,15 +588,17 @@ // Precompute output strides. if (NumOutputDims > 0) { if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - m_outputStrides[0] = 1; - for (int i = 1; i < NumOutputDims; ++i) { - m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; - } + m_outputStrides[0] = 1; + for (int i = 1; i < NumOutputDims; ++i) { + m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]); + } } else { - m_outputStrides.back() = 1; - for (int i = NumOutputDims - 2; i >= 0; --i) { - m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; - } + m_outputStrides[NumOutputDims - 1] = 1; + for (int i = NumOutputDims - 2; i >= 0; --i) { + m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]); + } } } @@ -432,27 +606,28 @@ if (NumInputDims > 0) { array<Index, NumInputDims> input_strides; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - input_strides[0] = 1; - for (int i = 1; i < NumInputDims; ++i) { - input_strides[i] = input_strides[i-1] * input_dims[i-1]; - } + input_strides[0] = 1; + for (int i = 1; i < NumInputDims; ++i) { + input_strides[i] = input_strides[i-1] * input_dims[i-1]; + } } else { - input_strides.back() = 1; - for (int i = NumInputDims - 2; i >= 0; --i) { - input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; - } + input_strides.back() = 1; + for (int i = NumInputDims - 2; i >= 0; --i) { + input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; + } } int outputIndex = 0; int reduceIndex = 0; for (int i = 0; i < NumInputDims; ++i) { - if (m_reduced[i]) { - m_reducedStrides[reduceIndex] = input_strides[i]; - ++reduceIndex; - } else { - m_preservedStrides[outputIndex] = input_strides[i]; - ++outputIndex; - } + if (m_reduced[i]) { + m_reducedStrides[reduceIndex] = input_strides[i]; + ++reduceIndex; + } else { + m_preservedStrides[outputIndex] = input_strides[i]; + m_output_to_input_dim_map[outputIndex] = i; + ++outputIndex; + } } } @@ -460,40 +635,56 @@ if (NumOutputDims == 0) { m_preservedStrides[0] = internal::array_prod(input_dims); } + + m_numValuesToReduce = + NumOutputDims == 0 + ? internal::array_prod(input_dims) + : (static_cast<int>(Layout) == static_cast<int>(ColMajor)) + ? m_preservedStrides[0] + : m_preservedStrides[NumOutputDims - 1]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } - static bool size_large_enough(Index total_size) { -#ifndef EIGEN_USE_COST_MODEL - return total_size > 1024 * 1024; -#else - return true || total_size; -#endif - } - - EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool evalSubExprsIfNeeded(CoeffReturnType* data) { + EIGEN_STRONG_INLINE + #if !defined(EIGEN_HIPCC) + // Marking this as EIGEN_DEVICE_FUNC for HIPCC requires also doing the same for all the functions + // being called within here, which then leads to proliferation of EIGEN_DEVICE_FUNC markings, one + // of which will eventually result in an NVCC error + EIGEN_DEVICE_FUNC + #endif + bool evalSubExprsIfNeeded(typename MakePointer_<CoeffReturnType>::Type data) { m_impl.evalSubExprsIfNeeded(NULL); // Use the FullReducer if possible. - if (RunningFullReduction && internal::FullReducer<Self, Op, Device>::HasOptimizedImplementation && + if ((RunningFullReduction && RunningOnSycl) ||(RunningFullReduction && + internal::FullReducer<Self, Op, Device>::HasOptimizedImplementation && ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || - (!RunningOnGPU && size_large_enough(internal::array_prod(m_impl.dimensions()))))) { - + !RunningOnGPU))) { bool need_assign = false; if (!data) { - m_result = static_cast<CoeffReturnType*>(m_device.allocate(sizeof(CoeffReturnType))); + m_result = static_cast<CoeffReturnType*>(m_device.allocate_temp(sizeof(CoeffReturnType))); data = m_result; need_assign = true; } - Op reducer(m_reducer); internal::FullReducer<Self, Op, Device>::run(*this, reducer, m_device, data); return need_assign; } + else if(RunningOnSycl){ + const Index num_values_to_reduce = internal::array_prod(m_reducedDims); + const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); + if (!data) { + data = static_cast<CoeffReturnType*>(m_device.allocate_temp(sizeof(CoeffReturnType) * num_coeffs_to_preserve)); + m_result = data; + } + Op reducer(m_reducer); + internal::InnerReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve); + return (m_result != NULL); + } // Attempt to use an optimized reduction. - else if (RunningOnGPU && data && (m_device.majorDeviceVersion() >= 3)) { + else if (RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) { bool reducing_inner_dims = true; for (int i = 0; i < NumReducedDims; ++i) { if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { @@ -506,8 +697,25 @@ (reducing_inner_dims || ReducingInnerMostDims)) { const Index num_values_to_reduce = internal::array_prod(m_reducedDims); const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); + if (!data) { + if (num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve && num_values_to_reduce > 128) { + data = static_cast<CoeffReturnType*>(m_device.allocate_temp(sizeof(CoeffReturnType) * num_coeffs_to_preserve)); + m_result = data; + } + else { + return true; + } + } Op reducer(m_reducer); - return internal::InnerReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve); + if (internal::InnerReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve)) { + if (m_result) { + m_device.deallocate_temp(m_result); + m_result = NULL; + } + return true; + } else { + return (m_result != NULL); + } } bool preserving_inner_dims = true; @@ -522,8 +730,25 @@ preserving_inner_dims) { const Index num_values_to_reduce = internal::array_prod(m_reducedDims); const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); + if (!data) { + if (num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve && num_values_to_reduce > 32) { + data = static_cast<CoeffReturnType*>(m_device.allocate_temp(sizeof(CoeffReturnType) * num_coeffs_to_preserve)); + m_result = data; + } + else { + return true; + } + } Op reducer(m_reducer); - return internal::OuterReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve); + if (internal::OuterReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve)) { + if (m_result) { + m_device.deallocate_temp(m_result); + m_result = NULL; + } + return true; + } else { + return (m_result != NULL); + } } } return true; @@ -532,19 +757,20 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); if (m_result) { - m_device.deallocate(m_result); + m_device.deallocate_temp(m_result); + m_result = NULL; } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - if (RunningFullReduction && m_result) { - return *m_result; + if ((RunningOnSycl || RunningFullReduction || RunningOnGPU) && m_result) { + return *(m_result + index); } Op reducer(m_reducer); if (ReducingInnerMostDims || RunningFullReduction) { const Index num_values_to_reduce = - (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? m_preservedStrides[0] : m_preservedStrides[NumPreservedStrides - 1]; + (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? m_preservedStrides[0] : m_preservedStrides[NumPreservedStrides - 1]; return internal::InnerMostDimReducer<Self, Op>::reduce(*this, firstInput(index), num_values_to_reduce, reducer); } else { @@ -558,13 +784,17 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) - eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(index + PacketSize - 1 < Index(internal::array_prod(dimensions()))); + + if (RunningOnGPU && m_result) { + return internal::pload<PacketReturnType>(m_result + index); + } EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; if (ReducingInnerMostDims) { const Index num_values_to_reduce = - (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? m_preservedStrides[0] : m_preservedStrides[NumPreservedStrides - 1]; + (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? m_preservedStrides[0] : m_preservedStrides[NumPreservedStrides - 1]; const Index firstIndex = firstInput(index); for (Index i = 0; i < PacketSize; ++i) { Op reducer(m_reducer); @@ -606,21 +836,308 @@ } } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.lastLevelCacheSize() / sizeof(Scalar)); + resources->push_back(internal::TensorOpResourceRequirements( + internal::kSkewedInnerDims, block_total_size_max)); + m_impl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void block( + OutputTensorBlock* output_block) const { + // Special case full reductions to avoid input block copy below. + if (NumInputDims == NumReducedDims) { + eigen_assert(output_block->first_coeff_index() == 0); + eigen_assert(output_block->block_sizes().TotalSize() == 1); + Op reducer(m_reducer); + output_block->data()[0] = internal::InnerMostDimReducer<Self, Op>::reduce( + *this, 0, m_numValuesToReduce, reducer); + return; + } + + // Calculate input tensor 'slice' required to reduce output block coeffs. + DSizes<Index, NumInputDims> input_slice_sizes(m_impl.dimensions()); + for (int i = 0; i < NumOutputDims; ++i) { + // Clip preserved input dimensions by output block size. + input_slice_sizes[m_output_to_input_dim_map[i]] = + output_block->block_sizes()[i]; + } + + // Shard input tensor slice into blocks (because it could be large if we + // need to reduce along several dimensions to calculate required output + // coefficients). + const Index max_coeff_count = + numext::mini<Index>(((m_device.firstLevelCacheSize()) / sizeof(Scalar)), + input_slice_sizes.TotalSize()); + + // Calculate max output shard size needed to keep working set of reducers + // in L1, while leaving enough space for reducer overhead and 'PacketSize' + // reductions. + DSizes<Index, NumInputDims> target_input_block_sizes; + CalculateTargetInputBlockShape(max_coeff_count, input_slice_sizes, + &target_input_block_sizes); + // Calculate indices for first preserved dimension. + const Index first_preserved_dim_output_index = + static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 0 + : NumOutputDims - 1; + const Index first_preserved_dim_input_index = + m_output_to_input_dim_map[first_preserved_dim_output_index]; + const bool inner_most_dim_preserved = + PreservingInnerMostDims || + (first_preserved_dim_input_index == + (static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? 0 + : NumInputDims - 1)); + + // Calculate output block inner/outer dimension sizes. + const Index output_block_inner_dim_size = + output_block->block_sizes()[first_preserved_dim_output_index]; + const Index output_block_outer_dim_size = + output_block->block_sizes().TotalSize() / output_block_inner_dim_size; + // Calculate shard size for first preserved dimension. + const Index output_shard_size = + target_input_block_sizes[first_preserved_dim_input_index]; + const Index num_output_shards = + (output_block_inner_dim_size + output_shard_size - 1) / + output_shard_size; + + // Initialize 'tensor_slice_offsets' from input coords of output index. + DSizes<Index, NumInputDims> tensor_slice_offsets; + GetInputCoordsForOutputIndex(output_block->first_coeff_index(), + &tensor_slice_offsets); + + // Store tensor slice offset in first preserved dimension to be used + // to update tensor slice extents in loop below. + const Index first_preserved_dim_offset_start = + tensor_slice_offsets[first_preserved_dim_input_index]; + + array<BlockIteratorState, NumOutputDims> block_iter_state; + + // Initialize state used to iterate through output coefficients + // and update 'tensor_slice_offsets' in outer preserved dims. + for (int i = 0; i < NumOutputDims - 1; ++i) { + const int dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i + 1 + : NumOutputDims - i - 2; + block_iter_state[i].input_dim = m_output_to_input_dim_map[dim]; + block_iter_state[i].output_size = output_block->block_sizes()[dim]; + block_iter_state[i].output_count = 0; + } + + // Allocate input block memory. + ScalarNoConst* input_block_data = static_cast<ScalarNoConst*>( + m_device.allocate(max_coeff_count * sizeof(Scalar))); + // Allocate reducer memory. + const bool packet_reductions_enabled = + (Self::InputPacketAccess & Self::ReducerTraits::PacketAccess); + const Index num_reducers = + (inner_most_dim_preserved && packet_reductions_enabled) + ? (output_shard_size / PacketSize + output_shard_size % PacketSize + + PacketSize) + : output_shard_size; + typedef internal::BlockReducer<Self, Op> BlockReducer; + BlockReducer* reducers = static_cast<BlockReducer*>( + m_device.allocate(num_reducers * sizeof(BlockReducer))); + + InputDimensions input_tensor_dims(m_impl.dimensions()); + for (Index output_outer_index = 0; + output_outer_index < output_block_outer_dim_size; + ++output_outer_index) { + for (Index output_shard_index = 0; output_shard_index < num_output_shards; + ++output_shard_index) { + // Initialize 'tensor_slice_extents' for this output shard. + DSizes<Index, NumInputDims> tensor_slice_extents(input_slice_sizes); + for (int i = 0; i < NumInputDims; ++i) { + if (i == first_preserved_dim_input_index) { + // Clip first preserved dim size to output shard size. + tensor_slice_extents[i] = numext::mini( + output_shard_size, + input_slice_sizes[i] - (tensor_slice_offsets[i] - + first_preserved_dim_offset_start)); + + } else if (!m_reduced[i]) { + // Clip outer preserved dims to size 1, so that we reduce a + // contiguous set of output coefficients. + tensor_slice_extents[i] = 1; + } + } + + // Initialize output coefficient reducers. + for (int i = 0; i < num_reducers; ++i) { + new (&reducers[i]) BlockReducer(m_reducer); + } + + typedef internal::TensorSliceBlockMapper<ScalarNoConst, Index, + NumInputDims, Layout> + TensorSliceBlockMapper; + + // TODO(andydavis) Consider removing 'input_block_stride_order' if we + // find that scattered reads are not worth supporting in + // TensorSliceBlockMapper. + TensorSliceBlockMapper block_mapper( + typename TensorSliceBlockMapper::Dimensions(input_tensor_dims), + tensor_slice_offsets, tensor_slice_extents, + target_input_block_sizes, DimensionList<Index, NumInputDims>()); + + const Index num_outputs_to_update = + tensor_slice_extents[first_preserved_dim_input_index]; + const Index preserved_dim_vector_reducer_count = + (inner_most_dim_preserved && packet_reductions_enabled) + ? num_outputs_to_update / PacketSize + : 0; + const Index preserved_dim_vector_coeff_count = + inner_most_dim_preserved + ? preserved_dim_vector_reducer_count * PacketSize + : 0; + const Index preserved_dim_reducer_limit = + (inner_most_dim_preserved && packet_reductions_enabled) + ? (preserved_dim_vector_reducer_count + + num_outputs_to_update % PacketSize) + : num_outputs_to_update; + + const Index total_block_count = block_mapper.total_block_count(); + for (Index b = 0; b < total_block_count; ++b) { + InputTensorBlock input_block = + block_mapper.GetBlockForIndex(b, input_block_data); + // Read. + m_impl.block(&input_block); + + Index num_values_to_reduce = 1; + for (Index i = 0; i < NumInputDims; ++i) { + if (m_reduced[i]) { + num_values_to_reduce *= input_block.block_sizes()[i]; + } + } + // Reduce. + if (inner_most_dim_preserved) { + const Index input_outer_dim_size = + input_block.block_sizes().TotalSize() / num_outputs_to_update; + for (Index input_outer_dim_index = 0; + input_outer_dim_index < input_outer_dim_size; + ++input_outer_dim_index) { + const Index input_outer_dim_base = + input_outer_dim_index * num_outputs_to_update; + for (Index i = 0; i < preserved_dim_vector_reducer_count; ++i) { + reducers[i].Reduce(input_outer_dim_base + i * PacketSize, + PacketSize, input_block.data()); + } + const Index scalar_reducer_base = + input_outer_dim_base + preserved_dim_vector_coeff_count; + for (Index i = preserved_dim_vector_reducer_count; + i < preserved_dim_reducer_limit; ++i) { + reducers[i].Reduce(scalar_reducer_base + i - + preserved_dim_vector_reducer_count, + 1, input_block.data()); + } + } + } else { + for (Index i = 0; i < num_outputs_to_update; ++i) { + reducers[i].Reduce(i * num_values_to_reduce, num_values_to_reduce, + input_block.data()); + } + } + } + + // Finalize all reducers for this output shard. + const Index output_base_index = + output_outer_index * output_block_inner_dim_size + + output_shard_index * output_shard_size; + if (inner_most_dim_preserved) { + EIGEN_ALIGN_MAX + typename internal::remove_const<CoeffReturnType>::type + values[PacketSize]; + for (Index i = 0; i < preserved_dim_vector_reducer_count; ++i) { + const Index reducer_base = output_base_index + i * PacketSize; + internal::pstore<CoeffReturnType, PacketReturnType>( + values, reducers[i].FinalizePacket()); + for (Index j = 0; j < PacketSize; ++j) { + output_block->data()[reducer_base + j] = values[j]; + } + } + const Index scalar_reducer_base = + output_base_index + preserved_dim_vector_coeff_count; + + for (Index i = preserved_dim_vector_reducer_count; + i < preserved_dim_reducer_limit; ++i) { + output_block->data()[scalar_reducer_base + i - + preserved_dim_vector_reducer_count] = + reducers[i].Finalize(); + } + } else { + for (int i = 0; i < num_outputs_to_update; ++i) { + output_block->data()[output_base_index + i] = + reducers[i].Finalize(); + } + } + + // Update 'tensor_slice_offsets' by num outputs for this output shard. + tensor_slice_offsets[first_preserved_dim_input_index] += + num_outputs_to_update; + } + // Update slice offset for inner preserved dim. + tensor_slice_offsets[first_preserved_dim_input_index] -= + output_block_inner_dim_size; + // Update slice offsets for remaining output dims. + for (int i = 0; i < NumOutputDims - 1; ++i) { + BlockIteratorState& b = block_iter_state[i]; + if (++b.output_count < b.output_size) { + ++tensor_slice_offsets[b.input_dim]; + break; + } + b.output_count = 0; + tensor_slice_offsets[b.input_dim] -= b.output_size - 1; + } + } + + // Free memory. + m_device.deallocate(input_block_data); + m_device.deallocate(reducers); + } + + EIGEN_DEVICE_FUNC typename MakePointer_<CoeffReturnType>::Type data() const { return m_result; } + +#if defined(EIGEN_USE_SYCL) + const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + const Device& device() const { return m_device; } + const Dims& xprDims() const { return m_xpr_dims; } +#endif private: template <int, typename, typename> friend struct internal::GenericDimReducer; - template <typename, typename, bool> friend struct internal::InnerMostDimReducer; + template <typename, typename, bool, bool> friend struct internal::InnerMostDimReducer; template <int, typename, typename, bool> friend struct internal::InnerMostDimPreserver; template <typename S, typename O, typename D, bool V> friend struct internal::FullReducer; #ifdef EIGEN_USE_THREADS template <typename S, typename O, bool V> friend struct internal::FullReducerShard; #endif -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) - template <int B, int N, typename S, typename R, typename I> friend void internal::FullReductionKernel(R, const S, I, typename S::CoeffReturnType*); - template <int NPT, typename S, typename R, typename I> friend void internal::InnerReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); - template <int NPT, typename S, typename R, typename I> friend void internal::OuterReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); +#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC)) + template <int B, int N, typename S, typename R, typename I_> KERNEL_FRIEND void internal::FullReductionKernel(R, const S, I_, typename S::CoeffReturnType*, unsigned int*); +#if defined(EIGEN_HAS_GPU_FP16) + template <typename S, typename R, typename I_> KERNEL_FRIEND void internal::ReductionInitFullReduxKernelHalfFloat(R, const S, I_, half2*); + template <int B, int N, typename S, typename R, typename I_> KERNEL_FRIEND void internal::FullReductionKernelHalfFloat(R, const S, I_, half*, half2*); + template <int NPT, typename S, typename R, typename I_> KERNEL_FRIEND void internal::InnerReductionKernelHalfFloat(R, const S, I_, I_, half*); #endif + template <int NPT, typename S, typename R, typename I_> KERNEL_FRIEND void internal::InnerReductionKernel(R, const S, I_, I_, typename S::CoeffReturnType*); + + template <int NPT, typename S, typename R, typename I_> KERNEL_FRIEND void internal::OuterReductionKernel(R, const S, I_, I_, typename S::CoeffReturnType*); +#endif + +#if defined(EIGEN_USE_SYCL) + template < typename HostExpr_, typename FunctorExpr_, typename Tuple_of_Acc_, typename Dims_, typename Op_, typename Index_> friend class TensorSycl::internal::ReductionFunctor; + template<typename CoeffReturnType_ ,typename OutAccessor_, typename HostExpr_, typename FunctorExpr_, typename Op_, typename Dims_, typename Index_, typename TupleType_> friend class TensorSycl::internal::FullReductionKernelFunctor; +#endif + + + template <typename S, typename O, typename D> friend struct internal::InnerReducer; + + struct BlockIteratorState { + Index input_dim; + Index output_size; + Index output_count; + }; // Returns the Index in the input tensor of the first value that needs to be // used to compute the reduction at output index "index". @@ -664,16 +1181,88 @@ return startInput; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void GetInputCoordsForOutputIndex( + Index index, + DSizes<Index, NumInputDims>* coords) const { + for (int i = 0; i < NumInputDims; ++i) { + (*coords)[i] = 0; + } + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumOutputDims - 1; i > 0; --i) { + const Index idx = index / m_fastOutputStrides[i]; + (*coords)[m_output_to_input_dim_map[i]] = idx; + index -= idx * m_outputStrides[i]; + } + (*coords)[m_output_to_input_dim_map[0]] = index; + } else { + for (int i = 0; i < NumOutputDims - 1; ++i) { + const Index idx = index / m_fastOutputStrides[i]; + (*coords)[m_output_to_input_dim_map[i]] = idx; + index -= idx * m_outputStrides[i]; + } + (*coords)[m_output_to_input_dim_map[NumOutputDims-1]] = index; + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void CalculateTargetInputBlockShape( + const Index max_coeff_count, + const DSizes<Index, NumInputDims>& input_slice_sizes, + DSizes<Index, NumInputDims>* target_input_block_sizes) const { + typedef internal::BlockReducer<Self, Op> BlockReducer; + // TODO(andydavis) Compute reducer overhead correctly for the case where + // we are preserving the inner most dimension, and a single reducer + // reduces a packet's worth of output coefficients. + const Index reducer_overhead = sizeof(BlockReducer) / sizeof(Scalar); + + Index coeff_to_allocate = max_coeff_count; + bool first_preserved_dim_allocated = false; + bool first_reduced_dim_allocated = false; + for (int i = 0; i < NumInputDims; ++i) { + const int dim = static_cast<int>(Layout) == static_cast<int>(ColMajor) + ? i + : NumInputDims - i - 1; + (*target_input_block_sizes)[dim] = 1; + if (m_reduced[dim]) { + // TODO(andydavis) Consider allocating to multiple reduced dimensions. + // Watch out for cases where reduced dimensions are not contiguous, + // which induces scattered reads. + if (!first_reduced_dim_allocated) { + (*target_input_block_sizes)[dim] = + numext::mini(input_slice_sizes[dim], coeff_to_allocate); + coeff_to_allocate /= (*target_input_block_sizes)[dim]; + first_reduced_dim_allocated = true; + } + } else if (!first_preserved_dim_allocated) { + // TODO(andydavis) Include output block size in this L1 working set + // calculation. + const Index alloc_size = numext::maxi( + static_cast<Index>(1), coeff_to_allocate / reducer_overhead); + (*target_input_block_sizes)[dim] = + numext::mini(input_slice_sizes[dim], alloc_size); + coeff_to_allocate = numext::maxi( + static_cast<Index>(1), + coeff_to_allocate / + ((*target_input_block_sizes)[dim] * reducer_overhead)); + first_preserved_dim_allocated = true; + } + } + } + // Bitmap indicating if an input dimension is reduced or not. array<bool, NumInputDims> m_reduced; // Dimensions of the output of the operation. Dimensions m_dimensions; // Precomputed strides for the output tensor. array<Index, NumOutputDims> m_outputStrides; + array<internal::TensorIntDivisor<Index>, NumOutputDims> m_fastOutputStrides; // Subset of strides of the input tensor for the non-reduced dimensions. // Indexed by output dimensions. static const int NumPreservedStrides = max_n_1<NumOutputDims>::size; array<Index, NumPreservedStrides> m_preservedStrides; + // Map from output to input dimension index. + array<Index, NumOutputDims> m_output_to_input_dim_map; + // How many values go into each reduction + Index m_numValuesToReduce; // Subset of strides of the input tensor for the reduced dimensions. // Indexed by reduced dimensions. @@ -689,14 +1278,23 @@ Op m_reducer; // For full reductions -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) +#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC)) static const bool RunningOnGPU = internal::is_same<Device, Eigen::GpuDevice>::value; + static const bool RunningOnSycl = false; +#elif defined(EIGEN_USE_SYCL) +static const bool RunningOnSycl = internal::is_same<typename internal::remove_all<Device>::type, Eigen::SyclDevice>::value; +static const bool RunningOnGPU = false; #else static const bool RunningOnGPU = false; + static const bool RunningOnSycl = false; #endif - CoeffReturnType* m_result; + typename MakePointer_<CoeffReturnType>::Type m_result; const Device& m_device; + +#if defined(EIGEN_USE_SYCL) + const Dims m_xpr_dims; +#endif }; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h index fd2587d..68780cd 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h
@@ -1,359 +1,6 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H -#define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H - -namespace Eigen { -namespace internal { - - -#if defined(EIGEN_USE_GPU) && defined(__CUDACC__) -// Full reducers for GPU, don't vectorize for now - -// Reducer function that enables multiple cuda thread to safely accumulate at the same -// output address. It basically reads the current value of the output variable, and -// attempts to update it with the new value. If in the meantime another cuda thread -// updated the content of the output address it will try again. -template <typename T, typename R> -__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) { -#if __CUDA_ARCH__ >= 300 - if (sizeof(T) == 4) - { - unsigned int oldval = *reinterpret_cast<unsigned int*>(output); - unsigned int newval = oldval; - reducer.reduce(accum, reinterpret_cast<T*>(&newval)); - if (newval == oldval) { - return; - } - unsigned int readback; - while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) { - oldval = readback; - newval = oldval; - reducer.reduce(accum, reinterpret_cast<T*>(&newval)); - if (newval == oldval) { - return; - } - } - } - else if (sizeof(T) == 8) { - unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output); - unsigned long long newval = oldval; - reducer.reduce(accum, reinterpret_cast<T*>(&newval)); - if (newval == oldval) { - return; - } - unsigned long long readback; - while ((readback = atomicCAS((unsigned long long*)output, oldval, newval)) != oldval) { - oldval = readback; - newval = oldval; - reducer.reduce(accum, reinterpret_cast<T*>(&newval)); - if (newval == oldval) { - return; - } - } - } - else { - assert(0 && "Wordsize not supported"); - } -#else - assert(0 && "Shouldn't be called on unsupported device"); -#endif -} - -template <typename T> -__device__ inline void atomicReduce(T* output, T accum, SumReducer<T>&) { -#if __CUDA_ARCH__ >= 300 - atomicAdd(output, accum); -#else - assert(0 && "Shouldn't be called on unsupported device"); -#endif -} - - -template <typename CoeffType, typename Index> -__global__ void ReductionInitKernel(const CoeffType val, Index num_preserved_coeffs, CoeffType* output) { - const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; - const Index num_threads = blockDim.x * gridDim.x; - for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { - output[i] = val; - } -} - -template <int BlockSize, int NumPerThread, typename Self, - typename Reducer, typename Index> -__global__ void FullReductionKernel(Reducer reducer, const Self input, Index num_coeffs, - typename Self::CoeffReturnType* output) { - const Index first_index = blockIdx.x * BlockSize * NumPerThread + threadIdx.x; - - // Initialize the output value if it wasn't initialized by the ReductionInitKernel - if (gridDim.x == 1 && first_index == 0) { - *output = reducer.initialize(); - } - - typename Self::CoeffReturnType accum = reducer.initialize(); - Index max_iter = numext::mini<Index>(num_coeffs - first_index, NumPerThread*BlockSize); - for (Index i = 0; i < max_iter; i+=BlockSize) { - const Index index = first_index + i; - eigen_assert(index < num_coeffs); - typename Self::CoeffReturnType val = input.m_impl.coeff(index); - reducer.reduce(val, &accum); - } - -#pragma unroll - for (int offset = warpSize/2; offset > 0; offset /= 2) { - reducer.reduce(__shfl_down(accum, offset), &accum); - } - - if ((threadIdx.x & (warpSize - 1)) == 0) { - atomicReduce(output, accum, reducer); - } -} - - -template <typename Self, typename Op, bool Vectorizable> -struct FullReducer<Self, Op, GpuDevice, Vectorizable> { - // Unfortunately nvidia doesn't support well exotic types such as complex, - // so reduce the scope of the optimized version of the code to the simple case - // of floats. - static const bool HasOptimizedImplementation = !Op::IsStateful && - internal::is_same<typename Self::CoeffReturnType, float>::value; - - template <typename OutputType> - static EIGEN_DEVICE_FUNC void run(const Self&, Op&, const GpuDevice&, OutputType*) { - assert(false && "Should only be called on floats"); - } - - static void run(const Self& self, Op& reducer, const GpuDevice& device, float* output) { - typedef typename Self::Index Index; - - const Index num_coeffs = array_prod(self.m_impl.dimensions()); - // Don't crash when we're called with an input tensor of size 0. - if (num_coeffs == 0) { - return; - } - - const int block_size = 256; - const int num_per_thread = 128; - const int num_blocks = divup<int>(num_coeffs, block_size * num_per_thread); - - if (num_blocks > 1) { - // We initialize the outputs outside the reduction kernel when we can't be sure that there - // won't be a race conditions between multiple thread blocks. - LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), - 1, 32, 0, device, reducer.initialize(), 1, output); - } - - LAUNCH_CUDA_KERNEL((FullReductionKernel<block_size, num_per_thread, Self, Op, Index>), - num_blocks, block_size, 0, device, reducer, self, num_coeffs, output); - } -}; - - -template <int NumPerThread, typename Self, - typename Reducer, typename Index> -__global__ void InnerReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, - typename Self::CoeffReturnType* output) { - eigen_assert(blockDim.y == 1); - eigen_assert(blockDim.z == 1); - eigen_assert(gridDim.y == 1); - eigen_assert(gridDim.z == 1); - - const int unroll_times = 16; - eigen_assert(NumPerThread % unroll_times == 0); - - const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread); - const Index num_input_blocks = input_col_blocks * num_preserved_coeffs; - - const Index num_threads = blockDim.x * gridDim.x; - const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; - - // Initialize the output values if they weren't initialized by the ReductionInitKernel - if (gridDim.x == 1) { - for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { - output[i] = reducer.initialize(); - } - } - - for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) { - const Index row = i / input_col_blocks; - - if (row < num_preserved_coeffs) { - const Index col_block = i % input_col_blocks; - const Index col_begin = col_block * blockDim.x * NumPerThread + threadIdx.x; - - float reduced_val = reducer.initialize(); - - for (Index j = 0; j < NumPerThread; j += unroll_times) { - const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1); - if (last_col >= num_coeffs_to_reduce) { - for (Index col = col_begin + blockDim.x * j; col < num_coeffs_to_reduce; col +=blockDim.x) { - const float val = input.m_impl.coeff(row * num_coeffs_to_reduce + col); - reducer.reduce(val, &reduced_val); - } - break; - } else { - // Faster version of the loop with no branches after unrolling. -#pragma unroll - for (int k = 0; k < unroll_times; ++k) { - const Index col = col_begin + blockDim.x * (j + k); - reducer.reduce(input.m_impl.coeff(row * num_coeffs_to_reduce + col), &reduced_val); - } - } - } - -#pragma unroll - for (int offset = warpSize/2; offset > 0; offset /= 2) { - reducer.reduce(__shfl_down(reduced_val, offset), &reduced_val); - } - - if ((threadIdx.x & (warpSize - 1)) == 0) { - atomicReduce(&(output[row]), reduced_val, reducer); - } - } - - __syncthreads(); - } -} - -template <typename Self, typename Op> -struct InnerReducer<Self, Op, GpuDevice> { - // Unfortunately nvidia doesn't support well exotic types such as complex, - // so reduce the scope of the optimized version of the code to the simple case - // of floats. - static const bool HasOptimizedImplementation = !Op::IsStateful && - internal::is_same<typename Self::CoeffReturnType, float>::value; - - template <typename Device, typename OutputType> - static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) { - assert(false && "Should only be called to reduce floats on a gpu device"); - return true; - } - - static bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { - typedef typename Self::Index Index; - - // It's faster to use the usual code. - if (num_coeffs_to_reduce <= 32) { - return true; - } - - const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; - const int block_size = 256; - const int num_per_thread = 128; - const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); - const int max_blocks = device.getNumCudaMultiProcessors() * - device.maxCudaThreadsPerMultiProcessor() / block_size; - const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); - - if (num_blocks > 1) { - // We initialize the outputs outside the reduction kernel when we can't be sure that there - // won't be a race conditions between multiple thread blocks. - const int dyn_blocks = divup<int>(num_preserved_vals, 1024); - const int max_blocks = device.getNumCudaMultiProcessors() * - device.maxCudaThreadsPerMultiProcessor() / 1024; - const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); - LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), - num_blocks, 1024, 0, device, reducer.initialize(), - num_preserved_vals, output); - } - - LAUNCH_CUDA_KERNEL((InnerReductionKernel<num_per_thread, Self, Op, Index>), - num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); - - return false; - } -}; - - -template <int NumPerThread, typename Self, - typename Reducer, typename Index> -__global__ void OuterReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, - typename Self::CoeffReturnType* output) { - const Index num_threads = blockDim.x * gridDim.x; - const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; - // Initialize the output values if they weren't initialized by the ReductionInitKernel - if (gridDim.x == 1) { - for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { - output[i] = reducer.initialize(); - } - } - - // Do the reduction. - const Index max_iter = num_preserved_coeffs * divup<Index>(num_coeffs_to_reduce, NumPerThread); - for (Index i = thread_id; i < max_iter; i += num_threads) { - const Index input_col = i % num_preserved_coeffs; - const Index input_row = (i / num_preserved_coeffs) * NumPerThread; - typename Self::CoeffReturnType reduced_val = reducer.initialize(); - const Index max_row = numext::mini(input_row + NumPerThread, num_coeffs_to_reduce); - for (Index j = input_row; j < max_row; j++) { - typename Self::CoeffReturnType val = input.m_impl.coeff(j * num_preserved_coeffs + input_col); - reducer.reduce(val, &reduced_val); - } - atomicReduce(&(output[input_col]), reduced_val, reducer); - } -} - - -template <typename Self, typename Op> -struct OuterReducer<Self, Op, GpuDevice> { - // Unfortunately nvidia doesn't support well exotic types such as complex, - // so reduce the scope of the optimized version of the code to the simple case - // of floats. - static const bool HasOptimizedImplementation = !Op::IsStateful && - internal::is_same<typename Self::CoeffReturnType, float>::value; - - template <typename Device, typename OutputType> - static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) { - assert(false && "Should only be called to reduce floats on a gpu device"); - return true; - } - - static bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { - typedef typename Self::Index Index; - - // It's faster to use the usual code. - if (num_coeffs_to_reduce <= 32) { - return true; - } - - const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; - const int block_size = 256; - const int num_per_thread = 16; - const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); - const int max_blocks = device.getNumCudaMultiProcessors() * - device.maxCudaThreadsPerMultiProcessor() / block_size; - const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); - - if (num_blocks > 1) { - // We initialize the outputs in the reduction kernel itself when we don't have to worry - // about race conditions between multiple thread blocks. - const int dyn_blocks = divup<int>(num_preserved_vals, 1024); - const int max_blocks = device.getNumCudaMultiProcessors() * - device.maxCudaThreadsPerMultiProcessor() / 1024; - const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); - LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), - num_blocks, 1024, 0, device, reducer.initialize(), - num_preserved_vals, output); - } - - LAUNCH_CUDA_KERNEL((OuterReductionKernel<num_per_thread, Self, Op, Index>), - num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); - - return false; - } -}; - +#if defined(__clang__) || defined(__GNUC__) +#warning "Deprecated header file, please either include the main Eigen/CXX11/Tensor header or the respective TensorReductionGpu.h file" #endif - -} // end namespace internal -} // end namespace Eigen - -#endif // EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H +#include "TensorReductionGpu.h"
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h new file mode 100644 index 0000000..0718ba2 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
@@ -0,0 +1,825 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_GPU_H +#define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_GPU_H + +namespace Eigen { +namespace internal { + + +#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC) +// Full reducers for GPU, don't vectorize for now + +// Reducer function that enables multiple gpu thread to safely accumulate at the same +// output address. It basically reads the current value of the output variable, and +// attempts to update it with the new value. If in the meantime another gpu thread +// updated the content of the output address it will try again. +template <typename T, typename R> +__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) { +#if (defined(EIGEN_HIP_DEVICE_COMPILE) && defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)) || (EIGEN_CUDA_ARCH >= 300) + if (sizeof(T) == 4) + { + unsigned int oldval = *reinterpret_cast<unsigned int*>(output); + unsigned int newval = oldval; + reducer.reduce(accum, reinterpret_cast<T*>(&newval)); + if (newval == oldval) { + return; + } + unsigned int readback; + while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) { + oldval = readback; + newval = oldval; + reducer.reduce(accum, reinterpret_cast<T*>(&newval)); + if (newval == oldval) { + return; + } + } + } + else if (sizeof(T) == 8) { + unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output); + unsigned long long newval = oldval; + reducer.reduce(accum, reinterpret_cast<T*>(&newval)); + if (newval == oldval) { + return; + } + unsigned long long readback; + while ((readback = atomicCAS((unsigned long long*)output, oldval, newval)) != oldval) { + oldval = readback; + newval = oldval; + reducer.reduce(accum, reinterpret_cast<T*>(&newval)); + if (newval == oldval) { + return; + } + } + } + else { + gpu_assert(0 && "Wordsize not supported"); + } +#else // EIGEN_CUDA_ARCH >= 300 + gpu_assert(0 && "Shouldn't be called on unsupported device"); +#endif // EIGEN_CUDA_ARCH >= 300 +} + +// We extend atomicExch to support extra data types +template <typename Type> +__device__ inline Type atomicExchCustom(Type* address, Type val) { + return atomicExch(address, val); +} + +template <> +__device__ inline double atomicExchCustom(double* address, double val) { + unsigned long long int* address_as_ull = reinterpret_cast<unsigned long long int*>(address); + return __longlong_as_double(atomicExch(address_as_ull, __double_as_longlong(val))); +} + +#ifdef EIGEN_HAS_GPU_FP16 +template <template <typename T> class R> +__device__ inline void atomicReduce(half2* output, half2 accum, R<half>& reducer) { + unsigned int oldval = *reinterpret_cast<unsigned int*>(output); + unsigned int newval = oldval; + reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval)); + if (newval == oldval) { + return; + } + unsigned int readback; + while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) { + oldval = readback; + newval = oldval; + reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval)); + if (newval == oldval) { + return; + } + } +} +#endif // EIGEN_HAS_GPU_FP16 + +template <> +__device__ inline void atomicReduce(float* output, float accum, SumReducer<float>&) { +#if (defined(EIGEN_HIP_DEVICE_COMPILE) && defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)) || (EIGEN_CUDA_ARCH >= 300) + atomicAdd(output, accum); +#else // EIGEN_CUDA_ARCH >= 300 + gpu_assert(0 && "Shouldn't be called on unsupported device"); +#endif // EIGEN_CUDA_ARCH >= 300 +} + + +template <typename CoeffType, typename Index> +__global__ void ReductionInitKernel(const CoeffType val, Index num_preserved_coeffs, CoeffType* output) { + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + const Index num_threads = blockDim.x * gridDim.x; + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = val; + } +} + + +template <int BlockSize, int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void FullReductionKernel(Reducer reducer, const Self input, Index num_coeffs, + typename Self::CoeffReturnType* output, unsigned int* semaphore) { +#if (defined(EIGEN_HIP_DEVICE_COMPILE) && defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)) || (EIGEN_CUDA_ARCH >= 300) + // Initialize the output value + const Index first_index = blockIdx.x * BlockSize * NumPerThread + threadIdx.x; + if (gridDim.x == 1) { + if (first_index == 0) { + *output = reducer.initialize(); + } + } + else { + if (threadIdx.x == 0) { + unsigned int block = atomicCAS(semaphore, 0u, 1u); + if (block == 0) { + // We're the first block to run, initialize the output value + atomicExchCustom(output, reducer.initialize()); + __threadfence(); + atomicExch(semaphore, 2u); + } + else { + // Wait for the first block to initialize the output value. + // Use atomicCAS here to ensure that the reads aren't cached + unsigned int val; + do { + val = atomicCAS(semaphore, 2u, 2u); + } + while (val < 2u); + } + } + } + + __syncthreads(); + + eigen_assert(gridDim.x == 1 || *semaphore >= 2u); + + typename Self::CoeffReturnType accum = reducer.initialize(); + Index max_iter = numext::mini<Index>(num_coeffs - first_index, NumPerThread*BlockSize); + for (Index i = 0; i < max_iter; i+=BlockSize) { + const Index index = first_index + i; + eigen_assert(index < num_coeffs); + typename Self::CoeffReturnType val = input.m_impl.coeff(index); + reducer.reduce(val, &accum); + } + +#pragma unroll + for (int offset = warpSize/2; offset > 0; offset /= 2) { + #if defined(EIGEN_HIPCC) + // use std::is_floating_point to determine the type of reduced_val + // This is needed because when Type == double, hipcc will give a "call to __shfl_down is ambguous" error + // and list the float and int versions of __shfl_down as the candidate functions. + if (std::is_floating_point<typename Self::CoeffReturnType>::value) { + reducer.reduce(__shfl_down(static_cast<float>(accum), offset, warpSize), &accum); + } else { + reducer.reduce(__shfl_down(static_cast<int>(accum), offset, warpSize), &accum); + } + #elif defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 + reducer.reduce(__shfl_down(accum, offset, warpSize), &accum); + #else + reducer.reduce(__shfl_down_sync(0xFFFFFFFF, accum, offset, warpSize), &accum); + #endif + } + + if ((threadIdx.x & (warpSize - 1)) == 0) { + atomicReduce(output, accum, reducer); + } + + if (gridDim.x > 1 && threadIdx.x == 0) { + // Let the last block reset the semaphore + atomicInc(semaphore, gridDim.x + 1); +#if defined(EIGEN_HIPCC) + __threadfence_system(); +#endif + } +#else // EIGEN_CUDA_ARCH >= 300 + gpu_assert(0 && "Shouldn't be called on unsupported device"); +#endif // EIGEN_CUDA_ARCH >= 300 +} + + +#ifdef EIGEN_HAS_GPU_FP16 +template <typename Self, + typename Reducer, typename Index> +__global__ void ReductionInitFullReduxKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs, half2* scratch) { + eigen_assert(blockDim.x == 1); + eigen_assert(gridDim.x == 1); + if (num_coeffs % 2 != 0) { + half lastCoeff = input.m_impl.coeff(num_coeffs-1); + *scratch = __halves2half2(lastCoeff, reducer.initialize()); + } else { + *scratch = reducer.template initializePacket<half2>(); + } +} + +template <typename Self, + typename Reducer, typename Index> +__global__ void ReductionInitKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs, half* output) { + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + const Index num_threads = blockDim.x * gridDim.x; + const Index num_packets = num_coeffs / 2; + for (Index i = thread_id; i < num_packets; i += num_threads) { + ((half2*)output)[i] = reducer.template initializePacket<half2>(); + } + + if (thread_id == 0 && num_coeffs % 2 != 0) { + output[num_coeffs-1] = reducer.initialize(); + } +} + +template <int BlockSize, int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void FullReductionKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs, + half* output, half2* scratch) { + eigen_assert(NumPerThread % 2 == 0); + + const Index first_index = blockIdx.x * BlockSize * NumPerThread + 2*threadIdx.x; + + // Initialize the output value if it wasn't initialized by the ReductionInitKernel + + if (gridDim.x == 1) { + if (first_index == 0) { + if (num_coeffs % 2 != 0) { + half last = input.m_impl.coeff(num_coeffs-1); + *scratch = __halves2half2(last, reducer.initialize()); + } else { + *scratch = reducer.template initializePacket<half2>(); + } + } + __syncthreads(); + } + + half2 accum = reducer.template initializePacket<half2>(); + const Index max_iter = numext::mini<Index>((num_coeffs - first_index) / 2, NumPerThread*BlockSize / 2); + for (Index i = 0; i < max_iter; i += BlockSize) { + const Index index = first_index + 2*i; + eigen_assert(index + 1 < num_coeffs); + half2 val = input.m_impl.template packet<Unaligned>(index); + reducer.reducePacket(val, &accum); + } + +#pragma unroll + for (int offset = warpSize/2; offset > 0; offset /= 2) { + #if defined(EIGEN_HIPCC) + // FIXME : remove this workaround once we have native half/half2 support for __shfl_down + union { int i; half2 h; } wka_in, wka_out; + wka_in.h = accum; + wka_out.i = __shfl_down(wka_in.i, offset, warpSize); + reducer.reducePacket(wka_out.h, &accum); + #elif defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 + reducer.reducePacket(__shfl_down(accum, offset, warpSize), &accum); + #else + int temp = __shfl_down_sync(0xFFFFFFFF, *(int*)(&accum), (unsigned)offset, warpSize); + reducer.reducePacket(*(half2*)(&temp), &accum); + #endif + } + + if ((threadIdx.x & (warpSize - 1)) == 0) { + atomicReduce(scratch, accum, reducer); + } + + if (gridDim.x == 1) { + __syncthreads(); + if (first_index == 0) { + half tmp = __low2half(*scratch); + reducer.reduce(__high2half(*scratch), &tmp); + *output = tmp; + } + } +} + +template <typename Op> +__global__ void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half2* scratch) { + eigen_assert(threadIdx.x == 1); + half tmp = __low2half(*scratch); + reducer.reduce(__high2half(*scratch), &tmp); + *output = tmp; +} + +#endif // EIGEN_HAS_GPU_FP16 + +template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void> +struct FullReductionLauncher { + static void run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index) { + gpu_assert(false && "Should only be called on doubles, floats and half floats"); + } +}; + +// Specialization for float and double +template <typename Self, typename Op, typename OutputType, bool PacketAccess> +struct FullReductionLauncher< + Self, Op, OutputType, PacketAccess, + typename internal::enable_if< + internal::is_same<float, OutputType>::value || + internal::is_same<double, OutputType>::value, + void>::type> { + static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs) { + + typedef typename Self::Index Index; + const int block_size = 256; + const int num_per_thread = 128; + const int num_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + + unsigned int* semaphore = NULL; + if (num_blocks > 1) { + semaphore = device.semaphore(); + } + + LAUNCH_GPU_KERNEL((FullReductionKernel<block_size, num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs, output, semaphore); + } +}; + +#ifdef EIGEN_HAS_GPU_FP16 +template <typename Self, typename Op> +struct FullReductionLauncher<Self, Op, Eigen::half, false> { + static void run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index) { + gpu_assert(false && "Should not be called since there is no packet accessor"); + } +}; + +template <typename Self, typename Op> +struct FullReductionLauncher<Self, Op, Eigen::half, true> { + static void run(const Self& self, Op& reducer, const GpuDevice& device, half* output, typename Self::Index num_coeffs) { + typedef typename Self::Index Index; + + const int block_size = 256; + const int num_per_thread = 128; + const int num_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + half2* scratch = static_cast<half2*>(device.scratchpad()); + + if (num_blocks > 1) { + // We initialize the output and the scrathpad outside the reduction kernel when we can't be sure that there + // won't be a race conditions between multiple thread blocks. + LAUNCH_GPU_KERNEL((ReductionInitFullReduxKernelHalfFloat<Self, Op, Index>), + 1, 1, 0, device, reducer, self, num_coeffs, scratch); + } + + LAUNCH_GPU_KERNEL((FullReductionKernelHalfFloat<block_size, num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs, output, scratch); + + if (num_blocks > 1) { + LAUNCH_GPU_KERNEL((ReductionCleanupKernelHalfFloat<Op>), + 1, 1, 0, device, reducer, output, scratch); + } + } +}; +#endif // EIGEN_HAS_GPU_FP16 + + +template <typename Self, typename Op, bool Vectorizable> +struct FullReducer<Self, Op, GpuDevice, Vectorizable> { + // Unfortunately nvidia doesn't support well exotic types such as complex, + // so reduce the scope of the optimized version of the code to the simple cases + // of doubles, floats and half floats +#ifdef EIGEN_HAS_GPU_FP16 + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful && + (internal::is_same<typename Self::CoeffReturnType, float>::value || + internal::is_same<typename Self::CoeffReturnType, double>::value || + (internal::is_same<typename Self::CoeffReturnType, Eigen::half>::value && reducer_traits<Op, GpuDevice>::PacketAccess)); +#else // EIGEN_HAS_GPU_FP16 + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful && + (internal::is_same<typename Self::CoeffReturnType, float>::value || + internal::is_same<typename Self::CoeffReturnType, double>::value); +#endif // EIGEN_HAS_GPU_FP16 + + template <typename OutputType> + static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output) { + gpu_assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats"); + const Index num_coeffs = array_prod(self.m_impl.dimensions()); + // Don't crash when we're called with an input tensor of size 0. + if (num_coeffs == 0) { + return; + } + + FullReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(self, reducer, device, output, num_coeffs); + } +}; + + +template <int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void InnerReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, + typename Self::CoeffReturnType* output) { +#if (defined(EIGEN_HIP_DEVICE_COMPILE) && defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)) || (EIGEN_CUDA_ARCH >= 300) + typedef typename Self::CoeffReturnType Type; + eigen_assert(blockDim.y == 1); + eigen_assert(blockDim.z == 1); + eigen_assert(gridDim.y == 1); + eigen_assert(gridDim.z == 1); + + const int unroll_times = 16; + eigen_assert(NumPerThread % unroll_times == 0); + + const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread); + const Index num_input_blocks = input_col_blocks * num_preserved_coeffs; + + const Index num_threads = blockDim.x * gridDim.x; + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + + // Initialize the output values if they weren't initialized by the ReductionInitKernel + if (gridDim.x == 1) { + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = reducer.initialize(); + } + __syncthreads(); + } + + for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) { + const Index row = i / input_col_blocks; + + if (row < num_preserved_coeffs) { + const Index col_block = i % input_col_blocks; + const Index col_begin = col_block * blockDim.x * NumPerThread + threadIdx.x; + + Type reduced_val = reducer.initialize(); + + for (Index j = 0; j < NumPerThread; j += unroll_times) { + const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1); + if (last_col >= num_coeffs_to_reduce) { + for (Index col = col_begin + blockDim.x * j; col < num_coeffs_to_reduce; col += blockDim.x) { + const Type val = input.m_impl.coeff(row * num_coeffs_to_reduce + col); + reducer.reduce(val, &reduced_val); + } + break; + } else { + // Faster version of the loop with no branches after unrolling. +#pragma unroll + for (int k = 0; k < unroll_times; ++k) { + const Index col = col_begin + blockDim.x * (j + k); + reducer.reduce(input.m_impl.coeff(row * num_coeffs_to_reduce + col), &reduced_val); + } + } + } + +#pragma unroll + for (int offset = warpSize/2; offset > 0; offset /= 2) { + #if defined(EIGEN_HIPCC) + // use std::is_floating_point to determine the type of reduced_val + // This is needed because when Type == double, hipcc will give a "call to __shfl_down is ambguous" error + // and list the float and int versions of __shfl_down as the candidate functions. + if (std::is_floating_point<Type>::value) { + reducer.reduce(__shfl_down(static_cast<float>(reduced_val), offset), &reduced_val); + } else { + reducer.reduce(__shfl_down(static_cast<int>(reduced_val), offset), &reduced_val); + } + #elif defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 + reducer.reduce(__shfl_down(reduced_val, offset), &reduced_val); + #else + reducer.reduce(__shfl_down_sync(0xFFFFFFFF, reduced_val, offset), &reduced_val); + #endif + } + + if ((threadIdx.x & (warpSize - 1)) == 0) { + atomicReduce(&(output[row]), reduced_val, reducer); + } + } + } +#else // EIGEN_CUDA_ARCH >= 300 + gpu_assert(0 && "Shouldn't be called on unsupported device"); +#endif // EIGEN_CUDA_ARCH >= 300 +} + +#ifdef EIGEN_HAS_GPU_FP16 + +template <int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void InnerReductionKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, + half* output) { + eigen_assert(blockDim.y == 1); + eigen_assert(blockDim.z == 1); + eigen_assert(gridDim.y == 1); + eigen_assert(gridDim.z == 1); + + const int unroll_times = 16; + eigen_assert(NumPerThread % unroll_times == 0); + eigen_assert(unroll_times % 2 == 0); + + const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread * 2); + const Index num_input_blocks = divup<Index>(input_col_blocks * num_preserved_coeffs, 2); + + const Index num_threads = blockDim.x * gridDim.x; + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + + // Initialize the output values if they weren't initialized by the ReductionInitKernel + if (gridDim.x == 1) { + Index i = 2*thread_id; + for (; i + 1 < num_preserved_coeffs; i += 2*num_threads) { + half* loc = output + i; + *((half2*)loc) = reducer.template initializePacket<half2>(); + } + if (i < num_preserved_coeffs) { + output[i] = reducer.initialize(); + } + __syncthreads(); + } + + for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) { + const Index row = 2 * (i / input_col_blocks); + + if (row + 1 < num_preserved_coeffs) { + const Index col_block = i % input_col_blocks; + const Index col_begin = 2 * (col_block * blockDim.x * NumPerThread + threadIdx.x); + + half2 reduced_val1 = reducer.template initializePacket<half2>(); + half2 reduced_val2 = reducer.template initializePacket<half2>(); + + for (Index j = 0; j < NumPerThread; j += unroll_times) { + const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1) * 2; + if (last_col >= num_coeffs_to_reduce) { + Index col = col_begin + blockDim.x * j; + for (; col + 1 < num_coeffs_to_reduce; col += blockDim.x) { + const half2 val1 = input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col); + reducer.reducePacket(val1, &reduced_val1); + const half2 val2 = input.m_impl.template packet<Unaligned>((row+1) * num_coeffs_to_reduce + col); + reducer.reducePacket(val2, &reduced_val2); + } + if (col < num_coeffs_to_reduce) { + // Peel; + const half last1 = input.m_impl.coeff(row * num_coeffs_to_reduce + col); + const half2 val1 = __halves2half2(last1, reducer.initialize()); + reducer.reducePacket(val1, &reduced_val1); + const half last2 = input.m_impl.coeff((row+1) * num_coeffs_to_reduce + col); + const half2 val2 = __halves2half2(last2, reducer.initialize()); + reducer.reducePacket(val2, &reduced_val2); + } + break; + } else { + // Faster version of the loop with no branches after unrolling. +#pragma unroll + for (int k = 0; k < unroll_times; ++k) { + const Index col = col_begin + blockDim.x * (j + k) * 2; + reducer.reducePacket(input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col), &reduced_val1); + reducer.reducePacket(input.m_impl.template packet<Unaligned>((row + 1)* num_coeffs_to_reduce + col), &reduced_val2); + } + } + } + +#pragma unroll + for (int offset = warpSize/2; offset > 0; offset /= 2) { + #if defined(EIGEN_HIPCC) + // FIXME : remove this workaround once we have native half/half2 support for __shfl_down + union { int i; half2 h; } wka_in, wka_out; + + wka_in.h = reduced_val1; + wka_out.i = __shfl_down(wka_in.i, offset, warpSize); + reducer.reducePacket(wka_out.h, &reduced_val1); + + wka_in.h = reduced_val2; + wka_out.i = __shfl_down(wka_in.i, offset, warpSize); + reducer.reducePacket(wka_out.h, &reduced_val2); + #elif defined(EIGEN_CUDACC_VER) && EIGEN_CUDACC_VER < 90000 + reducer.reducePacket(__shfl_down(reduced_val1, offset, warpSize), &reduced_val1); + reducer.reducePacket(__shfl_down(reduced_val2, offset, warpSize), &reduced_val2); + #else + int temp1 = __shfl_down_sync(0xFFFFFFFF, *(int*)(&reduced_val1), (unsigned)offset, warpSize); + int temp2 = __shfl_down_sync(0xFFFFFFFF, *(int*)(&reduced_val2), (unsigned)offset, warpSize); + reducer.reducePacket(*(half2*)(&temp1), &reduced_val1); + reducer.reducePacket(*(half2*)(&temp2), &reduced_val2); + #endif + } + + half val1 = __low2half(reduced_val1); + reducer.reduce(__high2half(reduced_val1), &val1); + half val2 = __low2half(reduced_val2); + reducer.reduce(__high2half(reduced_val2), &val2); + half2 val = __halves2half2(val1, val2); + + if ((threadIdx.x & (warpSize - 1)) == 0) { + half* loc = output + row; + atomicReduce((half2*)loc, val, reducer); + } + } + } +} + +#endif // EIGEN_HAS_GPU_FP16 + +template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void> +struct InnerReductionLauncher { + static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index, typename Self::Index) { + gpu_assert(false && "Should only be called to reduce doubles, floats and half floats on a gpu device"); + return true; + } +}; + +// Specialization for float and double +template <typename Self, typename Op, typename OutputType, bool PacketAccess> +struct InnerReductionLauncher< + Self, Op, OutputType, PacketAccess, + typename internal::enable_if< + internal::is_same<float, OutputType>::value || + internal::is_same<double, OutputType>::value, + void>::type> { + static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + typedef typename Self::Index Index; + + const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; + const int block_size = 256; + const int num_per_thread = 128; + const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / block_size; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + + if (num_blocks > 1) { + // We initialize the outputs outside the reduction kernel when we can't be sure that there + // won't be a race conditions between multiple thread blocks. + const int dyn_blocks = divup<int>(num_preserved_vals, 1024); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / 1024; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + LAUNCH_GPU_KERNEL((ReductionInitKernel<OutputType, Index>), + num_blocks, 1024, 0, device, reducer.initialize(), + num_preserved_vals, output); + } + + LAUNCH_GPU_KERNEL((InnerReductionKernel<num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); + + return false; + } +}; + +#ifdef EIGEN_HAS_GPU_FP16 +template <typename Self, typename Op> +struct InnerReductionLauncher<Self, Op, Eigen::half, false> { + static bool run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index, typename Self::Index) { + gpu_assert(false && "Should not be called since there is no packet accessor"); + return true; + } +}; + +template <typename Self, typename Op> +struct InnerReductionLauncher<Self, Op, Eigen::half, true> { + static bool run(const Self& self, Op& reducer, const GpuDevice& device, half* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + typedef typename Self::Index Index; + + if (num_preserved_vals % 2 != 0) { + // Not supported yet, revert to the slower code path + return true; + } + + const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; + const int block_size = /*256*/128; + const int num_per_thread = /*128*/64; + const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / block_size; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + + if (num_blocks > 1) { + // We initialize the outputs outside the reduction kernel when we can't be sure that there + // won't be a race conditions between multiple thread blocks. + const int dyn_blocks = divup<int>(num_preserved_vals, 1024); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / 1024; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + LAUNCH_GPU_KERNEL((ReductionInitKernelHalfFloat<Self, Op, Index>), + 1, 1, 0, device, reducer, self, num_preserved_vals, output); + } + + LAUNCH_GPU_KERNEL((InnerReductionKernelHalfFloat<num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); + + return false; + } +}; +#endif // EIGEN_HAS_GPU_FP16 + + +template <typename Self, typename Op> +struct InnerReducer<Self, Op, GpuDevice> { + // Unfortunately nvidia doesn't support well exotic types such as complex, + // so reduce the scope of the optimized version of the code to the simple case + // of floats and half floats. +#ifdef EIGEN_HAS_GPU_FP16 + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful && + (internal::is_same<typename Self::CoeffReturnType, float>::value || + internal::is_same<typename Self::CoeffReturnType, double>::value || + (internal::is_same<typename Self::CoeffReturnType, Eigen::half>::value && reducer_traits<Op, GpuDevice>::PacketAccess)); +#else // EIGEN_HAS_GPU_FP16 + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful && + (internal::is_same<typename Self::CoeffReturnType, float>::value || + internal::is_same<typename Self::CoeffReturnType, double>::value); +#endif // EIGEN_HAS_GPU_FP16 + + template <typename OutputType> + static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + gpu_assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats"); + const Index num_coeffs = array_prod(self.m_impl.dimensions()); + // Don't crash when we're called with an input tensor of size 0. + if (num_coeffs == 0) { + return true; + } + // It's faster to use the usual code. + if (num_coeffs_to_reduce <= 128) { + return true; + } + + return InnerReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(self, reducer, device, output, num_coeffs_to_reduce, num_preserved_vals); + } +}; + +template <int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void OuterReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, + typename Self::CoeffReturnType* output) { + const Index num_threads = blockDim.x * gridDim.x; + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + // Initialize the output values if they weren't initialized by the ReductionInitKernel + if (gridDim.x == 1) { + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = reducer.initialize(); + } + __syncthreads(); + } + + // Do the reduction. + const Index max_iter = num_preserved_coeffs * divup<Index>(num_coeffs_to_reduce, NumPerThread); + for (Index i = thread_id; i < max_iter; i += num_threads) { + const Index input_col = i % num_preserved_coeffs; + const Index input_row = (i / num_preserved_coeffs) * NumPerThread; + typename Self::CoeffReturnType reduced_val = reducer.initialize(); + const Index max_row = numext::mini(input_row + NumPerThread, num_coeffs_to_reduce); + for (Index j = input_row; j < max_row; j++) { + typename Self::CoeffReturnType val = input.m_impl.coeff(j * num_preserved_coeffs + input_col); + reducer.reduce(val, &reduced_val); + } + atomicReduce(&(output[input_col]), reduced_val, reducer); + } +} + + +template <typename Self, typename Op> +struct OuterReducer<Self, Op, GpuDevice> { + // Unfortunately nvidia doesn't support well exotic types such as complex, + // so reduce the scope of the optimized version of the code to the simple case + // of floats. + static const bool HasOptimizedImplementation = !Self::ReducerTraits::IsStateful && + (internal::is_same<typename Self::CoeffReturnType, float>::value || + internal::is_same<typename Self::CoeffReturnType, double>::value); + template <typename Device, typename OutputType> + static + #if !defined(EIGEN_HIPCC) + // FIXME : leaving this EIGEN_DEVICE_FUNC in, results in the following runtime error + // (in the cxx11_tensor_reduction_gpu test) + // + // terminate called after throwing an instance of 'std::runtime_error' + // what(): No device code available for function: _ZN5Eigen8internal20OuterReductionKernelIL... + // + // don't know why this happens (and why is it a runtime error instead of a compile time error) + // + // this will be fixed by HIP PR#457 + EIGEN_DEVICE_FUNC + #endif + bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) { + gpu_assert(false && "Should only be called to reduce doubles or floats on a gpu device"); + return true; + } + + static bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + typedef typename Self::Index Index; + + // It's faster to use the usual code. + if (num_coeffs_to_reduce <= 32) { + return true; + } + + const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; + const int block_size = 256; + const int num_per_thread = 16; + const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / block_size; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + + if (num_blocks > 1) { + // We initialize the outputs in the reduction kernel itself when we don't have to worry + // about race conditions between multiple thread blocks. + const int dyn_blocks = divup<int>(num_preserved_vals, 1024); + const int max_blocks = device.getNumGpuMultiProcessors() * + device.maxGpuThreadsPerMultiProcessor() / 1024; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + LAUNCH_GPU_KERNEL((ReductionInitKernel<float, Index>), + num_blocks, 1024, 0, device, reducer.initialize(), + num_preserved_vals, output); + } + + LAUNCH_GPU_KERNEL((OuterReductionKernel<num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); + + return false; + } +}; + +#endif // defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC) + + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_GPU_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionSycl.h new file mode 100644 index 0000000..a379f5a --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionSycl.h
@@ -0,0 +1,177 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclPlaceHolderExpr.h + * + * \brief: + * This is the specialisation of the placeholder expression based on the + * operation type + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_REDUCTION_SYCL_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_REDUCTION_SYCL_HPP + +namespace Eigen { +namespace internal { + +template<typename OP, typename CoeffReturnType> struct syclGenericBufferReducer{ +template<typename BufferTOut, typename BufferTIn> +static void run(OP op, BufferTOut& bufOut, ptrdiff_t out_offset, BufferTIn& bufI, const Eigen::SyclDevice& dev, size_t length, size_t local){ + do { + auto f = [length, local, op, out_offset, &bufOut, &bufI](cl::sycl::handler& h) mutable { + cl::sycl::nd_range<1> r{cl::sycl::range<1>{std::max(length, local)}, + cl::sycl::range<1>{std::min(length, local)}}; + /* Two accessors are used: one to the buffer that is being reduced, + * and a second to local memory, used to store intermediate data. */ + auto aI =bufI.template get_access<cl::sycl::access::mode::read_write>(h); + auto aOut =bufOut.template get_access<cl::sycl::access::mode::write>(h); + typedef decltype(aI) InputAccessor; + typedef decltype(aOut) OutputAccessor; + typedef cl::sycl::accessor<CoeffReturnType, 1, cl::sycl::access::mode::read_write,cl::sycl::access::target::local> LocalAccessor; + LocalAccessor scratch(cl::sycl::range<1>(local), h); + + /* The parallel_for invocation chosen is the variant with an nd_item + * parameter, since the code requires barriers for correctness. */ + h.parallel_for(r, TensorSycl::internal::GenericKernelReducer<CoeffReturnType, OP, OutputAccessor, InputAccessor, LocalAccessor>(op, aOut, out_offset, aI, scratch, length, local)); + }; + dev.sycl_queue().submit(f); + dev.asynchronousExec(); + + /* At this point, you could queue::wait_and_throw() to ensure that + * errors are caught quickly. However, this would likely impact + * performance negatively. */ + length = length / local; + + } while (length > 1); +} + +}; + +template<typename CoeffReturnType> struct syclGenericBufferReducer<Eigen::internal::MeanReducer<CoeffReturnType>, CoeffReturnType>{ +template<typename BufferTOut, typename BufferTIn> +static void run(Eigen::internal::MeanReducer<CoeffReturnType>, BufferTOut& bufOut,ptrdiff_t out_offset, BufferTIn& bufI, const Eigen::SyclDevice& dev, size_t length, size_t local){ + syclGenericBufferReducer<Eigen::internal::SumReducer<CoeffReturnType>, CoeffReturnType>::run(Eigen::internal::SumReducer<CoeffReturnType>(), + bufOut, out_offset, bufI, dev, length, local); +} +}; + +/// Self is useless here because in expression construction we are going to treat reduction as a leafnode. +/// we want to take reduction child and then build a construction and apply the full reducer function on it. Fullreducre applies the +/// reduction operation on the child of the reduction. once it is done the reduction is an empty shell and can be thrown away and treated as +// a leafNode. + +template <typename Self, typename Op, bool Vectorizable> +struct FullReducer<Self, Op, const Eigen::SyclDevice, Vectorizable> { + + typedef typename Self::CoeffReturnType CoeffReturnType; + static const bool HasOptimizedImplementation = false; + + static void run(const Self& self, Op& reducer, const Eigen::SyclDevice& dev, CoeffReturnType* output) { + typedef const typename Self::ChildType HostExpr; /// this is the child of reduction + typedef Eigen::TensorSycl::internal::FunctorExtractor<TensorEvaluator<HostExpr, const Eigen::SyclDevice> > FunctorExpr; + FunctorExpr functors = TensorSycl::internal::extractFunctors(self.impl()); + int red_factor =256; /// initial reduction. If the size is less than red_factor we only creates one thread. + size_t inputSize =self.impl().dimensions().TotalSize(); + size_t rng = inputSize/red_factor; // the total number of thread initially is half the size of the input + size_t remaining = inputSize% red_factor; + if(rng ==0) { + red_factor=1; + }; + size_t tileSize =dev.sycl_queue().get_device(). template get_info<cl::sycl::info::device::max_work_group_size>()/2; + size_t GRange=std::max((size_t )1, rng); + + // convert global range to power of 2 for redecution + GRange--; + GRange |= GRange >> 1; + GRange |= GRange >> 2; + GRange |= GRange >> 4; + GRange |= GRange >> 8; + GRange |= GRange >> 16; +#if __x86_64__ || __ppc64__ || _WIN64 + GRange |= GRange >> 32; +#endif + GRange++; + size_t outTileSize = tileSize; + /// if the shared memory is less than the GRange, we set shared_mem size to the TotalSize and in this case one kernel would be created for recursion to reduce all to one. + if (GRange < outTileSize) outTileSize=GRange; + /// creating the shared memory for calculating reduction. + /// This one is used to collect all the reduced value of shared memory as we don't have global barrier on GPU. Once it is saved we can + /// recursively apply reduction on it in order to reduce the whole. + auto temp_global_buffer =cl::sycl::buffer<CoeffReturnType, 1>(cl::sycl::range<1>(GRange)); + typedef typename Eigen::internal::remove_all<decltype(self.xprDims())>::type Dims; + // Dims dims= self.xprDims(); + //Op functor = reducer; + dev.sycl_queue().submit([&](cl::sycl::handler &cgh) { + // this is a workaround for gcc 4.8 bug + typedef decltype(TensorSycl::internal::createTupleOfAccessors(cgh, self.impl())) TupleType; + // create a tuple of accessors from Evaluator + TupleType tuple_of_accessors = TensorSycl::internal::createTupleOfAccessors(cgh, self.impl()); + auto tmp_global_accessor = temp_global_buffer. template get_access<cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer>(cgh); + typedef decltype(tmp_global_accessor) OutAccessor; + cgh.parallel_for( cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(outTileSize)), + TensorSycl::internal::FullReductionKernelFunctor<CoeffReturnType, OutAccessor, HostExpr, FunctorExpr, Op, Dims, size_t, TupleType> + (tmp_global_accessor, rng, remaining, red_factor, reducer, self.xprDims(), functors, tuple_of_accessors)); + }); + dev.asynchronousExec(); + + // getting final out buffer at the moment the created buffer is true because there is no need for assign + auto out_buffer =dev.get_sycl_buffer(output); + ptrdiff_t out_offset = dev.get_offset(output); + /// This is used to recursively reduce the tmp value to an element of 1; + syclGenericBufferReducer<Op, CoeffReturnType>::run(reducer, out_buffer, out_offset, temp_global_buffer,dev, GRange, outTileSize); + } + +}; + + +template <typename Self, typename Op> +struct InnerReducer<Self, Op, const Eigen::SyclDevice> { + + typedef typename Self::CoeffReturnType CoeffReturnType; + static const bool HasOptimizedImplementation = false; + + static bool run(const Self& self, Op& reducer, const Eigen::SyclDevice& dev, CoeffReturnType* output, typename Self::Index num_values_to_reduce, typename Self::Index num_coeffs_to_preserve) { + typedef const typename Self::ChildType HostExpr; /// this is the child of reduction + typedef Eigen::TensorSycl::internal::FunctorExtractor<TensorEvaluator<HostExpr, const Eigen::SyclDevice> > FunctorExpr; + FunctorExpr functors = TensorSycl::internal::extractFunctors(self.impl()); + typename Self::Index range, GRange, tileSize; + typedef typename Eigen::internal::remove_all<decltype(self.xprDims())>::type Dims; + + // getting final out buffer at the moment the created buffer is true because there is no need for assign + /// creating the shared memory for calculating reduction. + /// This one is used to collect all the reduced value of shared memory as we don't have global barrier on GPU. Once it is saved we can + /// recursively apply reduction on it in order to reduce the whole. + dev.parallel_for_setup(num_coeffs_to_preserve, tileSize, range, GRange); + dev.sycl_queue().submit([&](cl::sycl::handler &cgh) { + // this is workaround for gcc 4.8 bug. + typedef decltype(TensorSycl::internal::createTupleOfAccessors(cgh, self.impl())) Tuple_of_Acc; + // create a tuple of accessors from Evaluator + Tuple_of_Acc tuple_of_accessors = TensorSycl::internal::createTupleOfAccessors(cgh, self.impl()); + auto output_accessor = dev.template get_sycl_accessor<cl::sycl::access::mode::write>(cgh, output); + ptrdiff_t out_offset = dev.get_offset(output); + Index red_size = (num_values_to_reduce!=0)? num_values_to_reduce : static_cast<Index>(1); + cgh.parallel_for( cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), + TensorSycl::internal::ReductionFunctor<HostExpr, FunctorExpr, Tuple_of_Acc, Dims, Op, typename Self::Index> + (output_accessor, out_offset, functors, tuple_of_accessors, self.xprDims(), reducer, range, red_size)); + + }); + dev.asynchronousExec(); + return false; + } +}; + +} // end namespace internal +} // namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSOR_REDUCTION_SYCL_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h b/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h index bc92d9e..6e15e75 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h
@@ -31,7 +31,7 @@ int refCount() const { return m_refcount; } private: - // No copy, no assigment; + // No copy, no assignment; TensorLazyBaseEvaluator(const TensorLazyBaseEvaluator& other); TensorLazyBaseEvaluator& operator = (const TensorLazyBaseEvaluator& other); @@ -136,6 +136,8 @@ enum { IsAligned = false, PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, Layout = PlainObjectType::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -193,7 +195,7 @@ return m_evaluator->coeff(index); } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(Index firstIndex, IndexTypes... otherIndices) const { @@ -364,6 +366,8 @@ enum { IsAligned = false, PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorRef<Derived>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -411,6 +415,8 @@ enum { IsAligned = false, PacketAccess = false, + BlockAccess = false, + PreferBlockAccess = false, RawAccess = false };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h index 1a59cc8..b7fb969 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename ReverseDimensions, typename XprType> @@ -107,11 +108,13 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -122,7 +125,7 @@ : m_impl(op.expression(), device), m_reverse(op.reverse()) { // Reversing a scalar isn't supported yet. It would be a no-op anyway. - EIGEN_STATIC_ASSERT(NumDims > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); // Compute strides m_dimensions = m_impl.dimensions(); @@ -195,7 +198,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); // TODO(ndjaitly): write a better packing routine that uses @@ -222,7 +225,12 @@ TensorOpCost(0, 0, compute_cost, false /* vectorized */, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<ArgType, Device> & impl() const { return m_impl; } + /// added for sycl in order to construct the buffer from sycl device + ReverseDimensions functor() const { return m_reverse; } protected: Dimensions m_dimensions; @@ -247,6 +255,8 @@ enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -258,7 +268,7 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return this->m_dimensions; } @@ -269,7 +279,7 @@ template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); // This code is pilfered from TensorMorphing.h
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorScan.h b/unsupported/Eigen/CXX11/src/Tensor/TensorScan.h new file mode 100644 index 0000000..64f10d0 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorScan.h
@@ -0,0 +1,294 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Igor Babuschkin <igor@babuschk.in> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_SCAN_H +#define EIGEN_CXX11_TENSOR_TENSOR_SCAN_H + +namespace Eigen { + +namespace internal { + +template <typename Op, typename XprType> +struct traits<TensorScanOp<Op, XprType> > + : public traits<XprType> { + typedef typename XprType::Scalar Scalar; + typedef traits<XprType> XprTraits; + typedef typename XprTraits::StorageKind StorageKind; + typedef typename XprType::Nested Nested; + typedef typename remove_reference<Nested>::type _Nested; + static const int NumDimensions = XprTraits::NumDimensions; + static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; +}; + +template<typename Op, typename XprType> +struct eval<TensorScanOp<Op, XprType>, Eigen::Dense> +{ + typedef const TensorScanOp<Op, XprType>& type; +}; + +template<typename Op, typename XprType> +struct nested<TensorScanOp<Op, XprType>, 1, + typename eval<TensorScanOp<Op, XprType> >::type> +{ + typedef TensorScanOp<Op, XprType> type; +}; +} // end namespace internal + +/** \class TensorScan + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor scan class. + */ +template <typename Op, typename XprType> +class TensorScanOp + : public TensorBase<TensorScanOp<Op, XprType>, ReadOnlyAccessors> { +public: + typedef typename Eigen::internal::traits<TensorScanOp>::Scalar Scalar; + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename Eigen::internal::nested<TensorScanOp>::type Nested; + typedef typename Eigen::internal::traits<TensorScanOp>::StorageKind StorageKind; + typedef typename Eigen::internal::traits<TensorScanOp>::Index Index; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorScanOp( + const XprType& expr, const Index& axis, bool exclusive = false, const Op& op = Op()) + : m_expr(expr), m_axis(axis), m_accumulator(op), m_exclusive(exclusive) {} + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Index axis() const { return m_axis; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const XprType& expression() const { return m_expr; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Op accumulator() const { return m_accumulator; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + bool exclusive() const { return m_exclusive; } + +protected: + typename XprType::Nested m_expr; + const Index m_axis; + const Op m_accumulator; + const bool m_exclusive; +}; + +template <typename Self, typename Reducer, typename Device> +struct ScanLauncher; + +// Eval as rvalue +template <typename Op, typename ArgType, typename Device> +struct TensorEvaluator<const TensorScanOp<Op, ArgType>, Device> { + + typedef TensorScanOp<Op, ArgType> XprType; + typedef typename XprType::Index Index; + static const int NumDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value; + typedef DSizes<Index, NumDims> Dimensions; + typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + typedef TensorEvaluator<const TensorScanOp<Op, ArgType>, Device> Self; + + enum { + IsAligned = false, + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, + RawAccess = true + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, + const Device& device) + : m_impl(op.expression(), device), + m_device(device), + m_exclusive(op.exclusive()), + m_accumulator(op.accumulator()), + m_size(m_impl.dimensions()[op.axis()]), + m_stride(1), + m_output(NULL) { + + // Accumulating a scalar isn't supported. + EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); + eigen_assert(op.axis() >= 0 && op.axis() < NumDims); + + // Compute stride of scan axis + const Dimensions& dims = m_impl.dimensions(); + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = 0; i < op.axis(); ++i) { + m_stride = m_stride * dims[i]; + } + } else { + // dims can only be indexed through unsigned integers, + // so let's use an unsigned type to let the compiler knows. + // This prevents stupid warnings: ""'*((void*)(& evaluator)+64)[18446744073709551615]' may be used uninitialized in this function" + unsigned int axis = internal::convert_index<unsigned int>(op.axis()); + for (unsigned int i = NumDims - 1; i > axis; --i) { + m_stride = m_stride * dims[i]; + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { + return m_impl.dimensions(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Index& stride() const { + return m_stride; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Index& size() const { + return m_size; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Op& accumulator() const { + return m_accumulator; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool exclusive() const { + return m_exclusive; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& inner() const { + return m_impl; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Device& device() const { + return m_device; + } + + EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + m_impl.evalSubExprsIfNeeded(NULL); + ScanLauncher<Self, Op, Device> launcher; + if (data) { + launcher(*this, data); + return false; + } + + const Index total_size = internal::array_prod(dimensions()); + m_output = static_cast<CoeffReturnType*>(m_device.allocate(total_size * sizeof(Scalar))); + launcher(*this, m_output); + return true; + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const { + return internal::ploadt<PacketReturnType, LoadMode>(m_output + index); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Eigen::internal::traits<XprType>::PointerType data() const + { + return m_output; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const + { + return m_output[index]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool) const { + return TensorOpCost(sizeof(CoeffReturnType), 0, 0); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + if (m_output != NULL) { + m_device.deallocate(m_output); + m_output = NULL; + } + m_impl.cleanup(); + } + +protected: + TensorEvaluator<ArgType, Device> m_impl; + const Device& m_device; + const bool m_exclusive; + Op m_accumulator; + const Index m_size; + Index m_stride; + CoeffReturnType* m_output; +}; + +// CPU implementation of scan +// TODO(ibab) This single-threaded implementation should be parallelized, +// at least by running multiple scans at the same time. +template <typename Self, typename Reducer, typename Device> +struct ScanLauncher { + void operator()(Self& self, typename Self::CoeffReturnType *data) { + Index total_size = internal::array_prod(self.dimensions()); + + // We fix the index along the scan axis to 0 and perform a + // scan per remaining entry. The iteration is split into two nested + // loops to avoid an integer division by keeping track of each idx1 and idx2. + for (Index idx1 = 0; idx1 < total_size; idx1 += self.stride() * self.size()) { + for (Index idx2 = 0; idx2 < self.stride(); idx2++) { + // Calculate the starting offset for the scan + Index offset = idx1 + idx2; + + // Compute the scan along the axis, starting at the calculated offset + typename Self::CoeffReturnType accum = self.accumulator().initialize(); + for (Index idx3 = 0; idx3 < self.size(); idx3++) { + Index curr = offset + idx3 * self.stride(); + + if (self.exclusive()) { + data[curr] = self.accumulator().finalize(accum); + self.accumulator().reduce(self.inner().coeff(curr), &accum); + } else { + self.accumulator().reduce(self.inner().coeff(curr), &accum); + data[curr] = self.accumulator().finalize(accum); + } + } + } + } + } +}; + +#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC)) + +// GPU implementation of scan +// TODO(ibab) This placeholder implementation performs multiple scans in +// parallel, but it would be better to use a parallel scan algorithm and +// optimize memory access. +template <typename Self, typename Reducer> +__global__ void ScanKernel(Self self, Index total_size, typename Self::CoeffReturnType* data) { + // Compute offset as in the CPU version + Index val = threadIdx.x + blockIdx.x * blockDim.x; + Index offset = (val / self.stride()) * self.stride() * self.size() + val % self.stride(); + + if (offset + (self.size() - 1) * self.stride() < total_size) { + // Compute the scan along the axis, starting at the calculated offset + typename Self::CoeffReturnType accum = self.accumulator().initialize(); + for (Index idx = 0; idx < self.size(); idx++) { + Index curr = offset + idx * self.stride(); + if (self.exclusive()) { + data[curr] = self.accumulator().finalize(accum); + self.accumulator().reduce(self.inner().coeff(curr), &accum); + } else { + self.accumulator().reduce(self.inner().coeff(curr), &accum); + data[curr] = self.accumulator().finalize(accum); + } + } + } + __syncthreads(); + +} + +template <typename Self, typename Reducer> +struct ScanLauncher<Self, Reducer, GpuDevice> { + void operator()(const Self& self, typename Self::CoeffReturnType* data) { + Index total_size = internal::array_prod(self.dimensions()); + Index num_blocks = (total_size / self.size() + 63) / 64; + Index block_size = 64; + + LAUNCH_GPU_KERNEL((ScanKernel<Self, Reducer>), num_blocks, block_size, 0, self.device(), self, total_size, data); + } +}; +#endif // EIGEN_USE_GPU && (EIGEN_GPUCC) + +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_SCAN_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h b/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h index e765337..4169487 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename Shuffle, typename XprType> @@ -60,8 +61,8 @@ typedef typename Eigen::internal::traits<TensorShufflingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorShufflingOp>::Index Index; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorShufflingOp(const XprType& expr, const Shuffle& shuffle) - : m_xpr(expr), m_shuffle(shuffle) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorShufflingOp(const XprType& expr, const Shuffle& shfl) + : m_xpr(expr), m_shuffle(shfl) {} EIGEN_DEVICE_FUNC const Shuffle& shufflePermutation() const { return m_shuffle; } @@ -99,6 +100,7 @@ template<typename Shuffle, typename ArgType, typename Device> struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> { + typedef TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> Self; typedef TensorShufflingOp<Shuffle, ArgType> XprType; typedef typename XprType::Index Index; static const int NumDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value; @@ -106,45 +108,65 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = false, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), - Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = false, // to be implemented - RawAccess = false + IsAligned = false, + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, // to be implemented + RawAccess = false }; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device) + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + TensorBlock; + typedef internal::TensorBlockReader<ScalarNoConst, Index, NumDims, Layout> + TensorBlockReader; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, + const Device& device) + : m_device(device), + m_impl(op.expression(), device), + m_shuffle(op.shufflePermutation()) { const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); const Shuffle& shuffle = op.shufflePermutation(); + m_is_identity = true; for (int i = 0; i < NumDims; ++i) { m_dimensions[i] = input_dims[shuffle[i]]; + m_inverseShuffle[shuffle[i]] = i; + if (m_is_identity && shuffle[i] != i) { + m_is_identity = false; + } } - array<Index, NumDims> inputStrides; - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - inputStrides[0] = 1; + m_unshuffledInputStrides[0] = 1; m_outputStrides[0] = 1; + for (int i = 1; i < NumDims; ++i) { - inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1]; + m_unshuffledInputStrides[i] = + m_unshuffledInputStrides[i - 1] * input_dims[i - 1]; m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]); } } else { - inputStrides[NumDims - 1] = 1; + m_unshuffledInputStrides[NumDims - 1] = 1; m_outputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { - inputStrides[i] = inputStrides[i + 1] * input_dims[i + 1]; + m_unshuffledInputStrides[i] = + m_unshuffledInputStrides[i + 1] * input_dims[i + 1]; m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; + m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]); } } for (int i = 0; i < NumDims; ++i) { - m_inputStrides[i] = inputStrides[shuffle[i]]; + m_inputStrides[i] = m_unshuffledInputStrides[shuffle[i]]; } } @@ -160,46 +182,198 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - return m_impl.coeff(srcCoeff(index)); + if (m_is_identity) { + return m_impl.coeff(index); + } else { + return m_impl.coeff(srcCoeff(index)); + } } + template <int LoadMode, typename Self, bool ImplPacketAccess> + struct PacketLoader { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + static PacketReturnType Run(const Self& self, Index index) { + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + values[i] = self.coeff(index + i); + } + PacketReturnType rslt = internal::pload<PacketReturnType>(values); + return rslt; + } + }; + + template<int LoadMode, typename Self> + struct PacketLoader<LoadMode, Self, true> { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + static PacketReturnType Run(const Self& self, Index index) { + if (self.m_is_identity) { + return self.m_impl.template packet<LoadMode>(index); + } else { + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + values[i] = self.coeff(index + i); + } + PacketReturnType rslt = internal::pload<PacketReturnType>(values); + return rslt; + } + } + }; + template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) - eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) + eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); + return PacketLoader<LoadMode, Self, TensorEvaluator<ArgType, Device>::PacketAccess>::Run(*this, index); + } - EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; - for (int i = 0; i < PacketSize; ++i) { - values[i] = coeff(index+i); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void getResourceRequirements( + std::vector<internal::TensorOpResourceRequirements>* resources) const { + Eigen::Index block_total_size_max = numext::maxi<Eigen::Index>( + 1, m_device.firstLevelCacheSize() / sizeof(Scalar)); + resources->push_back(internal::TensorOpResourceRequirements( + internal::kUniformAllDims, block_total_size_max)); + m_impl.getResourceRequirements(resources); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void block( + TensorBlock* output_block) const { + if (m_impl.data() != NULL) { + // Fast path: we have direct access to the data, so shuffle as we read. + TensorBlockReader::Run(output_block, + srcCoeff(output_block->first_coeff_index()), + m_inverseShuffle, + m_unshuffledInputStrides, + m_impl.data()); + return; } - PacketReturnType rslt = internal::pload<PacketReturnType>(values); - return rslt; + + // Slow path: read unshuffled block from the input and shuffle in-place. + // Initialize input block sizes using input-to-output shuffle map. + DSizes<Index, NumDims> input_block_sizes; + for (Index i = 0; i < NumDims; ++i) { + input_block_sizes[i] = output_block->block_sizes()[m_inverseShuffle[i]]; + } + + // Calculate input block strides. + DSizes<Index, NumDims> input_block_strides; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + input_block_strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + input_block_strides[i] = + input_block_strides[i - 1] * input_block_sizes[i - 1]; + } + } else { + input_block_strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + input_block_strides[i] = + input_block_strides[i + 1] * input_block_sizes[i + 1]; + } + } + + // Read input block. + TensorBlock input_block(srcCoeff(output_block->first_coeff_index()), + input_block_sizes, + input_block_strides, + Dimensions(m_unshuffledInputStrides), + output_block->data()); + + m_impl.block(&input_block); + + // Naive In-place shuffle: random IO but block size is O(L1 cache size). + // TODO(andydavis) Improve the performance of this in-place shuffle. + const Index total_size = input_block_sizes.TotalSize(); + std::vector<bool> bitmap(total_size, false); + ScalarNoConst* data = const_cast<ScalarNoConst*>(output_block->data()); + const DSizes<Index, NumDims>& output_block_strides = + output_block->block_strides(); + for (Index input_index = 0; input_index < total_size; ++input_index) { + if (bitmap[input_index]) { + // Coefficient at this index has already been shuffled. + continue; + } + + Index output_index = GetBlockOutputIndex(input_index, input_block_strides, + output_block_strides); + if (output_index == input_index) { + // Coefficient already in place. + bitmap[output_index] = true; + continue; + } + + // The following loop starts at 'input_index', and shuffles + // coefficients into their shuffled location at 'output_index'. + // It skips through the array shuffling coefficients by following + // the shuffle cycle starting and ending a 'start_index'. + ScalarNoConst evicted_value; + ScalarNoConst shuffled_value = data[input_index]; + do { + evicted_value = data[output_index]; + data[output_index] = shuffled_value; + shuffled_value = evicted_value; + bitmap[output_index] = true; + output_index = GetBlockOutputIndex(output_index, input_block_strides, + output_block_strides); + } while (output_index != input_index); + + data[output_index] = shuffled_value; + bitmap[output_index] = true; + } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { - const double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() + + const double compute_cost = m_is_identity ? TensorOpCost::AddCost<Index>() : + NumDims * (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>()); return m_impl.costPerCoeff(vectorized) + - TensorOpCost(0, 0, compute_cost, false /* vectorized */, PacketSize); + TensorOpCost(0, 0, compute_cost, m_is_identity /* vectorized */, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + // required by sycl + EIGEN_STRONG_INLINE const Shuffle& shufflePermutation() const {return m_shuffle;} + // required by sycl + EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const {return m_impl;} protected: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index GetBlockOutputIndex( + Index input_index, + const DSizes<Index, NumDims>& input_block_strides, + const DSizes<Index, NumDims>& output_block_strides) const { + Index output_index = 0; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumDims - 1; i > 0; --i) { + const Index idx = input_index / input_block_strides[i]; + output_index += idx * output_block_strides[m_inverseShuffle[i]]; + input_index -= idx * input_block_strides[i]; + } + return output_index + input_index * + output_block_strides[m_inverseShuffle[0]]; + } else { + for (int i = 0; i < NumDims - 1; ++i) { + const Index idx = input_index / input_block_strides[i]; + output_index += idx * output_block_strides[m_inverseShuffle[i]]; + input_index -= idx * input_block_strides[i]; + } + return output_index + input_index * + output_block_strides[m_inverseShuffle[NumDims - 1]]; + } + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { - const Index idx = index / m_outputStrides[i]; + const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } return inputIndex + index * m_inputStrides[0]; } else { for (int i = 0; i < NumDims - 1; ++i) { - const Index idx = index / m_outputStrides[i]; + const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } @@ -208,9 +382,17 @@ } Dimensions m_dimensions; + bool m_is_identity; + array<Index, NumDims> m_inverseShuffle; array<Index, NumDims> m_outputStrides; + array<internal::TensorIntDivisor<Index>, NumDims> m_fastOutputStrides; array<Index, NumDims> m_inputStrides; + array<Index, NumDims> m_unshuffledInputStrides; + + const Device& m_device; TensorEvaluator<ArgType, Device> m_impl; + /// required by sycl + Shuffle m_shuffle; }; @@ -228,14 +410,24 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { - IsAligned = false, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), - RawAccess = false + IsAligned = false, + PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1), + BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess, + PreferBlockAccess = true, + Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false }; + typedef typename internal::remove_const<Scalar>::type ScalarNoConst; + + typedef internal::TensorBlock<ScalarNoConst, Index, NumDims, Layout> + TensorBlock; + typedef internal::TensorBlockWriter<ScalarNoConst, Index, NumDims, Layout> + TensorBlockWriter; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) { } @@ -248,7 +440,7 @@ template <int StoreMode> EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; internal::pstore<CoeffReturnType, PacketReturnType>(values, x); @@ -256,6 +448,14 @@ this->coeffRef(index+i) = values[i]; } } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock( + const TensorBlock& block) { + eigen_assert(this->m_impl.data() != NULL); + TensorBlockWriter::Run(block, this->srcCoeff(block.first_coeff_index()), + this->m_inverseShuffle, + this->m_unshuffledInputStrides, this->m_impl.data()); + } };
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h b/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h index 0e89033..e6a666f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h
@@ -31,12 +31,12 @@ * * \sa Tensor */ -template<typename T, typename Dimensions, int Options_> class TensorStorage; +template<typename T, typename Dimensions, int Options> class TensorStorage; // Pure fixed-size storage -template<typename T, int Options_, typename FixedDimensions> -class TensorStorage<T, FixedDimensions, Options_> +template<typename T, typename FixedDimensions, int Options_> +class TensorStorage { private: static const std::size_t Size = FixedDimensions::total_size; @@ -66,7 +66,7 @@ // pure dynamic -template<typename T, int Options_, typename IndexType, int NumIndices_> +template<typename T, typename IndexType, int NumIndices_, int Options_> class TensorStorage<T, DSizes<IndexType, NumIndices_>, Options_> { public: @@ -85,7 +85,7 @@ : m_data(internal::conditional_aligned_new_auto<T,(Options_&DontAlign)==0>(size)), m_dimensions(dimensions) { EIGEN_INTERNAL_TENSOR_STORAGE_CTOR_PLUGIN } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES template <typename... DenseIndex> EIGEN_DEVICE_FUNC TensorStorage(DenseIndex... indices) : m_dimensions(indices...) { m_data = internal::conditional_aligned_new_auto<T,(Options_&DontAlign)==0>(internal::array_prod(m_dimensions)); @@ -126,7 +126,7 @@ } else m_data = 0; - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN + EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_dimensions = nbDimensions; }
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h b/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h index 23248c6..221dc96 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h
@@ -31,6 +31,7 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; }; template<typename Strides, typename XprType> @@ -106,22 +107,24 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device) + : m_impl(op.expression(), device), m_strides(op.strides()) { m_dimensions = m_impl.dimensions(); for (int i = 0; i < NumDims; ++i) { - m_dimensions[i] = ceilf(static_cast<float>(m_dimensions[i]) / op.strides()[i]); + m_dimensions[i] =Eigen::numext::ceil(static_cast<float>(m_dimensions[i]) / op.strides()[i]); } const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); @@ -164,7 +167,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); Index inputIndices[] = {0, 0}; @@ -209,14 +212,25 @@ } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { - const double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() + - 2 * TensorOpCost::MulCost<Index>() + - TensorOpCost::DivCost<Index>()); - return m_impl.costPerCoeff(vectorized) + - TensorOpCost(0, 0, compute_cost, false /* vectorized */, PacketSize); + double compute_cost = (NumDims - 1) * (TensorOpCost::AddCost<Index>() + + TensorOpCost::MulCost<Index>() + + TensorOpCost::DivCost<Index>()) + + TensorOpCost::MulCost<Index>(); + if (vectorized) { + compute_cost *= 2; // packet() computes two indices + } + const int innerDim = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : (NumDims - 1); + return m_impl.costPerCoeff(vectorized && m_inputStrides[innerDim] == 1) + + // Computation is not vectorized per se, but it is done once per packet. + TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } + + /// required by sycl in order to extract the accessor + const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } + /// required by sycl in order to extract the accessor + Strides functor() const { return m_strides; } protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const @@ -244,9 +258,9 @@ array<Index, NumDims> m_outputStrides; array<Index, NumDims> m_inputStrides; TensorEvaluator<ArgType, Device> m_impl; + const Strides m_strides; }; - // Eval as lvalue template<typename Strides, typename ArgType, typename Device> struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device> @@ -261,6 +275,8 @@ enum { IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented RawAccess = false @@ -273,17 +289,22 @@ typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return this->m_impl.coeffRef(this->srcCoeff(index)); } + /// required by sycl in order to extract the accessor + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorEvaluator<ArgType, Device>& impl() const { return this->m_impl; } + /// required by sycl in order to extract the accessor + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Strides functor() const { return this->m_strides; } + template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < this->dimensions().TotalSize()); Index inputIndices[] = {0, 0};
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSycl.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSycl.h new file mode 100644 index 0000000..7b8bd2d --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSycl.h
@@ -0,0 +1,120 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: eigen@codeplay.com +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// General include header of SYCL target for Tensor Module +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_H +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_H + +#ifdef EIGEN_USE_SYCL + +// global pointer to set different attribute state for a class +template <class T> +struct MakeGlobalPointer { + typedef typename cl::sycl::global_ptr<T>::pointer_t Type; + typedef typename cl::sycl::global_ptr<T>::reference_t RefType; +}; + +// global pointer to set different attribute state for a class +template <class T> +struct MakeLocalPointer { + typedef typename cl::sycl::local_ptr<T>::pointer_t Type; + typedef typename cl::sycl::local_ptr<T>::reference_t RefType; +}; + + +namespace Eigen { + template<typename StrideDims, typename XprType> class TensorTupleReducerDeviceOp; + template<typename StrideDims, typename ArgType> struct TensorEvaluator<const TensorTupleReducerDeviceOp<StrideDims, ArgType>, SyclKernelDevice>; +namespace internal { + +#ifdef __SYCL_DEVICE_ONLY__ +template<typename A, typename B> struct TypeConversion { + template<typename T> + static typename MakeGlobalPointer<A>::Type get_address_space_pointer(typename MakeGlobalPointer<T>::Type p); + template<typename T> + static typename MakeLocalPointer<A>::Type get_address_space_pointer(typename MakeLocalPointer<T>::Type p); + + template<typename T> + static A* get_address_space_pointer(T* p); + typedef decltype(get_address_space_pointer(B())) type; +}; + +#endif +} +namespace TensorSycl { +namespace internal { + + template<typename CoeffReturnType, typename OP, typename OutputAccessor, typename InputAccessor, typename LocalAccessor> struct GenericKernelReducer; +/// This struct is used for special expression nodes with no operations (for example assign and selectOP). + struct NoOP; + +template<bool IsConst, typename T> struct GetType{ + typedef const T Type; +}; +template<typename T> struct GetType<false, T>{ + typedef T Type; +}; + +template <bool Conds, size_t X , size_t Y > struct ValueCondition { + static constexpr size_t Res =X; +}; +template<size_t X, size_t Y> struct ValueCondition<false, X, Y> { + static constexpr size_t Res =Y; +}; + +} +} +} + +// tuple construction +#include "TensorSyclTuple.h" + +// counting number of leaf at compile time +#include "TensorSyclLeafCount.h" + +// The index PlaceHolder takes the actual expression and replaces the actual +// data on it with the place holder. It uses the same pre-order expression tree +// traverse as the leaf count in order to give the right access number to each +// node in the expression +#include "TensorSyclPlaceHolderExpr.h" + +// creation of an accessor tuple from a tuple of SYCL buffers +#include "TensorSyclExtractAccessor.h" + +// this is used to change the address space type in tensor map for GPU +#include "TensorSyclConvertToDeviceExpression.h" + +// this is used to extract the functors +#include "TensorSyclExtractFunctors.h" + +// this is used to create tensormap on the device +// this is used to construct the expression on the device +#include "TensorSyclExprConstructor.h" + +/// this is used for extracting tensor reduction +#include "TensorReductionSycl.h" + +// TensorArgMaxSycl.h +#include "TensorArgMaxSycl.h" + +/// this is used for extracting tensor convolution +#include "TensorConvolutionSycl.h" + +// kernel execution using fusion +#include "TensorSyclRun.h" +//sycl functors +#include "TensorSyclFunctors.h" + +#include "TensorContractionSycl.h" + +#endif // end of EIGEN_USE_SYCL +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclConvertToDeviceExpression.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclConvertToDeviceExpression.h new file mode 100644 index 0000000..d6ac7b9 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclConvertToDeviceExpression.h
@@ -0,0 +1,205 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclConvertToDeviceExpression.h + * + * \brief: + * Conversion from host pointer to device pointer + * inside leaf nodes of the expression. + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_CONVERT_TO_DEVICE_EXPRESSION_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_CONVERT_TO_DEVICE_EXPRESSION_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { + +/// \struct ConvertToDeviceExpression +/// \brief This struct is used to convert the MakePointer in the host expression +/// to the MakeGlobalPointer for the device expression. For the leafNodes +/// containing the pointer. This is due to the fact that the address space of +/// the pointer T* is different on the host and the device. +template <typename Expr> +struct ConvertToDeviceExpression; + +template<template<class...> class NonOpCategory, bool IsConst, typename... Args> +struct NonOpConversion{ + typedef typename GetType<IsConst, NonOpCategory<typename ConvertToDeviceExpression<Args>::Type...> >::Type Type; +}; + + +template<template<class, template <class> class > class NonOpCategory, bool IsConst, typename Args> +struct DeviceConvertor{ + typedef typename GetType<IsConst, NonOpCategory<typename ConvertToDeviceExpression<Args>::Type, MakeGlobalPointer> >::Type Type; +}; + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node +/// type is TensorMap +#define TENSORMAPCONVERT(CVQual)\ +template <typename T, int Options_, template <class> class MakePointer_>\ +struct ConvertToDeviceExpression<CVQual TensorMap<T, Options_, MakePointer_> > {\ + typedef CVQual TensorMap<T, Options_, MakeGlobalPointer> Type;\ +}; + +TENSORMAPCONVERT(const) +TENSORMAPCONVERT() +#undef TENSORMAPCONVERT + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node +/// type is TensorCwiseNullaryOp, TensorCwiseUnaryOp, TensorCwiseBinaryOp, TensorCwiseTernaryOp, TensorBroadcastingOp +#define CATEGORYCONVERT(CVQual)\ +template <template<class, class...> class Category, typename OP, typename... subExprs>\ +struct ConvertToDeviceExpression<CVQual Category<OP, subExprs...> > {\ + typedef CVQual Category<OP, typename ConvertToDeviceExpression<subExprs>::Type... > Type;\ +}; +CATEGORYCONVERT(const) +CATEGORYCONVERT() +#undef CATEGORYCONVERT + + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node +/// type is TensorCwiseSelectOp +#define SELECTOPCONVERT(CVQual, Res)\ +template <typename IfExpr, typename ThenExpr, typename ElseExpr>\ +struct ConvertToDeviceExpression<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr> >\ +: NonOpConversion<TensorSelectOp, Res, IfExpr, ThenExpr, ElseExpr> {}; +SELECTOPCONVERT(const, true) +SELECTOPCONVERT(, false) +#undef SELECTOPCONVERT + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node +/// type is const AssingOP +#define ASSIGNCONVERT(CVQual, Res)\ +template <typename LHSExpr, typename RHSExpr>\ +struct ConvertToDeviceExpression<CVQual TensorAssignOp<LHSExpr, RHSExpr> >\ +: NonOpConversion<TensorAssignOp, Res, LHSExpr, RHSExpr>{}; + +ASSIGNCONVERT(const, true) +ASSIGNCONVERT(, false) +#undef ASSIGNCONVERT + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node +/// type is TensorEvalToOp +#define KERNELBROKERCONVERT(CVQual, Res, ExprNode)\ +template <typename Expr>\ +struct ConvertToDeviceExpression<CVQual ExprNode<Expr> > \ +: DeviceConvertor<ExprNode, Res, Expr>{}; + + +KERNELBROKERCONVERT(const, true, TensorEvalToOp) +KERNELBROKERCONVERT(, false, TensorEvalToOp) +#undef KERNELBROKERCONVERT + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node types are TensorForcedEvalOp and TensorLayoutSwapOp +#define KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(CVQual, ExprNode)\ +template <typename Expr>\ +struct ConvertToDeviceExpression<CVQual ExprNode<Expr> > {\ + typedef CVQual ExprNode< typename ConvertToDeviceExpression<Expr>::Type> Type;\ +}; + + +// TensorForcedEvalOp +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(const,TensorForcedEvalOp) +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(,TensorForcedEvalOp) + +// TensorLayoutSwapOp +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(const,TensorLayoutSwapOp) +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(,TensorLayoutSwapOp) + +//TensorIndexTupleOp +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(const,TensorIndexTupleOp) +KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP(,TensorIndexTupleOp) +#undef KERNELBROKERCONVERTFORCEDEVALLAYOUTSWAPINDEXTUPLEOP + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node type is TensorReductionOp +#define KERNELBROKERCONVERTREDUCTION(CVQual)\ +template <typename OP, typename Dim, typename subExpr, template <class> class MakePointer_>\ +struct ConvertToDeviceExpression<CVQual TensorReductionOp<OP, Dim, subExpr, MakePointer_> > {\ + typedef CVQual TensorReductionOp<OP, Dim, typename ConvertToDeviceExpression<subExpr>::Type, MakeGlobalPointer> Type;\ +}; + +KERNELBROKERCONVERTREDUCTION(const) +KERNELBROKERCONVERTREDUCTION() +#undef KERNELBROKERCONVERTREDUCTION + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node type is TensorReductionOp +#define KERNELBROKERCONVERTTUPLEREDUCTION(CVQual)\ +template <typename OP, typename Dim, typename subExpr>\ +struct ConvertToDeviceExpression<CVQual TensorTupleReducerOp<OP, Dim, subExpr> > {\ + typedef CVQual TensorTupleReducerOp<OP, Dim, typename ConvertToDeviceExpression<subExpr>::Type> Type;\ +}; + +KERNELBROKERCONVERTTUPLEREDUCTION(const) +KERNELBROKERCONVERTTUPLEREDUCTION() +#undef KERNELBROKERCONVERTTUPLEREDUCTION + +//TensorSlicingOp +#define KERNELBROKERCONVERTSLICEOP(CVQual)\ +template<typename StartIndices, typename Sizes, typename XprType>\ +struct ConvertToDeviceExpression<CVQual TensorSlicingOp <StartIndices, Sizes, XprType> >{\ + typedef CVQual TensorSlicingOp<StartIndices, Sizes, typename ConvertToDeviceExpression<XprType>::Type> Type;\ +}; + +KERNELBROKERCONVERTSLICEOP(const) +KERNELBROKERCONVERTSLICEOP() +#undef KERNELBROKERCONVERTSLICEOP + +//TensorStridingSlicingOp +#define KERNELBROKERCONVERTERSLICESTRIDEOP(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType>\ +struct ConvertToDeviceExpression<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> >{\ + typedef CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, typename ConvertToDeviceExpression<XprType>::Type> Type;\ +}; + +KERNELBROKERCONVERTERSLICESTRIDEOP(const) +KERNELBROKERCONVERTERSLICESTRIDEOP() +#undef KERNELBROKERCONVERTERSLICESTRIDEOP + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node type is TensorChippingOp +#define KERNELBROKERCONVERTCHIPPINGOP(CVQual)\ +template <DenseIndex DimId, typename Expr>\ +struct ConvertToDeviceExpression<CVQual TensorChippingOp<DimId, Expr> > {\ + typedef CVQual TensorChippingOp<DimId, typename ConvertToDeviceExpression<Expr>::Type> Type;\ +}; +KERNELBROKERCONVERTCHIPPINGOP(const) +KERNELBROKERCONVERTCHIPPINGOP() +#undef KERNELBROKERCONVERTCHIPPINGOP + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node type is TensorImagePatchOp +#define KERNELBROKERCONVERTIMAGEPATCHOP(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename XprType>\ +struct ConvertToDeviceExpression<CVQual TensorImagePatchOp<Rows, Cols, XprType> >{\ + typedef CVQual TensorImagePatchOp<Rows, Cols, typename ConvertToDeviceExpression<XprType>::Type> Type;\ +}; +KERNELBROKERCONVERTIMAGEPATCHOP(const) +KERNELBROKERCONVERTIMAGEPATCHOP() +#undef KERNELBROKERCONVERTIMAGEPATCHOP + + +/// specialisation of the \ref ConvertToDeviceExpression struct when the node type is TensorVolumePatchOp +#define KERNELBROKERCONVERTVOLUMEPATCHOP(CVQual)\ +template<DenseIndex Plannes, DenseIndex Rows, DenseIndex Cols, typename XprType>\ +struct ConvertToDeviceExpression<CVQual TensorVolumePatchOp<Plannes, Rows, Cols, XprType> >{\ + typedef CVQual TensorVolumePatchOp<Plannes, Rows, Cols, typename ConvertToDeviceExpression<XprType>::Type> Type;\ +}; +KERNELBROKERCONVERTVOLUMEPATCHOP(const) +KERNELBROKERCONVERTVOLUMEPATCHOP() +#undef KERNELBROKERCONVERTVOLUMEPATCHOP + +} // namespace internal +} // namespace TensorSycl +} // namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX1
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExprConstructor.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExprConstructor.h new file mode 100644 index 0000000..67003da --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExprConstructor.h
@@ -0,0 +1,514 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclExprConstructor.h + * + * \brief: + * This file re-create an expression on the SYCL device in order + * to use the original tensor evaluator. + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXPR_CONSTRUCTOR_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXPR_CONSTRUCTOR_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { + +template <typename Expr, typename Dims> +struct DeviceFixedSizeTensor; + +template <typename Expr, typename std::ptrdiff_t... Indices> +struct DeviceFixedSizeTensor<Expr, Eigen::Sizes<Indices...>>{ + template<typename Data> + static EIGEN_ALWAYS_INLINE Expr instantiate(Data& dt) {return Expr(ConvertToActualTypeSycl(typename Expr::Scalar, dt), Indices...);} +}; +/// this class is used by EvalToOp in order to create an lhs expression which is +/// a pointer from an accessor on device-only buffer +template <typename PtrType, size_t N, typename... Params> +struct EvalToLHSConstructor { + PtrType expr; + EvalToLHSConstructor(const utility::tuple::Tuple<Params...> &t) : expr(ConvertToActualTypeSycl(typename Eigen::internal::remove_all<PtrType>::type, utility::tuple::get<N>(t))) {} +}; + +/// struct ExprConstructor is used to reconstruct the expression on the device and +/// recreate the expression with MakeGlobalPointer containing the device address +/// space for the TensorMap pointers used in eval function. +/// It receives the original expression type, the functor of the node, the tuple +/// of accessors, and the device expression type to re-instantiate the +/// expression tree for the device +template <typename OrigExpr, typename IndexExpr, typename... Params> +struct ExprConstructor; + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorMap +#define TENSORMAP(CVQual)\ +template <typename T, int Options_,\ +template <class> class MakePointer_, size_t N, typename... Params>\ +struct ExprConstructor< CVQual TensorMap<T, Options_, MakeGlobalPointer>,\ +CVQual PlaceHolder<CVQual TensorMap<T, Options_, MakePointer_>, N>, Params...>{\ + typedef CVQual TensorMap<T, Options_, MakeGlobalPointer> Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + : expr(Type(ConvertToActualTypeSycl(typename Type::Scalar, utility::tuple::get<N>(t)), fd.dimensions())){}\ +}; + +TENSORMAP(const) +TENSORMAP() +#undef TENSORMAP + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorMap +#define TENSORMAPFIXEDSIZE(CVQual)\ +template <typename Scalar_, typename Dimensions_, int Options_2, typename IndexType, int Options_,\ +template <class> class MakePointer_, size_t N, typename... Params>\ +struct ExprConstructor< CVQual TensorMap<TensorFixedSize<Scalar_, Dimensions_, Options_2, IndexType>, Options_, MakeGlobalPointer>,\ +CVQual PlaceHolder<CVQual TensorMap<TensorFixedSize<Scalar_, Dimensions_, Options_2, IndexType>, Options_, MakePointer_>, N>, Params...>{\ + typedef CVQual TensorMap<TensorFixedSize<Scalar_, Dimensions_, Options_2, IndexType>, Options_, MakeGlobalPointer> Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &, const utility::tuple::Tuple<Params...> &t)\ + : expr(DeviceFixedSizeTensor<Type,Dimensions_>::instantiate(utility::tuple::get<N>(t))){}\ +}; + +TENSORMAPFIXEDSIZE(const) +TENSORMAPFIXEDSIZE() +#undef TENSORMAPFIXEDSIZE + +#define UNARYCATEGORY(CVQual)\ +template <template<class, class> class UnaryCategory, typename OP, typename OrigRHSExpr, typename RHSExpr, typename... Params>\ +struct ExprConstructor<CVQual UnaryCategory<OP, OrigRHSExpr>, CVQual UnaryCategory<OP, RHSExpr>, Params...> {\ + typedef ExprConstructor<OrigRHSExpr, RHSExpr, Params...> my_type;\ + my_type rhsExpr;\ + typedef CVQual UnaryCategory<OP, typename my_type::Type> Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : rhsExpr(funcD.rhsExpr, t), expr(rhsExpr.expr, funcD.func) {}\ +}; + +UNARYCATEGORY(const) +UNARYCATEGORY() +#undef UNARYCATEGORY + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorBinaryOp +#define BINARYCATEGORY(CVQual)\ +template <template<class, class, class> class BinaryCategory, typename OP, typename OrigLHSExpr, typename OrigRHSExpr, typename LHSExpr,\ +typename RHSExpr, typename... Params>\ +struct ExprConstructor<CVQual BinaryCategory<OP, OrigLHSExpr, OrigRHSExpr>, CVQual BinaryCategory<OP, LHSExpr, RHSExpr>, Params...> {\ + typedef ExprConstructor<OrigLHSExpr, LHSExpr, Params...> my_left_type;\ + typedef ExprConstructor<OrigRHSExpr, RHSExpr, Params...> my_right_type;\ + typedef CVQual BinaryCategory<OP, typename my_left_type::Type, typename my_right_type::Type> Type;\ + my_left_type lhsExpr;\ + my_right_type rhsExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : lhsExpr(funcD.lhsExpr, t),rhsExpr(funcD.rhsExpr, t), expr(lhsExpr.expr, rhsExpr.expr, funcD.func) {}\ +}; + +BINARYCATEGORY(const) +BINARYCATEGORY() +#undef BINARYCATEGORY + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorCwiseTernaryOp +#define TERNARYCATEGORY(CVQual)\ +template <template <class, class, class, class> class TernaryCategory, typename OP, typename OrigArg1Expr, typename OrigArg2Expr,typename OrigArg3Expr,\ +typename Arg1Expr, typename Arg2Expr, typename Arg3Expr, typename... Params>\ +struct ExprConstructor<CVQual TernaryCategory<OP, OrigArg1Expr, OrigArg2Expr, OrigArg3Expr>, CVQual TernaryCategory<OP, Arg1Expr, Arg2Expr, Arg3Expr>, Params...> {\ + typedef ExprConstructor<OrigArg1Expr, Arg1Expr, Params...> my_arg1_type;\ + typedef ExprConstructor<OrigArg2Expr, Arg2Expr, Params...> my_arg2_type;\ + typedef ExprConstructor<OrigArg3Expr, Arg3Expr, Params...> my_arg3_type;\ + typedef CVQual TernaryCategory<OP, typename my_arg1_type::Type, typename my_arg2_type::Type, typename my_arg3_type::Type> Type;\ + my_arg1_type arg1Expr;\ + my_arg2_type arg2Expr;\ + my_arg3_type arg3Expr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD,const utility::tuple::Tuple<Params...> &t)\ + : arg1Expr(funcD.arg1Expr, t), arg2Expr(funcD.arg2Expr, t), arg3Expr(funcD.arg3Expr, t), expr(arg1Expr.expr, arg2Expr.expr, arg3Expr.expr, funcD.func) {}\ +}; + +TERNARYCATEGORY(const) +TERNARYCATEGORY() +#undef TERNARYCATEGORY + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorCwiseSelectOp +#define SELECTOP(CVQual)\ +template <typename OrigIfExpr, typename OrigThenExpr, typename OrigElseExpr, typename IfExpr, typename ThenExpr, typename ElseExpr, typename... Params>\ +struct ExprConstructor< CVQual TensorSelectOp<OrigIfExpr, OrigThenExpr, OrigElseExpr>, CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, Params...> {\ + typedef ExprConstructor<OrigIfExpr, IfExpr, Params...> my_if_type;\ + typedef ExprConstructor<OrigThenExpr, ThenExpr, Params...> my_then_type;\ + typedef ExprConstructor<OrigElseExpr, ElseExpr, Params...> my_else_type;\ + typedef CVQual TensorSelectOp<typename my_if_type::Type, typename my_then_type::Type, typename my_else_type::Type> Type;\ + my_if_type ifExpr;\ + my_then_type thenExpr;\ + my_else_type elseExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : ifExpr(funcD.ifExpr, t), thenExpr(funcD.thenExpr, t), elseExpr(funcD.elseExpr, t), expr(ifExpr.expr, thenExpr.expr, elseExpr.expr) {}\ +}; + +SELECTOP(const) +SELECTOP() +#undef SELECTOP + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// const TensorAssignOp +#define ASSIGN(CVQual)\ +template <typename OrigLHSExpr, typename OrigRHSExpr, typename LHSExpr, typename RHSExpr, typename... Params>\ +struct ExprConstructor<CVQual TensorAssignOp<OrigLHSExpr, OrigRHSExpr>, CVQual TensorAssignOp<LHSExpr, RHSExpr>, Params...> {\ + typedef ExprConstructor<OrigLHSExpr, LHSExpr, Params...> my_left_type;\ + typedef ExprConstructor<OrigRHSExpr, RHSExpr, Params...> my_right_type;\ + typedef CVQual TensorAssignOp<typename my_left_type::Type, typename my_right_type::Type> Type;\ + my_left_type lhsExpr;\ + my_right_type rhsExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : lhsExpr(funcD.lhsExpr, t), rhsExpr(funcD.rhsExpr, t), expr(lhsExpr.expr, rhsExpr.expr) {}\ + }; + + ASSIGN(const) + ASSIGN() + #undef ASSIGN + + /// specialisation of the \ref ExprConstructor struct when the node type is + /// const TensorAssignOp + #define CONVERSIONEXPRCONST(CVQual)\ + template <typename OrigNestedExpr, typename ConvertType, typename NestedExpr, typename... Params>\ + struct ExprConstructor<CVQual TensorConversionOp<ConvertType, OrigNestedExpr>, CVQual TensorConversionOp<ConvertType, NestedExpr>, Params...> {\ + typedef ExprConstructor<OrigNestedExpr, NestedExpr, Params...> my_nested_type;\ + typedef CVQual TensorConversionOp<ConvertType, typename my_nested_type::Type> Type;\ + my_nested_type nestedExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : nestedExpr(funcD.subExpr, t), expr(nestedExpr.expr) {}\ + }; + + CONVERSIONEXPRCONST(const) + CONVERSIONEXPRCONST() + #undef CONVERSIONEXPRCONST + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorEvalToOp /// 0 here is the output number in the buffer +#define EVALTO(CVQual)\ +template <typename OrigExpr, typename Expr, typename... Params>\ +struct ExprConstructor<CVQual TensorEvalToOp<OrigExpr, MakeGlobalPointer>, CVQual TensorEvalToOp<Expr>, Params...> {\ + typedef ExprConstructor<OrigExpr, Expr, Params...> my_expr_type;\ + typedef typename TensorEvalToOp<OrigExpr, MakeGlobalPointer>::PointerType my_buffer_type;\ + typedef CVQual TensorEvalToOp<typename my_expr_type::Type, MakeGlobalPointer> Type;\ + my_expr_type nestedExpression;\ + EvalToLHSConstructor<my_buffer_type, 0, Params...> buffer;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : nestedExpression(funcD.xprExpr, t), buffer(t), expr(buffer.expr, nestedExpression.expr) {}\ +}; + +EVALTO(const) +EVALTO() +#undef EVALTO + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorForcedEvalOp +#define FORCEDEVAL(CVQual)\ +template <typename OrigExpr, typename DevExpr, size_t N, typename... Params>\ +struct ExprConstructor<CVQual TensorForcedEvalOp<OrigExpr>,\ +CVQual PlaceHolder<CVQual TensorForcedEvalOp<DevExpr>, N>, Params...> {\ + typedef TensorForcedEvalOp<OrigExpr> XprType;\ + typedef CVQual TensorMap<\ + Tensor<typename XprType::Scalar,XprType::NumDimensions, Eigen::internal::traits<XprType>::Layout,typename XprType::Index>,\ + Eigen::internal::traits<XprType>::Layout, \ + MakeGlobalPointer\ + > Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + : expr(Type(ConvertToActualTypeSycl(typename Type::Scalar, utility::tuple::get<N>(t)), fd.dimensions())) {}\ +}; + +FORCEDEVAL(const) +FORCEDEVAL() +#undef FORCEDEVAL + +#define TENSORCUSTOMUNARYOP(CVQual)\ +template <typename CustomUnaryFunc, typename OrigExpr, typename DevExpr, size_t N, typename... Params>\ +struct ExprConstructor<CVQual TensorCustomUnaryOp<CustomUnaryFunc, OrigExpr>,\ +CVQual PlaceHolder<CVQual TensorCustomUnaryOp<CustomUnaryFunc, DevExpr>, N>, Params...> {\ + typedef TensorCustomUnaryOp<CustomUnaryFunc, OrigExpr> XprType;\ + typedef CVQual TensorMap<\ + Tensor<typename XprType::Scalar,XprType::NumDimensions, Eigen::internal::traits<XprType>::Layout,typename XprType::Index>,\ + Eigen::internal::traits<XprType>::Layout, \ + MakeGlobalPointer\ + > Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + : expr(Type(ConvertToActualTypeSycl(typename Type::Scalar, utility::tuple::get<N>(t)), fd.dimensions())) {}\ +}; + +TENSORCUSTOMUNARYOP(const) +TENSORCUSTOMUNARYOP() +#undef TENSORCUSTOMUNARYOP + +/// specialisation of the \ref ExprConstructor struct when the node type is TensorReductionOp +#define SYCLREDUCTIONEXPR(CVQual)\ +template <typename OP, typename Dim, typename OrigExpr, typename DevExpr, size_t N, typename... Params>\ +struct ExprConstructor<CVQual TensorReductionOp<OP, Dim, OrigExpr, MakeGlobalPointer>,\ +CVQual PlaceHolder<CVQual TensorReductionOp<OP, Dim, DevExpr>, N>, Params...> {\ + static const auto NumIndices= ValueCondition< TensorReductionOp<OP, Dim, DevExpr, MakeGlobalPointer>::NumDimensions==0, 1, TensorReductionOp<OP, Dim, DevExpr, MakeGlobalPointer>::NumDimensions >::Res;\ + typedef CVQual TensorMap<Tensor<typename TensorReductionOp<OP, Dim, DevExpr, MakeGlobalPointer>::Scalar,\ + NumIndices, Eigen::internal::traits<TensorReductionOp<OP, Dim, DevExpr, MakeGlobalPointer>>::Layout, typename TensorReductionOp<OP, Dim, DevExpr>::Index>, Eigen::internal::traits<TensorReductionOp<OP, Dim, DevExpr, MakeGlobalPointer>>::Layout, MakeGlobalPointer> Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + :expr(Type(ConvertToActualTypeSycl(typename Type::Scalar, utility::tuple::get<N>(t)), fd.dimensions())) {}\ +}; + +SYCLREDUCTIONEXPR(const) +SYCLREDUCTIONEXPR() +#undef SYCLREDUCTIONEXPR + +/// specialisation of the \ref ExprConstructor struct when the node type is TensorTupleReducerOp +/// use reductionOp instead of the TensorTupleReducerOp in order to build the tensor map. Because the tensorMap is the output of Tensor ReductionOP. +#define SYCLTUPLEREDUCTIONEXPR(CVQual)\ +template <typename OP, typename Dim, typename OrigExpr, typename DevExpr, size_t N, typename... Params>\ +struct ExprConstructor<CVQual TensorTupleReducerOp<OP, Dim, OrigExpr>,\ +CVQual PlaceHolder<CVQual TensorTupleReducerOp<OP, Dim, DevExpr>, N>, Params...> {\ + static const auto NumRedDims= TensorReductionOp<OP, Dim, const TensorIndexTupleOp<OrigExpr> , MakeGlobalPointer>::NumDimensions;\ + static const auto NumIndices= ValueCondition<NumRedDims==0, 1, NumRedDims>::Res;\ +static const int Layout =static_cast<int>(Eigen::internal::traits<TensorReductionOp<OP, Dim, const TensorIndexTupleOp<OrigExpr>, MakeGlobalPointer>>::Layout);\ + typedef CVQual TensorMap<\ + Tensor<typename TensorIndexTupleOp<OrigExpr>::CoeffReturnType,NumIndices, Layout, typename TensorTupleReducerOp<OP, Dim, OrigExpr>::Index>,\ + Layout,\ + MakeGlobalPointer\ + > XprType;\ + typedef typename TensorEvaluator<const TensorIndexTupleOp<OrigExpr> , SyclKernelDevice>::Dimensions InputDimensions;\ + static const int NumDims = Eigen::internal::array_size<InputDimensions>::value;\ + typedef array<Index, NumDims> StrideDims;\ + typedef const TensorTupleReducerDeviceOp<StrideDims, XprType> Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + :expr(Type(XprType(ConvertToActualTypeSycl(typename XprType::CoeffReturnType, utility::tuple::get<N>(t)), fd.dimensions()),\ + fd.return_dim(), fd.strides(), fd.stride_mod(), fd.stride_div())) {\ + }\ +}; + +SYCLTUPLEREDUCTIONEXPR(const) +SYCLTUPLEREDUCTIONEXPR() +#undef SYCLTUPLEREDUCTIONEXPR + +/// specialisation of the \ref ExprConstructor struct when the node type is +/// TensorContractionOp, TensorConvolutionOp TensorCustomBinaryOp +#define SYCLCONTRACTCONVCUSBIOPS(CVQual, ExprNode)\ +template <typename Indices, typename OrigLhsXprType, typename OrigRhsXprType, typename LhsXprType, typename RhsXprType, size_t N, typename... Params>\ +struct ExprConstructor<CVQual ExprNode<Indices, OrigLhsXprType, OrigRhsXprType>,\ +CVQual PlaceHolder<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, N>, Params...> {\ + typedef ExprNode<Indices, OrigLhsXprType, OrigRhsXprType> XprTyp;\ + static const auto NumIndices= Eigen::internal::traits<XprTyp>::NumDimensions;\ + typedef CVQual TensorMap<\ + Tensor<typename XprTyp::Scalar,NumIndices, Eigen::internal::traits<XprTyp>::Layout, typename XprTyp::Index>,\ + Eigen::internal::traits<XprTyp>::Layout, \ + MakeGlobalPointer\ + > Type;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &fd, const utility::tuple::Tuple<Params...> &t)\ + :expr(Type(ConvertToActualTypeSycl(typename Type::Scalar, utility::tuple::get<N>(t)), fd.dimensions())) {}\ +}; + +//TensorContractionOp +SYCLCONTRACTCONVCUSBIOPS(const, TensorContractionOp) +SYCLCONTRACTCONVCUSBIOPS(, TensorContractionOp) +//TensorConvolutionOp +SYCLCONTRACTCONVCUSBIOPS(const, TensorConvolutionOp) +SYCLCONTRACTCONVCUSBIOPS(, TensorConvolutionOp) +//TensorCustomBinaryOp +SYCLCONTRACTCONVCUSBIOPS(const, TensorCustomBinaryOp) +SYCLCONTRACTCONVCUSBIOPS(, TensorCustomBinaryOp) +#undef SYCLCONTRACTCONVCUSBIOPS + +//TensorSlicingOp +#define SYCLSLICEOPEXPR(CVQual)\ +template<typename StartIndices, typename Sizes, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual TensorSlicingOp <StartIndices, Sizes, OrigXprType> , CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual TensorSlicingOp<StartIndices, Sizes, typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.startIndices(), funcD.dimensions()) {}\ +}; + +SYCLSLICEOPEXPR(const) +SYCLSLICEOPEXPR() +#undef SYCLSLICEOPEXPR + +//TensorStridingSlicingOp +#define SYCLSLICESTRIDEOPEXPR(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, OrigXprType>, CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.startIndices(), funcD.stopIndices(),funcD.strides()) {}\ +}; + +SYCLSLICESTRIDEOPEXPR(const) +SYCLSLICESTRIDEOPEXPR() +#undef SYCLSLICESTRIDEOPEXPR + +//TensorReshapingOp and TensorShufflingOp +#define SYCLRESHAPEANDSHUFFLEOPEXPRCONST(OPEXPR, CVQual)\ +template<typename Param, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual OPEXPR <Param, OrigXprType> , CVQual OPEXPR <Param, XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual OPEXPR <Param, typename my_xpr_type::Type> Type ;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.param()) {}\ +}; + +// TensorReshapingOp +SYCLRESHAPEANDSHUFFLEOPEXPRCONST(TensorReshapingOp, const) +SYCLRESHAPEANDSHUFFLEOPEXPRCONST(TensorReshapingOp, ) +// TensorShufflingOp +SYCLRESHAPEANDSHUFFLEOPEXPRCONST(TensorShufflingOp, const) +SYCLRESHAPEANDSHUFFLEOPEXPRCONST(TensorShufflingOp, ) +#undef SYCLRESHAPEANDSHUFFLEOPEXPRCONST + +//TensorPaddingOp +#define SYCLPADDINGOPEXPRCONST(OPEXPR, CVQual)\ +template<typename Param, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual OPEXPR <Param, OrigXprType> , CVQual OPEXPR <Param, XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual OPEXPR <Param, typename my_xpr_type::Type> Type ;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.param() , funcD.scalar_param()) {}\ +}; + +//TensorPaddingOp +SYCLPADDINGOPEXPRCONST(TensorPaddingOp, const) +SYCLPADDINGOPEXPRCONST(TensorPaddingOp, ) +#undef SYCLPADDINGOPEXPRCONST + +// TensorChippingOp +#define SYCLTENSORCHIPPINGOPEXPR(CVQual)\ +template<DenseIndex DimId, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual TensorChippingOp <DimId, OrigXprType> , CVQual TensorChippingOp<DimId, XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual TensorChippingOp<DimId, typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.offset(), funcD.dimId()) {}\ +}; + +SYCLTENSORCHIPPINGOPEXPR(const) +SYCLTENSORCHIPPINGOPEXPR() +#undef SYCLTENSORCHIPPINGOPEXPR + +// TensorImagePatchOp +#define SYCLTENSORIMAGEPATCHOPEXPR(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual TensorImagePatchOp<Rows, Cols, OrigXprType>, CVQual TensorImagePatchOp<Rows, Cols, XprType>, Params... > {\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual TensorImagePatchOp<Rows, Cols, typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.m_patch_rows, funcD.m_patch_cols, funcD.m_row_strides, funcD.m_col_strides,\ + funcD.m_in_row_strides, funcD.m_in_col_strides, funcD.m_row_inflate_strides, funcD.m_col_inflate_strides, funcD.m_padding_explicit, \ + funcD.m_padding_top, funcD.m_padding_bottom, funcD.m_padding_left, funcD.m_padding_right, funcD.m_padding_type, funcD.m_padding_value){}\ +}; + +SYCLTENSORIMAGEPATCHOPEXPR(const) +SYCLTENSORIMAGEPATCHOPEXPR() +#undef SYCLTENSORIMAGEPATCHOPEXPR + +// TensorVolumePatchOp +#define SYCLTENSORVOLUMEPATCHOPEXPR(CVQual)\ +template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual TensorVolumePatchOp<Planes, Rows, Cols, OrigXprType>, CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType>, Params... > {\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual TensorVolumePatchOp<Planes, Rows, Cols, typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr, funcD.m_patch_planes, funcD.m_patch_rows, funcD.m_patch_cols, funcD.m_plane_strides, funcD.m_row_strides, funcD.m_col_strides,\ + funcD.m_in_plane_strides, funcD.m_in_row_strides, funcD.m_in_col_strides,funcD.m_plane_inflate_strides, funcD.m_row_inflate_strides, funcD.m_col_inflate_strides, \ + funcD.m_padding_explicit, funcD.m_padding_top_z, funcD.m_padding_bottom_z, funcD.m_padding_top, funcD.m_padding_bottom, funcD.m_padding_left, funcD.m_padding_right, \ + funcD.m_padding_type, funcD.m_padding_value ){\ + }\ +}; + +SYCLTENSORVOLUMEPATCHOPEXPR(const) +SYCLTENSORVOLUMEPATCHOPEXPR() +#undef SYCLTENSORVOLUMEPATCHOPEXPR + +// TensorLayoutSwapOp and TensorIndexTupleOp +#define SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR(CVQual, ExprNode)\ +template<typename OrigXprType, typename XprType, typename... Params>\ +struct ExprConstructor<CVQual ExprNode <OrigXprType> , CVQual ExprNode<XprType>, Params... >{\ + typedef ExprConstructor<OrigXprType, XprType, Params...> my_xpr_type;\ + typedef CVQual ExprNode<typename my_xpr_type::Type> Type;\ + my_xpr_type xprExpr;\ + Type expr;\ + template <typename FuncDetector>\ + ExprConstructor(FuncDetector &funcD, const utility::tuple::Tuple<Params...> &t)\ + : xprExpr(funcD.xprExpr, t), expr(xprExpr.expr) {}\ +}; + +//TensorLayoutSwapOp +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR(const, TensorLayoutSwapOp) +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR(, TensorLayoutSwapOp) +//TensorIndexTupleOp +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR(const, TensorIndexTupleOp) +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR(, TensorIndexTupleOp) + +#undef SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXPR + +/// template deduction for \ref ExprConstructor struct +template <typename OrigExpr, typename IndexExpr, typename FuncD, typename... Params> +auto createDeviceExpression(FuncD &funcD, const utility::tuple::Tuple<Params...> &t) + -> decltype(ExprConstructor<OrigExpr, IndexExpr, Params...>(funcD, t)) { + return ExprConstructor<OrigExpr, IndexExpr, Params...>(funcD, t); +} + +} /// namespace TensorSycl +} /// namespace internal +} /// namespace Eigen + + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXPR_CONSTRUCTOR_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractAccessor.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractAccessor.h new file mode 100644 index 0000000..3c74d16 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractAccessor.h
@@ -0,0 +1,310 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclExtractAccessor.h + * + * \brief: + * ExtractAccessor takes Expression placeHolder expression and the tuple of sycl + * buffers as an input. Using pre-order tree traversal, ExtractAccessor + * recursively calls itself for its children in the expression tree. The + * leaf node in the PlaceHolder expression is nothing but a container preserving + * the order of the actual data in the tuple of sycl buffer. By invoking the + * extract accessor for the PlaceHolder<N>, an accessor is created for the Nth + * buffer in the tuple of buffers. This accessor is then added as an Nth + * element in the tuple of accessors. In this case we preserve the order of data + * in the expression tree. + * + * This is the specialisation of extract accessor method for different operation + * type in the PlaceHolder expression. + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_ACCESSOR_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_ACCESSOR_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { +#define RETURN_CPP11(expr) ->decltype(expr) {return expr;} + +/// struct ExtractAccessor: Extract Accessor Class is used to extract the +/// accessor from a buffer. +/// Depending on the type of the leaf node we can get a read accessor or a +/// read_write accessor +template <typename Evaluator> +struct ExtractAccessor; + +struct AccessorConstructor{ + template<typename Arg> static inline auto getTuple(cl::sycl::handler& cgh, const Arg& eval) + RETURN_CPP11(ExtractAccessor<Arg>::getTuple(cgh, eval)) + + template<typename Arg1, typename Arg2> static inline auto getTuple(cl::sycl::handler& cgh, const Arg1& eval1, const Arg2& eval2) + RETURN_CPP11(utility::tuple::append(ExtractAccessor<Arg1>::getTuple(cgh, eval1), ExtractAccessor<Arg2>::getTuple(cgh, eval2))) + + template<typename Arg1, typename Arg2, typename Arg3> static inline auto getTuple(cl::sycl::handler& cgh, const Arg1& eval1 , const Arg2& eval2 , const Arg3& eval3) + RETURN_CPP11(utility::tuple::append(ExtractAccessor<Arg1>::getTuple(cgh, eval1),utility::tuple::append(ExtractAccessor<Arg2>::getTuple(cgh, eval2), ExtractAccessor<Arg3>::getTuple(cgh, eval3)))) + + template< cl::sycl::access::mode AcM, typename Arg> static inline auto getAccessor(cl::sycl::handler& cgh, const Arg& eval) + RETURN_CPP11(utility::tuple::make_tuple(eval.device().template get_sycl_accessor<AcM>(cgh,eval.data()))) +}; + +/// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorCwiseNullaryOp, TensorCwiseUnaryOp and TensorBroadcastingOp +#define SYCLUNARYCATEGORYEXTACC(CVQual)\ +template <template<class, class> class UnaryCategory, typename OP, typename RHSExpr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual UnaryCategory<OP, RHSExpr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual UnaryCategory<OP, RHSExpr>, Dev>& eval)\ +RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLUNARYCATEGORYEXTACC(const) +SYCLUNARYCATEGORYEXTACC() +#undef SYCLUNARYCATEGORYEXTACC + + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorCwiseBinaryOp +#define SYCLBINARYCATEGORYEXTACC(CVQual)\ +template <template<class, class, class> class BinaryCategory, typename OP, typename LHSExpr, typename RHSExpr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual BinaryCategory<OP, LHSExpr, RHSExpr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual BinaryCategory<OP, LHSExpr, RHSExpr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.left_impl(), eval.right_impl()))\ +}; + +SYCLBINARYCATEGORYEXTACC(const) +SYCLBINARYCATEGORYEXTACC() +#undef SYCLBINARYCATEGORYEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is +/// const TensorCwiseTernaryOp +#define SYCLTERNARYCATEGORYEXTACC(CVQual)\ +template <template<class, class, class, class> class TernaryCategory, typename OP, typename Arg1Expr, typename Arg2Expr, typename Arg3Expr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TernaryCategory<OP, Arg1Expr, Arg2Expr, Arg3Expr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TernaryCategory<OP, Arg1Expr, Arg2Expr, Arg3Expr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.arg1Impl(), eval.arg2Impl(), eval.arg3Impl()))\ +}; + +SYCLTERNARYCATEGORYEXTACC(const) +SYCLTERNARYCATEGORYEXTACC() +#undef SYCLTERNARYCATEGORYEXTACC + + +/// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorCwiseSelectOp. This is a special case where there is no OP +#define SYCLSELECTOPEXTACC(CVQual)\ +template <typename IfExpr, typename ThenExpr, typename ElseExpr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.cond_impl(), eval.then_impl(), eval.else_impl()))\ +}; + +SYCLSELECTOPEXTACC(const) +SYCLSELECTOPEXTACC() +#undef SYCLSELECTOPEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorAssignOp +#define SYCLTENSORASSIGNOPEXTACC(CVQual)\ +template <typename LHSExpr, typename RHSExpr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorAssignOp<LHSExpr, RHSExpr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorAssignOp<LHSExpr, RHSExpr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.left_impl(), eval.right_impl()))\ +}; + + SYCLTENSORASSIGNOPEXTACC(const) + SYCLTENSORASSIGNOPEXTACC() + #undef SYCLTENSORASSIGNOPEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is const TensorMap +#define TENSORMAPEXPR(CVQual, ACCType)\ +template <typename PlainObjectType, int Options_, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorMap<PlainObjectType, Options_>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh,const TensorEvaluator<CVQual TensorMap<PlainObjectType, Options_>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<ACCType>(cgh, eval))\ +}; + +TENSORMAPEXPR(const, cl::sycl::access::mode::read) +TENSORMAPEXPR(, cl::sycl::access::mode::read_write) +#undef TENSORMAPEXPR + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorForcedEvalOp +#define SYCLFORCEDEVALEXTACC(CVQual)\ +template <typename Expr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorForcedEvalOp<Expr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorForcedEvalOp<Expr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<cl::sycl::access::mode::read>(cgh, eval))\ +}; + +SYCLFORCEDEVALEXTACC(const) +SYCLFORCEDEVALEXTACC() +#undef SYCLFORCEDEVALEXTACC + +//TensorCustomUnaryOp +#define SYCLCUSTOMUNARYOPEXTACC(CVQual)\ +template <typename CustomUnaryFunc, typename XprType, typename Dev >\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorCustomUnaryOp<CustomUnaryFunc, XprType>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorCustomUnaryOp<CustomUnaryFunc, XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<cl::sycl::access::mode::read>(cgh, eval))\ +}; + + +SYCLCUSTOMUNARYOPEXTACC(const) +SYCLCUSTOMUNARYOPEXTACC() +#undef SYCLCUSTOMUNARYOPEXTACC + +//TensorCustomBinaryOp +#define SYCLCUSTOMBINARYOPEXTACC(CVQual)\ +template <typename CustomBinaryFunc, typename LhsXprType, typename RhsXprType , typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<cl::sycl::access::mode::read>(cgh, eval))\ +}; + +SYCLCUSTOMBINARYOPEXTACC(const) +SYCLCUSTOMBINARYOPEXTACC() +#undef SYCLCUSTOMBIBARYOPEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorEvalToOp +#define SYCLEVALTOEXTACC(CVQual)\ +template <typename Expr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorEvalToOp<Expr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh,const TensorEvaluator<CVQual TensorEvalToOp<Expr>, Dev>& eval)\ + RETURN_CPP11(utility::tuple::append(AccessorConstructor::template getAccessor<cl::sycl::access::mode::write>(cgh, eval), AccessorConstructor::getTuple(cgh, eval.impl())))\ +}; + +SYCLEVALTOEXTACC(const) +SYCLEVALTOEXTACC() +#undef SYCLEVALTOEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorReductionOp +#define SYCLREDUCTIONEXTACC(CVQual, ExprNode)\ +template <typename OP, typename Dim, typename Expr, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual ExprNode<OP, Dim, Expr>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual ExprNode<OP, Dim, Expr>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<cl::sycl::access::mode::read>(cgh, eval))\ +}; +// TensorReductionOp +SYCLREDUCTIONEXTACC(const,TensorReductionOp) +SYCLREDUCTIONEXTACC(,TensorReductionOp) + +// TensorTupleReducerOp +SYCLREDUCTIONEXTACC(const,TensorTupleReducerOp) +SYCLREDUCTIONEXTACC(,TensorTupleReducerOp) +#undef SYCLREDUCTIONEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is TensorContractionOp and TensorConvolutionOp +#define SYCLCONTRACTIONCONVOLUTIONEXTACC(CVQual, ExprNode)\ +template<typename Indices, typename LhsXprType, typename RhsXprType, typename Dev>\ + struct ExtractAccessor<TensorEvaluator<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::template getAccessor<cl::sycl::access::mode::read>(cgh, eval))\ +}; +//TensorContractionOp +SYCLCONTRACTIONCONVOLUTIONEXTACC(const,TensorContractionOp) +SYCLCONTRACTIONCONVOLUTIONEXTACC(,TensorContractionOp) +//TensorConvolutionOp +SYCLCONTRACTIONCONVOLUTIONEXTACC(const,TensorConvolutionOp) +SYCLCONTRACTIONCONVOLUTIONEXTACC(,TensorConvolutionOp) +#undef SYCLCONTRACTIONCONVOLUTIONEXTACC + +/// specialisation of the \ref ExtractAccessor struct when the node type is +/// const TensorSlicingOp. +#define SYCLSLICEOPEXTACC(CVQual)\ +template <typename StartIndices, typename Sizes, typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, Dev> > {\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, Dev>& eval)\ + RETURN_CPP11( AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLSLICEOPEXTACC(const) +SYCLSLICEOPEXTACC() +#undef SYCLSLICEOPEXTACC +// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorStridingSlicingOp. +#define SYCLSLICESTRIDEOPEXTACC(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, Dev> >{\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLSLICESTRIDEOPEXTACC(const) +SYCLSLICESTRIDEOPEXTACC() +#undef SYCLSLICESTRIDEOPEXTACC + +// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorChippingOp. +#define SYCLTENSORCHIPPINGOPEXTACC(CVQual)\ +template<DenseIndex DimId, typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorChippingOp<DimId, XprType>, Dev> >{\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorChippingOp<DimId, XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLTENSORCHIPPINGOPEXTACC(const) +SYCLTENSORCHIPPINGOPEXTACC() +#undef SYCLTENSORCHIPPINGOPEXTACC + +// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorImagePatchOp. +#define SYCLTENSORIMAGEPATCHOPEXTACC(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorImagePatchOp<Rows, Cols, XprType>, Dev> >{\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorImagePatchOp<Rows, Cols, XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLTENSORIMAGEPATCHOPEXTACC(const) +SYCLTENSORIMAGEPATCHOPEXTACC() +#undef SYCLTENSORIMAGEPATCHOPEXTACC + +// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorVolumePatchOp. +#define SYCLTENSORVOLUMEPATCHOPEXTACC(CVQual)\ +template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType>, Dev> >{\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +SYCLTENSORVOLUMEPATCHOPEXTACC(const) +SYCLTENSORVOLUMEPATCHOPEXTACC() +#undef SYCLTENSORVOLUMEPATCHOPEXTACC + +// specialisation of the \ref ExtractAccessor struct when the node type is +/// TensorLayoutSwapOp, TensorIndexTupleOp +#define SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC(CVQual, ExprNode)\ +template<typename XprType, typename Dev>\ +struct ExtractAccessor<TensorEvaluator<CVQual ExprNode<XprType>, Dev> >{\ + static inline auto getTuple(cl::sycl::handler& cgh, const TensorEvaluator<CVQual ExprNode<XprType>, Dev>& eval)\ + RETURN_CPP11(AccessorConstructor::getTuple(cgh, eval.impl()))\ +}; + +// TensorLayoutSwapOp +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC(const,TensorLayoutSwapOp) +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC(,TensorLayoutSwapOp) +//TensorIndexTupleOp +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC(const,TensorIndexTupleOp) +SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC(,TensorIndexTupleOp) + +#undef SYCLTENSORLAYOUTSWAPINDEXTUPLEOPEXTACC + +/// template deduction for \ref ExtractAccessor +template <typename Evaluator> +auto createTupleOfAccessors(cl::sycl::handler& cgh, const Evaluator& eval) +-> decltype(ExtractAccessor<Evaluator>::getTuple(cgh, eval)) { + return ExtractAccessor<Evaluator>::getTuple(cgh, eval); +} + +} /// namespace TensorSycl +} /// namespace internal +} /// namespace Eigen +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_ACCESSOR_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractFunctors.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractFunctors.h new file mode 100644 index 0000000..09407e5 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclExtractFunctors.h
@@ -0,0 +1,467 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclextractFunctors.h + * + * \brief: + * Used to extract all the functors allocated to each node of the expression +*tree. + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_FUNCTORS_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_FUNCTORS_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { +/// struct FunctorExtractor: This struct is used to extract the functors +/// constructed on +/// the host-side, to pack them and reuse them in reconstruction of the +/// expression on the device. +/// We have to do that as in Eigen the functors are not stateless so we cannot +/// re-instantiate them on the device. +/// We have to pass instantiated functors to the device. +// This struct is used for leafNode (TensorMap) and nodes behaving like leafNode (TensorForcedEval). +#define DEFALTACTION(Evaluator)\ +typedef typename Evaluator::Dimensions Dimensions;\ +const Dimensions m_dimensions;\ +EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }\ +FunctorExtractor(const Evaluator& expr): m_dimensions(expr.dimensions()) {} + +template <typename Evaluator> struct FunctorExtractor{ + DEFALTACTION(Evaluator) +}; + + +/// specialisation of the \ref FunctorExtractor struct when the node type does not require anything +///TensorConversionOp +#define SYCLEXTRFUNCCONVERSION(ExprNode, CVQual)\ +template <typename ArgType1, typename ArgType2, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual ExprNode<ArgType1, ArgType2>, Dev> > {\ + FunctorExtractor<TensorEvaluator<ArgType2, Dev> > subExpr;\ + FunctorExtractor(const TensorEvaluator<CVQual ExprNode<ArgType1, ArgType2>, Dev>& expr)\ + : subExpr(expr.impl()) {}\ +}; + +SYCLEXTRFUNCCONVERSION(TensorConversionOp, const) +SYCLEXTRFUNCCONVERSION(TensorConversionOp, ) +#undef SYCLEXTRFUNCCONVERSION + +#define SYCLEXTRTENSORMAPFIXEDSIZE(CVQual)\ +template <typename Scalar_, typename Dimensions_, int Options_2, typename IndexType, int Options_, template <class> class MakePointer_, typename Dev>\ +struct FunctorExtractor< TensorEvaluator <CVQual TensorMap<TensorFixedSize<Scalar_, Dimensions_, Options_2, IndexType>, Options_, MakePointer_> , Dev> >{\ +FunctorExtractor(const TensorEvaluator <CVQual TensorMap<TensorFixedSize<Scalar_, Dimensions_, Options_2, IndexType>, Options_, MakePointer_> , Dev>& ){}\ +}; + +SYCLEXTRTENSORMAPFIXEDSIZE(const) +SYCLEXTRTENSORMAPFIXEDSIZE() +#undef SYCLEXTRTENSORMAPFIXEDSIZE + +/// specialisation of the \ref FunctorExtractor struct when the node type is +/// TensorCwiseNullaryOp, TensorCwiseUnaryOp, and TensorBroadcastingOp +#define SYCLEXTRFUNCUNARY(CVQual)\ +template <template <class, class> class UnaryCategory, typename OP, typename RHSExpr, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual UnaryCategory<OP, RHSExpr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<RHSExpr, Dev> > rhsExpr;\ + const OP func;\ + FunctorExtractor(const TensorEvaluator<CVQual UnaryCategory<OP, RHSExpr>, Dev>& expr)\ + : rhsExpr(expr.impl()), func(expr.functor()) {}\ +}; + +SYCLEXTRFUNCUNARY(const) +SYCLEXTRFUNCUNARY() +#undef SYCLEXTRFUNCUNARY + +/// specialisation of the \ref FunctorExtractor struct when the node type is +/// TensorCwiseBinaryOp +#define SYCLEXTRFUNCBIINARY(CVQual)\ +template <template<class, class, class> class BinaryCategory, typename OP, typename LHSExpr, typename RHSExpr, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual BinaryCategory<OP, LHSExpr, RHSExpr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<LHSExpr, Dev> > lhsExpr;\ + FunctorExtractor<TensorEvaluator<RHSExpr, Dev> > rhsExpr;\ + const OP func;\ + FunctorExtractor(const TensorEvaluator<CVQual BinaryCategory<OP, LHSExpr, RHSExpr>, Dev>& expr)\ + : lhsExpr(expr.left_impl()),rhsExpr(expr.right_impl()),func(expr.functor()) {}\ +}; + +SYCLEXTRFUNCBIINARY(const) +SYCLEXTRFUNCBIINARY() +#undef SYCLEXTRFUNCBIINARY + +/// specialisation of the \ref FunctorExtractor struct when the node type is TensorCwiseTernaryOp +#define SYCLEXTRFUNCTERNARY(CVQual)\ +template <template <class, class, class, class> class TernaryCategory, typename OP, typename Arg1Expr, typename Arg2Expr, typename Arg3Expr,typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual TernaryCategory<OP, Arg1Expr, Arg2Expr, Arg3Expr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<Arg1Expr, Dev> > arg1Expr;\ + FunctorExtractor<TensorEvaluator<Arg2Expr, Dev> > arg2Expr;\ + FunctorExtractor<TensorEvaluator<Arg3Expr, Dev> > arg3Expr;\ + const OP func;\ + FunctorExtractor(const TensorEvaluator<CVQual TernaryCategory<OP, Arg1Expr, Arg2Expr, Arg3Expr>, Dev>& expr)\ + : arg1Expr(expr.arg1Impl()), arg2Expr(expr.arg2Impl()), arg3Expr(expr.arg3Impl()), func(expr.functor()) {}\ +}; + +SYCLEXTRFUNCTERNARY(const) +SYCLEXTRFUNCTERNARY() +#undef SYCLEXTRFUNCTERNARY + + + +//TensorCustomOp must be specialised otherwise it will be captured by UnaryCategory while its action is different +//from the UnaryCategory and it is similar to the general FunctorExtractor. +/// specialisation of TensorCustomOp +#define SYCLEXTRFUNCCUSTOMUNARYOP(CVQual)\ +template <typename CustomUnaryFunc, typename ArgType, typename Dev >\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorCustomUnaryOp<CustomUnaryFunc, ArgType>, Dev> > {\ + typedef TensorEvaluator<CVQual TensorCustomUnaryOp<CustomUnaryFunc, ArgType>, Dev> Evaluator;\ + DEFALTACTION(Evaluator)\ +}; +//TensorCustomUnaryOp +SYCLEXTRFUNCCUSTOMUNARYOP(const) +SYCLEXTRFUNCCUSTOMUNARYOP() +#undef SYCLEXTRFUNCCUSTOMUNARYOP + +//TensorCustomBinaryOp +#define SYCLEXTRFUNCCUSTOMBIBARYOP(CVQual)\ +template <typename CustomBinaryFunc, typename ArgType1, typename ArgType2, typename Dev >\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorCustomBinaryOp<CustomBinaryFunc, ArgType1, ArgType2>, Dev> > {\ + typedef TensorEvaluator<CVQual TensorCustomBinaryOp<CustomBinaryFunc, ArgType1, ArgType2>, Dev> Evaluator;\ + DEFALTACTION(Evaluator)\ +}; +//TensorCustomBinaryOp +SYCLEXTRFUNCCUSTOMBIBARYOP(const) +SYCLEXTRFUNCCUSTOMBIBARYOP() +#undef SYCLEXTRFUNCCUSTOMBIBARYOP + + + +/// specialisation of the \ref FunctorExtractor struct when the node type is +/// TensorCwiseSelectOp. This is an specialisation without OP so it has to be separated. +#define SYCLEXTRFUNCSELECTOP(CVQual)\ +template <typename IfExpr, typename ThenExpr, typename ElseExpr, typename Dev>\ +struct FunctorExtractor< TensorEvaluator<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<IfExpr, Dev> > ifExpr;\ + FunctorExtractor<TensorEvaluator<ThenExpr, Dev> > thenExpr;\ + FunctorExtractor<TensorEvaluator<ElseExpr, Dev> > elseExpr;\ + FunctorExtractor(const TensorEvaluator<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, Dev>& expr)\ + : ifExpr(expr.cond_impl()), thenExpr(expr.then_impl()), elseExpr(expr.else_impl()) {}\ +}; + +SYCLEXTRFUNCSELECTOP(const) +SYCLEXTRFUNCSELECTOP() +#undef SYCLEXTRFUNCSELECTOP + +/// specialisation of the \ref FunctorExtractor struct when the node type is +/// const TensorAssignOp. This is an specialisation without OP so it has to be separated. +#define SYCLEXTRFUNCASSIGNOP(CVQual)\ +template <typename LHSExpr, typename RHSExpr, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorAssignOp<LHSExpr, RHSExpr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<LHSExpr, Dev> > lhsExpr;\ + FunctorExtractor<TensorEvaluator<RHSExpr, Dev> > rhsExpr;\ + FunctorExtractor(const TensorEvaluator<CVQual TensorAssignOp<LHSExpr, RHSExpr>, Dev>& expr)\ + : lhsExpr(expr.left_impl()), rhsExpr(expr.right_impl()) {}\ +}; +SYCLEXTRFUNCASSIGNOP(const) +SYCLEXTRFUNCASSIGNOP() +#undef SYCLEXTRFUNCASSIGNOP + +/// specialisation of the \ref FunctorExtractor struct when the node types are +/// TensorEvalToOp, TensorLayoutSwapOp. This is an specialisation without OP so it has to be separated. +#define SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(CVQual, ExprNode)\ +template <typename Expr, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual ExprNode<Expr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<Expr, Dev> > xprExpr;\ + FunctorExtractor(const TensorEvaluator<CVQual ExprNode<Expr>, Dev>& expr)\ + : xprExpr(expr.impl()) {}\ +}; +//TensorEvalToOp +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(const, TensorEvalToOp) +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(, TensorEvalToOp) +// TensorLayoutSwapOp +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(const, TensorLayoutSwapOp) +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(, TensorLayoutSwapOp) +// TensorIndexTupleOp +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(const, TensorIndexTupleOp) +SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE(, TensorIndexTupleOp) + +#undef SYCLEXTRFUNCEVALTOOPSWAPLAYOUTINDEXTUPLE + +template<typename Dim, size_t NumOutputDim> struct DimConstr { +template<typename InDim> + static EIGEN_STRONG_INLINE Dim getDim(InDim dims ) {return dims;} +}; + +template<typename Dim> struct DimConstr<Dim, 0> { + template<typename InDim> + static EIGEN_STRONG_INLINE Dim getDim(InDim dims ) {return Dim(static_cast<Dim>(dims.TotalSize()));} +}; +//TensorReductionOp +#define SYCLEXTRFUNCREDUCTIONOP(CVQual)\ +template<typename Op, typename Dims, typename ArgType, template <class> class MakePointer_, typename Device>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> >{\ + typedef TensorEvaluator<CVQual TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> Evaluator;\ + typedef typename Eigen::internal::conditional<Evaluator::NumOutputDims==0, DSizes<typename Evaluator::Index, 1>, typename Evaluator::Dimensions >::type Dimensions;\ + const Dimensions m_dimensions;\ + EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }\ + FunctorExtractor(const TensorEvaluator<CVQual TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device>& expr)\ + : m_dimensions(DimConstr<Dimensions, Evaluator::NumOutputDims>::getDim(expr.dimensions())) {}\ +}; +SYCLEXTRFUNCREDUCTIONOP(const) +SYCLEXTRFUNCREDUCTIONOP() +#undef SYCLEXTRFUNCREDUCTIONOP + +//TensorTupleReducerOp +#define SYCLEXTRFUNCTUPLEREDUCTIONOP(CVQual)\ +template<typename ReduceOp, typename Dims, typename ArgType, typename Device>\ + struct FunctorExtractor<TensorEvaluator<CVQual TensorTupleReducerOp<ReduceOp, Dims, ArgType>, Device> >{\ + typedef TensorEvaluator<CVQual TensorTupleReducerOp<ReduceOp, Dims, ArgType>, Device> Evaluator;\ + static const int NumOutputDims= Eigen::internal::traits<TensorTupleReducerOp<ReduceOp, Dims, ArgType> >::NumDimensions;\ + typedef typename Evaluator::StrideDims StrideDims;\ + typedef typename Evaluator::Index Index;\ + typedef typename Eigen::internal::conditional<NumOutputDims==0, DSizes<Index, 1>, typename Evaluator::Dimensions >::type Dimensions;\ + const Dimensions m_dimensions;\ + const Index m_return_dim;\ + const StrideDims m_strides;\ + const Index m_stride_mod;\ + const Index m_stride_div;\ + EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }\ + EIGEN_STRONG_INLINE Index return_dim() const {return m_return_dim;}\ + EIGEN_STRONG_INLINE const StrideDims strides() const {return m_strides;}\ + EIGEN_STRONG_INLINE const Index stride_mod() const {return m_stride_mod;}\ + EIGEN_STRONG_INLINE const Index stride_div() const {return m_stride_div;}\ + FunctorExtractor(const TensorEvaluator<CVQual TensorTupleReducerOp<ReduceOp, Dims, ArgType>, Device>& expr)\ + : m_dimensions(DimConstr<Dimensions, NumOutputDims>::getDim(expr.dimensions())), m_return_dim(expr.return_dim()),\ + m_strides(expr.strides()), m_stride_mod(expr.stride_mod()), m_stride_div(expr.stride_div()){}\ +}; + +SYCLEXTRFUNCTUPLEREDUCTIONOP(const) +SYCLEXTRFUNCTUPLEREDUCTIONOP() +#undef SYCLEXTRFUNCTUPLEREDUCTIONOP + +//TensorContractionOp and TensorConvolutionOp +#define SYCLEXTRFUNCCONTRACTCONVOLUTIONOP(CVQual, ExprNode)\ +template<typename Indices, typename LhsXprType, typename RhsXprType, typename Device>\ +struct FunctorExtractor<TensorEvaluator<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, Device>>{\ + typedef TensorEvaluator<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, Device> Evaluator;\ + typedef typename Evaluator::Dimensions Dimensions;\ + const Dimensions m_dimensions;\ + EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }\ + FunctorExtractor(const TensorEvaluator<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, Device>& expr)\ + : m_dimensions(expr.dimensions()) {}\ +}; + +//TensorContractionOp +SYCLEXTRFUNCCONTRACTCONVOLUTIONOP(const,TensorContractionOp) +SYCLEXTRFUNCCONTRACTCONVOLUTIONOP(,TensorContractionOp) +//TensorConvolutionOp +SYCLEXTRFUNCCONTRACTCONVOLUTIONOP(const,TensorConvolutionOp) +SYCLEXTRFUNCCONTRACTCONVOLUTIONOP(,TensorConvolutionOp) +#undef SYCLEXTRFUNCCONTRACTCONVOLUTIONOP + +/// specialisation of the \ref FunctorExtractor struct when the node type is +/// const TensorSlicingOp. This is an specialisation without OP so it has to be separated. +#define SYCLEXTRFUNCTSLICEOP(CVQual)\ +template <typename StartIndices, typename Sizes, typename XprType, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, Dev> > {\ + FunctorExtractor<TensorEvaluator<XprType, Dev> > xprExpr;\ + const StartIndices m_offsets;\ + const Sizes m_dimensions;\ + FunctorExtractor(const TensorEvaluator<CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, Dev>& expr)\ + : xprExpr(expr.impl()), m_offsets(expr.startIndices()), m_dimensions(expr.dimensions()) {}\ + EIGEN_STRONG_INLINE const StartIndices& startIndices() const {return m_offsets;}\ + EIGEN_STRONG_INLINE const Sizes& dimensions() const {return m_dimensions;}\ +}; + +SYCLEXTRFUNCTSLICEOP(const) +SYCLEXTRFUNCTSLICEOP() +#undef SYCLEXTRFUNCTSLICEOP + +//TensorStridingSlicingOp +#define SYCLEXTRFUNCTSLICESTRIDEOP(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, Dev> >{\ + FunctorExtractor<TensorEvaluator<XprType, Dev> > xprExpr;\ + const StartIndices m_startIndices;\ + const StopIndices m_stopIndices;\ + const Strides m_strides;\ + FunctorExtractor(const TensorEvaluator<CVQual TensorStridingSlicingOp<StartIndices, StopIndices,Strides, XprType>, Dev>& expr)\ + : xprExpr(expr.impl()), m_startIndices(expr.exprStartIndices()), m_stopIndices(expr.exprStopIndices()), m_strides(expr.strides()) {}\ + EIGEN_STRONG_INLINE const StartIndices& startIndices() const { return m_startIndices; }\ + EIGEN_STRONG_INLINE const StartIndices& stopIndices() const { return m_stopIndices; }\ + EIGEN_STRONG_INLINE const StartIndices& strides() const { return m_strides; }\ +}; + +SYCLEXTRFUNCTSLICESTRIDEOP(const) +SYCLEXTRFUNCTSLICESTRIDEOP() +#undef SYCLEXTRFUNCTSLICESTRIDEOP + +// Had to separate TensorReshapingOp and TensorShufflingOp. Otherwise it will be mistaken by UnaryCategory +#define SYCLRESHAPEANDSHUFFLEOPFUNCEXT(OPEXPR, FUNCCALL, CVQual)\ +template<typename Param, typename XprType, typename Dev>\ +struct FunctorExtractor<Eigen::TensorEvaluator<CVQual Eigen::OPEXPR<Param, XprType>, Dev> > {\ + FunctorExtractor<Eigen::TensorEvaluator<XprType, Dev> > xprExpr;\ + const Param m_param;\ + EIGEN_STRONG_INLINE const Param& param() const { return m_param; }\ + FunctorExtractor(const Eigen::TensorEvaluator<CVQual Eigen::OPEXPR<Param, XprType>, Dev>& expr)\ + : xprExpr(expr.impl()), m_param(expr.FUNCCALL) {}\ +}; + +//TensorReshapingOp +SYCLRESHAPEANDSHUFFLEOPFUNCEXT(TensorReshapingOp, dimensions(), const) +SYCLRESHAPEANDSHUFFLEOPFUNCEXT(TensorReshapingOp, dimensions(), ) + +//TensorShufflingOp +SYCLRESHAPEANDSHUFFLEOPFUNCEXT(TensorShufflingOp, shufflePermutation(), const) +SYCLRESHAPEANDSHUFFLEOPFUNCEXT(TensorShufflingOp, shufflePermutation(), ) +#undef SYCLRESHAPEANDSHUFFLEOPFUNCEXT + +// Had to separate reshapeOP otherwise it will be mistaken by UnaryCategory +#define PADDINGOPFUNCEXT(OPEXPR, FUNCCALL, SCALARFUNCCALL, CVQual)\ +template<typename Param, typename XprType, typename Dev>\ +struct FunctorExtractor<Eigen::TensorEvaluator<CVQual Eigen::OPEXPR<Param, XprType>, Dev> > {\ + FunctorExtractor<Eigen::TensorEvaluator<XprType, Dev> > xprExpr;\ + const Param m_param;\ + typedef typename Eigen::TensorEvaluator<CVQual Eigen::OPEXPR<Param, XprType>, Dev>::Scalar Scalar;\ + const Scalar m_scalar_param;\ + EIGEN_STRONG_INLINE const Param& param() const { return m_param; }\ + EIGEN_STRONG_INLINE const Scalar& scalar_param() const { return m_scalar_param; }\ + FunctorExtractor(const Eigen::TensorEvaluator<CVQual Eigen::OPEXPR<Param, XprType>, Dev>& expr)\ + : xprExpr(expr.impl()), m_param(expr.FUNCCALL), m_scalar_param(expr.SCALARFUNCCALL) {}\ +}; + +PADDINGOPFUNCEXT(TensorPaddingOp, padding(), padding_value(), const) +PADDINGOPFUNCEXT(TensorPaddingOp, padding(), padding_value(), ) +#undef PADDINGOPFUNCEXT + +/// specialisation of the \ref FunctorExtractor struct when the node type is TensorContractionOp and TensorConcatenationOp +/// for TensorContractionOp the LHS and RHS here are the original one no need to apply condition on their type. +#define SYCLEXTRFUNCCONTRACTCONCAT(OPEXPR, FUNCCALL, CVQual)\ +template <typename Param, typename LHSExpr, typename RHSExpr, typename Dev>\ +struct FunctorExtractor<TensorEvaluator<CVQual OPEXPR<Param, LHSExpr, RHSExpr>, Dev> > {\ + FunctorExtractor<TensorEvaluator<LHSExpr, Dev> > lhsExpr;\ + FunctorExtractor<TensorEvaluator<RHSExpr, Dev> > rhsExpr;\ + const Param func;\ + FunctorExtractor(const TensorEvaluator<CVQual OPEXPR<Param, LHSExpr, RHSExpr>, Dev>& expr)\ + : lhsExpr(expr.left_impl()),rhsExpr(expr.right_impl()),func(expr.FUNCCALL) {}\ +}; + +// TensorConcatenationOp +SYCLEXTRFUNCCONTRACTCONCAT(TensorConcatenationOp, axis(), const) +SYCLEXTRFUNCCONTRACTCONCAT(TensorConcatenationOp, axis(),) +#undef SYCLEXTRFUNCCONTRACTCONCAT + +//TensorChippingOp +#define SYCLEXTRFUNCCHIPPINGOP(CVQual)\ +template<DenseIndex DimId, typename XprType, typename Device>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorChippingOp<DimId, XprType>, Device> >{\ + FunctorExtractor<Eigen::TensorEvaluator<XprType, Device> > xprExpr;\ + const DenseIndex m_dim;\ + const DenseIndex m_offset;\ + EIGEN_STRONG_INLINE const DenseIndex& dimId() const { return m_dim; }\ + EIGEN_STRONG_INLINE const DenseIndex& offset() const { return m_offset; }\ + FunctorExtractor(const TensorEvaluator<CVQual TensorChippingOp<DimId, XprType>, Device>& expr)\ + : xprExpr(expr.impl()), m_dim(expr.dimId()), m_offset(expr.offset()) {}\ +}; + +SYCLEXTRFUNCCHIPPINGOP(const) +SYCLEXTRFUNCCHIPPINGOP() +#undef SYCLEXTRFUNCCHIPPINGOP + +//TensorImagePatchOp +#define SYCLEXTRFUNCIMAGEPATCHOP(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename XprType, typename Device>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorImagePatchOp<Rows, Cols, XprType>, Device> >{\ +typedef CVQual TensorImagePatchOp<Rows, Cols, XprType> Self;\ +FunctorExtractor<Eigen::TensorEvaluator<XprType, Device> > xprExpr;\ +const DenseIndex m_patch_rows;\ +const DenseIndex m_patch_cols;\ +const DenseIndex m_row_strides;\ +const DenseIndex m_col_strides;\ +const DenseIndex m_in_row_strides;\ +const DenseIndex m_in_col_strides;\ +const DenseIndex m_row_inflate_strides;\ +const DenseIndex m_col_inflate_strides;\ +const bool m_padding_explicit;\ +const DenseIndex m_padding_top;\ +const DenseIndex m_padding_bottom;\ +const DenseIndex m_padding_left;\ +const DenseIndex m_padding_right;\ +const PaddingType m_padding_type;\ +const typename Self::Scalar m_padding_value;\ +FunctorExtractor(const TensorEvaluator<Self, Device>& expr)\ +: xprExpr(expr.impl()), m_patch_rows(expr.xpr().patch_rows()), m_patch_cols(expr.xpr().patch_cols()),\ + m_row_strides(expr.xpr().row_strides()), m_col_strides(expr.xpr().col_strides()),\ + m_in_row_strides(expr.xpr().in_row_strides()), m_in_col_strides(expr.xpr().in_col_strides()),\ + m_row_inflate_strides(expr.xpr().row_inflate_strides()), m_col_inflate_strides(expr.xpr().col_inflate_strides()),\ + m_padding_explicit(expr.xpr().padding_explicit()),m_padding_top(expr.xpr().padding_top()),\ + m_padding_bottom(expr.xpr().padding_bottom()), m_padding_left(expr.xpr().padding_left()),\ + m_padding_right(expr.xpr().padding_right()), m_padding_type(expr.xpr().padding_type()),\ + m_padding_value(expr.xpr().padding_value()){}\ +}; + +SYCLEXTRFUNCIMAGEPATCHOP(const) +SYCLEXTRFUNCIMAGEPATCHOP() +#undef SYCLEXTRFUNCIMAGEPATCHOP + +/// TensorVolumePatchOp +#define SYCLEXTRFUNCVOLUMEPATCHOP(CVQual)\ +template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType, typename Device>\ +struct FunctorExtractor<TensorEvaluator<CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType>, Device> >{\ +typedef CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType> Self;\ +FunctorExtractor<Eigen::TensorEvaluator<XprType, Device> > xprExpr;\ +const DenseIndex m_patch_planes;\ +const DenseIndex m_patch_rows;\ +const DenseIndex m_patch_cols;\ +const DenseIndex m_plane_strides;\ +const DenseIndex m_row_strides;\ +const DenseIndex m_col_strides;\ +const DenseIndex m_in_plane_strides;\ +const DenseIndex m_in_row_strides;\ +const DenseIndex m_in_col_strides;\ +const DenseIndex m_plane_inflate_strides;\ +const DenseIndex m_row_inflate_strides;\ +const DenseIndex m_col_inflate_strides;\ +const bool m_padding_explicit;\ +const DenseIndex m_padding_top_z;\ +const DenseIndex m_padding_bottom_z;\ +const DenseIndex m_padding_top;\ +const DenseIndex m_padding_bottom;\ +const DenseIndex m_padding_left;\ +const DenseIndex m_padding_right;\ +const PaddingType m_padding_type;\ +const typename Self::Scalar m_padding_value;\ +FunctorExtractor(const TensorEvaluator<Self, Device>& expr)\ +: xprExpr(expr.impl()), m_patch_planes(expr.xpr().patch_planes()), m_patch_rows(expr.xpr().patch_rows()), m_patch_cols(expr.xpr().patch_cols()),\ + m_plane_strides(expr.xpr().plane_strides()), m_row_strides(expr.xpr().row_strides()), m_col_strides(expr.xpr().col_strides()),\ + m_in_plane_strides(expr.xpr().in_plane_strides()), m_in_row_strides(expr.xpr().in_row_strides()), m_in_col_strides(expr.xpr().in_col_strides()),\ + m_plane_inflate_strides(expr.xpr().plane_inflate_strides()),m_row_inflate_strides(expr.xpr().row_inflate_strides()),\ + m_col_inflate_strides(expr.xpr().col_inflate_strides()), m_padding_explicit(expr.xpr().padding_explicit()),\ + m_padding_top_z(expr.xpr().padding_top_z()), m_padding_bottom_z(expr.xpr().padding_bottom_z()), \ + m_padding_top(expr.xpr().padding_top()), m_padding_bottom(expr.xpr().padding_bottom()), m_padding_left(expr.xpr().padding_left()),\ + m_padding_right(expr.xpr().padding_right()), m_padding_type(expr.xpr().padding_type()),m_padding_value(expr.xpr().padding_value()){}\ +}; +SYCLEXTRFUNCVOLUMEPATCHOP(const) +SYCLEXTRFUNCVOLUMEPATCHOP() +#undef SYCLEXTRFUNCVOLUMEPATCHOP + + +/// template deduction function for FunctorExtractor +template <typename Evaluator> +auto inline extractFunctors(const Evaluator& evaluator)-> FunctorExtractor<Evaluator> { + return FunctorExtractor<Evaluator>(evaluator); +} +} // namespace internal +} // namespace TensorSycl +} // namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_EXTRACT_FUNCTORS_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclFunctors.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclFunctors.h new file mode 100644 index 0000000..a447c3f --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclFunctors.h
@@ -0,0 +1,248 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: eigen@codeplay.com +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// General include header of SYCL target for Tensor Module +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCLFUNCTORS_H +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCLFUNCTORS_H + +namespace Eigen { +namespace TensorSycl { +namespace internal { + + template<typename CoeffReturnType, typename OP, typename OutputAccessor, typename InputAccessor, typename LocalAccessor> struct GenericKernelReducer{ + OP op; + OutputAccessor aOut; + ptrdiff_t out_offset; + InputAccessor aI; + LocalAccessor scratch; + size_t length, local; + GenericKernelReducer(OP op_, OutputAccessor aOut_, ptrdiff_t out_offset_, InputAccessor aI_, LocalAccessor scratch_, size_t length_, size_t local_) + : op(op_), aOut(aOut_), out_offset(out_offset_), aI(aI_), scratch(scratch_), length(length_), local(local_){} + void operator()(cl::sycl::nd_item<1> itemID) { + size_t globalid = itemID.get_global(0); + size_t localid = itemID.get_local(0); + /* All threads collectively read from global memory into local. + * The barrier ensures all threads' IO is resolved before + * execution continues (strictly speaking, all threads within + * a single work-group - there is no co-ordination between + * work-groups, only work-items). */ + if (globalid < length) { + scratch[localid] = aI[globalid]; + } + itemID.barrier(cl::sycl::access::fence_space::local_space); + + /* Apply the reduction operation between the current local + * id and the one on the other half of the vector. */ + if (globalid < length) { + auto min = (length < local) ? length : local; + for (size_t offset = min / 2; offset > 0; offset /= 2) { + if (localid < offset) { + auto accum = op.initialize(); + op.reduce(scratch[localid], &accum); + op.reduce(scratch[localid + offset], &accum); + op.finalize(accum); + scratch[localid]=accum; + //scratch[localid] += scratch[localid + offset]; + } + itemID.barrier(cl::sycl::access::fence_space::local_space); + } + /* The final result will be stored in local id 0. */ + if (localid == 0) { + aI[itemID.get_group(0)] = scratch[localid]; + if((length<=local) && globalid ==0){ + auto aOutPtr = ConvertToActualTypeSycl(CoeffReturnType, aOut); + aOutPtr[0 + ConvertToActualSyclOffset(CoeffReturnType, out_offset)]=scratch[0]; + } + } + } + } + + }; + +/// ReductionFunctor +template < typename HostExpr, typename FunctorExpr, typename Tuple_of_Acc, typename Dims, typename Op, typename Index> class ReductionFunctor { + public: + typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; + typedef cl::sycl::accessor<uint8_t, 1, cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer> write_accessor; + ReductionFunctor(write_accessor output_accessor_, ptrdiff_t out_offset_, FunctorExpr functors_, Tuple_of_Acc tuple_of_accessors_,Dims dims_, Op functor_, Index range_, Index) + :output_accessor(output_accessor_), out_offset(out_offset_), functors(functors_), tuple_of_accessors(tuple_of_accessors_), dims(dims_), functor(functor_), range(range_) {} + void operator()(cl::sycl::nd_item<1> itemID) { + + typedef typename ConvertToDeviceExpression<const HostExpr>::Type DevExpr; + auto device_expr = createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + /// reduction cannot be captured automatically through our device conversion recursion. The reason is that reduction has two behaviour + /// the first behaviour is when it is used as a root to launch the sub-kernel. The second one is when it is treated as a leafnode to pass the + /// calculated result to its parent kernel. While the latter is automatically detected through our device expression generator. The former is created here. + const auto device_self_expr= Eigen::TensorReductionOp<Op, Dims, decltype(device_expr.expr) ,MakeGlobalPointer>(device_expr.expr, dims, functor); + /// This is the evaluator for device_self_expr. This is exactly similar to the self which has been passed to run function. The difference is + /// the device_evaluator is detectable and recognisable on the device. + typedef Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice> DeviceSelf; + auto device_self_evaluator = Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice>(device_self_expr, Eigen::SyclKernelDevice()); + auto output_accessor_ptr =ConvertToActualTypeSycl(typename DeviceSelf::CoeffReturnType, output_accessor); + /// const cast added as a naive solution to solve the qualifier drop error + auto globalid=static_cast<Index>(itemID.get_global_linear_id()); + if (globalid< range) { + typename DeviceSelf::CoeffReturnType accum = functor.initialize(); + Eigen::internal::GenericDimReducer<DeviceSelf::NumReducedDims-1, DeviceSelf, Op>::reduce(device_self_evaluator, device_self_evaluator.firstInput(static_cast<typename DevExpr::Index>(globalid)),const_cast<Op&>(functor), &accum); + functor.finalize(accum); + output_accessor_ptr[globalid + ConvertToActualSyclOffset(typename DeviceSelf::CoeffReturnType, out_offset)]= accum; + } + } + private: + write_accessor output_accessor; + ptrdiff_t out_offset; + FunctorExpr functors; + Tuple_of_Acc tuple_of_accessors; + Dims dims; + Op functor; + Index range; +}; + +template < typename HostExpr, typename FunctorExpr, typename Tuple_of_Acc, typename Dims, typename Index> +class ReductionFunctor<HostExpr, FunctorExpr, Tuple_of_Acc, Dims, Eigen::internal::MeanReducer<typename HostExpr::CoeffReturnType>, Index> { + public: + typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; + typedef cl::sycl::accessor<uint8_t, 1, cl::sycl::access::mode::write, cl::sycl::access::target::global_buffer> write_accessor; + typedef Eigen::internal::SumReducer<typename HostExpr::CoeffReturnType> Op; + ReductionFunctor(write_accessor output_accessor_, ptrdiff_t out_offset_, FunctorExpr functors_, Tuple_of_Acc tuple_of_accessors_,Dims dims_, + Eigen::internal::MeanReducer<typename HostExpr::CoeffReturnType>, Index range_, Index num_values_to_reduce_) + :output_accessor(output_accessor_), out_offset(out_offset_), functors(functors_), tuple_of_accessors(tuple_of_accessors_), dims(dims_), functor(Op()), range(range_), num_values_to_reduce(num_values_to_reduce_) {} + void operator()(cl::sycl::nd_item<1> itemID) { + + typedef typename ConvertToDeviceExpression<const HostExpr>::Type DevExpr; + auto device_expr = createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + /// reduction cannot be captured automatically through our device conversion recursion. The reason is that reduction has two behaviour + /// the first behaviour is when it is used as a root to launch the sub-kernel. The second one is when it is treated as a leafnode to pass the + /// calculated result to its parent kernel. While the latter is automatically detected through our device expression generator. The former is created here. + const auto device_self_expr= Eigen::TensorReductionOp<Op, Dims, decltype(device_expr.expr) ,MakeGlobalPointer>(device_expr.expr, dims, functor); + /// This is the evaluator for device_self_expr. This is exactly similar to the self which has been passed to run function. The difference is + /// the device_evaluator is detectable and recognisable on the device. + typedef Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice> DeviceSelf; + auto device_self_evaluator = Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice>(device_self_expr, Eigen::SyclKernelDevice()); + auto output_accessor_ptr =ConvertToActualTypeSycl(typename DeviceSelf::CoeffReturnType, output_accessor); + /// const cast added as a naive solution to solve the qualifier drop error + auto globalid=static_cast<Index>(itemID.get_global_linear_id()); + if (globalid< range) { + typename DeviceSelf::CoeffReturnType accum = functor.initialize(); + Eigen::internal::GenericDimReducer<DeviceSelf::NumReducedDims-1, DeviceSelf, Op>::reduce(device_self_evaluator, device_self_evaluator.firstInput(static_cast<typename DevExpr::Index>(globalid)),const_cast<Op&>(functor), &accum); + functor.finalize(accum); + output_accessor_ptr[globalid+ ConvertToActualSyclOffset(typename DeviceSelf::CoeffReturnType, out_offset)]= accum/num_values_to_reduce; + } + } + private: + write_accessor output_accessor; + ptrdiff_t out_offset; + FunctorExpr functors; + Tuple_of_Acc tuple_of_accessors; + Dims dims; + Op functor; + Index range; + Index num_values_to_reduce; +}; + +template<typename CoeffReturnType ,typename OutAccessor, typename HostExpr, typename FunctorExpr, typename Op, typename Dims, typename Index, typename TupleType> +class FullReductionKernelFunctor{ +public: + typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; + OutAccessor tmp_global_accessor; + Index rng , remaining, red_factor; + Op op; + Dims dims; + FunctorExpr functors; + TupleType tuple_of_accessors; + + FullReductionKernelFunctor(OutAccessor acc, Index rng_, Index remaining_, Index red_factor_, Op op_, Dims dims_, FunctorExpr functors_, TupleType t_acc) + :tmp_global_accessor(acc), rng(rng_), remaining(remaining_), red_factor(red_factor_),op(op_), dims(dims_), functors(functors_), tuple_of_accessors(t_acc){} + + void operator()(cl::sycl::nd_item<1> itemID) { + + typedef typename TensorSycl::internal::ConvertToDeviceExpression<const HostExpr>::Type DevExpr; + auto device_expr = TensorSycl::internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + /// reduction cannot be captured automatically through our device conversion recursion. The reason is that reduction has two behaviour + /// the first behaviour is when it is used as a root to launch the sub-kernel. The second one is when it is treated as a leafnode to pass the + /// calculated result to its parent kernel. While the latter is automatically detected through our device expression generator. The former is created here. + const auto device_self_expr= Eigen::TensorReductionOp<Op, Dims, decltype(device_expr.expr) ,MakeGlobalPointer>(device_expr.expr, dims, op); + /// This is the evaluator for device_self_expr. This is exactly similar to the self which has been passed to run function. The difference is + /// the device_evaluator is detectable and recognisable on the device. + auto device_self_evaluator = Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice>(device_self_expr, Eigen::SyclKernelDevice()); + /// const cast added as a naive solution to solve the qualifier drop error + auto globalid=itemID.get_global_linear_id(); + + tmp_global_accessor.get_pointer()[globalid]=(globalid<rng) ? Eigen::internal::InnerMostDimReducer<decltype(device_self_evaluator), Op, false>::reduce(device_self_evaluator, static_cast<typename DevExpr::Index>(red_factor*globalid), red_factor, const_cast<Op&>(op)) + : static_cast<CoeffReturnType>(op.initialize()); + + if(remaining!=0 && globalid==0 ){ + // this will add the rest of input buffer when the input size is not devidable to red_factor. + auto remaining_reduce =Eigen::internal::InnerMostDimReducer<decltype(device_self_evaluator), Op, false>:: + reduce(device_self_evaluator, static_cast<typename DevExpr::Index>(red_factor*(rng)), static_cast<typename DevExpr::Index>(remaining), const_cast<Op&>(op)); + auto accum = op.initialize(); + op.reduce(tmp_global_accessor.get_pointer()[0], &accum); + op.reduce(remaining_reduce, &accum); + op.finalize(accum); + tmp_global_accessor.get_pointer()[0]=accum; + + } + } +}; + +template<typename CoeffReturnType ,typename OutAccessor, typename HostExpr, typename FunctorExpr, typename Dims, typename Index, typename TupleType> +class FullReductionKernelFunctor<CoeffReturnType, OutAccessor, HostExpr, FunctorExpr, Eigen::internal::MeanReducer<CoeffReturnType>, Dims, Index, TupleType>{ +public: + typedef typename TensorSycl::internal::createPlaceHolderExpression<HostExpr>::Type PlaceHolderExpr; + typedef Eigen::internal::SumReducer<CoeffReturnType> Op; + + OutAccessor tmp_global_accessor; + Index rng , remaining, red_factor; + Op op; + Dims dims; + FunctorExpr functors; + TupleType tuple_of_accessors; + + FullReductionKernelFunctor(OutAccessor acc, Index rng_, Index remaining_, Index red_factor_, Eigen::internal::MeanReducer<CoeffReturnType>, Dims dims_, FunctorExpr functors_, TupleType t_acc) + :tmp_global_accessor(acc), rng(rng_), remaining(remaining_), red_factor(red_factor_),op(Op()), dims(dims_), functors(functors_), tuple_of_accessors(t_acc){} + + void operator()(cl::sycl::nd_item<1> itemID) { + + typedef typename TensorSycl::internal::ConvertToDeviceExpression<const HostExpr>::Type DevExpr; + auto device_expr = TensorSycl::internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + /// reduction cannot be captured automatically through our device conversion recursion. The reason is that reduction has two behaviour + /// the first behaviour is when it is used as a root to launch the sub-kernel. The second one is when it is treated as a leafnode to pass the + /// calculated result to its parent kernel. While the latter is automatically detected through our device expression generator. The former is created here. + const auto device_self_expr= Eigen::TensorReductionOp<Op, Dims, decltype(device_expr.expr) ,MakeGlobalPointer>(device_expr.expr, dims, op); + /// This is the evaluator for device_self_expr. This is exactly similar to the self which has been passed to run function. The difference is + /// the device_evaluator is detectable and recognisable on the device. + auto device_self_evaluator = Eigen::TensorEvaluator<decltype(device_self_expr), Eigen::SyclKernelDevice>(device_self_expr, Eigen::SyclKernelDevice()); + /// const cast added as a naive solution to solve the qualifier drop error + auto globalid=itemID.get_global_linear_id(); + auto scale = (rng*red_factor) + remaining; + + tmp_global_accessor.get_pointer()[globalid]= (globalid<rng)? ((Eigen::internal::InnerMostDimReducer<decltype(device_self_evaluator), Op, false>::reduce(device_self_evaluator, static_cast<typename DevExpr::Index>(red_factor*globalid), red_factor, const_cast<Op&>(op)))/scale) + :static_cast<CoeffReturnType>(op.initialize())/scale; + + if(remaining!=0 && globalid==0 ){ + // this will add the rest of input buffer when the input size is not devidable to red_factor. + auto remaining_reduce =Eigen::internal::InnerMostDimReducer<decltype(device_self_evaluator), Op, false>::reduce(device_self_evaluator, static_cast<typename DevExpr::Index>(red_factor*(rng)), static_cast<typename DevExpr::Index>(remaining), const_cast<Op&>(op)); + auto accum = op.initialize(); + tmp_global_accessor.get_pointer()[0]= tmp_global_accessor.get_pointer()[0]*scale; + op.reduce(tmp_global_accessor.get_pointer()[0], &accum); + op.reduce(remaining_reduce, &accum); + op.finalize(accum); + tmp_global_accessor.get_pointer()[0]=accum/scale; + + } + } +}; + +} +} +} +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCLFUNCTORS_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclLeafCount.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclLeafCount.h new file mode 100644 index 0000000..234580c --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclLeafCount.h
@@ -0,0 +1,213 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclLeafCount.h + * + * \brief: + * The leaf count used the pre-order expression tree traverse in order to name + * count the number of leaf nodes in the expression + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_LEAF_COUNT_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_LEAF_COUNT_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { +/// \brief LeafCount used to counting terminal nodes. The total number of +/// leaf nodes is used by MakePlaceHolderExprHelper to find the order +/// of the leaf node in a expression tree at compile time. +template <typename Expr> +struct LeafCount; + +template<typename... Args> struct CategoryCount; + +template<> struct CategoryCount<> +{ + static const size_t Count =0; +}; + +template<typename Arg, typename... Args> +struct CategoryCount<Arg,Args...>{ + static const size_t Count = LeafCount<Arg>::Count + CategoryCount<Args...>::Count; +}; + +/// specialisation of the \ref LeafCount struct when the node type is const TensorMap +#define SYCLTENSORMAPLEAFCOUNT(CVQual)\ +template <typename PlainObjectType, int Options_, template <class> class MakePointer_>\ +struct LeafCount<CVQual TensorMap<PlainObjectType, Options_, MakePointer_> > {\ + static const size_t Count =1;\ +}; + +SYCLTENSORMAPLEAFCOUNT(const) +SYCLTENSORMAPLEAFCOUNT() +#undef SYCLTENSORMAPLEAFCOUNT + +// TensorCwiseUnaryOp, TensorCwiseNullaryOp, TensorCwiseBinaryOp, TensorCwiseTernaryOp, and TensorBroadcastingOp +#define SYCLCATEGORYLEAFCOUNT(CVQual)\ +template <template <class, class...> class CategoryExpr, typename OP, typename... RHSExpr>\ +struct LeafCount<CVQual CategoryExpr<OP, RHSExpr...> >: CategoryCount<RHSExpr...> {}; + +SYCLCATEGORYLEAFCOUNT(const) +SYCLCATEGORYLEAFCOUNT() +#undef SYCLCATEGORYLEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is const TensorSelectOp is an exception +#define SYCLSELECTOPLEAFCOUNT(CVQual)\ +template <typename IfExpr, typename ThenExpr, typename ElseExpr>\ +struct LeafCount<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr> > : CategoryCount<IfExpr, ThenExpr, ElseExpr> {}; + +SYCLSELECTOPLEAFCOUNT(const) +SYCLSELECTOPLEAFCOUNT() +#undef SYCLSELECTOPLEAFCOUNT + + +/// specialisation of the \ref LeafCount struct when the node type is TensorAssignOp +#define SYCLLEAFCOUNTASSIGNOP(CVQual)\ +template <typename LHSExpr, typename RHSExpr>\ +struct LeafCount<CVQual TensorAssignOp<LHSExpr, RHSExpr> >: CategoryCount<LHSExpr,RHSExpr> {}; + +SYCLLEAFCOUNTASSIGNOP(const) +SYCLLEAFCOUNTASSIGNOP() +#undef SYCLLEAFCOUNTASSIGNOP + +/// specialisation of the \ref LeafCount struct when the node type is const TensorForcedEvalOp +#define SYCLFORCEDEVALLEAFCOUNT(CVQual)\ +template <typename Expr>\ +struct LeafCount<CVQual TensorForcedEvalOp<Expr> > {\ + static const size_t Count =1;\ +}; + +SYCLFORCEDEVALLEAFCOUNT(const) +SYCLFORCEDEVALLEAFCOUNT() +#undef SYCLFORCEDEVALLEAFCOUNT + +#define SYCLCUSTOMUNARYOPLEAFCOUNT(CVQual)\ +template <typename CustomUnaryFunc, typename XprType>\ +struct LeafCount<CVQual TensorCustomUnaryOp<CustomUnaryFunc, XprType> > {\ +static const size_t Count =1;\ +}; + +SYCLCUSTOMUNARYOPLEAFCOUNT(const) +SYCLCUSTOMUNARYOPLEAFCOUNT() +#undef SYCLCUSTOMUNARYOPLEAFCOUNT + + +#define SYCLCUSTOMBINARYOPLEAFCOUNT(CVQual)\ +template <typename CustomBinaryFunc, typename LhsXprType, typename RhsXprType>\ +struct LeafCount<CVQual TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType> > {\ +static const size_t Count =1;\ +}; +SYCLCUSTOMBINARYOPLEAFCOUNT( const) +SYCLCUSTOMBINARYOPLEAFCOUNT() +#undef SYCLCUSTOMBINARYOPLEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is TensorEvalToOp +#define EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(CVQual , ExprNode, Num)\ +template <typename Expr>\ +struct LeafCount<CVQual ExprNode<Expr> > {\ + static const size_t Count = Num + CategoryCount<Expr>::Count;\ +}; + +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(const, TensorEvalToOp, 1) +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(, TensorEvalToOp, 1) +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(const, TensorLayoutSwapOp, 0) +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(, TensorLayoutSwapOp, 0) + +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(const, TensorIndexTupleOp, 0) +EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT(, TensorIndexTupleOp, 0) + +#undef EVALTOLAYOUTSWAPINDEXTUPLELEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is const TensorReductionOp +#define REDUCTIONLEAFCOUNT(CVQual, ExprNode)\ +template <typename OP, typename Dim, typename Expr>\ +struct LeafCount<CVQual ExprNode<OP, Dim, Expr> > {\ + static const size_t Count =1;\ +}; + +// TensorReductionOp +REDUCTIONLEAFCOUNT(const,TensorReductionOp) +REDUCTIONLEAFCOUNT(,TensorReductionOp) + +// tensor Argmax -TensorTupleReducerOp +REDUCTIONLEAFCOUNT(const, TensorTupleReducerOp) +REDUCTIONLEAFCOUNT(, TensorTupleReducerOp) + +#undef REDUCTIONLEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is const TensorContractionOp +#define CONTRACTIONCONVOLUTIONLEAFCOUNT(CVQual, ExprNode)\ +template <typename Indices, typename LhsXprType, typename RhsXprType>\ +struct LeafCount<CVQual ExprNode<Indices, LhsXprType, RhsXprType> > {\ + static const size_t Count =1;\ +}; + +CONTRACTIONCONVOLUTIONLEAFCOUNT(const,TensorContractionOp) +CONTRACTIONCONVOLUTIONLEAFCOUNT(,TensorContractionOp) +CONTRACTIONCONVOLUTIONLEAFCOUNT(const,TensorConvolutionOp) +CONTRACTIONCONVOLUTIONLEAFCOUNT(,TensorConvolutionOp) +#undef CONTRACTIONCONVOLUTIONLEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is TensorSlicingOp +#define SLICEOPLEAFCOUNT(CVQual)\ +template <typename StartIndices, typename Sizes, typename XprType>\ +struct LeafCount<CVQual TensorSlicingOp<StartIndices, Sizes, XprType> >:CategoryCount<XprType>{}; + +SLICEOPLEAFCOUNT(const) +SLICEOPLEAFCOUNT() +#undef SLICEOPLEAFCOUNT + +/// specialisation of the \ref LeafCount struct when the node type is TensorChippingOp +#define CHIPPINGOPLEAFCOUNT(CVQual)\ +template <DenseIndex DimId, typename XprType>\ +struct LeafCount<CVQual TensorChippingOp<DimId, XprType> >:CategoryCount<XprType>{}; + +CHIPPINGOPLEAFCOUNT(const) +CHIPPINGOPLEAFCOUNT() +#undef CHIPPINGOPLEAFCOUNT + +///TensorStridingSlicingOp +#define SLICESTRIDEOPLEAFCOUNT(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType>\ +struct LeafCount<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType> >:CategoryCount<XprType>{}; + +SLICESTRIDEOPLEAFCOUNT(const) +SLICESTRIDEOPLEAFCOUNT() +#undef SLICESTRIDEOPLEAFCOUNT + +//TensorImagePatchOp +#define TENSORIMAGEPATCHOPLEAFCOUNT(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename XprType>\ +struct LeafCount<CVQual TensorImagePatchOp<Rows, Cols, XprType> >:CategoryCount<XprType>{}; + + +TENSORIMAGEPATCHOPLEAFCOUNT(const) +TENSORIMAGEPATCHOPLEAFCOUNT() +#undef TENSORIMAGEPATCHOPLEAFCOUNT + +// TensorVolumePatchOp +#define TENSORVOLUMEPATCHOPLEAFCOUNT(CVQual)\ +template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType>\ +struct LeafCount<CVQual TensorVolumePatchOp<Planes, Rows, Cols, XprType> >:CategoryCount<XprType>{}; + +TENSORVOLUMEPATCHOPLEAFCOUNT(const) +TENSORVOLUMEPATCHOPLEAFCOUNT() +#undef TENSORVOLUMEPATCHOPLEAFCOUNT + +} /// namespace TensorSycl +} /// namespace internal +} /// namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_LEAF_COUNT_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclPlaceHolderExpr.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclPlaceHolderExpr.h new file mode 100644 index 0000000..9d5708f --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclPlaceHolderExpr.h
@@ -0,0 +1,302 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclPlaceHolderExpr.h + * + * \brief: + * This is the specialisation of the placeholder expression based on the + * operation type + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_PLACEHOLDER_EXPR_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_PLACEHOLDER_EXPR_HPP + +namespace Eigen { +namespace TensorSycl { +namespace internal { + +/// \struct PlaceHolder +/// \brief PlaceHolder is used to replace the \ref TensorMap in the expression +/// tree. +/// PlaceHolder contains the order of the leaf node in the expression tree. +template <typename Scalar, size_t N> +struct PlaceHolder { + static constexpr size_t I = N; + typedef Scalar Type; +}; + +/// \sttruct PlaceHolderExpression +/// \brief it is used to create the PlaceHolder expression. The PlaceHolder +/// expression is a copy of expression type in which the TensorMap of the has +/// been replaced with PlaceHolder. +template <typename Expr, size_t N> +struct PlaceHolderExpression; + +template<size_t N, typename... Args> +struct CalculateIndex; + +template<size_t N, typename Arg> +struct CalculateIndex<N, Arg>{ + typedef typename PlaceHolderExpression<Arg, N>::Type ArgType; + typedef utility::tuple::Tuple<ArgType> ArgsTuple; +}; + +template<size_t N, typename Arg1, typename Arg2> +struct CalculateIndex<N, Arg1, Arg2>{ + static const size_t Arg2LeafCount = LeafCount<Arg2>::Count; + typedef typename PlaceHolderExpression<Arg1, N - Arg2LeafCount>::Type Arg1Type; + typedef typename PlaceHolderExpression<Arg2, N>::Type Arg2Type; + typedef utility::tuple::Tuple<Arg1Type, Arg2Type> ArgsTuple; +}; + +template<size_t N, typename Arg1, typename Arg2, typename Arg3> +struct CalculateIndex<N, Arg1, Arg2, Arg3> { + static const size_t Arg3LeafCount = LeafCount<Arg3>::Count; + static const size_t Arg2LeafCount = LeafCount<Arg2>::Count; + typedef typename PlaceHolderExpression<Arg1, N - Arg3LeafCount - Arg2LeafCount>::Type Arg1Type; + typedef typename PlaceHolderExpression<Arg2, N - Arg3LeafCount>::Type Arg2Type; + typedef typename PlaceHolderExpression<Arg3, N>::Type Arg3Type; + typedef utility::tuple::Tuple<Arg1Type, Arg2Type, Arg3Type> ArgsTuple; +}; + +template<template<class...> class Category , class OP, class TPL> +struct CategoryHelper; + +template<template<class...> class Category , class OP, class ...T > +struct CategoryHelper<Category, OP, utility::tuple::Tuple<T...> > { + typedef Category<OP, T... > Type; +}; + +template<template<class...> class Category , class ...T > +struct CategoryHelper<Category, NoOP, utility::tuple::Tuple<T...> > { + typedef Category<T... > Type; +}; + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorCwiseNullaryOp, TensorCwiseUnaryOp, TensorBroadcastingOp, TensorCwiseBinaryOp, TensorCwiseTernaryOp +#define OPEXPRCATEGORY(CVQual)\ +template <template <class, class... > class Category, typename OP, typename... SubExpr, size_t N>\ +struct PlaceHolderExpression<CVQual Category<OP, SubExpr...>, N>{\ + typedef CVQual typename CategoryHelper<Category, OP, typename CalculateIndex<N, SubExpr...>::ArgsTuple>::Type Type;\ +}; + +OPEXPRCATEGORY(const) +OPEXPRCATEGORY() +#undef OPEXPRCATEGORY + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorCwiseSelectOp +#define SELECTEXPR(CVQual)\ +template <typename IfExpr, typename ThenExpr, typename ElseExpr, size_t N>\ +struct PlaceHolderExpression<CVQual TensorSelectOp<IfExpr, ThenExpr, ElseExpr>, N> {\ + typedef CVQual typename CategoryHelper<TensorSelectOp, NoOP, typename CalculateIndex<N, IfExpr, ThenExpr, ElseExpr>::ArgsTuple>::Type Type;\ +}; + +SELECTEXPR(const) +SELECTEXPR() +#undef SELECTEXPR + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorAssignOp +#define ASSIGNEXPR(CVQual)\ +template <typename LHSExpr, typename RHSExpr, size_t N>\ +struct PlaceHolderExpression<CVQual TensorAssignOp<LHSExpr, RHSExpr>, N> {\ + typedef CVQual typename CategoryHelper<TensorAssignOp, NoOP, typename CalculateIndex<N, LHSExpr, RHSExpr>::ArgsTuple>::Type Type;\ +}; + +ASSIGNEXPR(const) +ASSIGNEXPR() +#undef ASSIGNEXPR + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorMap +#define TENSORMAPEXPR(CVQual)\ +template <typename T, int Options_, template <class> class MakePointer_, size_t N>\ +struct PlaceHolderExpression< CVQual TensorMap< T, Options_, MakePointer_>, N> {\ + typedef CVQual PlaceHolder<CVQual TensorMap<T, Options_, MakePointer_>, N> Type;\ +}; + +TENSORMAPEXPR(const) +TENSORMAPEXPR() +#undef TENSORMAPEXPR + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorForcedEvalOp +#define FORCEDEVAL(CVQual)\ +template <typename Expr, size_t N>\ +struct PlaceHolderExpression<CVQual TensorForcedEvalOp<Expr>, N> {\ + typedef CVQual PlaceHolder<CVQual TensorForcedEvalOp<Expr>, N> Type;\ +}; + +FORCEDEVAL(const) +FORCEDEVAL() +#undef FORCEDEVAL + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorForcedEvalOp +#define CUSTOMUNARYOPEVAL(CVQual)\ +template <typename CustomUnaryFunc, typename XprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorCustomUnaryOp<CustomUnaryFunc, XprType>, N> {\ + typedef CVQual PlaceHolder<CVQual TensorCustomUnaryOp<CustomUnaryFunc, XprType>, N> Type;\ +}; + +CUSTOMUNARYOPEVAL(const) +CUSTOMUNARYOPEVAL() +#undef CUSTOMUNARYOPEVAL + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorForcedEvalOp +#define CUSTOMBINARYOPEVAL(CVQual)\ +template <typename CustomBinaryFunc, typename LhsXprType, typename RhsXprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType>, N> {\ + typedef CVQual PlaceHolder<CVQual TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType>, N> Type;\ +}; + +CUSTOMBINARYOPEVAL(const) +CUSTOMBINARYOPEVAL() +#undef CUSTOMBINARYOPEVAL + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensoroOp, TensorLayoutSwapOp, and TensorIndexTupleOp +#define EVALTOLAYOUTSWAPINDEXTUPLE(CVQual, ExprNode)\ +template <typename Expr, size_t N>\ +struct PlaceHolderExpression<CVQual ExprNode<Expr>, N> {\ + typedef CVQual ExprNode<typename CalculateIndex <N, Expr>::ArgType> Type;\ +}; + +// TensorEvalToOp +EVALTOLAYOUTSWAPINDEXTUPLE(const, TensorEvalToOp) +EVALTOLAYOUTSWAPINDEXTUPLE(, TensorEvalToOp) +//TensorLayoutSwapOp +EVALTOLAYOUTSWAPINDEXTUPLE(const, TensorLayoutSwapOp) +EVALTOLAYOUTSWAPINDEXTUPLE(, TensorLayoutSwapOp) +//TensorIndexTupleOp +EVALTOLAYOUTSWAPINDEXTUPLE(const, TensorIndexTupleOp) +EVALTOLAYOUTSWAPINDEXTUPLE(, TensorIndexTupleOp) + +#undef EVALTOLAYOUTSWAPINDEXTUPLE + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorChippingOp +#define CHIPPINGOP(CVQual)\ +template <DenseIndex DimId, typename Expr, size_t N>\ +struct PlaceHolderExpression<CVQual TensorChippingOp<DimId, Expr>, N> {\ + typedef CVQual TensorChippingOp< DimId, typename CalculateIndex <N, Expr>::ArgType> Type;\ +}; + +CHIPPINGOP(const) +CHIPPINGOP() +#undef CHIPPINGOP + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorReductionOp and TensorTupleReducerOp (Argmax) +#define SYCLREDUCTION(CVQual, ExprNode)\ +template <typename OP, typename Dims, typename Expr, size_t N>\ +struct PlaceHolderExpression<CVQual ExprNode<OP, Dims, Expr>, N>{\ + typedef CVQual PlaceHolder<CVQual ExprNode<OP, Dims,Expr>, N> Type;\ +}; + +// tensor reduction +SYCLREDUCTION(const, TensorReductionOp) +SYCLREDUCTION(, TensorReductionOp) + +// tensor Argmax -TensorTupleReducerOp +SYCLREDUCTION(const, TensorTupleReducerOp) +SYCLREDUCTION(, TensorTupleReducerOp) +#undef SYCLREDUCTION + + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorReductionOp +#define SYCLCONTRACTIONCONVOLUTIONPLH(CVQual, ExprNode)\ +template <typename Indices, typename LhsXprType, typename RhsXprType, size_t N>\ +struct PlaceHolderExpression<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, N>{\ + typedef CVQual PlaceHolder<CVQual ExprNode<Indices, LhsXprType, RhsXprType>, N> Type;\ +}; +SYCLCONTRACTIONCONVOLUTIONPLH(const, TensorContractionOp) +SYCLCONTRACTIONCONVOLUTIONPLH(,TensorContractionOp) +SYCLCONTRACTIONCONVOLUTIONPLH(const, TensorConvolutionOp) +SYCLCONTRACTIONCONVOLUTIONPLH(,TensorConvolutionOp) +#undef SYCLCONTRACTIONCONVOLUTIONPLH + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorCwiseSelectOp +#define SLICEOPEXPR(CVQual)\ +template <typename StartIndices, typename Sizes, typename XprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorSlicingOp<StartIndices, Sizes, XprType>, N> {\ + typedef CVQual TensorSlicingOp<StartIndices, Sizes, typename CalculateIndex<N, XprType>::ArgType> Type;\ +}; + +SLICEOPEXPR(const) +SLICEOPEXPR() +#undef SLICEOPEXPR + + +#define SYCLSLICESTRIDEOPPLH(CVQual)\ +template<typename StartIndices, typename StopIndices, typename Strides, typename XprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, XprType>, N> {\ + typedef CVQual TensorStridingSlicingOp<StartIndices, StopIndices, Strides, typename CalculateIndex<N, XprType>::ArgType> Type;\ +}; + +SYCLSLICESTRIDEOPPLH(const) +SYCLSLICESTRIDEOPPLH() +#undef SYCLSLICESTRIDEOPPLH + + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorImagePatchOp +#define SYCLTENSORIMAGEPATCHOP(CVQual)\ +template<DenseIndex Rows, DenseIndex Cols, typename XprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorImagePatchOp<Rows, Cols, XprType>, N> {\ + typedef CVQual TensorImagePatchOp<Rows, Cols, typename CalculateIndex <N, XprType>::ArgType> Type;\ +}; + +SYCLTENSORIMAGEPATCHOP(const) +SYCLTENSORIMAGEPATCHOP() +#undef SYCLTENSORIMAGEPATCHOP + + + +/// specialisation of the \ref PlaceHolderExpression when the node is +/// TensorVolumePatchOp +#define SYCLTENSORVOLUMEPATCHOP(CVQual)\ +template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType, size_t N>\ +struct PlaceHolderExpression<CVQual TensorVolumePatchOp<Planes,Rows, Cols, XprType>, N> {\ + typedef CVQual TensorVolumePatchOp<Planes,Rows, Cols, typename CalculateIndex <N, XprType>::ArgType> Type;\ +}; + +SYCLTENSORVOLUMEPATCHOP(const) +SYCLTENSORVOLUMEPATCHOP() +#undef SYCLTENSORVOLUMEPATCHOP + + +/// template deduction for \ref PlaceHolderExpression struct +template <typename Expr> +struct createPlaceHolderExpression { + static const size_t TotalLeaves = LeafCount<Expr>::Count; + typedef typename PlaceHolderExpression<Expr, TotalLeaves - 1>::Type Type; +}; + +} // internal +} // TensorSycl +} // namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_PLACEHOLDER_EXPR_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclRun.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclRun.h new file mode 100644 index 0000000..29c7818 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclRun.h
@@ -0,0 +1,96 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Cummins Chris PhD student at The University of Edinburgh. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensorSyclRun.h + * + * \brief: + * Schedule_kernel invoke an specialised version of kernel struct. The + * specialisation is based on the data dimension in sycl buffer + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_SYCLRUN_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_SYCLRUN_HPP + +namespace Eigen { +namespace TensorSycl { +template<typename Expr, typename FunctorExpr, typename TupleType > struct ExecExprFunctorKernel{ + typedef typename internal::createPlaceHolderExpression<Expr>::Type PlaceHolderExpr; + + typedef typename Expr::Index Index; + FunctorExpr functors; + TupleType tuple_of_accessors; + Index range; + ExecExprFunctorKernel(Index range_, FunctorExpr functors_, TupleType tuple_of_accessors_) + : functors(functors_), tuple_of_accessors(tuple_of_accessors_), range(range_){} + void operator()(cl::sycl::nd_item<1> itemID) { + typedef typename internal::ConvertToDeviceExpression<Expr>::Type DevExpr; + auto device_expr =internal::createDeviceExpression<DevExpr, PlaceHolderExpr>(functors, tuple_of_accessors); + auto device_evaluator = Eigen::TensorEvaluator<decltype(device_expr.expr), Eigen::SyclKernelDevice>(device_expr.expr, Eigen::SyclKernelDevice()); + typename DevExpr::Index gId = static_cast<typename DevExpr::Index>(itemID.get_global_linear_id()); + if (gId < range) + device_evaluator.evalScalar(gId); + } +}; + +/// The run function in tensor sycl convert the expression tree to a buffer +/// based expression tree; +/// creates the expression tree for the device with accessor to buffers; +/// construct the kernel and submit it to the sycl queue. +/// std::array does not have TotalSize. So I have to get the size through template specialisation. +template<typename , typename Dimensions> struct DimensionSize{ + static auto getDimSize(const Dimensions& dim)->decltype(dim.TotalSize()){ + return dim.TotalSize(); + } +}; +#define DIMSIZEMACRO(CVQual)\ +template<typename Index, size_t NumDims> struct DimensionSize<Index, CVQual std::array<Index, NumDims>>{\ + static inline Index getDimSize(const std::array<Index, NumDims>& dim){\ + return (NumDims == 0) ? 1 : ::Eigen::internal::array_prod(dim);\ + }\ +}; + +DIMSIZEMACRO(const) +DIMSIZEMACRO() +#undef DIMSIZEMACRO + + +template <typename Expr, typename Dev> +void run(Expr &expr, Dev &dev) { + Eigen::TensorEvaluator<Expr, Dev> evaluator(expr, dev); + const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); + if (needs_assign) { + typedef Eigen::TensorSycl::internal::FunctorExtractor<Eigen::TensorEvaluator<Expr, Dev> > FunctorExpr; + FunctorExpr functors = internal::extractFunctors(evaluator); + dev.sycl_queue().submit([&](cl::sycl::handler &cgh) { + // create a tuple of accessors from Evaluator + typedef decltype(internal::createTupleOfAccessors<Eigen::TensorEvaluator<Expr, Dev> >(cgh, evaluator)) TupleType; + TupleType tuple_of_accessors = internal::createTupleOfAccessors<Eigen::TensorEvaluator<Expr, Dev> >(cgh, evaluator); + typename Expr::Index range, GRange, tileSize; + typename Expr::Index total_size = static_cast<typename Expr::Index>(DimensionSize<typename Expr::Index, typename Eigen::TensorEvaluator<Expr, Dev>::Dimensions>::getDimSize(evaluator.dimensions())); + dev.parallel_for_setup(total_size, tileSize, range, GRange); + + cgh.parallel_for(cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), + ExecExprFunctorKernel<Expr,FunctorExpr,TupleType>(range + , functors, tuple_of_accessors + )); + }); + dev.asynchronousExec(); + } + evaluator.cleanup(); +} +} // namespace TensorSycl +} // namespace Eigen + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_SYCLRUN_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorSyclTuple.h b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclTuple.h new file mode 100644 index 0000000..5385c7e --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorSyclTuple.h
@@ -0,0 +1,239 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/***************************************************************** + * TensroSyclTuple.h + * + * \brief: + * Minimal implementation of std::tuple that can be used inside a SYCL kernel. + * +*****************************************************************/ + +#ifndef UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_TUPLE_HPP +#define UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_TUPLE_HPP + +namespace utility { +namespace tuple { +/// \struct StaticIf +/// \brief The StaticIf struct is used to statically choose the type based on the +/// condition. +template <bool, typename T = void> struct StaticIf; +/// \brief specialisation of the \ref StaticIf when the condition is true +template <typename T> +struct StaticIf<true, T> { + typedef T type; +}; + +/// \struct Tuple +/// \brief is a fixed-size collection of heterogeneous values +/// \tparam Ts... - the types of the elements that the tuple stores. +/// Empty list is supported. +template <class... Ts> +struct Tuple {}; + +/// \brief specialisation of the \ref Tuple class when the tuple has at least +/// one element. +/// \tparam T : the type of the first element in the tuple. +/// \tparam Ts... the rest of the elements in the tuple. Ts... can be empty. +template <class T, class... Ts> +struct Tuple<T, Ts...> { + Tuple(T t, Ts... ts) : head(t), tail(ts...) {} + T head; + Tuple<Ts...> tail; +}; + +///\ struct ElemTypeHolder +/// \brief ElemTypeHolder class is used to specify the types of the +/// elements inside the tuple +/// \tparam size_t the number of elements inside the tuple +/// \tparam class the tuple class +template <size_t, class> +struct ElemTypeHolder; + +/// \brief specialisation of the \ref ElemTypeHolder class when the number of +/// elements inside the tuple is 1 +template <class T, class... Ts> +struct ElemTypeHolder<0, Tuple<T, Ts...> > { + typedef T type; +}; + +/// \brief specialisation of the \ref ElemTypeHolder class when the number of +/// elements inside the tuple is bigger than 1. It recursively calls itself to +/// detect the type of each element in the tuple +/// \tparam T : the type of the first element in the tuple. +/// \tparam Ts... the rest of the elements in the tuple. Ts... can be empty. +/// \tparam K is the Kth element in the tuple +template <size_t k, class T, class... Ts> +struct ElemTypeHolder<k, Tuple<T, Ts...> > { + typedef typename ElemTypeHolder<k - 1, Tuple<Ts...> >::type type; +}; + +/// get +/// \brief Extracts the first element from the tuple. +/// K=0 represents the first element of the tuple. The tuple cannot be empty. +/// \tparam Ts... are the type of the elements in the tuple. +/// \param t is the tuple whose contents to extract +/// \return typename ElemTypeHolder<0, Tuple<Ts...> >::type &>::type + +#define TERMINATE_CONDS_TUPLE_GET(CVQual) \ +template <size_t k, class... Ts> \ +typename StaticIf<k == 0, CVQual typename ElemTypeHolder<0, Tuple<Ts...> >::type &>::type \ +get(CVQual Tuple<Ts...> &t) { \ + static_assert(sizeof...(Ts)!=0, "The requseted value is bigger than the size of the tuple"); \ + return t.head; \ +} + +TERMINATE_CONDS_TUPLE_GET(const) +TERMINATE_CONDS_TUPLE_GET() +#undef TERMINATE_CONDS_TUPLE_GET +/// get +/// \brief Extracts the Kth element from the tuple. +///\tparam K is an integer value in [0,sizeof...(Types)). +/// \tparam T is the (sizeof...(Types) -(K+1)) element in the tuple +/// \tparam Ts... are the type of the elements in the tuple. +/// \param t is the tuple whose contents to extract +/// \return typename ElemTypeHolder<K, Tuple<Ts...> >::type &>::type +#define RECURSIVE_TUPLE_GET(CVQual) \ +template <size_t k, class T, class... Ts> \ +typename StaticIf<k != 0, CVQual typename ElemTypeHolder<k, Tuple<T, Ts...> >::type &>::type \ +get(CVQual Tuple<T, Ts...> &t) { \ + return utility::tuple::get<k - 1>(t.tail); \ +} +RECURSIVE_TUPLE_GET(const) +RECURSIVE_TUPLE_GET() +#undef RECURSIVE_TUPLE_GET + +/// make_tuple +/// \brief Creates a tuple object, deducing the target type from the types of +/// arguments. +/// \tparam Args the type of the arguments to construct the tuple from +/// \param args zero or more arguments to construct the tuple from +/// \return Tuple<Args...> +template <typename... Args> +Tuple<Args...> make_tuple(Args... args) { + return Tuple<Args...>(args...); +} + +/// size +/// \brief Provides access to the number of elements in a tuple as a +/// compile-time constant expression. +/// \tparam Args the type of the arguments to construct the tuple from +/// \return size_t +template <typename... Args> +static constexpr size_t size(Tuple<Args...> &) { + return sizeof...(Args); +} + +/// \struct IndexList +/// \brief Creates a list of index from the elements in the tuple +/// \tparam Is... a list of index from [0 to sizeof...(tuple elements)) +template <size_t... Is> +struct IndexList {}; + +/// \struct RangeBuilder +/// \brief Collects internal details for generating index ranges [MIN, MAX) +/// Declare primary template for index range builder +/// \tparam MIN is the starting index in the tuple +/// \tparam N represents sizeof..(elements)- sizeof...(Is) +/// \tparam Is... are the list of generated index so far +template <size_t MIN, size_t N, size_t... Is> +struct RangeBuilder; + +// FIXME Doxygen has problems with recursive inheritance +#ifndef EIGEN_PARSED_BY_DOXYGEN +/// \brief base Step: Specialisation of the \ref RangeBuilder when the +/// MIN==MAX. In this case the Is... is [0 to sizeof...(tuple elements)) +/// \tparam MIN is the starting index of the tuple +/// \tparam Is is [0 to sizeof...(tuple elements)) +template <size_t MIN, size_t... Is> +struct RangeBuilder<MIN, MIN, Is...> { + typedef IndexList<Is...> type; +}; + +/// Induction step: Specialisation of the RangeBuilder class when N!=MIN +/// in this case we are recursively subtracting N by one and adding one +/// index to Is... list until MIN==N +/// \tparam MIN is the starting index in the tuple +/// \tparam N represents sizeof..(elements)- sizeof...(Is) +/// \tparam Is... are the list of generated index so far +template <size_t MIN, size_t N, size_t... Is> +struct RangeBuilder : public RangeBuilder<MIN, N - 1, N - 1, Is...> {}; +#endif // EIGEN_PARSED_BY_DOXYGEN + +/// \brief IndexRange that returns a [MIN, MAX) index range +/// \tparam MIN is the starting index in the tuple +/// \tparam MAX is the size of the tuple +template <size_t MIN, size_t MAX> +struct IndexRange: RangeBuilder<MIN, MAX>::type {}; + +/// append_base +/// \brief unpacking the elements of the input tuple t and creating a new tuple +/// by adding element a at the end of it. +///\tparam Args... the type of the elements inside the tuple t +/// \tparam T the type of the new element going to be added at the end of tuple +/// \tparam I... is the list of index from [0 to sizeof...(t)) +/// \param t the tuple on which we want to append a. +/// \param a the new elements going to be added to the tuple +/// \return Tuple<Args..., T> +template <typename... Args, typename T, size_t... I> +Tuple<Args..., T> append_base(Tuple<Args...> t, T a,IndexList<I...>) { + return utility::tuple::make_tuple(get<I>(t)..., a); +} + +/// append +/// \brief the deduction function for \ref append_base that automatically +/// generate the \ref IndexRange +///\tparam Args... the type of the elements inside the tuple t +/// \tparam T the type of the new element going to be added at the end of tuple +/// \param t the tuple on which we want to append a. +/// \param a the new elements going to be added to the tuple +/// \return Tuple<Args..., T> +template <typename... Args, typename T> +Tuple<Args..., T> append(Tuple<Args...> t, T a) { + return utility::tuple::append_base(t, a, IndexRange<0, sizeof...(Args)>()); +} + +/// append_base +/// \brief This is a specialisation of \ref append_base when we want to +/// concatenate +/// tuple t2 at the end of the tuple t1. Here we unpack both tuples, generate the +/// IndexRange for each of them and create an output tuple T that contains both +/// elements of t1 and t2. +///\tparam Args1... the type of the elements inside the tuple t1 +///\tparam Args2... the type of the elements inside the tuple t2 +/// \tparam I1... is the list of index from [0 to sizeof...(t1)) +/// \tparam I2... is the list of index from [0 to sizeof...(t2)) +/// \param t1 is the tuple on which we want to append t2. +/// \param t2 is the tuple that is going to be added on t1. +/// \return Tuple<Args1..., Args2...> +template <typename... Args1, typename... Args2, size_t... I1, size_t... I2> +Tuple<Args1..., Args2...> append_base(Tuple<Args1...> t1, Tuple<Args2...> t2, IndexList<I1...>, IndexList<I2...>) { + return utility::tuple::make_tuple(get<I1>(t1)...,get<I2>(t2)...); +} + +/// append +/// \brief deduction function for \ref append_base when we are appending tuple +/// t1 by tuple t2. In this case the \ref IndexRange for both tuple are +/// automatically generated. +///\tparam Args1... the type of the elements inside the tuple t1 +///\tparam Args2... the type of the elements inside the tuple t2 +/// \param t1 is the tuple on which we want to append t2. +/// \param t2 is the tuple that is going to be added on t1. +/// \return Tuple<Args1..., Args2...> +template <typename... Args1, typename... Args2> +Tuple<Args1..., Args2...> append(Tuple<Args1...> t1,Tuple<Args2...> t2) { + return utility::tuple::append_base(t1, t2, IndexRange<0, sizeof...(Args1)>(), IndexRange<0, sizeof...(Args2)>()); +} +} // tuple +} // utility + +#endif // UNSUPPORTED_EIGEN_CXX11_SRC_TENSOR_TENSORSYCL_TUPLE_HPP
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h b/unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h new file mode 100644 index 0000000..9fc54a4 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h
@@ -0,0 +1,290 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gagan Goel <gagan.nith@gmail.com> +// Copyright (C) 2017 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_TRACE_H +#define EIGEN_CXX11_TENSOR_TENSOR_TRACE_H + +namespace Eigen { + +/** \class TensorTrace + * \ingroup CXX11_Tensor_Module + * + * \brief Tensor Trace class. + * + * + */ + +namespace internal { +template<typename Dims, typename XprType> +struct traits<TensorTraceOp<Dims, XprType> > : public traits<XprType> +{ + typedef typename XprType::Scalar Scalar; + typedef traits<XprType> XprTraits; + typedef typename XprTraits::StorageKind StorageKind; + typedef typename XprTraits::Index Index; + typedef typename XprType::Nested Nested; + typedef typename remove_reference<Nested>::type _Nested; + static const int NumDimensions = XprTraits::NumDimensions - array_size<Dims>::value; + static const int Layout = XprTraits::Layout; +}; + +template<typename Dims, typename XprType> +struct eval<TensorTraceOp<Dims, XprType>, Eigen::Dense> +{ + typedef const TensorTraceOp<Dims, XprType>& type; +}; + +template<typename Dims, typename XprType> +struct nested<TensorTraceOp<Dims, XprType>, 1, typename eval<TensorTraceOp<Dims, XprType> >::type> +{ + typedef TensorTraceOp<Dims, XprType> type; +}; + +} // end namespace internal + + +template<typename Dims, typename XprType> +class TensorTraceOp : public TensorBase<TensorTraceOp<Dims, XprType> > +{ + public: + typedef typename Eigen::internal::traits<TensorTraceOp>::Scalar Scalar; + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename Eigen::internal::nested<TensorTraceOp>::type Nested; + typedef typename Eigen::internal::traits<TensorTraceOp>::StorageKind StorageKind; + typedef typename Eigen::internal::traits<TensorTraceOp>::Index Index; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorTraceOp(const XprType& expr, const Dims& dims) + : m_xpr(expr), m_dims(dims) { + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const Dims& dims() const { return m_dims; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const typename internal::remove_all<typename XprType::Nested>::type& expression() const { return m_xpr; } + + protected: + typename XprType::Nested m_xpr; + const Dims m_dims; +}; + + +// Eval as rvalue +template<typename Dims, typename ArgType, typename Device> +struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> +{ + typedef TensorTraceOp<Dims, ArgType> XprType; + static const int NumInputDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value; + static const int NumReducedDims = internal::array_size<Dims>::value; + static const int NumOutputDims = NumInputDims - NumReducedDims; + typedef typename XprType::Index Index; + typedef DSizes<Index, NumOutputDims> Dimensions; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + + enum { + IsAligned = false, + PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + BlockAccess = false, + PreferBlockAccess = false, + Layout = TensorEvaluator<ArgType, Device>::Layout, + CoordAccess = false, + RawAccess = false + }; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) + : m_impl(op.expression(), device), m_traceDim(1), m_device(device) + { + + EIGEN_STATIC_ASSERT((NumOutputDims >= 0), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumReducedDims >= 2) || ((NumReducedDims == 0) && (NumInputDims == 0)), YOU_MADE_A_PROGRAMMING_MISTAKE); + + for (int i = 0; i < NumInputDims; ++i) { + m_reduced[i] = false; + } + + const Dims& op_dims = op.dims(); + for (int i = 0; i < NumReducedDims; ++i) { + eigen_assert(op_dims[i] >= 0); + eigen_assert(op_dims[i] < NumInputDims); + m_reduced[op_dims[i]] = true; + } + + // All the dimensions should be distinct to compute the trace + int num_distinct_reduce_dims = 0; + for (int i = 0; i < NumInputDims; ++i) { + if (m_reduced[i]) { + ++num_distinct_reduce_dims; + } + } + + eigen_assert(num_distinct_reduce_dims == NumReducedDims); + + // Compute the dimensions of the result. + const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); + + int output_index = 0; + int reduced_index = 0; + for (int i = 0; i < NumInputDims; ++i) { + if (m_reduced[i]) { + m_reducedDims[reduced_index] = input_dims[i]; + if (reduced_index > 0) { + // All the trace dimensions must have the same size + eigen_assert(m_reducedDims[0] == m_reducedDims[reduced_index]); + } + ++reduced_index; + } + else { + m_dimensions[output_index] = input_dims[i]; + ++output_index; + } + } + + if (NumReducedDims != 0) { + m_traceDim = m_reducedDims[0]; + } + + // Compute the output strides + if (NumOutputDims > 0) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + m_outputStrides[0] = 1; + for (int i = 1; i < NumOutputDims; ++i) { + m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; + } + } + else { + m_outputStrides.back() = 1; + for (int i = NumOutputDims - 2; i >= 0; --i) { + m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; + } + } + } + + // Compute the input strides + if (NumInputDims > 0) { + array<Index, NumInputDims> input_strides; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + input_strides[0] = 1; + for (int i = 1; i < NumInputDims; ++i) { + input_strides[i] = input_strides[i - 1] * input_dims[i - 1]; + } + } + else { + input_strides.back() = 1; + for (int i = NumInputDims - 2; i >= 0; --i) { + input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; + } + } + + output_index = 0; + reduced_index = 0; + for (int i = 0; i < NumInputDims; ++i) { + if(m_reduced[i]) { + m_reducedStrides[reduced_index] = input_strides[i]; + ++reduced_index; + } + else { + m_preservedStrides[output_index] = input_strides[i]; + ++output_index; + } + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { + return m_dimensions; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* /*data*/) { + m_impl.evalSubExprsIfNeeded(NULL); + return true; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { + m_impl.cleanup(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const + { + // Initialize the result + CoeffReturnType result = internal::cast<int, CoeffReturnType>(0); + Index index_stride = 0; + for (int i = 0; i < NumReducedDims; ++i) { + index_stride += m_reducedStrides[i]; + } + + // If trace is requested along all dimensions, starting index would be 0 + Index cur_index = 0; + if (NumOutputDims != 0) + cur_index = firstInput(index); + for (Index i = 0; i < m_traceDim; ++i) { + result += m_impl.coeff(cur_index); + cur_index += index_stride; + } + + return result; + } + + template<int LoadMode> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { + + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE); + eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); + + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[PacketSize]; + for (int i = 0; i < PacketSize; ++i) { + values[i] = coeff(index + i); + } + PacketReturnType result = internal::ploadt<PacketReturnType, LoadMode>(values); + return result; + } + + protected: + // Given the output index, finds the first index in the input tensor used to compute the trace + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const { + Index startInput = 0; + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + for (int i = NumOutputDims - 1; i > 0; --i) { + const Index idx = index / m_outputStrides[i]; + startInput += idx * m_preservedStrides[i]; + index -= idx * m_outputStrides[i]; + } + startInput += index * m_preservedStrides[0]; + } + else { + for (int i = 0; i < NumOutputDims - 1; ++i) { + const Index idx = index / m_outputStrides[i]; + startInput += idx * m_preservedStrides[i]; + index -= idx * m_outputStrides[i]; + } + startInput += index * m_preservedStrides[NumOutputDims - 1]; + } + return startInput; + } + + Dimensions m_dimensions; + TensorEvaluator<ArgType, Device> m_impl; + // Initialize the size of the trace dimension + Index m_traceDim; + const Device& m_device; + array<bool, NumInputDims> m_reduced; + array<Index, NumReducedDims> m_reducedDims; + array<Index, NumOutputDims> m_outputStrides; + array<Index, NumReducedDims> m_reducedStrides; + array<Index, NumOutputDims> m_preservedStrides; +}; + + +} // End namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_TRACE_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h b/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h index 2f06f84..0a394c8 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h
@@ -28,7 +28,7 @@ #else 0 #endif - || + | #if EIGEN_MAX_ALIGN_BYTES>0 is_dynamic_size_storage #else @@ -40,7 +40,7 @@ }; public: - enum { ret = packet_access_bit}; + enum { ret = packet_access_bit }; }; @@ -54,8 +54,15 @@ static const int Layout = Options_ & RowMajor ? RowMajor : ColMajor; enum { Options = Options_, - Flags = compute_tensor_flags<Scalar_, Options_>::ret | (is_const<Scalar_>::value ? 0 : LvalueBit), + Flags = compute_tensor_flags<Scalar_, Options_>::ret | (is_const<Scalar_>::value ? 0 : LvalueBit) }; + template <typename T> struct MakePointer { + typedef T* Type; + typedef T& RefType; + typedef T ScalarType; + + }; + typedef typename MakePointer<Scalar>::Type PointerType; }; @@ -69,13 +76,20 @@ static const int Layout = Options_ & RowMajor ? RowMajor : ColMajor; enum { Options = Options_, - Flags = compute_tensor_flags<Scalar_, Options_>::ret | (is_const<Scalar_>::value ? 0: LvalueBit), + Flags = compute_tensor_flags<Scalar_, Options_>::ret | (is_const<Scalar_>::value ? 0: LvalueBit) }; + template <typename T> struct MakePointer { + typedef T* Type; + typedef T& RefType; + typedef T ScalarType; + + }; + typedef typename MakePointer<Scalar>::Type PointerType; }; -template<typename PlainObjectType, int Options_> -struct traits<TensorMap<PlainObjectType, Options_> > +template<typename PlainObjectType, int Options_, template <class> class MakePointer_> +struct traits<TensorMap<PlainObjectType, Options_, MakePointer_> > : public traits<PlainObjectType> { typedef traits<PlainObjectType> BaseTraits; @@ -86,8 +100,18 @@ static const int Layout = BaseTraits::Layout; enum { Options = Options_, - Flags = BaseTraits::Flags, + Flags = BaseTraits::Flags }; + template <class T> struct MakePointer { + // Intermediate typedef to workaround MSVC issue. + typedef MakePointer_<T> MakePointerT; + typedef typename MakePointerT::Type Type; + typedef typename MakePointerT::RefType RefType; + typedef typename MakePointerT::ScalarType ScalarType; + + + }; + typedef typename MakePointer<Scalar>::Type PointerType; }; template<typename PlainObjectType> @@ -102,8 +126,9 @@ static const int Layout = BaseTraits::Layout; enum { Options = BaseTraits::Options, - Flags = BaseTraits::Flags, + Flags = BaseTraits::Flags }; + typedef typename BaseTraits::PointerType PointerType; }; @@ -131,16 +156,16 @@ typedef const TensorFixedSize<Scalar_, Dimensions, Options, IndexType_>& type; }; -template<typename PlainObjectType, int Options> -struct eval<TensorMap<PlainObjectType, Options>, Eigen::Dense> +template<typename PlainObjectType, int Options, template <class> class MakePointer> +struct eval<TensorMap<PlainObjectType, Options, MakePointer>, Eigen::Dense> { - typedef const TensorMap<PlainObjectType, Options>& type; + typedef const TensorMap<PlainObjectType, Options, MakePointer>& type; }; -template<typename PlainObjectType, int Options> -struct eval<const TensorMap<PlainObjectType, Options>, Eigen::Dense> +template<typename PlainObjectType, int Options, template <class> class MakePointer> +struct eval<const TensorMap<PlainObjectType, Options, MakePointer>, Eigen::Dense> { - typedef const TensorMap<PlainObjectType, Options>& type; + typedef const TensorMap<PlainObjectType, Options, MakePointer>& type; }; template<typename PlainObjectType> @@ -186,16 +211,16 @@ }; -template <typename PlainObjectType, int Options> -struct nested<TensorMap<PlainObjectType, Options> > +template <typename PlainObjectType, int Options, template <class> class MakePointer> +struct nested<TensorMap<PlainObjectType, Options, MakePointer> > { - typedef const TensorMap<PlainObjectType, Options>& type; + typedef const TensorMap<PlainObjectType, Options, MakePointer>& type; }; -template <typename PlainObjectType, int Options> -struct nested<const TensorMap<PlainObjectType, Options> > +template <typename PlainObjectType, int Options, template <class> class MakePointer> +struct nested<const TensorMap<PlainObjectType, Options, MakePointer> > { - typedef const TensorMap<PlainObjectType, Options>& type; + typedef const TensorMap<PlainObjectType, Options, MakePointer>& type; }; template <typename PlainObjectType> @@ -253,7 +278,7 @@ // Pc=0. typedef enum { PADDING_VALID = 1, - PADDING_SAME = 2, + PADDING_SAME = 2 } PaddingType; } // end namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h b/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h index 5950f38..d23f2e4 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h
@@ -20,8 +20,10 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE operator uint64_t() const { return n; } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static_val() { } + template <typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static_val(const T& v) { + EIGEN_UNUSED_VARIABLE(v); eigen_assert(v == n); } }; @@ -53,7 +55,7 @@ template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE explicit TensorUInt128(const T& x) : high(0), low(x) { - eigen_assert((static_cast<typename conditional<sizeof(T) == 8, uint64_t, uint32_t>::type>(x) <= static_cast<typename conditional<sizeof(LOW) == 8, uint64_t, uint32_t>::type>(NumTraits<LOW>::highest()))); + eigen_assert((static_cast<typename conditional<sizeof(T) == 8, uint64_t, uint32_t>::type>(x) <= NumTraits<uint64_t>::highest())); eigen_assert(x >= 0); } @@ -74,21 +76,21 @@ template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static bool operator == (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +bool operator == (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { return (lhs.high == rhs.high) & (lhs.low == rhs.low); } template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static bool operator != (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +bool operator != (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { return (lhs.high != rhs.high) | (lhs.low != rhs.low); } template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static bool operator >= (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +bool operator >= (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { if (lhs.high != rhs.high) { return lhs.high > rhs.high; @@ -98,7 +100,7 @@ template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static bool operator < (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +bool operator < (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { if (lhs.high != rhs.high) { return lhs.high < rhs.high; @@ -108,7 +110,7 @@ template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static TensorUInt128<uint64_t, uint64_t> operator + (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +TensorUInt128<uint64_t, uint64_t> operator + (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { TensorUInt128<uint64_t, uint64_t> result(lhs.high + rhs.high, lhs.low + rhs.low); if (result.low < rhs.low) { @@ -119,7 +121,7 @@ template <typename HL, typename LL, typename HR, typename LR> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -static TensorUInt128<uint64_t, uint64_t> operator - (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +TensorUInt128<uint64_t, uint64_t> operator - (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { TensorUInt128<uint64_t, uint64_t> result(lhs.high - rhs.high, lhs.low - rhs.low); if (result.low > lhs.low) { @@ -130,8 +132,8 @@ template <typename HL, typename LL, typename HR, typename LR> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -static TensorUInt128<uint64_t, uint64_t> operator * (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +TensorUInt128<uint64_t, uint64_t> operator * (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { // Split each 128-bit integer into 4 32-bit integers, and then do the // multiplications by hand as follow: @@ -205,8 +207,8 @@ } template <typename HL, typename LL, typename HR, typename LR> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE -static TensorUInt128<uint64_t, uint64_t> operator / (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) +static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +TensorUInt128<uint64_t, uint64_t> operator / (const TensorUInt128<HL, LL>& lhs, const TensorUInt128<HR, LR>& rhs) { if (rhs == TensorUInt128<static_val<0>, static_val<1> >(1)) { return TensorUInt128<uint64_t, uint64_t>(lhs.high, lhs.low);
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h index e735fc7..c1b7a58 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h
@@ -22,6 +22,7 @@ * dimensions. */ namespace internal { + template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType> struct traits<TensorVolumePatchOp<Planes, Rows, Cols, XprType> > : public traits<XprType> { @@ -33,6 +34,8 @@ typedef typename remove_reference<Nested>::type _Nested; static const int NumDimensions = XprTraits::NumDimensions + 1; static const int Layout = XprTraits::Layout; + typedef typename XprTraits::PointerType PointerType; + }; template<DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename XprType> @@ -65,12 +68,12 @@ DenseIndex in_plane_strides, DenseIndex in_row_strides, DenseIndex in_col_strides, DenseIndex plane_inflate_strides, DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, PaddingType padding_type, Scalar padding_value) - : m_xpr(expr), m_patch_planes(patch_planes), m_patch_rows(patch_rows), m_patch_cols(patch_cols), - m_plane_strides(plane_strides), m_row_strides(row_strides), m_col_strides(col_strides), - m_in_plane_strides(in_plane_strides), m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), - m_plane_inflate_strides(plane_inflate_strides), m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), - m_padding_explicit(false), m_padding_top_z(0), m_padding_bottom_z(0), m_padding_top(0), m_padding_bottom(0), m_padding_left(0), m_padding_right(0), - m_padding_type(padding_type), m_padding_value(padding_value) {} + : m_xpr(expr), m_patch_planes(patch_planes), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_plane_strides(plane_strides), m_row_strides(row_strides), m_col_strides(col_strides), + m_in_plane_strides(in_plane_strides), m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_plane_inflate_strides(plane_inflate_strides), m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), + m_padding_explicit(false), m_padding_top_z(0), m_padding_bottom_z(0), m_padding_top(0), m_padding_bottom(0), m_padding_left(0), m_padding_right(0), + m_padding_type(padding_type), m_padding_value(padding_value) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorVolumePatchOp(const XprType& expr, DenseIndex patch_planes, DenseIndex patch_rows, DenseIndex patch_cols, DenseIndex plane_strides, DenseIndex row_strides, DenseIndex col_strides, @@ -80,13 +83,31 @@ DenseIndex padding_top, DenseIndex padding_bottom, DenseIndex padding_left, DenseIndex padding_right, Scalar padding_value) - : m_xpr(expr), m_patch_planes(patch_planes), m_patch_rows(patch_rows), m_patch_cols(patch_cols), - m_plane_strides(plane_strides), m_row_strides(row_strides), m_col_strides(col_strides), - m_in_plane_strides(in_plane_strides), m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), - m_plane_inflate_strides(plane_inflate_strides), m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), - m_padding_explicit(true), m_padding_top_z(padding_top_z), m_padding_bottom_z(padding_bottom_z), m_padding_top(padding_top), m_padding_bottom(padding_bottom), - m_padding_left(padding_left), m_padding_right(padding_right), - m_padding_type(PADDING_VALID), m_padding_value(padding_value) {} + : m_xpr(expr), m_patch_planes(patch_planes), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_plane_strides(plane_strides), m_row_strides(row_strides), m_col_strides(col_strides), + m_in_plane_strides(in_plane_strides), m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_plane_inflate_strides(plane_inflate_strides), m_row_inflate_strides(row_inflate_strides), m_col_inflate_strides(col_inflate_strides), + m_padding_explicit(true), m_padding_top_z(padding_top_z), m_padding_bottom_z(padding_bottom_z), m_padding_top(padding_top), m_padding_bottom(padding_bottom), + m_padding_left(padding_left), m_padding_right(padding_right), + m_padding_type(PADDING_VALID), m_padding_value(padding_value) {} + +#ifdef EIGEN_USE_SYCL // this is work around for sycl as Eigen could not use c++11 deligate constructor feature +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorVolumePatchOp(const XprType& expr, DenseIndex patch_planes, DenseIndex patch_rows, DenseIndex patch_cols, + DenseIndex plane_strides, DenseIndex row_strides, DenseIndex col_strides, + DenseIndex in_plane_strides, DenseIndex in_row_strides, DenseIndex in_col_strides, + DenseIndex plane_inflate_strides, DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, + bool padding_explicit, DenseIndex padding_top_z, DenseIndex padding_bottom_z, + DenseIndex padding_top, DenseIndex padding_bottom, DenseIndex padding_left, + DenseIndex padding_right, PaddingType padding_type, Scalar padding_value) + : m_xpr(expr), m_patch_planes(patch_planes), m_patch_rows(patch_rows), m_patch_cols(patch_cols), + m_plane_strides(plane_strides), m_row_strides(row_strides), m_col_strides(col_strides), + m_in_plane_strides(in_plane_strides), m_in_row_strides(in_row_strides), m_in_col_strides(in_col_strides), + m_plane_inflate_strides(plane_inflate_strides), m_row_inflate_strides(row_inflate_strides), + m_col_inflate_strides(col_inflate_strides), m_padding_explicit(padding_explicit), m_padding_top_z(padding_top_z), + m_padding_bottom_z(padding_bottom_z), m_padding_top(padding_top), m_padding_bottom(padding_bottom), m_padding_left(padding_left), + m_padding_right(padding_right), m_padding_type(padding_type), m_padding_value(padding_value) {} + +#endif EIGEN_DEVICE_FUNC DenseIndex patch_planes() const { return m_patch_planes; } @@ -173,21 +194,29 @@ typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size; + static const int PacketSize = PacketType<CoeffReturnType, Device>::size; enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, BlockAccess = false, + PreferBlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, RawAccess = false }; +#ifdef __SYCL_DEVICE_ONLY__ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( const XprType op, const Device& device) +#else + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( const XprType& op, const Device& device) +#endif - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_impl(op.expression(), device) +#ifdef EIGEN_USE_SYCL + , m_op(op) +#endif { - EIGEN_STATIC_ASSERT(NumDims >= 5, YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((NumDims >= 5), YOU_MADE_A_PROGRAMMING_MISTAKE); m_paddingValue = op.padding_value(); @@ -322,6 +351,7 @@ // Fast representations of different variables. m_fastOtherStride = internal::TensorIntDivisor<Index>(m_otherStride); + m_fastPatchStride = internal::TensorIntDivisor<Index>(m_patchStride); m_fastColStride = internal::TensorIntDivisor<Index>(m_colStride); m_fastRowStride = internal::TensorIntDivisor<Index>(m_rowStride); @@ -338,7 +368,6 @@ m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims-1]); } } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* /*data*/) { @@ -408,7 +437,7 @@ template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index+PacketSize-1 < dimensions().TotalSize()); if (m_in_row_strides != 1 || m_in_col_strides != 1 || m_row_inflate_strides != 1 || m_col_inflate_strides != 1 || @@ -502,10 +531,15 @@ return TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC typename Eigen::internal::traits<XprType>::PointerType data() const { return NULL; } const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } +#ifdef EIGEN_USE_SYCL + // Required by SYCL in order to construct the expression on the device + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const XprType& xpr() const { return m_op; } +#endif + Index planePaddingTop() const { return m_planePaddingTop; } Index rowPaddingTop() const { return m_rowPaddingTop; } Index colPaddingLeft() const { return m_colPaddingLeft; } @@ -535,7 +569,7 @@ Dimensions m_dimensions; - // Parameters passed to the costructor. + // Parameters passed to the constructor. Index m_plane_strides; Index m_row_strides; Index m_col_strides; @@ -600,6 +634,12 @@ Scalar m_paddingValue; TensorEvaluator<ArgType, Device> m_impl; + +#ifdef EIGEN_USE_SYCL +// Required by SYCL in order to construct the expression on the device + XprType m_op; +#endif + };
diff --git a/unsupported/Eigen/CXX11/src/TensorSymmetry/CMakeLists.txt b/unsupported/Eigen/CXX11/src/TensorSymmetry/CMakeLists.txt deleted file mode 100644 index 6e871a8..0000000 --- a/unsupported/Eigen/CXX11/src/TensorSymmetry/CMakeLists.txt +++ /dev/null
@@ -1,8 +0,0 @@ -FILE(GLOB Eigen_CXX11_TensorSymmetry_SRCS "*.h") - -INSTALL(FILES - ${Eigen_CXX11_TensorSymmetry_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11/src/TensorSymmetry COMPONENT Devel - ) - -add_subdirectory(util)
diff --git a/unsupported/Eigen/CXX11/src/TensorSymmetry/util/CMakeLists.txt b/unsupported/Eigen/CXX11/src/TensorSymmetry/util/CMakeLists.txt deleted file mode 100644 index dc9fc78..0000000 --- a/unsupported/Eigen/CXX11/src/TensorSymmetry/util/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_CXX11_TensorSymmetry_util_SRCS "*.h") - -INSTALL(FILES - ${Eigen_CXX11_TensorSymmetry_util_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/CXX11/src/TensorSymmetry/util COMPONENT Devel - )
diff --git a/unsupported/Eigen/CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h b/unsupported/Eigen/CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h index 0fe0b7c..54bf9db 100644 --- a/unsupported/Eigen/CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h +++ b/unsupported/Eigen/CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h
@@ -17,7 +17,7 @@ namespace group_theory { /** \internal - * \file CXX11/Tensor/util/TemplateGroupTheory.h + * \file CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h * This file contains C++ templates that implement group theory algorithms. * * The algorithms allow for a compile-time analysis of finite groups. @@ -167,7 +167,9 @@ typename elements, bool dont_add_current_element // = false > -struct dimino_first_step_elements_helper : +struct dimino_first_step_elements_helper +#ifndef EIGEN_PARSED_BY_DOXYGEN + : // recursive inheritance is too difficult for Doxygen public dimino_first_step_elements_helper< Multiply, Equality, @@ -187,6 +189,7 @@ typename elements > struct dimino_first_step_elements_helper<Multiply, Equality, id, g, current_element, elements, true> +#endif // EIGEN_PARSED_BY_DOXYGEN { typedef elements type; constexpr static int global_flags = Equality<current_element, id>::global_flags; @@ -241,7 +244,7 @@ * multiplying all elements in the given subgroup with the new * coset representative. Note that the first element of the * subgroup is always the identity element, so the first element of - * ther result of this template is going to be the coset + * the result of this template is going to be the coset * representative itself. * * Note that this template accepts an additional boolean parameter
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/Barrier.h b/unsupported/Eigen/CXX11/src/ThreadPool/Barrier.h new file mode 100644 index 0000000..bae68e1 --- /dev/null +++ b/unsupported/Eigen/CXX11/src/ThreadPool/Barrier.h
@@ -0,0 +1,64 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Barrier is an object that allows one or more threads to wait until +// Notify has been called a specified number of times. + +#ifndef EIGEN_CXX11_THREADPOOL_BARRIER_H +#define EIGEN_CXX11_THREADPOOL_BARRIER_H + +namespace Eigen { + +class Barrier { + public: + Barrier(unsigned int count) : state_(count << 1), notified_(false) { + eigen_plain_assert(((count << 1) >> 1) == count); + } + ~Barrier() { eigen_plain_assert((state_ >> 1) == 0); } + + void Notify() { + unsigned int v = state_.fetch_sub(2, std::memory_order_acq_rel) - 2; + if (v != 1) { + eigen_plain_assert(((v + 2) & ~1) != 0); + return; // either count has not dropped to 0, or waiter is not waiting + } + std::unique_lock<std::mutex> l(mu_); + eigen_plain_assert(!notified_); + notified_ = true; + cv_.notify_all(); + } + + void Wait() { + unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel); + if ((v >> 1) == 0) return; + std::unique_lock<std::mutex> l(mu_); + while (!notified_) { + cv_.wait(l); + } + } + + private: + std::mutex mu_; + std::condition_variable cv_; + std::atomic<unsigned int> state_; // low bit is waiter flag + bool notified_; +}; + +// Notification is an object that allows a user to to wait for another +// thread to signal a notification that an event has occurred. +// +// Multiple threads can wait on the same Notification object, +// but only one caller must call Notify() on the object. +struct Notification : Barrier { + Notification() : Barrier(1){}; +}; + +} // namespace Eigen + +#endif // EIGEN_CXX11_THREADPOOL_BARRIER_H
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h b/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h index 6dd64f1..7a9ebe4 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h
@@ -33,10 +33,10 @@ // ec.Notify(true); // // Notify is cheap if there are no waiting threads. Prewait/CommitWait are not -// cheap, but they are executed only if the preceeding predicate check has +// cheap, but they are executed only if the preceding predicate check has // failed. // -// Algorihtm outline: +// Algorithm outline: // There are two main variables: predicate (managed by user) and state_. // Operation closely resembles Dekker mutual algorithm: // https://en.wikipedia.org/wiki/Dekker%27s_algorithm @@ -50,15 +50,15 @@ public: class Waiter; - EventCount(std::vector<Waiter>& waiters) : waiters_(waiters) { - eigen_assert(waiters.size() < (1 << kWaiterBits) - 1); + EventCount(MaxSizeVector<Waiter>& waiters) : waiters_(waiters) { + eigen_plain_assert(waiters.size() < (1 << kWaiterBits) - 1); // Initialize epoch to something close to overflow to test overflow. state_ = kStackMask | (kEpochMask - kEpochInc * waiters.size() * 2); } ~EventCount() { // Ensure there are no waiters. - eigen_assert((state_.load() & (kStackMask | kWaiterMask)) == kStackMask); + eigen_plain_assert((state_.load() & (kStackMask | kWaiterMask)) == kStackMask); } // Prewait prepares for waiting. @@ -79,7 +79,7 @@ uint64_t state = state_.load(std::memory_order_seq_cst); for (;;) { if (int64_t((state & kEpochMask) - epoch) < 0) { - // The preceeding waiter has not decided on its fate. Wait until it + // The preceding waiter has not decided on its fate. Wait until it // calls either CancelWait or CommitWait, or is notified. EIGEN_THREAD_YIELD(); state = state_.load(std::memory_order_seq_cst); @@ -88,7 +88,7 @@ // We've already been notified. if (int64_t((state & kEpochMask) - epoch) > 0) return; // Remove this thread from prewait counter and add it to the waiter list. - eigen_assert((state & kWaiterMask) != 0); + eigen_plain_assert((state & kWaiterMask) != 0); uint64_t newstate = state - kWaiterInc + kEpochInc; newstate = (newstate & ~kStackMask) | (w - &waiters_[0]); if ((state & kStackMask) == kStackMask) @@ -110,7 +110,7 @@ uint64_t state = state_.load(std::memory_order_relaxed); for (;;) { if (int64_t((state & kEpochMask) - epoch) < 0) { - // The preceeding waiter has not decided on its fate. Wait until it + // The preceding waiter has not decided on its fate. Wait until it // calls either CancelWait or CommitWait, or is notified. EIGEN_THREAD_YIELD(); state = state_.load(std::memory_order_relaxed); @@ -119,7 +119,7 @@ // We've already been notified. if (int64_t((state & kEpochMask) - epoch) > 0) return; // Remove this thread from prewait counter. - eigen_assert((state & kWaiterMask) != 0); + eigen_plain_assert((state & kWaiterMask) != 0); if (state_.compare_exchange_weak(state, state - kWaiterInc + kEpochInc, std::memory_order_relaxed)) return; @@ -128,7 +128,7 @@ // Notify wakes one or all waiting threads. // Must be called after changing the associated wait predicate. - void Notify(bool all) { + void Notify(bool notifyAll) { std::atomic_thread_fence(std::memory_order_seq_cst); uint64_t state = state_.load(std::memory_order_acquire); for (;;) { @@ -137,7 +137,7 @@ return; uint64_t waiters = (state & kWaiterMask) >> kWaiterShift; uint64_t newstate; - if (all) { + if (notifyAll) { // Reset prewait counter and empty wait list. newstate = (state & kEpochMask) + (kEpochInc * waiters) + kStackMask; } else if (waiters) { @@ -157,10 +157,10 @@ } if (state_.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { - if (!all && waiters) return; // unblocked pre-wait thread + if (!notifyAll && waiters) return; // unblocked pre-wait thread if ((state & kStackMask) == kStackMask) return; Waiter* w = &waiters_[state & kStackMask]; - if (!all) w->next.store(nullptr, std::memory_order_relaxed); + if (!notifyAll) w->next.store(nullptr, std::memory_order_relaxed); Unpark(w); return; } @@ -169,7 +169,9 @@ class Waiter { friend class EventCount; - std::atomic<Waiter*> next; + // Align to 128 byte boundary to prevent false sharing with other Waiter + // objects in the same vector. + EIGEN_ALIGN_TO_BOUNDARY(128) std::atomic<Waiter*> next; std::mutex mu; std::condition_variable cv; uint64_t epoch; @@ -179,8 +181,6 @@ kWaiting, kSignaled, }; - // Prevent false sharing with other Waiter objects in the same vector. - char pad_[128]; }; private: @@ -200,7 +200,7 @@ static const uint64_t kEpochMask = ((1ull << kEpochBits) - 1) << kEpochShift; static const uint64_t kEpochInc = 1ull << kEpochShift; std::atomic<uint64_t> state_; - std::vector<Waiter>& waiters_; + MaxSizeVector<Waiter>& waiters_; void Park(Waiter* w) { std::unique_lock<std::mutex> lock(w->mu);
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/NonBlockingThreadPool.h b/unsupported/Eigen/CXX11/src/ThreadPool/NonBlockingThreadPool.h index 1c471a1..8fafcda 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/NonBlockingThreadPool.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/NonBlockingThreadPool.h
@@ -10,53 +10,115 @@ #ifndef EIGEN_CXX11_THREADPOOL_NONBLOCKING_THREAD_POOL_H #define EIGEN_CXX11_THREADPOOL_NONBLOCKING_THREAD_POOL_H - namespace Eigen { template <typename Environment> -class NonBlockingThreadPoolTempl : public Eigen::ThreadPoolInterface { +class ThreadPoolTempl : public Eigen::ThreadPoolInterface { public: typedef typename Environment::Task Task; typedef RunQueue<Task, 1024> Queue; - NonBlockingThreadPoolTempl(int num_threads, Environment env = Environment()) + ThreadPoolTempl(int num_threads, Environment env = Environment()) + : ThreadPoolTempl(num_threads, true, env) {} + + ThreadPoolTempl(int num_threads, bool allow_spinning, + Environment env = Environment()) : env_(env), - threads_(num_threads), - queues_(num_threads), + num_threads_(num_threads), + allow_spinning_(allow_spinning), + thread_data_(num_threads), + all_coprimes_(num_threads), waiters_(num_threads), - blocked_(), - spinning_(), - done_(), + blocked_(0), + spinning_(0), + done_(false), + cancelled_(false), ec_(waiters_) { - for (int i = 0; i < num_threads; i++) queues_.push_back(new Queue()); - for (int i = 0; i < num_threads; i++) - threads_.push_back(env_.CreateThread([this, i]() { WorkerLoop(i); })); + waiters_.resize(num_threads_); + // Calculate coprimes of all numbers [1, num_threads]. + // Coprimes are used for random walks over all threads in Steal + // and NonEmptyQueueIndex. Iteration is based on the fact that if we take + // a random starting thread index t and calculate num_threads - 1 subsequent + // indices as (t + coprime) % num_threads, we will cover all threads without + // repetitions (effectively getting a presudo-random permutation of thread + // indices). + eigen_plain_assert(num_threads_ < kMaxThreads); + for (int i = 1; i <= num_threads_; ++i) { + all_coprimes_.emplace_back(i); + ComputeCoprimes(i, &all_coprimes_.back()); + } +#ifndef EIGEN_THREAD_LOCAL + init_barrier_.reset(new Barrier(num_threads_)); +#endif + thread_data_.resize(num_threads_); + for (int i = 0; i < num_threads_; i++) { + SetStealPartition(i, EncodePartition(0, num_threads_)); + thread_data_[i].thread.reset( + env_.CreateThread([this, i]() { WorkerLoop(i); })); + } +#ifndef EIGEN_THREAD_LOCAL + // Wait for workers to initialize per_thread_map_. Otherwise we might race + // with them in Schedule or CurrentThreadId. + init_barrier_->Wait(); +#endif } - ~NonBlockingThreadPoolTempl() { - done_.store(true, std::memory_order_relaxed); + ~ThreadPoolTempl() { + done_ = true; + // Now if all threads block without work, they will start exiting. // But note that threads can continue to work arbitrary long, // block, submit new work, unblock and otherwise live full life. - ec_.Notify(true); - - // Join threads explicitly to avoid destruction order issues. - for (size_t i = 0; i < threads_.size(); i++) delete threads_[i]; - for (size_t i = 0; i < threads_.size(); i++) delete queues_[i]; + if (!cancelled_) { + ec_.Notify(true); + } else { + // Since we were cancelled, there might be entries in the queues. + // Empty them to prevent their destructor from asserting. + for (size_t i = 0; i < thread_data_.size(); i++) { + thread_data_[i].queue.Flush(); + } + } + // Join threads explicitly (by destroying) to avoid destruction order within + // this class. + for (size_t i = 0; i < thread_data_.size(); ++i) + thread_data_[i].thread.reset(); } - void Schedule(std::function<void()> fn) { + void SetStealPartitions(const std::vector<std::pair<unsigned, unsigned>>& partitions) { + eigen_plain_assert(partitions.size() == static_cast<std::size_t>(num_threads_)); + + // Pass this information to each thread queue. + for (int i = 0; i < num_threads_; i++) { + const auto& pair = partitions[i]; + unsigned start = pair.first, end = pair.second; + AssertBounds(start, end); + unsigned val = EncodePartition(start, end); + SetStealPartition(i, val); + } + } + + void Schedule(std::function<void()> fn) EIGEN_OVERRIDE { + ScheduleWithHint(std::move(fn), 0, num_threads_); + } + + void ScheduleWithHint(std::function<void()> fn, int start, + int limit) override { Task t = env_.CreateTask(std::move(fn)); PerThread* pt = GetPerThread(); if (pt->pool == this) { // Worker thread of this pool, push onto the thread's queue. - Queue* q = queues_[pt->index]; - t = q->PushFront(std::move(t)); + Queue& q = thread_data_[pt->thread_id].queue; + t = q.PushFront(std::move(t)); } else { // A free-standing thread (or worker of another pool), push onto a random // queue. - Queue* q = queues_[Rand(&pt->rand) % queues_.size()]; - t = q->PushBack(std::move(t)); + eigen_plain_assert(start < limit); + eigen_plain_assert(limit <= num_threads_); + int num_queues = limit - start; + int rnd = Rand(&pt->rand) % num_queues; + eigen_plain_assert(start + rnd < limit); + Queue& q = thread_data_[start + rnd].queue; + t = q.PushBack(std::move(t)); } // Note: below we touch this after making w available to worker threads. // Strictly speaking, this can lead to a racy-use-after-free. Consider that @@ -65,128 +127,278 @@ // completes overall computations, which in turn leads to destruction of // this. We expect that such scenario is prevented by program, that is, // this is kept alive while any threads can potentially be in Schedule. - if (!t.f) + if (!t.f) { ec_.Notify(false); - else + } else { env_.ExecuteTask(t); // Push failed, execute directly. + } + } + + void Cancel() EIGEN_OVERRIDE { + cancelled_ = true; + done_ = true; + + // Let each thread know it's been cancelled. +#ifdef EIGEN_THREAD_ENV_SUPPORTS_CANCELLATION + for (size_t i = 0; i < thread_data_.size(); i++) { + thread_data_[i].thread->OnCancel(); + } +#endif + + // Wake up the threads without work to let them exit on their own. + ec_.Notify(true); + } + + int NumThreads() const EIGEN_FINAL { return num_threads_; } + + int CurrentThreadId() const EIGEN_FINAL { + const PerThread* pt = const_cast<ThreadPoolTempl*>(this)->GetPerThread(); + if (pt->pool == this) { + return pt->thread_id; + } else { + return -1; + } } private: + // Create a single atomic<int> that encodes start and limit information for + // each thread. + // We expect num_threads_ < 65536, so we can store them in a single + // std::atomic<unsigned>. + // Exposed publicly as static functions so that external callers can reuse + // this encode/decode logic for maintaining their own thread-safe copies of + // scheduling and steal domain(s). + static const int kMaxPartitionBits = 16; + static const int kMaxThreads = 1 << kMaxPartitionBits; + + inline unsigned EncodePartition(unsigned start, unsigned limit) { + return (start << kMaxPartitionBits) | limit; + } + + inline void DecodePartition(unsigned val, unsigned* start, unsigned* limit) { + *limit = val & (kMaxThreads - 1); + val >>= kMaxPartitionBits; + *start = val; + } + + void AssertBounds(int start, int end) { + eigen_plain_assert(start >= 0); + eigen_plain_assert(start < end); // non-zero sized partition + eigen_plain_assert(end <= num_threads_); + } + + inline void SetStealPartition(size_t i, unsigned val) { + thread_data_[i].steal_partition.store(val, std::memory_order_relaxed); + } + + inline unsigned GetStealPartition(int i) { + return thread_data_[i].steal_partition.load(std::memory_order_relaxed); + } + + void ComputeCoprimes(int N, MaxSizeVector<unsigned>* coprimes) { + for (int i = 1; i <= N; i++) { + unsigned a = i; + unsigned b = N; + // If GCD(a, b) == 1, then a and b are coprimes. + while (b != 0) { + unsigned tmp = a; + a = b; + b = tmp % b; + } + if (a == 1) { + coprimes->push_back(i); + } + } + } + typedef typename Environment::EnvThread Thread; struct PerThread { - bool inited; - NonBlockingThreadPoolTempl* pool; // Parent pool, or null for normal threads. - unsigned index; // Worker thread index in pool. - unsigned rand; // Random generator state. + constexpr PerThread() : pool(NULL), rand(0), thread_id(-1) {} + ThreadPoolTempl* pool; // Parent pool, or null for normal threads. + uint64_t rand; // Random generator state. + int thread_id; // Worker thread index in pool. +#ifndef EIGEN_THREAD_LOCAL + // Prevent false sharing. + char pad_[128]; +#endif + }; + + struct ThreadData { + constexpr ThreadData() : thread(), steal_partition(0), queue() {} + std::unique_ptr<Thread> thread; + std::atomic<unsigned> steal_partition; + Queue queue; }; Environment env_; - MaxSizeVector<Thread*> threads_; - MaxSizeVector<Queue*> queues_; - std::vector<EventCount::Waiter> waiters_; + const int num_threads_; + const bool allow_spinning_; + MaxSizeVector<ThreadData> thread_data_; + MaxSizeVector<MaxSizeVector<unsigned>> all_coprimes_; + MaxSizeVector<EventCount::Waiter> waiters_; std::atomic<unsigned> blocked_; std::atomic<bool> spinning_; std::atomic<bool> done_; + std::atomic<bool> cancelled_; EventCount ec_; +#ifndef EIGEN_THREAD_LOCAL + std::unique_ptr<Barrier> init_barrier_; + std::mutex per_thread_map_mutex_; // Protects per_thread_map_. + std::unordered_map<uint64_t, std::unique_ptr<PerThread>> per_thread_map_; +#endif // Main worker thread loop. - void WorkerLoop(unsigned index) { + void WorkerLoop(int thread_id) { +#ifndef EIGEN_THREAD_LOCAL + std::unique_ptr<PerThread> new_pt(new PerThread()); + per_thread_map_mutex_.lock(); + eigen_plain_assert(per_thread_map_.emplace(GlobalThreadIdHash(), std::move(new_pt)).second); + per_thread_map_mutex_.unlock(); + init_barrier_->Notify(); + init_barrier_->Wait(); +#endif PerThread* pt = GetPerThread(); pt->pool = this; - pt->index = index; - Queue* q = queues_[index]; - EventCount::Waiter* waiter = &waiters_[index]; - std::vector<Task> stolen; - for (;;) { - Task t; - if (!stolen.empty()) { - t = std::move(stolen.back()); - stolen.pop_back(); + pt->rand = GlobalThreadIdHash(); + pt->thread_id = thread_id; + Queue& q = thread_data_[thread_id].queue; + EventCount::Waiter* waiter = &waiters_[thread_id]; + // TODO(dvyukov,rmlarsen): The time spent in NonEmptyQueueIndex() is + // proportional to num_threads_ and we assume that new work is scheduled at + // a constant rate, so we set spin_count to 5000 / num_threads_. The + // constant was picked based on a fair dice roll, tune it. + const int spin_count = + allow_spinning_ && num_threads_ > 0 ? 5000 / num_threads_ : 0; + if (num_threads_ == 1) { + // For num_threads_ == 1 there is no point in going through the expensive + // steal loop. Moreover, since NonEmptyQueueIndex() calls PopBack() on the + // victim queues it might reverse the order in which ops are executed + // compared to the order in which they are scheduled, which tends to be + // counter-productive for the types of I/O workloads the single thread + // pools tend to be used for. + while (!cancelled_) { + Task t = q.PopFront(); + for (int i = 0; i < spin_count && !t.f; i++) { + if (!cancelled_.load(std::memory_order_relaxed)) { + t = q.PopFront(); + } + } + if (!t.f) { + if (!WaitForWork(waiter, &t)) { + return; + } + } + if (t.f) { + env_.ExecuteTask(t); + } } - if (!t.f) t = q->PopFront(); - if (!t.f) { - if (Steal(&stolen)) { - t = std::move(stolen.back()); - stolen.pop_back(); - while (stolen.size()) { - Task t1 = q->PushFront(std::move(stolen.back())); - stolen.pop_back(); - if (t1.f) { - // There is not much we can do in this case. Just execute the - // remaining directly. - stolen.push_back(std::move(t1)); - break; + } else { + while (!cancelled_) { + Task t = q.PopFront(); + if (!t.f) { + t = LocalSteal(); + if (!t.f) { + t = GlobalSteal(); + if (!t.f) { + // Leave one thread spinning. This reduces latency. + if (allow_spinning_ && !spinning_ && !spinning_.exchange(true)) { + for (int i = 0; i < spin_count && !t.f; i++) { + if (!cancelled_.load(std::memory_order_relaxed)) { + t = GlobalSteal(); + } else { + return; + } + } + spinning_ = false; + } + if (!t.f) { + if (!WaitForWork(waiter, &t)) { + return; + } + } } } } - } - if (t.f) { - env_.ExecuteTask(t); - continue; - } - // Leave one thread spinning. This reduces latency. - if (!spinning_ && !spinning_.exchange(true)) { - bool nowork = true; - for (int i = 0; i < 1000; i++) { - if (!OutOfWork()) { - nowork = false; - break; - } + if (t.f) { + env_.ExecuteTask(t); } - spinning_ = false; - if (!nowork) continue; } - if (!WaitForWork(waiter)) return; } } - // Steal tries to steal work from other worker threads in best-effort manner. - bool Steal(std::vector<Task>* stolen) { - if (queues_.size() == 1) return false; + // Steal tries to steal work from other worker threads in the range [start, + // limit) in best-effort manner. + Task Steal(unsigned start, unsigned limit) { PerThread* pt = GetPerThread(); - unsigned lastq = pt->index; - for (unsigned i = queues_.size(); i > 0; i--) { - unsigned victim = Rand(&pt->rand) % queues_.size(); - if (victim == lastq && queues_.size() > 2) { - i++; - continue; + const size_t size = limit - start; + unsigned r = Rand(&pt->rand); + unsigned victim = r % size; + unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()]; + + for (unsigned i = 0; i < size; i++) { + eigen_plain_assert(start + victim < limit); + Task t = thread_data_[start + victim].queue.PopBack(); + if (t.f) { + return t; } - // Steal half of elements from a victim queue. - // It is typical to steal just one element, but that assumes that work is - // recursively subdivided in halves so that the stolen element is exactly - // half of work. If work elements are equally-sized, then is makes sense - // to steal half of elements at once and then work locally for a while. - if (queues_[victim]->PopBackHalf(stolen)) return true; - lastq = victim; + victim += inc; + if (victim >= size) { + victim -= size; + } } - // Just to make sure that we did not miss anything. - for (unsigned i = queues_.size(); i > 0; i--) - if (queues_[i - 1]->PopBackHalf(stolen)) return true; - return false; + return Task(); } - // WaitForWork blocks until new work is available, or if it is time to exit. - bool WaitForWork(EventCount::Waiter* waiter) { - // We already did best-effort emptiness check in Steal, so prepare blocking. + // Steals work within threads belonging to the partition. + Task LocalSteal() { + PerThread* pt = GetPerThread(); + unsigned partition = GetStealPartition(pt->thread_id); + unsigned start, limit; + DecodePartition(partition, &start, &limit); + AssertBounds(start, limit); + + return Steal(start, limit); + } + + // Steals work from any other thread in the pool. + Task GlobalSteal() { + return Steal(0, num_threads_); + } + + + // WaitForWork blocks until new work is available (returns true), or if it is + // time to exit (returns false). Can optionally return a task to execute in t + // (in such case t.f != nullptr on return). + bool WaitForWork(EventCount::Waiter* waiter, Task* t) { + eigen_plain_assert(!t->f); + // We already did best-effort emptiness check in Steal, so prepare for + // blocking. ec_.Prewait(waiter); - // Now do reliable emptiness check. - if (!OutOfWork()) { + // Now do a reliable emptiness check. + int victim = NonEmptyQueueIndex(); + if (victim != -1) { ec_.CancelWait(waiter); - return true; + if (cancelled_) { + return false; + } else { + *t = thread_data_[victim].queue.PopBack(); + return true; + } } // Number of blocked threads is used as termination condition. // If we are shutting down and all worker threads blocked without work, // that's we are done. blocked_++; - if (done_ && blocked_ == threads_.size()) { + // TODO is blocked_ required to be unsigned? + if (done_ && blocked_ == static_cast<unsigned>(num_threads_)) { ec_.CancelWait(waiter); // Almost done, but need to re-check queues. // Consider that all queues are empty and all worker threads are preempted // right after incrementing blocked_ above. Now a free-standing thread // submits work and calls destructor (which sets done_). If we don't // re-check queues, we will exit leaving the work unexecuted. - if (!OutOfWork()) { + if (NonEmptyQueueIndex() != -1) { // Note: we must not pop from queues before we decrement blocked_, // otherwise the following scenario is possible. Consider that instead // of checking for emptiness we popped the only element from queues. @@ -205,27 +417,58 @@ return true; } - bool OutOfWork() { - for (unsigned i = 0; i < queues_.size(); i++) - if (!queues_[i]->Empty()) return false; - return true; + int NonEmptyQueueIndex() { + PerThread* pt = GetPerThread(); + // We intentionally design NonEmptyQueueIndex to steal work from + // anywhere in the queue so threads don't block in WaitForWork() forever + // when all threads in their partition go to sleep. Steal is still local. + const size_t size = thread_data_.size(); + unsigned r = Rand(&pt->rand); + unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()]; + unsigned victim = r % size; + for (unsigned i = 0; i < size; i++) { + if (!thread_data_[victim].queue.Empty()) { + return victim; + } + victim += inc; + if (victim >= size) { + victim -= size; + } + } + return -1; } - PerThread* GetPerThread() { + static EIGEN_STRONG_INLINE uint64_t GlobalThreadIdHash() { + return std::hash<std::thread::id>()(std::this_thread::get_id()); + } + + EIGEN_STRONG_INLINE PerThread* GetPerThread() { +#ifndef EIGEN_THREAD_LOCAL + static PerThread dummy; + auto it = per_thread_map_.find(GlobalThreadIdHash()); + if (it == per_thread_map_.end()) { + return &dummy; + } else { + return it->second.get(); + } +#else EIGEN_THREAD_LOCAL PerThread per_thread_; PerThread* pt = &per_thread_; - if (pt->inited) return pt; - pt->inited = true; - pt->rand = std::hash<std::thread::id>()(std::this_thread::get_id()); return pt; +#endif } - static unsigned Rand(unsigned* state) { - return *state = *state * 1103515245 + 12345; + static EIGEN_STRONG_INLINE unsigned Rand(uint64_t* state) { + uint64_t current = *state; + // Update the internal state + *state = current * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL; + // Generate the random output (using the PCG-XSH-RS scheme) + return static_cast<unsigned>((current ^ (current >> 22)) >> + (22 + (current >> 61))); } }; -typedef NonBlockingThreadPoolTempl<StlThreadEnvironment> NonBlockingThreadPool; +typedef ThreadPoolTempl<StlThreadEnvironment> ThreadPool; } // namespace Eigen
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/RunQueue.h b/unsupported/Eigen/CXX11/src/ThreadPool/RunQueue.h index 0544a6e..ecdc35f 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/RunQueue.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/RunQueue.h
@@ -10,7 +10,6 @@ #ifndef EIGEN_CXX11_THREADPOOL_RUNQUEUE_H_ #define EIGEN_CXX11_THREADPOOL_RUNQUEUE_H_ - namespace Eigen { // RunQueue is a fixed-size, partially non-blocking deque or Work items. @@ -38,16 +37,16 @@ template <typename Work, unsigned kSize> class RunQueue { public: - RunQueue() : front_(), back_() { + RunQueue() : front_(0), back_(0) { // require power-of-two for fast masking - eigen_assert((kSize & (kSize - 1)) == 0); - eigen_assert(kSize > 2); // why would you do this? - eigen_assert(kSize <= (64 << 10)); // leave enough space for counter + eigen_plain_assert((kSize & (kSize - 1)) == 0); + eigen_plain_assert(kSize > 2); // why would you do this? + eigen_plain_assert(kSize <= (64 << 10)); // leave enough space for counter for (unsigned i = 0; i < kSize; i++) array_[i].state.store(kEmpty, std::memory_order_relaxed); } - ~RunQueue() { eigen_assert(Size() == 0); } + ~RunQueue() { eigen_plain_assert(Size() == 0); } // PushFront inserts w at the beginning of the queue. // If queue is full returns w, otherwise returns default-constructed Work. @@ -100,7 +99,7 @@ // PopBack removes and returns the last elements in the queue. // Can fail spuriously. Work PopBack() { - if (Empty()) return 0; + if (Empty()) return Work(); std::unique_lock<std::mutex> lock(mutex_, std::try_to_lock); if (!lock) return Work(); unsigned back = back_.load(std::memory_order_relaxed); @@ -131,15 +130,14 @@ Elem* e = &array_[mid & kMask]; uint8_t s = e->state.load(std::memory_order_relaxed); if (n == 0) { - if (s != kReady || - !e->state.compare_exchange_strong(s, kBusy, - std::memory_order_acquire)) + if (s != kReady || !e->state.compare_exchange_strong( + s, kBusy, std::memory_order_acquire)) continue; start = mid; } else { // Note: no need to store temporal kBusy, we exclusively own these // elements. - eigen_assert(s == kReady); + eigen_plain_assert(s == kReady); } result->push_back(std::move(e->w)); e->state.store(kEmpty, std::memory_order_release); @@ -152,30 +150,18 @@ // Size returns current queue size. // Can be called by any thread at any time. - unsigned Size() const { - // Emptiness plays critical role in thread pool blocking. So we go to great - // effort to not produce false positives (claim non-empty queue as empty). - for (;;) { - // Capture a consistent snapshot of front/tail. - unsigned front = front_.load(std::memory_order_acquire); - unsigned back = back_.load(std::memory_order_acquire); - unsigned front1 = front_.load(std::memory_order_relaxed); - if (front != front1) continue; - int size = (front & kMask2) - (back & kMask2); - // Fix overflow. - if (size < 0) size += 2 * kSize; - // Order of modification in push/pop is crafted to make the queue look - // larger than it is during concurrent modifications. E.g. pop can - // decrement size before the corresponding push has incremented it. - // So the computed size can be up to kSize + 1, fix it. - if (size > static_cast<int>(kSize)) size = kSize; - return size; - } - } + unsigned Size() const { return SizeOrNotEmpty<true>(); } // Empty tests whether container is empty. // Can be called by any thread at any time. - bool Empty() const { return Size() == 0; } + bool Empty() const { return SizeOrNotEmpty<false>() == 0; } + + // Delete all the elements from the queue. + void Flush() { + while (!Empty()) { + PopFront(); + } + } private: static const unsigned kMask = kSize - 1; @@ -191,7 +177,7 @@ }; std::mutex mutex_; // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of - // front/back, repsectively. The remaining bits contain modification counters + // front/back, respectively. The remaining bits contain modification counters // that are incremented on Push operations. This allows us to (1) distinguish // between empty and full conditions (if we would use log(kSize) bits for // position, these conditions would be indistinguishable); (2) obtain @@ -201,6 +187,47 @@ std::atomic<unsigned> back_; Elem array_[kSize]; + // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false, + // only whether the size is 0 is guaranteed to be correct. + // Can be called by any thread at any time. + template<bool NeedSizeEstimate> + unsigned SizeOrNotEmpty() const { + // Emptiness plays critical role in thread pool blocking. So we go to great + // effort to not produce false positives (claim non-empty queue as empty). + unsigned front = front_.load(std::memory_order_acquire); + for (;;) { + // Capture a consistent snapshot of front/tail. + unsigned back = back_.load(std::memory_order_acquire); + unsigned front1 = front_.load(std::memory_order_relaxed); + if (front != front1) { + front = front1; + std::atomic_thread_fence(std::memory_order_acquire); + continue; + } + if (NeedSizeEstimate) { + return CalculateSize(front, back); + } else { + // This value will be 0 if the queue is empty, and undefined otherwise. + unsigned maybe_zero = ((front ^ back) & kMask2); + eigen_assert(maybe_zero == 0 ? CalculateSize(front, back) == 0 : true); + return maybe_zero; + } + } + } + + EIGEN_ALWAYS_INLINE + unsigned CalculateSize(unsigned front, unsigned back) const { + int size = (front & kMask2) - (back & kMask2); + // Fix overflow. + if (size < 0) size += 2 * kSize; + // Order of modification in push/pop is crafted to make the queue look + // larger than it is during concurrent modifications. E.g. push can + // increment size before the corresponding pop has decremented it. + // So the computed size can be up to kSize + 1, fix it. + if (size > static_cast<int>(kSize)) size = kSize; + return static_cast<unsigned>(size); + } + RunQueue(const RunQueue&) = delete; void operator=(const RunQueue&) = delete; };
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/SimpleThreadPool.h b/unsupported/Eigen/CXX11/src/ThreadPool/SimpleThreadPool.h deleted file mode 100644 index 17fd165..0000000 --- a/unsupported/Eigen/CXX11/src/ThreadPool/SimpleThreadPool.h +++ /dev/null
@@ -1,127 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_CXX11_THREADPOOL_SIMPLE_THREAD_POOL_H -#define EIGEN_CXX11_THREADPOOL_SIMPLE_THREAD_POOL_H - -namespace Eigen { - -// The implementation of the ThreadPool type ensures that the Schedule method -// runs the functions it is provided in FIFO order when the scheduling is done -// by a single thread. -// Environment provides a way to create threads and also allows to intercept -// task submission and execution. -template <typename Environment> -class SimpleThreadPoolTempl : public ThreadPoolInterface { - public: - // Construct a pool that contains "num_threads" threads. - explicit SimpleThreadPoolTempl(int num_threads, Environment env = Environment()) - : env_(env), threads_(num_threads), waiters_(num_threads) { - for (int i = 0; i < num_threads; i++) { - threads_.push_back(env.CreateThread([this]() { WorkerLoop(); })); - } - } - - // Wait until all scheduled work has finished and then destroy the - // set of threads. - ~SimpleThreadPoolTempl() { - { - // Wait for all work to get done. - std::unique_lock<std::mutex> l(mu_); - while (!pending_.empty()) { - empty_.wait(l); - } - exiting_ = true; - - // Wakeup all waiters. - for (auto w : waiters_) { - w->ready = true; - w->task.f = nullptr; - w->cv.notify_one(); - } - } - - // Wait for threads to finish. - for (auto t : threads_) { - delete t; - } - } - - // Schedule fn() for execution in the pool of threads. The functions are - // executed in the order in which they are scheduled. - void Schedule(std::function<void()> fn) { - Task t = env_.CreateTask(std::move(fn)); - std::unique_lock<std::mutex> l(mu_); - if (waiters_.empty()) { - pending_.push_back(std::move(t)); - } else { - Waiter* w = waiters_.back(); - waiters_.pop_back(); - w->ready = true; - w->task = std::move(t); - w->cv.notify_one(); - } - } - - protected: - void WorkerLoop() { - std::unique_lock<std::mutex> l(mu_); - Waiter w; - Task t; - while (!exiting_) { - if (pending_.empty()) { - // Wait for work to be assigned to me - w.ready = false; - waiters_.push_back(&w); - while (!w.ready) { - w.cv.wait(l); - } - t = w.task; - w.task.f = nullptr; - } else { - // Pick up pending work - t = std::move(pending_.front()); - pending_.pop_front(); - if (pending_.empty()) { - empty_.notify_all(); - } - } - if (t.f) { - mu_.unlock(); - env_.ExecuteTask(t); - t.f = nullptr; - mu_.lock(); - } - } - } - - private: - typedef typename Environment::Task Task; - typedef typename Environment::EnvThread Thread; - - struct Waiter { - std::condition_variable cv; - Task task; - bool ready; - }; - - Environment env_; - std::mutex mu_; - MaxSizeVector<Thread*> threads_; // All threads - MaxSizeVector<Waiter*> waiters_; // Stack of waiting threads. - std::deque<Task> pending_; // Queue of pending work - std::condition_variable empty_; // Signaled on pending_.empty() - bool exiting_ = false; -}; - -typedef SimpleThreadPoolTempl<StlThreadEnvironment> SimpleThreadPool; - -} // namespace Eigen - -#endif // EIGEN_CXX11_THREADPOOL_SIMPLE_THREAD_POOL_H
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadCancel.h b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadCancel.h new file mode 100644 index 0000000..a05685f --- /dev/null +++ b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadCancel.h
@@ -0,0 +1,23 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H +#define EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H + +// Try to come up with a portable way to cancel a thread +#if EIGEN_OS_GNULINUX + #define EIGEN_THREAD_CANCEL(t) \ + pthread_cancel(t.native_handle()); + #define EIGEN_SUPPORTS_THREAD_CANCELLATION 1 +#else +#define EIGEN_THREAD_CANCEL(t) +#endif + + +#endif // EIGEN_CXX11_THREADPOOL_THREAD_CANCEL_H
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadEnvironment.h b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadEnvironment.h index d2204ad..d94a064 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadEnvironment.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadEnvironment.h
@@ -21,14 +21,16 @@ // destructor must join the thread. class EnvThread { public: - EnvThread(std::function<void()> f) : thr_(f) {} + EnvThread(std::function<void()> f) : thr_(std::move(f)) {} ~EnvThread() { thr_.join(); } + // This function is called when the threadpool is cancelled. + void OnCancel() { } private: std::thread thr_; }; - EnvThread* CreateThread(std::function<void()> f) { return new EnvThread(f); } + EnvThread* CreateThread(std::function<void()> f) { return new EnvThread(std::move(f)); } Task CreateTask(std::function<void()> f) { return Task{std::move(f)}; } void ExecuteTask(const Task& t) { t.f(); } };
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadLocal.h b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadLocal.h index cfa2217..7229839 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadLocal.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadLocal.h
@@ -10,13 +10,46 @@ #ifndef EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H #define EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H -// Try to come up with a portable implementation of thread local variables -#if EIGEN_COMP_GNUC && EIGEN_GNUC_AT_MOST(4, 7) -#define EIGEN_THREAD_LOCAL static __thread -#elif EIGEN_COMP_CLANG -#define EIGEN_THREAD_LOCAL static __thread -#else +#if EIGEN_MAX_CPP_VER >= 11 && \ + ((EIGEN_COMP_GNUC && EIGEN_GNUC_AT_LEAST(4, 8)) || \ + __has_feature(cxx_thread_local) || \ + (EIGEN_COMP_MSVC >= 1900) ) #define EIGEN_THREAD_LOCAL static thread_local #endif +// Disable TLS for Apple and Android builds with older toolchains. +#if defined(__APPLE__) +// Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, +// __IPHONE_8_0. +#include <Availability.h> +#include <TargetConditionals.h> +#endif +// Checks whether C++11's `thread_local` storage duration specifier is +// supported. +#if defined(__apple_build_version__) && \ + ((__apple_build_version__ < 8000042) || \ + (TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0)) +// Notes: Xcode's clang did not support `thread_local` until version +// 8, and even then not for all iOS < 9.0. +#undef EIGEN_THREAD_LOCAL + +#elif defined(__ANDROID__) && EIGEN_COMP_CLANG +// There are platforms for which TLS should not be used even though the compiler +// makes it seem like it's supported (Android NDK < r12b for example). +// This is primarily because of linker problems and toolchain misconfiguration: +// TLS isn't supported until NDK r12b per +// https://developer.android.com/ndk/downloads/revision_history.html +// Since NDK r16, `__NDK_MAJOR__` and `__NDK_MINOR__` are defined in +// <android/ndk-version.h>. For NDK < r16, users should define these macros, +// e.g. `-D__NDK_MAJOR__=11 -D__NKD_MINOR__=0` for NDK r11. +#if __has_include(<android/ndk-version.h>) +#include <android/ndk-version.h> +#endif // __has_include(<android/ndk-version.h>) +#if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ + defined(__NDK_MINOR__) && \ + ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) +#undef EIGEN_THREAD_LOCAL +#endif +#endif // defined(__ANDROID__) && defined(__clang__) + #endif // EIGEN_CXX11_THREADPOOL_THREAD_LOCAL_H
diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadPoolInterface.h b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadPoolInterface.h index 38b40ac..25030dc 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/ThreadPoolInterface.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/ThreadPoolInterface.h
@@ -16,8 +16,30 @@ // custom thread pools underneath. class ThreadPoolInterface { public: + // Submits a closure to be run by a thread in the pool. virtual void Schedule(std::function<void()> fn) = 0; + // Submits a closure to be run by threads in the range [start, end) in the + // pool. + virtual void ScheduleWithHint(std::function<void()> fn, int /*start*/, + int /*end*/) { + // Just defer to Schedule in case sub-classes aren't interested in + // overriding this functionality. + Schedule(fn); + } + + // If implemented, stop processing the closures that have been enqueued. + // Currently running closures may still be processed. + // If not implemented, does nothing. + virtual void Cancel() {} + + // Returns the number of threads in the pool. + virtual int NumThreads() const = 0; + + // Returns a logical thread index between 0 and NumThreads() - 1 if called + // from one of the threads in the pool. Returns -1 otherwise. + virtual int CurrentThreadId() const = 0; + virtual ~ThreadPoolInterface() {} };
diff --git a/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h b/unsupported/Eigen/CXX11/src/util/CXX11Meta.h similarity index 77% rename from unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h rename to unsupported/Eigen/CXX11/src/util/CXX11Meta.h index c582e21..6c95d0a 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h +++ b/unsupported/Eigen/CXX11/src/util/CXX11Meta.h
@@ -10,12 +10,22 @@ #ifndef EIGEN_CXX11META_H #define EIGEN_CXX11META_H +#include <vector> +#include "EmulateArray.h" + +// Emulate the cxx11 functionality that we need if the compiler doesn't support it. +// Visual studio 2015 doesn't advertise itself as cxx11 compliant, although it +// supports enough of the standard for our needs +#if __cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900 + +#include "CXX11Workarounds.h" + namespace Eigen { namespace internal { /** \internal - * \file CXX11/Core/util/CXX11Meta.h + * \file CXX11/util/CXX11Meta.h * This file contains generic metaprogramming classes which are not specifically related to Eigen. * This file expands upon Core/util/Meta.h and adds support for C++11 specific features. */ @@ -30,7 +40,7 @@ struct numeric_list { constexpr static std::size_t count = sizeof...(nn); }; template<typename T, T n, T... nn> -struct numeric_list<T, n, nn...> { constexpr static std::size_t count = sizeof...(nn) + 1; constexpr static T first_value = n; }; +struct numeric_list<T, n, nn...> { static const std::size_t count = sizeof...(nn) + 1; const static T first_value = n; }; /* numeric list constructors * @@ -94,9 +104,9 @@ template<int n> struct h_skip { template<typename T, T... ii> - constexpr static inline typename h_skip_helper_numeric<T, n, ii...>::type helper(numeric_list<T, ii...>) { return typename h_skip_helper_numeric<T, n, ii...>::type(); } + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_numeric<T, n, ii...>::type helper(numeric_list<T, ii...>) { return typename h_skip_helper_numeric<T, n, ii...>::type(); } template<typename... tt> - constexpr static inline typename h_skip_helper_type<n, tt...>::type helper(type_list<tt...>) { return typename h_skip_helper_type<n, tt...>::type(); } + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_type<n, tt...>::type helper(type_list<tt...>) { return typename h_skip_helper_type<n, tt...>::type(); } }; template<int n, typename a> struct skip { typedef decltype(h_skip<n>::helper(a())) type; }; @@ -113,6 +123,10 @@ template<typename T, int n, T a, T... as> struct get<n, numeric_list<T, a, as...>> : get<n-1, numeric_list<T, as...>> {}; template<typename T, T a, T... as> struct get<0, numeric_list<T, a, as...>> { constexpr static T value = a; }; +template<std::size_t n, typename T, T a, T... as> constexpr T array_get(const numeric_list<T, a, as...>&) { + return get<(int)n, numeric_list<T, a, as...>>::value; +} + /* always get type, regardless of dummy; good for parameter pack expansion */ template<typename T, T dummy, typename t> struct id_numeric { typedef t type; }; @@ -254,7 +268,7 @@ typename Reducer > struct reduce<Reducer> { - constexpr static inline int run() { return Reducer::Identity; } + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE int run() { return Reducer::Identity; } }; template< @@ -262,7 +276,7 @@ typename A > struct reduce<Reducer, A> { - constexpr static inline A run(A a) { return a; } + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE A run(A a) { return a; } }; template< @@ -271,7 +285,7 @@ typename... Ts > struct reduce<Reducer, A, Ts...> { - constexpr static inline auto run(A a, Ts... ts) -> decltype(Reducer::run(a, reduce<Reducer, Ts...>::run(ts...))) { + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, Ts... ts) -> decltype(Reducer::run(a, reduce<Reducer, Ts...>::run(ts...))) { return Reducer::run(a, reduce<Reducer, Ts...>::run(ts...)); } }; @@ -279,29 +293,29 @@ /* generic binary operations */ struct sum_op { - template<typename A, typename B> EIGEN_DEVICE_FUNC constexpr static inline auto run(A a, B b) -> decltype(a + b) { return a + b; } + template<typename A, typename B> EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a + b) { return a + b; } static constexpr int Identity = 0; }; struct product_op { - template<typename A, typename B> EIGEN_DEVICE_FUNC constexpr static inline auto run(A a, B b) -> decltype(a * b) { return a * b; } + template<typename A, typename B> EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a * b) { return a * b; } static constexpr int Identity = 1; }; -struct logical_and_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a && b) { return a && b; } }; -struct logical_or_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a || b) { return a || b; } }; +struct logical_and_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a && b) { return a && b; } }; +struct logical_or_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a || b) { return a || b; } }; -struct equal_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a == b) { return a == b; } }; -struct not_equal_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a != b) { return a != b; } }; -struct lesser_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a < b) { return a < b; } }; -struct lesser_equal_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a <= b) { return a <= b; } }; -struct greater_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a > b) { return a > b; } }; -struct greater_equal_op { template<typename A, typename B> constexpr static inline auto run(A a, B b) -> decltype(a >= b) { return a >= b; } }; +struct equal_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a == b) { return a == b; } }; +struct not_equal_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a != b) { return a != b; } }; +struct lesser_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a < b) { return a < b; } }; +struct lesser_equal_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a <= b) { return a <= b; } }; +struct greater_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a > b) { return a > b; } }; +struct greater_equal_op { template<typename A, typename B> constexpr static EIGEN_STRONG_INLINE auto run(A a, B b) -> decltype(a >= b) { return a >= b; } }; /* generic unary operations */ -struct not_op { template<typename A> constexpr static inline auto run(A a) -> decltype(!a) { return !a; } }; -struct negation_op { template<typename A> constexpr static inline auto run(A a) -> decltype(-a) { return -a; } }; -struct greater_equal_zero_op { template<typename A> constexpr static inline auto run(A a) -> decltype(a >= 0) { return a >= 0; } }; +struct not_op { template<typename A> constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(!a) { return !a; } }; +struct negation_op { template<typename A> constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(-a) { return -a; } }; +struct greater_equal_zero_op { template<typename A> constexpr static EIGEN_STRONG_INLINE auto run(A a) -> decltype(a >= 0) { return a >= 0; } }; /* reductions for lists */ @@ -310,13 +324,13 @@ // together in front... (13.0 doesn't work with array_prod/array_reduce/... anyway, but 13.1 // does... template<typename... Ts> -constexpr inline decltype(reduce<product_op, Ts...>::run((*((Ts*)0))...)) arg_prod(Ts... ts) +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE decltype(reduce<product_op, Ts...>::run((*((Ts*)0))...)) arg_prod(Ts... ts) { return reduce<product_op, Ts...>::run(ts...); } template<typename... Ts> -constexpr inline decltype(reduce<sum_op, Ts...>::run((*((Ts*)0))...)) arg_sum(Ts... ts) +constexpr EIGEN_STRONG_INLINE decltype(reduce<sum_op, Ts...>::run((*((Ts*)0))...)) arg_sum(Ts... ts) { return reduce<sum_op, Ts...>::run(ts...); } @@ -324,13 +338,13 @@ /* reverse arrays */ template<typename Array, int... n> -constexpr inline Array h_array_reverse(Array arr, numeric_list<int, n...>) +constexpr EIGEN_STRONG_INLINE Array h_array_reverse(Array arr, numeric_list<int, n...>) { return {{array_get<sizeof...(n) - n - 1>(arr)...}}; } template<typename T, std::size_t N> -constexpr inline array<T, N> array_reverse(array<T, N> arr) +constexpr EIGEN_STRONG_INLINE array<T, N> array_reverse(array<T, N> arr) { return h_array_reverse(arr, typename gen_numeric_list<int, N>::type()); } @@ -345,7 +359,7 @@ // an infinite loop) template<typename Reducer, typename T, std::size_t N, std::size_t n = N - 1> struct h_array_reduce { - EIGEN_DEVICE_FUNC constexpr static inline auto run(array<T, N> arr, T identity) -> decltype(Reducer::run(h_array_reduce<Reducer, T, N, n - 1>::run(arr, identity), array_get<n>(arr))) + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE auto run(array<T, N> arr, T identity) -> decltype(Reducer::run(h_array_reduce<Reducer, T, N, n - 1>::run(arr, identity), array_get<n>(arr))) { return Reducer::run(h_array_reduce<Reducer, T, N, n - 1>::run(arr, identity), array_get<n>(arr)); } @@ -354,7 +368,7 @@ template<typename Reducer, typename T, std::size_t N> struct h_array_reduce<Reducer, T, N, 0> { - EIGEN_DEVICE_FUNC constexpr static inline T run(const array<T, N>& arr, T) + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE T run(const array<T, N>& arr, T) { return array_get<0>(arr); } @@ -363,14 +377,14 @@ template<typename Reducer, typename T> struct h_array_reduce<Reducer, T, 0> { - EIGEN_DEVICE_FUNC constexpr static inline T run(const array<T, 0>&, T identity) + EIGEN_DEVICE_FUNC constexpr static EIGEN_STRONG_INLINE T run(const array<T, 0>&, T identity) { return identity; } }; template<typename Reducer, typename T, std::size_t N> -EIGEN_DEVICE_FUNC constexpr inline auto array_reduce(const array<T, N>& arr, T identity) -> decltype(h_array_reduce<Reducer, T, N>::run(arr, identity)) +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_reduce(const array<T, N>& arr, T identity) -> decltype(h_array_reduce<Reducer, T, N>::run(arr, identity)) { return h_array_reduce<Reducer, T, N>::run(arr, identity); } @@ -378,13 +392,13 @@ /* standard array reductions */ template<typename T, std::size_t N> -EIGEN_DEVICE_FUNC constexpr inline auto array_sum(const array<T, N>& arr) -> decltype(array_reduce<sum_op, T, N>(arr, static_cast<T>(0))) +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_sum(const array<T, N>& arr) -> decltype(array_reduce<sum_op, T, N>(arr, static_cast<T>(0))) { return array_reduce<sum_op, T, N>(arr, static_cast<T>(0)); } template<typename T, std::size_t N> -EIGEN_DEVICE_FUNC constexpr inline auto array_prod(const array<T, N>& arr) -> decltype(array_reduce<product_op, T, N>(arr, static_cast<T>(1))) +EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE auto array_prod(const array<T, N>& arr) -> decltype(array_reduce<product_op, T, N>(arr, static_cast<T>(1))) { return array_reduce<product_op, T, N>(arr, static_cast<T>(1)); } @@ -400,13 +414,13 @@ /* zip an array */ template<typename Op, typename A, typename B, std::size_t N, int... n> -constexpr inline array<decltype(Op::run(A(), B())),N> h_array_zip(array<A, N> a, array<B, N> b, numeric_list<int, n...>) +constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A(), B())),N> h_array_zip(array<A, N> a, array<B, N> b, numeric_list<int, n...>) { return array<decltype(Op::run(A(), B())),N>{{ Op::run(array_get<n>(a), array_get<n>(b))... }}; } template<typename Op, typename A, typename B, std::size_t N> -constexpr inline array<decltype(Op::run(A(), B())),N> array_zip(array<A, N> a, array<B, N> b) +constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A(), B())),N> array_zip(array<A, N> a, array<B, N> b) { return h_array_zip<Op>(a, b, typename gen_numeric_list<int, N>::type()); } @@ -414,13 +428,13 @@ /* zip an array and reduce the result */ template<typename Reducer, typename Op, typename A, typename B, std::size_t N, int... n> -constexpr inline auto h_array_zip_and_reduce(array<A, N> a, array<B, N> b, numeric_list<int, n...>) -> decltype(reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A(), B()))>::type...>::run(Op::run(array_get<n>(a), array_get<n>(b))...)) +constexpr EIGEN_STRONG_INLINE auto h_array_zip_and_reduce(array<A, N> a, array<B, N> b, numeric_list<int, n...>) -> decltype(reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A(), B()))>::type...>::run(Op::run(array_get<n>(a), array_get<n>(b))...)) { return reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A(), B()))>::type...>::run(Op::run(array_get<n>(a), array_get<n>(b))...); } template<typename Reducer, typename Op, typename A, typename B, std::size_t N> -constexpr inline auto array_zip_and_reduce(array<A, N> a, array<B, N> b) -> decltype(h_array_zip_and_reduce<Reducer, Op, A, B, N>(a, b, typename gen_numeric_list<int, N>::type())) +constexpr EIGEN_STRONG_INLINE auto array_zip_and_reduce(array<A, N> a, array<B, N> b) -> decltype(h_array_zip_and_reduce<Reducer, Op, A, B, N>(a, b, typename gen_numeric_list<int, N>::type())) { return h_array_zip_and_reduce<Reducer, Op, A, B, N>(a, b, typename gen_numeric_list<int, N>::type()); } @@ -428,13 +442,13 @@ /* apply stuff to an array */ template<typename Op, typename A, std::size_t N, int... n> -constexpr inline array<decltype(Op::run(A())),N> h_array_apply(array<A, N> a, numeric_list<int, n...>) +constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A())),N> h_array_apply(array<A, N> a, numeric_list<int, n...>) { return array<decltype(Op::run(A())),N>{{ Op::run(array_get<n>(a))... }}; } template<typename Op, typename A, std::size_t N> -constexpr inline array<decltype(Op::run(A())),N> array_apply(array<A, N> a) +constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A())),N> array_apply(array<A, N> a) { return h_array_apply<Op>(a, typename gen_numeric_list<int, N>::type()); } @@ -442,13 +456,13 @@ /* apply stuff to an array and reduce */ template<typename Reducer, typename Op, typename A, std::size_t N, int... n> -constexpr inline auto h_array_apply_and_reduce(array<A, N> arr, numeric_list<int, n...>) -> decltype(reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A()))>::type...>::run(Op::run(array_get<n>(arr))...)) +constexpr EIGEN_STRONG_INLINE auto h_array_apply_and_reduce(array<A, N> arr, numeric_list<int, n...>) -> decltype(reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A()))>::type...>::run(Op::run(array_get<n>(arr))...)) { return reduce<Reducer, typename id_numeric<int,n,decltype(Op::run(A()))>::type...>::run(Op::run(array_get<n>(arr))...); } template<typename Reducer, typename Op, typename A, std::size_t N> -constexpr inline auto array_apply_and_reduce(array<A, N> a) -> decltype(h_array_apply_and_reduce<Reducer, Op, A, N>(a, typename gen_numeric_list<int, N>::type())) +constexpr EIGEN_STRONG_INLINE auto array_apply_and_reduce(array<A, N> a) -> decltype(h_array_apply_and_reduce<Reducer, Op, A, N>(a, typename gen_numeric_list<int, N>::type())) { return h_array_apply_and_reduce<Reducer, Op, A, N>(a, typename gen_numeric_list<int, N>::type()); } @@ -462,7 +476,7 @@ struct h_repeat { template<typename t, int... ii> - constexpr static inline array<t, n> run(t v, numeric_list<int, ii...>) + constexpr static EIGEN_STRONG_INLINE array<t, n> run(t v, numeric_list<int, ii...>) { return {{ typename id_numeric<int, ii, t>::type(v)... }}; } @@ -523,4 +537,10 @@ } // end namespace Eigen +#else // Non C++11, fallback to emulation mode + +#include "EmulateCXX11Meta.h" + +#endif + #endif // EIGEN_CXX11META_H
diff --git a/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h b/unsupported/Eigen/CXX11/src/util/CXX11Workarounds.h similarity index 91% rename from unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h rename to unsupported/Eigen/CXX11/src/util/CXX11Workarounds.h index fe4d228..f1c0284 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h +++ b/unsupported/Eigen/CXX11/src/util/CXX11Workarounds.h
@@ -47,9 +47,9 @@ */ -template<std::size_t I, class T> constexpr inline T& array_get(std::vector<T>& a) { return a[I]; } -template<std::size_t I, class T> constexpr inline T&& array_get(std::vector<T>&& a) { return a[I]; } -template<std::size_t I, class T> constexpr inline T const& array_get(std::vector<T> const& a) { return a[I]; } +template<std::size_t I_, class T> constexpr inline T& array_get(std::vector<T>& a) { return a[I_]; } +template<std::size_t I_, class T> constexpr inline T&& array_get(std::vector<T>&& a) { return a[I_]; } +template<std::size_t I_, class T> constexpr inline T const& array_get(std::vector<T> const& a) { return a[I_]; } /* Suppose you have a template of the form * template<typename T> struct X;
diff --git a/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h b/unsupported/Eigen/CXX11/src/util/EmulateArray.h similarity index 79% rename from unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h rename to unsupported/Eigen/CXX11/src/util/EmulateArray.h index 579519b..834b20b 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h +++ b/unsupported/Eigen/CXX11/src/util/EmulateArray.h
@@ -15,15 +15,20 @@ // The array class is only available starting with cxx11. Emulate our own here // if needed. Beware, msvc still doesn't advertise itself as a c++11 compiler! // Moreover, CUDA doesn't support the STL containers, so we use our own instead. -#if (__cplusplus <= 199711L && EIGEN_COMP_MSVC < 1900) || defined(__CUDACC__) || defined(EIGEN_AVOID_STL_ARRAY) +#if (__cplusplus <= 199711L && EIGEN_COMP_MSVC < 1900) || defined(EIGEN_GPUCC) || defined(EIGEN_AVOID_STL_ARRAY) namespace Eigen { template <typename T, size_t n> class array { public: EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE T& operator[] (size_t index) { return values[index]; } + EIGEN_STRONG_INLINE T& operator[] (size_t index) { eigen_internal_assert(index < size()); return values[index]; } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const T& operator[] (size_t index) const { return values[index]; } + EIGEN_STRONG_INLINE const T& operator[] (size_t index) const { eigen_internal_assert(index < size()); return values[index]; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& at(size_t index) { eigen_assert(index < size()); return values[index]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& at(size_t index) const { eigen_assert(index < size()); return values[index]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& front() { return values[0]; } @@ -117,7 +122,7 @@ values[7] = v8; } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(std::initializer_list<T> l) { eigen_assert(l.size() == n); @@ -167,8 +172,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array() : dummy() { } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES EIGEN_DEVICE_FUNC array(std::initializer_list<T> l) : dummy() { + EIGEN_UNUSED_VARIABLE(l); eigen_assert(l.size() == 0); } #endif @@ -191,30 +197,26 @@ namespace internal { -template<std::size_t I, class T, std::size_t N> +template<std::size_t I_, class T, std::size_t N> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& array_get(array<T,N>& a) { - return a[I]; + return a[I_]; } -template<std::size_t I, class T, std::size_t N> +template<std::size_t I_, class T, std::size_t N> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& array_get(const array<T,N>& a) { - return a[I]; + return a[I_]; } -template <typename T> struct array_size; template<class T, std::size_t N> struct array_size<array<T,N> > { - static const size_t value = N; + enum { value = N }; }; -template <typename T> struct array_size; template<class T, std::size_t N> struct array_size<array<T,N>& > { - static const size_t value = N; + enum { value = N }; }; -template <typename T> struct array_size; template<class T, std::size_t N> struct array_size<const array<T,N> > { - static const size_t value = N; + enum { value = N }; }; -template <typename T> struct array_size; template<class T, std::size_t N> struct array_size<const array<T,N>& > { - static const size_t value = N; + enum { value = N }; }; } // end namespace internal @@ -222,7 +224,7 @@ #else -// The compiler supports c++11, and we're not targetting cuda: use std::array as Eigen array +// The compiler supports c++11, and we're not targeting cuda: use std::array as Eigen::array #include <array> namespace Eigen { @@ -238,34 +240,22 @@ * this may not be constexpr */ #if defined(__GLIBCXX__) && __GLIBCXX__ < 20120322 -#define STD_GET_ARR_HACK a._M_instance[I] +#define STD_GET_ARR_HACK a._M_instance[I_] #elif defined(_LIBCPP_VERSION) -#define STD_GET_ARR_HACK a.__elems_[I] +#define STD_GET_ARR_HACK a.__elems_[I_] #else -#define STD_GET_ARR_HACK std::template get<I, T, N>(a) +#define STD_GET_ARR_HACK std::template get<I_, T, N>(a) #endif -template<std::size_t I, class T, std::size_t N> constexpr inline T& array_get(std::array<T,N>& a) { return (T&) STD_GET_ARR_HACK; } -template<std::size_t I, class T, std::size_t N> constexpr inline T&& array_get(std::array<T,N>&& a) { return (T&&) STD_GET_ARR_HACK; } -template<std::size_t I, class T, std::size_t N> constexpr inline T const& array_get(std::array<T,N> const& a) { return (T const&) STD_GET_ARR_HACK; } +template<std::size_t I_, class T, std::size_t N> constexpr inline T& array_get(std::array<T,N>& a) { return (T&) STD_GET_ARR_HACK; } +template<std::size_t I_, class T, std::size_t N> constexpr inline T&& array_get(std::array<T,N>&& a) { return (T&&) STD_GET_ARR_HACK; } +template<std::size_t I_, class T, std::size_t N> constexpr inline T const& array_get(std::array<T,N> const& a) { return (T const&) STD_GET_ARR_HACK; } #undef STD_GET_ARR_HACK -template <typename T> struct array_size; -template<class T, std::size_t N> struct array_size<const std::array<T,N> > { - static const size_t value = N; -}; -template <typename T> struct array_size; -template<class T, std::size_t N> struct array_size<std::array<T,N> > { - static const size_t value = N; -}; } // end namespace internal } // end namespace Eigen #endif - - - - #endif // EIGEN_EMULATE_ARRAY_H
diff --git a/unsupported/Eigen/CXX11/src/Core/util/EmulateCXX11Meta.h b/unsupported/Eigen/CXX11/src/util/EmulateCXX11Meta.h similarity index 96% rename from unsupported/Eigen/CXX11/src/Core/util/EmulateCXX11Meta.h rename to unsupported/Eigen/CXX11/src/util/EmulateCXX11Meta.h index d685d4f..d02d86f 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/EmulateCXX11Meta.h +++ b/unsupported/Eigen/CXX11/src/util/EmulateCXX11Meta.h
@@ -17,7 +17,7 @@ namespace internal { /** \internal - * \file CXX11/Core/util/EmulateCXX11Meta.h + * \file CXX11/util/EmulateCXX11Meta.h * This file emulates a subset of the functionality provided by CXXMeta.h for * compilers that don't yet support cxx11 such as nvcc. */ @@ -166,13 +166,13 @@ return array; } -template<std::size_t I, class Head, class Tail> +template<std::size_t I_, class Head, class Tail> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Head::type array_get(type_list<Head, Tail>&) { - return get<I, type_list<Head, Tail> >::value; + return get<I_, type_list<Head, Tail> >::value; } -template<std::size_t I, class Head, class Tail> +template<std::size_t I_, class Head, class Tail> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Head::type array_get(const type_list<Head, Tail>&) { - return get<I, type_list<Head, Tail> >::value; + return get<I_, type_list<Head, Tail> >::value; } template <class NList> @@ -188,7 +188,7 @@ } template<typename t> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE t array_prod(const array<t, 0>& /*a*/) { - return 0; + return 1; } template<typename t> @@ -200,13 +200,13 @@ } -template<std::size_t I, class T> +template<std::size_t I_, class T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& array_get(std::vector<T>& a) { - return a[I]; + return a[I_]; } -template<std::size_t I, class T> +template<std::size_t I_, class T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& array_get(const std::vector<T>& a) { - return a[I]; + return a[I_]; } struct sum_op {
diff --git a/unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h b/unsupported/Eigen/CXX11/src/util/MaxSizeVector.h similarity index 70% rename from unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h rename to unsupported/Eigen/CXX11/src/util/MaxSizeVector.h index 551124b..277ab14 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h +++ b/unsupported/Eigen/CXX11/src/util/MaxSizeVector.h
@@ -29,39 +29,70 @@ */ template <typename T> class MaxSizeVector { + static const size_t alignment = EIGEN_PLAIN_ENUM_MAX(EIGEN_ALIGNOF(T), sizeof(void*)); public: // Construct a new MaxSizeVector, reserve n elements. EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit MaxSizeVector(size_t n) : reserve_(n), size_(0), - data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) { - for (size_t i = 0; i < n; ++i) { new (&data_[i]) T; } + data_(static_cast<T*>(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) { } // Construct a new MaxSizeVector, reserve and resize to n. // Copy the init value to all elements. EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - explicit MaxSizeVector(size_t n, const T& init) + MaxSizeVector(size_t n, const T& init) : reserve_(n), size_(n), - data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) { - for (size_t i = 0; i < n; ++i) { new (&data_[i]) T(init); } + data_(static_cast<T*>(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) { + size_t i = 0; + EIGEN_TRY + { + for(; i < size_; ++i) { new (&data_[i]) T(init); } + } + EIGEN_CATCH(...) + { + // Construction failed, destruct in reverse order: + for(; (i+1) > 0; --i) { data_[i-1].~T(); } + internal::handmade_aligned_free(data_); + EIGEN_THROW; + } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~MaxSizeVector() { - for (size_t i = 0; i < size_; ++i) { - data_[i].~T(); + for (size_t i = size_; i > 0; --i) { + data_[i-1].~T(); } - internal::aligned_free(data_); + internal::handmade_aligned_free(data_); + } + + void resize(size_t n) { + eigen_assert(n <= reserve_); + for (; size_ < n; ++size_) { + new (&data_[size_]) T; + } + for (; size_ > n; --size_) { + data_[size_-1].~T(); + } + eigen_assert(size_ == n); } // Append new elements (up to reserved size). EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void push_back(const T& t) { eigen_assert(size_ < reserve_); - data_[size_++] = t; + new (&data_[size_++]) T(t); } + // For C++03 compatibility this only takes one argument + template<class X> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void emplace_back(const X& x) { + eigen_assert(size_ < reserve_); + new (&data_[size_++]) T(x); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[] (size_t i) const { eigen_assert(i < size_); @@ -88,11 +119,8 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pop_back() { - // NOTE: This does not destroy the value at the end the way - // std::vector's version of pop_back() does. That happens when - // the Vector is destroyed. eigen_assert(size_ > 0); - size_--; + data_[--size_].~T(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
diff --git a/unsupported/Eigen/EulerAngles b/unsupported/Eigen/EulerAngles new file mode 100644 index 0000000..f8f1c5d --- /dev/null +++ b/unsupported/Eigen/EulerAngles
@@ -0,0 +1,43 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Tal Hadad <tal_hd@hotmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_EULERANGLES_MODULE_H +#define EIGEN_EULERANGLES_MODULE_H + + +#include "../../Eigen/Core" +#include "../../Eigen/Geometry" + +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + +namespace Eigen { + +/** + * \defgroup EulerAngles_Module EulerAngles module + * \brief This module provides generic euler angles rotation. + * + * Euler angles are a way to represent 3D rotation. + * + * In order to use this module in your code, include this header: + * \code + * #include <unsupported/Eigen/EulerAngles> + * \endcode + * + * See \ref EulerAngles for more information. + * + */ + +} + +#include "src/EulerAngles/EulerSystem.h" +#include "src/EulerAngles/EulerAngles.h" + +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + +#endif // EIGEN_EULERANGLES_MODULE_H
diff --git a/unsupported/Eigen/FFT b/unsupported/Eigen/FFT index 2c45b39..d9ad21a 100644 --- a/unsupported/Eigen/FFT +++ b/unsupported/Eigen/FFT
@@ -13,7 +13,7 @@ #include <complex> #include <vector> #include <map> -#include <Eigen/Core> +#include "../../Eigen/Core" /** @@ -68,6 +68,8 @@ */ +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #ifdef EIGEN_FFTW_DEFAULT // FFTW: faster, GPL -- incompatible with Eigen in LGPL form, bigger code size # include <fftw3.h> @@ -289,6 +291,7 @@ void inv( MatrixBase<OutputDerived> & dst, const MatrixBase<ComplexDerived> & src, Index nfft=-1) { typedef typename ComplexDerived::Scalar src_type; + typedef typename ComplexDerived::RealScalar real_type; typedef typename OutputDerived::Scalar dst_type; const bool realfft= (NumTraits<dst_type>::IsComplex == 0); EIGEN_STATIC_ASSERT_VECTOR_ONLY(OutputDerived) @@ -329,9 +332,9 @@ tmp.head(nhead) = src.head(nhead); tmp.tail(ntail) = src.tail(ntail); if (resize_input<0) { //shrinking -- create the Nyquist bin as the average of the two bins that fold into it - tmp(nhead) = ( src(nfft/2) + src( src.size() - nfft/2 ) )*src_type(.5); + tmp(nhead) = ( src(nfft/2) + src( src.size() - nfft/2 ) )*real_type(.5); }else{ // expanding -- split the old Nyquist bin into two halves - tmp(nhead) = src(nhead) * src_type(.5); + tmp(nhead) = src(nhead) * real_type(.5); tmp(tmp.size()-nhead) = tmp(nhead); } } @@ -414,5 +417,8 @@ } } + +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + #endif /* vim: set filetype=cpp et sw=2 ts=2 ai: */
diff --git a/unsupported/Eigen/IterativeSolvers b/unsupported/Eigen/IterativeSolvers index 31e880b..0fa129a 100644 --- a/unsupported/Eigen/IterativeSolvers +++ b/unsupported/Eigen/IterativeSolvers
@@ -10,7 +10,9 @@ #ifndef EIGEN_ITERATIVE_SOLVERS_MODULE_H #define EIGEN_ITERATIVE_SOLVERS_MODULE_H -#include <Eigen/Sparse> +#include "../../Eigen/Sparse" +#include "../../Eigen/Jacobi" +#include "../../Eigen/Householder" /** * \defgroup IterativeSolvers_Module Iterative solvers module @@ -24,19 +26,21 @@ */ //@{ +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #ifndef EIGEN_MPL2_ONLY #include "src/IterativeSolvers/IterationController.h" #include "src/IterativeSolvers/ConstrainedConjGrad.h" #endif #include "src/IterativeSolvers/IncompleteLU.h" -#include "../../Eigen/Jacobi" -#include "../../Eigen/Householder" #include "src/IterativeSolvers/GMRES.h" #include "src/IterativeSolvers/DGMRES.h" //#include "src/IterativeSolvers/SSORPreconditioner.h" #include "src/IterativeSolvers/MINRES.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + //@} #endif // EIGEN_ITERATIVE_SOLVERS_MODULE_H
diff --git a/unsupported/Eigen/KroneckerProduct b/unsupported/Eigen/KroneckerProduct index c932c06..5f5afb8 100644 --- a/unsupported/Eigen/KroneckerProduct +++ b/unsupported/Eigen/KroneckerProduct
@@ -13,6 +13,8 @@ #include "../../Eigen/src/Core/util/DisableStupidWarnings.h" +#include "../../Eigen/src/SparseCore/SparseUtil.h" + namespace Eigen { /**
diff --git a/unsupported/Eigen/LevenbergMarquardt b/unsupported/Eigen/LevenbergMarquardt index 0fe2680..1090505 100644 --- a/unsupported/Eigen/LevenbergMarquardt +++ b/unsupported/Eigen/LevenbergMarquardt
@@ -12,12 +12,12 @@ // #include <vector> -#include <Eigen/Core> -#include <Eigen/Jacobi> -#include <Eigen/QR> -#include <unsupported/Eigen/NumericalDiff> +#include "../../Eigen/Core" +#include "../../Eigen/Jacobi" +#include "../../Eigen/QR" +#include "NumericalDiff" -#include <Eigen/SparseQR> +#include "../../Eigen/SparseQR" /** * \defgroup LevenbergMarquardt_Module Levenberg-Marquardt module @@ -29,7 +29,10 @@ * */ -#include "Eigen/SparseCore" +#include "../../Eigen/SparseCore" + +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #ifndef EIGEN_PARSED_BY_DOXYGEN #include "src/LevenbergMarquardt/LMqrsolv.h" @@ -41,5 +44,6 @@ #include "src/LevenbergMarquardt/LevenbergMarquardt.h" #include "src/LevenbergMarquardt/LMonestep.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_LEVENBERGMARQUARDT_MODULE
diff --git a/unsupported/Eigen/MPRealSupport b/unsupported/Eigen/MPRealSupport index 8903688..c4ea4ec 100644 --- a/unsupported/Eigen/MPRealSupport +++ b/unsupported/Eigen/MPRealSupport
@@ -12,7 +12,7 @@ #ifndef EIGEN_MPREALSUPPORT_MODULE_H #define EIGEN_MPREALSUPPORT_MODULE_H -#include <Eigen/Core> +#include "../../Eigen/Core" #include <mpreal.h> namespace Eigen { @@ -67,27 +67,35 @@ IsSigned = 1, IsComplex = 0, RequireInitialization = 1, - ReadCost = 10, - AddCost = 10, - MulCost = 40 + ReadCost = HugeCost, + AddCost = HugeCost, + MulCost = HugeCost }; typedef mpfr::mpreal Real; typedef mpfr::mpreal NonInteger; - inline static Real highest (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::maxval(Precision); } - inline static Real lowest (long Precision = mpfr::mpreal::get_default_prec()) { return -mpfr::maxval(Precision); } + static inline Real highest (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::maxval(Precision); } + static inline Real lowest (long Precision = mpfr::mpreal::get_default_prec()) { return -mpfr::maxval(Precision); } // Constants - inline static Real Pi (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_pi(Precision); } - inline static Real Euler (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_euler(Precision); } - inline static Real Log2 (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_log2(Precision); } - inline static Real Catalan (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_catalan(Precision); } + static inline Real Pi (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_pi(Precision); } + static inline Real Euler (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_euler(Precision); } + static inline Real Log2 (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_log2(Precision); } + static inline Real Catalan (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::const_catalan(Precision); } - inline static Real epsilon (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::machine_epsilon(Precision); } - inline static Real epsilon (const Real& x) { return mpfr::machine_epsilon(x); } + static inline Real epsilon (long Precision = mpfr::mpreal::get_default_prec()) { return mpfr::machine_epsilon(Precision); } + static inline Real epsilon (const Real& x) { return mpfr::machine_epsilon(x); } - inline static Real dummy_precision() +#ifdef MPREAL_HAVE_DYNAMIC_STD_NUMERIC_LIMITS + static inline int digits10 (long Precision = mpfr::mpreal::get_default_prec()) { return std::numeric_limits<Real>::digits10(Precision); } + static inline int digits10 (const Real& x) { return std::numeric_limits<Real>::digits10(x); } + + static inline int digits () { return std::numeric_limits<Real>::digits(); } + static inline int digits (const Real& x) { return std::numeric_limits<Real>::digits(x); } +#endif + + static inline Real dummy_precision() { mpfr_prec_t weak_prec = ((mpfr::mpreal::get_default_prec()-1) * 90) / 100; return mpfr::machine_epsilon(weak_prec); @@ -154,6 +162,7 @@ typedef ResScalar LhsPacket; typedef ResScalar RhsPacket; typedef ResScalar ResPacket; + typedef LhsPacket LhsPacket4Packing; };
diff --git a/unsupported/Eigen/MatrixFunctions b/unsupported/Eigen/MatrixFunctions index 0320606..20c23d1 100644 --- a/unsupported/Eigen/MatrixFunctions +++ b/unsupported/Eigen/MatrixFunctions
@@ -14,9 +14,9 @@ #include <cfloat> #include <list> -#include <Eigen/Core> -#include <Eigen/LU> -#include <Eigen/Eigenvalues> +#include "../../Eigen/Core" +#include "../../Eigen/LU" +#include "../../Eigen/Eigenvalues" /** * \defgroup MatrixFunctions_Module Matrix functions module @@ -53,12 +53,16 @@ * */ +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #include "src/MatrixFunctions/MatrixExponential.h" #include "src/MatrixFunctions/MatrixFunction.h" #include "src/MatrixFunctions/MatrixSquareRoot.h" #include "src/MatrixFunctions/MatrixLogarithm.h" #include "src/MatrixFunctions/MatrixPower.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + /** \page matrixbaseextra_page @@ -161,8 +165,8 @@ \include MatrixExponential.cpp Output: \verbinclude MatrixExponential.out -\note \p M has to be a matrix of \c float, \c double, \c long double -\c complex<float>, \c complex<double>, or \c complex<long double> . +\note \p M has to be a matrix of \c float, \c double, `long double` +\c complex<float>, \c complex<double>, or `complex<long double>` . \subsection matrixbase_log MatrixBase::log() @@ -219,9 +223,8 @@ \include MatrixLogarithm.cpp Output: \verbinclude MatrixLogarithm.out -\note \p M has to be a matrix of \c float, \c double, <tt>long -double</tt>, \c complex<float>, \c complex<double>, or \c complex<long -double> . +\note \p M has to be a matrix of \c float, \c double, `long +double`, \c complex<float>, \c complex<double>, or `complex<long double>`. \sa MatrixBase::exp(), MatrixBase::matrixFunction(), class MatrixLogarithmAtomic, MatrixBase::sqrt(). @@ -326,9 +329,9 @@ \include MatrixPower_optimal.cpp Output: \verbinclude MatrixPower_optimal.out -\note \p M has to be a matrix of \c float, \c double, <tt>long -double</tt>, \c complex<float>, \c complex<double>, or \c complex<long -double> . +\note \p M has to be a matrix of \c float, \c double, `long +double`, \c complex<float>, \c complex<double>, or +\c complex<long double> . \sa MatrixBase::exp(), MatrixBase::log(), class MatrixPower.
diff --git a/unsupported/Eigen/MoreVectorization b/unsupported/Eigen/MoreVectorization index 470e724..7662b47 100644 --- a/unsupported/Eigen/MoreVectorization +++ b/unsupported/Eigen/MoreVectorization
@@ -9,7 +9,7 @@ #ifndef EIGEN_MOREVECTORIZATION_MODULE_H #define EIGEN_MOREVECTORIZATION_MODULE_H -#include <Eigen/Core> +#include "../../Eigen/Core" namespace Eigen {
diff --git a/unsupported/Eigen/NonLinearOptimization b/unsupported/Eigen/NonLinearOptimization index 600ab4c..961f192 100644 --- a/unsupported/Eigen/NonLinearOptimization +++ b/unsupported/Eigen/NonLinearOptimization
@@ -12,10 +12,10 @@ #include <vector> -#include <Eigen/Core> -#include <Eigen/Jacobi> -#include <Eigen/QR> -#include <unsupported/Eigen/NumericalDiff> +#include "../../Eigen/Core" +#include "../../Eigen/Jacobi" +#include "../../Eigen/QR" +#include "NumericalDiff" /** * \defgroup NonLinearOptimization_Module Non linear optimization module @@ -30,12 +30,12 @@ * actually linear. But if this is so, you should probably better use other * methods more fitted to this special case. * - * One algorithm allows to find an extremum of such a system (Levenberg - * Marquardt algorithm) and the second one is used to find + * One algorithm allows to find a least-squares solution of such a system + * (Levenberg-Marquardt algorithm) and the second one is used to find * a zero for the system (Powell hybrid "dogleg" method). * * This code is a port of minpack (http://en.wikipedia.org/wiki/MINPACK). - * Minpack is a very famous, old, robust and well-reknown package, written in + * Minpack is a very famous, old, robust and well renowned package, written in * fortran. Those implementations have been carefully tuned, tested, and used * for several decades. * @@ -58,35 +58,41 @@ * There are two kinds of tests : those that come from examples bundled with cminpack. * They guaranty we get the same results as the original algorithms (value for 'x', * for the number of evaluations of the function, and for the number of evaluations - * of the jacobian if ever). + * of the Jacobian if ever). * * Other tests were added by myself at the very beginning of the - * process and check the results for levenberg-marquardt using the reference data + * process and check the results for Levenberg-Marquardt using the reference data * on http://www.itl.nist.gov/div898/strd/nls/nls_main.shtml. Since then i've - * carefully checked that the same results were obtained when modifiying the + * carefully checked that the same results were obtained when modifying the * code. Please note that we do not always get the exact same decimals as they do, * but this is ok : they use 128bits float, and we do the tests using the C type 'double', * which is 64 bits on most platforms (x86 and amd64, at least). - * I've performed those tests on several other implementations of levenberg-marquardt, and + * I've performed those tests on several other implementations of Levenberg-Marquardt, and * (c)minpack performs VERY well compared to those, both in accuracy and speed. * * The documentation for running the tests is on the wiki * http://eigen.tuxfamily.org/index.php?title=Tests * - * \section API API : overview of methods + * \section API API: overview of methods * - * Both algorithms can use either the jacobian (provided by the user) or compute - * an approximation by themselves (actually using Eigen \ref NumericalDiff_Module). - * The part of API referring to the latter use 'NumericalDiff' in the method names - * (exemple: LevenbergMarquardt.minimizeNumericalDiff() ) + * Both algorithms needs a functor computing the Jacobian. It can be computed by + * hand, using auto-differentiation (see \ref AutoDiff_Module), or using numerical + * differences (see \ref NumericalDiff_Module). For instance: + *\code + * MyFunc func; + * NumericalDiff<MyFunc> func_with_num_diff(func); + * LevenbergMarquardt<NumericalDiff<MyFunc> > lm(func_with_num_diff); + * \endcode + * For HybridNonLinearSolver, the method solveNumericalDiff() does the above wrapping for + * you. * * The methods LevenbergMarquardt.lmder1()/lmdif1()/lmstr1() and * HybridNonLinearSolver.hybrj1()/hybrd1() are specific methods from the original * minpack package that you probably should NOT use until you are porting a code that - * was previously using minpack. They just define a 'simple' API with default values + * was previously using minpack. They just define a 'simple' API with default values * for some parameters. * - * All algorithms are provided using Two APIs : + * All algorithms are provided using two APIs : * - one where the user inits the algorithm, and uses '*OneStep()' as much as he wants : * this way the caller have control over the steps * - one where the user just calls a method (optimize() or solve()) which will @@ -94,7 +100,7 @@ * convenience. * * As an example, the method LevenbergMarquardt::minimize() is - * implemented as follow : + * implemented as follow: * \code * Status LevenbergMarquardt<FunctorType,Scalar>::minimize(FVectorType &x, const int mode) * {
diff --git a/unsupported/Eigen/NumericalDiff b/unsupported/Eigen/NumericalDiff index 433334c..0668f96 100644 --- a/unsupported/Eigen/NumericalDiff +++ b/unsupported/Eigen/NumericalDiff
@@ -10,7 +10,7 @@ #ifndef EIGEN_NUMERICALDIFF_MODULE #define EIGEN_NUMERICALDIFF_MODULE -#include <Eigen/Core> +#include "../../Eigen/Core" namespace Eigen {
diff --git a/unsupported/Eigen/OpenGLSupport b/unsupported/Eigen/OpenGLSupport index 87f5094..f8c2130 100644 --- a/unsupported/Eigen/OpenGLSupport +++ b/unsupported/Eigen/OpenGLSupport
@@ -10,7 +10,7 @@ #ifndef EIGEN_OPENGL_MODULE #define EIGEN_OPENGL_MODULE -#include <Eigen/Geometry> +#include "../../Eigen/Geometry" #if defined(__APPLE_CC__) #include <OpenGL/gl.h> @@ -25,7 +25,7 @@ * * This module provides wrapper functions for a couple of OpenGL functions * which simplify the way to pass Eigen's object to openGL. - * Here is an exmaple: + * Here is an example: * * \code * // You need to add path_to_eigen/unsupported to your include path. @@ -184,7 +184,7 @@ } inline void glRotate(const Rotation2D<double>& rot) { - glRotated(rot.angle()*180.0/EIGEN_PI, 0.0, 0.0, 1.0); + glRotated(rot.angle()*180.0/double(EIGEN_PI), 0.0, 0.0, 1.0); } template<typename Derived> void glRotate(const RotationBase<Derived,3>& rot)
diff --git a/unsupported/Eigen/Polynomials b/unsupported/Eigen/Polynomials index cece563..146e5c4 100644 --- a/unsupported/Eigen/Polynomials +++ b/unsupported/Eigen/Polynomials
@@ -9,11 +9,11 @@ #ifndef EIGEN_POLYNOMIALS_MODULE_H #define EIGEN_POLYNOMIALS_MODULE_H -#include <Eigen/Core> +#include "../../Eigen/Core" -#include <Eigen/src/Core/util/DisableStupidWarnings.h> +#include "../../Eigen/Eigenvalues" -#include <Eigen/Eigenvalues> +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" // Note that EIGEN_HIDE_HEAVY_CODE has to be defined per module #if (defined EIGEN_EXTERN_INSTANTIATIONS) && (EIGEN_EXTERN_INSTANTIATIONS>=2) @@ -132,7 +132,7 @@ Output: \verbinclude PolynomialSolver1.out */ -#include <Eigen/src/Core/util/ReenableStupidWarnings.h> +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_POLYNOMIALS_MODULE_H /* vim: set filetype=cpp et sw=2 ts=2 ai: */
diff --git a/unsupported/Eigen/Skyline b/unsupported/Eigen/Skyline index 71a68cb..ebdf143 100644 --- a/unsupported/Eigen/Skyline +++ b/unsupported/Eigen/Skyline
@@ -10,9 +10,9 @@ #define EIGEN_SKYLINE_MODULE_H -#include "Eigen/Core" +#include "../../Eigen/Core" -#include "Eigen/src/Core/util/DisableStupidWarnings.h" +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" #include <map> #include <cstdlib> @@ -34,6 +34,6 @@ #include "src/Skyline/SkylineInplaceLU.h" #include "src/Skyline/SkylineProduct.h" -#include "Eigen/src/Core/util/ReenableStupidWarnings.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_SKYLINE_MODULE_H
diff --git a/unsupported/Eigen/SpecialFunctions b/unsupported/Eigen/SpecialFunctions new file mode 100644 index 0000000..44fd99b --- /dev/null +++ b/unsupported/Eigen/SpecialFunctions
@@ -0,0 +1,67 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <g.gael@free.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPECIALFUNCTIONS_MODULE +#define EIGEN_SPECIALFUNCTIONS_MODULE + +#include <math.h> + +#include "../../Eigen/Core" + +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + +namespace Eigen { + +/** + * \defgroup SpecialFunctions_Module Special math functions module + * + * This module features additional coefficient-wise math functions available + * within the numext:: namespace for the scalar version, and as method and/or free + * functions of Array. Those include: + * + * - erf + * - erfc + * - lgamma + * - igamma + * - igamma_der_a + * - gamma_sample_der_alpha + * - igammac + * - digamma + * - polygamma + * - zeta + * - betainc + * - i0e + * - i1e + * + * \code + * #include <unsupported/Eigen/SpecialFunctions> + * \endcode + */ +//@{ + +} + +#include "src/SpecialFunctions/SpecialFunctionsImpl.h" +#include "src/SpecialFunctions/SpecialFunctionsPacketMath.h" +#include "src/SpecialFunctions/SpecialFunctionsHalf.h" +#include "src/SpecialFunctions/SpecialFunctionsFunctors.h" +#include "src/SpecialFunctions/SpecialFunctionsArrayAPI.h" + +#if defined EIGEN_VECTORIZE_GPU + #include "src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h" +#endif + +namespace Eigen { +//@} +} + + +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + +#endif // EIGEN_SPECIALFUNCTIONS_MODULE
diff --git a/unsupported/Eigen/Splines b/unsupported/Eigen/Splines index 322e6b9..2ca5813 100644 --- a/unsupported/Eigen/Splines +++ b/unsupported/Eigen/Splines
@@ -24,8 +24,12 @@ */ } +#include "../../Eigen/src/Core/util/DisableStupidWarnings.h" + #include "src/Splines/SplineFwd.h" #include "src/Splines/Spline.h" #include "src/Splines/SplineFitting.h" +#include "../../Eigen/src/Core/util/ReenableStupidWarnings.h" + #endif // EIGEN_SPLINES_MODULE_H
diff --git a/unsupported/Eigen/src/AutoDiff/AutoDiffJacobian.h b/unsupported/Eigen/src/AutoDiff/AutoDiffJacobian.h index 1a61e33..33b6c39 100644 --- a/unsupported/Eigen/src/AutoDiff/AutoDiffJacobian.h +++ b/unsupported/Eigen/src/AutoDiff/AutoDiffJacobian.h
@@ -20,37 +20,60 @@ AutoDiffJacobian(const Functor& f) : Functor(f) {} // forward constructors +#if EIGEN_HAS_VARIADIC_TEMPLATES + template<typename... T> + AutoDiffJacobian(const T& ...Values) : Functor(Values...) {} +#else template<typename T0> AutoDiffJacobian(const T0& a0) : Functor(a0) {} template<typename T0, typename T1> AutoDiffJacobian(const T0& a0, const T1& a1) : Functor(a0, a1) {} template<typename T0, typename T1, typename T2> AutoDiffJacobian(const T0& a0, const T1& a1, const T2& a2) : Functor(a0, a1, a2) {} - - enum { - InputsAtCompileTime = Functor::InputsAtCompileTime, - ValuesAtCompileTime = Functor::ValuesAtCompileTime - }; +#endif typedef typename Functor::InputType InputType; typedef typename Functor::ValueType ValueType; - typedef typename Functor::JacobianType JacobianType; - typedef typename JacobianType::Scalar Scalar; + typedef typename ValueType::Scalar Scalar; + + enum { + InputsAtCompileTime = InputType::RowsAtCompileTime, + ValuesAtCompileTime = ValueType::RowsAtCompileTime + }; + + typedef Matrix<Scalar, ValuesAtCompileTime, InputsAtCompileTime> JacobianType; typedef typename JacobianType::Index Index; - typedef Matrix<Scalar,InputsAtCompileTime,1> DerivativeType; + typedef Matrix<Scalar, InputsAtCompileTime, 1> DerivativeType; typedef AutoDiffScalar<DerivativeType> ActiveScalar; - typedef Matrix<ActiveScalar, InputsAtCompileTime, 1> ActiveInput; typedef Matrix<ActiveScalar, ValuesAtCompileTime, 1> ActiveValue; +#if EIGEN_HAS_VARIADIC_TEMPLATES + // Some compilers don't accept variadic parameters after a default parameter, + // i.e., we can't just write _jac=0 but we need to overload operator(): + EIGEN_STRONG_INLINE + void operator() (const InputType& x, ValueType* v) const + { + this->operator()(x, v, 0); + } + template<typename... ParamsType> + void operator() (const InputType& x, ValueType* v, JacobianType* _jac, + const ParamsType&... Params) const +#else void operator() (const InputType& x, ValueType* v, JacobianType* _jac=0) const +#endif { eigen_assert(v!=0); + if (!_jac) { +#if EIGEN_HAS_VARIADIC_TEMPLATES + Functor::operator()(x, v, Params...); +#else Functor::operator()(x, v); +#endif return; } @@ -61,12 +84,16 @@ if(InputsAtCompileTime==Dynamic) for (Index j=0; j<jac.rows(); j++) - av[j].derivatives().resize(this->inputs()); + av[j].derivatives().resize(x.rows()); for (Index i=0; i<jac.cols(); i++) - ax[i].derivatives() = DerivativeType::Unit(this->inputs(),i); + ax[i].derivatives() = DerivativeType::Unit(x.rows(),i); +#if EIGEN_HAS_VARIADIC_TEMPLATES + Functor::operator()(ax, &av, Params...); +#else Functor::operator()(ax, &av); +#endif for (Index i=0; i<jac.rows(); i++) { @@ -74,8 +101,6 @@ jac.row(i) = av[i].derivatives(); } } -protected: - }; }
diff --git a/unsupported/Eigen/src/AutoDiff/AutoDiffScalar.h b/unsupported/Eigen/src/AutoDiff/AutoDiffScalar.h index 481dfa9..2db9147 100755 --- a/unsupported/Eigen/src/AutoDiff/AutoDiffScalar.h +++ b/unsupported/Eigen/src/AutoDiff/AutoDiffScalar.h
@@ -30,6 +30,13 @@ } // end namespace internal +template<typename _DerType> class AutoDiffScalar; + +template<typename NewDerType> +inline AutoDiffScalar<NewDerType> MakeAutoDiffScalar(const typename NewDerType::Scalar& value, const NewDerType &der) { + return AutoDiffScalar<NewDerType>(value,der); +} + /** \class AutoDiffScalar * \brief A scalar type replacement with automatic differentation capability * @@ -60,7 +67,7 @@ class AutoDiffScalar : public internal::auto_diff_special_op <_DerType, !internal::is_same<typename internal::traits<typename internal::remove_all<_DerType>::type>::Scalar, - typename NumTraits<typename internal::traits<typename internal::remove_all<_DerType>::type>::Scalar>::Real>::value> + typename NumTraits<typename internal::traits<typename internal::remove_all<_DerType>::type>::Scalar>::Real>::value> { public: typedef internal::auto_diff_special_op @@ -101,7 +108,9 @@ template<typename OtherDerType> AutoDiffScalar(const AutoDiffScalar<OtherDerType>& other #ifndef EIGEN_PARSED_BY_DOXYGEN - , typename internal::enable_if<internal::is_same<Scalar,typename OtherDerType::Scalar>::value,void*>::type = 0 + , typename internal::enable_if< + internal::is_same<Scalar, typename internal::traits<typename internal::remove_all<OtherDerType>::type>::Scalar>::value + && internal::is_convertible<OtherDerType,DerType>::value , void*>::type = 0 #endif ) : m_value(other.value()), m_derivatives(other.derivatives()) @@ -257,20 +266,16 @@ -m_derivatives); } - inline const AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> > + inline const AutoDiffScalar<EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product) > operator*(const Scalar& other) const { - return AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> >( - m_value * other, - (m_derivatives * other)); + return MakeAutoDiffScalar(m_value * other, m_derivatives * other); } - friend inline const AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> > + friend inline const AutoDiffScalar<EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product) > operator*(const Scalar& other, const AutoDiffScalar& a) { - return AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> >( - a.value() * other, - a.derivatives() * other); + return MakeAutoDiffScalar(a.value() * other, a.derivatives() * other); } // inline const AutoDiffScalar<typename CwiseUnaryOp<internal::scalar_multiple_op<Real>, DerType>::Type > @@ -289,20 +294,16 @@ // a.derivatives() * other); // } - inline const AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> > + inline const AutoDiffScalar<EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product) > operator/(const Scalar& other) const { - return AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> >( - m_value / other, - (m_derivatives * (Scalar(1)/other))); + return MakeAutoDiffScalar(m_value / other, (m_derivatives * (Scalar(1)/other))); } - friend inline const AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> > + friend inline const AutoDiffScalar<EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product) > operator/(const Scalar& other, const AutoDiffScalar& a) { - return AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType> >( - other / a.value(), - a.derivatives() * (Scalar(-other) / (a.value()*a.value()))); + return MakeAutoDiffScalar(other / a.value(), a.derivatives() * (Scalar(-other) / (a.value()*a.value()))); } // inline const AutoDiffScalar<typename CwiseUnaryOp<internal::scalar_multiple_op<Real>, DerType>::Type > @@ -322,34 +323,29 @@ // } template<typename OtherDerType> - inline const AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, - const CwiseBinaryOp<internal::scalar_difference_op<Scalar>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const typename internal::remove_all<OtherDerType>::type > > > > + inline const AutoDiffScalar<EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE( + CwiseBinaryOp<internal::scalar_difference_op<Scalar> EIGEN_COMMA + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product) EIGEN_COMMA + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(typename internal::remove_all<OtherDerType>::type,Scalar,product) >,Scalar,product) > operator/(const AutoDiffScalar<OtherDerType>& other) const { internal::make_coherent(m_derivatives, other.derivatives()); - return AutoDiffScalar<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, - const CwiseBinaryOp<internal::scalar_difference_op<Scalar>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const typename internal::remove_all<OtherDerType>::type > > > >( + return MakeAutoDiffScalar( m_value / other.value(), - ((m_derivatives * other.value()) - (m_value * other.derivatives())) + ((m_derivatives * other.value()) - (other.derivatives() * m_value)) * (Scalar(1)/(other.value()*other.value()))); } template<typename OtherDerType> inline const AutoDiffScalar<CwiseBinaryOp<internal::scalar_sum_op<Scalar>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const typename internal::remove_all<OtherDerType>::type> > > + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DerType,Scalar,product), + const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(typename internal::remove_all<OtherDerType>::type,Scalar,product) > > operator*(const AutoDiffScalar<OtherDerType>& other) const { internal::make_coherent(m_derivatives, other.derivatives()); - return AutoDiffScalar<const CwiseBinaryOp<internal::scalar_sum_op<Scalar>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const DerType>, - const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const typename internal::remove_all<OtherDerType>::type > > >( + return MakeAutoDiffScalar( m_value * other.value(), - (m_derivatives * other.value()) + (m_value * other.derivatives())); + (m_derivatives * other.value()) + (other.derivatives() * m_value)); } inline AutoDiffScalar& operator*=(const Scalar& other) @@ -426,18 +422,18 @@ } - inline const AutoDiffScalar<typename CwiseUnaryOp<scalar_multiple2_op<Scalar,Real>, DerType>::Type > + inline const AutoDiffScalar<typename CwiseUnaryOp<bind2nd_op<scalar_product_op<Scalar,Real> >, DerType>::Type > operator*(const Real& other) const { - return AutoDiffScalar<typename CwiseUnaryOp<scalar_multiple2_op<Scalar,Real>, DerType>::Type >( + return AutoDiffScalar<typename CwiseUnaryOp<bind2nd_op<scalar_product_op<Scalar,Real> >, DerType>::Type >( derived().value() * other, derived().derivatives() * other); } - friend inline const AutoDiffScalar<typename CwiseUnaryOp<scalar_multiple2_op<Scalar,Real>, DerType>::Type > + friend inline const AutoDiffScalar<typename CwiseUnaryOp<bind1st_op<scalar_product_op<Real,Scalar> >, DerType>::Type > operator*(const Real& other, const AutoDiffScalar<_DerType>& a) { - return AutoDiffScalar<typename CwiseUnaryOp<scalar_multiple2_op<Scalar,Real>, DerType>::Type >( + return AutoDiffScalar<typename CwiseUnaryOp<bind1st_op<scalar_product_op<Real,Scalar> >, DerType>::Type >( a.value() * other, a.derivatives() * other); } @@ -501,43 +497,45 @@ } }; -template<typename A_Scalar, int A_Rows, int A_Cols, int A_Options, int A_MaxRows, int A_MaxCols> -struct scalar_product_traits<Matrix<A_Scalar, A_Rows, A_Cols, A_Options, A_MaxRows, A_MaxCols>,A_Scalar> -{ - enum { Defined = 1 }; - typedef Matrix<A_Scalar, A_Rows, A_Cols, A_Options, A_MaxRows, A_MaxCols> ReturnType; -}; - -template<typename A_Scalar, int A_Rows, int A_Cols, int A_Options, int A_MaxRows, int A_MaxCols> -struct scalar_product_traits<A_Scalar, Matrix<A_Scalar, A_Rows, A_Cols, A_Options, A_MaxRows, A_MaxCols> > -{ - enum { Defined = 1 }; - typedef Matrix<A_Scalar, A_Rows, A_Cols, A_Options, A_MaxRows, A_MaxCols> ReturnType; -}; - -template<typename DerType> -struct scalar_product_traits<AutoDiffScalar<DerType>,typename DerType::Scalar> -{ - enum { Defined = 1 }; - typedef AutoDiffScalar<DerType> ReturnType; -}; - -template<typename DerType> -struct scalar_product_traits<typename DerType::Scalar,AutoDiffScalar<DerType> > -{ - enum { Defined = 1 }; - typedef AutoDiffScalar<DerType> ReturnType; -}; - } // end namespace internal +template<typename DerType, typename BinOp> +struct ScalarBinaryOpTraits<AutoDiffScalar<DerType>,typename DerType::Scalar,BinOp> +{ + typedef AutoDiffScalar<DerType> ReturnType; +}; + +template<typename DerType, typename BinOp> +struct ScalarBinaryOpTraits<typename DerType::Scalar,AutoDiffScalar<DerType>, BinOp> +{ + typedef AutoDiffScalar<DerType> ReturnType; +}; + + +// The following is an attempt to let Eigen's known about expression template, but that's more tricky! + +// template<typename DerType, typename BinOp> +// struct ScalarBinaryOpTraits<AutoDiffScalar<DerType>,AutoDiffScalar<DerType>, BinOp> +// { +// enum { Defined = 1 }; +// typedef AutoDiffScalar<typename DerType::PlainObject> ReturnType; +// }; +// +// template<typename DerType1,typename DerType2, typename BinOp> +// struct ScalarBinaryOpTraits<AutoDiffScalar<DerType1>,AutoDiffScalar<DerType2>, BinOp> +// { +// enum { Defined = 1 };//internal::is_same<typename DerType1::Scalar,typename DerType2::Scalar>::value }; +// typedef AutoDiffScalar<typename DerType1::PlainObject> ReturnType; +// }; + #define EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(FUNC,CODE) \ template<typename DerType> \ - inline const Eigen::AutoDiffScalar<Eigen::CwiseUnaryOp<Eigen::internal::scalar_multiple_op<typename Eigen::internal::traits<typename Eigen::internal::remove_all<DerType>::type>::Scalar>, const typename Eigen::internal::remove_all<DerType>::type> > \ + inline const Eigen::AutoDiffScalar< \ + EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(typename Eigen::internal::remove_all<DerType>::type, typename Eigen::internal::traits<typename Eigen::internal::remove_all<DerType>::type>::Scalar, product) > \ FUNC(const Eigen::AutoDiffScalar<DerType>& x) { \ using namespace Eigen; \ typedef typename Eigen::internal::traits<typename Eigen::internal::remove_all<DerType>::type>::Scalar Scalar; \ - typedef AutoDiffScalar<CwiseUnaryOp<Eigen::internal::scalar_multiple_op<Scalar>, const typename Eigen::internal::remove_all<DerType>::type> > ReturnType; \ + EIGEN_UNUSED_VARIABLE(sizeof(Scalar)); \ CODE; \ } @@ -548,56 +546,75 @@ template<typename DerType> inline typename DerType::Scalar imag(const AutoDiffScalar<DerType>&) { return 0.; } template<typename DerType, typename T> -inline AutoDiffScalar<DerType> (min)(const AutoDiffScalar<DerType>& x, const T& y) { return (x <= y ? x : y); } +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (min)(const AutoDiffScalar<DerType>& x, const T& y) { + typedef AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> ADS; + return (x <= y ? ADS(x) : ADS(y)); +} template<typename DerType, typename T> -inline AutoDiffScalar<DerType> (max)(const AutoDiffScalar<DerType>& x, const T& y) { return (x >= y ? x : y); } +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (max)(const AutoDiffScalar<DerType>& x, const T& y) { + typedef AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> ADS; + return (x >= y ? ADS(x) : ADS(y)); +} template<typename DerType, typename T> -inline AutoDiffScalar<DerType> (min)(const T& x, const AutoDiffScalar<DerType>& y) { return (x < y ? x : y); } +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (min)(const T& x, const AutoDiffScalar<DerType>& y) { + typedef AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> ADS; + return (x < y ? ADS(x) : ADS(y)); +} template<typename DerType, typename T> -inline AutoDiffScalar<DerType> (max)(const T& x, const AutoDiffScalar<DerType>& y) { return (x > y ? x : y); } +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (max)(const T& x, const AutoDiffScalar<DerType>& y) { + typedef AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> ADS; + return (x > y ? ADS(x) : ADS(y)); +} +template<typename DerType> +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (min)(const AutoDiffScalar<DerType>& x, const AutoDiffScalar<DerType>& y) { + return (x.value() < y.value() ? x : y); +} +template<typename DerType> +inline AutoDiffScalar<typename Eigen::internal::remove_all<DerType>::type::PlainObject> (max)(const AutoDiffScalar<DerType>& x, const AutoDiffScalar<DerType>& y) { + return (x.value() >= y.value() ? x : y); +} + EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(abs, using std::abs; - return ReturnType(abs(x.value()), x.derivatives() * (x.value()<0 ? -1 : 1) );) + return Eigen::MakeAutoDiffScalar(abs(x.value()), x.derivatives() * (x.value()<0 ? -1 : 1) );) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(abs2, using numext::abs2; - return ReturnType(abs2(x.value()), x.derivatives() * (Scalar(2)*x.value()));) + return Eigen::MakeAutoDiffScalar(abs2(x.value()), x.derivatives() * (Scalar(2)*x.value()));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(sqrt, using std::sqrt; Scalar sqrtx = sqrt(x.value()); - return ReturnType(sqrtx,x.derivatives() * (Scalar(0.5) / sqrtx));) + return Eigen::MakeAutoDiffScalar(sqrtx,x.derivatives() * (Scalar(0.5) / sqrtx));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(cos, using std::cos; using std::sin; - return ReturnType(cos(x.value()), x.derivatives() * (-sin(x.value())));) + return Eigen::MakeAutoDiffScalar(cos(x.value()), x.derivatives() * (-sin(x.value())));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(sin, using std::sin; using std::cos; - return ReturnType(sin(x.value()),x.derivatives() * cos(x.value()));) + return Eigen::MakeAutoDiffScalar(sin(x.value()),x.derivatives() * cos(x.value()));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(exp, using std::exp; Scalar expx = exp(x.value()); - return ReturnType(expx,x.derivatives() * expx);) + return Eigen::MakeAutoDiffScalar(expx,x.derivatives() * expx);) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(log, using std::log; - return ReturnType(log(x.value()),x.derivatives() * (Scalar(1)/x.value()));) + return Eigen::MakeAutoDiffScalar(log(x.value()),x.derivatives() * (Scalar(1)/x.value()));) template<typename DerType> -inline const Eigen::AutoDiffScalar<Eigen::CwiseUnaryOp<Eigen::internal::scalar_multiple_op<typename internal::traits<typename internal::remove_all<DerType>::type>::Scalar>, const typename internal::remove_all<DerType>::type> > -pow(const Eigen::AutoDiffScalar<DerType>& x, typename internal::traits<typename internal::remove_all<DerType>::type>::Scalar y) +inline const Eigen::AutoDiffScalar< +EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(typename internal::remove_all<DerType>::type,typename internal::traits<typename internal::remove_all<DerType>::type>::Scalar,product) > +pow(const Eigen::AutoDiffScalar<DerType> &x, const typename internal::traits<typename internal::remove_all<DerType>::type>::Scalar &y) { using namespace Eigen; - typedef typename internal::remove_all<DerType>::type DerTypeCleaned; - typedef typename Eigen::internal::traits<DerTypeCleaned>::Scalar Scalar; - return AutoDiffScalar<CwiseUnaryOp<Eigen::internal::scalar_multiple_op<Scalar>, const DerTypeCleaned> >( - std::pow(x.value(),y), - x.derivatives() * (y * std::pow(x.value(),y-1))); + using std::pow; + return Eigen::MakeAutoDiffScalar(pow(x.value(),y), x.derivatives() * (y * pow(x.value(),y-1))); } @@ -622,27 +639,44 @@ EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(tan, using std::tan; using std::cos; - return ReturnType(tan(x.value()),x.derivatives() * (Scalar(1)/numext::abs2(cos(x.value()))));) + return Eigen::MakeAutoDiffScalar(tan(x.value()),x.derivatives() * (Scalar(1)/numext::abs2(cos(x.value()))));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(asin, using std::sqrt; using std::asin; - return ReturnType(asin(x.value()),x.derivatives() * (Scalar(1)/sqrt(1-numext::abs2(x.value()))));) + return Eigen::MakeAutoDiffScalar(asin(x.value()),x.derivatives() * (Scalar(1)/sqrt(1-numext::abs2(x.value()))));) EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(acos, using std::sqrt; using std::acos; - return ReturnType(acos(x.value()),x.derivatives() * (Scalar(-1)/sqrt(1-numext::abs2(x.value()))));) + return Eigen::MakeAutoDiffScalar(acos(x.value()),x.derivatives() * (Scalar(-1)/sqrt(1-numext::abs2(x.value()))));) + +EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(tanh, + using std::cosh; + using std::tanh; + return Eigen::MakeAutoDiffScalar(tanh(x.value()),x.derivatives() * (Scalar(1)/numext::abs2(cosh(x.value()))));) + +EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(sinh, + using std::sinh; + using std::cosh; + return Eigen::MakeAutoDiffScalar(sinh(x.value()),x.derivatives() * cosh(x.value()));) + +EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY(cosh, + using std::sinh; + using std::cosh; + return Eigen::MakeAutoDiffScalar(cosh(x.value()),x.derivatives() * sinh(x.value()));) #undef EIGEN_AUTODIFF_DECLARE_GLOBAL_UNARY template<typename DerType> struct NumTraits<AutoDiffScalar<DerType> > - : NumTraits< typename NumTraits<typename DerType::Scalar>::Real > + : NumTraits< typename NumTraits<typename internal::remove_all<DerType>::type::Scalar>::Real > { - typedef AutoDiffScalar<Matrix<typename NumTraits<typename DerType::Scalar>::Real,DerType::RowsAtCompileTime,DerType::ColsAtCompileTime, - DerType::Options, DerType::MaxRowsAtCompileTime, DerType::MaxColsAtCompileTime> > Real; + typedef typename internal::remove_all<DerType>::type DerTypeCleaned; + typedef AutoDiffScalar<Matrix<typename NumTraits<typename DerTypeCleaned::Scalar>::Real,DerTypeCleaned::RowsAtCompileTime,DerTypeCleaned::ColsAtCompileTime, + 0, DerTypeCleaned::MaxRowsAtCompileTime, DerTypeCleaned::MaxColsAtCompileTime> > Real; typedef AutoDiffScalar<DerType> NonInteger; typedef AutoDiffScalar<DerType> Nested; + typedef typename NumTraits<typename DerTypeCleaned::Scalar>::Literal Literal; enum{ RequireInitialization = 1 }; @@ -650,4 +684,16 @@ } +namespace std { + +template <typename T> +class numeric_limits<Eigen::AutoDiffScalar<T> > + : public numeric_limits<typename T::Scalar> {}; + +template <typename T> +class numeric_limits<Eigen::AutoDiffScalar<T&> > + : public numeric_limits<typename T::Scalar> {}; + +} // namespace std + #endif // EIGEN_AUTODIFF_SCALAR_H
diff --git a/unsupported/Eigen/src/AutoDiff/CMakeLists.txt b/unsupported/Eigen/src/AutoDiff/CMakeLists.txt deleted file mode 100644 index ad91fd9..0000000 --- a/unsupported/Eigen/src/AutoDiff/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_AutoDiff_SRCS "*.h") - -INSTALL(FILES - ${Eigen_AutoDiff_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/AutoDiff COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/BVH/CMakeLists.txt b/unsupported/Eigen/src/BVH/CMakeLists.txt deleted file mode 100644 index b377d86..0000000 --- a/unsupported/Eigen/src/BVH/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_BVH_SRCS "*.h") - -INSTALL(FILES - ${Eigen_BVH_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/BVH COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/BVH/KdBVH.h b/unsupported/Eigen/src/BVH/KdBVH.h index 1b8d758..2d5b76a 100644 --- a/unsupported/Eigen/src/BVH/KdBVH.h +++ b/unsupported/Eigen/src/BVH/KdBVH.h
@@ -35,6 +35,7 @@ { outBoxes.insert(outBoxes.end(), boxBegin, boxEnd); eigen_assert(outBoxes.size() == objects.size()); + EIGEN_ONLY_USED_FOR_DEBUG(objects); } }; @@ -170,7 +171,7 @@ typedef internal::vector_int_pair<Scalar, Dim> VIPair; typedef std::vector<VIPair, aligned_allocator<VIPair> > VIPairList; typedef Matrix<Scalar, Dim, 1> VectorType; - struct VectorComparator //compares vectors, or, more specificall, VIPairs along a particular dimension + struct VectorComparator //compares vectors, or more specifically, VIPairs along a particular dimension { VectorComparator(int inDim) : dim(inDim) {} inline bool operator()(const VIPair &v1, const VIPair &v2) const { return v1.first[dim] < v2.first[dim]; }
diff --git a/unsupported/Eigen/src/CMakeLists.txt b/unsupported/Eigen/src/CMakeLists.txt deleted file mode 100644 index a7e8c75..0000000 --- a/unsupported/Eigen/src/CMakeLists.txt +++ /dev/null
@@ -1,15 +0,0 @@ -ADD_SUBDIRECTORY(AutoDiff) -ADD_SUBDIRECTORY(BVH) -ADD_SUBDIRECTORY(Eigenvalues) -ADD_SUBDIRECTORY(FFT) -ADD_SUBDIRECTORY(IterativeSolvers) -ADD_SUBDIRECTORY(LevenbergMarquardt) -ADD_SUBDIRECTORY(MatrixFunctions) -ADD_SUBDIRECTORY(MoreVectorization) -ADD_SUBDIRECTORY(NonLinearOptimization) -ADD_SUBDIRECTORY(NumericalDiff) -ADD_SUBDIRECTORY(Polynomials) -ADD_SUBDIRECTORY(Skyline) -ADD_SUBDIRECTORY(SparseExtra) -ADD_SUBDIRECTORY(KroneckerProduct) -ADD_SUBDIRECTORY(Splines)
diff --git a/unsupported/Eigen/src/Eigenvalues/ArpackSelfAdjointEigenSolver.h b/unsupported/Eigen/src/Eigenvalues/ArpackSelfAdjointEigenSolver.h index 3b6a69a..3c6cfb1 100644 --- a/unsupported/Eigen/src/Eigenvalues/ArpackSelfAdjointEigenSolver.h +++ b/unsupported/Eigen/src/Eigenvalues/ArpackSelfAdjointEigenSolver.h
@@ -25,7 +25,7 @@ #ifndef EIGEN_ARPACKGENERALIZEDSELFADJOINTEIGENSOLVER_H #define EIGEN_ARPACKGENERALIZEDSELFADJOINTEIGENSOLVER_H -#include <Eigen/Dense> +#include "../../../../Eigen/Dense" namespace Eigen { @@ -300,7 +300,7 @@ /** \brief Reports whether previous computation was successful. * - * \returns \c Success if computation was succesful, \c NoConvergence otherwise. + * \returns \c Success if computation was successful, \c NoConvergence otherwise. */ ComputationInfo info() const { @@ -628,15 +628,15 @@ m_info = Success; } - delete select; + delete[] select; } - delete v; - delete iparam; - delete ipntr; - delete workd; - delete workl; - delete resid; + delete[] v; + delete[] iparam; + delete[] ipntr; + delete[] workd; + delete[] workl; + delete[] resid; m_isInitialized = true;
diff --git a/unsupported/Eigen/src/Eigenvalues/CMakeLists.txt b/unsupported/Eigen/src/Eigenvalues/CMakeLists.txt deleted file mode 100644 index 1d4387c..0000000 --- a/unsupported/Eigen/src/Eigenvalues/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Eigenvalues_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Eigenvalues_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/Eigenvalues COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/EulerAngles/CMakeLists.txt b/unsupported/Eigen/src/EulerAngles/CMakeLists.txt new file mode 100644 index 0000000..40af550 --- /dev/null +++ b/unsupported/Eigen/src/EulerAngles/CMakeLists.txt
@@ -0,0 +1,6 @@ +FILE(GLOB Eigen_EulerAngles_SRCS "*.h") + +INSTALL(FILES + ${Eigen_EulerAngles_SRCS} + DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/EulerAngles COMPONENT Devel + )
diff --git a/unsupported/Eigen/src/EulerAngles/EulerAngles.h b/unsupported/Eigen/src/EulerAngles/EulerAngles.h new file mode 100644 index 0000000..e43cdb7 --- /dev/null +++ b/unsupported/Eigen/src/EulerAngles/EulerAngles.h
@@ -0,0 +1,355 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Tal Hadad <tal_hd@hotmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_EULERANGLESCLASS_H// TODO: Fix previous "EIGEN_EULERANGLES_H" definition? +#define EIGEN_EULERANGLESCLASS_H + +namespace Eigen +{ + /** \class EulerAngles + * + * \ingroup EulerAngles_Module + * + * \brief Represents a rotation in a 3 dimensional space as three Euler angles. + * + * Euler rotation is a set of three rotation of three angles over three fixed axes, defined by the EulerSystem given as a template parameter. + * + * Here is how intrinsic Euler angles works: + * - first, rotate the axes system over the alpha axis in angle alpha + * - then, rotate the axes system over the beta axis(which was rotated in the first stage) in angle beta + * - then, rotate the axes system over the gamma axis(which was rotated in the two stages above) in angle gamma + * + * \note This class support only intrinsic Euler angles for simplicity, + * see EulerSystem how to easily overcome this for extrinsic systems. + * + * ### Rotation representation and conversions ### + * + * It has been proved(see Wikipedia link below) that every rotation can be represented + * by Euler angles, but there is no single representation (e.g. unlike rotation matrices). + * Therefore, you can convert from Eigen rotation and to them + * (including rotation matrices, which is not called "rotations" by Eigen design). + * + * Euler angles usually used for: + * - convenient human representation of rotation, especially in interactive GUI. + * - gimbal systems and robotics + * - efficient encoding(i.e. 3 floats only) of rotation for network protocols. + * + * However, Euler angles are slow comparing to quaternion or matrices, + * because their unnatural math definition, although it's simple for human. + * To overcome this, this class provide easy movement from the math friendly representation + * to the human friendly representation, and vise-versa. + * + * All the user need to do is a safe simple C++ type conversion, + * and this class take care for the math. + * Additionally, some axes related computation is done in compile time. + * + * #### Euler angles ranges in conversions #### + * Rotations representation as EulerAngles are not single (unlike matrices), + * and even have infinite EulerAngles representations.<BR> + * For example, add or subtract 2*PI from either angle of EulerAngles + * and you'll get the same rotation. + * This is the general reason for infinite representation, + * but it's not the only general reason for not having a single representation. + * + * When converting rotation to EulerAngles, this class convert it to specific ranges + * When converting some rotation to EulerAngles, the rules for ranges are as follow: + * - If the rotation we converting from is an EulerAngles + * (even when it represented as RotationBase explicitly), angles ranges are __undefined__. + * - otherwise, alpha and gamma angles will be in the range [-PI, PI].<BR> + * As for Beta angle: + * - If the system is Tait-Bryan, the beta angle will be in the range [-PI/2, PI/2]. + * - otherwise: + * - If the beta axis is positive, the beta angle will be in the range [0, PI] + * - If the beta axis is negative, the beta angle will be in the range [-PI, 0] + * + * \sa EulerAngles(const MatrixBase<Derived>&) + * \sa EulerAngles(const RotationBase<Derived, 3>&) + * + * ### Convenient user typedefs ### + * + * Convenient typedefs for EulerAngles exist for float and double scalar, + * in a form of EulerAngles{A}{B}{C}{scalar}, + * e.g. \ref EulerAnglesXYZd, \ref EulerAnglesZYZf. + * + * Only for positive axes{+x,+y,+z} Euler systems are have convenient typedef. + * If you need negative axes{-x,-y,-z}, it is recommended to create you own typedef with + * a word that represent what you need. + * + * ### Example ### + * + * \include EulerAngles.cpp + * Output: \verbinclude EulerAngles.out + * + * ### Additional reading ### + * + * If you're want to get more idea about how Euler system work in Eigen see EulerSystem. + * + * More information about Euler angles: https://en.wikipedia.org/wiki/Euler_angles + * + * \tparam _Scalar the scalar type, i.e. the type of the angles. + * + * \tparam _System the EulerSystem to use, which represents the axes of rotation. + */ + template <typename _Scalar, class _System> + class EulerAngles : public RotationBase<EulerAngles<_Scalar, _System>, 3> + { + public: + typedef RotationBase<EulerAngles<_Scalar, _System>, 3> Base; + + /** the scalar type of the angles */ + typedef _Scalar Scalar; + typedef typename NumTraits<Scalar>::Real RealScalar; + + /** the EulerSystem to use, which represents the axes of rotation. */ + typedef _System System; + + typedef Matrix<Scalar,3,3> Matrix3; /*!< the equivalent rotation matrix type */ + typedef Matrix<Scalar,3,1> Vector3; /*!< the equivalent 3 dimension vector type */ + typedef Quaternion<Scalar> QuaternionType; /*!< the equivalent quaternion type */ + typedef AngleAxis<Scalar> AngleAxisType; /*!< the equivalent angle-axis type */ + + /** \returns the axis vector of the first (alpha) rotation */ + static Vector3 AlphaAxisVector() { + const Vector3& u = Vector3::Unit(System::AlphaAxisAbs - 1); + return System::IsAlphaOpposite ? -u : u; + } + + /** \returns the axis vector of the second (beta) rotation */ + static Vector3 BetaAxisVector() { + const Vector3& u = Vector3::Unit(System::BetaAxisAbs - 1); + return System::IsBetaOpposite ? -u : u; + } + + /** \returns the axis vector of the third (gamma) rotation */ + static Vector3 GammaAxisVector() { + const Vector3& u = Vector3::Unit(System::GammaAxisAbs - 1); + return System::IsGammaOpposite ? -u : u; + } + + private: + Vector3 m_angles; + + public: + /** Default constructor without initialization. */ + EulerAngles() {} + /** Constructs and initialize an EulerAngles (\p alpha, \p beta, \p gamma). */ + EulerAngles(const Scalar& alpha, const Scalar& beta, const Scalar& gamma) : + m_angles(alpha, beta, gamma) {} + + // TODO: Test this constructor + /** Constructs and initialize an EulerAngles from the array data {alpha, beta, gamma} */ + explicit EulerAngles(const Scalar* data) : m_angles(data) {} + + /** Constructs and initializes an EulerAngles from either: + * - a 3x3 rotation matrix expression(i.e. pure orthogonal matrix with determinant of +1), + * - a 3D vector expression representing Euler angles. + * + * \note If \p other is a 3x3 rotation matrix, the angles range rules will be as follow:<BR> + * Alpha and gamma angles will be in the range [-PI, PI].<BR> + * As for Beta angle: + * - If the system is Tait-Bryan, the beta angle will be in the range [-PI/2, PI/2]. + * - otherwise: + * - If the beta axis is positive, the beta angle will be in the range [0, PI] + * - If the beta axis is negative, the beta angle will be in the range [-PI, 0] + */ + template<typename Derived> + explicit EulerAngles(const MatrixBase<Derived>& other) { *this = other; } + + /** Constructs and initialize Euler angles from a rotation \p rot. + * + * \note If \p rot is an EulerAngles (even when it represented as RotationBase explicitly), + * angles ranges are __undefined__. + * Otherwise, alpha and gamma angles will be in the range [-PI, PI].<BR> + * As for Beta angle: + * - If the system is Tait-Bryan, the beta angle will be in the range [-PI/2, PI/2]. + * - otherwise: + * - If the beta axis is positive, the beta angle will be in the range [0, PI] + * - If the beta axis is negative, the beta angle will be in the range [-PI, 0] + */ + template<typename Derived> + EulerAngles(const RotationBase<Derived, 3>& rot) { System::CalcEulerAngles(*this, rot.toRotationMatrix()); } + + /*EulerAngles(const QuaternionType& q) + { + // TODO: Implement it in a faster way for quaternions + // According to http://www.euclideanspace.com/maths/geometry/rotations/conversions/quaternionToEuler/ + // we can compute only the needed matrix cells and then convert to euler angles. (see ZYX example below) + // Currently we compute all matrix cells from quaternion. + + // Special case only for ZYX + //Scalar y2 = q.y() * q.y(); + //m_angles[0] = std::atan2(2*(q.w()*q.z() + q.x()*q.y()), (1 - 2*(y2 + q.z()*q.z()))); + //m_angles[1] = std::asin( 2*(q.w()*q.y() - q.z()*q.x())); + //m_angles[2] = std::atan2(2*(q.w()*q.x() + q.y()*q.z()), (1 - 2*(q.x()*q.x() + y2))); + }*/ + + /** \returns The angle values stored in a vector (alpha, beta, gamma). */ + const Vector3& angles() const { return m_angles; } + /** \returns A read-write reference to the angle values stored in a vector (alpha, beta, gamma). */ + Vector3& angles() { return m_angles; } + + /** \returns The value of the first angle. */ + Scalar alpha() const { return m_angles[0]; } + /** \returns A read-write reference to the angle of the first angle. */ + Scalar& alpha() { return m_angles[0]; } + + /** \returns The value of the second angle. */ + Scalar beta() const { return m_angles[1]; } + /** \returns A read-write reference to the angle of the second angle. */ + Scalar& beta() { return m_angles[1]; } + + /** \returns The value of the third angle. */ + Scalar gamma() const { return m_angles[2]; } + /** \returns A read-write reference to the angle of the third angle. */ + Scalar& gamma() { return m_angles[2]; } + + /** \returns The Euler angles rotation inverse (which is as same as the negative), + * (-alpha, -beta, -gamma). + */ + EulerAngles inverse() const + { + EulerAngles res; + res.m_angles = -m_angles; + return res; + } + + /** \returns The Euler angles rotation negative (which is as same as the inverse), + * (-alpha, -beta, -gamma). + */ + EulerAngles operator -() const + { + return inverse(); + } + + /** Set \c *this from either: + * - a 3x3 rotation matrix expression(i.e. pure orthogonal matrix with determinant of +1), + * - a 3D vector expression representing Euler angles. + * + * See EulerAngles(const MatrixBase<Derived, 3>&) for more information about + * angles ranges output. + */ + template<class Derived> + EulerAngles& operator=(const MatrixBase<Derived>& other) + { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, typename Derived::Scalar>::value), + YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) + + internal::eulerangles_assign_impl<System, Derived>::run(*this, other.derived()); + return *this; + } + + // TODO: Assign and construct from another EulerAngles (with different system) + + /** Set \c *this from a rotation. + * + * See EulerAngles(const RotationBase<Derived, 3>&) for more information about + * angles ranges output. + */ + template<typename Derived> + EulerAngles& operator=(const RotationBase<Derived, 3>& rot) { + System::CalcEulerAngles(*this, rot.toRotationMatrix()); + return *this; + } + + /** \returns \c true if \c *this is approximately equal to \a other, within the precision + * determined by \a prec. + * + * \sa MatrixBase::isApprox() */ + bool isApprox(const EulerAngles& other, + const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const + { return angles().isApprox(other.angles(), prec); } + + /** \returns an equivalent 3x3 rotation matrix. */ + Matrix3 toRotationMatrix() const + { + // TODO: Calc it faster + return static_cast<QuaternionType>(*this).toRotationMatrix(); + } + + /** Convert the Euler angles to quaternion. */ + operator QuaternionType() const + { + return + AngleAxisType(alpha(), AlphaAxisVector()) * + AngleAxisType(beta(), BetaAxisVector()) * + AngleAxisType(gamma(), GammaAxisVector()); + } + + friend std::ostream& operator<<(std::ostream& s, const EulerAngles<Scalar, System>& eulerAngles) + { + s << eulerAngles.angles().transpose(); + return s; + } + + /** \returns \c *this with scalar type casted to \a NewScalarType */ + template <typename NewScalarType> + EulerAngles<NewScalarType, System> cast() const + { + EulerAngles<NewScalarType, System> e; + e.angles() = angles().template cast<NewScalarType>(); + return e; + } + }; + +#define EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(AXES, SCALAR_TYPE, SCALAR_POSTFIX) \ + /** \ingroup EulerAngles_Module */ \ + typedef EulerAngles<SCALAR_TYPE, EulerSystem##AXES> EulerAngles##AXES##SCALAR_POSTFIX; + +#define EIGEN_EULER_ANGLES_TYPEDEFS(SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(XYZ, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(XYX, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(XZY, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(XZX, SCALAR_TYPE, SCALAR_POSTFIX) \ + \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(YZX, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(YZY, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(YXZ, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(YXY, SCALAR_TYPE, SCALAR_POSTFIX) \ + \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(ZXY, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(ZXZ, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(ZYX, SCALAR_TYPE, SCALAR_POSTFIX) \ + EIGEN_EULER_ANGLES_SINGLE_TYPEDEF(ZYZ, SCALAR_TYPE, SCALAR_POSTFIX) + +EIGEN_EULER_ANGLES_TYPEDEFS(float, f) +EIGEN_EULER_ANGLES_TYPEDEFS(double, d) + + namespace internal + { + template<typename _Scalar, class _System> + struct traits<EulerAngles<_Scalar, _System> > + { + typedef _Scalar Scalar; + }; + + // set from a rotation matrix + template<class System, class Other> + struct eulerangles_assign_impl<System,Other,3,3> + { + typedef typename Other::Scalar Scalar; + static void run(EulerAngles<Scalar, System>& e, const Other& m) + { + System::CalcEulerAngles(e, m); + } + }; + + // set from a vector of Euler angles + template<class System, class Other> + struct eulerangles_assign_impl<System,Other,3,1> + { + typedef typename Other::Scalar Scalar; + static void run(EulerAngles<Scalar, System>& e, const Other& vec) + { + e.angles() = vec; + } + }; + } +} + +#endif // EIGEN_EULERANGLESCLASS_H
diff --git a/unsupported/Eigen/src/EulerAngles/EulerSystem.h b/unsupported/Eigen/src/EulerAngles/EulerSystem.h new file mode 100644 index 0000000..2a833b0 --- /dev/null +++ b/unsupported/Eigen/src/EulerAngles/EulerSystem.h
@@ -0,0 +1,305 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Tal Hadad <tal_hd@hotmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_EULERSYSTEM_H +#define EIGEN_EULERSYSTEM_H + +namespace Eigen +{ + // Forward declarations + template <typename _Scalar, class _System> + class EulerAngles; + + namespace internal + { + // TODO: Add this trait to the Eigen internal API? + template <int Num, bool IsPositive = (Num > 0)> + struct Abs + { + enum { value = Num }; + }; + + template <int Num> + struct Abs<Num, false> + { + enum { value = -Num }; + }; + + template <int Axis> + struct IsValidAxis + { + enum { value = Axis != 0 && Abs<Axis>::value <= 3 }; + }; + + template<typename System, + typename Other, + int OtherRows=Other::RowsAtCompileTime, + int OtherCols=Other::ColsAtCompileTime> + struct eulerangles_assign_impl; + } + + #define EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT(COND,MSG) typedef char static_assertion_##MSG[(COND)?1:-1] + + /** \brief Representation of a fixed signed rotation axis for EulerSystem. + * + * \ingroup EulerAngles_Module + * + * Values here represent: + * - The axis of the rotation: X, Y or Z. + * - The sign (i.e. direction of the rotation along the axis): positive(+) or negative(-) + * + * Therefore, this could express all the axes {+X,+Y,+Z,-X,-Y,-Z} + * + * For positive axis, use +EULER_{axis}, and for negative axis use -EULER_{axis}. + */ + enum EulerAxis + { + EULER_X = 1, /*!< the X axis */ + EULER_Y = 2, /*!< the Y axis */ + EULER_Z = 3 /*!< the Z axis */ + }; + + /** \class EulerSystem + * + * \ingroup EulerAngles_Module + * + * \brief Represents a fixed Euler rotation system. + * + * This meta-class goal is to represent the Euler system in compilation time, for EulerAngles. + * + * You can use this class to get two things: + * - Build an Euler system, and then pass it as a template parameter to EulerAngles. + * - Query some compile time data about an Euler system. (e.g. Whether it's Tait-Bryan) + * + * Euler rotation is a set of three rotation on fixed axes. (see \ref EulerAngles) + * This meta-class store constantly those signed axes. (see \ref EulerAxis) + * + * ### Types of Euler systems ### + * + * All and only valid 3 dimension Euler rotation over standard + * signed axes{+X,+Y,+Z,-X,-Y,-Z} are supported: + * - all axes X, Y, Z in each valid order (see below what order is valid) + * - rotation over the axis is supported both over the positive and negative directions. + * - both Tait-Bryan and proper/classic Euler angles (i.e. the opposite). + * + * Since EulerSystem support both positive and negative directions, + * you may call this rotation distinction in other names: + * - _right handed_ or _left handed_ + * - _counterclockwise_ or _clockwise_ + * + * Notice all axed combination are valid, and would trigger a static assertion. + * Same unsigned axes can't be neighbors, e.g. {X,X,Y} is invalid. + * This yield two and only two classes: + * - _Tait-Bryan_ - all unsigned axes are distinct, e.g. {X,Y,Z} + * - _proper/classic Euler angles_ - The first and the third unsigned axes is equal, + * and the second is different, e.g. {X,Y,X} + * + * ### Intrinsic vs extrinsic Euler systems ### + * + * Only intrinsic Euler systems are supported for simplicity. + * If you want to use extrinsic Euler systems, + * just use the equal intrinsic opposite order for axes and angles. + * I.e axes (A,B,C) becomes (C,B,A), and angles (a,b,c) becomes (c,b,a). + * + * ### Convenient user typedefs ### + * + * Convenient typedefs for EulerSystem exist (only for positive axes Euler systems), + * in a form of EulerSystem{A}{B}{C}, e.g. \ref EulerSystemXYZ. + * + * ### Additional reading ### + * + * More information about Euler angles: https://en.wikipedia.org/wiki/Euler_angles + * + * \tparam _AlphaAxis the first fixed EulerAxis + * + * \tparam _BetaAxis the second fixed EulerAxis + * + * \tparam _GammaAxis the third fixed EulerAxis + */ + template <int _AlphaAxis, int _BetaAxis, int _GammaAxis> + class EulerSystem + { + public: + // It's defined this way and not as enum, because I think + // that enum is not guerantee to support negative numbers + + /** The first rotation axis */ + static const int AlphaAxis = _AlphaAxis; + + /** The second rotation axis */ + static const int BetaAxis = _BetaAxis; + + /** The third rotation axis */ + static const int GammaAxis = _GammaAxis; + + enum + { + AlphaAxisAbs = internal::Abs<AlphaAxis>::value, /*!< the first rotation axis unsigned */ + BetaAxisAbs = internal::Abs<BetaAxis>::value, /*!< the second rotation axis unsigned */ + GammaAxisAbs = internal::Abs<GammaAxis>::value, /*!< the third rotation axis unsigned */ + + IsAlphaOpposite = (AlphaAxis < 0) ? 1 : 0, /*!< whether alpha axis is negative */ + IsBetaOpposite = (BetaAxis < 0) ? 1 : 0, /*!< whether beta axis is negative */ + IsGammaOpposite = (GammaAxis < 0) ? 1 : 0, /*!< whether gamma axis is negative */ + + // Parity is even if alpha axis X is followed by beta axis Y, or Y is followed + // by Z, or Z is followed by X; otherwise it is odd. + IsOdd = ((AlphaAxisAbs)%3 == (BetaAxisAbs - 1)%3) ? 0 : 1, /*!< whether the Euler system is odd */ + IsEven = IsOdd ? 0 : 1, /*!< whether the Euler system is even */ + + IsTaitBryan = ((unsigned)AlphaAxisAbs != (unsigned)GammaAxisAbs) ? 1 : 0 /*!< whether the Euler system is Tait-Bryan */ + }; + + private: + + EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT(internal::IsValidAxis<AlphaAxis>::value, + ALPHA_AXIS_IS_INVALID); + + EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT(internal::IsValidAxis<BetaAxis>::value, + BETA_AXIS_IS_INVALID); + + EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT(internal::IsValidAxis<GammaAxis>::value, + GAMMA_AXIS_IS_INVALID); + + EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT((unsigned)AlphaAxisAbs != (unsigned)BetaAxisAbs, + ALPHA_AXIS_CANT_BE_EQUAL_TO_BETA_AXIS); + + EIGEN_EULER_ANGLES_CLASS_STATIC_ASSERT((unsigned)BetaAxisAbs != (unsigned)GammaAxisAbs, + BETA_AXIS_CANT_BE_EQUAL_TO_GAMMA_AXIS); + + static const int + // I, J, K are the pivot indexes permutation for the rotation matrix, that match this Euler system. + // They are used in this class converters. + // They are always different from each other, and their possible values are: 0, 1, or 2. + I_ = AlphaAxisAbs - 1, + J_ = (AlphaAxisAbs - 1 + 1 + IsOdd)%3, + K_ = (AlphaAxisAbs - 1 + 2 - IsOdd)%3 + ; + + // TODO: Get @mat parameter in form that avoids double evaluation. + template <typename Derived> + static void CalcEulerAngles_imp(Matrix<typename MatrixBase<Derived>::Scalar, 3, 1>& res, const MatrixBase<Derived>& mat, internal::true_type /*isTaitBryan*/) + { + using std::atan2; + using std::sqrt; + + typedef typename Derived::Scalar Scalar; + + const Scalar plusMinus = IsEven? 1 : -1; + const Scalar minusPlus = IsOdd? 1 : -1; + + const Scalar Rsum = sqrt((mat(I_,I_) * mat(I_,I_) + mat(I_,J_) * mat(I_,J_) + mat(J_,K_) * mat(J_,K_) + mat(K_,K_) * mat(K_,K_))/2); + res[1] = atan2(plusMinus * mat(I_,K_), Rsum); + + // There is a singularity when cos(beta) == 0 + if(Rsum > 4 * NumTraits<Scalar>::epsilon()) {// cos(beta) != 0 + res[0] = atan2(minusPlus * mat(J_, K_), mat(K_, K_)); + res[2] = atan2(minusPlus * mat(I_, J_), mat(I_, I_)); + } + else if(plusMinus * mat(I_, K_) > 0) {// cos(beta) == 0 and sin(beta) == 1 + Scalar spos = mat(J_, I_) + plusMinus * mat(K_, J_); // 2*sin(alpha + plusMinus * gamma + Scalar cpos = mat(J_, J_) + minusPlus * mat(K_, I_); // 2*cos(alpha + plusMinus * gamma) + Scalar alphaPlusMinusGamma = atan2(spos, cpos); + res[0] = alphaPlusMinusGamma; + res[2] = 0; + } + else {// cos(beta) == 0 and sin(beta) == -1 + Scalar sneg = plusMinus * (mat(K_, J_) + minusPlus * mat(J_, I_)); // 2*sin(alpha + minusPlus*gamma) + Scalar cneg = mat(J_, J_) + plusMinus * mat(K_, I_); // 2*cos(alpha + minusPlus*gamma) + Scalar alphaMinusPlusBeta = atan2(sneg, cneg); + res[0] = alphaMinusPlusBeta; + res[2] = 0; + } + } + + template <typename Derived> + static void CalcEulerAngles_imp(Matrix<typename MatrixBase<Derived>::Scalar,3,1>& res, + const MatrixBase<Derived>& mat, internal::false_type /*isTaitBryan*/) + { + using std::atan2; + using std::sqrt; + + typedef typename Derived::Scalar Scalar; + + const Scalar plusMinus = IsEven? 1 : -1; + const Scalar minusPlus = IsOdd? 1 : -1; + + const Scalar Rsum = sqrt((mat(I_, J_) * mat(I_, J_) + mat(I_, K_) * mat(I_, K_) + mat(J_, I_) * mat(J_, I_) + mat(K_, I_) * mat(K_, I_)) / 2); + + res[1] = atan2(Rsum, mat(I_, I_)); + + // There is a singularity when sin(beta) == 0 + if(Rsum > 4 * NumTraits<Scalar>::epsilon()) {// sin(beta) != 0 + res[0] = atan2(mat(J_, I_), minusPlus * mat(K_, I_)); + res[2] = atan2(mat(I_, J_), plusMinus * mat(I_, K_)); + } + else if(mat(I_, I_) > 0) {// sin(beta) == 0 and cos(beta) == 1 + Scalar spos = plusMinus * mat(K_, J_) + minusPlus * mat(J_, K_); // 2*sin(alpha + gamma) + Scalar cpos = mat(J_, J_) + mat(K_, K_); // 2*cos(alpha + gamma) + res[0] = atan2(spos, cpos); + res[2] = 0; + } + else {// sin(beta) == 0 and cos(beta) == -1 + Scalar sneg = plusMinus * mat(K_, J_) + plusMinus * mat(J_, K_); // 2*sin(alpha - gamma) + Scalar cneg = mat(J_, J_) - mat(K_, K_); // 2*cos(alpha - gamma) + res[0] = atan2(sneg, cneg); + res[2] = 0; + } + } + + template<typename Scalar> + static void CalcEulerAngles( + EulerAngles<Scalar, EulerSystem>& res, + const typename EulerAngles<Scalar, EulerSystem>::Matrix3& mat) + { + CalcEulerAngles_imp( + res.angles(), mat, + typename internal::conditional<IsTaitBryan, internal::true_type, internal::false_type>::type()); + + if (IsAlphaOpposite) + res.alpha() = -res.alpha(); + + if (IsBetaOpposite) + res.beta() = -res.beta(); + + if (IsGammaOpposite) + res.gamma() = -res.gamma(); + } + + template <typename _Scalar, class _System> + friend class Eigen::EulerAngles; + + template<typename System, + typename Other, + int OtherRows, + int OtherCols> + friend struct internal::eulerangles_assign_impl; + }; + +#define EIGEN_EULER_SYSTEM_TYPEDEF(A, B, C) \ + /** \ingroup EulerAngles_Module */ \ + typedef EulerSystem<EULER_##A, EULER_##B, EULER_##C> EulerSystem##A##B##C; + + EIGEN_EULER_SYSTEM_TYPEDEF(X,Y,Z) + EIGEN_EULER_SYSTEM_TYPEDEF(X,Y,X) + EIGEN_EULER_SYSTEM_TYPEDEF(X,Z,Y) + EIGEN_EULER_SYSTEM_TYPEDEF(X,Z,X) + + EIGEN_EULER_SYSTEM_TYPEDEF(Y,Z,X) + EIGEN_EULER_SYSTEM_TYPEDEF(Y,Z,Y) + EIGEN_EULER_SYSTEM_TYPEDEF(Y,X,Z) + EIGEN_EULER_SYSTEM_TYPEDEF(Y,X,Y) + + EIGEN_EULER_SYSTEM_TYPEDEF(Z,X,Y) + EIGEN_EULER_SYSTEM_TYPEDEF(Z,X,Z) + EIGEN_EULER_SYSTEM_TYPEDEF(Z,Y,X) + EIGEN_EULER_SYSTEM_TYPEDEF(Z,Y,Z) +} + +#endif // EIGEN_EULERSYSTEM_H
diff --git a/unsupported/Eigen/src/FFT/CMakeLists.txt b/unsupported/Eigen/src/FFT/CMakeLists.txt deleted file mode 100644 index edcffcb..0000000 --- a/unsupported/Eigen/src/FFT/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_FFT_SRCS "*.h") - -INSTALL(FILES - ${Eigen_FFT_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/FFT COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/FFT/ei_kissfft_impl.h b/unsupported/Eigen/src/FFT/ei_kissfft_impl.h index be51b4e..079e886 100644 --- a/unsupported/Eigen/src/FFT/ei_kissfft_impl.h +++ b/unsupported/Eigen/src/FFT/ei_kissfft_impl.h
@@ -316,8 +316,8 @@ // use optimized mode for even real fwd( dst, reinterpret_cast<const Complex*> (src), ncfft); - Complex dc = dst[0].real() + dst[0].imag(); - Complex nyquist = dst[0].real() - dst[0].imag(); + Complex dc(dst[0].real() + dst[0].imag()); + Complex nyquist(dst[0].real() - dst[0].imag()); int k; for ( k=1;k <= ncfft2 ; ++k ) { Complex fpk = dst[k];
diff --git a/unsupported/Eigen/src/IterativeSolvers/CMakeLists.txt b/unsupported/Eigen/src/IterativeSolvers/CMakeLists.txt deleted file mode 100644 index 7986afc..0000000 --- a/unsupported/Eigen/src/IterativeSolvers/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_IterativeSolvers_SRCS "*.h") - -INSTALL(FILES - ${Eigen_IterativeSolvers_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/IterativeSolvers COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/IterativeSolvers/ConstrainedConjGrad.h b/unsupported/Eigen/src/IterativeSolvers/ConstrainedConjGrad.h index dc0093e..5f7cdf2 100644 --- a/unsupported/Eigen/src/IterativeSolvers/ConstrainedConjGrad.h +++ b/unsupported/Eigen/src/IterativeSolvers/ConstrainedConjGrad.h
@@ -31,7 +31,7 @@ #ifndef EIGEN_CONSTRAINEDCG_H #define EIGEN_CONSTRAINEDCG_H -#include <Eigen/Core> +#include "../../../../Eigen/Core" namespace Eigen { @@ -99,7 +99,7 @@ /** \ingroup IterativeSolvers_Module * Constrained conjugate gradient * - * Computes the minimum of \f$ 1/2((Ax).x) - bx \f$ under the contraint \f$ Cx \le f \f$ + * Computes the minimum of \f$ 1/2((Ax).x) - bx \f$ under the constraint \f$ Cx \le f \f$ */ template<typename TMatrix, typename CMatrix, typename VectorX, typename VectorB, typename VectorF>
diff --git a/unsupported/Eigen/src/IterativeSolvers/DGMRES.h b/unsupported/Eigen/src/IterativeSolvers/DGMRES.h index bae04fc..2ab56b5 100644 --- a/unsupported/Eigen/src/IterativeSolvers/DGMRES.h +++ b/unsupported/Eigen/src/IterativeSolvers/DGMRES.h
@@ -10,7 +10,7 @@ #ifndef EIGEN_DGMRES_H #define EIGEN_DGMRES_H -#include <Eigen/Eigenvalues> +#include "../../../../Eigen/Eigenvalues" namespace Eigen { @@ -39,7 +39,6 @@ void sortWithPermutation (VectorType& vec, IndexType& perm, typename IndexType::Scalar& ncut) { eigen_assert(vec.size() == perm.size()); - typedef typename IndexType::Scalar Index; bool flag; for (Index k = 0; k < ncut; k++) { @@ -89,7 +88,7 @@ * [1] D. NUENTSA WAKAM and F. PACULL, Memory Efficient Hybrid * Algebraic Solvers for Linear Systems Arising from Compressible * Flows, Computers and Fluids, In Press, - * http://dx.doi.org/10.1016/j.compfluid.2012.03.023 + * https://doi.org/10.1016/j.compfluid.2012.03.023 * [2] K. Burrage and J. Erhel, On the performance of various * adaptive preconditioned GMRES strategies, 5(1998), 101-121. * [3] J. Erhel, K. Burrage and B. Pohl, Restarted GMRES @@ -110,9 +109,9 @@ using Base::m_tolerance; public: using Base::_solve_impl; + using Base::_solve_with_guess_impl; typedef _MatrixType MatrixType; typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::Index Index; typedef typename MatrixType::StorageIndex StorageIndex; typedef typename MatrixType::RealScalar RealScalar; typedef _Preconditioner Preconditioner; @@ -143,44 +142,30 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const - { - bool failed = false; - for(int j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - dgmres(matrix(), b.col(j), xj, Base::m_preconditioner); - } - m_info = failed ? NumericalIssue - : m_error <= Base::m_tolerance ? Success - : NoConvergence; - m_isInitialized = true; + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const + { + EIGEN_STATIC_ASSERT(Rhs::ColsAtCompileTime==1 || Dest::ColsAtCompileTime==1, YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX); + + m_iterations = Base::maxIterations(); + m_error = Base::m_tolerance; + + dgmres(matrix(), b, x, Base::m_preconditioner); } - /** \internal */ - template<typename Rhs,typename Dest> - void _solve_impl(const Rhs& b, MatrixBase<Dest>& x) const - { - x = b; - _solve_with_guess_impl(b,x.derived()); - } /** * Get the restart value */ - int restart() { return m_restart; } + Index restart() { return m_restart; } /** * Set the restart value (default is 30) */ - void set_restart(const int restart) { m_restart=restart; } + void set_restart(const Index restart) { m_restart=restart; } /** * Set the number of eigenvalues to deflate at each restart */ - void setEigenv(const int neig) + void setEigenv(const Index neig) { m_neig = neig; if (neig+1 > m_maxNeig) m_maxNeig = neig+1; // To allow for complex conjugates @@ -189,12 +174,12 @@ /** * Get the size of the deflation subspace size */ - int deflSize() {return m_r; } + Index deflSize() {return m_r; } /** * Set the maximum size of the deflation subspace */ - void setMaxEigenv(const int maxNeig) { m_maxNeig = maxNeig; } + void setMaxEigenv(const Index maxNeig) { m_maxNeig = maxNeig; } protected: // DGMRES algorithm @@ -202,27 +187,27 @@ void dgmres(const MatrixType& mat,const Rhs& rhs, Dest& x, const Preconditioner& precond) const; // Perform one cycle of GMRES template<typename Dest> - int dgmresCycle(const MatrixType& mat, const Preconditioner& precond, Dest& x, DenseVector& r0, RealScalar& beta, const RealScalar& normRhs, int& nbIts) const; + Index dgmresCycle(const MatrixType& mat, const Preconditioner& precond, Dest& x, DenseVector& r0, RealScalar& beta, const RealScalar& normRhs, Index& nbIts) const; // Compute data to use for deflation - int dgmresComputeDeflationData(const MatrixType& mat, const Preconditioner& precond, const Index& it, StorageIndex& neig) const; + Index dgmresComputeDeflationData(const MatrixType& mat, const Preconditioner& precond, const Index& it, StorageIndex& neig) const; // Apply deflation to a vector template<typename RhsType, typename DestType> - int dgmresApplyDeflation(const RhsType& In, DestType& Out) const; + Index dgmresApplyDeflation(const RhsType& In, DestType& Out) const; ComplexVector schurValues(const ComplexSchur<DenseMatrix>& schurofH) const; ComplexVector schurValues(const RealSchur<DenseMatrix>& schurofH) const; // Init data for deflation void dgmresInitDeflation(Index& rows) const; mutable DenseMatrix m_V; // Krylov basis vectors mutable DenseMatrix m_H; // Hessenberg matrix - mutable DenseMatrix m_Hes; // Initial hessenberg matrix wihout Givens rotations applied + mutable DenseMatrix m_Hes; // Initial hessenberg matrix without Givens rotations applied mutable Index m_restart; // Maximum size of the Krylov subspace mutable DenseMatrix m_U; // Vectors that form the basis of the invariant subspace mutable DenseMatrix m_MU; // matrix operator applied to m_U (for next cycles) mutable DenseMatrix m_T; /* T=U^T*M^{-1}*A*U */ mutable PartialPivLU<DenseMatrix> m_luT; // LU factorization of m_T mutable StorageIndex m_neig; //Number of eigenvalues to extract at each restart - mutable int m_r; // Current number of deflated eigenvalues, size of m_U - mutable int m_maxNeig; // Maximum number of eigenvalues to deflate + mutable Index m_r; // Current number of deflated eigenvalues, size of m_U + mutable Index m_maxNeig; // Maximum number of eigenvalues to deflate mutable RealScalar m_lambdaN; //Modulus of the largest eigenvalue of A mutable bool m_isDeflAllocated; mutable bool m_isDeflInitialized; @@ -243,18 +228,30 @@ void DGMRES<_MatrixType, _Preconditioner>::dgmres(const MatrixType& mat,const Rhs& rhs, Dest& x, const Preconditioner& precond) const { + const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); + + RealScalar normRhs = rhs.norm(); + if(normRhs <= considerAsZero) + { + x.setZero(); + m_error = 0; + return; + } + //Initialization - int n = mat.rows(); + m_isDeflInitialized = false; + Index n = mat.rows(); DenseVector r0(n); - int nbIts = 0; + Index nbIts = 0; m_H.resize(m_restart+1, m_restart); m_Hes.resize(m_restart, m_restart); m_V.resize(n,m_restart+1); - //Initial residual vector and intial norm - x = precond.solve(x); + //Initial residual vector and initial norm + if(x.squaredNorm()==0) + x = precond.solve(rhs); r0 = rhs - mat * x; RealScalar beta = r0.norm(); - RealScalar normRhs = rhs.norm(); + m_error = beta/normRhs; if(m_error < m_tolerance) m_info = Success; @@ -267,8 +264,10 @@ dgmresCycle(mat, precond, x, r0, beta, normRhs, nbIts); // Compute the new residual vector for the restart - if (nbIts < m_iterations && m_info == NoConvergence) - r0 = rhs - mat * x; + if (nbIts < m_iterations && m_info == NoConvergence) { + r0 = rhs - mat * x; + beta = r0.norm(); + } } } @@ -284,7 +283,7 @@ */ template< typename _MatrixType, typename _Preconditioner> template<typename Dest> -int DGMRES<_MatrixType, _Preconditioner>::dgmresCycle(const MatrixType& mat, const Preconditioner& precond, Dest& x, DenseVector& r0, RealScalar& beta, const RealScalar& normRhs, int& nbIts) const +Index DGMRES<_MatrixType, _Preconditioner>::dgmresCycle(const MatrixType& mat, const Preconditioner& precond, Dest& x, DenseVector& r0, RealScalar& beta, const RealScalar& normRhs, Index& nbIts) const { //Initialization DenseVector g(m_restart+1); // Right hand side of the least square problem @@ -293,8 +292,8 @@ m_V.col(0) = r0/beta; m_info = NoConvergence; std::vector<JacobiRotation<Scalar> >gr(m_restart); // Givens rotations - int it = 0; // Number of inner iterations - int n = mat.rows(); + Index it = 0; // Number of inner iterations + Index n = mat.rows(); DenseVector tv1(n), tv2(n); //Temporary vectors while (m_info == NoConvergence && it < m_restart && nbIts < m_iterations) { @@ -312,7 +311,7 @@ // Orthogonalize it with the previous basis in the basis using modified Gram-Schmidt Scalar coef; - for (int i = 0; i <= it; ++i) + for (Index i = 0; i <= it; ++i) { coef = tv1.dot(m_V.col(i)); tv1 = tv1 - coef * m_V.col(i); @@ -328,7 +327,7 @@ // FIXME Check for happy breakdown // Update Hessenberg matrix with Givens rotations - for (int i = 1; i <= it; ++i) + for (Index i = 1; i <= it; ++i) { m_H.col(it).applyOnTheLeft(i-1,i,gr[i-1].adjoint()); } @@ -394,7 +393,6 @@ template< typename _MatrixType, typename _Preconditioner> inline typename DGMRES<_MatrixType, _Preconditioner>::ComplexVector DGMRES<_MatrixType, _Preconditioner>::schurValues(const RealSchur<DenseMatrix>& schurofH) const { - typedef typename MatrixType::Index Index; const DenseMatrix& T = schurofH.matrixT(); Index it = T.rows(); ComplexVector eig(it); @@ -418,7 +416,7 @@ } template< typename _MatrixType, typename _Preconditioner> -int DGMRES<_MatrixType, _Preconditioner>::dgmresComputeDeflationData(const MatrixType& mat, const Preconditioner& precond, const Index& it, StorageIndex& neig) const +Index DGMRES<_MatrixType, _Preconditioner>::dgmresComputeDeflationData(const MatrixType& mat, const Preconditioner& precond, const Index& it, StorageIndex& neig) const { // First, find the Schur form of the Hessenberg matrix H typename internal::conditional<NumTraits<Scalar>::IsComplex, ComplexSchur<DenseMatrix>, RealSchur<DenseMatrix> >::type schurofH; @@ -433,8 +431,8 @@ // Reorder the absolute values of Schur values DenseRealVector modulEig(it); - for (int j=0; j<it; ++j) modulEig(j) = std::abs(eig(j)); - perm.setLinSpaced(it,0,it-1); + for (Index j=0; j<it; ++j) modulEig(j) = std::abs(eig(j)); + perm.setLinSpaced(it,0,internal::convert_index<StorageIndex>(it-1)); internal::sortWithPermutation(modulEig, perm, neig); if (!m_lambdaN) @@ -442,7 +440,7 @@ m_lambdaN = (std::max)(modulEig.maxCoeff(), m_lambdaN); } //Count the real number of extracted eigenvalues (with complex conjugates) - int nbrEig = 0; + Index nbrEig = 0; while (nbrEig < neig) { if(eig(perm(it-nbrEig-1)).imag() == RealScalar(0)) nbrEig++; @@ -451,7 +449,7 @@ // Extract the Schur vectors corresponding to the smallest Ritz values DenseMatrix Sr(it, nbrEig); Sr.setZero(); - for (int j = 0; j < nbrEig; j++) + for (Index j = 0; j < nbrEig; j++) { Sr.col(j) = schurofH.matrixU().col(perm(it-j-1)); } @@ -462,8 +460,8 @@ if (m_r) { // Orthogonalize X against m_U using modified Gram-Schmidt - for (int j = 0; j < nbrEig; j++) - for (int k =0; k < m_r; k++) + for (Index j = 0; j < nbrEig; j++) + for (Index k =0; k < m_r; k++) X.col(j) = X.col(j) - (m_U.col(k).dot(X.col(j)))*m_U.col(k); } @@ -473,7 +471,7 @@ dgmresInitDeflation(m); DenseMatrix MX(m, nbrEig); DenseVector tv1(m); - for (int j = 0; j < nbrEig; j++) + for (Index j = 0; j < nbrEig; j++) { tv1 = mat * X.col(j); MX.col(j) = precond.solve(tv1); @@ -488,8 +486,8 @@ } // Save X into m_U and m_MX in m_MU - for (int j = 0; j < nbrEig; j++) m_U.col(m_r+j) = X.col(j); - for (int j = 0; j < nbrEig; j++) m_MU.col(m_r+j) = MX.col(j); + for (Index j = 0; j < nbrEig; j++) m_U.col(m_r+j) = X.col(j); + for (Index j = 0; j < nbrEig; j++) m_MU.col(m_r+j) = MX.col(j); // Increase the size of the invariant subspace m_r += nbrEig; @@ -502,7 +500,7 @@ } template<typename _MatrixType, typename _Preconditioner> template<typename RhsType, typename DestType> -int DGMRES<_MatrixType, _Preconditioner>::dgmresApplyDeflation(const RhsType &x, DestType &y) const +Index DGMRES<_MatrixType, _Preconditioner>::dgmresApplyDeflation(const RhsType &x, DestType &y) const { DenseVector x1 = m_U.leftCols(m_r).transpose() * x; y = x + m_U.leftCols(m_r) * ( m_lambdaN * m_luT.solve(x1) - x1);
diff --git a/unsupported/Eigen/src/IterativeSolvers/GMRES.h b/unsupported/Eigen/src/IterativeSolvers/GMRES.h index fbe21fc..ff91209 100644 --- a/unsupported/Eigen/src/IterativeSolvers/GMRES.h +++ b/unsupported/Eigen/src/IterativeSolvers/GMRES.h
@@ -21,7 +21,7 @@ * * Parameters: * \param mat matrix of linear system of equations -* \param Rhs right hand side vector of linear system of equations +* \param rhs right hand side vector of linear system of equations * \param x on input: initial guess, on output: solution * \param precond preconditioner used * \param iters on input: maximum number of iterations to perform @@ -62,7 +62,16 @@ typedef typename Dest::RealScalar RealScalar; typedef typename Dest::Scalar Scalar; typedef Matrix < Scalar, Dynamic, 1 > VectorType; - typedef Matrix < Scalar, Dynamic, Dynamic > FMatrixType; + typedef Matrix < Scalar, Dynamic, Dynamic, ColMajor> FMatrixType; + + const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); + + if(rhs.norm() <= considerAsZero) + { + x.setZero(); + tol_error = 0; + return true; + } RealScalar tol = tol_error; const Index maxIters = iters; @@ -157,7 +166,8 @@ // insert coefficients into upper matrix triangle H.col(k-1).head(k) = v.head(k); - bool stop = (k==m || abs(w(k)) < tol * r0Norm || iters == maxIters); + tol_error = abs(w(k)) / r0Norm; + bool stop = (k==m || tol_error < tol || iters == maxIters); if (stop || k == restart) { @@ -306,31 +316,14 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const { - bool failed = false; - for(Index j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - if(!internal::gmres(matrix(), b.col(j), xj, Base::m_preconditioner, m_iterations, m_restart, m_error)) - failed = true; - } - m_info = failed ? NumericalIssue + m_iterations = Base::maxIterations(); + m_error = Base::m_tolerance; + bool ret = internal::gmres(matrix(), b, x, Base::m_preconditioner, m_iterations, m_restart, m_error); + m_info = (!ret) ? NumericalIssue : m_error <= Base::m_tolerance ? Success : NoConvergence; - m_isInitialized = true; - } - - /** \internal */ - template<typename Rhs,typename Dest> - void _solve_impl(const Rhs& b, MatrixBase<Dest> &x) const - { - x = b; - if(x.squaredNorm() == 0) return; // Check Zero right hand side - _solve_with_guess_impl(b,x.derived()); } protected:
diff --git a/unsupported/Eigen/src/IterativeSolvers/MINRES.h b/unsupported/Eigen/src/IterativeSolvers/MINRES.h index 256990c..5db454d 100644 --- a/unsupported/Eigen/src/IterativeSolvers/MINRES.h +++ b/unsupported/Eigen/src/IterativeSolvers/MINRES.h
@@ -3,6 +3,7 @@ // // Copyright (C) 2012 Giacomo Po <gpo@ucla.edu> // Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2018 David Hyde <dabh@stanford.edu> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -64,8 +65,6 @@ eigen_assert(beta_new2 >= 0.0 && "PRECONDITIONER IS NOT POSITIVE DEFINITE"); RealScalar beta_new(sqrt(beta_new2)); const RealScalar beta_one(beta_new); - v_new /= beta_new; - w_new /= beta_new; // Initialize other variables RealScalar c(1.0); // the cosine of the Givens rotation RealScalar c_old(1.0); @@ -83,18 +82,18 @@ /* Note that there are 4 variants on the Lanczos algorithm. These are * described in Paige, C. C. (1972). Computational variants of * the Lanczos method for the eigenproblem. IMA Journal of Applied - * Mathematics, 10(3), 373–381. The current implementation corresponds + * Mathematics, 10(3), 373-381. The current implementation corresponds * to the case A(2,7) in the paper. It also corresponds to - * algorithm 6.14 in Y. Saad, Iterative Methods for Sparse Linear + * algorithm 6.14 in Y. Saad, Iterative Methods for Sparse Linear * Systems, 2003 p.173. For the preconditioned version see * A. Greenbaum, Iterative Methods for Solving Linear Systems, SIAM (1987). */ const RealScalar beta(beta_new); v_old = v; // update: at first time step, this makes v_old = 0 so value of beta doesn't matter -// const VectorType v_old(v); // NOT SURE IF CREATING v_old EVERY ITERATION IS EFFICIENT + v_new /= beta_new; // overwrite v_new for next iteration + w_new /= beta_new; // overwrite w_new for next iteration v = v_new; // update w = w_new; // update -// const VectorType w(w_new); // NOT SURE IF CREATING w EVERY ITERATION IS EFFICIENT v_new.noalias() = mat*w - beta*v_old; // compute v_new const RealScalar alpha = v_new.dot(w); v_new -= alpha*v; // overwrite v_new @@ -102,8 +101,6 @@ beta_new2 = v_new.dot(w_new); // compute beta_new eigen_assert(beta_new2 >= 0.0 && "PRECONDITIONER IS NOT POSITIVE DEFINITE"); beta_new = sqrt(beta_new2); // compute beta_new - v_new /= beta_new; // overwrite v_new for next iteration - w_new /= beta_new; // overwrite w_new for next iteration // Givens rotation const RealScalar r2 =s*alpha+c*c_old*beta; // s, s_old, c and c_old are still from previous iteration @@ -117,7 +114,6 @@ // Update solution p_oold = p_old; -// const VectorType p_oold(p_old); // NOT SURE IF CREATING p_oold EVERY ITERATION IS EFFICIENT p_old = p; p.noalias()=(w-r2*p_old-r3*p_oold) /r1; // IS NOALIAS REQUIRED? x += beta_one*c*eta*p; @@ -237,7 +233,7 @@ /** \internal */ template<typename Rhs,typename Dest> - void _solve_with_guess_impl(const Rhs& b, Dest& x) const + void _solve_vector_with_guess_impl(const Rhs& b, Dest& x) const { typedef typename Base::MatrixWrapper MatrixWrapper; typedef typename Base::ActualMatrixType ActualMatrixType; @@ -257,28 +253,11 @@ m_iterations = Base::maxIterations(); m_error = Base::m_tolerance; RowMajorWrapper row_mat(matrix()); - for(int j=0; j<b.cols(); ++j) - { - m_iterations = Base::maxIterations(); - m_error = Base::m_tolerance; - - typename Dest::ColXpr xj(x,j); - internal::minres(SelfAdjointWrapper(row_mat), b.col(j), xj, - Base::m_preconditioner, m_iterations, m_error); - } - - m_isInitialized = true; + internal::minres(SelfAdjointWrapper(row_mat), b, x, + Base::m_preconditioner, m_iterations, m_error); m_info = m_error <= Base::m_tolerance ? Success : NoConvergence; } - /** \internal */ - template<typename Rhs,typename Dest> - void _solve_impl(const Rhs& b, MatrixBase<Dest> &x) const - { - x.setZero(); - _solve_with_guess_impl(b,x.derived()); - } - protected: }; @@ -286,4 +265,3 @@ } // end namespace Eigen #endif // EIGEN_MINRES_H -
diff --git a/unsupported/Eigen/src/IterativeSolvers/Scaling.h b/unsupported/Eigen/src/IterativeSolvers/Scaling.h index d113e6e..9b3eb53 100644 --- a/unsupported/Eigen/src/IterativeSolvers/Scaling.h +++ b/unsupported/Eigen/src/IterativeSolvers/Scaling.h
@@ -104,12 +104,18 @@ for (int i = 0; i < m; ++i) { Dr(i) = std::sqrt(Dr(i)); + } + for (int i = 0; i < n; ++i) + { Dc(i) = std::sqrt(Dc(i)); } // Save the scaling factors for (int i = 0; i < m; ++i) { m_left(i) /= Dr(i); + } + for (int i = 0; i < n; ++i) + { m_right(i) /= Dc(i); } // Scale the rows and the columns of the matrix
diff --git a/unsupported/Eigen/src/KroneckerProduct/CMakeLists.txt b/unsupported/Eigen/src/KroneckerProduct/CMakeLists.txt deleted file mode 100644 index 4daefeb..0000000 --- a/unsupported/Eigen/src/KroneckerProduct/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_KroneckerProduct_SRCS "*.h") - -INSTALL(FILES - ${Eigen_KroneckerProduct_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/KroneckerProduct COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/KroneckerProduct/KroneckerTensorProduct.h b/unsupported/Eigen/src/KroneckerProduct/KroneckerTensorProduct.h index 4d3e535..582fa85 100644 --- a/unsupported/Eigen/src/KroneckerProduct/KroneckerTensorProduct.h +++ b/unsupported/Eigen/src/KroneckerProduct/KroneckerTensorProduct.h
@@ -203,7 +203,7 @@ { typedef typename remove_all<_Lhs>::type Lhs; typedef typename remove_all<_Rhs>::type Rhs; - typedef typename scalar_product_traits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; + typedef typename ScalarBinaryOpTraits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; typedef typename promote_index_type<typename Lhs::StorageIndex, typename Rhs::StorageIndex>::type StorageIndex; enum { @@ -222,7 +222,7 @@ typedef MatrixXpr XprKind; typedef typename remove_all<_Lhs>::type Lhs; typedef typename remove_all<_Rhs>::type Rhs; - typedef typename scalar_product_traits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; + typedef typename ScalarBinaryOpTraits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; typedef typename cwise_promote_storage_type<typename traits<Lhs>::StorageKind, typename traits<Rhs>::StorageKind, scalar_product_op<typename Lhs::Scalar, typename Rhs::Scalar> >::ret StorageKind; typedef typename promote_index_type<typename Lhs::StorageIndex, typename Rhs::StorageIndex>::type StorageIndex; @@ -239,7 +239,7 @@ RemovedBits = ~(EvalToRowMajor ? 0 : RowMajorBit), Flags = ((LhsFlags | RhsFlags) & HereditaryBits & RemovedBits) - | EvalBeforeNestingBit | EvalBeforeAssigningBit, + | EvalBeforeNestingBit, CoeffReadCost = HugeCost };
diff --git a/unsupported/Eigen/src/LevenbergMarquardt/CMakeLists.txt b/unsupported/Eigen/src/LevenbergMarquardt/CMakeLists.txt deleted file mode 100644 index d969085..0000000 --- a/unsupported/Eigen/src/LevenbergMarquardt/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_LevenbergMarquardt_SRCS "*.h") - -INSTALL(FILES - ${Eigen_LevenbergMarquardt_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/LevenbergMarquardt COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/LevenbergMarquardt/LMqrsolv.h b/unsupported/Eigen/src/LevenbergMarquardt/LMqrsolv.h index ae9d793..1234858 100644 --- a/unsupported/Eigen/src/LevenbergMarquardt/LMqrsolv.h +++ b/unsupported/Eigen/src/LevenbergMarquardt/LMqrsolv.h
@@ -73,7 +73,7 @@ qtbpj = -givens.s() * wa[k] + givens.c() * qtbpj; wa[k] = temp; - /* accumulate the tranformation in the row of s. */ + /* accumulate the transformation in the row of s. */ for (i = k+1; i<n; ++i) { temp = givens.c() * s(i,k) + givens.s() * sdiag[i]; sdiag[i] = -givens.s() * s(i,k) + givens.c() * sdiag[i];
diff --git a/unsupported/Eigen/src/LevenbergMarquardt/LevenbergMarquardt.h b/unsupported/Eigen/src/LevenbergMarquardt/LevenbergMarquardt.h index b30e0a9..62561da 100644 --- a/unsupported/Eigen/src/LevenbergMarquardt/LevenbergMarquardt.h +++ b/unsupported/Eigen/src/LevenbergMarquardt/LevenbergMarquardt.h
@@ -117,7 +117,7 @@ typedef typename JacobianType::RealScalar RealScalar; typedef typename QRSolver::StorageIndex PermIndex; typedef Matrix<Scalar,Dynamic,1> FVectorType; - typedef PermutationMatrix<Dynamic,Dynamic> PermutationType; + typedef PermutationMatrix<Dynamic,Dynamic,int> PermutationType; public: LevenbergMarquardt(FunctorType& functor) : m_functor(functor),m_nfev(0),m_njev(0),m_fnorm(0.0),m_gnorm(0), @@ -233,9 +233,9 @@ /** * \brief Reports whether the minimization was successful - * \returns \c Success if the minimization was succesful, + * \returns \c Success if the minimization was successful, * \c NumericalIssue if a numerical problem arises during the - * minimization process, for exemple during the QR factorization + * minimization process, for example during the QR factorization * \c NoConvergence if the minimization did not converge after * the maximum number of function evaluation allowed * \c InvalidInput if the input matrix is invalid @@ -304,7 +304,7 @@ // m_fjac.reserve(VectorXi::Constant(n,5)); // FIXME Find a better alternative if (!m_useExternalScaling) m_diag.resize(n); - eigen_assert( (!m_useExternalScaling || m_diag.size()==n) || "When m_useExternalScaling is set, the caller must provide a valid 'm_diag'"); + eigen_assert( (!m_useExternalScaling || m_diag.size()==n) && "When m_useExternalScaling is set, the caller must provide a valid 'm_diag'"); m_qtf.resize(n); /* Function Body */
diff --git a/unsupported/Eigen/src/MatrixFunctions/CMakeLists.txt b/unsupported/Eigen/src/MatrixFunctions/CMakeLists.txt deleted file mode 100644 index cdde64d..0000000 --- a/unsupported/Eigen/src/MatrixFunctions/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_MatrixFunctions_SRCS "*.h") - -INSTALL(FILES - ${Eigen_MatrixFunctions_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/MatrixFunctions COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/MatrixFunctions/MatrixExponential.h b/unsupported/Eigen/src/MatrixFunctions/MatrixExponential.h index bbb7e57..77cbedc 100644 --- a/unsupported/Eigen/src/MatrixFunctions/MatrixExponential.h +++ b/unsupported/Eigen/src/MatrixFunctions/MatrixExponential.h
@@ -61,11 +61,12 @@ * After exit, \f$ (V+U)(V-U)^{-1} \f$ is the Padé * approximant of \f$ \exp(A) \f$ around \f$ A = 0 \f$. */ -template <typename MatrixType> -void matrix_exp_pade3(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade3(const MatA& A, MatU& U, MatV& V) { - typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; - const RealScalar b[] = {120., 60., 12., 1.}; + typedef typename MatA::PlainObject MatrixType; + typedef typename NumTraits<typename traits<MatA>::Scalar>::Real RealScalar; + const RealScalar b[] = {120.L, 60.L, 12.L, 1.L}; const MatrixType A2 = A * A; const MatrixType tmp = b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols()); U.noalias() = A * tmp; @@ -77,11 +78,12 @@ * After exit, \f$ (V+U)(V-U)^{-1} \f$ is the Padé * approximant of \f$ \exp(A) \f$ around \f$ A = 0 \f$. */ -template <typename MatrixType> -void matrix_exp_pade5(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade5(const MatA& A, MatU& U, MatV& V) { + typedef typename MatA::PlainObject MatrixType; typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; - const RealScalar b[] = {30240., 15120., 3360., 420., 30., 1.}; + const RealScalar b[] = {30240.L, 15120.L, 3360.L, 420.L, 30.L, 1.L}; const MatrixType A2 = A * A; const MatrixType A4 = A2 * A2; const MatrixType tmp = b[5] * A4 + b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols()); @@ -94,11 +96,12 @@ * After exit, \f$ (V+U)(V-U)^{-1} \f$ is the Padé * approximant of \f$ \exp(A) \f$ around \f$ A = 0 \f$. */ -template <typename MatrixType> -void matrix_exp_pade7(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade7(const MatA& A, MatU& U, MatV& V) { + typedef typename MatA::PlainObject MatrixType; typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; - const RealScalar b[] = {17297280., 8648640., 1995840., 277200., 25200., 1512., 56., 1.}; + const RealScalar b[] = {17297280.L, 8648640.L, 1995840.L, 277200.L, 25200.L, 1512.L, 56.L, 1.L}; const MatrixType A2 = A * A; const MatrixType A4 = A2 * A2; const MatrixType A6 = A4 * A2; @@ -114,12 +117,13 @@ * After exit, \f$ (V+U)(V-U)^{-1} \f$ is the Padé * approximant of \f$ \exp(A) \f$ around \f$ A = 0 \f$. */ -template <typename MatrixType> -void matrix_exp_pade9(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade9(const MatA& A, MatU& U, MatV& V) { + typedef typename MatA::PlainObject MatrixType; typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; - const RealScalar b[] = {17643225600., 8821612800., 2075673600., 302702400., 30270240., - 2162160., 110880., 3960., 90., 1.}; + const RealScalar b[] = {17643225600.L, 8821612800.L, 2075673600.L, 302702400.L, 30270240.L, + 2162160.L, 110880.L, 3960.L, 90.L, 1.L}; const MatrixType A2 = A * A; const MatrixType A4 = A2 * A2; const MatrixType A6 = A4 * A2; @@ -135,13 +139,14 @@ * After exit, \f$ (V+U)(V-U)^{-1} \f$ is the Padé * approximant of \f$ \exp(A) \f$ around \f$ A = 0 \f$. */ -template <typename MatrixType> -void matrix_exp_pade13(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade13(const MatA& A, MatU& U, MatV& V) { + typedef typename MatA::PlainObject MatrixType; typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; - const RealScalar b[] = {64764752532480000., 32382376266240000., 7771770303897600., - 1187353796428800., 129060195264000., 10559470521600., 670442572800., - 33522128640., 1323241920., 40840800., 960960., 16380., 182., 1.}; + const RealScalar b[] = {64764752532480000.L, 32382376266240000.L, 7771770303897600.L, + 1187353796428800.L, 129060195264000.L, 10559470521600.L, 670442572800.L, + 33522128640.L, 1323241920.L, 40840800.L, 960960.L, 16380.L, 182.L, 1.L}; const MatrixType A2 = A * A; const MatrixType A4 = A2 * A2; const MatrixType A6 = A4 * A2; @@ -162,9 +167,10 @@ * This function activates only if your long double is double-double or quadruple. */ #if LDBL_MANT_DIG > 64 -template <typename MatrixType> -void matrix_exp_pade17(const MatrixType &A, MatrixType &U, MatrixType &V) +template <typename MatA, typename MatU, typename MatV> +void matrix_exp_pade17(const MatA& A, MatU& U, MatV& V) { + typedef typename MatA::PlainObject MatrixType; typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; const RealScalar b[] = {830034394580628357120000.L, 415017197290314178560000.L, 100610229646136770560000.L, 15720348382208870400000.L, @@ -204,15 +210,16 @@ template <typename MatrixType> struct matrix_exp_computeUV<MatrixType, float> { - static void run(const MatrixType& arg, MatrixType& U, MatrixType& V, int& squarings) + template <typename ArgType> + static void run(const ArgType& arg, MatrixType& U, MatrixType& V, int& squarings) { using std::frexp; using std::pow; const float l1norm = arg.cwiseAbs().colwise().sum().maxCoeff(); squarings = 0; - if (l1norm < 4.258730016922831e-001) { + if (l1norm < 4.258730016922831e-001f) { matrix_exp_pade3(arg, U, V); - } else if (l1norm < 1.880152677804762e+000) { + } else if (l1norm < 1.880152677804762e+000f) { matrix_exp_pade5(arg, U, V); } else { const float maxnorm = 3.925724783138660f; @@ -227,11 +234,13 @@ template <typename MatrixType> struct matrix_exp_computeUV<MatrixType, double> { - static void run(const MatrixType& arg, MatrixType& U, MatrixType& V, int& squarings) + typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar; + template <typename ArgType> + static void run(const ArgType& arg, MatrixType& U, MatrixType& V, int& squarings) { using std::frexp; using std::pow; - const double l1norm = arg.cwiseAbs().colwise().sum().maxCoeff(); + const RealScalar l1norm = arg.cwiseAbs().colwise().sum().maxCoeff(); squarings = 0; if (l1norm < 1.495585217958292e-002) { matrix_exp_pade3(arg, U, V); @@ -242,10 +251,10 @@ } else if (l1norm < 2.097847961257068e+000) { matrix_exp_pade9(arg, U, V); } else { - const double maxnorm = 5.371920351148152; + const RealScalar maxnorm = 5.371920351148152; frexp(l1norm / maxnorm, &squarings); if (squarings < 0) squarings = 0; - MatrixType A = arg.unaryExpr(MatrixExponentialScalingOp<double>(squarings)); + MatrixType A = arg.unaryExpr(MatrixExponentialScalingOp<RealScalar>(squarings)); matrix_exp_pade13(A, U, V); } } @@ -254,7 +263,8 @@ template <typename MatrixType> struct matrix_exp_computeUV<MatrixType, long double> { - static void run(const MatrixType& arg, MatrixType& U, MatrixType& V, int& squarings) + template <typename ArgType> + static void run(const ArgType& arg, MatrixType& U, MatrixType& V, int& squarings) { #if LDBL_MANT_DIG == 53 // double precision matrix_exp_computeUV<MatrixType, double>::run(arg, U, V, squarings); @@ -304,7 +314,7 @@ matrix_exp_pade17(A, U, V); } -#elif LDBL_MANT_DIG <= 112 // quadruple precison +#elif LDBL_MANT_DIG <= 113 // quadruple precision if (l1norm < 1.639394610288918690547467954466970e-005L) { matrix_exp_pade3(arg, U, V); @@ -317,6 +327,7 @@ } else if (l1norm < 1.125358383453143065081397882891878e+000L) { matrix_exp_pade13(arg, U, V); } else { + const long double maxnorm = 2.884233277829519311757165057717815L; frexp(l1norm / maxnorm, &squarings); if (squarings < 0) squarings = 0; MatrixType A = arg.unaryExpr(MatrixExponentialScalingOp<long double>(squarings)); @@ -333,32 +344,41 @@ } }; +template<typename T> struct is_exp_known_type : false_type {}; +template<> struct is_exp_known_type<float> : true_type {}; +template<> struct is_exp_known_type<double> : true_type {}; +#if LDBL_MANT_DIG <= 113 +template<> struct is_exp_known_type<long double> : true_type {}; +#endif + +template <typename ArgType, typename ResultType> +void matrix_exp_compute(const ArgType& arg, ResultType &result, true_type) // natively supported scalar type +{ + typedef typename ArgType::PlainObject MatrixType; + MatrixType U, V; + int squarings; + matrix_exp_computeUV<MatrixType>::run(arg, U, V, squarings); // Pade approximant is (U+V) / (-U+V) + MatrixType numer = U + V; + MatrixType denom = -U + V; + result = denom.partialPivLu().solve(numer); + for (int i=0; i<squarings; i++) + result *= result; // undo scaling by repeated squaring +} + /* Computes the matrix exponential * * \param arg argument of matrix exponential (should be plain object) * \param result variable in which result will be stored */ -template <typename MatrixType, typename ResultType> -void matrix_exp_compute(const MatrixType& arg, ResultType &result) +template <typename ArgType, typename ResultType> +void matrix_exp_compute(const ArgType& arg, ResultType &result, false_type) // default { -#if LDBL_MANT_DIG > 112 // rarely happens + typedef typename ArgType::PlainObject MatrixType; typedef typename traits<MatrixType>::Scalar Scalar; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename std::complex<RealScalar> ComplexScalar; - if (sizeof(RealScalar) > 14) { - result = arg.matrixFunction(internal::stem_function_exp<ComplexScalar>); - return; - } -#endif - MatrixType U, V; - int squarings; - matrix_exp_computeUV<MatrixType>::run(arg, U, V, squarings); // Pade approximant is (U+V) / (-U+V) - MatrixType numer = U + V; - MatrixType denom = -U + V; - result = denom.partialPivLu().solve(numer); - for (int i=0; i<squarings; i++) - result *= result; // undo scaling by repeated squaring + result = arg.matrixFunction(internal::stem_function_exp<ComplexScalar>); } } // end namespace Eigen::internal @@ -376,7 +396,6 @@ template<typename Derived> struct MatrixExponentialReturnValue : public ReturnByValue<MatrixExponentialReturnValue<Derived> > { - typedef typename Derived::Index Index; public: /** \brief Constructor. * @@ -392,7 +411,7 @@ inline void evalTo(ResultType& result) const { const typename internal::nested_eval<Derived, 10>::type tmp(m_src); - internal::matrix_exp_compute(tmp, result); + internal::matrix_exp_compute(tmp, result, internal::is_exp_known_type<typename Derived::Scalar>()); } Index rows() const { return m_src.rows(); }
diff --git a/unsupported/Eigen/src/MatrixFunctions/MatrixFunction.h b/unsupported/Eigen/src/MatrixFunctions/MatrixFunction.h index 8f7a6f3..cc12ab6 100644 --- a/unsupported/Eigen/src/MatrixFunctions/MatrixFunction.h +++ b/unsupported/Eigen/src/MatrixFunctions/MatrixFunction.h
@@ -7,8 +7,8 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_MATRIX_FUNCTION -#define EIGEN_MATRIX_FUNCTION +#ifndef EIGEN_MATRIX_FUNCTION_H +#define EIGEN_MATRIX_FUNCTION_H #include "StemFunction.h" @@ -53,7 +53,7 @@ typename NumTraits<typename MatrixType::Scalar>::Real matrix_function_compute_mu(const MatrixType& A) { typedef typename plain_col_type<MatrixType>::type VectorType; - typename MatrixType::Index rows = A.rows(); + Index rows = A.rows(); const MatrixType N = MatrixType::Identity(rows, rows) - A; VectorType e = VectorType::Ones(rows); N.template triangularView<Upper>().solveInPlace(e); @@ -65,7 +65,6 @@ { // TODO: Use that A is upper triangular typedef typename NumTraits<Scalar>::Real RealScalar; - typedef typename MatrixType::Index Index; Index rows = A.rows(); Scalar avgEival = A.trace() / Scalar(RealScalar(rows)); MatrixType Ashifted = A - avgEival * MatrixType::Identity(rows, rows); @@ -73,10 +72,10 @@ MatrixType F = m_f(avgEival, 0) * MatrixType::Identity(rows, rows); MatrixType P = Ashifted; MatrixType Fincr; - for (Index s = 1; s < 1.1 * rows + 10; s++) { // upper limit is fairly arbitrary + for (Index s = 1; double(s) < 1.1 * double(rows) + 10.0; s++) { // upper limit is fairly arbitrary Fincr = m_f(avgEival, static_cast<int>(s)) * P; F += Fincr; - P = Scalar(RealScalar(1.0/(s + 1))) * P * Ashifted; + P = Scalar(RealScalar(1)/RealScalar(s + 1)) * P * Ashifted; // test whether Taylor series converged const RealScalar F_norm = F.cwiseAbs().rowwise().sum().maxCoeff(); @@ -131,7 +130,7 @@ template <typename EivalsType, typename Cluster> void matrix_function_partition_eigenvalues(const EivalsType& eivals, std::list<Cluster>& clusters) { - typedef typename EivalsType::Index Index; + typedef typename EivalsType::RealScalar RealScalar; for (Index i=0; i<eivals.rows(); ++i) { // Find cluster containing i-th ei'val, adding a new cluster if necessary typename std::list<Cluster>::iterator qi = matrix_function_find_cluster(i, clusters); @@ -145,7 +144,7 @@ // Look for other element to add to the set for (Index j=i+1; j<eivals.rows(); ++j) { - if (abs(eivals(j) - eivals(i)) <= matrix_function_separation + if (abs(eivals(j) - eivals(i)) <= RealScalar(matrix_function_separation) && std::find(qi->begin(), qi->end(), j) == qi->end()) { typename std::list<Cluster>::iterator qj = matrix_function_find_cluster(j, clusters); if (qj == clusters.end()) { @@ -178,7 +177,7 @@ { blockStart.resize(clusterSize.rows()); blockStart(0) = 0; - for (typename VectorType::Index i = 1; i < clusterSize.rows(); i++) { + for (Index i = 1; i < clusterSize.rows(); i++) { blockStart(i) = blockStart(i-1) + clusterSize(i-1); } } @@ -187,7 +186,6 @@ template <typename EivalsType, typename ListOfClusters, typename VectorType> void matrix_function_compute_map(const EivalsType& eivals, const ListOfClusters& clusters, VectorType& eivalToCluster) { - typedef typename EivalsType::Index Index; eivalToCluster.resize(eivals.rows()); Index clusterIndex = 0; for (typename ListOfClusters::const_iterator cluster = clusters.begin(); cluster != clusters.end(); ++cluster) { @@ -204,7 +202,6 @@ template <typename DynVectorType, typename VectorType> void matrix_function_compute_permutation(const DynVectorType& blockStart, const DynVectorType& eivalToCluster, VectorType& permutation) { - typedef typename VectorType::Index Index; DynVectorType indexNextEntry = blockStart; permutation.resize(eivalToCluster.rows()); for (Index i = 0; i < eivalToCluster.rows(); i++) { @@ -218,7 +215,6 @@ template <typename VectorType, typename MatrixType> void matrix_function_permute_schur(VectorType& permutation, MatrixType& U, MatrixType& T) { - typedef typename VectorType::Index Index; for (Index i = 0; i < permutation.rows() - 1; i++) { Index j; for (j = i; j < permutation.rows(); j++) { @@ -246,7 +242,7 @@ void matrix_function_compute_block_atomic(const MatrixType& T, AtomicType& atomic, const VectorType& blockStart, const VectorType& clusterSize, MatrixType& fT) { fT.setZero(T.rows(), T.cols()); - for (typename VectorType::Index i = 0; i < clusterSize.rows(); ++i) { + for (Index i = 0; i < clusterSize.rows(); ++i) { fT.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i)) = atomic.compute(T.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i))); } @@ -284,7 +280,6 @@ eigen_assert(C.rows() == A.rows()); eigen_assert(C.cols() == B.rows()); - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; Index m = A.rows(); @@ -329,11 +324,8 @@ { typedef internal::traits<MatrixType> Traits; typedef typename MatrixType::Scalar Scalar; - typedef typename MatrixType::Index Index; - static const int RowsAtCompileTime = Traits::RowsAtCompileTime; - static const int ColsAtCompileTime = Traits::ColsAtCompileTime; static const int Options = MatrixType::Options; - typedef Matrix<Scalar, Dynamic, Dynamic, Options, RowsAtCompileTime, ColsAtCompileTime> DynMatrixType; + typedef Matrix<Scalar, Dynamic, Dynamic, Options, Traits::RowsAtCompileTime, Traits::ColsAtCompileTime> DynMatrixType; for (Index k = 1; k < clusterSize.rows(); k++) { for (Index i = 0; i < clusterSize.rows() - k; i++) { @@ -397,17 +389,16 @@ template <typename MatrixType> struct matrix_function_compute<MatrixType, 0> { - template <typename AtomicType, typename ResultType> - static void run(const MatrixType& A, AtomicType& atomic, ResultType &result) + template <typename MatA, typename AtomicType, typename ResultType> + static void run(const MatA& A, AtomicType& atomic, ResultType &result) { typedef internal::traits<MatrixType> Traits; typedef typename Traits::Scalar Scalar; static const int Rows = Traits::RowsAtCompileTime, Cols = Traits::ColsAtCompileTime; - static const int Options = MatrixType::Options; static const int MaxRows = Traits::MaxRowsAtCompileTime, MaxCols = Traits::MaxColsAtCompileTime; typedef std::complex<Scalar> ComplexScalar; - typedef Matrix<ComplexScalar, Rows, Cols, Options, MaxRows, MaxCols> ComplexMatrix; + typedef Matrix<ComplexScalar, Rows, Cols, 0, MaxRows, MaxCols> ComplexMatrix; ComplexMatrix CA = A.template cast<ComplexScalar>(); ComplexMatrix Cresult; @@ -422,14 +413,14 @@ template <typename MatrixType> struct matrix_function_compute<MatrixType, 1> { - template <typename AtomicType, typename ResultType> - static void run(const MatrixType& A, AtomicType& atomic, ResultType &result) + template <typename MatA, typename AtomicType, typename ResultType> + static void run(const MatA& A, AtomicType& atomic, ResultType &result) { typedef internal::traits<MatrixType> Traits; - typedef typename MatrixType::Index Index; // compute Schur decomposition of A - const ComplexSchur<MatrixType> schurOfA(A); + const ComplexSchur<MatrixType> schurOfA(A); + eigen_assert(schurOfA.info()==Success); MatrixType T = schurOfA.matrixT(); MatrixType U = schurOfA.matrixU(); @@ -481,7 +472,6 @@ { public: typedef typename Derived::Scalar Scalar; - typedef typename Derived::Index Index; typedef typename internal::stem_function<Scalar>::type StemFunction; protected: @@ -506,16 +496,13 @@ typedef typename internal::nested_eval<Derived, 10>::type NestedEvalType; typedef typename internal::remove_all<NestedEvalType>::type NestedEvalTypeClean; typedef internal::traits<NestedEvalTypeClean> Traits; - static const int RowsAtCompileTime = Traits::RowsAtCompileTime; - static const int ColsAtCompileTime = Traits::ColsAtCompileTime; - static const int Options = NestedEvalTypeClean::Options; typedef std::complex<typename NumTraits<Scalar>::Real> ComplexScalar; - typedef Matrix<ComplexScalar, Dynamic, Dynamic, Options, RowsAtCompileTime, ColsAtCompileTime> DynMatrixType; + typedef Matrix<ComplexScalar, Dynamic, Dynamic, 0, Traits::RowsAtCompileTime, Traits::ColsAtCompileTime> DynMatrixType; typedef internal::MatrixFunctionAtomic<DynMatrixType> AtomicType; AtomicType atomic(m_f); - internal::matrix_function_compute<NestedEvalTypeClean>::run(m_A, atomic, result); + internal::matrix_function_compute<typename NestedEvalTypeClean::PlainObject>::run(m_A, atomic, result); } Index rows() const { return m_A.rows(); } @@ -579,4 +566,4 @@ } // end namespace Eigen -#endif // EIGEN_MATRIX_FUNCTION +#endif // EIGEN_MATRIX_FUNCTION_H
diff --git a/unsupported/Eigen/src/MatrixFunctions/MatrixLogarithm.h b/unsupported/Eigen/src/MatrixFunctions/MatrixLogarithm.h index e43e86e..e917013 100644 --- a/unsupported/Eigen/src/MatrixFunctions/MatrixLogarithm.h +++ b/unsupported/Eigen/src/MatrixFunctions/MatrixLogarithm.h
@@ -37,6 +37,7 @@ void matrix_log_compute_2x2(const MatrixType& A, MatrixType& result) { typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::RealScalar RealScalar; using std::abs; using std::ceil; using std::imag; @@ -54,15 +55,15 @@ { result(0,1) = A(0,1) / A(0,0); } - else if ((abs(A(0,0)) < 0.5*abs(A(1,1))) || (abs(A(0,0)) > 2*abs(A(1,1)))) + else if ((abs(A(0,0)) < RealScalar(0.5)*abs(A(1,1))) || (abs(A(0,0)) > 2*abs(A(1,1)))) { result(0,1) = A(0,1) * (logA11 - logA00) / y; } else { // computation in previous branch is inaccurate if A(1,1) \approx A(0,0) - int unwindingNumber = static_cast<int>(ceil((imag(logA11 - logA00) - EIGEN_PI) / (2*EIGEN_PI))); - result(0,1) = A(0,1) * (numext::log1p(y/A(0,0)) + Scalar(0,2*EIGEN_PI*unwindingNumber)) / y; + RealScalar unwindingNumber = ceil((imag(logA11 - logA00) - RealScalar(EIGEN_PI)) / RealScalar(2*EIGEN_PI)); + result(0,1) = A(0,1) * (numext::log1p(y/A(0,0)) + Scalar(0,RealScalar(2*EIGEN_PI)*unwindingNumber)) / y; } } @@ -134,7 +135,8 @@ const int minPadeDegree = 3; const int maxPadeDegree = 11; assert(degree >= minPadeDegree && degree <= maxPadeDegree); - + // FIXME this creates float-conversion-warnings if these are enabled. + // Either manually convert each value, or disable the warning locally const RealScalar nodes[][maxPadeDegree] = { { 0.1127016653792583114820734600217600L, 0.5000000000000000000000000000000000L, // degree 3 0.8872983346207416885179265399782400L }, @@ -231,12 +233,13 @@ int degree; MatrixType T = A, sqrtT; - int maxPadeDegree = matrix_log_max_pade_degree<Scalar>::value; - const RealScalar maxNormForPade = maxPadeDegree<= 5? 5.3149729967117310e-1: // single precision - maxPadeDegree<= 7? 2.6429608311114350e-1: // double precision + const int maxPadeDegree = matrix_log_max_pade_degree<Scalar>::value; + const RealScalar maxNormForPade = RealScalar( + maxPadeDegree<= 5? 5.3149729967117310e-1L: // single precision + maxPadeDegree<= 7? 2.6429608311114350e-1L: // double precision maxPadeDegree<= 8? 2.32777776523703892094e-1L: // extended precision maxPadeDegree<=10? 1.05026503471351080481093652651105e-1L: // double-double - 1.1880960220216759245467951592883642e-1L; // quadruple precision + 1.1880960220216759245467951592883642e-1L); // quadruple precision while (true) { RealScalar normTminusI = (T - MatrixType::Identity(T.rows(), T.rows())).cwiseAbs().colwise().sum().maxCoeff(); @@ -253,7 +256,7 @@ } matrix_log_compute_pade(result, T, degree); - result *= pow(RealScalar(2), numberOfSquareRoots); + result *= pow(RealScalar(2), RealScalar(numberOfSquareRoots)); // TODO replace by bitshift if possible } /** \ingroup MatrixFunctions_Module @@ -323,7 +326,7 @@ /** \brief Compute the matrix logarithm. * - * \param[out] result Logarithm of \p A, where \A is as specified in the constructor. + * \param[out] result Logarithm of \c A, where \c A is as specified in the constructor. */ template <typename ResultType> inline void evalTo(ResultType& result) const @@ -331,15 +334,12 @@ typedef typename internal::nested_eval<Derived, 10>::type DerivedEvalType; typedef typename internal::remove_all<DerivedEvalType>::type DerivedEvalTypeClean; typedef internal::traits<DerivedEvalTypeClean> Traits; - static const int RowsAtCompileTime = Traits::RowsAtCompileTime; - static const int ColsAtCompileTime = Traits::ColsAtCompileTime; - static const int Options = DerivedEvalTypeClean::Options; typedef std::complex<typename NumTraits<Scalar>::Real> ComplexScalar; - typedef Matrix<ComplexScalar, Dynamic, Dynamic, Options, RowsAtCompileTime, ColsAtCompileTime> DynMatrixType; + typedef Matrix<ComplexScalar, Dynamic, Dynamic, 0, Traits::RowsAtCompileTime, Traits::ColsAtCompileTime> DynMatrixType; typedef internal::MatrixLogarithmAtomic<DynMatrixType> AtomicType; AtomicType atomic; - internal::matrix_function_compute<DerivedEvalTypeClean>::run(m_A, atomic, result); + internal::matrix_function_compute<typename DerivedEvalTypeClean::PlainObject>::run(m_A, atomic, result); } Index rows() const { return m_A.rows(); }
diff --git a/unsupported/Eigen/src/MatrixFunctions/MatrixPower.h b/unsupported/Eigen/src/MatrixFunctions/MatrixPower.h index f37d31c..d7672d7 100644 --- a/unsupported/Eigen/src/MatrixFunctions/MatrixPower.h +++ b/unsupported/Eigen/src/MatrixFunctions/MatrixPower.h
@@ -40,7 +40,6 @@ { public: typedef typename MatrixType::RealScalar RealScalar; - typedef typename MatrixType::Index Index; /** * \brief Constructor. @@ -57,8 +56,8 @@ * \param[out] result */ template<typename ResultType> - inline void evalTo(ResultType& res) const - { m_pow.compute(res, m_p); } + inline void evalTo(ResultType& result) const + { m_pow.compute(result, m_p); } Index rows() const { return m_pow.rows(); } Index cols() const { return m_pow.cols(); } @@ -81,7 +80,7 @@ * * \note Currently this class is only used by MatrixPower. One may * insist that this be nested into MatrixPower. This class is here to - * faciliate future development of triangular matrix functions. + * facilitate future development of triangular matrix functions. */ template<typename MatrixType> class MatrixPowerAtomic : internal::noncopyable @@ -94,7 +93,6 @@ typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; typedef std::complex<RealScalar> ComplexScalar; - typedef typename MatrixType::Index Index; typedef Block<MatrixType,Dynamic,Dynamic> ResultType; const MatrixType& m_A; @@ -162,11 +160,11 @@ void MatrixPowerAtomic<MatrixType>::computePade(int degree, const MatrixType& IminusT, ResultType& res) const { int i = 2*degree; - res = (m_p-degree) / (2*i-2) * IminusT; + res = (m_p-RealScalar(degree)) / RealScalar(2*i-2) * IminusT; for (--i; i; --i) { res = (MatrixType::Identity(IminusT.rows(), IminusT.cols()) + res).template triangularView<Upper>() - .solve((i==1 ? -m_p : i&1 ? (-m_p-i/2)/(2*i) : (m_p-i/2)/(2*i-2)) * IminusT).eval(); + .solve((i==1 ? -m_p : i&1 ? (-m_p-RealScalar(i/2))/RealScalar(2*i) : (m_p-RealScalar(i/2))/RealScalar(2*i-2)) * IminusT).eval(); } res += MatrixType::Identity(IminusT.rows(), IminusT.cols()); } @@ -196,11 +194,12 @@ { using std::ldexp; const int digits = std::numeric_limits<RealScalar>::digits; - const RealScalar maxNormForPade = digits <= 24? 4.3386528e-1f: // sigle precision - digits <= 53? 2.789358995219730e-1: // double precision - digits <= 64? 2.4471944416607995472e-1L: // extended precision - digits <= 106? 1.1016843812851143391275867258512e-1L: // double-double - 9.134603732914548552537150753385375e-2L; // quadruple precision + const RealScalar maxNormForPade = RealScalar( + digits <= 24? 4.3386528e-1L // single precision + : digits <= 53? 2.789358995219730e-1L // double precision + : digits <= 64? 2.4471944416607995472e-1L // extended precision + : digits <= 106? 1.1016843812851143391275867258512e-1L // double-double + : 9.134603732914548552537150753385375e-2L); // quadruple precision MatrixType IminusT, sqrtT, T = m_A.template triangularView<Upper>(); RealScalar normIminusT; int degree, degree2, numberOfSquareRoots = 0; @@ -264,7 +263,7 @@ 1.999045567181744e-1L, 2.789358995219730e-1L }; #elif LDBL_MANT_DIG <= 64 const int maxPadeDegree = 8; - const double maxNormForPade[] = { 6.3854693117491799460e-3L /* degree = 3 */ , 2.6394893435456973676e-2L, + const long double maxNormForPade[] = { 6.3854693117491799460e-3L /* degree = 3 */ , 2.6394893435456973676e-2L, 6.4216043030404063729e-2L, 1.1701165502926694307e-1L, 1.7904284231268670284e-1L, 2.4471944416607995472e-1L }; #elif LDBL_MANT_DIG <= 106 const int maxPadeDegree = 10; @@ -298,8 +297,8 @@ ComplexScalar logCurr = log(curr); ComplexScalar logPrev = log(prev); - int unwindingNumber = ceil((numext::imag(logCurr - logPrev) - EIGEN_PI) / (2*EIGEN_PI)); - ComplexScalar w = numext::log1p((curr-prev)/prev)/RealScalar(2) + ComplexScalar(0, EIGEN_PI*unwindingNumber); + RealScalar unwindingNumber = ceil((numext::imag(logCurr - logPrev) - RealScalar(EIGEN_PI)) / RealScalar(2*EIGEN_PI)); + ComplexScalar w = numext::log1p((curr-prev)/prev)/RealScalar(2) + ComplexScalar(0, RealScalar(EIGEN_PI)*unwindingNumber); return RealScalar(2) * exp(RealScalar(0.5) * p * (logCurr + logPrev)) * sinh(p * w) / (curr - prev); } @@ -340,7 +339,6 @@ private: typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; - typedef typename MatrixType::Index Index; public: /** @@ -383,7 +381,7 @@ private: typedef std::complex<RealScalar> ComplexScalar; - typedef Matrix<ComplexScalar, Dynamic, Dynamic, MatrixType::Options, + typedef Matrix<ComplexScalar, Dynamic, Dynamic, 0, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime> ComplexMatrix; /** \brief Reference to the base of matrix power. */ @@ -600,7 +598,6 @@ public: typedef typename Derived::PlainObject PlainObject; typedef typename Derived::RealScalar RealScalar; - typedef typename Derived::Index Index; /** * \brief Constructor. @@ -618,8 +615,8 @@ * constructor. */ template<typename ResultType> - inline void evalTo(ResultType& res) const - { MatrixPower<PlainObject>(m_A.eval()).compute(res, m_p); } + inline void evalTo(ResultType& result) const + { MatrixPower<PlainObject>(m_A.eval()).compute(result, m_p); } Index rows() const { return m_A.rows(); } Index cols() const { return m_A.cols(); } @@ -648,7 +645,6 @@ public: typedef typename Derived::PlainObject PlainObject; typedef typename std::complex<typename Derived::RealScalar> ComplexScalar; - typedef typename Derived::Index Index; /** * \brief Constructor. @@ -669,8 +665,8 @@ * constructor. */ template<typename ResultType> - inline void evalTo(ResultType& res) const - { res = (m_p * m_A.log()).exp(); } + inline void evalTo(ResultType& result) const + { result = (m_p * m_A.log()).exp(); } Index rows() const { return m_A.rows(); } Index cols() const { return m_A.cols(); }
diff --git a/unsupported/Eigen/src/MatrixFunctions/MatrixSquareRoot.h b/unsupported/Eigen/src/MatrixFunctions/MatrixSquareRoot.h index 9f08c61..34bf789 100644 --- a/unsupported/Eigen/src/MatrixFunctions/MatrixSquareRoot.h +++ b/unsupported/Eigen/src/MatrixFunctions/MatrixSquareRoot.h
@@ -17,7 +17,7 @@ // pre: T.block(i,i,2,2) has complex conjugate eigenvalues // post: sqrtT.block(i,i,2,2) is square root of T.block(i,i,2,2) template <typename MatrixType, typename ResultType> -void matrix_sqrt_quasi_triangular_2x2_diagonal_block(const MatrixType& T, typename MatrixType::Index i, ResultType& sqrtT) +void matrix_sqrt_quasi_triangular_2x2_diagonal_block(const MatrixType& T, Index i, ResultType& sqrtT) { // TODO: This case (2-by-2 blocks with complex conjugate eigenvalues) is probably hidden somewhere // in EigenSolver. If we expose it, we could call it directly from here. @@ -32,7 +32,7 @@ // all blocks of sqrtT to left of and below (i,j) are correct // post: sqrtT(i,j) has the correct value template <typename MatrixType, typename ResultType> -void matrix_sqrt_quasi_triangular_1x1_off_diagonal_block(const MatrixType& T, typename MatrixType::Index i, typename MatrixType::Index j, ResultType& sqrtT) +void matrix_sqrt_quasi_triangular_1x1_off_diagonal_block(const MatrixType& T, Index i, Index j, ResultType& sqrtT) { typedef typename traits<MatrixType>::Scalar Scalar; Scalar tmp = (sqrtT.row(i).segment(i+1,j-i-1) * sqrtT.col(j).segment(i+1,j-i-1)).value(); @@ -41,7 +41,7 @@ // similar to compute1x1offDiagonalBlock() template <typename MatrixType, typename ResultType> -void matrix_sqrt_quasi_triangular_1x2_off_diagonal_block(const MatrixType& T, typename MatrixType::Index i, typename MatrixType::Index j, ResultType& sqrtT) +void matrix_sqrt_quasi_triangular_1x2_off_diagonal_block(const MatrixType& T, Index i, Index j, ResultType& sqrtT) { typedef typename traits<MatrixType>::Scalar Scalar; Matrix<Scalar,1,2> rhs = T.template block<1,2>(i,j); @@ -54,7 +54,7 @@ // similar to compute1x1offDiagonalBlock() template <typename MatrixType, typename ResultType> -void matrix_sqrt_quasi_triangular_2x1_off_diagonal_block(const MatrixType& T, typename MatrixType::Index i, typename MatrixType::Index j, ResultType& sqrtT) +void matrix_sqrt_quasi_triangular_2x1_off_diagonal_block(const MatrixType& T, Index i, Index j, ResultType& sqrtT) { typedef typename traits<MatrixType>::Scalar Scalar; Matrix<Scalar,2,1> rhs = T.template block<2,1>(i,j); @@ -65,21 +65,6 @@ sqrtT.template block<2,1>(i,j) = A.fullPivLu().solve(rhs); } -// similar to compute1x1offDiagonalBlock() -template <typename MatrixType, typename ResultType> -void matrix_sqrt_quasi_triangular_2x2_off_diagonal_block(const MatrixType& T, typename MatrixType::Index i, typename MatrixType::Index j, ResultType& sqrtT) -{ - typedef typename traits<MatrixType>::Scalar Scalar; - Matrix<Scalar,2,2> A = sqrtT.template block<2,2>(i,i); - Matrix<Scalar,2,2> B = sqrtT.template block<2,2>(j,j); - Matrix<Scalar,2,2> C = T.template block<2,2>(i,j); - if (j-i > 2) - C -= sqrtT.block(i, i+2, 2, j-i-2) * sqrtT.block(i+2, j, j-i-2, 2); - Matrix<Scalar,2,2> X; - matrix_sqrt_quasi_triangular_solve_auxiliary_equation(X, A, B, C); - sqrtT.template block<2,2>(i,j) = X; -} - // solves the equation A X + X B = C where all matrices are 2-by-2 template <typename MatrixType> void matrix_sqrt_quasi_triangular_solve_auxiliary_equation(MatrixType& X, const MatrixType& A, const MatrixType& B, const MatrixType& C) @@ -98,13 +83,13 @@ coeffMatrix.coeffRef(2,3) = B.coeff(1,0); coeffMatrix.coeffRef(3,1) = A.coeff(1,0); coeffMatrix.coeffRef(3,2) = B.coeff(0,1); - + Matrix<Scalar,4,1> rhs; rhs.coeffRef(0) = C.coeff(0,0); rhs.coeffRef(1) = C.coeff(0,1); rhs.coeffRef(2) = C.coeff(1,0); rhs.coeffRef(3) = C.coeff(1,1); - + Matrix<Scalar,4,1> result; result = coeffMatrix.fullPivLu().solve(rhs); @@ -114,6 +99,20 @@ X.coeffRef(1,1) = result.coeff(3); } +// similar to compute1x1offDiagonalBlock() +template <typename MatrixType, typename ResultType> +void matrix_sqrt_quasi_triangular_2x2_off_diagonal_block(const MatrixType& T, Index i, Index j, ResultType& sqrtT) +{ + typedef typename traits<MatrixType>::Scalar Scalar; + Matrix<Scalar,2,2> A = sqrtT.template block<2,2>(i,i); + Matrix<Scalar,2,2> B = sqrtT.template block<2,2>(j,j); + Matrix<Scalar,2,2> C = T.template block<2,2>(i,j); + if (j-i > 2) + C -= sqrtT.block(i, i+2, 2, j-i-2) * sqrtT.block(i+2, j, j-i-2, 2); + Matrix<Scalar,2,2> X; + matrix_sqrt_quasi_triangular_solve_auxiliary_equation(X, A, B, C); + sqrtT.template block<2,2>(i,j) = X; +} // pre: T is quasi-upper-triangular and sqrtT is a zero matrix of the same size // post: the diagonal blocks of sqrtT are the square roots of the diagonal blocks of T @@ -121,7 +120,6 @@ void matrix_sqrt_quasi_triangular_diagonal(const MatrixType& T, ResultType& sqrtT) { using std::sqrt; - typedef typename MatrixType::Index Index; const Index size = T.rows(); for (Index i = 0; i < size; i++) { if (i == size - 1 || T.coeff(i+1, i) == 0) { @@ -140,7 +138,6 @@ template <typename MatrixType, typename ResultType> void matrix_sqrt_quasi_triangular_off_diagonal(const MatrixType& T, ResultType& sqrtT) { - typedef typename MatrixType::Index Index; const Index size = T.rows(); for (Index j = 1; j < size; j++) { if (T.coeff(j, j-1) != 0) // if T(j-1:j, j-1:j) is a 2-by-2 block @@ -207,8 +204,7 @@ void matrix_sqrt_triangular(const MatrixType &arg, ResultType &result) { using std::sqrt; - typedef typename MatrixType::Index Index; - typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::Scalar Scalar; eigen_assert(arg.rows() == arg.cols()); @@ -319,7 +315,6 @@ : public ReturnByValue<MatrixSquareRootReturnValue<Derived> > { protected: - typedef typename Derived::Index Index; typedef typename internal::ref_selector<Derived>::type DerivedNested; public:
diff --git a/unsupported/Eigen/src/MoreVectorization/CMakeLists.txt b/unsupported/Eigen/src/MoreVectorization/CMakeLists.txt deleted file mode 100644 index 1b887cc..0000000 --- a/unsupported/Eigen/src/MoreVectorization/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_MoreVectorization_SRCS "*.h") - -INSTALL(FILES - ${Eigen_MoreVectorization_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/MoreVectorization COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/NonLinearOptimization/CMakeLists.txt b/unsupported/Eigen/src/NonLinearOptimization/CMakeLists.txt deleted file mode 100644 index 9322dda..0000000 --- a/unsupported/Eigen/src/NonLinearOptimization/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_NonLinearOptimization_SRCS "*.h") - -INSTALL(FILES - ${Eigen_NonLinearOptimization_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/NonLinearOptimization COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/NonLinearOptimization/HybridNonLinearSolver.h b/unsupported/Eigen/src/NonLinearOptimization/HybridNonLinearSolver.h index b8ba6dd..8fe3ed8 100644 --- a/unsupported/Eigen/src/NonLinearOptimization/HybridNonLinearSolver.h +++ b/unsupported/Eigen/src/NonLinearOptimization/HybridNonLinearSolver.h
@@ -150,7 +150,7 @@ fjac.resize(n, n); if (!useExternalScaling) diag.resize(n); - eigen_assert( (!useExternalScaling || diag.size()==n) || "When useExternalScaling is set, the caller must provide a valid 'diag'"); + eigen_assert( (!useExternalScaling || diag.size()==n) && "When useExternalScaling is set, the caller must provide a valid 'diag'"); /* Function Body */ nfev = 0; @@ -390,7 +390,7 @@ fvec.resize(n); if (!useExternalScaling) diag.resize(n); - eigen_assert( (!useExternalScaling || diag.size()==n) || "When useExternalScaling is set, the caller must provide a valid 'diag'"); + eigen_assert( (!useExternalScaling || diag.size()==n) && "When useExternalScaling is set, the caller must provide a valid 'diag'"); /* Function Body */ nfev = 0;
diff --git a/unsupported/Eigen/src/NonLinearOptimization/LevenbergMarquardt.h b/unsupported/Eigen/src/NonLinearOptimization/LevenbergMarquardt.h index 69106dd..fe3b79c 100644 --- a/unsupported/Eigen/src/NonLinearOptimization/LevenbergMarquardt.h +++ b/unsupported/Eigen/src/NonLinearOptimization/LevenbergMarquardt.h
@@ -179,7 +179,7 @@ fjac.resize(m, n); if (!useExternalScaling) diag.resize(n); - eigen_assert( (!useExternalScaling || diag.size()==n) || "When useExternalScaling is set, the caller must provide a valid 'diag'"); + eigen_assert( (!useExternalScaling || diag.size()==n) && "When useExternalScaling is set, the caller must provide a valid 'diag'"); qtf.resize(n); /* Function Body */ @@ -215,7 +215,7 @@ { using std::abs; using std::sqrt; - + eigen_assert(x.size()==n); // check the caller is not cheating us /* calculate the jacobian matrix. */ @@ -398,7 +398,7 @@ fjac.resize(n, n); if (!useExternalScaling) diag.resize(n); - eigen_assert( (!useExternalScaling || diag.size()==n) || "When useExternalScaling is set, the caller must provide a valid 'diag'"); + eigen_assert( (!useExternalScaling || diag.size()==n) && "When useExternalScaling is set, the caller must provide a valid 'diag'"); qtf.resize(n); /* Function Body */
diff --git a/unsupported/Eigen/src/NonLinearOptimization/qrsolv.h b/unsupported/Eigen/src/NonLinearOptimization/qrsolv.h index feafd62..4f2f560 100644 --- a/unsupported/Eigen/src/NonLinearOptimization/qrsolv.h +++ b/unsupported/Eigen/src/NonLinearOptimization/qrsolv.h
@@ -61,7 +61,7 @@ qtbpj = -givens.s() * wa[k] + givens.c() * qtbpj; wa[k] = temp; - /* accumulate the tranformation in the row of s. */ + /* accumulate the transformation in the row of s. */ for (i = k+1; i<n; ++i) { temp = givens.c() * s(i,k) + givens.s() * sdiag[i]; sdiag[i] = -givens.s() * s(i,k) + givens.c() * sdiag[i];
diff --git a/unsupported/Eigen/src/NonLinearOptimization/r1updt.h b/unsupported/Eigen/src/NonLinearOptimization/r1updt.h index f287660..09fc652 100644 --- a/unsupported/Eigen/src/NonLinearOptimization/r1updt.h +++ b/unsupported/Eigen/src/NonLinearOptimization/r1updt.h
@@ -22,7 +22,7 @@ Scalar temp; JacobiRotation<Scalar> givens; - // r1updt had a broader usecase, but we dont use it here. And, more + // r1updt had a broader usecase, but we don't use it here. And, more // importantly, we can not test it. eigen_assert(m==n); eigen_assert(u.size()==m);
diff --git a/unsupported/Eigen/src/NumericalDiff/CMakeLists.txt b/unsupported/Eigen/src/NumericalDiff/CMakeLists.txt deleted file mode 100644 index 1199aca..0000000 --- a/unsupported/Eigen/src/NumericalDiff/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_NumericalDiff_SRCS "*.h") - -INSTALL(FILES - ${Eigen_NumericalDiff_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/NumericalDiff COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/Polynomials/CMakeLists.txt b/unsupported/Eigen/src/Polynomials/CMakeLists.txt deleted file mode 100644 index 51f13f3..0000000 --- a/unsupported/Eigen/src/Polynomials/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Polynomials_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Polynomials_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/Polynomials COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/Polynomials/Companion.h b/unsupported/Eigen/src/Polynomials/Companion.h index b515c29..6ab8f97 100644 --- a/unsupported/Eigen/src/Polynomials/Companion.h +++ b/unsupported/Eigen/src/Polynomials/Companion.h
@@ -75,8 +75,7 @@ void setPolynomial( const VectorType& poly ) { const Index deg = poly.size()-1; - m_monic = -1/poly[deg] * poly.head(deg); - //m_bl_diag.setIdentity( deg-1 ); + m_monic = -poly.head(deg)/poly[deg]; m_bl_diag.setOnes(deg-1); } @@ -89,13 +88,13 @@ { const Index deg = m_monic.size(); const Index deg_1 = deg-1; - DenseCompanionMatrixType companion(deg,deg); - companion << + DenseCompanionMatrixType companMat(deg,deg); + companMat << ( LeftBlock(deg,deg_1) << LeftBlockFirstRow::Zero(1,deg_1), BottomLeftBlock::Identity(deg-1,deg-1)*m_bl_diag.asDiagonal() ).finished() , m_monic; - return companion; + return companMat; } @@ -104,20 +103,20 @@ /** Helper function for the balancing algorithm. * \returns true if the row and the column, having colNorm and rowNorm * as norms, are balanced, false otherwise. - * colB and rowB are repectively the multipliers for + * colB and rowB are respectively the multipliers for * the column and the row in order to balance them. * */ - bool balanced( Scalar colNorm, Scalar rowNorm, - bool& isBalanced, Scalar& colB, Scalar& rowB ); + bool balanced( RealScalar colNorm, RealScalar rowNorm, + bool& isBalanced, RealScalar& colB, RealScalar& rowB ); /** Helper function for the balancing algorithm. * \returns true if the row and the column, having colNorm and rowNorm * as norms, are balanced, false otherwise. - * colB and rowB are repectively the multipliers for + * colB and rowB are respectively the multipliers for * the column and the row in order to balance them. * */ - bool balancedR( Scalar colNorm, Scalar rowNorm, - bool& isBalanced, Scalar& colB, Scalar& rowB ); + bool balancedR( RealScalar colNorm, RealScalar rowNorm, + bool& isBalanced, RealScalar& colB, RealScalar& rowB ); public: /** @@ -139,10 +138,10 @@ template< typename _Scalar, int _Deg > inline -bool companion<_Scalar,_Deg>::balanced( Scalar colNorm, Scalar rowNorm, - bool& isBalanced, Scalar& colB, Scalar& rowB ) +bool companion<_Scalar,_Deg>::balanced( RealScalar colNorm, RealScalar rowNorm, + bool& isBalanced, RealScalar& colB, RealScalar& rowB ) { - if( Scalar(0) == colNorm || Scalar(0) == rowNorm ){ return true; } + if( RealScalar(0) == colNorm || RealScalar(0) == rowNorm ){ return true; } else { //To find the balancing coefficients, if the radix is 2, @@ -150,29 +149,29 @@ // \f$ 2^{2\sigma-1} < rowNorm / colNorm \le 2^{2\sigma+1} \f$ // then the balancing coefficient for the row is \f$ 1/2^{\sigma} \f$ // and the balancing coefficient for the column is \f$ 2^{\sigma} \f$ - rowB = rowNorm / radix<Scalar>(); - colB = Scalar(1); - const Scalar s = colNorm + rowNorm; + rowB = rowNorm / radix<RealScalar>(); + colB = RealScalar(1); + const RealScalar s = colNorm + rowNorm; while (colNorm < rowB) { - colB *= radix<Scalar>(); - colNorm *= radix2<Scalar>(); + colB *= radix<RealScalar>(); + colNorm *= radix2<RealScalar>(); } - rowB = rowNorm * radix<Scalar>(); + rowB = rowNorm * radix<RealScalar>(); while (colNorm >= rowB) { - colB /= radix<Scalar>(); - colNorm /= radix2<Scalar>(); + colB /= radix<RealScalar>(); + colNorm /= radix2<RealScalar>(); } //This line is used to avoid insubstantial balancing - if ((rowNorm + colNorm) < Scalar(0.95) * s * colB) + if ((rowNorm + colNorm) < RealScalar(0.95) * s * colB) { isBalanced = false; - rowB = Scalar(1) / colB; + rowB = RealScalar(1) / colB; return false; } else{ @@ -182,21 +181,21 @@ template< typename _Scalar, int _Deg > inline -bool companion<_Scalar,_Deg>::balancedR( Scalar colNorm, Scalar rowNorm, - bool& isBalanced, Scalar& colB, Scalar& rowB ) +bool companion<_Scalar,_Deg>::balancedR( RealScalar colNorm, RealScalar rowNorm, + bool& isBalanced, RealScalar& colB, RealScalar& rowB ) { - if( Scalar(0) == colNorm || Scalar(0) == rowNorm ){ return true; } + if( RealScalar(0) == colNorm || RealScalar(0) == rowNorm ){ return true; } else { /** * Set the norm of the column and the row to the geometric mean * of the row and column norm */ - const _Scalar q = colNorm/rowNorm; + const RealScalar q = colNorm/rowNorm; if( !isApprox( q, _Scalar(1) ) ) { rowB = sqrt( colNorm/rowNorm ); - colB = Scalar(1)/rowB; + colB = RealScalar(1)/rowB; isBalanced = false; return false; @@ -219,8 +218,8 @@ while( !hasConverged ) { hasConverged = true; - Scalar colNorm,rowNorm; - Scalar colB,rowB; + RealScalar colNorm,rowNorm; + RealScalar colB,rowB; //First row, first column excluding the diagonal //==============================================
diff --git a/unsupported/Eigen/src/Polynomials/PolynomialSolver.h b/unsupported/Eigen/src/Polynomials/PolynomialSolver.h index 03198ec..5e0ecbb 100644 --- a/unsupported/Eigen/src/Polynomials/PolynomialSolver.h +++ b/unsupported/Eigen/src/Polynomials/PolynomialSolver.h
@@ -99,7 +99,7 @@ */ inline const RootType& greatestRoot() const { - std::greater<Scalar> greater; + std::greater<RealScalar> greater; return selectComplexRoot_withRespectToNorm( greater ); } @@ -108,7 +108,7 @@ */ inline const RootType& smallestRoot() const { - std::less<Scalar> less; + std::less<RealScalar> less; return selectComplexRoot_withRespectToNorm( less ); } @@ -126,7 +126,7 @@ for( Index i=0; i<m_roots.size(); ++i ) { - if( abs( m_roots[i].imag() ) < absImaginaryThreshold ) + if( abs( m_roots[i].imag() ) <= absImaginaryThreshold ) { if( !hasArealRoot ) { @@ -144,10 +144,10 @@ } } } - else + else if(!hasArealRoot) { if( abs( m_roots[i].imag() ) < abs( m_roots[res].imag() ) ){ - res = i; } + res = i;} } } return numext::real_ref(m_roots[res]); @@ -167,7 +167,7 @@ for( Index i=0; i<m_roots.size(); ++i ) { - if( abs( m_roots[i].imag() ) < absImaginaryThreshold ) + if( abs( m_roots[i].imag() ) <= absImaginaryThreshold ) { if( !hasArealRoot ) { @@ -213,7 +213,7 @@ bool& hasArealRoot, const RealScalar& absImaginaryThreshold = NumTraits<Scalar>::dummy_precision() ) const { - std::greater<Scalar> greater; + std::greater<RealScalar> greater; return selectRealRoot_withRespectToAbsRealPart( greater, hasArealRoot, absImaginaryThreshold ); } @@ -236,7 +236,7 @@ bool& hasArealRoot, const RealScalar& absImaginaryThreshold = NumTraits<Scalar>::dummy_precision() ) const { - std::less<Scalar> less; + std::less<RealScalar> less; return selectRealRoot_withRespectToAbsRealPart( less, hasArealRoot, absImaginaryThreshold ); } @@ -259,7 +259,7 @@ bool& hasArealRoot, const RealScalar& absImaginaryThreshold = NumTraits<Scalar>::dummy_precision() ) const { - std::greater<Scalar> greater; + std::greater<RealScalar> greater; return selectRealRoot_withRespectToRealPart( greater, hasArealRoot, absImaginaryThreshold ); } @@ -282,7 +282,7 @@ bool& hasArealRoot, const RealScalar& absImaginaryThreshold = NumTraits<Scalar>::dummy_precision() ) const { - std::less<Scalar> less; + std::less<RealScalar> less; return selectRealRoot_withRespectToRealPart( less, hasArealRoot, absImaginaryThreshold ); } @@ -327,7 +327,7 @@ * However, almost always, correct accuracy is reached even in these cases for 64bit * (double) floating types and small polynomial degree (<20). */ -template< typename _Scalar, int _Deg > +template<typename _Scalar, int _Deg> class PolynomialSolver : public PolynomialSolverBase<_Scalar,_Deg> { public: @@ -337,7 +337,10 @@ EIGEN_POLYNOMIAL_SOLVER_BASE_INHERITED_TYPES( PS_Base ) typedef Matrix<Scalar,_Deg,_Deg> CompanionMatrixType; - typedef EigenSolver<CompanionMatrixType> EigenSolverType; + typedef typename internal::conditional<NumTraits<Scalar>::IsComplex, + ComplexEigenSolver<CompanionMatrixType>, + EigenSolver<CompanionMatrixType> >::type EigenSolverType; + typedef typename internal::conditional<NumTraits<Scalar>::IsComplex, Scalar, std::complex<Scalar> >::type ComplexScalar; public: /** Computes the complex roots of a new polynomial. */ @@ -352,6 +355,25 @@ companion.balance(); m_eigenSolver.compute( companion.denseMatrix() ); m_roots = m_eigenSolver.eigenvalues(); + // cleanup noise in imaginary part of real roots: + // if the imaginary part is rather small compared to the real part + // and that cancelling the imaginary part yield a smaller evaluation, + // then it's safe to keep the real part only. + RealScalar coarse_prec = RealScalar(std::pow(4,poly.size()+1))*NumTraits<RealScalar>::epsilon(); + for(Index i = 0; i<m_roots.size(); ++i) + { + if( internal::isMuchSmallerThan(numext::abs(numext::imag(m_roots[i])), + numext::abs(numext::real(m_roots[i])), + coarse_prec) ) + { + ComplexScalar as_real_root = ComplexScalar(numext::real(m_roots[i])); + if( numext::abs(poly_eval(poly, as_real_root)) + <= numext::abs(poly_eval(poly, m_roots[i]))) + { + m_roots[i] = as_real_root; + } + } + } } else if(poly.size () == 2) {
diff --git a/unsupported/Eigen/src/Polynomials/PolynomialUtils.h b/unsupported/Eigen/src/Polynomials/PolynomialUtils.h index 40ba65b..394e857 100644 --- a/unsupported/Eigen/src/Polynomials/PolynomialUtils.h +++ b/unsupported/Eigen/src/Polynomials/PolynomialUtils.h
@@ -20,8 +20,8 @@ * e.g. \f$ 1 + 3x^2 \f$ is stored as a vector \f$ [ 1, 0, 3 ] \f$. * \param[in] x : the value to evaluate the polynomial at. * - * <i><b>Note for stability:</b></i> - * <dd> \f$ |x| \le 1 \f$ </dd> + * \note for stability: + * \f$ |x| \le 1 \f$ */ template <typename Polynomials, typename T> inline @@ -67,8 +67,8 @@ * by degrees i.e. poly[i] is the coefficient of degree i of the polynomial * e.g. \f$ 1 + 3x^2 \f$ is stored as a vector \f$ [ 1, 0, 3 ] \f$. * - * <i><b>Precondition:</b></i> - * <dd> the leading coefficient of the input polynomial poly must be non zero </dd> + * \pre + * the leading coefficient of the input polynomial poly must be non zero */ template <typename Polynomial> inline
diff --git a/unsupported/Eigen/src/Skyline/CMakeLists.txt b/unsupported/Eigen/src/Skyline/CMakeLists.txt deleted file mode 100644 index 3bf1b0d..0000000 --- a/unsupported/Eigen/src/Skyline/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Skyline_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Skyline_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/Skyline COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/Skyline/SkylineInplaceLU.h b/unsupported/Eigen/src/Skyline/SkylineInplaceLU.h index a1f54ed..bda057a 100644 --- a/unsupported/Eigen/src/Skyline/SkylineInplaceLU.h +++ b/unsupported/Eigen/src/Skyline/SkylineInplaceLU.h
@@ -41,7 +41,7 @@ /** Sets the relative threshold value used to prune zero coefficients during the decomposition. * - * Setting a value greater than zero speeds up computation, and yields to an imcomplete + * Setting a value greater than zero speeds up computation, and yields to an incomplete * factorization with fewer non zero coefficients. Such approximate factors are especially * useful to initialize an iterative solver. *
diff --git a/unsupported/Eigen/src/Skyline/SkylineMatrix.h b/unsupported/Eigen/src/Skyline/SkylineMatrix.h index a2a8933..f77d79a 100644 --- a/unsupported/Eigen/src/Skyline/SkylineMatrix.h +++ b/unsupported/Eigen/src/Skyline/SkylineMatrix.h
@@ -206,26 +206,26 @@ if (col > row) //upper matrix { const Index minOuterIndex = inner - m_data.upperProfile(inner); - eigen_assert(outer >= minOuterIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(outer >= minOuterIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.upper(m_colStartIndex[inner] + outer - (inner - m_data.upperProfile(inner))); } if (col < row) //lower matrix { const Index minInnerIndex = outer - m_data.lowerProfile(outer); - eigen_assert(inner >= minInnerIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(inner >= minInnerIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.lower(m_rowStartIndex[outer] + inner - (outer - m_data.lowerProfile(outer))); } } else { if (outer > inner) //upper matrix { const Index maxOuterIndex = inner + m_data.upperProfile(inner); - eigen_assert(outer <= maxOuterIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(outer <= maxOuterIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.upper(m_colStartIndex[inner] + (outer - inner)); } if (outer < inner) //lower matrix { const Index maxInnerIndex = outer + m_data.lowerProfile(outer); - eigen_assert(inner <= maxInnerIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(inner <= maxInnerIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.lower(m_rowStartIndex[outer] + (inner - outer)); } } @@ -300,11 +300,11 @@ if (IsRowMajor) { const Index minInnerIndex = outer - m_data.lowerProfile(outer); - eigen_assert(inner >= minInnerIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(inner >= minInnerIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.lower(m_rowStartIndex[outer] + inner - (outer - m_data.lowerProfile(outer))); } else { const Index maxInnerIndex = outer + m_data.lowerProfile(outer); - eigen_assert(inner <= maxInnerIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(inner <= maxInnerIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.lower(m_rowStartIndex[outer] + (inner - outer)); } } @@ -336,11 +336,11 @@ if (IsRowMajor) { const Index minOuterIndex = inner - m_data.upperProfile(inner); - eigen_assert(outer >= minOuterIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(outer >= minOuterIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.upper(m_colStartIndex[inner] + outer - (inner - m_data.upperProfile(inner))); } else { const Index maxOuterIndex = inner + m_data.upperProfile(inner); - eigen_assert(outer <= maxOuterIndex && "you try to acces a coeff that do not exist in the storage"); + eigen_assert(outer <= maxOuterIndex && "You tried to access a coeff that does not exist in the storage"); return this->m_data.upper(m_colStartIndex[inner] + (outer - inner)); } }
diff --git a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h index 0e8350a..536a0c3 100644 --- a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h +++ b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h
@@ -931,7 +931,7 @@ } /** - * \returns the starting position of the block <id> in the array of values + * \returns the starting position of the block \p id in the array of values */ Index blockPtr(Index id) const {
diff --git a/unsupported/Eigen/src/SparseExtra/CMakeLists.txt b/unsupported/Eigen/src/SparseExtra/CMakeLists.txt deleted file mode 100644 index 7ea32ca..0000000 --- a/unsupported/Eigen/src/SparseExtra/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_SparseExtra_SRCS "*.h") - -INSTALL(FILES - ${Eigen_SparseExtra_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/SparseExtra COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h b/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h index 037a13f..42c99e4 100644 --- a/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h +++ b/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h
@@ -187,7 +187,7 @@ /** Does nothing: provided for compatibility with SparseMatrix */ inline void finalize() {} - /** Suppress all nonzeros which are smaller than \a reference under the tolerence \a epsilon */ + /** Suppress all nonzeros which are smaller than \a reference under the tolerance \a epsilon */ void prune(Scalar reference, RealScalar epsilon = NumTraits<RealScalar>::dummy_precision()) { for (Index j=0; j<outerSize(); ++j) @@ -224,31 +224,43 @@ } } - /** The class DynamicSparseMatrix is deprectaed */ + /** The class DynamicSparseMatrix is deprecated */ EIGEN_DEPRECATED inline DynamicSparseMatrix() : m_innerSize(0), m_data(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif eigen_assert(innerSize()==0 && outerSize()==0); } - /** The class DynamicSparseMatrix is deprectaed */ + /** The class DynamicSparseMatrix is deprecated */ EIGEN_DEPRECATED inline DynamicSparseMatrix(Index rows, Index cols) : m_innerSize(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif resize(rows, cols); } - /** The class DynamicSparseMatrix is deprectaed */ + /** The class DynamicSparseMatrix is deprecated */ template<typename OtherDerived> EIGEN_DEPRECATED explicit inline DynamicSparseMatrix(const SparseMatrixBase<OtherDerived>& other) : m_innerSize(0) { - Base::operator=(other.derived()); + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif + Base::operator=(other.derived()); } inline DynamicSparseMatrix(const DynamicSparseMatrix& other) : Base(), m_innerSize(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif *this = other.derived(); }
diff --git a/unsupported/Eigen/src/SparseExtra/MarketIO.h b/unsupported/Eigen/src/SparseExtra/MarketIO.h index cdc14f8..1618b09 100644 --- a/unsupported/Eigen/src/SparseExtra/MarketIO.h +++ b/unsupported/Eigen/src/SparseExtra/MarketIO.h
@@ -12,38 +12,38 @@ #define EIGEN_SPARSE_MARKET_IO_H #include <iostream> +#include <vector> namespace Eigen { namespace internal { - template <typename Scalar> - inline bool GetMarketLine (std::stringstream& line, Index& M, Index& N, Index& i, Index& j, Scalar& value) + template <typename Scalar, typename StorageIndex> + inline void GetMarketLine (const char* line, StorageIndex& i, StorageIndex& j, Scalar& value) { - line >> i >> j >> value; - i--; - j--; - if(i>=0 && j>=0 && i<M && j<N) - { - return true; - } - else - return false; + std::stringstream sline(line); + sline >> i >> j >> value; } - template <typename Scalar> - inline bool GetMarketLine (std::stringstream& line, Index& M, Index& N, Index& i, Index& j, std::complex<Scalar>& value) + + template<> inline void GetMarketLine (const char* line, int& i, int& j, float& value) + { std::sscanf(line, "%d %d %g", &i, &j, &value); } + + template<> inline void GetMarketLine (const char* line, int& i, int& j, double& value) + { std::sscanf(line, "%d %d %lg", &i, &j, &value); } + + template<> inline void GetMarketLine (const char* line, int& i, int& j, std::complex<float>& value) + { std::sscanf(line, "%d %d %g %g", &i, &j, &numext::real_ref(value), &numext::imag_ref(value)); } + + template<> inline void GetMarketLine (const char* line, int& i, int& j, std::complex<double>& value) + { std::sscanf(line, "%d %d %lg %lg", &i, &j, &numext::real_ref(value), &numext::imag_ref(value)); } + + template <typename Scalar, typename StorageIndex> + inline void GetMarketLine (const char* line, StorageIndex& i, StorageIndex& j, std::complex<Scalar>& value) { + std::stringstream sline(line); Scalar valR, valI; - line >> i >> j >> valR >> valI; - i--; - j--; - if(i>=0 && j>=0 && i<M && j<N) - { - value = std::complex<Scalar>(valR, valI); - return true; - } - else - return false; + sline >> i >> j >> valR >> valI; + value = std::complex<Scalar>(valR,valI); } template <typename RealScalar> @@ -81,13 +81,13 @@ } } - template<typename Scalar> - inline void PutMatrixElt(Scalar value, int row, int col, std::ofstream& out) + template<typename Scalar, typename StorageIndex> + inline void PutMatrixElt(Scalar value, StorageIndex row, StorageIndex col, std::ofstream& out) { out << row << " "<< col << " " << value << "\n"; } - template<typename Scalar> - inline void PutMatrixElt(std::complex<Scalar> value, int row, int col, std::ofstream& out) + template<typename Scalar, typename StorageIndex> + inline void PutMatrixElt(std::complex<Scalar> value, StorageIndex row, StorageIndex col, std::ofstream& out) { out << row << " " << col << " " << value.real() << " " << value.imag() << "\n"; } @@ -104,11 +104,12 @@ out << value.real << " " << value.imag()<< "\n"; } -} // end namepsace internal +} // end namespace internal inline bool getMarketHeader(const std::string& filename, int& sym, bool& iscomplex, bool& isvector) { sym = 0; + iscomplex = false; isvector = false; std::ifstream in(filename.c_str(),std::ios::in); if(!in) @@ -133,17 +134,20 @@ bool loadMarket(SparseMatrixType& mat, const std::string& filename) { typedef typename SparseMatrixType::Scalar Scalar; - typedef typename SparseMatrixType::Index Index; + typedef typename SparseMatrixType::StorageIndex StorageIndex; std::ifstream input(filename.c_str(),std::ios::in); if(!input) return false; + + char rdbuffer[4096]; + input.rdbuf()->pubsetbuf(rdbuffer, 4096); const int maxBuffersize = 2048; char buffer[maxBuffersize]; bool readsizes = false; - typedef Triplet<Scalar,Index> T; + typedef Triplet<Scalar,StorageIndex> T; std::vector<T> elements; Index M(-1), N(-1), NNZ(-1); @@ -154,33 +158,36 @@ //NOTE An appropriate test should be done on the header to get the symmetry if(buffer[0]=='%') continue; - - std::stringstream line(buffer); - + if(!readsizes) { + std::stringstream line(buffer); line >> M >> N >> NNZ; if(M > 0 && N > 0 && NNZ > 0) { readsizes = true; - //std::cout << "sizes: " << M << "," << N << "," << NNZ << "\n"; mat.resize(M,N); mat.reserve(NNZ); } } else { - Index i(-1), j(-1); + StorageIndex i(-1), j(-1); Scalar value; - if( internal::GetMarketLine(line, M, N, i, j, value) ) + internal::GetMarketLine(buffer, i, j, value); + + i--; + j--; + if(i>=0 && j>=0 && i<M && j<N) { - ++ count; + ++count; elements.push_back(T(i,j,value)); } - else + else std::cerr << "Invalid read: " << i << "," << j << "\n"; } } + mat.setFromTriplets(elements.begin(), elements.end()); if(count!=NNZ) std::cerr << count << "!=" << NNZ << "\n"; @@ -225,12 +232,13 @@ bool saveMarket(const SparseMatrixType& mat, const std::string& filename, int sym = 0) { typedef typename SparseMatrixType::Scalar Scalar; + typedef typename SparseMatrixType::RealScalar RealScalar; std::ofstream out(filename.c_str(),std::ios::out); if(!out) return false; out.flags(std::ios_base::scientific); - out.precision(64); + out.precision(std::numeric_limits<RealScalar>::digits10 + 2); std::string header; internal::putMarketHeader<Scalar>(header, sym); out << header << std::endl; @@ -241,7 +249,6 @@ { ++ count; internal::PutMatrixElt(it.value(), it.row()+1, it.col()+1, out); - // out << it.row()+1 << " " << it.col()+1 << " " << it.value() << "\n"; } out.close(); return true; @@ -250,13 +257,14 @@ template<typename VectorType> bool saveMarketVector (const VectorType& vec, const std::string& filename) { - typedef typename VectorType::Scalar Scalar; + typedef typename VectorType::Scalar Scalar; + typedef typename VectorType::RealScalar RealScalar; std::ofstream out(filename.c_str(),std::ios::out); if(!out) return false; out.flags(std::ios_base::scientific); - out.precision(64); + out.precision(std::numeric_limits<RealScalar>::digits10 + 2); if(internal::is_same<Scalar, std::complex<float> >::value || internal::is_same<Scalar, std::complex<double> >::value) out << "%%MatrixMarket matrix array complex general\n"; else
diff --git a/unsupported/Eigen/src/SparseExtra/RandomSetter.h b/unsupported/Eigen/src/SparseExtra/RandomSetter.h index ee97299..7542cf7 100644 --- a/unsupported/Eigen/src/SparseExtra/RandomSetter.h +++ b/unsupported/Eigen/src/SparseExtra/RandomSetter.h
@@ -249,10 +249,10 @@ } } // prefix sum - Index count = 0; + StorageIndex count = 0; for (Index j=0; j<mp_target->outerSize(); ++j) { - Index tmp = positions[j]; + StorageIndex tmp = positions[j]; mp_target->outerIndexPtr()[j] = count; positions[j] = count; count += tmp; @@ -281,7 +281,7 @@ mp_target->innerIndexPtr()[i+1] = mp_target->innerIndexPtr()[i]; --i; } - mp_target->innerIndexPtr()[i+1] = inner; + mp_target->innerIndexPtr()[i+1] = internal::convert_index<StorageIndex>(inner); mp_target->valuePtr()[i+1] = it->second.value; } }
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsArrayAPI.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsArrayAPI.h new file mode 100644 index 0000000..ed6d832 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsArrayAPI.h
@@ -0,0 +1,212 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#ifndef EIGEN_SPECIALFUNCTIONS_ARRAYAPI_H +#define EIGEN_SPECIALFUNCTIONS_ARRAYAPI_H + +namespace Eigen { + +/** \cpp11 \returns an expression of the coefficient-wise igamma(\a a, \a x) to the given arrays. + * + * This function computes the coefficient-wise incomplete gamma function. + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of igammac(T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::igammac(), Eigen::lgamma() + */ +template<typename Derived,typename ExponentDerived> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_op<typename Derived::Scalar>, const Derived, const ExponentDerived> +igamma(const Eigen::ArrayBase<Derived>& a, const Eigen::ArrayBase<ExponentDerived>& x) +{ + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_op<typename Derived::Scalar>, const Derived, const ExponentDerived>( + a.derived(), + x.derived() + ); +} + +/** \cpp11 \returns an expression of the coefficient-wise igamma_der_a(\a a, \a x) to the given arrays. + * + * This function computes the coefficient-wise derivative of the incomplete + * gamma function with respect to the parameter a. + * + * \note This function supports only float and double scalar types in c++11 + * mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations + * of igamma_der_a(T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::igamma(), Eigen::lgamma() + */ +template <typename Derived, typename ExponentDerived> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_der_a_op<typename Derived::Scalar>, const Derived, const ExponentDerived> +igamma_der_a(const Eigen::ArrayBase<Derived>& a, const Eigen::ArrayBase<ExponentDerived>& x) { + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_igamma_der_a_op<typename Derived::Scalar>, const Derived, const ExponentDerived>( + a.derived(), + x.derived()); +} + +/** \cpp11 \returns an expression of the coefficient-wise gamma_sample_der_alpha(\a alpha, \a sample) to the given arrays. + * + * This function computes the coefficient-wise derivative of the sample + * of a Gamma(alpha, 1) random variable with respect to the parameter alpha. + * + * \note This function supports only float and double scalar types in c++11 + * mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations + * of gamma_sample_der_alpha(T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::igamma(), Eigen::lgamma() + */ +template <typename AlphaDerived, typename SampleDerived> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_gamma_sample_der_alpha_op<typename AlphaDerived::Scalar>, const AlphaDerived, const SampleDerived> +gamma_sample_der_alpha(const Eigen::ArrayBase<AlphaDerived>& alpha, const Eigen::ArrayBase<SampleDerived>& sample) { + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_gamma_sample_der_alpha_op<typename AlphaDerived::Scalar>, const AlphaDerived, const SampleDerived>( + alpha.derived(), + sample.derived()); +} + +/** \cpp11 \returns an expression of the coefficient-wise igammac(\a a, \a x) to the given arrays. + * + * This function computes the coefficient-wise complementary incomplete gamma function. + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of igammac(T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::igamma(), Eigen::lgamma() + */ +template<typename Derived,typename ExponentDerived> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_igammac_op<typename Derived::Scalar>, const Derived, const ExponentDerived> +igammac(const Eigen::ArrayBase<Derived>& a, const Eigen::ArrayBase<ExponentDerived>& x) +{ + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_igammac_op<typename Derived::Scalar>, const Derived, const ExponentDerived>( + a.derived(), + x.derived() + ); +} + +/** \cpp11 \returns an expression of the coefficient-wise polygamma(\a n, \a x) to the given arrays. + * + * It returns the \a n -th derivative of the digamma(psi) evaluated at \c x. + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of polygamma(T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::digamma() + */ +// * \warning Be careful with the order of the parameters: x.polygamma(n) is equivalent to polygamma(n,x) +// * \sa ArrayBase::polygamma() +template<typename DerivedN,typename DerivedX> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_polygamma_op<typename DerivedX::Scalar>, const DerivedN, const DerivedX> +polygamma(const Eigen::ArrayBase<DerivedN>& n, const Eigen::ArrayBase<DerivedX>& x) +{ + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_polygamma_op<typename DerivedX::Scalar>, const DerivedN, const DerivedX>( + n.derived(), + x.derived() + ); +} + +/** \cpp11 \returns an expression of the coefficient-wise betainc(\a x, \a a, \a b) to the given arrays. + * + * This function computes the regularized incomplete beta function (integral). + * + * \note This function supports only float and double scalar types in c++11 mode. To support other scalar types, + * or float/double in non c++11 mode, the user has to provide implementations of betainc(T,T,T) for any scalar + * type T to be supported. + * + * \sa Eigen::betainc(), Eigen::lgamma() + */ +template<typename ArgADerived, typename ArgBDerived, typename ArgXDerived> +EIGEN_STRONG_INLINE const Eigen::CwiseTernaryOp<Eigen::internal::scalar_betainc_op<typename ArgXDerived::Scalar>, const ArgADerived, const ArgBDerived, const ArgXDerived> +betainc(const Eigen::ArrayBase<ArgADerived>& a, const Eigen::ArrayBase<ArgBDerived>& b, const Eigen::ArrayBase<ArgXDerived>& x) +{ + return Eigen::CwiseTernaryOp<Eigen::internal::scalar_betainc_op<typename ArgXDerived::Scalar>, const ArgADerived, const ArgBDerived, const ArgXDerived>( + a.derived(), + b.derived(), + x.derived() + ); +} + + +/** \returns an expression of the coefficient-wise zeta(\a x, \a q) to the given arrays. + * + * It returns the Riemann zeta function of two arguments \a x and \a q: + * + * \param x is the exposent, it must be > 1 + * \param q is the shift, it must be > 0 + * + * \note This function supports only float and double scalar types. To support other scalar types, the user has + * to provide implementations of zeta(T,T) for any scalar type T to be supported. + * + * \sa ArrayBase::zeta() + */ +template<typename DerivedX,typename DerivedQ> +EIGEN_STRONG_INLINE const Eigen::CwiseBinaryOp<Eigen::internal::scalar_zeta_op<typename DerivedX::Scalar>, const DerivedX, const DerivedQ> +zeta(const Eigen::ArrayBase<DerivedX>& x, const Eigen::ArrayBase<DerivedQ>& q) +{ + return Eigen::CwiseBinaryOp<Eigen::internal::scalar_zeta_op<typename DerivedX::Scalar>, const DerivedX, const DerivedQ>( + x.derived(), + q.derived() + ); +} + +/** \returns an expression of the coefficient-wise i0e(\a x) to the given + * arrays. + * + * It returns the exponentially scaled modified Bessel + * function of order zero. + * + * \param x is the argument + * + * \note This function supports only float and double scalar types. To support + * other scalar types, the user has to provide implementations of i0e(T) for + * any scalar type T to be supported. + * + * \sa ArrayBase::i0e() + */ +template <typename Derived> +EIGEN_STRONG_INLINE const Eigen::CwiseUnaryOp< + Eigen::internal::scalar_i0e_op<typename Derived::Scalar>, const Derived> +i0e(const Eigen::ArrayBase<Derived>& x) { + return Eigen::CwiseUnaryOp< + Eigen::internal::scalar_i0e_op<typename Derived::Scalar>, + const Derived>(x.derived()); +} + +/** \returns an expression of the coefficient-wise i1e(\a x) to the given + * arrays. + * + * It returns the exponentially scaled modified Bessel + * function of order one. + * + * \param x is the argument + * + * \note This function supports only float and double scalar types. To support + * other scalar types, the user has to provide implementations of i1e(T) for + * any scalar type T to be supported. + * + * \sa ArrayBase::i1e() + */ +template <typename Derived> +EIGEN_STRONG_INLINE const Eigen::CwiseUnaryOp< + Eigen::internal::scalar_i1e_op<typename Derived::Scalar>, const Derived> +i1e(const Eigen::ArrayBase<Derived>& x) { + return Eigen::CwiseUnaryOp< + Eigen::internal::scalar_i1e_op<typename Derived::Scalar>, + const Derived>(x.derived()); +} + +} // end namespace Eigen + +#endif // EIGEN_SPECIALFUNCTIONS_ARRAYAPI_H
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsFunctors.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsFunctors.h new file mode 100644 index 0000000..c6fac91 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsFunctors.h
@@ -0,0 +1,344 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Eugene Brevdo <ebrevdo@gmail.com> +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPECIALFUNCTIONS_FUNCTORS_H +#define EIGEN_SPECIALFUNCTIONS_FUNCTORS_H + +namespace Eigen { + +namespace internal { + + +/** \internal + * \brief Template functor to compute the incomplete gamma function igamma(a, x) + * + * \sa class CwiseBinaryOp, Cwise::igamma + */ +template<typename Scalar> struct scalar_igamma_op : binary_op_base<Scalar,Scalar> +{ + EIGEN_EMPTY_STRUCT_CTOR(scalar_igamma_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& x) const { + using numext::igamma; return igamma(a, x); + } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& x) const { + return internal::pigamma(a, x); + } +}; +template<typename Scalar> +struct functor_traits<scalar_igamma_op<Scalar> > { + enum { + // Guesstimate + Cost = 20 * NumTraits<Scalar>::MulCost + 10 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasIGamma + }; +}; + +/** \internal + * \brief Template functor to compute the derivative of the incomplete gamma + * function igamma_der_a(a, x) + * + * \sa class CwiseBinaryOp, Cwise::igamma_der_a + */ +template <typename Scalar> +struct scalar_igamma_der_a_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_igamma_der_a_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& a, const Scalar& x) const { + using numext::igamma_der_a; + return igamma_der_a(a, x); + } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& x) const { + return internal::pigamma_der_a(a, x); + } +}; +template <typename Scalar> +struct functor_traits<scalar_igamma_der_a_op<Scalar> > { + enum { + // 2x the cost of igamma + Cost = 40 * NumTraits<Scalar>::MulCost + 20 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasIGammaDerA + }; +}; + +/** \internal + * \brief Template functor to compute the derivative of the sample + * of a Gamma(alpha, 1) random variable with respect to the parameter alpha + * gamma_sample_der_alpha(alpha, sample) + * + * \sa class CwiseBinaryOp, Cwise::gamma_sample_der_alpha + */ +template <typename Scalar> +struct scalar_gamma_sample_der_alpha_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_gamma_sample_der_alpha_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& alpha, const Scalar& sample) const { + using numext::gamma_sample_der_alpha; + return gamma_sample_der_alpha(alpha, sample); + } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& alpha, const Packet& sample) const { + return internal::pgamma_sample_der_alpha(alpha, sample); + } +}; +template <typename Scalar> +struct functor_traits<scalar_gamma_sample_der_alpha_op<Scalar> > { + enum { + // 2x the cost of igamma, minus the lgamma cost (the lgamma cancels out) + Cost = 30 * NumTraits<Scalar>::MulCost + 15 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasGammaSampleDerAlpha + }; +}; + +/** \internal + * \brief Template functor to compute the complementary incomplete gamma function igammac(a, x) + * + * \sa class CwiseBinaryOp, Cwise::igammac + */ +template<typename Scalar> struct scalar_igammac_op : binary_op_base<Scalar,Scalar> +{ + EIGEN_EMPTY_STRUCT_CTOR(scalar_igammac_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& x) const { + using numext::igammac; return igammac(a, x); + } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& x) const + { + return internal::pigammac(a, x); + } +}; +template<typename Scalar> +struct functor_traits<scalar_igammac_op<Scalar> > { + enum { + // Guesstimate + Cost = 20 * NumTraits<Scalar>::MulCost + 10 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasIGammac + }; +}; + + +/** \internal + * \brief Template functor to compute the incomplete beta integral betainc(a, b, x) + * + */ +template<typename Scalar> struct scalar_betainc_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_betainc_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& x, const Scalar& a, const Scalar& b) const { + using numext::betainc; return betainc(x, a, b); + } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& x, const Packet& a, const Packet& b) const + { + return internal::pbetainc(x, a, b); + } +}; +template<typename Scalar> +struct functor_traits<scalar_betainc_op<Scalar> > { + enum { + // Guesstimate + Cost = 400 * NumTraits<Scalar>::MulCost + 400 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasBetaInc + }; +}; + + +/** \internal + * \brief Template functor to compute the natural log of the absolute + * value of Gamma of a scalar + * \sa class CwiseUnaryOp, Cwise::lgamma() + */ +template<typename Scalar> struct scalar_lgamma_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_lgamma_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { + using numext::lgamma; return lgamma(a); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { return internal::plgamma(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_lgamma_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasLGamma + }; +}; + +/** \internal + * \brief Template functor to compute psi, the derivative of lgamma of a scalar. + * \sa class CwiseUnaryOp, Cwise::digamma() + */ +template<typename Scalar> struct scalar_digamma_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_digamma_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { + using numext::digamma; return digamma(a); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { return internal::pdigamma(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_digamma_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasDiGamma + }; +}; + +/** \internal + * \brief Template functor to compute the Riemann Zeta function of two arguments. + * \sa class CwiseUnaryOp, Cwise::zeta() + */ +template<typename Scalar> struct scalar_zeta_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_zeta_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& x, const Scalar& q) const { + using numext::zeta; return zeta(x, q); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x, const Packet& q) const { return internal::pzeta(x, q); } +}; +template<typename Scalar> +struct functor_traits<scalar_zeta_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasZeta + }; +}; + +/** \internal + * \brief Template functor to compute the polygamma function. + * \sa class CwiseUnaryOp, Cwise::polygamma() + */ +template<typename Scalar> struct scalar_polygamma_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_polygamma_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& n, const Scalar& x) const { + using numext::polygamma; return polygamma(n, x); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& n, const Packet& x) const { return internal::ppolygamma(n, x); } +}; +template<typename Scalar> +struct functor_traits<scalar_polygamma_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasPolygamma + }; +}; + +/** \internal + * \brief Template functor to compute the Gauss error function of a + * scalar + * \sa class CwiseUnaryOp, Cwise::erf() + */ +template<typename Scalar> struct scalar_erf_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_erf_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { + using numext::erf; return erf(a); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { return internal::perf(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_erf_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasErf + }; +}; + +/** \internal + * \brief Template functor to compute the Complementary Error Function + * of a scalar + * \sa class CwiseUnaryOp, Cwise::erfc() + */ +template<typename Scalar> struct scalar_erfc_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_erfc_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { + using numext::erfc; return erfc(a); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { return internal::perfc(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_erfc_op<Scalar> > +{ + enum { + // Guesstimate + Cost = 10 * NumTraits<Scalar>::MulCost + 5 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasErfc + }; +}; + +/** \internal + * \brief Template functor to compute the exponentially scaled modified Bessel + * function of order zero + * \sa class CwiseUnaryOp, Cwise::i0e() + */ +template <typename Scalar> +struct scalar_i0e_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_i0e_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& x) const { + using numext::i0e; + return i0e(x); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const { + return internal::pi0e(x); + } +}; +template <typename Scalar> +struct functor_traits<scalar_i0e_op<Scalar> > { + enum { + // On average, a Chebyshev polynomial of order N=20 is computed. + // The cost is N multiplications and 2N additions. + Cost = 20 * NumTraits<Scalar>::MulCost + 40 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasI0e + }; +}; + +/** \internal + * \brief Template functor to compute the exponentially scaled modified Bessel + * function of order zero + * \sa class CwiseUnaryOp, Cwise::i1e() + */ +template <typename Scalar> +struct scalar_i1e_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_i1e_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& x) const { + using numext::i1e; + return i1e(x); + } + typedef typename packet_traits<Scalar>::type Packet; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const { + return internal::pi1e(x); + } +}; +template <typename Scalar> +struct functor_traits<scalar_i1e_op<Scalar> > { + enum { + // On average, a Chebyshev polynomial of order N=20 is computed. + // The cost is N multiplications and 2N additions. + Cost = 20 * NumTraits<Scalar>::MulCost + 40 * NumTraits<Scalar>::AddCost, + PacketAccess = packet_traits<Scalar>::HasI1e + }; +}; + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_SPECIALFUNCTIONS_FUNCTORS_H
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsHalf.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsHalf.h new file mode 100644 index 0000000..fbdfd29 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsHalf.h
@@ -0,0 +1,63 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPECIALFUNCTIONS_HALF_H +#define EIGEN_SPECIALFUNCTIONS_HALF_H + +namespace Eigen { +namespace numext { + +#if EIGEN_HAS_C99_MATH +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half lgamma(const Eigen::half& a) { + return Eigen::half(Eigen::numext::lgamma(static_cast<float>(a))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half digamma(const Eigen::half& a) { + return Eigen::half(Eigen::numext::digamma(static_cast<float>(a))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half zeta(const Eigen::half& x, const Eigen::half& q) { + return Eigen::half(Eigen::numext::zeta(static_cast<float>(x), static_cast<float>(q))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half polygamma(const Eigen::half& n, const Eigen::half& x) { + return Eigen::half(Eigen::numext::polygamma(static_cast<float>(n), static_cast<float>(x))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half erf(const Eigen::half& a) { + return Eigen::half(Eigen::numext::erf(static_cast<float>(a))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half erfc(const Eigen::half& a) { + return Eigen::half(Eigen::numext::erfc(static_cast<float>(a))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half igamma(const Eigen::half& a, const Eigen::half& x) { + return Eigen::half(Eigen::numext::igamma(static_cast<float>(a), static_cast<float>(x))); +} +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half igamma_der_a(const Eigen::half& a, const Eigen::half& x) { + return Eigen::half(Eigen::numext::igamma_der_a(static_cast<float>(a), static_cast<float>(x))); +} +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half gamma_sample_der_alpha(const Eigen::half& alpha, const Eigen::half& sample) { + return Eigen::half(Eigen::numext::gamma_sample_der_alpha(static_cast<float>(alpha), static_cast<float>(sample))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half igammac(const Eigen::half& a, const Eigen::half& x) { + return Eigen::half(Eigen::numext::igammac(static_cast<float>(a), static_cast<float>(x))); +} +template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half betainc(const Eigen::half& a, const Eigen::half& b, const Eigen::half& x) { + return Eigen::half(Eigen::numext::betainc(static_cast<float>(a), static_cast<float>(b), static_cast<float>(x))); +} +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half i0e(const Eigen::half& x) { + return Eigen::half(Eigen::numext::i0e(static_cast<float>(x))); +} +template <> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half i1e(const Eigen::half& x) { + return Eigen::half(Eigen::numext::i1e(static_cast<float>(x))); +} +#endif + +} // end namespace numext +} // end namespace Eigen + +#endif // EIGEN_SPECIALFUNCTIONS_HALF_H
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h new file mode 100644 index 0000000..5784cbc --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h
@@ -0,0 +1,2170 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Eugene Brevdo <ebrevdo@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPECIAL_FUNCTIONS_H +#define EIGEN_SPECIAL_FUNCTIONS_H + +namespace Eigen { +namespace internal { + +// Parts of this code are based on the Cephes Math Library. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1992, 2000 by Stephen L. Moshier +// +// Permission has been kindly provided by the original author +// to incorporate the Cephes software into the Eigen codebase: +// +// From: Stephen Moshier +// To: Eugene Brevdo +// Subject: Re: Permission to wrap several cephes functions in Eigen +// +// Hello Eugene, +// +// Thank you for writing. +// +// If your licensing is similar to BSD, the formal way that has been +// handled is simply to add a statement to the effect that you are incorporating +// the Cephes software by permission of the author. +// +// Good luck with your project, +// Steve + +namespace cephes { + +/* polevl (modified for Eigen) + * + * Evaluate polynomial + * + * + * + * SYNOPSIS: + * + * int N; + * Scalar x, y, coef[N+1]; + * + * y = polevl<decltype(x), N>( x, coef); + * + * + * + * DESCRIPTION: + * + * Evaluates polynomial of degree N: + * + * 2 N + * y = C + C x + C x +...+ C x + * 0 1 2 N + * + * Coefficients are stored in reverse order: + * + * coef[0] = C , ..., coef[N] = C . + * N 0 + * + * The function p1evl() assumes that coef[N] = 1.0 and is + * omitted from the array. Its calling arguments are + * otherwise the same as polevl(). + * + * + * The Eigen implementation is templatized. For best speed, store + * coef as a const array (constexpr), e.g. + * + * const double coef[] = {1.0, 2.0, 3.0, ...}; + * + */ +template <typename Scalar, int N> +struct polevl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar x, const Scalar coef[]) { + EIGEN_STATIC_ASSERT((N > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); + + return polevl<Scalar, N - 1>::run(x, coef) * x + coef[N]; + } +}; + +template <typename Scalar> +struct polevl<Scalar, 0> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar, const Scalar coef[]) { + return coef[0]; + } +}; + +/* chbevl (modified for Eigen) + * + * Evaluate Chebyshev series + * + * + * + * SYNOPSIS: + * + * int N; + * Scalar x, y, coef[N], chebevl(); + * + * y = chbevl( x, coef, N ); + * + * + * + * DESCRIPTION: + * + * Evaluates the series + * + * N-1 + * - ' + * y = > coef[i] T (x/2) + * - i + * i=0 + * + * of Chebyshev polynomials Ti at argument x/2. + * + * Coefficients are stored in reverse order, i.e. the zero + * order term is last in the array. Note N is the number of + * coefficients, not the order. + * + * If coefficients are for the interval a to b, x must + * have been transformed to x -> 2(2x - b - a)/(b-a) before + * entering the routine. This maps x from (a, b) to (-1, 1), + * over which the Chebyshev polynomials are defined. + * + * If the coefficients are for the inverted interval, in + * which (a, b) is mapped to (1/b, 1/a), the transformation + * required is x -> 2(2ab/x - b - a)/(b-a). If b is infinity, + * this becomes x -> 4a/x - 1. + * + * + * + * SPEED: + * + * Taking advantage of the recurrence properties of the + * Chebyshev polynomials, the routine requires one more + * addition per loop than evaluating a nested polynomial of + * the same degree. + * + */ +template <typename Scalar, int N> +struct chebevl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar x, const Scalar coef[]) { + Scalar b0 = coef[0]; + Scalar b1 = 0; + Scalar b2; + + for (int i = 1; i < N; i++) { + b2 = b1; + b1 = b0; + b0 = x * b1 - b2 + coef[i]; + } + + return Scalar(0.5) * (b0 - b2); + } +}; + +} // end namespace cephes + +/**************************************************************************** + * Implementation of lgamma, requires C++11/C99 * + ****************************************************************************/ + +template <typename Scalar> +struct lgamma_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <typename Scalar> +struct lgamma_retval { + typedef Scalar type; +}; + +#if EIGEN_HAS_C99_MATH +template <> +struct lgamma_impl<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(float x) { +#if !defined(EIGEN_GPU_COMPILE_PHASE) && (defined(_BSD_SOURCE) || defined(_SVID_SOURCE)) && !defined(__APPLE__) + int dummy; + return ::lgammaf_r(x, &dummy); +#elif defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::lgamma(x); +#else + return ::lgammaf(x); +#endif + } +}; + +template <> +struct lgamma_impl<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(double x) { +#if !defined(EIGEN_GPU_COMPILE_PHASE) && (defined(_BSD_SOURCE) || defined(_SVID_SOURCE)) && !defined(__APPLE__) + int dummy; + return ::lgamma_r(x, &dummy); +#elif defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::lgamma(x); +#else + return ::lgamma(x); +#endif + } +}; +#endif + +/**************************************************************************** + * Implementation of digamma (psi), based on Cephes * + ****************************************************************************/ + +template <typename Scalar> +struct digamma_retval { + typedef Scalar type; +}; + +/* + * + * Polynomial evaluation helper for the Psi (digamma) function. + * + * digamma_impl_maybe_poly::run(s) evaluates the asymptotic Psi expansion for + * input Scalar s, assuming s is above 10.0. + * + * If s is above a certain threshold for the given Scalar type, zero + * is returned. Otherwise the polynomial is evaluated with enough + * coefficients for results matching Scalar machine precision. + * + * + */ +template <typename Scalar> +struct digamma_impl_maybe_poly { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + + +template <> +struct digamma_impl_maybe_poly<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(const float s) { + const float A[] = { + -4.16666666666666666667E-3f, + 3.96825396825396825397E-3f, + -8.33333333333333333333E-3f, + 8.33333333333333333333E-2f + }; + + float z; + if (s < 1.0e8f) { + z = 1.0f / (s * s); + return z * cephes::polevl<float, 3>::run(z, A); + } else return 0.0f; + } +}; + +template <> +struct digamma_impl_maybe_poly<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(const double s) { + const double A[] = { + 8.33333333333333333333E-2, + -2.10927960927960927961E-2, + 7.57575757575757575758E-3, + -4.16666666666666666667E-3, + 3.96825396825396825397E-3, + -8.33333333333333333333E-3, + 8.33333333333333333333E-2 + }; + + double z; + if (s < 1.0e17) { + z = 1.0 / (s * s); + return z * cephes::polevl<double, 6>::run(z, A); + } + else return 0.0; + } +}; + +template <typename Scalar> +struct digamma_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar x) { + /* + * + * Psi (digamma) function (modified for Eigen) + * + * + * SYNOPSIS: + * + * double x, y, psi(); + * + * y = psi( x ); + * + * + * DESCRIPTION: + * + * d - + * psi(x) = -- ln | (x) + * dx + * + * is the logarithmic derivative of the gamma function. + * For integer x, + * n-1 + * - + * psi(n) = -EUL + > 1/k. + * - + * k=1 + * + * If x is negative, it is transformed to a positive argument by the + * reflection formula psi(1-x) = psi(x) + pi cot(pi x). + * For general positive x, the argument is made greater than 10 + * using the recurrence psi(x+1) = psi(x) + 1/x. + * Then the following asymptotic expansion is applied: + * + * inf. B + * - 2k + * psi(x) = log(x) - 1/2x - > ------- + * - 2k + * k=1 2k x + * + * where the B2k are Bernoulli numbers. + * + * ACCURACY (float): + * Relative error (except absolute when |psi| < 1): + * arithmetic domain # trials peak rms + * IEEE 0,30 30000 1.3e-15 1.4e-16 + * IEEE -30,0 40000 1.5e-15 2.2e-16 + * + * ACCURACY (double): + * Absolute error, relative when |psi| > 1 : + * arithmetic domain # trials peak rms + * IEEE -33,0 30000 8.2e-7 1.2e-7 + * IEEE 0,33 100000 7.3e-7 7.7e-8 + * + * ERROR MESSAGES: + * message condition value returned + * psi singularity x integer <=0 INFINITY + */ + + Scalar p, q, nz, s, w, y; + bool negative = false; + + const Scalar maxnum = NumTraits<Scalar>::infinity(); + const Scalar m_pi = Scalar(EIGEN_PI); + + const Scalar zero = Scalar(0); + const Scalar one = Scalar(1); + const Scalar half = Scalar(0.5); + nz = zero; + + if (x <= zero) { + negative = true; + q = x; + p = numext::floor(q); + if (p == q) { + return maxnum; + } + /* Remove the zeros of tan(m_pi x) + * by subtracting the nearest integer from x + */ + nz = q - p; + if (nz != half) { + if (nz > half) { + p += one; + nz = q - p; + } + nz = m_pi / numext::tan(m_pi * nz); + } + else { + nz = zero; + } + x = one - x; + } + + /* use the recurrence psi(x+1) = psi(x) + 1/x. */ + s = x; + w = zero; + while (s < Scalar(10)) { + w += one / s; + s += one; + } + + y = digamma_impl_maybe_poly<Scalar>::run(s); + + y = numext::log(s) - (half / s) - y - w; + + return (negative) ? y - nz : y; + } +}; + +/**************************************************************************** + * Implementation of erf, requires C++11/C99 * + ****************************************************************************/ + +template <typename Scalar> +struct erf_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <typename Scalar> +struct erf_retval { + typedef Scalar type; +}; + +#if EIGEN_HAS_C99_MATH +template <> +struct erf_impl<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(float x) { +#if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::erf(x); +#else + return ::erff(x); +#endif + } +}; + +template <> +struct erf_impl<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(double x) { +#if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::erf(x); +#else + return ::erf(x); +#endif + } +}; +#endif // EIGEN_HAS_C99_MATH + +/*************************************************************************** +* Implementation of erfc, requires C++11/C99 * +****************************************************************************/ + +template <typename Scalar> +struct erfc_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <typename Scalar> +struct erfc_retval { + typedef Scalar type; +}; + +#if EIGEN_HAS_C99_MATH +template <> +struct erfc_impl<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(const float x) { +#if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::erfc(x); +#else + return ::erfcf(x); +#endif + } +}; + +template <> +struct erfc_impl<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(const double x) { +#if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) + return cl::sycl::erfc(x); +#else + return ::erfc(x); +#endif + } +}; +#endif // EIGEN_HAS_C99_MATH + +/************************************************************************************************************** + * Implementation of igammac (complemented incomplete gamma integral), based on Cephes but requires C++11/C99 * + **************************************************************************************************************/ + +template <typename Scalar> +struct igammac_retval { + typedef Scalar type; +}; + +// NOTE: cephes_helper is also used to implement zeta +template <typename Scalar> +struct cephes_helper { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar machep() { assert(false && "machep not supported for this type"); return 0.0; } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar big() { assert(false && "big not supported for this type"); return 0.0; } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar biginv() { assert(false && "biginv not supported for this type"); return 0.0; } +}; + +template <> +struct cephes_helper<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float machep() { + return NumTraits<float>::epsilon() / 2; // 1.0 - machep == 1.0 + } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float big() { + // use epsneg (1.0 - epsneg == 1.0) + return 1.0f / (NumTraits<float>::epsilon() / 2); + } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float biginv() { + // epsneg + return machep(); + } +}; + +template <> +struct cephes_helper<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double machep() { + return NumTraits<double>::epsilon() / 2; // 1.0 - machep == 1.0 + } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double big() { + return 1.0 / NumTraits<double>::epsilon(); + } + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double biginv() { + // inverse of eps + return NumTraits<double>::epsilon(); + } +}; + +enum IgammaComputationMode { VALUE, DERIVATIVE, SAMPLE_DERIVATIVE }; + +template <typename Scalar, IgammaComputationMode mode> +EIGEN_DEVICE_FUNC +int igamma_num_iterations() { + /* Returns the maximum number of internal iterations for igamma computation. + */ + if (mode == VALUE) { + return 2000; + } + + if (internal::is_same<Scalar, float>::value) { + return 200; + } else if (internal::is_same<Scalar, double>::value) { + return 500; + } else { + return 2000; + } +} + +template <typename Scalar, IgammaComputationMode mode> +struct igammac_cf_impl { + /* Computes igamc(a, x) or derivative (depending on the mode) + * using the continued fraction expansion of the complementary + * incomplete Gamma function. + * + * Preconditions: + * a > 0 + * x >= 1 + * x >= a + */ + EIGEN_DEVICE_FUNC + static Scalar run(Scalar a, Scalar x) { + const Scalar zero = 0; + const Scalar one = 1; + const Scalar two = 2; + const Scalar machep = cephes_helper<Scalar>::machep(); + const Scalar big = cephes_helper<Scalar>::big(); + const Scalar biginv = cephes_helper<Scalar>::biginv(); + + if ((numext::isinf)(x)) { + return zero; + } + + // continued fraction + Scalar y = one - a; + Scalar z = x + y + one; + Scalar c = zero; + Scalar pkm2 = one; + Scalar qkm2 = x; + Scalar pkm1 = x + one; + Scalar qkm1 = z * x; + Scalar ans = pkm1 / qkm1; + + Scalar dpkm2_da = zero; + Scalar dqkm2_da = zero; + Scalar dpkm1_da = zero; + Scalar dqkm1_da = -x; + Scalar dans_da = (dpkm1_da - ans * dqkm1_da) / qkm1; + + for (int i = 0; i < igamma_num_iterations<Scalar, mode>(); i++) { + c += one; + y += one; + z += two; + + Scalar yc = y * c; + Scalar pk = pkm1 * z - pkm2 * yc; + Scalar qk = qkm1 * z - qkm2 * yc; + + Scalar dpk_da = dpkm1_da * z - pkm1 - dpkm2_da * yc + pkm2 * c; + Scalar dqk_da = dqkm1_da * z - qkm1 - dqkm2_da * yc + qkm2 * c; + + if (qk != zero) { + Scalar ans_prev = ans; + ans = pk / qk; + + Scalar dans_da_prev = dans_da; + dans_da = (dpk_da - ans * dqk_da) / qk; + + if (mode == VALUE) { + if (numext::abs(ans_prev - ans) <= machep * numext::abs(ans)) { + break; + } + } else { + if (numext::abs(dans_da - dans_da_prev) <= machep) { + break; + } + } + } + + pkm2 = pkm1; + pkm1 = pk; + qkm2 = qkm1; + qkm1 = qk; + + dpkm2_da = dpkm1_da; + dpkm1_da = dpk_da; + dqkm2_da = dqkm1_da; + dqkm1_da = dqk_da; + + if (numext::abs(pk) > big) { + pkm2 *= biginv; + pkm1 *= biginv; + qkm2 *= biginv; + qkm1 *= biginv; + + dpkm2_da *= biginv; + dpkm1_da *= biginv; + dqkm2_da *= biginv; + dqkm1_da *= biginv; + } + } + + /* Compute x**a * exp(-x) / gamma(a) */ + Scalar logax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a); + Scalar dlogax_da = numext::log(x) - digamma_impl<Scalar>::run(a); + Scalar ax = numext::exp(logax); + Scalar dax_da = ax * dlogax_da; + + switch (mode) { + case VALUE: + return ans * ax; + case DERIVATIVE: + return ans * dax_da + dans_da * ax; + case SAMPLE_DERIVATIVE: + default: // this is needed to suppress clang warning + return -(dans_da + ans * dlogax_da) * x; + } + } +}; + +template <typename Scalar, IgammaComputationMode mode> +struct igamma_series_impl { + /* Computes igam(a, x) or its derivative (depending on the mode) + * using the series expansion of the incomplete Gamma function. + * + * Preconditions: + * x > 0 + * a > 0 + * !(x > 1 && x > a) + */ + EIGEN_DEVICE_FUNC + static Scalar run(Scalar a, Scalar x) { + const Scalar zero = 0; + const Scalar one = 1; + const Scalar machep = cephes_helper<Scalar>::machep(); + + /* power series */ + Scalar r = a; + Scalar c = one; + Scalar ans = one; + + Scalar dc_da = zero; + Scalar dans_da = zero; + + for (int i = 0; i < igamma_num_iterations<Scalar, mode>(); i++) { + r += one; + Scalar term = x / r; + Scalar dterm_da = -x / (r * r); + dc_da = term * dc_da + dterm_da * c; + dans_da += dc_da; + c *= term; + ans += c; + + if (mode == VALUE) { + if (c <= machep * ans) { + break; + } + } else { + if (numext::abs(dc_da) <= machep * numext::abs(dans_da)) { + break; + } + } + } + + /* Compute x**a * exp(-x) / gamma(a + 1) */ + Scalar logax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a + one); + Scalar dlogax_da = numext::log(x) - digamma_impl<Scalar>::run(a + one); + Scalar ax = numext::exp(logax); + Scalar dax_da = ax * dlogax_da; + + switch (mode) { + case VALUE: + return ans * ax; + case DERIVATIVE: + return ans * dax_da + dans_da * ax; + case SAMPLE_DERIVATIVE: + default: // this is needed to suppress clang warning + return -(dans_da + ans * dlogax_da) * x / a; + } + } +}; + +#if !EIGEN_HAS_C99_MATH + +template <typename Scalar> +struct igammac_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar a, Scalar x) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +#else + +template <typename Scalar> +struct igammac_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar a, Scalar x) { + /* igamc() + * + * Incomplete gamma integral (modified for Eigen) + * + * + * + * SYNOPSIS: + * + * double a, x, y, igamc(); + * + * y = igamc( a, x ); + * + * DESCRIPTION: + * + * The function is defined by + * + * + * igamc(a,x) = 1 - igam(a,x) + * + * inf. + * - + * 1 | | -t a-1 + * = ----- | e t dt. + * - | | + * | (a) - + * x + * + * + * In this implementation both arguments must be positive. + * The integral is evaluated by either a power series or + * continued fraction expansion, depending on the relative + * values of a and x. + * + * ACCURACY (float): + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0,30 30000 7.8e-6 5.9e-7 + * + * + * ACCURACY (double): + * + * Tested at random a, x. + * a x Relative error: + * arithmetic domain domain # trials peak rms + * IEEE 0.5,100 0,100 200000 1.9e-14 1.7e-15 + * IEEE 0.01,0.5 0,100 200000 1.4e-13 1.6e-15 + * + */ + /* + Cephes Math Library Release 2.2: June, 1992 + Copyright 1985, 1987, 1992 by Stephen L. Moshier + Direct inquiries to 30 Frost Street, Cambridge, MA 02140 + */ + const Scalar zero = 0; + const Scalar one = 1; + const Scalar nan = NumTraits<Scalar>::quiet_NaN(); + + if ((x < zero) || (a <= zero)) { + // domain error + return nan; + } + + if ((numext::isnan)(a) || (numext::isnan)(x)) { // propagate nans + return nan; + } + + if ((x < one) || (x < a)) { + return (one - igamma_series_impl<Scalar, VALUE>::run(a, x)); + } + + return igammac_cf_impl<Scalar, VALUE>::run(a, x); + } +}; + +#endif // EIGEN_HAS_C99_MATH + +/************************************************************************************************ + * Implementation of igamma (incomplete gamma integral), based on Cephes but requires C++11/C99 * + ************************************************************************************************/ + +#if !EIGEN_HAS_C99_MATH + +template <typename Scalar, IgammaComputationMode mode> +struct igamma_generic_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar a, Scalar x) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +#else + +template <typename Scalar, IgammaComputationMode mode> +struct igamma_generic_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar a, Scalar x) { + /* Depending on the mode, returns + * - VALUE: incomplete Gamma function igamma(a, x) + * - DERIVATIVE: derivative of incomplete Gamma function d/da igamma(a, x) + * - SAMPLE_DERIVATIVE: implicit derivative of a Gamma random variable + * x ~ Gamma(x | a, 1), dx/da = -1 / Gamma(x | a, 1) * d igamma(a, x) / dx + * + * Derivatives are implemented by forward-mode differentiation. + */ + const Scalar zero = 0; + const Scalar one = 1; + const Scalar nan = NumTraits<Scalar>::quiet_NaN(); + + if (x == zero) return zero; + + if ((x < zero) || (a <= zero)) { // domain error + return nan; + } + + if ((numext::isnan)(a) || (numext::isnan)(x)) { // propagate nans + return nan; + } + + if ((x > one) && (x > a)) { + Scalar ret = igammac_cf_impl<Scalar, mode>::run(a, x); + if (mode == VALUE) { + return one - ret; + } else { + return -ret; + } + } + + return igamma_series_impl<Scalar, mode>::run(a, x); + } +}; + +#endif // EIGEN_HAS_C99_MATH + +template <typename Scalar> +struct igamma_retval { + typedef Scalar type; +}; + +template <typename Scalar> +struct igamma_impl : igamma_generic_impl<Scalar, VALUE> { + /* igam() + * Incomplete gamma integral. + * + * The CDF of Gamma(a, 1) random variable at the point x. + * + * Accuracy estimation. For each a in [10^-2, 10^-1...10^3] we sample + * 50 Gamma random variables x ~ Gamma(x | a, 1), a total of 300 points. + * The ground truth is computed by mpmath. Mean absolute error: + * float: 1.26713e-05 + * double: 2.33606e-12 + * + * Cephes documentation below. + * + * SYNOPSIS: + * + * double a, x, y, igam(); + * + * y = igam( a, x ); + * + * DESCRIPTION: + * + * The function is defined by + * + * x + * - + * 1 | | -t a-1 + * igam(a,x) = ----- | e t dt. + * - | | + * | (a) - + * 0 + * + * + * In this implementation both arguments must be positive. + * The integral is evaluated by either a power series or + * continued fraction expansion, depending on the relative + * values of a and x. + * + * ACCURACY (double): + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0,30 200000 3.6e-14 2.9e-15 + * IEEE 0,100 300000 9.9e-14 1.5e-14 + * + * + * ACCURACY (float): + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0,30 20000 7.8e-6 5.9e-7 + * + */ + /* + Cephes Math Library Release 2.2: June, 1992 + Copyright 1985, 1987, 1992 by Stephen L. Moshier + Direct inquiries to 30 Frost Street, Cambridge, MA 02140 + */ + + /* left tail of incomplete gamma function: + * + * inf. k + * a -x - x + * x e > ---------- + * - - + * k=0 | (a+k+1) + * + */ +}; + +template <typename Scalar> +struct igamma_der_a_retval : igamma_retval<Scalar> {}; + +template <typename Scalar> +struct igamma_der_a_impl : igamma_generic_impl<Scalar, DERIVATIVE> { + /* Derivative of the incomplete Gamma function with respect to a. + * + * Computes d/da igamma(a, x) by forward differentiation of the igamma code. + * + * Accuracy estimation. For each a in [10^-2, 10^-1...10^3] we sample + * 50 Gamma random variables x ~ Gamma(x | a, 1), a total of 300 points. + * The ground truth is computed by mpmath. Mean absolute error: + * float: 6.17992e-07 + * double: 4.60453e-12 + * + * Reference: + * R. Moore. "Algorithm AS 187: Derivatives of the incomplete gamma + * integral". Journal of the Royal Statistical Society. 1982 + */ +}; + +template <typename Scalar> +struct gamma_sample_der_alpha_retval : igamma_retval<Scalar> {}; + +template <typename Scalar> +struct gamma_sample_der_alpha_impl + : igamma_generic_impl<Scalar, SAMPLE_DERIVATIVE> { + /* Derivative of a Gamma random variable sample with respect to alpha. + * + * Consider a sample of a Gamma random variable with the concentration + * parameter alpha: sample ~ Gamma(alpha, 1). The reparameterization + * derivative that we want to compute is dsample / dalpha = + * d igammainv(alpha, u) / dalpha, where u = igamma(alpha, sample). + * However, this formula is numerically unstable and expensive, so instead + * we use implicit differentiation: + * + * igamma(alpha, sample) = u, where u ~ Uniform(0, 1). + * Apply d / dalpha to both sides: + * d igamma(alpha, sample) / dalpha + * + d igamma(alpha, sample) / dsample * dsample/dalpha = 0 + * d igamma(alpha, sample) / dalpha + * + Gamma(sample | alpha, 1) dsample / dalpha = 0 + * dsample/dalpha = - (d igamma(alpha, sample) / dalpha) + * / Gamma(sample | alpha, 1) + * + * Here Gamma(sample | alpha, 1) is the PDF of the Gamma distribution + * (note that the derivative of the CDF w.r.t. sample is the PDF). + * See the reference below for more details. + * + * The derivative of igamma(alpha, sample) is computed by forward + * differentiation of the igamma code. Division by the Gamma PDF is performed + * in the same code, increasing the accuracy and speed due to cancellation + * of some terms. + * + * Accuracy estimation. For each alpha in [10^-2, 10^-1...10^3] we sample + * 50 Gamma random variables sample ~ Gamma(sample | alpha, 1), a total of 300 + * points. The ground truth is computed by mpmath. Mean absolute error: + * float: 2.1686e-06 + * double: 1.4774e-12 + * + * Reference: + * M. Figurnov, S. Mohamed, A. Mnih "Implicit Reparameterization Gradients". + * 2018 + */ +}; + +/***************************************************************************** + * Implementation of Riemann zeta function of two arguments, based on Cephes * + *****************************************************************************/ + +template <typename Scalar> +struct zeta_retval { + typedef Scalar type; +}; + +template <typename Scalar> +struct zeta_impl_series { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <> +struct zeta_impl_series<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE bool run(float& a, float& b, float& s, const float x, const float machep) { + int i = 0; + while(i < 9) + { + i += 1; + a += 1.0f; + b = numext::pow( a, -x ); + s += b; + if( numext::abs(b/s) < machep ) + return true; + } + + //Return whether we are done + return false; + } +}; + +template <> +struct zeta_impl_series<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE bool run(double& a, double& b, double& s, const double x, const double machep) { + int i = 0; + while( (i < 9) || (a <= 9.0) ) + { + i += 1; + a += 1.0; + b = numext::pow( a, -x ); + s += b; + if( numext::abs(b/s) < machep ) + return true; + } + + //Return whether we are done + return false; + } +}; + +template <typename Scalar> +struct zeta_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar x, Scalar q) { + /* zeta.c + * + * Riemann zeta function of two arguments + * + * + * + * SYNOPSIS: + * + * double x, q, y, zeta(); + * + * y = zeta( x, q ); + * + * + * + * DESCRIPTION: + * + * + * + * inf. + * - -x + * zeta(x,q) = > (k+q) + * - + * k=0 + * + * where x > 1 and q is not a negative integer or zero. + * The Euler-Maclaurin summation formula is used to obtain + * the expansion + * + * n + * - -x + * zeta(x,q) = > (k+q) + * - + * k=1 + * + * 1-x inf. B x(x+1)...(x+2j) + * (n+q) 1 - 2j + * + --------- - ------- + > -------------------- + * x-1 x - x+2j+1 + * 2(n+q) j=1 (2j)! (n+q) + * + * where the B2j are Bernoulli numbers. Note that (see zetac.c) + * zeta(x,1) = zetac(x) + 1. + * + * + * + * ACCURACY: + * + * Relative error for single precision: + * arithmetic domain # trials peak rms + * IEEE 0,25 10000 6.9e-7 1.0e-7 + * + * Large arguments may produce underflow in powf(), in which + * case the results are inaccurate. + * + * REFERENCE: + * + * Gradshteyn, I. S., and I. M. Ryzhik, Tables of Integrals, + * Series, and Products, p. 1073; Academic Press, 1980. + * + */ + + int i; + Scalar p, r, a, b, k, s, t, w; + + const Scalar A[] = { + Scalar(12.0), + Scalar(-720.0), + Scalar(30240.0), + Scalar(-1209600.0), + Scalar(47900160.0), + Scalar(-1.8924375803183791606e9), /*1.307674368e12/691*/ + Scalar(7.47242496e10), + Scalar(-2.950130727918164224e12), /*1.067062284288e16/3617*/ + Scalar(1.1646782814350067249e14), /*5.109094217170944e18/43867*/ + Scalar(-4.5979787224074726105e15), /*8.028576626982912e20/174611*/ + Scalar(1.8152105401943546773e17), /*1.5511210043330985984e23/854513*/ + Scalar(-7.1661652561756670113e18) /*1.6938241367317436694528e27/236364091*/ + }; + + const Scalar maxnum = NumTraits<Scalar>::infinity(); + const Scalar zero = 0.0, half = 0.5, one = 1.0; + const Scalar machep = cephes_helper<Scalar>::machep(); + const Scalar nan = NumTraits<Scalar>::quiet_NaN(); + + if( x == one ) + return maxnum; + + if( x < one ) + { + return nan; + } + + if( q <= zero ) + { + if(q == numext::floor(q)) + { + return maxnum; + } + p = x; + r = numext::floor(p); + if (p != r) + return nan; + } + + /* Permit negative q but continue sum until n+q > +9 . + * This case should be handled by a reflection formula. + * If q<0 and x is an integer, there is a relation to + * the polygamma function. + */ + s = numext::pow( q, -x ); + a = q; + b = zero; + // Run the summation in a helper function that is specific to the floating precision + if (zeta_impl_series<Scalar>::run(a, b, s, x, machep)) { + return s; + } + + w = a; + s += b*w/(x-one); + s -= half * b; + a = one; + k = zero; + for( i=0; i<12; i++ ) + { + a *= x + k; + b /= w; + t = a*b/A[i]; + s = s + t; + t = numext::abs(t/s); + if( t < machep ) { + break; + } + k += one; + a *= x + k; + b /= w; + k += one; + } + return s; + } +}; + +/**************************************************************************** + * Implementation of polygamma function, requires C++11/C99 * + ****************************************************************************/ + +template <typename Scalar> +struct polygamma_retval { + typedef Scalar type; +}; + +#if !EIGEN_HAS_C99_MATH + +template <typename Scalar> +struct polygamma_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar n, Scalar x) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +#else + +template <typename Scalar> +struct polygamma_impl { + EIGEN_DEVICE_FUNC + static Scalar run(Scalar n, Scalar x) { + Scalar zero = 0.0, one = 1.0; + Scalar nplus = n + one; + const Scalar nan = NumTraits<Scalar>::quiet_NaN(); + + // Check that n is an integer + if (numext::floor(n) != n) { + return nan; + } + // Just return the digamma function for n = 1 + else if (n == zero) { + return digamma_impl<Scalar>::run(x); + } + // Use the same implementation as scipy + else { + Scalar factorial = numext::exp(lgamma_impl<Scalar>::run(nplus)); + return numext::pow(-one, nplus) * factorial * zeta_impl<Scalar>::run(nplus, x); + } + } +}; + +#endif // EIGEN_HAS_C99_MATH + +/************************************************************************************************ + * Implementation of betainc (incomplete beta integral), based on Cephes but requires C++11/C99 * + ************************************************************************************************/ + +template <typename Scalar> +struct betainc_retval { + typedef Scalar type; +}; + +#if !EIGEN_HAS_C99_MATH + +template <typename Scalar> +struct betainc_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar a, Scalar b, Scalar x) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +#else + +template <typename Scalar> +struct betainc_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar, Scalar, Scalar) { + /* betaincf.c + * + * Incomplete beta integral + * + * + * SYNOPSIS: + * + * float a, b, x, y, betaincf(); + * + * y = betaincf( a, b, x ); + * + * + * DESCRIPTION: + * + * Returns incomplete beta integral of the arguments, evaluated + * from zero to x. The function is defined as + * + * x + * - - + * | (a+b) | | a-1 b-1 + * ----------- | t (1-t) dt. + * - - | | + * | (a) | (b) - + * 0 + * + * The domain of definition is 0 <= x <= 1. In this + * implementation a and b are restricted to positive values. + * The integral from x to 1 may be obtained by the symmetry + * relation + * + * 1 - betainc( a, b, x ) = betainc( b, a, 1-x ). + * + * The integral is evaluated by a continued fraction expansion. + * If a < 1, the function calls itself recursively after a + * transformation to increase a to a+1. + * + * ACCURACY (float): + * + * Tested at random points (a,b,x) with a and b in the indicated + * interval and x between 0 and 1. + * + * arithmetic domain # trials peak rms + * Relative error: + * IEEE 0,30 10000 3.7e-5 5.1e-6 + * IEEE 0,100 10000 1.7e-4 2.5e-5 + * The useful domain for relative error is limited by underflow + * of the single precision exponential function. + * Absolute error: + * IEEE 0,30 100000 2.2e-5 9.6e-7 + * IEEE 0,100 10000 6.5e-5 3.7e-6 + * + * Larger errors may occur for extreme ratios of a and b. + * + * ACCURACY (double): + * arithmetic domain # trials peak rms + * IEEE 0,5 10000 6.9e-15 4.5e-16 + * IEEE 0,85 250000 2.2e-13 1.7e-14 + * IEEE 0,1000 30000 5.3e-12 6.3e-13 + * IEEE 0,10000 250000 9.3e-11 7.1e-12 + * IEEE 0,100000 10000 8.7e-10 4.8e-11 + * Outputs smaller than the IEEE gradual underflow threshold + * were excluded from these statistics. + * + * ERROR MESSAGES: + * message condition value returned + * incbet domain x<0, x>1 nan + * incbet underflow nan + */ + + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +/* Continued fraction expansion #1 for incomplete beta integral (small_branch = True) + * Continued fraction expansion #2 for incomplete beta integral (small_branch = False) + */ +template <typename Scalar> +struct incbeta_cfe { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(Scalar a, Scalar b, Scalar x, bool small_branch) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, float>::value || + internal::is_same<Scalar, double>::value), + THIS_TYPE_IS_NOT_SUPPORTED); + const Scalar big = cephes_helper<Scalar>::big(); + const Scalar machep = cephes_helper<Scalar>::machep(); + const Scalar biginv = cephes_helper<Scalar>::biginv(); + + const Scalar zero = 0; + const Scalar one = 1; + const Scalar two = 2; + + Scalar xk, pk, pkm1, pkm2, qk, qkm1, qkm2; + Scalar k1, k2, k3, k4, k5, k6, k7, k8, k26update; + Scalar ans; + int n; + + const int num_iters = (internal::is_same<Scalar, float>::value) ? 100 : 300; + const Scalar thresh = + (internal::is_same<Scalar, float>::value) ? machep : Scalar(3) * machep; + Scalar r = (internal::is_same<Scalar, float>::value) ? zero : one; + + if (small_branch) { + k1 = a; + k2 = a + b; + k3 = a; + k4 = a + one; + k5 = one; + k6 = b - one; + k7 = k4; + k8 = a + two; + k26update = one; + } else { + k1 = a; + k2 = b - one; + k3 = a; + k4 = a + one; + k5 = one; + k6 = a + b; + k7 = a + one; + k8 = a + two; + k26update = -one; + x = x / (one - x); + } + + pkm2 = zero; + qkm2 = one; + pkm1 = one; + qkm1 = one; + ans = one; + n = 0; + + do { + xk = -(x * k1 * k2) / (k3 * k4); + pk = pkm1 + pkm2 * xk; + qk = qkm1 + qkm2 * xk; + pkm2 = pkm1; + pkm1 = pk; + qkm2 = qkm1; + qkm1 = qk; + + xk = (x * k5 * k6) / (k7 * k8); + pk = pkm1 + pkm2 * xk; + qk = qkm1 + qkm2 * xk; + pkm2 = pkm1; + pkm1 = pk; + qkm2 = qkm1; + qkm1 = qk; + + if (qk != zero) { + r = pk / qk; + if (numext::abs(ans - r) < numext::abs(r) * thresh) { + return r; + } + ans = r; + } + + k1 += one; + k2 += k26update; + k3 += two; + k4 += two; + k5 += one; + k6 -= k26update; + k7 += two; + k8 += two; + + if ((numext::abs(qk) + numext::abs(pk)) > big) { + pkm2 *= biginv; + pkm1 *= biginv; + qkm2 *= biginv; + qkm1 *= biginv; + } + if ((numext::abs(qk) < biginv) || (numext::abs(pk) < biginv)) { + pkm2 *= big; + pkm1 *= big; + qkm2 *= big; + qkm1 *= big; + } + } while (++n < num_iters); + + return ans; + } +}; + +/* Helper functions depending on the Scalar type */ +template <typename Scalar> +struct betainc_helper {}; + +template <> +struct betainc_helper<float> { + /* Core implementation, assumes a large (> 1.0) */ + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float incbsa(float aa, float bb, + float xx) { + float ans, a, b, t, x, onemx; + bool reversed_a_b = false; + + onemx = 1.0f - xx; + + /* see if x is greater than the mean */ + if (xx > (aa / (aa + bb))) { + reversed_a_b = true; + a = bb; + b = aa; + t = xx; + x = onemx; + } else { + a = aa; + b = bb; + t = onemx; + x = xx; + } + + /* Choose expansion for optimal convergence */ + if (b > 10.0f) { + if (numext::abs(b * x / a) < 0.3f) { + t = betainc_helper<float>::incbps(a, b, x); + if (reversed_a_b) t = 1.0f - t; + return t; + } + } + + ans = x * (a + b - 2.0f) / (a - 1.0f); + if (ans < 1.0f) { + ans = incbeta_cfe<float>::run(a, b, x, true /* small_branch */); + t = b * numext::log(t); + } else { + ans = incbeta_cfe<float>::run(a, b, x, false /* small_branch */); + t = (b - 1.0f) * numext::log(t); + } + + t += a * numext::log(x) + lgamma_impl<float>::run(a + b) - + lgamma_impl<float>::run(a) - lgamma_impl<float>::run(b); + t += numext::log(ans / a); + t = numext::exp(t); + + if (reversed_a_b) t = 1.0f - t; + return t; + } + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float incbps(float a, float b, float x) { + float t, u, y, s; + const float machep = cephes_helper<float>::machep(); + + y = a * numext::log(x) + (b - 1.0f) * numext::log1p(-x) - numext::log(a); + y -= lgamma_impl<float>::run(a) + lgamma_impl<float>::run(b); + y += lgamma_impl<float>::run(a + b); + + t = x / (1.0f - x); + s = 0.0f; + u = 1.0f; + do { + b -= 1.0f; + if (b == 0.0f) { + break; + } + a += 1.0f; + u *= t * b / a; + s += u; + } while (numext::abs(u) > machep); + + return numext::exp(y) * (1.0f + s); + } +}; + +template <> +struct betainc_impl<float> { + EIGEN_DEVICE_FUNC + static float run(float a, float b, float x) { + const float nan = NumTraits<float>::quiet_NaN(); + float ans, t; + + if (a <= 0.0f) return nan; + if (b <= 0.0f) return nan; + if ((x <= 0.0f) || (x >= 1.0f)) { + if (x == 0.0f) return 0.0f; + if (x == 1.0f) return 1.0f; + // mtherr("betaincf", DOMAIN); + return nan; + } + + /* transformation for small aa */ + if (a <= 1.0f) { + ans = betainc_helper<float>::incbsa(a + 1.0f, b, x); + t = a * numext::log(x) + b * numext::log1p(-x) + + lgamma_impl<float>::run(a + b) - lgamma_impl<float>::run(a + 1.0f) - + lgamma_impl<float>::run(b); + return (ans + numext::exp(t)); + } else { + return betainc_helper<float>::incbsa(a, b, x); + } + } +}; + +template <> +struct betainc_helper<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double incbps(double a, double b, double x) { + const double machep = cephes_helper<double>::machep(); + + double s, t, u, v, n, t1, z, ai; + + ai = 1.0 / a; + u = (1.0 - b) * x; + v = u / (a + 1.0); + t1 = v; + t = u; + n = 2.0; + s = 0.0; + z = machep * ai; + while (numext::abs(v) > z) { + u = (n - b) * x / n; + t *= u; + v = t / (a + n); + s += v; + n += 1.0; + } + s += t1; + s += ai; + + u = a * numext::log(x); + // TODO: gamma() is not directly implemented in Eigen. + /* + if ((a + b) < maxgam && numext::abs(u) < maxlog) { + t = gamma(a + b) / (gamma(a) * gamma(b)); + s = s * t * pow(x, a); + } else { + */ + t = lgamma_impl<double>::run(a + b) - lgamma_impl<double>::run(a) - + lgamma_impl<double>::run(b) + u + numext::log(s); + return s = numext::exp(t); + } +}; + +template <> +struct betainc_impl<double> { + EIGEN_DEVICE_FUNC + static double run(double aa, double bb, double xx) { + const double nan = NumTraits<double>::quiet_NaN(); + const double machep = cephes_helper<double>::machep(); + // const double maxgam = 171.624376956302725; + + double a, b, t, x, xc, w, y; + bool reversed_a_b = false; + + if (aa <= 0.0 || bb <= 0.0) { + return nan; // goto domerr; + } + + if ((xx <= 0.0) || (xx >= 1.0)) { + if (xx == 0.0) return (0.0); + if (xx == 1.0) return (1.0); + // mtherr("incbet", DOMAIN); + return nan; + } + + if ((bb * xx) <= 1.0 && xx <= 0.95) { + return betainc_helper<double>::incbps(aa, bb, xx); + } + + w = 1.0 - xx; + + /* Reverse a and b if x is greater than the mean. */ + if (xx > (aa / (aa + bb))) { + reversed_a_b = true; + a = bb; + b = aa; + xc = xx; + x = w; + } else { + a = aa; + b = bb; + xc = w; + x = xx; + } + + if (reversed_a_b && (b * x) <= 1.0 && x <= 0.95) { + t = betainc_helper<double>::incbps(a, b, x); + if (t <= machep) { + t = 1.0 - machep; + } else { + t = 1.0 - t; + } + return t; + } + + /* Choose expansion for better convergence. */ + y = x * (a + b - 2.0) - (a - 1.0); + if (y < 0.0) { + w = incbeta_cfe<double>::run(a, b, x, true /* small_branch */); + } else { + w = incbeta_cfe<double>::run(a, b, x, false /* small_branch */) / xc; + } + + /* Multiply w by the factor + a b _ _ _ + x (1-x) | (a+b) / ( a | (a) | (b) ) . */ + + y = a * numext::log(x); + t = b * numext::log(xc); + // TODO: gamma is not directly implemented in Eigen. + /* + if ((a + b) < maxgam && numext::abs(y) < maxlog && numext::abs(t) < maxlog) + { + t = pow(xc, b); + t *= pow(x, a); + t /= a; + t *= w; + t *= gamma(a + b) / (gamma(a) * gamma(b)); + } else { + */ + /* Resort to logarithms. */ + y += t + lgamma_impl<double>::run(a + b) - lgamma_impl<double>::run(a) - + lgamma_impl<double>::run(b); + y += numext::log(w / a); + t = numext::exp(y); + + /* } */ + // done: + + if (reversed_a_b) { + if (t <= machep) { + t = 1.0 - machep; + } else { + t = 1.0 - t; + } + } + return t; + } +}; + +#endif // EIGEN_HAS_C99_MATH + +/**************************************************************************** + * Implementation of Bessel function, based on Cephes * + ****************************************************************************/ + +template <typename Scalar> +struct i0e_retval { + typedef Scalar type; +}; + +template <typename Scalar> +struct i0e_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <> +struct i0e_impl<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(float x) { + /* i0ef.c + * + * Modified Bessel function of order zero, + * exponentially scaled + * + * + * + * SYNOPSIS: + * + * float x, y, i0ef(); + * + * y = i0ef( x ); + * + * + * + * DESCRIPTION: + * + * Returns exponentially scaled modified Bessel function + * of order zero of the argument. + * + * The function is defined as i0e(x) = exp(-|x|) j0( ix ). + * + * + * + * ACCURACY: + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0,30 100000 3.7e-7 7.0e-8 + * See i0f(). + * + */ + const float A[] = {-1.30002500998624804212E-8f, 6.04699502254191894932E-8f, + -2.67079385394061173391E-7f, 1.11738753912010371815E-6f, + -4.41673835845875056359E-6f, 1.64484480707288970893E-5f, + -5.75419501008210370398E-5f, 1.88502885095841655729E-4f, + -5.76375574538582365885E-4f, 1.63947561694133579842E-3f, + -4.32430999505057594430E-3f, 1.05464603945949983183E-2f, + -2.37374148058994688156E-2f, 4.93052842396707084878E-2f, + -9.49010970480476444210E-2f, 1.71620901522208775349E-1f, + -3.04682672343198398683E-1f, 6.76795274409476084995E-1f}; + + const float B[] = {3.39623202570838634515E-9f, 2.26666899049817806459E-8f, + 2.04891858946906374183E-7f, 2.89137052083475648297E-6f, + 6.88975834691682398426E-5f, 3.36911647825569408990E-3f, + 8.04490411014108831608E-1f}; + if (x < 0.0f) { + x = -x; + } + + if (x <= 8.0f) { + float y = 0.5f * x - 2.0f; + return cephes::chebevl<float, 18>::run(y, A); + } + + return cephes::chebevl<float, 7>::run(32.0f / x - 2.0f, B) / numext::sqrt(x); + } +}; + +template <> +struct i0e_impl<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(double x) { + /* i0e.c + * + * Modified Bessel function of order zero, + * exponentially scaled + * + * + * + * SYNOPSIS: + * + * double x, y, i0e(); + * + * y = i0e( x ); + * + * + * + * DESCRIPTION: + * + * Returns exponentially scaled modified Bessel function + * of order zero of the argument. + * + * The function is defined as i0e(x) = exp(-|x|) j0( ix ). + * + * + * + * ACCURACY: + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0,30 30000 5.4e-16 1.2e-16 + * See i0(). + * + */ + const double A[] = {-4.41534164647933937950E-18, 3.33079451882223809783E-17, + -2.43127984654795469359E-16, 1.71539128555513303061E-15, + -1.16853328779934516808E-14, 7.67618549860493561688E-14, + -4.85644678311192946090E-13, 2.95505266312963983461E-12, + -1.72682629144155570723E-11, 9.67580903537323691224E-11, + -5.18979560163526290666E-10, 2.65982372468238665035E-9, + -1.30002500998624804212E-8, 6.04699502254191894932E-8, + -2.67079385394061173391E-7, 1.11738753912010371815E-6, + -4.41673835845875056359E-6, 1.64484480707288970893E-5, + -5.75419501008210370398E-5, 1.88502885095841655729E-4, + -5.76375574538582365885E-4, 1.63947561694133579842E-3, + -4.32430999505057594430E-3, 1.05464603945949983183E-2, + -2.37374148058994688156E-2, 4.93052842396707084878E-2, + -9.49010970480476444210E-2, 1.71620901522208775349E-1, + -3.04682672343198398683E-1, 6.76795274409476084995E-1}; + const double B[] = { + -7.23318048787475395456E-18, -4.83050448594418207126E-18, + 4.46562142029675999901E-17, 3.46122286769746109310E-17, + -2.82762398051658348494E-16, -3.42548561967721913462E-16, + 1.77256013305652638360E-15, 3.81168066935262242075E-15, + -9.55484669882830764870E-15, -4.15056934728722208663E-14, + 1.54008621752140982691E-14, 3.85277838274214270114E-13, + 7.18012445138366623367E-13, -1.79417853150680611778E-12, + -1.32158118404477131188E-11, -3.14991652796324136454E-11, + 1.18891471078464383424E-11, 4.94060238822496958910E-10, + 3.39623202570838634515E-9, 2.26666899049817806459E-8, + 2.04891858946906374183E-7, 2.89137052083475648297E-6, + 6.88975834691682398426E-5, 3.36911647825569408990E-3, + 8.04490411014108831608E-1}; + + if (x < 0.0) { + x = -x; + } + + if (x <= 8.0) { + double y = (x / 2.0) - 2.0; + return cephes::chebevl<double, 30>::run(y, A); + } + + return cephes::chebevl<double, 25>::run(32.0 / x - 2.0, B) / + numext::sqrt(x); + } +}; + +template <typename Scalar> +struct i1e_retval { + typedef Scalar type; +}; + +template <typename Scalar> +struct i1e_impl { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Scalar) { + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, Scalar>::value == false), + THIS_TYPE_IS_NOT_SUPPORTED); + return Scalar(0); + } +}; + +template <> +struct i1e_impl<float> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE float run(float x) { + /* i1ef.c + * + * Modified Bessel function of order one, + * exponentially scaled + * + * + * + * SYNOPSIS: + * + * float x, y, i1ef(); + * + * y = i1ef( x ); + * + * + * + * DESCRIPTION: + * + * Returns exponentially scaled modified Bessel function + * of order one of the argument. + * + * The function is defined as i1(x) = -i exp(-|x|) j1( ix ). + * + * + * + * ACCURACY: + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0, 30 30000 1.5e-6 1.5e-7 + * See i1(). + * + */ + const float A[] = {9.38153738649577178388E-9f, -4.44505912879632808065E-8f, + 2.00329475355213526229E-7f, -8.56872026469545474066E-7f, + 3.47025130813767847674E-6f, -1.32731636560394358279E-5f, + 4.78156510755005422638E-5f, -1.61760815825896745588E-4f, + 5.12285956168575772895E-4f, -1.51357245063125314899E-3f, + 4.15642294431288815669E-3f, -1.05640848946261981558E-2f, + 2.47264490306265168283E-2f, -5.29459812080949914269E-2f, + 1.02643658689847095384E-1f, -1.76416518357834055153E-1f, + 2.52587186443633654823E-1f}; + + const float B[] = {-3.83538038596423702205E-9f, -2.63146884688951950684E-8f, + -2.51223623787020892529E-7f, -3.88256480887769039346E-6f, + -1.10588938762623716291E-4f, -9.76109749136146840777E-3f, + 7.78576235018280120474E-1f}; + + float z = numext::abs(x); + + if (z <= 8.0f) { + float y = 0.5f * z - 2.0f; + z = cephes::chebevl<float, 17>::run(y, A) * z; + } else { + z = cephes::chebevl<float, 7>::run(32.0f / z - 2.0f, B) / numext::sqrt(z); + } + + if (x < 0.0f) { + z = -z; + } + + return z; + } +}; + +template <> +struct i1e_impl<double> { + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE double run(double x) { + /* i1e.c + * + * Modified Bessel function of order one, + * exponentially scaled + * + * + * + * SYNOPSIS: + * + * double x, y, i1e(); + * + * y = i1e( x ); + * + * + * + * DESCRIPTION: + * + * Returns exponentially scaled modified Bessel function + * of order one of the argument. + * + * The function is defined as i1(x) = -i exp(-|x|) j1( ix ). + * + * + * + * ACCURACY: + * + * Relative error: + * arithmetic domain # trials peak rms + * IEEE 0, 30 30000 2.0e-15 2.0e-16 + * See i1(). + * + */ + const double A[] = {2.77791411276104639959E-18, -2.11142121435816608115E-17, + 1.55363195773620046921E-16, -1.10559694773538630805E-15, + 7.60068429473540693410E-15, -5.04218550472791168711E-14, + 3.22379336594557470981E-13, -1.98397439776494371520E-12, + 1.17361862988909016308E-11, -6.66348972350202774223E-11, + 3.62559028155211703701E-10, -1.88724975172282928790E-9, + 9.38153738649577178388E-9, -4.44505912879632808065E-8, + 2.00329475355213526229E-7, -8.56872026469545474066E-7, + 3.47025130813767847674E-6, -1.32731636560394358279E-5, + 4.78156510755005422638E-5, -1.61760815825896745588E-4, + 5.12285956168575772895E-4, -1.51357245063125314899E-3, + 4.15642294431288815669E-3, -1.05640848946261981558E-2, + 2.47264490306265168283E-2, -5.29459812080949914269E-2, + 1.02643658689847095384E-1, -1.76416518357834055153E-1, + 2.52587186443633654823E-1}; + const double B[] = { + 7.51729631084210481353E-18, 4.41434832307170791151E-18, + -4.65030536848935832153E-17, -3.20952592199342395980E-17, + 2.96262899764595013876E-16, 3.30820231092092828324E-16, + -1.88035477551078244854E-15, -3.81440307243700780478E-15, + 1.04202769841288027642E-14, 4.27244001671195135429E-14, + -2.10154184277266431302E-14, -4.08355111109219731823E-13, + -7.19855177624590851209E-13, 2.03562854414708950722E-12, + 1.41258074366137813316E-11, 3.25260358301548823856E-11, + -1.89749581235054123450E-11, -5.58974346219658380687E-10, + -3.83538038596423702205E-9, -2.63146884688951950684E-8, + -2.51223623787020892529E-7, -3.88256480887769039346E-6, + -1.10588938762623716291E-4, -9.76109749136146840777E-3, + 7.78576235018280120474E-1}; + + double z = numext::abs(x); + + if (z <= 8.0) { + double y = (z / 2.0) - 2.0; + z = cephes::chebevl<double, 29>::run(y, A) * z; + } else { + z = cephes::chebevl<double, 25>::run(32.0 / z - 2.0, B) / numext::sqrt(z); + } + + if (x < 0.0) { + z = -z; + } + + return z; + } +}; + +} // end namespace internal + +namespace numext { + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(lgamma, Scalar) + lgamma(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(lgamma, Scalar)::run(x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(digamma, Scalar) + digamma(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(digamma, Scalar)::run(x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(zeta, Scalar) +zeta(const Scalar& x, const Scalar& q) { + return EIGEN_MATHFUNC_IMPL(zeta, Scalar)::run(x, q); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(polygamma, Scalar) +polygamma(const Scalar& n, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(polygamma, Scalar)::run(n, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(erf, Scalar) + erf(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(erf, Scalar)::run(x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(erfc, Scalar) + erfc(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(erfc, Scalar)::run(x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(igamma, Scalar) + igamma(const Scalar& a, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(igamma, Scalar)::run(a, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(igamma_der_a, Scalar) + igamma_der_a(const Scalar& a, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(igamma_der_a, Scalar)::run(a, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(gamma_sample_der_alpha, Scalar) + gamma_sample_der_alpha(const Scalar& a, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(gamma_sample_der_alpha, Scalar)::run(a, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(igammac, Scalar) + igammac(const Scalar& a, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(igammac, Scalar)::run(a, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(betainc, Scalar) + betainc(const Scalar& a, const Scalar& b, const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(betainc, Scalar)::run(a, b, x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(i0e, Scalar) + i0e(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(i0e, Scalar)::run(x); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC inline EIGEN_MATHFUNC_RETVAL(i1e, Scalar) + i1e(const Scalar& x) { + return EIGEN_MATHFUNC_IMPL(i1e, Scalar)::run(x); +} + +} // end namespace numext + + +} // end namespace Eigen + +#endif // EIGEN_SPECIAL_FUNCTIONS_H
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsPacketMath.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsPacketMath.h new file mode 100644 index 0000000..465f41d --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsPacketMath.h
@@ -0,0 +1,85 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPECIALFUNCTIONS_PACKETMATH_H +#define EIGEN_SPECIALFUNCTIONS_PACKETMATH_H + +namespace Eigen { + +namespace internal { + +/** \internal \returns the ln(|gamma(\a a)|) (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet plgamma(const Packet& a) { using numext::lgamma; return lgamma(a); } + +/** \internal \returns the derivative of lgamma, psi(\a a) (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pdigamma(const Packet& a) { using numext::digamma; return digamma(a); } + +/** \internal \returns the zeta function of two arguments (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pzeta(const Packet& x, const Packet& q) { using numext::zeta; return zeta(x, q); } + +/** \internal \returns the polygamma function (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet ppolygamma(const Packet& n, const Packet& x) { using numext::polygamma; return polygamma(n, x); } + +/** \internal \returns the erf(\a a) (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet perf(const Packet& a) { using numext::erf; return erf(a); } + +/** \internal \returns the erfc(\a a) (coeff-wise) */ +template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet perfc(const Packet& a) { using numext::erfc; return erfc(a); } + +/** \internal \returns the incomplete gamma function igamma(\a a, \a x) */ +template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Packet pigamma(const Packet& a, const Packet& x) { using numext::igamma; return igamma(a, x); } + +/** \internal \returns the derivative of the incomplete gamma function + * igamma_der_a(\a a, \a x) */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pigamma_der_a(const Packet& a, const Packet& x) { + using numext::igamma_der_a; return igamma_der_a(a, x); +} + +/** \internal \returns compute the derivative of the sample + * of Gamma(alpha, 1) random variable with respect to the parameter a + * gamma_sample_der_alpha(\a alpha, \a sample) */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pgamma_sample_der_alpha(const Packet& alpha, const Packet& sample) { + using numext::gamma_sample_der_alpha; return gamma_sample_der_alpha(alpha, sample); +} + +/** \internal \returns the complementary incomplete gamma function igammac(\a a, \a x) */ +template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Packet pigammac(const Packet& a, const Packet& x) { using numext::igammac; return igammac(a, x); } + +/** \internal \returns the complementary incomplete gamma function betainc(\a a, \a b, \a x) */ +template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +Packet pbetainc(const Packet& a, const Packet& b,const Packet& x) { using numext::betainc; return betainc(a, b, x); } + +/** \internal \returns the exponentially scaled modified Bessel function of + * order zero i0e(\a a) (coeff-wise) */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pi0e(const Packet& x) { using numext::i0e; return i0e(x); } + +/** \internal \returns the exponentially scaled modified Bessel function of + * order one i1e(\a a) (coeff-wise) */ +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS +Packet pi1e(const Packet& x) { using numext::i1e; return i1e(x); } + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_SPECIALFUNCTIONS_PACKETMATH_H +
diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h new file mode 100644 index 0000000..40abcee --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h
@@ -0,0 +1,226 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_GPU_SPECIALFUNCTIONS_H +#define EIGEN_GPU_SPECIALFUNCTIONS_H + +namespace Eigen { + +namespace internal { + +// Make sure this is only available when targeting a GPU: we don't want to +// introduce conflicts between these packet_traits definitions and the ones +// we'll use on the host side (SSE, AVX, ...) +#if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 plgamma<float4>(const float4& a) +{ + return make_float4(lgammaf(a.x), lgammaf(a.y), lgammaf(a.z), lgammaf(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 plgamma<double2>(const double2& a) +{ + using numext::lgamma; + return make_double2(lgamma(a.x), lgamma(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pdigamma<float4>(const float4& a) +{ + using numext::digamma; + return make_float4(digamma(a.x), digamma(a.y), digamma(a.z), digamma(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pdigamma<double2>(const double2& a) +{ + using numext::digamma; + return make_double2(digamma(a.x), digamma(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pzeta<float4>(const float4& x, const float4& q) +{ + using numext::zeta; + return make_float4(zeta(x.x, q.x), zeta(x.y, q.y), zeta(x.z, q.z), zeta(x.w, q.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pzeta<double2>(const double2& x, const double2& q) +{ + using numext::zeta; + return make_double2(zeta(x.x, q.x), zeta(x.y, q.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 ppolygamma<float4>(const float4& n, const float4& x) +{ + using numext::polygamma; + return make_float4(polygamma(n.x, x.x), polygamma(n.y, x.y), polygamma(n.z, x.z), polygamma(n.w, x.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 ppolygamma<double2>(const double2& n, const double2& x) +{ + using numext::polygamma; + return make_double2(polygamma(n.x, x.x), polygamma(n.y, x.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 perf<float4>(const float4& a) +{ + return make_float4(erff(a.x), erff(a.y), erff(a.z), erff(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 perf<double2>(const double2& a) +{ + using numext::erf; + return make_double2(erf(a.x), erf(a.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 perfc<float4>(const float4& a) +{ + using numext::erfc; + return make_float4(erfc(a.x), erfc(a.y), erfc(a.z), erfc(a.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 perfc<double2>(const double2& a) +{ + using numext::erfc; + return make_double2(erfc(a.x), erfc(a.y)); +} + + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pigamma<float4>(const float4& a, const float4& x) +{ + using numext::igamma; + return make_float4( + igamma(a.x, x.x), + igamma(a.y, x.y), + igamma(a.z, x.z), + igamma(a.w, x.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pigamma<double2>(const double2& a, const double2& x) +{ + using numext::igamma; + return make_double2(igamma(a.x, x.x), igamma(a.y, x.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pigamma_der_a<float4>( + const float4& a, const float4& x) { + using numext::igamma_der_a; + return make_float4(igamma_der_a(a.x, x.x), igamma_der_a(a.y, x.y), + igamma_der_a(a.z, x.z), igamma_der_a(a.w, x.w)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pigamma_der_a<double2>(const double2& a, const double2& x) { + using numext::igamma_der_a; + return make_double2(igamma_der_a(a.x, x.x), igamma_der_a(a.y, x.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pgamma_sample_der_alpha<float4>( + const float4& alpha, const float4& sample) { + using numext::gamma_sample_der_alpha; + return make_float4( + gamma_sample_der_alpha(alpha.x, sample.x), + gamma_sample_der_alpha(alpha.y, sample.y), + gamma_sample_der_alpha(alpha.z, sample.z), + gamma_sample_der_alpha(alpha.w, sample.w)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pgamma_sample_der_alpha<double2>(const double2& alpha, const double2& sample) { + using numext::gamma_sample_der_alpha; + return make_double2( + gamma_sample_der_alpha(alpha.x, sample.x), + gamma_sample_der_alpha(alpha.y, sample.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pigammac<float4>(const float4& a, const float4& x) +{ + using numext::igammac; + return make_float4( + igammac(a.x, x.x), + igammac(a.y, x.y), + igammac(a.z, x.z), + igammac(a.w, x.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pigammac<double2>(const double2& a, const double2& x) +{ + using numext::igammac; + return make_double2(igammac(a.x, x.x), igammac(a.y, x.y)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +float4 pbetainc<float4>(const float4& a, const float4& b, const float4& x) +{ + using numext::betainc; + return make_float4( + betainc(a.x, b.x, x.x), + betainc(a.y, b.y, x.y), + betainc(a.z, b.z, x.z), + betainc(a.w, b.w, x.w)); +} + +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE +double2 pbetainc<double2>(const double2& a, const double2& b, const double2& x) +{ + using numext::betainc; + return make_double2(betainc(a.x, b.x, x.x), betainc(a.y, b.y, x.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pi0e<float4>(const float4& x) { + using numext::i0e; + return make_float4(i0e(x.x), i0e(x.y), i0e(x.z), i0e(x.w)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pi0e<double2>(const double2& x) { + using numext::i0e; + return make_double2(i0e(x.x), i0e(x.y)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pi1e<float4>(const float4& x) { + using numext::i1e; + return make_float4(i1e(x.x), i1e(x.y), i1e(x.z), i1e(x.w)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 +pi1e<double2>(const double2& x) { + using numext::i1e; + return make_double2(i1e(x.x), i1e(x.y)); +} + +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_GPU_SPECIALFUNCTIONS_H
diff --git a/unsupported/Eigen/src/Splines/CMakeLists.txt b/unsupported/Eigen/src/Splines/CMakeLists.txt deleted file mode 100644 index 55c6271..0000000 --- a/unsupported/Eigen/src/Splines/CMakeLists.txt +++ /dev/null
@@ -1,6 +0,0 @@ -FILE(GLOB Eigen_Splines_SRCS "*.h") - -INSTALL(FILES - ${Eigen_Splines_SRCS} - DESTINATION ${INCLUDE_INSTALL_DIR}/unsupported/Eigen/src/Splines COMPONENT Devel - )
diff --git a/unsupported/Eigen/src/Splines/Spline.h b/unsupported/Eigen/src/Splines/Spline.h index d1636f4..79edd52 100644 --- a/unsupported/Eigen/src/Splines/Spline.h +++ b/unsupported/Eigen/src/Splines/Spline.h
@@ -94,7 +94,7 @@ const KnotVectorType& knots() const { return m_knots; } /** - * \brief Returns the knots of the underlying spline. + * \brief Returns the ctrls of the underlying spline. **/ const ControlPointVectorType& ctrls() const { return m_ctrls; } @@ -191,7 +191,7 @@ DenseIndex span(Scalar u) const; /** - * \brief Computes the spang within the provided knot vector in which u is falling. + * \brief Computes the span within the provided knot vector in which u is falling. **/ static DenseIndex Span(typename SplineTraits<Spline>::Scalar u, DenseIndex degree, const typename SplineTraits<Spline>::KnotVectorType& knots); @@ -249,15 +249,13 @@ DenseIndex degree, const typename Spline<_Scalar, _Dim, _Degree>::KnotVectorType& knots) { - typedef typename Spline<_Scalar, _Dim, _Degree>::BasisVectorType BasisVectorType; - const DenseIndex p = degree; const DenseIndex i = Spline::Span(u, degree, knots); const KnotVectorType& U = knots; BasisVectorType left(p+1); left(0) = Scalar(0); - BasisVectorType right(p+1); right(0) = Scalar(0); + BasisVectorType right(p+1); right(0) = Scalar(0); VectorBlock<BasisVectorType,Degree>(left,1,p) = u - VectorBlock<const KnotVectorType,Degree>(U,i+1-p,p).reverse(); VectorBlock<BasisVectorType,Degree>(right,1,p) = VectorBlock<const KnotVectorType,Degree>(U,i+1,p) - u; @@ -380,9 +378,6 @@ typedef Spline<_Scalar, _Dim, _Degree> SplineType; enum { Order = SplineTraits<SplineType>::OrderAtCompileTime }; - typedef typename SplineTraits<SplineType>::Scalar Scalar; - typedef typename SplineTraits<SplineType>::BasisVectorType BasisVectorType; - const DenseIndex span = SplineType::Span(u, p, U); const DenseIndex n = (std::min)(p, order); @@ -394,7 +389,7 @@ Matrix<Scalar,Order,Order> ndu(p+1,p+1); - double saved, temp; + Scalar saved, temp; // FIXME These were double instead of Scalar. Was there a reason for that? ndu(0,0) = 1.0; @@ -433,7 +428,7 @@ // Compute the k-th derivative for (DenseIndex k=1; k<=static_cast<DenseIndex>(n); ++k) { - double d = 0.0; + Scalar d = 0.0; DenseIndex rk,pk,j1,j2; rk = r-k; pk = p-k;
diff --git a/unsupported/Eigen/src/Splines/SplineFitting.h b/unsupported/Eigen/src/Splines/SplineFitting.h index c761a9b..850f78a 100644 --- a/unsupported/Eigen/src/Splines/SplineFitting.h +++ b/unsupported/Eigen/src/Splines/SplineFitting.h
@@ -17,8 +17,8 @@ #include "SplineFwd.h" -#include <Eigen/LU> -#include <Eigen/QR> +#include "../../../../Eigen/LU" +#include "../../../../Eigen/QR" namespace Eigen { @@ -181,7 +181,7 @@ * \ingroup Splines_Module * * \param[in] pts The data points to which a spline should be fit. - * \param[out] chord_lengths The resulting chord lenggth vector. + * \param[out] chord_lengths The resulting chord length vector. * * \sa Les Piegl and Wayne Tiller, The NURBS book (2nd ed.), 1997, 9.2.1 Global Curve Interpolation to Point Data **/
diff --git a/unsupported/Eigen/src/Splines/SplineFwd.h b/unsupported/Eigen/src/Splines/SplineFwd.h index 0a95fbf..00d6b49 100644 --- a/unsupported/Eigen/src/Splines/SplineFwd.h +++ b/unsupported/Eigen/src/Splines/SplineFwd.h
@@ -10,7 +10,7 @@ #ifndef EIGEN_SPLINES_FWD_H #define EIGEN_SPLINES_FWD_H -#include <Eigen/Core> +#include "../../../../Eigen/Core" namespace Eigen {
diff --git a/unsupported/README.txt b/unsupported/README.txt index 83479ff..70793bf 100644 --- a/unsupported/README.txt +++ b/unsupported/README.txt
@@ -20,7 +20,7 @@ - must rely on Eigen, - must be highly related to math, - should have some general purpose in the sense that it could - potentially become an offical Eigen module (or be merged into another one). + potentially become an official Eigen module (or be merged into another one). In doubt feel free to contact us. For instance, if your addons is very too specific but it shows an interesting way of using Eigen, then it could be a nice demo.
diff --git a/unsupported/bench/bench_svd.cpp b/unsupported/bench/bench_svd.cpp index 01d8231..e7028a2 100644 --- a/unsupported/bench/bench_svd.cpp +++ b/unsupported/bench/bench_svd.cpp
@@ -70,7 +70,7 @@ std::cout<< std::endl; timerJacobi.reset(); timerBDC.reset(); - cout << " Computes rotaion matrix" <<endl; + cout << " Computes rotation matrix" <<endl; for (int k=1; k<=NUMBER_SAMPLE; ++k) { timerBDC.start();
diff --git a/unsupported/doc/examples/BVH_Example.cpp b/unsupported/doc/examples/BVH_Example.cpp index 6b6fac0..afb0c94 100644 --- a/unsupported/doc/examples/BVH_Example.cpp +++ b/unsupported/doc/examples/BVH_Example.cpp
@@ -6,9 +6,7 @@ typedef AlignedBox<double, 2> Box2d; namespace Eigen { - namespace internal { - Box2d bounding_box(const Vector2d &v) { return Box2d(v, v); } //compute the bounding box of a single point - } + Box2d bounding_box(const Vector2d &v) { return Box2d(v, v); } //compute the bounding box of a single point } struct PointPointMinimizer //how to compute squared distances between points and rectangles
diff --git a/unsupported/doc/examples/EulerAngles.cpp b/unsupported/doc/examples/EulerAngles.cpp new file mode 100644 index 0000000..3f8ca8c --- /dev/null +++ b/unsupported/doc/examples/EulerAngles.cpp
@@ -0,0 +1,46 @@ +#include <unsupported/Eigen/EulerAngles> +#include <iostream> + +using namespace Eigen; + +int main() +{ + // A common Euler system by many armies around the world, + // where the first one is the azimuth(the angle from the north - + // the same angle that is show in compass) + // and the second one is elevation(the angle from the horizon) + // and the third one is roll(the angle between the horizontal body + // direction and the plane ground surface) + // Keep remembering we're using radian angles here! + typedef EulerSystem<-EULER_Z, EULER_Y, EULER_X> MyArmySystem; + typedef EulerAngles<double, MyArmySystem> MyArmyAngles; + + MyArmyAngles vehicleAngles( + 3.14/*PI*/ / 2, /* heading to east, notice that this angle is counter-clockwise */ + -0.3, /* going down from a mountain */ + 0.1); /* slightly rolled to the right */ + + // Some Euler angles representation that our plane use. + EulerAnglesZYZd planeAngles(0.78474, 0.5271, -0.513794); + + MyArmyAngles planeAnglesInMyArmyAngles(planeAngles); + + std::cout << "vehicle angles(MyArmy): " << vehicleAngles << std::endl; + std::cout << "plane angles(ZYZ): " << planeAngles << std::endl; + std::cout << "plane angles(MyArmy): " << planeAnglesInMyArmyAngles << std::endl; + + // Now lets rotate the plane a little bit + std::cout << "==========================================================\n"; + std::cout << "rotating plane now!\n"; + std::cout << "==========================================================\n"; + + Quaterniond planeRotated = AngleAxisd(-0.342, Vector3d::UnitY()) * planeAngles; + + planeAngles = planeRotated; + planeAnglesInMyArmyAngles = planeRotated; + + std::cout << "new plane angles(ZYZ): " << planeAngles << std::endl; + std::cout << "new plane angles(MyArmy): " << planeAnglesInMyArmyAngles << std::endl; + + return 0; +}
diff --git a/unsupported/doc/examples/FFT.cpp b/unsupported/doc/examples/FFT.cpp index fcbf812..85e8a02 100644 --- a/unsupported/doc/examples/FFT.cpp +++ b/unsupported/doc/examples/FFT.cpp
@@ -61,14 +61,14 @@ void RandomFill(std::vector<T> & vec) { for (size_t k=0;k<vec.size();++k) - vec[k] = T( rand() )/T(RAND_MAX) - .5; + vec[k] = T( rand() )/T(RAND_MAX) - T(.5); } template <typename T> void RandomFill(std::vector<std::complex<T> > & vec) { for (size_t k=0;k<vec.size();++k) - vec[k] = std::complex<T> ( T( rand() )/T(RAND_MAX) - .5, T( rand() )/T(RAND_MAX) - .5); + vec[k] = std::complex<T> ( T( rand() )/T(RAND_MAX) - T(.5), T( rand() )/T(RAND_MAX) - T(.5)); } template <typename T_time,typename T_freq> @@ -85,7 +85,7 @@ vector<T_time> timebuf2; fft.inv(timebuf2,freqbuf); - long double rmse = mag2(timebuf - timebuf2) / mag2(timebuf); + T_time rmse = mag2(timebuf - timebuf2) / mag2(timebuf); cout << "roundtrip rmse: " << rmse << endl; }
diff --git a/unsupported/test/BVH.cpp b/unsupported/test/BVH.cpp index ff5b329..d8c39d5 100644 --- a/unsupported/test/BVH.cpp +++ b/unsupported/test/BVH.cpp
@@ -192,7 +192,7 @@ }; -void test_BVH() +EIGEN_DECLARE_TEST(BVH) { for(int i = 0; i < g_repeat; i++) { #ifdef EIGEN_TEST_PART_1
diff --git a/unsupported/test/CMakeLists.txt b/unsupported/test/CMakeLists.txt index b9e1b34..e8e1dc8 100644 --- a/unsupported/test/CMakeLists.txt +++ b/unsupported/test/CMakeLists.txt
@@ -1,26 +1,28 @@ -# generate split test header file only if it does not yet exist -# in order to prevent a rebuild everytime cmake is configured -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h "") - foreach(i RANGE 1 999) - file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h - "#ifdef EIGEN_TEST_PART_${i}\n" - "#define CALL_SUBTEST_${i}(FUNC) CALL_SUBTEST(FUNC)\n" - "#else\n" - "#define CALL_SUBTEST_${i}(FUNC)\n" - "#endif\n\n" - ) - endforeach() +# The file split_test_helper.h was generated at first run, +# it is now included in test/ +if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/split_test_helper.h) endif() set_property(GLOBAL PROPERTY EIGEN_CURRENT_SUBPROJECT "Unsupported") add_custom_target(BuildUnsupported) -include_directories(../../test ../../unsupported ../../Eigen +include_directories(../../test ../../unsupported ../../Eigen ${CMAKE_CURRENT_BINARY_DIR}/../../test) find_package (Threads) - + +find_package(Xsmm) +if(XSMM_FOUND) + add_definitions("-DEIGEN_USE_LIBXSMM") + include_directories(${XSMM_INCLUDES}) + link_directories(${XSMM_LIBRARIES}) + set(EXTERNAL_LIBS ${EXTERNAL_LIBS} xsmm) + ei_add_property(EIGEN_TESTED_BACKENDS "Xsmm, ") +else(XSMM_FOUND) + ei_add_property(EIGEN_MISSING_BACKENDS "Xsmm, ") +endif(XSMM_FOUND) + find_package(GoogleHash) if(GOOGLEHASH_FOUND) add_definitions("-DEIGEN_GOOGLEHASH_SUPPORT") @@ -30,11 +32,16 @@ ei_add_property(EIGEN_MISSING_BACKENDS "GoogleHash, ") endif(GOOGLEHASH_FOUND) + find_package(Adolc) if(ADOLC_FOUND) include_directories(${ADOLC_INCLUDES}) ei_add_property(EIGEN_TESTED_BACKENDS "Adolc, ") - ei_add_test(forward_adolc "" ${ADOLC_LIBRARIES}) + if(EIGEN_TEST_CXX11) + ei_add_test(forward_adolc "" ${ADOLC_LIBRARIES}) + else() + message(STATUS "Adolc found, but tests require C++11 mode") + endif() else(ADOLC_FOUND) ei_add_property(EIGEN_MISSING_BACKENDS "Adolc, ") endif(ADOLC_FOUND) @@ -47,9 +54,7 @@ ei_add_test(autodiff_scalar) ei_add_test(autodiff) -if (NOT CMAKE_CXX_COMPILER MATCHES "clang\\+\\+$") ei_add_test(BVH) -endif() ei_add_test(matrix_exponential) ei_add_test(matrix_function) @@ -59,9 +64,11 @@ ei_add_test(FFT) +ei_add_test(EulerAngles) + find_package(MPFR 2.3.0) find_package(GMP) -if(MPFR_FOUND AND EIGEN_COMPILER_SUPPORT_CXX11) +if(MPFR_FOUND AND EIGEN_COMPILER_SUPPORT_CPP11) include_directories(${MPFR_INCLUDES} ./mpreal) ei_add_property(EIGEN_TESTED_BACKENDS "MPFR C++, ") set(EIGEN_MPFR_TEST_LIBRARIES ${MPFR_LIBRARIES} ${GMP_LIBRARIES}) @@ -106,39 +113,91 @@ ei_add_test(polynomialutils) ei_add_test(splines) ei_add_test(gmres) +ei_add_test(dgmres) ei_add_test(minres) ei_add_test(levenberg_marquardt) ei_add_test(kronecker_product) +ei_add_test(special_functions) + +# TODO: The following test names are prefixed with the cxx11 string, since historically +# the tests depended on c++11. This isn't the case anymore so we ought to rename them. +# FIXME: Old versions of MSVC fail to compile this code, so we just disable these tests +# when using visual studio. We should make the check more strict to enable the tests for +# newer versions of MSVC. +if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") +ei_add_test(cxx11_tensor_dimension) +ei_add_test(cxx11_tensor_map) +ei_add_test(cxx11_tensor_assign) +ei_add_test(cxx11_tensor_block_access) +ei_add_test(cxx11_tensor_comparisons) +ei_add_test(cxx11_tensor_forced_eval) +ei_add_test(cxx11_tensor_math) +ei_add_test(cxx11_tensor_const) +ei_add_test(cxx11_tensor_intdiv) +ei_add_test(cxx11_tensor_casts) +ei_add_test(cxx11_tensor_empty) +ei_add_test(cxx11_tensor_sugar) +ei_add_test(cxx11_tensor_roundings) +ei_add_test(cxx11_tensor_layout_swap) +ei_add_test(cxx11_tensor_io) +ei_add_test(cxx11_maxsizevector) +endif() if(EIGEN_TEST_CXX11) - # It should be safe to always run these tests as there is some fallback code for - # older compiler that don't support cxx11. - set(CMAKE_CXX_STANDARD 11) + if(EIGEN_TEST_SYCL) + if(EIGEN_SYCL_TRISYCL) + set(CMAKE_CXX_STANDARD 14) + set(STD_CXX_FLAG "-std=c++1z") + else(EIGEN_SYCL_TRISYCL) + # It should be safe to always run these tests as there is some fallback code for + # older compiler that don't support cxx11. + # This is already set if EIGEN_TEST_CXX11 is enabled: + # set(CMAKE_CXX_STANDARD 11) + # set(STD_CXX_FLAG "-std=c++11") + endif(EIGEN_SYCL_TRISYCL) - ei_add_test(cxx11_float16) + ei_add_test_sycl(cxx11_tensor_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_forced_eval_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_broadcast_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_device_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_reduction_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_morphing_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_shuffling_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_padding_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_builtins_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_contract_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_concatenation_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_reverse_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_convolution_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_striding_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_chipping_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_layout_swap_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_inflation_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_generator_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_patch_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_image_patch_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_volume_patch_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_argmax_sycl ${STD_CXX_FLAG}) + ei_add_test_sycl(cxx11_tensor_custom_op_sycl ${STD_CXX_FLAG}) + endif(EIGEN_TEST_SYCL) + ei_add_test(cxx11_eventcount "-pthread" "${CMAKE_THREAD_LIBS_INIT}") ei_add_test(cxx11_runqueue "-pthread" "${CMAKE_THREAD_LIBS_INIT}") + ei_add_test(cxx11_non_blocking_thread_pool "-pthread" "${CMAKE_THREAD_LIBS_INIT}") + ei_add_test(cxx11_meta) ei_add_test(cxx11_tensor_simple) # ei_add_test(cxx11_tensor_symmetry) - ei_add_test(cxx11_tensor_assign) - ei_add_test(cxx11_tensor_dimension) ei_add_test(cxx11_tensor_index_list) ei_add_test(cxx11_tensor_mixed_indices) - ei_add_test(cxx11_tensor_comparisons) ei_add_test(cxx11_tensor_contraction) ei_add_test(cxx11_tensor_convolution) ei_add_test(cxx11_tensor_expr) - ei_add_test(cxx11_tensor_math) - ei_add_test(cxx11_tensor_forced_eval) ei_add_test(cxx11_tensor_fixed_size) - ei_add_test(cxx11_tensor_const) ei_add_test(cxx11_tensor_of_const_values) ei_add_test(cxx11_tensor_of_complex) ei_add_test(cxx11_tensor_of_strings) - ei_add_test(cxx11_tensor_intdiv) ei_add_test(cxx11_tensor_lvalue) - ei_add_test(cxx11_tensor_map) ei_add_test(cxx11_tensor_broadcasting) ei_add_test(cxx11_tensor_chipping) ei_add_test(cxx11_tensor_concatenation) @@ -154,25 +213,21 @@ ei_add_test(cxx11_tensor_striding) ei_add_test(cxx11_tensor_notification "-pthread" "${CMAKE_THREAD_LIBS_INIT}") ei_add_test(cxx11_tensor_thread_pool "-pthread" "${CMAKE_THREAD_LIBS_INIT}") + ei_add_test(cxx11_tensor_executor "-pthread" "${CMAKE_THREAD_LIBS_INIT}") ei_add_test(cxx11_tensor_ref) ei_add_test(cxx11_tensor_random) - ei_add_test(cxx11_tensor_casts) - ei_add_test(cxx11_tensor_roundings) - ei_add_test(cxx11_tensor_reverse) - ei_add_test(cxx11_tensor_layout_swap) - ei_add_test(cxx11_tensor_io) ei_add_test(cxx11_tensor_generator) ei_add_test(cxx11_tensor_custom_op) ei_add_test(cxx11_tensor_custom_index) - ei_add_test(cxx11_tensor_sugar) ei_add_test(cxx11_tensor_fft) ei_add_test(cxx11_tensor_ifft) - ei_add_test(cxx11_tensor_empty) - - if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") - # This test requires __uint128_t which is only available on 64bit systems - ei_add_test(cxx11_tensor_uint128) - endif() + ei_add_test(cxx11_tensor_scan) + ei_add_test(cxx11_tensor_trace) + ei_add_test(cxx11_tensor_move) +if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + # This test requires __uint128_t which is only available on 64bit systems + ei_add_test(cxx11_tensor_uint128) +endif() endif() @@ -182,37 +237,115 @@ # Make sure to compile without the -pedantic, -Wundef, -Wnon-virtual-dtor # and -fno-check-new flags since they trigger thousands of compilation warnings # in the CUDA runtime + # Also remove -ansi that is incompatible with std=c++11. string(REPLACE "-pedantic" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE "-Wundef" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE "-Wnon-virtual-dtor" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE "-fno-check-new" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + string(REPLACE "-ansi" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") message(STATUS "Flags used to compile cuda code: " ${CMAKE_CXX_FLAGS}) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - set(CUDA_NVCC_FLAGS "-ccbin /usr/bin/clang" CACHE STRING "nvcc flags" FORCE) + set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_C_COMPILER}" CACHE STRING "nvcc flags" FORCE) endif() if(EIGEN_TEST_CUDA_CLANG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 --cuda-gpu-arch=sm_${EIGEN_CUDA_COMPUTE_ARCH}") endif() - set(CUDA_NVCC_FLAGS "-std=c++11 --relaxed-constexpr -arch compute_${EIGEN_CUDA_COMPUTE_ARCH} -Xcudafe \"--display_error_number\"") + set(EIGEN_CUDA_RELAXED_CONSTEXPR "--expt-relaxed-constexpr") + if (${CUDA_VERSION} STREQUAL "7.0") + set(EIGEN_CUDA_RELAXED_CONSTEXPR "--relaxed-constexpr") + endif() + + if(( (NOT EIGEN_TEST_CXX11) OR (CMAKE_VERSION VERSION_LESS 3.3)) AND EIGEN_TEST_CXX11) + set(EIGEN_CUDA_CXX11_FLAG "-std=c++11") + else() + # otherwise the flag has already been added because of the above set(CMAKE_CXX_STANDARD 11) + set(EIGEN_CUDA_CXX11_FLAG "") + endif() + + set(CUDA_NVCC_FLAGS "${EIGEN_CUDA_CXX11_FLAG} ${EIGEN_CUDA_RELAXED_CONSTEXPR} -arch compute_${EIGEN_CUDA_COMPUTE_ARCH} -Xcudafe \"--display_error_number\" ${CUDA_NVCC_FLAGS}") cuda_include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CUDA_TOOLKIT_ROOT_DIR}/include") set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu") - ei_add_test(cxx11_tensor_device) - ei_add_test(cxx11_tensor_cuda) - ei_add_test(cxx11_tensor_contract_cuda) - ei_add_test(cxx11_tensor_reduction_cuda) - ei_add_test(cxx11_tensor_argmax_cuda) - ei_add_test(cxx11_tensor_cast_float16_cuda) + ei_add_test(cxx11_tensor_complex_gpu) + ei_add_test(cxx11_tensor_complex_cwise_ops_gpu) + ei_add_test(cxx11_tensor_reduction_gpu) + ei_add_test(cxx11_tensor_argmax_gpu) + ei_add_test(cxx11_tensor_cast_float16_gpu) + ei_add_test(cxx11_tensor_scan_gpu) + + # Contractions require arch 3.0 or higher + if (${EIGEN_CUDA_COMPUTE_ARCH} GREATER 29) + ei_add_test(cxx11_tensor_device) + ei_add_test(cxx11_tensor_gpu) + ei_add_test(cxx11_tensor_contract_gpu) + ei_add_test(cxx11_tensor_of_float16_gpu) + endif() # The random number generation code requires arch 3.5 or greater. if (${EIGEN_CUDA_COMPUTE_ARCH} GREATER 34) - ei_add_test(cxx11_tensor_random_cuda) + ei_add_test(cxx11_tensor_random_gpu) endif() - ei_add_test(cxx11_tensor_of_float16_cuda) unset(EIGEN_ADD_TEST_FILENAME_EXTENSION) endif() + +# Add HIP specific tests +if (EIGEN_TEST_HIP) + + set(HIP_PATH "/opt/rocm/hip" CACHE STRING "Path to the HIP installation.") + + if (EXISTS ${HIP_PATH}) + + list(APPEND CMAKE_MODULE_PATH ${HIP_PATH}/cmake) + + find_package(HIP REQUIRED) + if (HIP_FOUND) + + execute_process(COMMAND ${HIP_PATH}/bin/hipconfig --platform OUTPUT_VARIABLE HIP_PLATFORM) + + if (${HIP_PLATFORM} STREQUAL "hcc") + + include_directories(${CMAKE_CURRENT_BINARY_DIR}) + include_directories(${HIP_PATH}/include) + + set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu") + # + # complex datatype is not yet supported by HIP + # so leaving out those tests for now + # + # ei_add_test(cxx11_tensor_complex_gpu) + # ei_add_test(cxx11_tensor_complex_cwise_ops_gpu) + # + ei_add_test(cxx11_tensor_reduction_gpu) + ei_add_test(cxx11_tensor_argmax_gpu) + ei_add_test(cxx11_tensor_cast_float16_gpu) + ei_add_test(cxx11_tensor_scan_gpu) + ei_add_test(cxx11_tensor_device) + + ei_add_test(cxx11_tensor_gpu) + ei_add_test(cxx11_tensor_contract_gpu) + ei_add_test(cxx11_tensor_of_float16_gpu) + ei_add_test(cxx11_tensor_random_gpu) + + unset(EIGEN_ADD_TEST_FILENAME_EXTENSION) + + elseif (${HIP_PLATFORM} STREQUAL "nvcc") + message(FATAL_ERROR "HIP_PLATFORM = nvcc is not supported within Eigen") + else () + message(FATAL_ERROR "Unknown HIP_PLATFORM = ${HIP_PLATFORM}") + endif() + + endif(HIP_FOUND) + + else () + + message(FATAL_ERROR "EIGEN_TEST_HIP is ON, but the specified HIP_PATH (${HIP_PATH}) does not exist") + + endif() + +endif(EIGEN_TEST_HIP) +
diff --git a/unsupported/test/EulerAngles.cpp b/unsupported/test/EulerAngles.cpp new file mode 100644 index 0000000..4ddb5a2 --- /dev/null +++ b/unsupported/test/EulerAngles.cpp
@@ -0,0 +1,296 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Tal Hadad <tal_hd@hotmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +#include <unsupported/Eigen/EulerAngles> + +using namespace Eigen; + +// Unfortunately, we need to specialize it in order to work. (We could add it in main.h test framework) +template <typename Scalar, class System> +bool verifyIsApprox(const Eigen::EulerAngles<Scalar, System>& a, const Eigen::EulerAngles<Scalar, System>& b) +{ + return verifyIsApprox(a.angles(), b.angles()); +} + +// Verify that x is in the approxed range [a, b] +#define VERIFY_APPROXED_RANGE(a, x, b) \ + do { \ + VERIFY_IS_APPROX_OR_LESS_THAN(a, x); \ + VERIFY_IS_APPROX_OR_LESS_THAN(x, b); \ + } while(0) + +const char X = EULER_X; +const char Y = EULER_Y; +const char Z = EULER_Z; + +template<typename Scalar, class EulerSystem> +void verify_euler(const EulerAngles<Scalar, EulerSystem>& e) +{ + typedef EulerAngles<Scalar, EulerSystem> EulerAnglesType; + typedef Matrix<Scalar,3,3> Matrix3; + typedef Matrix<Scalar,3,1> Vector3; + typedef Quaternion<Scalar> QuaternionType; + typedef AngleAxis<Scalar> AngleAxisType; + + const Scalar ONE = Scalar(1); + const Scalar HALF_PI = Scalar(EIGEN_PI / 2); + const Scalar PI = Scalar(EIGEN_PI); + + // It's very important calc the acceptable precision depending on the distance from the pole. + const Scalar longitudeRadius = std::abs( + EulerSystem::IsTaitBryan ? + std::cos(e.beta()) : + std::sin(e.beta()) + ); + Scalar precision = test_precision<Scalar>() / longitudeRadius; + + Scalar betaRangeStart, betaRangeEnd; + if (EulerSystem::IsTaitBryan) + { + betaRangeStart = -HALF_PI; + betaRangeEnd = HALF_PI; + } + else + { + if (!EulerSystem::IsBetaOpposite) + { + betaRangeStart = 0; + betaRangeEnd = PI; + } + else + { + betaRangeStart = -PI; + betaRangeEnd = 0; + } + } + + const Vector3 I_ = EulerAnglesType::AlphaAxisVector(); + const Vector3 J_ = EulerAnglesType::BetaAxisVector(); + const Vector3 K_ = EulerAnglesType::GammaAxisVector(); + + // Is approx checks + VERIFY(e.isApprox(e)); + VERIFY_IS_APPROX(e, e); + VERIFY_IS_NOT_APPROX(e, EulerAnglesType(e.alpha() + ONE, e.beta() + ONE, e.gamma() + ONE)); + + const Matrix3 m(e); + VERIFY_IS_APPROX(Scalar(m.determinant()), ONE); + + EulerAnglesType ebis(m); + + // When no roll(acting like polar representation), we have the best precision. + // One of those cases is when the Euler angles are on the pole, and because it's singular case, + // the computation returns no roll. + if (ebis.beta() == 0) + precision = test_precision<Scalar>(); + + // Check that eabis in range + VERIFY_APPROXED_RANGE(-PI, ebis.alpha(), PI); + VERIFY_APPROXED_RANGE(betaRangeStart, ebis.beta(), betaRangeEnd); + VERIFY_APPROXED_RANGE(-PI, ebis.gamma(), PI); + + const Matrix3 mbis(AngleAxisType(ebis.alpha(), I_) * AngleAxisType(ebis.beta(), J_) * AngleAxisType(ebis.gamma(), K_)); + VERIFY_IS_APPROX(Scalar(mbis.determinant()), ONE); + VERIFY_IS_APPROX(mbis, ebis.toRotationMatrix()); + /*std::cout << "===================\n" << + "e: " << e << std::endl << + "eabis: " << eabis.transpose() << std::endl << + "m: " << m << std::endl << + "mbis: " << mbis << std::endl << + "X: " << (m * Vector3::UnitX()).transpose() << std::endl << + "X: " << (mbis * Vector3::UnitX()).transpose() << std::endl;*/ + VERIFY(m.isApprox(mbis, precision)); + + // Test if ea and eabis are the same + // Need to check both singular and non-singular cases + // There are two singular cases. + // 1. When I==K and sin(ea(1)) == 0 + // 2. When I!=K and cos(ea(1)) == 0 + + // TODO: Make this test work well, and use range saturation function. + /*// If I==K, and ea[1]==0, then there no unique solution. + // The remark apply in the case where I!=K, and |ea[1]| is close to +-pi/2. + if( (i!=k || ea[1]!=0) && (i==k || !internal::isApprox(abs(ea[1]),Scalar(EIGEN_PI/2),test_precision<Scalar>())) ) + VERIFY_IS_APPROX(ea, eabis);*/ + + // Quaternions + const QuaternionType q(e); + ebis = q; + const QuaternionType qbis(ebis); + VERIFY(internal::isApprox<Scalar>(std::abs(q.dot(qbis)), ONE, precision)); + //VERIFY_IS_APPROX(eabis, eabis2);// Verify that the euler angles are still the same + + // A suggestion for simple product test when will be supported. + /*EulerAnglesType e2(PI/2, PI/2, PI/2); + Matrix3 m2(e2); + VERIFY_IS_APPROX(e*e2, m*m2);*/ +} + +template<signed char A, signed char B, signed char C, typename Scalar> +void verify_euler_vec(const Matrix<Scalar,3,1>& ea) +{ + verify_euler(EulerAngles<Scalar, EulerSystem<A, B, C> >(ea[0], ea[1], ea[2])); +} + +template<signed char A, signed char B, signed char C, typename Scalar> +void verify_euler_all_neg(const Matrix<Scalar,3,1>& ea) +{ + verify_euler_vec<+A,+B,+C>(ea); + verify_euler_vec<+A,+B,-C>(ea); + verify_euler_vec<+A,-B,+C>(ea); + verify_euler_vec<+A,-B,-C>(ea); + + verify_euler_vec<-A,+B,+C>(ea); + verify_euler_vec<-A,+B,-C>(ea); + verify_euler_vec<-A,-B,+C>(ea); + verify_euler_vec<-A,-B,-C>(ea); +} + +template<typename Scalar> void check_all_var(const Matrix<Scalar,3,1>& ea) +{ + verify_euler_all_neg<X,Y,Z>(ea); + verify_euler_all_neg<X,Y,X>(ea); + verify_euler_all_neg<X,Z,Y>(ea); + verify_euler_all_neg<X,Z,X>(ea); + + verify_euler_all_neg<Y,Z,X>(ea); + verify_euler_all_neg<Y,Z,Y>(ea); + verify_euler_all_neg<Y,X,Z>(ea); + verify_euler_all_neg<Y,X,Y>(ea); + + verify_euler_all_neg<Z,X,Y>(ea); + verify_euler_all_neg<Z,X,Z>(ea); + verify_euler_all_neg<Z,Y,X>(ea); + verify_euler_all_neg<Z,Y,Z>(ea); +} + +template<typename Scalar> void check_singular_cases(const Scalar& singularBeta) +{ + typedef Matrix<Scalar,3,1> Vector3; + const Scalar PI = Scalar(EIGEN_PI); + + for (Scalar epsilon = NumTraits<Scalar>::epsilon(); epsilon < 1; epsilon *= Scalar(1.2)) + { + check_all_var(Vector3(PI/4, singularBeta, PI/3)); + check_all_var(Vector3(PI/4, singularBeta - epsilon, PI/3)); + check_all_var(Vector3(PI/4, singularBeta - Scalar(1.5)*epsilon, PI/3)); + check_all_var(Vector3(PI/4, singularBeta - 2*epsilon, PI/3)); + check_all_var(Vector3(PI*Scalar(0.8), singularBeta - epsilon, Scalar(0.9)*PI)); + check_all_var(Vector3(PI*Scalar(-0.9), singularBeta + epsilon, PI*Scalar(0.3))); + check_all_var(Vector3(PI*Scalar(-0.6), singularBeta + Scalar(1.5)*epsilon, PI*Scalar(0.3))); + check_all_var(Vector3(PI*Scalar(-0.5), singularBeta + 2*epsilon, PI*Scalar(0.4))); + check_all_var(Vector3(PI*Scalar(0.9), singularBeta + epsilon, Scalar(0.8)*PI)); + } + + // This one for sanity, it had a problem with near pole cases in float scalar. + check_all_var(Vector3(PI*Scalar(0.8), singularBeta - Scalar(1E-6), Scalar(0.9)*PI)); +} + +template<typename Scalar> void eulerangles_manual() +{ + typedef Matrix<Scalar,3,1> Vector3; + typedef Matrix<Scalar,Dynamic,1> VectorX; + const Vector3 Zero = Vector3::Zero(); + const Scalar PI = Scalar(EIGEN_PI); + + check_all_var(Zero); + + // singular cases + check_singular_cases(PI/2); + check_singular_cases(-PI/2); + + check_singular_cases(Scalar(0)); + check_singular_cases(Scalar(-0)); + + check_singular_cases(PI); + check_singular_cases(-PI); + + // non-singular cases + VectorX alpha = VectorX::LinSpaced(Eigen::Sequential, 20, Scalar(-0.99) * PI, PI); + VectorX beta = VectorX::LinSpaced(Eigen::Sequential, 20, Scalar(-0.49) * PI, Scalar(0.49) * PI); + VectorX gamma = VectorX::LinSpaced(Eigen::Sequential, 20, Scalar(-0.99) * PI, PI); + for (int i = 0; i < alpha.size(); ++i) { + for (int j = 0; j < beta.size(); ++j) { + for (int k = 0; k < gamma.size(); ++k) { + check_all_var(Vector3(alpha(i), beta(j), gamma(k))); + } + } + } +} + +template<typename Scalar> void eulerangles_rand() +{ + typedef Matrix<Scalar,3,3> Matrix3; + typedef Matrix<Scalar,3,1> Vector3; + typedef Array<Scalar,3,1> Array3; + typedef Quaternion<Scalar> Quaternionx; + typedef AngleAxis<Scalar> AngleAxisType; + + Scalar a = internal::random<Scalar>(-Scalar(EIGEN_PI), Scalar(EIGEN_PI)); + Quaternionx q1; + q1 = AngleAxisType(a, Vector3::Random().normalized()); + Matrix3 m; + m = q1; + + Vector3 ea = m.eulerAngles(0,1,2); + check_all_var(ea); + ea = m.eulerAngles(0,1,0); + check_all_var(ea); + + // Check with purely random Quaternion: + q1.coeffs() = Quaternionx::Coefficients::Random().normalized(); + m = q1; + ea = m.eulerAngles(0,1,2); + check_all_var(ea); + ea = m.eulerAngles(0,1,0); + check_all_var(ea); + + // Check with random angles in range [0:pi]x[-pi:pi]x[-pi:pi]. + ea = (Array3::Random() + Array3(1,0,0))*Scalar(EIGEN_PI)*Array3(0.5,1,1); + check_all_var(ea); + + ea[2] = ea[0] = internal::random<Scalar>(0,Scalar(EIGEN_PI)); + check_all_var(ea); + + ea[0] = ea[1] = internal::random<Scalar>(0,Scalar(EIGEN_PI)); + check_all_var(ea); + + ea[1] = 0; + check_all_var(ea); + + ea.head(2).setZero(); + check_all_var(ea); + + ea.setZero(); + check_all_var(ea); +} + +EIGEN_DECLARE_TEST(EulerAngles) +{ + // Simple cast test + EulerAnglesXYZd onesEd(1, 1, 1); + EulerAnglesXYZf onesEf = onesEd.cast<float>(); + VERIFY_IS_APPROX(onesEd, onesEf.cast<double>()); + + // Simple Construction from Vector3 test + VERIFY_IS_APPROX(onesEd, EulerAnglesXYZd(Vector3d::Ones())); + + CALL_SUBTEST_1( eulerangles_manual<float>() ); + CALL_SUBTEST_2( eulerangles_manual<double>() ); + + for(int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_3( eulerangles_rand<float>() ); + CALL_SUBTEST_4( eulerangles_rand<double>() ); + } + + // TODO: Add tests for auto diff + // TODO: Add tests for complex numbers +}
diff --git a/unsupported/test/FFTW.cpp b/unsupported/test/FFTW.cpp index d3718e2..cfe559e 100644 --- a/unsupported/test/FFTW.cpp +++ b/unsupported/test/FFTW.cpp
@@ -18,11 +18,11 @@ template < typename T> -complex<long double> promote(complex<T> x) { return complex<long double>(x.real(),x.imag()); } +complex<long double> promote(complex<T> x) { return complex<long double>((long double)x.real(),(long double)x.imag()); } -complex<long double> promote(float x) { return complex<long double>( x); } -complex<long double> promote(double x) { return complex<long double>( x); } -complex<long double> promote(long double x) { return complex<long double>( x); } +complex<long double> promote(float x) { return complex<long double>((long double)x); } +complex<long double> promote(double x) { return complex<long double>((long double)x); } +complex<long double> promote(long double x) { return complex<long double>((long double)x); } template <typename VT1,typename VT2> @@ -33,7 +33,7 @@ long double pi = acos((long double)-1 ); for (size_t k0=0;k0<(size_t)fftbuf.size();++k0) { complex<long double> acc = 0; - long double phinc = -2.*k0* pi / timebuf.size(); + long double phinc = (long double)(-2.)*k0* pi / timebuf.size(); for (size_t k1=0;k1<(size_t)timebuf.size();++k1) { acc += promote( timebuf[k1] ) * exp( complex<long double>(0,k1*phinc) ); } @@ -54,8 +54,8 @@ long double difpower=0; size_t n = (min)( buf1.size(),buf2.size() ); for (size_t k=0;k<n;++k) { - totalpower += (numext::abs2( buf1[k] ) + numext::abs2(buf2[k]) )/2.; - difpower += numext::abs2(buf1[k] - buf2[k]); + totalpower += (long double)((numext::abs2( buf1[k] ) + numext::abs2(buf2[k]) )/2); + difpower += (long double)(numext::abs2(buf1[k] - buf2[k])); } return sqrt(difpower/totalpower); } @@ -93,19 +93,19 @@ fft.SetFlag(fft.HalfSpectrum ); fft.fwd( freqBuf,tbuf); VERIFY((size_t)freqBuf.size() == (size_t)( (nfft>>1)+1) ); - VERIFY( fft_rmse(freqBuf,tbuf) < test_precision<T>() );// gross check + VERIFY( T(fft_rmse(freqBuf,tbuf)) < test_precision<T>() );// gross check fft.ClearFlag(fft.HalfSpectrum ); fft.fwd( freqBuf,tbuf); VERIFY( (size_t)freqBuf.size() == (size_t)nfft); - VERIFY( fft_rmse(freqBuf,tbuf) < test_precision<T>() );// gross check + VERIFY( T(fft_rmse(freqBuf,tbuf)) < test_precision<T>() );// gross check if (nfft&1) return; // odd FFTs get the wrong size inverse FFT ScalarVector tbuf2; fft.inv( tbuf2 , freqBuf); - VERIFY( dif_rmse(tbuf,tbuf2) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(tbuf,tbuf2)) < test_precision<T>() );// gross check // verify that the Unscaled flag takes effect @@ -121,12 +121,12 @@ //for (size_t i=0;i<(size_t) tbuf.size();++i) // cout << "freqBuf=" << freqBuf[i] << " in2=" << tbuf3[i] << " - in=" << tbuf[i] << " => " << (tbuf3[i] - tbuf[i] ) << endl; - VERIFY( dif_rmse(tbuf,tbuf3) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(tbuf,tbuf3)) < test_precision<T>() );// gross check // verify that ClearFlag works fft.ClearFlag(fft.Unscaled); fft.inv( tbuf2 , freqBuf); - VERIFY( dif_rmse(tbuf,tbuf2) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(tbuf,tbuf2)) < test_precision<T>() );// gross check } template <typename T> @@ -152,10 +152,10 @@ inbuf[k]= Complex( (T)(rand()/(double)RAND_MAX - .5), (T)(rand()/(double)RAND_MAX - .5) ); fft.fwd( outbuf , inbuf); - VERIFY( fft_rmse(outbuf,inbuf) < test_precision<T>() );// gross check + VERIFY( T(fft_rmse(outbuf,inbuf)) < test_precision<T>() );// gross check fft.inv( buf3 , outbuf); - VERIFY( dif_rmse(inbuf,buf3) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(inbuf,buf3)) < test_precision<T>() );// gross check // verify that the Unscaled flag takes effect ComplexVector buf4; @@ -163,12 +163,12 @@ fft.inv( buf4 , outbuf); for (int k=0;k<nfft;++k) buf4[k] *= T(1./nfft); - VERIFY( dif_rmse(inbuf,buf4) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(inbuf,buf4)) < test_precision<T>() );// gross check // verify that ClearFlag works fft.ClearFlag(fft.Unscaled); fft.inv( buf3 , outbuf); - VERIFY( dif_rmse(inbuf,buf3) < test_precision<T>() );// gross check + VERIFY( T(dif_rmse(inbuf,buf3)) < test_precision<T>() );// gross check } template <typename T> @@ -225,7 +225,7 @@ VERIFY( (in1-in).norm() < test_precision<float>() ); } -void test_FFTW() +EIGEN_DECLARE_TEST(FFTW) { CALL_SUBTEST( test_return_by_value(32) ); //CALL_SUBTEST( ( test_complex2d<float,4,8> () ) ); CALL_SUBTEST( ( test_complex2d<double,4,8> () ) );
diff --git a/unsupported/test/NonLinearOptimization.cpp b/unsupported/test/NonLinearOptimization.cpp index 6a5ed05..c667b72 100644 --- a/unsupported/test/NonLinearOptimization.cpp +++ b/unsupported/test/NonLinearOptimization.cpp
@@ -12,11 +12,18 @@ // It is intended to be done for this test only. #include <Eigen/src/Core/util/DisableStupidWarnings.h> -using std::sqrt; - // tolerance for chekcing number of iterations #define LM_EVAL_COUNT_TOL 4/3 +#define LM_CHECK_N_ITERS(SOLVER,NFEV,NJEV) { \ + ++g_test_level; \ + VERIFY_IS_EQUAL(SOLVER.nfev, NFEV); \ + VERIFY_IS_EQUAL(SOLVER.njev, NJEV); \ + --g_test_level; \ + VERIFY(SOLVER.nfev <= NFEV * LM_EVAL_COUNT_TOL); \ + VERIFY(SOLVER.njev <= NJEV * LM_EVAL_COUNT_TOL); \ + } + int fcn_chkder(const VectorXd &x, VectorXd &fvec, MatrixXd &fjac, int iflag) { /* subroutine fcn for chkder example. */ @@ -182,8 +189,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 6); - VERIFY_IS_EQUAL(lm.njev, 5); + LM_CHECK_N_ITERS(lm, 6, 5); // check norm VERIFY_IS_APPROX(lm.fvec.blueNorm(), 0.09063596); @@ -211,8 +217,7 @@ // check return values VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 6); - VERIFY_IS_EQUAL(lm.njev, 5); + LM_CHECK_N_ITERS(lm, 6, 5); // check norm fnorm = lm.fvec.blueNorm(); @@ -296,8 +301,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(solver.nfev, 11); - VERIFY_IS_EQUAL(solver.njev, 1); + LM_CHECK_N_ITERS(solver, 11, 1); // check norm VERIFY_IS_APPROX(solver.fvec.blueNorm(), 1.192636e-08); @@ -331,8 +335,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(solver.nfev, 11); - VERIFY_IS_EQUAL(solver.njev, 1); + LM_CHECK_N_ITERS(solver, 11, 1); // check norm VERIFY_IS_APPROX(solver.fvec.blueNorm(), 1.192636e-08); @@ -487,8 +490,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 6); - VERIFY_IS_EQUAL(lm.njev, 5); + LM_CHECK_N_ITERS(lm, 6, 5); // check norm VERIFY_IS_APPROX(lm.fvec.blueNorm(), 0.09063596); @@ -516,8 +518,7 @@ // check return values VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 6); - VERIFY_IS_EQUAL(lm.njev, 5); + LM_CHECK_N_ITERS(lm, 6, 5); // check norm fnorm = lm.fvec.blueNorm(); @@ -567,7 +568,7 @@ // do the computation lmdif_functor functor; - DenseIndex nfev; + DenseIndex nfev = -1; // initialize to avoid maybe-uninitialized warning info = LevenbergMarquardt<lmdif_functor>::lmdif1(functor, x, &nfev); // check return value @@ -688,8 +689,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 10); - VERIFY_IS_EQUAL(lm.njev, 8); + LM_CHECK_N_ITERS(lm, 10, 8); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.1304802941E+02); // check x @@ -709,8 +709,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 7); - VERIFY_IS_EQUAL(lm.njev, 6); + LM_CHECK_N_ITERS(lm, 7, 6); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.1304802941E+02); // check x @@ -768,8 +767,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 19); - VERIFY_IS_EQUAL(lm.njev, 15); + LM_CHECK_N_ITERS(lm, 19, 15); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.2455138894E-01); // check x @@ -785,8 +783,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 5); - VERIFY_IS_EQUAL(lm.njev, 4); + LM_CHECK_N_ITERS(lm, 5, 4); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.2455138894E-01); // check x @@ -858,8 +855,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 11); - VERIFY_IS_EQUAL(lm.njev, 10); + LM_CHECK_N_ITERS(lm, 11, 10); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.5324382854E+00); // check x @@ -880,8 +876,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 11); - VERIFY_IS_EQUAL(lm.njev, 10); + LM_CHECK_N_ITERS(lm, 11, 10); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.5324382854E+00); // check x @@ -944,8 +939,7 @@ // check return value VERIFY_IS_EQUAL(info, 3); - VERIFY_IS_EQUAL(lm.nfev, 9); - VERIFY_IS_EQUAL(lm.njev, 7); + LM_CHECK_N_ITERS(lm, 9, 7); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.6419295283E-02); // check x @@ -961,8 +955,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 4); - VERIFY_IS_EQUAL(lm.njev, 3); + LM_CHECK_N_ITERS(lm, 4, 3); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.6419295283E-02); // check x @@ -1022,8 +1015,7 @@ // check return value VERIFY_IS_EQUAL(info, 2); - VERIFY_IS_EQUAL(lm.nfev, 79); - VERIFY_IS_EQUAL(lm.njev, 72); + LM_CHECK_N_ITERS(lm, 79, 72); // check norm^2 std::cout.precision(30); std::cout << lm.fvec.squaredNorm() << "\n"; @@ -1045,8 +1037,7 @@ // check return value VERIFY_IS_EQUAL(info, 2); - VERIFY_IS_EQUAL(lm.nfev, 9); - VERIFY_IS_EQUAL(lm.njev, 8); + LM_CHECK_N_ITERS(lm, 9, 8); // check norm^2 VERIFY(lm.fvec.squaredNorm() <= 1.4307867721E-25); // check x @@ -1110,8 +1101,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 10); - VERIFY_IS_EQUAL(lm.njev, 8); + LM_CHECK_N_ITERS(lm, 10, 8); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.0565229338E+00); // check x @@ -1128,8 +1118,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 6); - VERIFY_IS_EQUAL(lm.njev, 5); + LM_CHECK_N_ITERS(lm, 6, 5); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.0565229338E+00); // check x @@ -1188,8 +1177,7 @@ // check return value VERIFY_IS_EQUAL(info, 2); - VERIFY_IS_EQUAL(lm.nfev, 284 ); - VERIFY_IS_EQUAL(lm.njev, 249 ); + LM_CHECK_N_ITERS(lm, 284, 249); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.7945855171E+01); // check x @@ -1206,8 +1194,7 @@ // check return value VERIFY_IS_EQUAL(info, 3); - VERIFY_IS_EQUAL(lm.nfev, 126); - VERIFY_IS_EQUAL(lm.njev, 116); + LM_CHECK_N_ITERS(lm, 126, 116); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.7945855171E+01); // check x @@ -1267,8 +1254,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY(lm.nfev < 31); // 31 - VERIFY(lm.njev < 25); // 25 + LM_CHECK_N_ITERS(lm, 31, 25); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.1680088766E+03); // check x @@ -1286,9 +1272,8 @@ info = lm.minimize(x); // check return value - VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 15 ); - VERIFY_IS_EQUAL(lm.njev, 14 ); + VERIFY_IS_EQUAL(info, 1); + LM_CHECK_N_ITERS(lm, 15, 14); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.1680088766E+03); // check x @@ -1358,12 +1343,7 @@ // check return value VERIFY_IS_EQUAL(info, 2); - ++g_test_level; - VERIFY_IS_EQUAL(lm.nfev, 602); // 602 - VERIFY_IS_EQUAL(lm.njev, 545); // 545 - --g_test_level; - VERIFY(lm.nfev < 602 * LM_EVAL_COUNT_TOL); - VERIFY(lm.njev < 545 * LM_EVAL_COUNT_TOL); + LM_CHECK_N_ITERS(lm, 602, 545); /* * Second try @@ -1375,8 +1355,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 18); - VERIFY_IS_EQUAL(lm.njev, 15); + LM_CHECK_N_ITERS(lm, 18, 15); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.4648946975E-05); // check x @@ -1440,9 +1419,8 @@ info = lm.minimize(x); // check return value - VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 490 ); - VERIFY_IS_EQUAL(lm.njev, 376 ); + VERIFY_IS_EQUAL(info, 1); + LM_CHECK_N_ITERS(lm, 490, 376); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 3.0750560385E-04); // check x @@ -1461,8 +1439,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 18); - VERIFY_IS_EQUAL(lm.njev, 16); + LM_CHECK_N_ITERS(lm, 18, 16); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 3.0750560385E-04); // check x @@ -1527,8 +1504,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 758); - VERIFY_IS_EQUAL(lm.njev, 744); + LM_CHECK_N_ITERS(lm, 758, 744); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.2404744073E-04); // check x @@ -1545,8 +1521,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 203); - VERIFY_IS_EQUAL(lm.njev, 192); + LM_CHECK_N_ITERS(lm, 203, 192); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.2404744073E-04); // check x @@ -1615,8 +1590,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 39); - VERIFY_IS_EQUAL(lm.njev, 36); + LM_CHECK_N_ITERS(lm, 39,36); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.6427082397E+03); // check x @@ -1640,8 +1614,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 29); - VERIFY_IS_EQUAL(lm.njev, 28); + LM_CHECK_N_ITERS(lm, 29, 28); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 5.6427082397E+03); // check x @@ -1707,8 +1680,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 27); - VERIFY_IS_EQUAL(lm.njev, 20); + LM_CHECK_N_ITERS(lm, 27, 20); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.7864049080E+03); // check x @@ -1729,8 +1701,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 9); - VERIFY_IS_EQUAL(lm.njev, 8); + LM_CHECK_N_ITERS(lm, 9, 8); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 8.7864049080E+03); // check x @@ -1792,8 +1763,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 18); - VERIFY_IS_EQUAL(lm.njev, 15); + LM_CHECK_N_ITERS(lm, 18, 15); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.4635887487E-03); // check x @@ -1810,8 +1780,7 @@ // check return value VERIFY_IS_EQUAL(info, 1); - VERIFY_IS_EQUAL(lm.nfev, 7); - VERIFY_IS_EQUAL(lm.njev, 6); + LM_CHECK_N_ITERS(lm, 7, 6); // check norm^2 VERIFY_IS_APPROX(lm.fvec.squaredNorm(), 1.4635887487E-03); // check x @@ -1820,7 +1789,7 @@ VERIFY_IS_APPROX(x[2], 4.5154121844E+02); } -void test_NonLinearOptimization() +EIGEN_DECLARE_TEST(NonLinearOptimization) { // Tests using the examples provided by (c)minpack CALL_SUBTEST/*_1*/(testChkder());
diff --git a/unsupported/test/NumericalDiff.cpp b/unsupported/test/NumericalDiff.cpp index 27d8880..6d83641 100644 --- a/unsupported/test/NumericalDiff.cpp +++ b/unsupported/test/NumericalDiff.cpp
@@ -24,7 +24,7 @@ int m_inputs, m_values; Functor() : m_inputs(InputsAtCompileTime), m_values(ValuesAtCompileTime) {} - Functor(int inputs, int values) : m_inputs(inputs), m_values(values) {} + Functor(int inputs_, int values_) : m_inputs(inputs_), m_values(values_) {} int inputs() const { return m_inputs; } int values() const { return m_values; } @@ -107,7 +107,7 @@ VERIFY_IS_APPROX(jac, actual_jac); } -void test_NumericalDiff() +EIGEN_DECLARE_TEST(NumericalDiff) { CALL_SUBTEST(test_forward()); CALL_SUBTEST(test_central());
diff --git a/unsupported/test/alignedvector3.cpp b/unsupported/test/alignedvector3.cpp index 252cb1d..fcc89da 100644 --- a/unsupported/test/alignedvector3.cpp +++ b/unsupported/test/alignedvector3.cpp
@@ -76,7 +76,7 @@ VERIFY(ss1.str()==ss2.str()); } -void test_alignedvector3() +EIGEN_DECLARE_TEST(alignedvector3) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST( alignedvector3<float>() );
diff --git a/unsupported/test/autodiff.cpp b/unsupported/test/autodiff.cpp index 374f86d..bafea6a 100644 --- a/unsupported/test/autodiff.cpp +++ b/unsupported/test/autodiff.cpp
@@ -16,7 +16,8 @@ using namespace std; // return x+std::sin(y); EIGEN_ASM_COMMENT("mybegin"); - return static_cast<Scalar>(x*2 - 1 + pow(1+x,2) + 2*sqrt(y*y+0) - 4 * sin(0+x) + 2 * cos(y+0) - exp(-0.5*x*x+0)); + // pow(float, int) promotes to pow(double, double) + return x*2 - 1 + static_cast<Scalar>(pow(1+x,2)) + 2*sqrt(y*y+0) - 4 * sin(0+x) + 2 * cos(y+0) - exp(Scalar(-0.5)*x*x+0); //return x+2*y*x;//x*2 -std::pow(x,2);//(2*y/x);// - y*2; EIGEN_ASM_COMMENT("myend"); } @@ -43,7 +44,7 @@ int m_inputs, m_values; TestFunc1() : m_inputs(InputsAtCompileTime), m_values(ValuesAtCompileTime) {} - TestFunc1(int inputs, int values) : m_inputs(inputs), m_values(values) {} + TestFunc1(int inputs_, int values_) : m_inputs(inputs_), m_values(values_) {} int inputs() const { return m_inputs; } int values() const { return m_values; } @@ -104,6 +105,89 @@ } }; + +#if EIGEN_HAS_VARIADIC_TEMPLATES +/* Test functor for the C++11 features. */ +template <typename Scalar> +struct integratorFunctor +{ + typedef Matrix<Scalar, 2, 1> InputType; + typedef Matrix<Scalar, 2, 1> ValueType; + + /* + * Implementation starts here. + */ + integratorFunctor(const Scalar gain) : _gain(gain) {} + integratorFunctor(const integratorFunctor& f) : _gain(f._gain) {} + const Scalar _gain; + + template <typename T1, typename T2> + void operator() (const T1 &input, T2 *output, const Scalar dt) const + { + T2 &o = *output; + + /* Integrator to test the AD. */ + o[0] = input[0] + input[1] * dt * _gain; + o[1] = input[1] * _gain; + } + + /* Only needed for the test */ + template <typename T1, typename T2, typename T3> + void operator() (const T1 &input, T2 *output, T3 *jacobian, const Scalar dt) const + { + T2 &o = *output; + + /* Integrator to test the AD. */ + o[0] = input[0] + input[1] * dt * _gain; + o[1] = input[1] * _gain; + + if (jacobian) + { + T3 &j = *jacobian; + + j(0, 0) = 1; + j(0, 1) = dt * _gain; + j(1, 0) = 0; + j(1, 1) = _gain; + } + } + +}; + +template<typename Func> void forward_jacobian_cpp11(const Func& f) +{ + typedef typename Func::ValueType::Scalar Scalar; + typedef typename Func::ValueType ValueType; + typedef typename Func::InputType InputType; + typedef typename AutoDiffJacobian<Func>::JacobianType JacobianType; + + InputType x = InputType::Random(InputType::RowsAtCompileTime); + ValueType y, yref; + JacobianType j, jref; + + const Scalar dt = internal::random<double>(); + + jref.setZero(); + yref.setZero(); + f(x, &yref, &jref, dt); + + //std::cerr << "y, yref, jref: " << "\n"; + //std::cerr << y.transpose() << "\n\n"; + //std::cerr << yref << "\n\n"; + //std::cerr << jref << "\n\n"; + + AutoDiffJacobian<Func> autoj(f); + autoj(x, &y, &j, dt); + + //std::cerr << "y j (via autodiff): " << "\n"; + //std::cerr << y.transpose() << "\n\n"; + //std::cerr << j << "\n\n"; + + VERIFY_IS_APPROX(y, yref); + VERIFY_IS_APPROX(j, jref); +} +#endif + template<typename Func> void forward_jacobian(const Func& f) { typename Func::InputType x = Func::InputType::Random(f.inputs()); @@ -127,7 +211,6 @@ VERIFY_IS_APPROX(j, jref); } - // TODO also check actual derivatives! template <int> void test_autodiff_scalar() @@ -140,6 +223,7 @@ VERIFY_IS_APPROX(res.value(), foo(p.x(),p.y())); } + // TODO also check actual derivatives! template <int> void test_autodiff_vector() @@ -150,7 +234,7 @@ VectorAD ap = p.cast<AD>(); ap.x().derivatives() = Vector2f::UnitX(); ap.y().derivatives() = Vector2f::UnitY(); - + AD res = foo<VectorAD>(ap); VERIFY_IS_APPROX(res.value(), foo(p)); } @@ -163,6 +247,9 @@ CALL_SUBTEST(( forward_jacobian(TestFunc1<double,3,2>()) )); CALL_SUBTEST(( forward_jacobian(TestFunc1<double,3,3>()) )); CALL_SUBTEST(( forward_jacobian(TestFunc1<double>(3,3)) )); +#if EIGEN_HAS_VARIADIC_TEMPLATES + CALL_SUBTEST(( forward_jacobian_cpp11(integratorFunctor<double>(10)) )); +#endif } @@ -204,11 +291,70 @@ VERIFY_IS_APPROX(y.value().derivatives()(1), s4*std::cos(s1*s3+s2*s4)); VERIFY_IS_APPROX(y.derivatives()(0).derivatives(), -std::sin(s1*s3+s2*s4)*Vector2d(s3*s3,s4*s3)); VERIFY_IS_APPROX(y.derivatives()(1).derivatives(), -std::sin(s1*s3+s2*s4)*Vector2d(s3*s4,s4*s4)); + + ADD z = x(0)*x(1); + VERIFY_IS_APPROX(z.derivatives()(0).derivatives(), Vector2d(0,1)); + VERIFY_IS_APPROX(z.derivatives()(1).derivatives(), Vector2d(1,0)); } +double bug_1222() { + typedef Eigen::AutoDiffScalar<Eigen::Vector3d> AD; + const double _cv1_3 = 1.0; + const AD chi_3 = 1.0; + // this line did not work, because operator+ returns ADS<DerType&>, which then cannot be converted to ADS<DerType> + const AD denom = chi_3 + _cv1_3; + return denom.value(); +} +#ifdef EIGEN_TEST_PART_5 -void test_autodiff() +double bug_1223() { + using std::min; + typedef Eigen::AutoDiffScalar<Eigen::Vector3d> AD; + + const double _cv1_3 = 1.0; + const AD chi_3 = 1.0; + const AD denom = 1.0; + + // failed because implementation of min attempts to construct ADS<DerType&> via constructor AutoDiffScalar(const Real& value) + // without initializing m_derivatives (which is a reference in this case) + #define EIGEN_TEST_SPACE + const AD t = min EIGEN_TEST_SPACE (denom / chi_3, 1.0); + + const AD t2 = min EIGEN_TEST_SPACE (denom / (chi_3 * _cv1_3), 1.0); + + return t.value() + t2.value(); +} + +// regression test for some compilation issues with specializations of ScalarBinaryOpTraits +void bug_1260() { + Matrix4d A = Matrix4d::Ones(); + Vector4d v = Vector4d::Ones(); + A*v; +} + +// check a compilation issue with numext::max +double bug_1261() { + typedef AutoDiffScalar<Matrix2d> AD; + typedef Matrix<AD,2,1> VectorAD; + + VectorAD v(0.,0.); + const AD maxVal = v.maxCoeff(); + const AD minVal = v.minCoeff(); + return maxVal.value() + minVal.value(); +} + +double bug_1264() { + typedef AutoDiffScalar<Vector2d> AD; + const AD s = 0.; + const Matrix<AD, 3, 1> v1(0.,0.,0.); + const Matrix<AD, 3, 1> v2 = (s + 3.0) * v1; + return v2(0).value(); +} + +#endif + +EIGEN_DECLARE_TEST(autodiff) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( test_autodiff_scalar<1>() ); @@ -216,5 +362,10 @@ CALL_SUBTEST_3( test_autodiff_jacobian<1>() ); CALL_SUBTEST_4( test_autodiff_hessian<1>() ); } + + CALL_SUBTEST_5( bug_1222() ); + CALL_SUBTEST_5( bug_1223() ); + CALL_SUBTEST_5( bug_1260() ); + CALL_SUBTEST_5( bug_1261() ); }
diff --git a/unsupported/test/autodiff_scalar.cpp b/unsupported/test/autodiff_scalar.cpp index c631c73..e81a778 100644 --- a/unsupported/test/autodiff_scalar.cpp +++ b/unsupported/test/autodiff_scalar.cpp
@@ -36,13 +36,66 @@ VERIFY_IS_APPROX(res.derivatives(), x.derivatives()); } +template<typename Scalar> void check_hyperbolic_functions() +{ + using std::sinh; + using std::cosh; + using std::tanh; + typedef Matrix<Scalar, 1, 1> Deriv1; + typedef AutoDiffScalar<Deriv1> AD; + Deriv1 p = Deriv1::Random(); + AD val(p.x(),Deriv1::UnitX()); + Scalar cosh_px = std::cosh(p.x()); + AD res1 = tanh(val); + VERIFY_IS_APPROX(res1.value(), std::tanh(p.x())); + VERIFY_IS_APPROX(res1.derivatives().x(), Scalar(1.0) / (cosh_px * cosh_px)); + AD res2 = sinh(val); + VERIFY_IS_APPROX(res2.value(), std::sinh(p.x())); + VERIFY_IS_APPROX(res2.derivatives().x(), cosh_px); -void test_autodiff_scalar() + AD res3 = cosh(val); + VERIFY_IS_APPROX(res3.value(), cosh_px); + VERIFY_IS_APPROX(res3.derivatives().x(), std::sinh(p.x())); + + // Check constant values. + const Scalar sample_point = Scalar(1) / Scalar(3); + val = AD(sample_point,Deriv1::UnitX()); + res1 = tanh(val); + VERIFY_IS_APPROX(res1.derivatives().x(), Scalar(0.896629559604914)); + + res2 = sinh(val); + VERIFY_IS_APPROX(res2.derivatives().x(), Scalar(1.056071867829939)); + + res3 = cosh(val); + VERIFY_IS_APPROX(res3.derivatives().x(), Scalar(0.339540557256150)); +} + +template <typename Scalar> +void check_limits_specialization() +{ + typedef Eigen::Matrix<Scalar, 1, 1> Deriv; + typedef Eigen::AutoDiffScalar<Deriv> AD; + + typedef std::numeric_limits<AD> A; + typedef std::numeric_limits<Scalar> B; + + // workaround "unused typedef" warning: + VERIFY(!bool(internal::is_same<B, A>::value)); + +#if EIGEN_HAS_CXX11 + VERIFY(bool(std::is_base_of<B, A>::value)); +#endif +} + +EIGEN_DECLARE_TEST(autodiff_scalar) { for(int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1( check_atan2<float>() ); CALL_SUBTEST_2( check_atan2<double>() ); + CALL_SUBTEST_3( check_hyperbolic_functions<float>() ); + CALL_SUBTEST_4( check_hyperbolic_functions<double>() ); + CALL_SUBTEST_5( check_limits_specialization<double>()); } }
diff --git a/unsupported/test/cxx11_eventcount.cpp b/unsupported/test/cxx11_eventcount.cpp index f16cc6f..2f14186 100644 --- a/unsupported/test/cxx11_eventcount.cpp +++ b/unsupported/test/cxx11_eventcount.cpp
@@ -25,7 +25,8 @@ static void test_basic_eventcount() { - std::vector<EventCount::Waiter> waiters(1); + MaxSizeVector<EventCount::Waiter> waiters(1); + waiters.resize(1); EventCount ec(waiters); EventCount::Waiter& w = waiters[0]; ec.Notify(false); @@ -81,7 +82,8 @@ static const int kEvents = 1 << 16; static const int kQueues = 10; - std::vector<EventCount::Waiter> waiters(kThreads); + MaxSizeVector<EventCount::Waiter> waiters(kThreads); + waiters.resize(kThreads); EventCount ec(waiters); TestQueue queues[kQueues]; @@ -133,7 +135,7 @@ } } -void test_cxx11_eventcount() +EIGEN_DECLARE_TEST(cxx11_eventcount) { CALL_SUBTEST(test_basic_eventcount()); CALL_SUBTEST(test_stress_eventcount());
diff --git a/unsupported/test/cxx11_float16.cpp b/unsupported/test/cxx11_float16.cpp deleted file mode 100644 index 273dcbc..0000000 --- a/unsupported/test/cxx11_float16.cpp +++ /dev/null
@@ -1,192 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_float16 - -#include "main.h" -#include <Eigen/src/Core/arch/CUDA/Half.h> - -using Eigen::half; - -void test_conversion() -{ - // Conversion from float. - VERIFY_IS_EQUAL(half(1.0f).x, 0x3c00); - VERIFY_IS_EQUAL(half(0.5f).x, 0x3800); - VERIFY_IS_EQUAL(half(0.33333f).x, 0x3555); - VERIFY_IS_EQUAL(half(0.0f).x, 0x0000); - VERIFY_IS_EQUAL(half(-0.0f).x, 0x8000); - VERIFY_IS_EQUAL(half(65504.0f).x, 0x7bff); - VERIFY_IS_EQUAL(half(65536.0f).x, 0x7c00); // Becomes infinity. - - // Denormals. - VERIFY_IS_EQUAL(half(-5.96046e-08f).x, 0x8001); - VERIFY_IS_EQUAL(half(5.96046e-08f).x, 0x0001); - VERIFY_IS_EQUAL(half(1.19209e-07f).x, 0x0002); - - // Verify round-to-nearest-even behavior. - float val1 = float(half(__half{0x3c00})); - float val2 = float(half(__half{0x3c01})); - float val3 = float(half(__half{0x3c02})); - VERIFY_IS_EQUAL(half(0.5 * (val1 + val2)).x, 0x3c00); - VERIFY_IS_EQUAL(half(0.5 * (val2 + val3)).x, 0x3c02); - - // Conversion from int. - VERIFY_IS_EQUAL(half(-1).x, 0xbc00); - VERIFY_IS_EQUAL(half(0).x, 0x0000); - VERIFY_IS_EQUAL(half(1).x, 0x3c00); - VERIFY_IS_EQUAL(half(2).x, 0x4000); - VERIFY_IS_EQUAL(half(3).x, 0x4200); - - // Conversion from bool. - VERIFY_IS_EQUAL(half(false).x, 0x0000); - VERIFY_IS_EQUAL(half(true).x, 0x3c00); - - // Conversion to float. - VERIFY_IS_EQUAL(float(half(__half{0x0000})), 0.0f); - VERIFY_IS_EQUAL(float(half(__half{0x3c00})), 1.0f); - - // Denormals. - VERIFY_IS_APPROX(float(half(__half{0x8001})), -5.96046e-08f); - VERIFY_IS_APPROX(float(half(__half{0x0001})), 5.96046e-08f); - VERIFY_IS_APPROX(float(half(__half{0x0002})), 1.19209e-07f); - - // NaNs and infinities. - VERIFY(!(numext::isinf)(float(half(65504.0f)))); // Largest finite number. - VERIFY(!(numext::isnan)(float(half(0.0f)))); - VERIFY((numext::isinf)(float(half(__half{0xfc00})))); - VERIFY((numext::isnan)(float(half(__half{0xfc01})))); - VERIFY((numext::isinf)(float(half(__half{0x7c00})))); - VERIFY((numext::isnan)(float(half(__half{0x7c01})))); - -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY((numext::isnan)(float(half(0.0 / 0.0)))); - VERIFY((numext::isinf)(float(half(1.0 / 0.0)))); - VERIFY((numext::isinf)(float(half(-1.0 / 0.0)))); -#endif - - // Exactly same checks as above, just directly on the half representation. - VERIFY(!(numext::isinf)(half(__half{0x7bff}))); - VERIFY(!(numext::isnan)(half(__half{0x0000}))); - VERIFY((numext::isinf)(half(__half{0xfc00}))); - VERIFY((numext::isnan)(half(__half{0xfc01}))); - VERIFY((numext::isinf)(half(__half{0x7c00}))); - VERIFY((numext::isnan)(half(__half{0x7c01}))); - -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY((numext::isnan)(half(0.0 / 0.0))); - VERIFY((numext::isinf)(half(1.0 / 0.0))); - VERIFY((numext::isinf)(half(-1.0 / 0.0))); -#endif -} - -void test_arithmetic() -{ - VERIFY_IS_EQUAL(float(half(2) + half(2)), 4); - VERIFY_IS_EQUAL(float(half(2) + half(-2)), 0); - VERIFY_IS_APPROX(float(half(0.33333f) + half(0.66667f)), 1.0f); - VERIFY_IS_EQUAL(float(half(2.0f) * half(-5.5f)), -11.0f); - VERIFY_IS_APPROX(float(half(1.0f) / half(3.0f)), 0.33333f); - VERIFY_IS_EQUAL(float(-half(4096.0f)), -4096.0f); - VERIFY_IS_EQUAL(float(-half(-4096.0f)), 4096.0f); -} - -void test_comparison() -{ - VERIFY(half(1.0f) > half(0.5f)); - VERIFY(half(0.5f) < half(1.0f)); - VERIFY(!(half(1.0f) < half(0.5f))); - VERIFY(!(half(0.5f) > half(1.0f))); - - VERIFY(!(half(4.0f) > half(4.0f))); - VERIFY(!(half(4.0f) < half(4.0f))); - - VERIFY(!(half(0.0f) < half(-0.0f))); - VERIFY(!(half(-0.0f) < half(0.0f))); - VERIFY(!(half(0.0f) > half(-0.0f))); - VERIFY(!(half(-0.0f) > half(0.0f))); - - VERIFY(half(0.2f) > half(-1.0f)); - VERIFY(half(-1.0f) < half(0.2f)); - VERIFY(half(-16.0f) < half(-15.0f)); - - VERIFY(half(1.0f) == half(1.0f)); - VERIFY(half(1.0f) != half(2.0f)); - - // Comparisons with NaNs and infinities. -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY(!(half(0.0 / 0.0) == half(0.0 / 0.0))); - VERIFY(half(0.0 / 0.0) != half(0.0 / 0.0)); - - VERIFY(!(half(1.0) == half(0.0 / 0.0))); - VERIFY(!(half(1.0) < half(0.0 / 0.0))); - VERIFY(!(half(1.0) > half(0.0 / 0.0))); - VERIFY(half(1.0) != half(0.0 / 0.0)); - - VERIFY(half(1.0) < half(1.0 / 0.0)); - VERIFY(half(1.0) > half(-1.0 / 0.0)); -#endif -} - -void test_basic_functions() -{ - VERIFY_IS_EQUAL(float(numext::abs(half(3.5f))), 3.5f); - VERIFY_IS_EQUAL(float(numext::abs(half(-3.5f))), 3.5f); - - VERIFY_IS_EQUAL(float(numext::floor(half(3.5f))), 3.0f); - VERIFY_IS_EQUAL(float(numext::floor(half(-3.5f))), -4.0f); - - VERIFY_IS_EQUAL(float(numext::ceil(half(3.5f))), 4.0f); - VERIFY_IS_EQUAL(float(numext::ceil(half(-3.5f))), -3.0f); - - VERIFY_IS_APPROX(float(numext::sqrt(half(0.0f))), 0.0f); - VERIFY_IS_APPROX(float(numext::sqrt(half(4.0f))), 2.0f); - - VERIFY_IS_APPROX(float(numext::pow(half(0.0f), half(1.0f))), 0.0f); - VERIFY_IS_APPROX(float(numext::pow(half(2.0f), half(2.0f))), 4.0f); - - VERIFY_IS_EQUAL(float(numext::exp(half(0.0f))), 1.0f); - VERIFY_IS_APPROX(float(numext::exp(half(EIGEN_PI))), float(20.0 + EIGEN_PI)); - - VERIFY_IS_EQUAL(float(numext::log(half(1.0f))), 0.0f); - VERIFY_IS_APPROX(float(numext::log(half(10.0f))), 2.30273f); -} - -void test_trigonometric_functions() -{ - VERIFY_IS_APPROX(numext::cos(half(0.0f)), half(cosf(0.0f))); - VERIFY_IS_APPROX(numext::cos(half(EIGEN_PI)), half(cosf(EIGEN_PI))); - //VERIFY_IS_APPROX(numext::cos(half(EIGEN_PI/2)), half(cosf(EIGEN_PI/2))); - //VERIFY_IS_APPROX(numext::cos(half(3*EIGEN_PI/2)), half(cosf(3*EIGEN_PI/2))); - VERIFY_IS_APPROX(numext::cos(half(3.5f)), half(cosf(3.5f))); - - VERIFY_IS_APPROX(numext::sin(half(0.0f)), half(sinf(0.0f))); - // VERIFY_IS_APPROX(numext::sin(half(EIGEN_PI)), half(sinf(EIGEN_PI))); - VERIFY_IS_APPROX(numext::sin(half(EIGEN_PI/2)), half(sinf(EIGEN_PI/2))); - VERIFY_IS_APPROX(numext::sin(half(3*EIGEN_PI/2)), half(sinf(3*EIGEN_PI/2))); - VERIFY_IS_APPROX(numext::sin(half(3.5f)), half(sinf(3.5f))); - - VERIFY_IS_APPROX(numext::tan(half(0.0f)), half(tanf(0.0f))); - // VERIFY_IS_APPROX(numext::tan(half(EIGEN_PI)), half(tanf(EIGEN_PI))); - // VERIFY_IS_APPROX(numext::tan(half(EIGEN_PI/2)), half(tanf(EIGEN_PI/2))); - //VERIFY_IS_APPROX(numext::tan(half(3*EIGEN_PI/2)), half(tanf(3*EIGEN_PI/2))); - VERIFY_IS_APPROX(numext::tan(half(3.5f)), half(tanf(3.5f))); -} - -void test_cxx11_float16() -{ - CALL_SUBTEST(test_conversion()); - CALL_SUBTEST(test_arithmetic()); - CALL_SUBTEST(test_comparison()); - CALL_SUBTEST(test_basic_functions()); - CALL_SUBTEST(test_trigonometric_functions()); -}
diff --git a/unsupported/test/cxx11_maxsizevector.cpp b/unsupported/test/cxx11_maxsizevector.cpp new file mode 100644 index 0000000..46b689a --- /dev/null +++ b/unsupported/test/cxx11_maxsizevector.cpp
@@ -0,0 +1,77 @@ +#include "main.h" + +#include <exception> // std::exception + +#include <unsupported/Eigen/CXX11/Tensor> + +struct Foo +{ + static Index object_count; + static Index object_limit; + EIGEN_ALIGN_TO_BOUNDARY(128) int dummy; + + Foo(int x=0) : dummy(x) + { +#ifdef EIGEN_EXCEPTIONS + // TODO: Is this the correct way to handle this? + if (Foo::object_count > Foo::object_limit) { std::cout << "\nThrow!\n"; throw Foo::Fail(); } +#endif + std::cout << '+'; + ++Foo::object_count; + eigen_assert((internal::UIntPtr(this) & (127)) == 0); + } + Foo(const Foo&) + { + std::cout << 'c'; + ++Foo::object_count; + eigen_assert((internal::UIntPtr(this) & (127)) == 0); + } + + ~Foo() + { + std::cout << '~'; + --Foo::object_count; + } + + class Fail : public std::exception {}; +}; + +Index Foo::object_count = 0; +Index Foo::object_limit = 0; + + + +EIGEN_DECLARE_TEST(cxx11_maxsizevector) +{ + typedef MaxSizeVector<Foo> VectorX; + Foo::object_count = 0; + for(int r = 0; r < g_repeat; r++) { + Index rows = internal::random<Index>(3,30); + Foo::object_limit = internal::random<Index>(0, rows - 2); + std::cout << "object_limit = " << Foo::object_limit << std::endl; + bool exception_raised = false; +#ifdef EIGEN_EXCEPTIONS + try + { +#endif + std::cout << "\nVectorX m(" << rows << ");\n"; + VectorX vect(rows); + for(int i=0; i<rows; ++i) + vect.push_back(Foo()); +#ifdef EIGEN_EXCEPTIONS + VERIFY(false); // not reached if exceptions are enabled + } + catch (const Foo::Fail&) { exception_raised = true; } + VERIFY(exception_raised); +#endif + VERIFY_IS_EQUAL(Index(0), Foo::object_count); + + { + Foo::object_limit = rows+1; + VectorX vect2(rows, Foo()); + VERIFY_IS_EQUAL(Foo::object_count, rows); + } + VERIFY_IS_EQUAL(Index(0), Foo::object_count); + std::cout << '\n'; + } +}
diff --git a/unsupported/test/cxx11_meta.cpp b/unsupported/test/cxx11_meta.cpp index ecac3ad..510e110 100644 --- a/unsupported/test/cxx11_meta.cpp +++ b/unsupported/test/cxx11_meta.cpp
@@ -10,7 +10,7 @@ #include "main.h" #include <array> -#include <Eigen/CXX11/Core> +#include <Eigen/CXX11/src/util/CXX11Meta.h> using Eigen::internal::is_same; using Eigen::internal::type_list; @@ -340,7 +340,7 @@ VERIFY_IS_EQUAL((instantiate_by_c_array<dummy_inst, int, 5>(data).c), 5); } -void test_cxx11_meta() +EIGEN_DECLARE_TEST(cxx11_meta) { CALL_SUBTEST(test_gen_numeric_list()); CALL_SUBTEST(test_concat());
diff --git a/unsupported/test/cxx11_non_blocking_thread_pool.cpp b/unsupported/test/cxx11_non_blocking_thread_pool.cpp new file mode 100644 index 0000000..90b330f --- /dev/null +++ b/unsupported/test/cxx11_non_blocking_thread_pool.cpp
@@ -0,0 +1,178 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Dmitry Vyukov <dvyukov@google.com> +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_USE_THREADS +#include "main.h" +#include "Eigen/CXX11/ThreadPool" +#include "Eigen/CXX11/Tensor" + +static void test_create_destroy_empty_pool() +{ + // Just create and destroy the pool. This will wind up and tear down worker + // threads. Ensure there are no issues in that logic. + for (int i = 0; i < 16; ++i) { + ThreadPool tp(i); + } +} + + +static void test_parallelism(bool allow_spinning) +{ + // Test we never-ever fail to match available tasks with idle threads. + const int kThreads = 16; // code below expects that this is a multiple of 4 + ThreadPool tp(kThreads, allow_spinning); + VERIFY_IS_EQUAL(tp.NumThreads(), kThreads); + VERIFY_IS_EQUAL(tp.CurrentThreadId(), -1); + for (int iter = 0; iter < 100; ++iter) { + std::atomic<int> running(0); + std::atomic<int> done(0); + std::atomic<int> phase(0); + // Schedule kThreads tasks and ensure that they all are running. + for (int i = 0; i < kThreads; ++i) { + tp.Schedule([&]() { + const int thread_id = tp.CurrentThreadId(); + VERIFY_GE(thread_id, 0); + VERIFY_LE(thread_id, kThreads - 1); + running++; + while (phase < 1) { + } + done++; + }); + } + while (running != kThreads) { + } + running = 0; + phase = 1; + // Now, while the previous tasks exit, schedule another kThreads tasks and + // ensure that they are running. + for (int i = 0; i < kThreads; ++i) { + tp.Schedule([&, i]() { + running++; + while (phase < 2) { + } + // When all tasks are running, half of tasks exit, quarter of tasks + // continue running and quarter of tasks schedule another 2 tasks each. + // Concurrently main thread schedules another quarter of tasks. + // This gives us another kThreads tasks and we ensure that they all + // are running. + if (i < kThreads / 2) { + } else if (i < 3 * kThreads / 4) { + running++; + while (phase < 3) { + } + done++; + } else { + for (int j = 0; j < 2; ++j) { + tp.Schedule([&]() { + running++; + while (phase < 3) { + } + done++; + }); + } + } + done++; + }); + } + while (running != kThreads) { + } + running = 0; + phase = 2; + for (int i = 0; i < kThreads / 4; ++i) { + tp.Schedule([&]() { + running++; + while (phase < 3) { + } + done++; + }); + } + while (running != kThreads) { + } + phase = 3; + while (done != 3 * kThreads) { + } + } +} + + +static void test_cancel() +{ + ThreadPool tp(2); + + // Schedule a large number of closure that each sleeps for one second. This + // will keep the thread pool busy for much longer than the default test timeout. + for (int i = 0; i < 1000; ++i) { + tp.Schedule([]() { EIGEN_SLEEP(2000); }); + } + + // Cancel the processing of all the closures that are still pending. + tp.Cancel(); +} + +static void test_pool_partitions() { + const int kThreads = 2; + ThreadPool tp(kThreads); + + // Assign each thread to its own partition, so that stealing other work only + // occurs globally when a thread is idle. + std::vector<std::pair<unsigned, unsigned>> steal_partitions(kThreads); + for (int i = 0; i < kThreads; ++i) { + steal_partitions[i] = std::make_pair(i, i + 1); + } + tp.SetStealPartitions(steal_partitions); + + std::atomic<int> running(0); + std::atomic<int> done(0); + std::atomic<int> phase(0); + + // Schedule kThreads tasks and ensure that they all are running. + for (int i = 0; i < kThreads; ++i) { + tp.Schedule([&]() { + const int thread_id = tp.CurrentThreadId(); + VERIFY_GE(thread_id, 0); + VERIFY_LE(thread_id, kThreads - 1); + ++running; + while (phase < 1) { + } + ++done; + }); + } + while (running != kThreads) { + } + // Schedule each closure to only run on thread 'i' and verify that it does. + for (int i = 0; i < kThreads; ++i) { + tp.ScheduleWithHint( + [&, i]() { + ++running; + const int thread_id = tp.CurrentThreadId(); + VERIFY_IS_EQUAL(thread_id, i); + while (phase < 2) { + } + ++done; + }, + i, i + 1); + } + running = 0; + phase = 1; + while (running != kThreads) { + } + running = 0; + phase = 2; +} + + +EIGEN_DECLARE_TEST(cxx11_non_blocking_thread_pool) +{ + CALL_SUBTEST(test_create_destroy_empty_pool()); + CALL_SUBTEST(test_parallelism(true)); + CALL_SUBTEST(test_parallelism(false)); + CALL_SUBTEST(test_cancel()); + CALL_SUBTEST(test_pool_partitions()); +}
diff --git a/unsupported/test/cxx11_runqueue.cpp b/unsupported/test/cxx11_runqueue.cpp index d1770ee..8fc5a30 100644 --- a/unsupported/test/cxx11_runqueue.cpp +++ b/unsupported/test/cxx11_runqueue.cpp
@@ -33,73 +33,81 @@ VERIFY_IS_EQUAL(0u, q.Size()); VERIFY_IS_EQUAL(0, q.PopFront()); std::vector<int> stolen; - VERIFY_IS_EQUAL(0, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(0u, q.PopBackHalf(&stolen)); VERIFY_IS_EQUAL(0u, stolen.size()); // Push one front, pop one front. VERIFY_IS_EQUAL(0, q.PushFront(1)); - VERIFY_IS_EQUAL(1, q.Size()); + VERIFY_IS_EQUAL(1u, q.Size()); VERIFY_IS_EQUAL(1, q.PopFront()); - VERIFY_IS_EQUAL(0, q.Size()); + VERIFY_IS_EQUAL(0u, q.Size()); // Push front to overflow. VERIFY_IS_EQUAL(0, q.PushFront(2)); - VERIFY_IS_EQUAL(1, q.Size()); + VERIFY_IS_EQUAL(1u, q.Size()); VERIFY_IS_EQUAL(0, q.PushFront(3)); - VERIFY_IS_EQUAL(2, q.Size()); + VERIFY_IS_EQUAL(2u, q.Size()); VERIFY_IS_EQUAL(0, q.PushFront(4)); - VERIFY_IS_EQUAL(3, q.Size()); + VERIFY_IS_EQUAL(3u, q.Size()); VERIFY_IS_EQUAL(0, q.PushFront(5)); - VERIFY_IS_EQUAL(4, q.Size()); + VERIFY_IS_EQUAL(4u, q.Size()); VERIFY_IS_EQUAL(6, q.PushFront(6)); - VERIFY_IS_EQUAL(4, q.Size()); + VERIFY_IS_EQUAL(4u, q.Size()); VERIFY_IS_EQUAL(5, q.PopFront()); - VERIFY_IS_EQUAL(3, q.Size()); + VERIFY_IS_EQUAL(3u, q.Size()); VERIFY_IS_EQUAL(4, q.PopFront()); - VERIFY_IS_EQUAL(2, q.Size()); + VERIFY_IS_EQUAL(2u, q.Size()); VERIFY_IS_EQUAL(3, q.PopFront()); - VERIFY_IS_EQUAL(1, q.Size()); + VERIFY_IS_EQUAL(1u, q.Size()); VERIFY_IS_EQUAL(2, q.PopFront()); - VERIFY_IS_EQUAL(0, q.Size()); + VERIFY_IS_EQUAL(0u, q.Size()); VERIFY_IS_EQUAL(0, q.PopFront()); // Push one back, pop one back. VERIFY_IS_EQUAL(0, q.PushBack(7)); - VERIFY_IS_EQUAL(1, q.Size()); - VERIFY_IS_EQUAL(1, q.PopBackHalf(&stolen)); - VERIFY_IS_EQUAL(1, stolen.size()); + VERIFY_IS_EQUAL(1u, q.Size()); + VERIFY_IS_EQUAL(1u, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(1u, stolen.size()); VERIFY_IS_EQUAL(7, stolen[0]); - VERIFY_IS_EQUAL(0, q.Size()); + VERIFY_IS_EQUAL(0u, q.Size()); stolen.clear(); // Push back to overflow. VERIFY_IS_EQUAL(0, q.PushBack(8)); - VERIFY_IS_EQUAL(1, q.Size()); + VERIFY_IS_EQUAL(1u, q.Size()); VERIFY_IS_EQUAL(0, q.PushBack(9)); - VERIFY_IS_EQUAL(2, q.Size()); + VERIFY_IS_EQUAL(2u, q.Size()); VERIFY_IS_EQUAL(0, q.PushBack(10)); - VERIFY_IS_EQUAL(3, q.Size()); + VERIFY_IS_EQUAL(3u, q.Size()); VERIFY_IS_EQUAL(0, q.PushBack(11)); - VERIFY_IS_EQUAL(4, q.Size()); + VERIFY_IS_EQUAL(4u, q.Size()); VERIFY_IS_EQUAL(12, q.PushBack(12)); - VERIFY_IS_EQUAL(4, q.Size()); + VERIFY_IS_EQUAL(4u, q.Size()); // Pop back in halves. - VERIFY_IS_EQUAL(2, q.PopBackHalf(&stolen)); - VERIFY_IS_EQUAL(2, stolen.size()); + VERIFY_IS_EQUAL(2u, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(2u, stolen.size()); VERIFY_IS_EQUAL(10, stolen[0]); VERIFY_IS_EQUAL(11, stolen[1]); - VERIFY_IS_EQUAL(2, q.Size()); + VERIFY_IS_EQUAL(2u, q.Size()); stolen.clear(); - VERIFY_IS_EQUAL(1, q.PopBackHalf(&stolen)); - VERIFY_IS_EQUAL(1, stolen.size()); + VERIFY_IS_EQUAL(1u, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(1u, stolen.size()); VERIFY_IS_EQUAL(9, stolen[0]); - VERIFY_IS_EQUAL(1, q.Size()); + VERIFY_IS_EQUAL(1u, q.Size()); stolen.clear(); - VERIFY_IS_EQUAL(1, q.PopBackHalf(&stolen)); - VERIFY_IS_EQUAL(1, stolen.size()); + VERIFY_IS_EQUAL(1u, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(1u, stolen.size()); VERIFY_IS_EQUAL(8, stolen[0]); stolen.clear(); - VERIFY_IS_EQUAL(0, q.PopBackHalf(&stolen)); - VERIFY_IS_EQUAL(0, stolen.size()); + VERIFY_IS_EQUAL(0u, q.PopBackHalf(&stolen)); + VERIFY_IS_EQUAL(0u, stolen.size()); // Empty again. VERIFY(q.Empty()); - VERIFY_IS_EQUAL(0, q.Size()); + VERIFY_IS_EQUAL(0u, q.Size()); + VERIFY_IS_EQUAL(0, q.PushFront(1)); + VERIFY_IS_EQUAL(0, q.PushFront(2)); + VERIFY_IS_EQUAL(0, q.PushFront(3)); + VERIFY_IS_EQUAL(1, q.PopBack()); + VERIFY_IS_EQUAL(2, q.PopBack()); + VERIFY_IS_EQUAL(3, q.PopBack()); + VERIFY(q.Empty()); + VERIFY_IS_EQUAL(0u, q.Size()); } // Empty tests that the queue is not claimed to be empty when is is in fact not. @@ -130,7 +138,7 @@ stolen.clear(); break; } - VERIFY_IS_EQUAL(0, stolen.size()); + VERIFY_IS_EQUAL(0u, stolen.size()); } } } @@ -219,7 +227,7 @@ VERIFY(total.load() == 0); } -void test_cxx11_runqueue() +EIGEN_DECLARE_TEST(cxx11_runqueue) { CALL_SUBTEST_1(test_basic_runqueue()); CALL_SUBTEST_2(test_empty_runqueue());
diff --git a/unsupported/test/cxx11_tensor_argmax.cpp b/unsupported/test/cxx11_tensor_argmax.cpp index 482dfa7..4a0c896 100644 --- a/unsupported/test/cxx11_tensor_argmax.cpp +++ b/unsupported/test/cxx11_tensor_argmax.cpp
@@ -64,7 +64,7 @@ Tensor<Tuple<DenseIndex, float>, 0, DataLayout> reduced; DimensionList<DenseIndex, 4> dims; reduced = index_tuples.reduce( - dims, internal::ArgMaxTupleReducer<Tuple<DenseIndex, float>>()); + dims, internal::ArgMaxTupleReducer<Tuple<DenseIndex, float> >()); Tensor<float, 0, DataLayout> maxi = tensor.maximum(); @@ -74,7 +74,7 @@ for (int d = 0; d < 3; ++d) reduce_dims[d] = d; Tensor<Tuple<DenseIndex, float>, 1, DataLayout> reduced_by_dims(7); reduced_by_dims = index_tuples.reduce( - reduce_dims, internal::ArgMaxTupleReducer<Tuple<DenseIndex, float>>()); + reduce_dims, internal::ArgMaxTupleReducer<Tuple<DenseIndex, float> >()); Tensor<float, 1, DataLayout> max_by_dims = tensor.maximum(reduce_dims); @@ -96,7 +96,7 @@ Tensor<Tuple<DenseIndex, float>, 0, DataLayout> reduced; DimensionList<DenseIndex, 4> dims; reduced = index_tuples.reduce( - dims, internal::ArgMinTupleReducer<Tuple<DenseIndex, float>>()); + dims, internal::ArgMinTupleReducer<Tuple<DenseIndex, float> >()); Tensor<float, 0, DataLayout> mini = tensor.minimum(); @@ -106,7 +106,7 @@ for (int d = 0; d < 3; ++d) reduce_dims[d] = d; Tensor<Tuple<DenseIndex, float>, 1, DataLayout> reduced_by_dims(7); reduced_by_dims = index_tuples.reduce( - reduce_dims, internal::ArgMinTupleReducer<Tuple<DenseIndex, float>>()); + reduce_dims, internal::ArgMinTupleReducer<Tuple<DenseIndex, float> >()); Tensor<float, 1, DataLayout> min_by_dims = tensor.minimum(reduce_dims); @@ -273,7 +273,7 @@ } } -void test_cxx11_tensor_argmax() +EIGEN_DECLARE_TEST(cxx11_tensor_argmax) { CALL_SUBTEST(test_simple_index_tuples<RowMajor>()); CALL_SUBTEST(test_simple_index_tuples<ColMajor>());
diff --git a/unsupported/test/cxx11_tensor_argmax_cuda.cu b/unsupported/test/cxx11_tensor_argmax_gpu.cu similarity index 67% rename from unsupported/test/cxx11_tensor_argmax_cuda.cu rename to unsupported/test/cxx11_tensor_argmax_gpu.cu index 41ccbe9..79f4066 100644 --- a/unsupported/test/cxx11_tensor_argmax_cuda.cu +++ b/unsupported/test/cxx11_tensor_argmax_gpu.cu
@@ -9,16 +9,18 @@ #define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_FUNC cxx11_tensor_cuda + #define EIGEN_USE_GPU #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> +#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + using Eigen::Tensor; template <int Layout> -void test_cuda_simple_argmax() +void test_gpu_simple_argmax() { Tensor<double, 3, Layout> in(Eigen::array<DenseIndex, 3>(72,53,97)); Tensor<DenseIndex, 1, Layout> out_max(Eigen::array<DenseIndex, 1>(1)); @@ -34,13 +36,13 @@ double* d_in; DenseIndex* d_out_max; DenseIndex* d_out_min; - cudaMalloc((void**)(&d_in), in_bytes); - cudaMalloc((void**)(&d_out_max), out_bytes); - cudaMalloc((void**)(&d_out_min), out_bytes); + gpuMalloc((void**)(&d_in), in_bytes); + gpuMalloc((void**)(&d_out_max), out_bytes); + gpuMalloc((void**)(&d_out_min), out_bytes); - cudaMemcpy(d_in, in.data(), in_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in, in.data(), in_bytes, gpuMemcpyHostToDevice); - Eigen::CudaStreamDevice stream; + Eigen::GpuStreamDevice stream; Eigen::GpuDevice gpu_device(&stream); Eigen::TensorMap<Eigen::Tensor<double, 3, Layout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 3>(72,53,97)); @@ -50,20 +52,20 @@ gpu_out_max.device(gpu_device) = gpu_in.argmax(); gpu_out_min.device(gpu_device) = gpu_in.argmin(); - assert(cudaMemcpyAsync(out_max.data(), d_out_max, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaMemcpyAsync(out_min.data(), d_out_min, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + assert(gpuMemcpyAsync(out_max.data(), d_out_max, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuMemcpyAsync(out_min.data(), d_out_min, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); VERIFY_IS_EQUAL(out_max(Eigen::array<DenseIndex, 1>(0)), 72*53*97 - 1); VERIFY_IS_EQUAL(out_min(Eigen::array<DenseIndex, 1>(0)), 0); - cudaFree(d_in); - cudaFree(d_out_max); - cudaFree(d_out_min); + gpuFree(d_in); + gpuFree(d_out_max); + gpuFree(d_out_min); } template <int DataLayout> -void test_cuda_argmax_dim() +void test_gpu_argmax_dim() { Tensor<float, 4, DataLayout> tensor(2,3,5,7); std::vector<int> dims; @@ -97,12 +99,12 @@ float* d_in; DenseIndex* d_out; - cudaMalloc((void**)(&d_in), in_bytes); - cudaMalloc((void**)(&d_out), out_bytes); + gpuMalloc((void**)(&d_in), in_bytes); + gpuMalloc((void**)(&d_out), out_bytes); - cudaMemcpy(d_in, tensor.data(), in_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in, tensor.data(), in_bytes, gpuMemcpyHostToDevice); - Eigen::CudaStreamDevice stream; + Eigen::GpuStreamDevice stream; Eigen::GpuDevice gpu_device(&stream); Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 4>(2, 3, 5, 7)); @@ -110,13 +112,13 @@ gpu_out.device(gpu_device) = gpu_in.argmax(dim); - assert(cudaMemcpyAsync(tensor_arg.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + assert(gpuMemcpyAsync(tensor_arg.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); - VERIFY_IS_EQUAL(tensor_arg.dimensions().TotalSize(), + VERIFY_IS_EQUAL(tensor_arg.size(), size_t(2*3*5*7 / tensor.dimension(dim))); - for (size_t n = 0; n < tensor_arg.dimensions().TotalSize(); ++n) { + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { // Expect max to be in the first index of the reduced dimension VERIFY_IS_EQUAL(tensor_arg.data()[n], 0); } @@ -134,25 +136,25 @@ } } - cudaMemcpy(d_in, tensor.data(), in_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in, tensor.data(), in_bytes, gpuMemcpyHostToDevice); gpu_out.device(gpu_device) = gpu_in.argmax(dim); - assert(cudaMemcpyAsync(tensor_arg.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + assert(gpuMemcpyAsync(tensor_arg.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); - for (size_t n = 0; n < tensor_arg.dimensions().TotalSize(); ++n) { + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { // Expect max to be in the last index of the reduced dimension VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1); } - cudaFree(d_in); - cudaFree(d_out); + gpuFree(d_in); + gpuFree(d_out); } } template <int DataLayout> -void test_cuda_argmin_dim() +void test_gpu_argmin_dim() { Tensor<float, 4, DataLayout> tensor(2,3,5,7); std::vector<int> dims; @@ -186,12 +188,12 @@ float* d_in; DenseIndex* d_out; - cudaMalloc((void**)(&d_in), in_bytes); - cudaMalloc((void**)(&d_out), out_bytes); + gpuMalloc((void**)(&d_in), in_bytes); + gpuMalloc((void**)(&d_out), out_bytes); - cudaMemcpy(d_in, tensor.data(), in_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in, tensor.data(), in_bytes, gpuMemcpyHostToDevice); - Eigen::CudaStreamDevice stream; + Eigen::GpuStreamDevice stream; Eigen::GpuDevice gpu_device(&stream); Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 4>(2, 3, 5, 7)); @@ -199,13 +201,13 @@ gpu_out.device(gpu_device) = gpu_in.argmin(dim); - assert(cudaMemcpyAsync(tensor_arg.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + assert(gpuMemcpyAsync(tensor_arg.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); - VERIFY_IS_EQUAL(tensor_arg.dimensions().TotalSize(), - size_t(2*3*5*7 / tensor.dimension(dim))); + VERIFY_IS_EQUAL(tensor_arg.size(), + 2*3*5*7 / tensor.dimension(dim)); - for (size_t n = 0; n < tensor_arg.dimensions().TotalSize(); ++n) { + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { // Expect min to be in the first index of the reduced dimension VERIFY_IS_EQUAL(tensor_arg.data()[n], 0); } @@ -223,29 +225,29 @@ } } - cudaMemcpy(d_in, tensor.data(), in_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in, tensor.data(), in_bytes, gpuMemcpyHostToDevice); gpu_out.device(gpu_device) = gpu_in.argmin(dim); - assert(cudaMemcpyAsync(tensor_arg.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + assert(gpuMemcpyAsync(tensor_arg.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); - for (size_t n = 0; n < tensor_arg.dimensions().TotalSize(); ++n) { + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { // Expect max to be in the last index of the reduced dimension VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1); } - cudaFree(d_in); - cudaFree(d_out); + gpuFree(d_in); + gpuFree(d_out); } } -void test_cxx11_tensor_cuda() +EIGEN_DECLARE_TEST(cxx11_tensor_argmax_gpu) { - CALL_SUBTEST_1(test_cuda_simple_argmax<RowMajor>()); - CALL_SUBTEST_1(test_cuda_simple_argmax<ColMajor>()); - CALL_SUBTEST_2(test_cuda_argmax_dim<RowMajor>()); - CALL_SUBTEST_2(test_cuda_argmax_dim<ColMajor>()); - CALL_SUBTEST_3(test_cuda_argmin_dim<RowMajor>()); - CALL_SUBTEST_3(test_cuda_argmin_dim<ColMajor>()); + CALL_SUBTEST_1(test_gpu_simple_argmax<RowMajor>()); + CALL_SUBTEST_1(test_gpu_simple_argmax<ColMajor>()); + CALL_SUBTEST_2(test_gpu_argmax_dim<RowMajor>()); + CALL_SUBTEST_2(test_gpu_argmax_dim<ColMajor>()); + CALL_SUBTEST_3(test_gpu_argmin_dim<RowMajor>()); + CALL_SUBTEST_3(test_gpu_argmin_dim<ColMajor>()); }
diff --git a/unsupported/test/cxx11_tensor_argmax_sycl.cpp b/unsupported/test/cxx11_tensor_argmax_sycl.cpp new file mode 100644 index 0000000..0bbb0f6 --- /dev/null +++ b/unsupported/test/cxx11_tensor_argmax_sycl.cpp
@@ -0,0 +1,245 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +template <typename DataType, int Layout, typename DenseIndex> +static void test_sycl_simple_argmax(const Eigen::SyclDevice &sycl_device){ + + Tensor<DataType, 3, Layout, DenseIndex> in(Eigen::array<DenseIndex, 3>{{2,2,2}}); + Tensor<DenseIndex, 0, Layout, DenseIndex> out_max; + Tensor<DenseIndex, 0, Layout, DenseIndex> out_min; + in.setRandom(); + in *= in.constant(100.0); + in(0, 0, 0) = -1000.0; + in(1, 1, 1) = 1000.0; + + std::size_t in_bytes = in.size() * sizeof(DataType); + std::size_t out_bytes = out_max.size() * sizeof(DenseIndex); + + DataType * d_in = static_cast<DataType*>(sycl_device.allocate(in_bytes)); + DenseIndex* d_out_max = static_cast<DenseIndex*>(sycl_device.allocate(out_bytes)); + DenseIndex* d_out_min = static_cast<DenseIndex*>(sycl_device.allocate(out_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 3, Layout, DenseIndex> > gpu_in(d_in, Eigen::array<DenseIndex, 3>{{2,2,2}}); + Eigen::TensorMap<Eigen::Tensor<DenseIndex, 0, Layout, DenseIndex> > gpu_out_max(d_out_max); + Eigen::TensorMap<Eigen::Tensor<DenseIndex, 0, Layout, DenseIndex> > gpu_out_min(d_out_min); + sycl_device.memcpyHostToDevice(d_in, in.data(),in_bytes); + + gpu_out_max.device(sycl_device) = gpu_in.argmax(); + gpu_out_min.device(sycl_device) = gpu_in.argmin(); + + sycl_device.memcpyDeviceToHost(out_max.data(), d_out_max, out_bytes); + sycl_device.memcpyDeviceToHost(out_min.data(), d_out_min, out_bytes); + + VERIFY_IS_EQUAL(out_max(), 2*2*2 - 1); + VERIFY_IS_EQUAL(out_min(), 0); + + sycl_device.deallocate(d_in); + sycl_device.deallocate(d_out_max); + sycl_device.deallocate(d_out_min); +} + + +template <typename DataType, int DataLayout, typename DenseIndex> +static void test_sycl_argmax_dim(const Eigen::SyclDevice &sycl_device) +{ + DenseIndex sizeDim0=9; + DenseIndex sizeDim1=3; + DenseIndex sizeDim2=5; + DenseIndex sizeDim3=7; + Tensor<DataType, 4, DataLayout, DenseIndex> tensor(sizeDim0,sizeDim1,sizeDim2,sizeDim3); + + std::vector<DenseIndex> dims; + dims.push_back(sizeDim0); dims.push_back(sizeDim1); dims.push_back(sizeDim2); dims.push_back(sizeDim3); + for (DenseIndex dim = 0; dim < 4; ++dim) { + + array<DenseIndex, 3> out_shape; + for (DenseIndex d = 0; d < 3; ++d) out_shape[d] = (d < dim) ? dims[d] : dims[d+1]; + + Tensor<DenseIndex, 3, DataLayout, DenseIndex> tensor_arg(out_shape); + + array<DenseIndex, 4> ix; + for (DenseIndex i = 0; i < sizeDim0; ++i) { + for (DenseIndex j = 0; j < sizeDim1; ++j) { + for (DenseIndex k = 0; k < sizeDim2; ++k) { + for (DenseIndex l = 0; l < sizeDim3; ++l) { + ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l; + // suppose dim == 1, then for all i, k, l, set tensor(i, 0, k, l) = 10.0 + tensor(ix)=(ix[dim] != 0)?-1.0:10.0; + } + } + } + } + + std::size_t in_bytes = tensor.size() * sizeof(DataType); + std::size_t out_bytes = tensor_arg.size() * sizeof(DenseIndex); + + + DataType * d_in = static_cast<DataType*>(sycl_device.allocate(in_bytes)); + DenseIndex* d_out= static_cast<DenseIndex*>(sycl_device.allocate(out_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, DenseIndex> > gpu_in(d_in, Eigen::array<DenseIndex, 4>{{sizeDim0,sizeDim1,sizeDim2,sizeDim3}}); + Eigen::TensorMap<Eigen::Tensor<DenseIndex, 3, DataLayout, DenseIndex> > gpu_out(d_out, out_shape); + + sycl_device.memcpyHostToDevice(d_in, tensor.data(),in_bytes); + gpu_out.device(sycl_device) = gpu_in.argmax(dim); + sycl_device.memcpyDeviceToHost(tensor_arg.data(), d_out, out_bytes); + + VERIFY_IS_EQUAL(static_cast<size_t>(tensor_arg.size()), + size_t(sizeDim0*sizeDim1*sizeDim2*sizeDim3 / tensor.dimension(dim))); + + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { + // Expect max to be in the first index of the reduced dimension + VERIFY_IS_EQUAL(tensor_arg.data()[n], 0); + } + + sycl_device.synchronize(); + + for (DenseIndex i = 0; i < sizeDim0; ++i) { + for (DenseIndex j = 0; j < sizeDim1; ++j) { + for (DenseIndex k = 0; k < sizeDim2; ++k) { + for (DenseIndex l = 0; l < sizeDim3; ++l) { + ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l; + // suppose dim == 1, then for all i, k, l, set tensor(i, 2, k, l) = 20.0 + tensor(ix)=(ix[dim] != tensor.dimension(dim) - 1)?-1.0:20.0; + } + } + } + } + + sycl_device.memcpyHostToDevice(d_in, tensor.data(),in_bytes); + gpu_out.device(sycl_device) = gpu_in.argmax(dim); + sycl_device.memcpyDeviceToHost(tensor_arg.data(), d_out, out_bytes); + + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { + // Expect max to be in the last index of the reduced dimension + VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1); + } + sycl_device.deallocate(d_in); + sycl_device.deallocate(d_out); + } +} + +template <typename DataType, int DataLayout, typename DenseIndex> +static void test_sycl_argmin_dim(const Eigen::SyclDevice &sycl_device) +{ + DenseIndex sizeDim0=9; + DenseIndex sizeDim1=3; + DenseIndex sizeDim2=5; + DenseIndex sizeDim3=7; + Tensor<DataType, 4, DataLayout, DenseIndex> tensor(sizeDim0,sizeDim1,sizeDim2,sizeDim3); + + std::vector<DenseIndex> dims; + dims.push_back(sizeDim0); dims.push_back(sizeDim1); dims.push_back(sizeDim2); dims.push_back(sizeDim3); + for (DenseIndex dim = 0; dim < 4; ++dim) { + + array<DenseIndex, 3> out_shape; + for (DenseIndex d = 0; d < 3; ++d) out_shape[d] = (d < dim) ? dims[d] : dims[d+1]; + + Tensor<DenseIndex, 3, DataLayout, DenseIndex> tensor_arg(out_shape); + + array<DenseIndex, 4> ix; + for (DenseIndex i = 0; i < sizeDim0; ++i) { + for (DenseIndex j = 0; j < sizeDim1; ++j) { + for (DenseIndex k = 0; k < sizeDim2; ++k) { + for (DenseIndex l = 0; l < sizeDim3; ++l) { + ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l; + // suppose dim == 1, then for all i, k, l, set tensor(i, 0, k, l) = 10.0 + tensor(ix)=(ix[dim] != 0)?1.0:-10.0; + } + } + } + } + + std::size_t in_bytes = tensor.size() * sizeof(DataType); + std::size_t out_bytes = tensor_arg.size() * sizeof(DenseIndex); + + + DataType * d_in = static_cast<DataType*>(sycl_device.allocate(in_bytes)); + DenseIndex* d_out= static_cast<DenseIndex*>(sycl_device.allocate(out_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, DenseIndex> > gpu_in(d_in, Eigen::array<DenseIndex, 4>{{sizeDim0,sizeDim1,sizeDim2,sizeDim3}}); + Eigen::TensorMap<Eigen::Tensor<DenseIndex, 3, DataLayout, DenseIndex> > gpu_out(d_out, out_shape); + + sycl_device.memcpyHostToDevice(d_in, tensor.data(),in_bytes); + gpu_out.device(sycl_device) = gpu_in.argmin(dim); + sycl_device.memcpyDeviceToHost(tensor_arg.data(), d_out, out_bytes); + + VERIFY_IS_EQUAL(static_cast<size_t>(tensor_arg.size()), + size_t(sizeDim0*sizeDim1*sizeDim2*sizeDim3 / tensor.dimension(dim))); + + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { + // Expect max to be in the first index of the reduced dimension + VERIFY_IS_EQUAL(tensor_arg.data()[n], 0); + } + + sycl_device.synchronize(); + + for (DenseIndex i = 0; i < sizeDim0; ++i) { + for (DenseIndex j = 0; j < sizeDim1; ++j) { + for (DenseIndex k = 0; k < sizeDim2; ++k) { + for (DenseIndex l = 0; l < sizeDim3; ++l) { + ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l; + // suppose dim == 1, then for all i, k, l, set tensor(i, 2, k, l) = 20.0 + tensor(ix)=(ix[dim] != tensor.dimension(dim) - 1)?1.0:-20.0; + } + } + } + } + + sycl_device.memcpyHostToDevice(d_in, tensor.data(),in_bytes); + gpu_out.device(sycl_device) = gpu_in.argmin(dim); + sycl_device.memcpyDeviceToHost(tensor_arg.data(), d_out, out_bytes); + + for (DenseIndex n = 0; n < tensor_arg.size(); ++n) { + // Expect max to be in the last index of the reduced dimension + VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1); + } + sycl_device.deallocate(d_in); + sycl_device.deallocate(d_out); + } +} + + + + +template<typename DataType, typename Device_Selector> void sycl_argmax_test_per_device(const Device_Selector& d){ + QueueInterface queueInterface(d); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_sycl_simple_argmax<DataType, RowMajor, int64_t>(sycl_device); + test_sycl_simple_argmax<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_argmax_dim<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_argmax_dim<DataType, RowMajor, int64_t>(sycl_device); + test_sycl_argmin_dim<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_argmin_dim<DataType, RowMajor, int64_t>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_argmax_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_argmax_test_per_device<double>(device)); + } + +}
diff --git a/unsupported/test/cxx11_tensor_assign.cpp b/unsupported/test/cxx11_tensor_assign.cpp index e5cf61f..ce9d243 100644 --- a/unsupported/test/cxx11_tensor_assign.cpp +++ b/unsupported/test/cxx11_tensor_assign.cpp
@@ -286,7 +286,7 @@ } static void test_std_initializers_tensor() { -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES +#if EIGEN_HAS_VARIADIC_TEMPLATES Tensor<int, 1> a(3); a.setValues({0, 1, 2}); VERIFY_IS_EQUAL(a(0), 0); @@ -358,7 +358,7 @@ #endif // EIGEN_HAS_VARIADIC_TEMPLATES } -void test_cxx11_tensor_assign() +EIGEN_DECLARE_TEST(cxx11_tensor_assign) { CALL_SUBTEST(test_1d()); CALL_SUBTEST(test_2d());
diff --git a/unsupported/test/cxx11_tensor_block_access.cpp b/unsupported/test/cxx11_tensor_block_access.cpp new file mode 100644 index 0000000..0fb189e --- /dev/null +++ b/unsupported/test/cxx11_tensor_block_access.cpp
@@ -0,0 +1,1221 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Andy Davis <andydavis@google.com> +// Copyright (C) 2018 Eugene Zhulenev <ezhulenev@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +#include <algorithm> +#include <set> + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; +using Eigen::Index; +using Eigen::RowMajor; +using Eigen::ColMajor; + + +template<typename T> +static const T& choose(int layout, const T& col, const T& row) { + return layout == ColMajor ? col : row; +} + +static internal::TensorBlockShapeType RandomShape() { + return internal::random<bool>() + ? internal::kUniformAllDims + : internal::kSkewedInnerDims; +} + +template <int NumDims> +static Index RandomTargetSize(const DSizes<Index, NumDims>& dims) { + return internal::random<Index>(1, dims.TotalSize()); +} + +template <int NumDims> +static DSizes<Index, NumDims> RandomDims() { + array<Index, NumDims> dims; + for (int i = 0; i < NumDims; ++i) { + dims[i] = internal::random<int>(1, 20); + } + return DSizes<Index, NumDims>(dims); +} + +/** Dummy data type to test TensorBlock copy ops. */ +struct Data { + Data() : value(0) {} + explicit Data(int v) : value(v) { } + int value; +}; + +bool operator==(const Data& lhs, const Data& rhs) { + return lhs.value == rhs.value; +} + +std::ostream& operator<<(std::ostream& os, const Data& d) { + os << "Data: value=" << d.value; + return os; +} + +template <typename T> +static T* GenerateRandomData(const Index& size) { + T* data = new T[size]; + for (int i = 0; i < size; ++i) { + data[i] = internal::random<T>(); + } + return data; +} + +template <> +Data* GenerateRandomData(const Index& size) { + Data* data = new Data[size]; + for (int i = 0; i < size; ++i) { + data[i] = Data(internal::random<int>(1, 100)); + } + return data; +} + +template <int NumDims> +static void Debug(DSizes<Index, NumDims> dims) { + for (int i = 0; i < NumDims; ++i) { + std::cout << dims[i] << "; "; + } + std::cout << std::endl; +} + +template <int Layout> +static void test_block_mapper_sanity() +{ + typedef internal::TensorBlockMapper<int, Index, 2, Layout> TensorBlockMapper; + + DSizes<Index, 2> tensor_dims(100, 100); + + // Test uniform blocks. + TensorBlockMapper uniform_block_mapper( + tensor_dims, internal::kUniformAllDims, 100); + + VERIFY_IS_EQUAL(uniform_block_mapper.total_block_count(), 100); + VERIFY_IS_EQUAL(uniform_block_mapper.block_dims_total_size(), 100); + + // 10x10 blocks + typename TensorBlockMapper::Block uniform_b0 = uniform_block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(uniform_b0.block_sizes().at(0), 10); + VERIFY_IS_EQUAL(uniform_b0.block_sizes().at(1), 10); + // Depending on a layout we stride by cols rows. + VERIFY_IS_EQUAL(uniform_b0.block_strides().at(0), choose(Layout, 1, 10)); + VERIFY_IS_EQUAL(uniform_b0.block_strides().at(1), choose(Layout, 10, 1)); + // Tensor strides depend only on a layout and not on the block size. + VERIFY_IS_EQUAL(uniform_b0.tensor_strides().at(0), choose(Layout, 1, 100)); + VERIFY_IS_EQUAL(uniform_b0.tensor_strides().at(1), choose(Layout, 100, 1)); + + // Test skewed to inner dims blocks. + TensorBlockMapper skewed_block_mapper( + tensor_dims, internal::kSkewedInnerDims, 100); + + VERIFY_IS_EQUAL(skewed_block_mapper.total_block_count(), 100); + VERIFY_IS_EQUAL(skewed_block_mapper.block_dims_total_size(), 100); + + // 1x100 (100x1) rows/cols depending on a tensor layout. + typename TensorBlockMapper::Block skewed_b0 = skewed_block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(skewed_b0.block_sizes().at(0), choose(Layout, 100, 1)); + VERIFY_IS_EQUAL(skewed_b0.block_sizes().at(1), choose(Layout, 1, 100)); + // Depending on a layout we stride by cols rows. + VERIFY_IS_EQUAL(skewed_b0.block_strides().at(0), choose(Layout, 1, 100)); + VERIFY_IS_EQUAL(skewed_b0.block_strides().at(1), choose(Layout, 100, 1)); + // Tensor strides depend only on a layout and not on the block size. + VERIFY_IS_EQUAL(skewed_b0.tensor_strides().at(0), choose(Layout, 1, 100)); + VERIFY_IS_EQUAL(skewed_b0.tensor_strides().at(1), choose(Layout, 100, 1)); +} + +// Given a TensorBlock "visit" every element accessible though it, and a keep an +// index in the visited set. Verify that every coeff accessed only once. +template <typename T, int Layout, int NumDims> +static void UpdateCoeffSet( + const internal::TensorBlock<T, Index, NumDims, Layout>& block, + Index first_coeff_index, int dim_index, std::set<Index>* visited_coeffs) { + const DSizes<Index, NumDims>& block_sizes = block.block_sizes(); + const DSizes<Index, NumDims>& tensor_strides = block.tensor_strides(); + + for (int i = 0; i < block_sizes[dim_index]; ++i) { + if (tensor_strides[dim_index] == 1) { + typedef std::pair<std::set<Index>::iterator, bool> ReturnType; + ReturnType inserted = visited_coeffs->insert(first_coeff_index + i); + VERIFY_IS_EQUAL(inserted.second, true); + } else { + int next_dim_index = dim_index + choose(Layout, -1, 1); + UpdateCoeffSet<T, Layout, NumDims>(block, first_coeff_index, + next_dim_index, visited_coeffs); + first_coeff_index += tensor_strides[dim_index]; + } + } +} + +template <typename T, int NumDims, int Layout> +static void test_block_mapper_maps_every_element() { + typedef internal::TensorBlock<T, Index, NumDims, Layout> TensorBlock; + typedef internal::TensorBlockMapper<T, Index, NumDims, Layout> TensorBlockMapper; + + DSizes<Index, NumDims> dims = RandomDims<NumDims>(); + + // Keep track of elements indices available via block access. + std::set<Index> coeff_set; + + // Try different combinations of block types and sizes. + TensorBlockMapper block_mapper(dims, RandomShape(), RandomTargetSize(dims)); + + for (int i = 0; i < block_mapper.total_block_count(); ++i) { + TensorBlock block = block_mapper.GetBlockForIndex(i, NULL); + UpdateCoeffSet<T, Layout, NumDims>(block, block.first_coeff_index(), + choose(Layout, NumDims - 1, 0), + &coeff_set); + } + + // Verify that every coefficient in the original Tensor is accessible through + // TensorBlock only once. + Index total_coeffs = dims.TotalSize(); + VERIFY_IS_EQUAL(Index(coeff_set.size()), total_coeffs); + VERIFY_IS_EQUAL(*coeff_set.begin(), 0); + VERIFY_IS_EQUAL(*coeff_set.rbegin(), total_coeffs - 1); +} + +template <typename T, int NumDims, int Layout> +static void test_slice_block_mapper_maps_every_element() { + typedef internal::TensorBlock<T, Index, NumDims, Layout> TensorBlock; + typedef internal::TensorSliceBlockMapper<T, Index, NumDims, Layout> TensorSliceBlockMapper; + + DSizes<Index, NumDims> tensor_dims = RandomDims<NumDims>(); + DSizes<Index, NumDims> tensor_slice_offsets = RandomDims<NumDims>(); + DSizes<Index, NumDims> tensor_slice_extents = RandomDims<NumDims>(); + + // Make sure that tensor offsets + extents do not overflow. + for (int i = 0; i < NumDims; ++i) { + tensor_slice_offsets[i] = + numext::mini(tensor_dims[i] - 1, tensor_slice_offsets[i]); + tensor_slice_extents[i] = numext::mini( + tensor_slice_extents[i], tensor_dims[i] - tensor_slice_offsets[i]); + } + + // Keep track of elements indices available via block access. + std::set<Index> coeff_set; + + int total_coeffs = static_cast<int>(tensor_slice_extents.TotalSize()); + + // Pick a random dimension sizes for the tensor blocks. + DSizes<Index, NumDims> block_sizes; + for (int i = 0; i < NumDims; ++i) { + block_sizes[i] = internal::random<Index>(1, tensor_slice_extents[i]); + } + + TensorSliceBlockMapper block_mapper(tensor_dims, tensor_slice_offsets, + tensor_slice_extents, block_sizes, + DimensionList<Index, NumDims>()); + + for (int i = 0; i < block_mapper.total_block_count(); ++i) { + TensorBlock block = block_mapper.GetBlockForIndex(i, NULL); + UpdateCoeffSet<T, Layout, NumDims>(block, block.first_coeff_index(), + choose(Layout, NumDims - 1, 0), + &coeff_set); + } + + VERIFY_IS_EQUAL(Index(coeff_set.size()), total_coeffs); +} + +template <typename T, int NumDims, int Layout> +static void test_block_io_copy_data_from_source_to_target() { + typedef internal::TensorBlock<T, Index, NumDims, Layout> TensorBlock; + typedef internal::TensorBlockMapper<T, Index, NumDims, Layout> + TensorBlockMapper; + + typedef internal::TensorBlockReader<T, Index, NumDims, Layout> + TensorBlockReader; + typedef internal::TensorBlockWriter<T, Index, NumDims, Layout> + TensorBlockWriter; + + DSizes<Index, NumDims> input_tensor_dims = RandomDims<NumDims>(); + const Index input_tensor_size = input_tensor_dims.TotalSize(); + + T* input_data = GenerateRandomData<T>(input_tensor_size); + T* output_data = new T[input_tensor_size]; + + TensorBlockMapper block_mapper(input_tensor_dims, RandomShape(), + RandomTargetSize(input_tensor_dims)); + T* block_data = new T[block_mapper.block_dims_total_size()]; + + for (int i = 0; i < block_mapper.total_block_count(); ++i) { + TensorBlock block = block_mapper.GetBlockForIndex(i, block_data); + TensorBlockReader::Run(&block, input_data); + TensorBlockWriter::Run(block, output_data); + } + + for (int i = 0; i < input_tensor_size; ++i) { + VERIFY_IS_EQUAL(input_data[i], output_data[i]); + } + + delete[] input_data; + delete[] output_data; + delete[] block_data; +} + +template <int Layout, int NumDims> +static Index GetInputIndex(Index output_index, + const array<Index, NumDims>& output_to_input_dim_map, + const array<Index, NumDims>& input_strides, + const array<Index, NumDims>& output_strides) { + int input_index = 0; + if (Layout == ColMajor) { + for (int i = NumDims - 1; i > 0; --i) { + const Index idx = output_index / output_strides[i]; + input_index += idx * input_strides[output_to_input_dim_map[i]]; + output_index -= idx * output_strides[i]; + } + return input_index + + output_index * input_strides[output_to_input_dim_map[0]]; + } else { + for (int i = 0; i < NumDims - 1; ++i) { + const Index idx = output_index / output_strides[i]; + input_index += idx * input_strides[output_to_input_dim_map[i]]; + output_index -= idx * output_strides[i]; + } + return input_index + + output_index * input_strides[output_to_input_dim_map[NumDims - 1]]; + } +} + +template <int Layout, int NumDims> +static array<Index, NumDims> ComputeStrides( + const array<Index, NumDims>& sizes) { + array<Index, NumDims> strides; + if (Layout == ColMajor) { + strides[0] = 1; + for (int i = 1; i < NumDims; ++i) { + strides[i] = strides[i - 1] * sizes[i - 1]; + } + } else { + strides[NumDims - 1] = 1; + for (int i = NumDims - 2; i >= 0; --i) { + strides[i] = strides[i + 1] * sizes[i + 1]; + } + } + return strides; +} + +template <typename T, int NumDims, int Layout> +static void test_block_io_copy_using_reordered_dimensions() { + typedef internal::TensorBlock<T, Index, NumDims, Layout> TensorBlock; + typedef internal::TensorBlockMapper<T, Index, NumDims, Layout> + TensorBlockMapper; + + typedef internal::TensorBlockReader<T, Index, NumDims, Layout> + TensorBlockReader; + typedef internal::TensorBlockWriter<T, Index, NumDims, Layout> + TensorBlockWriter; + + DSizes<Index, NumDims> input_tensor_dims = RandomDims<NumDims>(); + const Index input_tensor_size = input_tensor_dims.TotalSize(); + + // Create a random input tensor. + T* input_data = GenerateRandomData<T>(input_tensor_size); + + // Create a random dimension re-ordering/shuffle. + std::vector<Index> shuffle; + for (int i = 0; i < NumDims; ++i) shuffle.push_back(i); + std::random_shuffle(shuffle.begin(), shuffle.end()); + + DSizes<Index, NumDims> output_tensor_dims; + array<Index, NumDims> input_to_output_dim_map; + array<Index, NumDims> output_to_input_dim_map; + for (Index i = 0; i < NumDims; ++i) { + output_tensor_dims[shuffle[i]] = input_tensor_dims[i]; + input_to_output_dim_map[i] = shuffle[i]; + output_to_input_dim_map[shuffle[i]] = i; + } + + // Random block shape and size. + TensorBlockMapper block_mapper(output_tensor_dims, RandomShape(), + RandomTargetSize(input_tensor_dims)); + + T* block_data = new T[block_mapper.block_dims_total_size()]; + T* output_data = new T[input_tensor_size]; + + array<Index, NumDims> input_tensor_strides = + ComputeStrides<Layout, NumDims>(input_tensor_dims); + array<Index, NumDims> output_tensor_strides = + ComputeStrides<Layout, NumDims>(output_tensor_dims); + + for (Index i = 0; i < block_mapper.total_block_count(); ++i) { + TensorBlock block = block_mapper.GetBlockForIndex(i, block_data); + const Index first_coeff_index = GetInputIndex<Layout, NumDims>( + block.first_coeff_index(), output_to_input_dim_map, + input_tensor_strides, output_tensor_strides); + TensorBlockReader::Run(&block, first_coeff_index, input_to_output_dim_map, + input_tensor_strides, input_data); + TensorBlockWriter::Run(block, first_coeff_index, input_to_output_dim_map, + input_tensor_strides, output_data); + } + + for (int i = 0; i < input_tensor_size; ++i) { + VERIFY_IS_EQUAL(input_data[i], output_data[i]); + } + + delete[] input_data; + delete[] block_data; + delete[] output_data; +} + +// This is the special case for reading data with reordering, when dimensions +// before/after reordering are the same. Squeezing reads along inner dimensions +// in this case is illegal, because we reorder innermost dimension. +template <int Layout> +static void test_block_io_copy_using_reordered_dimensions_do_not_squeeze() +{ + typedef internal::TensorBlock<float, Index, 3, Layout> TensorBlock; + typedef internal::TensorBlockReader<float, Index, 3, Layout> + TensorBlockReader; + + DSizes<Index, 3> tensor_dims; + tensor_dims[0] = 7; + tensor_dims[1] = 9; + tensor_dims[2] = 7; + + DSizes<Index, 3> block_dims = tensor_dims; + + DSizes<Index, 3> tensor_to_block_dim_map; + tensor_to_block_dim_map[0] = 2; + tensor_to_block_dim_map[1] = 1; + tensor_to_block_dim_map[2] = 0; + + DSizes<Index, 3> tensor_strides(ComputeStrides<Layout, 3>(tensor_dims)); + DSizes<Index, 3> block_strides(ComputeStrides<Layout, 3>(block_dims)); + + const Index tensor_size = tensor_dims.TotalSize(); + float* tensor_data = GenerateRandomData<float>(tensor_size); + float* block_data = new float[tensor_size]; + + TensorBlock block(0, block_dims, block_strides, tensor_strides, block_data); + TensorBlockReader::Run(&block, + 0, + tensor_to_block_dim_map, + tensor_strides, + tensor_data); + + TensorMap<Tensor<float, 3, Layout> > block_tensor(block_data, block_dims); + TensorMap<Tensor<float, 3, Layout> > tensor_tensor(tensor_data, tensor_dims); + + for (Index d0 = 0; d0 < tensor_dims[0]; ++d0) { + for (Index d1 = 0; d1 < tensor_dims[1]; ++d1) { + for (Index d2 = 0; d2 < tensor_dims[2]; ++d2) { + float block_value = block_tensor(d2, d1, d0); + float tensor_value = tensor_tensor(d0, d1, d2); + VERIFY_IS_EQUAL(block_value, tensor_value); + } + } + } + + delete[] block_data; + delete[] tensor_data; +} + +// This is the special case for reading data with reordering, when dimensions +// before/after reordering are the same. Squeezing reads in this case is allowed +// because we reorder outer dimensions. +template <int Layout> +static void test_block_io_copy_using_reordered_dimensions_squeeze() +{ + typedef internal::TensorBlock<float, Index, 4, Layout> TensorBlock; + typedef internal::TensorBlockReader<float, Index, 4, Layout> + TensorBlockReader; + + DSizes<Index, 4> tensor_dims; + tensor_dims[0] = 7; + tensor_dims[1] = 5; + tensor_dims[2] = 9; + tensor_dims[3] = 9; + + DSizes<Index, 4> block_dims = tensor_dims; + + DSizes<Index, 4> tensor_to_block_dim_map; + tensor_to_block_dim_map[0] = 0; + tensor_to_block_dim_map[1] = 1; + tensor_to_block_dim_map[2] = 3; + tensor_to_block_dim_map[3] = 2; + + DSizes<Index, 4> tensor_strides(ComputeStrides<Layout, 4>(tensor_dims)); + DSizes<Index, 4> block_strides(ComputeStrides<Layout, 4>(block_dims)); + + const Index tensor_size = tensor_dims.TotalSize(); + float* tensor_data = GenerateRandomData<float>(tensor_size); + float* block_data = new float[tensor_size]; + + TensorBlock block(0, block_dims, block_strides, tensor_strides, block_data); + TensorBlockReader::Run(&block, + 0, + tensor_to_block_dim_map, + tensor_strides, + tensor_data); + + TensorMap<Tensor<float, 4, Layout> > block_tensor(block_data, block_dims); + TensorMap<Tensor<float, 4, Layout> > tensor_tensor(tensor_data, tensor_dims); + + for (Index d0 = 0; d0 < tensor_dims[0]; ++d0) { + for (Index d1 = 0; d1 < tensor_dims[1]; ++d1) { + for (Index d2 = 0; d2 < tensor_dims[2]; ++d2) { + for (Index d3 = 0; d3 < tensor_dims[3]; ++d3) { + float block_value = block_tensor(d0, d1, d3, d2); + float tensor_value = tensor_tensor(d0, d1, d2, d3); + VERIFY_IS_EQUAL(block_value, tensor_value); + } + } + } + } + + delete[] block_data; + delete[] tensor_data; +} + +template<typename Scalar, typename StorageIndex, int Dim> +class EqualityChecker +{ + const Scalar* input_data; + const DSizes<StorageIndex, Dim> &input_dims, &input_strides, &output_dims, &output_strides; + void check_recursive(const Scalar* input, const Scalar* output, int depth=0) const + { + if(depth==Dim) + { + VERIFY_IS_EQUAL(*input, *output); + return; + } + + for(int i=0; i<output_dims[depth]; ++i) + { + check_recursive(input + i % input_dims[depth] * input_strides[depth], output + i*output_strides[depth], depth+1); + } + } +public: + EqualityChecker(const Scalar* input_data_, + const DSizes<StorageIndex, Dim> &input_dims_, const DSizes<StorageIndex, Dim> &input_strides_, + const DSizes<StorageIndex, Dim> &output_dims_, const DSizes<StorageIndex, Dim> &output_strides_) + : input_data(input_data_) + , input_dims(input_dims_), input_strides(input_strides_) + , output_dims(output_dims_), output_strides(output_strides_) + {} + + void operator()(const Scalar* output_data) const + { + check_recursive(input_data, output_data); + } +}; + +template <int Layout> +static void test_block_io_zero_stride() +{ + typedef internal::TensorBlock<float, Index, 5, Layout> TensorBlock; + typedef internal::TensorBlockReader<float, Index, 5, Layout> + TensorBlockReader; + typedef internal::TensorBlockWriter<float, Index, 5, Layout> + TensorBlockWriter; + + DSizes<Index, 5> rnd_dims = RandomDims<5>(); + + DSizes<Index, 5> input_tensor_dims = rnd_dims; + input_tensor_dims[0] = 1; + input_tensor_dims[2] = 1; + input_tensor_dims[4] = 1; + const Index input_tensor_size = input_tensor_dims.TotalSize(); + float* input_data = GenerateRandomData<float>(input_tensor_size); + + DSizes<Index, 5> output_tensor_dims = rnd_dims; + + DSizes<Index, 5> input_tensor_strides( + ComputeStrides<Layout, 5>(input_tensor_dims)); + DSizes<Index, 5> output_tensor_strides( + ComputeStrides<Layout, 5>(output_tensor_dims)); + + DSizes<Index, 5> input_tensor_strides_with_zeros(input_tensor_strides); + input_tensor_strides_with_zeros[0] = 0; + input_tensor_strides_with_zeros[2] = 0; + input_tensor_strides_with_zeros[4] = 0; + + // Verify that data was correctly read/written from/into the block. + const EqualityChecker<float, Index, 5> verify_is_equal(input_data, input_tensor_dims, input_tensor_strides, output_tensor_dims, output_tensor_strides); + + { + float* output_data = new float[output_tensor_dims.TotalSize()]; + TensorBlock read_block(0, output_tensor_dims, output_tensor_strides, + input_tensor_strides_with_zeros, output_data); + TensorBlockReader::Run(&read_block, input_data); + verify_is_equal(output_data); + delete[] output_data; + } + + { + float* output_data = new float[output_tensor_dims.TotalSize()]; + TensorBlock write_block(0, output_tensor_dims, + input_tensor_strides_with_zeros, + output_tensor_strides, input_data); + TensorBlockWriter::Run(write_block, output_data); + verify_is_equal(output_data); + delete[] output_data; + } + + delete[] input_data; +} + +template <int Layout> +static void test_block_io_squeeze_ones() { + typedef internal::TensorBlock<float, Index, 5, Layout> TensorBlock; + typedef internal::TensorBlockReader<float, Index, 5, Layout> + TensorBlockReader; + typedef internal::TensorBlockWriter<float, Index, 5, Layout> + TensorBlockWriter; + + // Total size > 1. + { + DSizes<Index, 5> block_sizes(1, 2, 1, 2, 1); + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensor. + float* input_data = GenerateRandomData<float>(total_size); + DSizes<Index, 5> strides(ComputeStrides<Layout, 5>(block_sizes)); + + { + float* output_data = new float[block_sizes.TotalSize()]; + TensorBlock read_block(0, block_sizes, strides, strides, output_data); + TensorBlockReader::Run(&read_block, input_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], input_data[i]); + } + delete[] output_data; + } + + { + float* output_data = new float[block_sizes.TotalSize()]; + TensorBlock write_block(0, block_sizes, strides, strides, input_data); + TensorBlockWriter::Run(write_block, output_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], input_data[i]); + } + delete[] output_data; + } + } + + // Total size == 1. + { + DSizes<Index, 5> block_sizes(1, 1, 1, 1, 1); + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensor. + float* input_data = GenerateRandomData<float>(total_size); + DSizes<Index, 5> strides(ComputeStrides<Layout, 5>(block_sizes)); + + { + float* output_data = new float[block_sizes.TotalSize()]; + TensorBlock read_block(0, block_sizes, strides, strides, output_data); + TensorBlockReader::Run(&read_block, input_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], input_data[i]); + } + delete[] output_data; + } + + { + float* output_data = new float[block_sizes.TotalSize()]; + TensorBlock write_block(0, block_sizes, strides, strides, input_data); + TensorBlockWriter::Run(write_block, output_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], input_data[i]); + } + delete[] output_data; + } + } +} + +template <typename T, int NumDims, int Layout> +static void test_block_cwise_unary_io_basic() { + typedef internal::scalar_square_op<T> UnaryFunctor; + typedef internal::TensorBlockCwiseUnaryIO<UnaryFunctor, Index, T, NumDims, + Layout> + TensorBlockCwiseUnaryIO; + + DSizes<Index, NumDims> block_sizes = RandomDims<NumDims>(); + DSizes<Index, NumDims> strides(ComputeStrides<Layout, NumDims>(block_sizes)); + + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensors. + T* input_data = GenerateRandomData<T>(total_size); + + T* output_data = new T[total_size]; + UnaryFunctor functor; + TensorBlockCwiseUnaryIO::Run(functor, block_sizes, strides, output_data, + strides, input_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], functor(input_data[i])); + } + + delete[] input_data; + delete[] output_data; +} + +template <int Layout> +static void test_block_cwise_unary_io_squeeze_ones() { + typedef internal::scalar_square_op<float> UnaryFunctor; + typedef internal::TensorBlockCwiseUnaryIO<UnaryFunctor, Index, float, 5, + Layout> + TensorBlockCwiseUnaryIO; + + DSizes<Index, 5> block_sizes(1, 2, 1, 3, 1); + DSizes<Index, 5> strides(ComputeStrides<Layout, 5>(block_sizes)); + + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensors. + float* input_data = GenerateRandomData<float>(total_size); + + float* output_data = new float[total_size]; + UnaryFunctor functor; + TensorBlockCwiseUnaryIO::Run(functor, block_sizes, strides, output_data, + strides, input_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], functor(input_data[i])); + } + + delete[] input_data; + delete[] output_data; +} + +template <int Layout> +static void test_block_cwise_unary_io_zero_strides() { + typedef internal::scalar_square_op<float> UnaryFunctor; + typedef internal::TensorBlockCwiseUnaryIO<UnaryFunctor, Index, float, 5, + Layout> + TensorBlockCwiseUnaryIO; + + DSizes<Index, 5> rnd_dims = RandomDims<5>(); + + DSizes<Index, 5> input_sizes = rnd_dims; + input_sizes[0] = 1; + input_sizes[2] = 1; + input_sizes[4] = 1; + + DSizes<Index, 5> input_strides(ComputeStrides<Layout, 5>(input_sizes)); + input_strides[0] = 0; + input_strides[2] = 0; + input_strides[4] = 0; + + // Generate random data. + float* input_data = GenerateRandomData<float>(input_sizes.TotalSize()); + + DSizes<Index, 5> output_sizes = rnd_dims; + DSizes<Index, 5> output_strides(ComputeStrides<Layout, 5>(output_sizes)); + + const Index output_total_size = output_sizes.TotalSize(); + float* output_data = new float[output_total_size]; + + UnaryFunctor functor; + TensorBlockCwiseUnaryIO::Run(functor, output_sizes, output_strides, + output_data, input_strides, input_data); + for (int i = 0; i < rnd_dims[0]; ++i) { + for (int j = 0; j < rnd_dims[1]; ++j) { + for (int k = 0; k < rnd_dims[2]; ++k) { + for (int l = 0; l < rnd_dims[3]; ++l) { + for (int m = 0; m < rnd_dims[4]; ++m) { + Index output_index = i * output_strides[0] + j * output_strides[1] + + k * output_strides[2] + l * output_strides[3] + + m * output_strides[4]; + Index input_index = i * input_strides[0] + j * input_strides[1] + + k * input_strides[2] + l * input_strides[3] + + m * input_strides[4]; + VERIFY_IS_EQUAL(output_data[output_index], + functor(input_data[input_index])); + } + } + } + } + } + + delete[] input_data; + delete[] output_data; +} + +template <typename T, int NumDims, int Layout> +static void test_block_cwise_binary_io_basic() { + typedef internal::scalar_sum_op<T> BinaryFunctor; + typedef internal::TensorBlockCwiseBinaryIO<BinaryFunctor, Index, T, NumDims, + Layout> + TensorBlockCwiseBinaryIO; + + DSizes<Index, NumDims> block_sizes = RandomDims<NumDims>(); + DSizes<Index, NumDims> strides(ComputeStrides<Layout, NumDims>(block_sizes)); + + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensors. + T* left_data = GenerateRandomData<T>(total_size); + T* right_data = GenerateRandomData<T>(total_size); + + T* output_data = new T[total_size]; + BinaryFunctor functor; + TensorBlockCwiseBinaryIO::Run(functor, block_sizes, strides, output_data, + strides, left_data, strides, right_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], functor(left_data[i], right_data[i])); + } + + delete[] left_data; + delete[] right_data; + delete[] output_data; +} + +template <int Layout> +static void test_block_cwise_binary_io_squeeze_ones() { + typedef internal::scalar_sum_op<float> BinaryFunctor; + typedef internal::TensorBlockCwiseBinaryIO<BinaryFunctor, Index, float, 5, + Layout> + TensorBlockCwiseBinaryIO; + + DSizes<Index, 5> block_sizes(1, 2, 1, 3, 1); + DSizes<Index, 5> strides(ComputeStrides<Layout, 5>(block_sizes)); + + const Index total_size = block_sizes.TotalSize(); + + // Create a random input tensors. + float* left_data = GenerateRandomData<float>(total_size); + float* right_data = GenerateRandomData<float>(total_size); + + float* output_data = new float[total_size]; + BinaryFunctor functor; + TensorBlockCwiseBinaryIO::Run(functor, block_sizes, strides, output_data, + strides, left_data, strides, right_data); + for (int i = 0; i < total_size; ++i) { + VERIFY_IS_EQUAL(output_data[i], functor(left_data[i], right_data[i])); + } + + delete[] left_data; + delete[] right_data; + delete[] output_data; +} + +template <int Layout> +static void test_block_cwise_binary_io_zero_strides() { + typedef internal::scalar_sum_op<float> BinaryFunctor; + typedef internal::TensorBlockCwiseBinaryIO<BinaryFunctor, Index, float, 5, + Layout> + TensorBlockCwiseBinaryIO; + + DSizes<Index, 5> rnd_dims = RandomDims<5>(); + + DSizes<Index, 5> left_sizes = rnd_dims; + left_sizes[0] = 1; + left_sizes[2] = 1; + left_sizes[4] = 1; + + DSizes<Index, 5> left_strides(ComputeStrides<Layout, 5>(left_sizes)); + left_strides[0] = 0; + left_strides[2] = 0; + left_strides[4] = 0; + + DSizes<Index, 5> right_sizes = rnd_dims; + right_sizes[1] = 1; + right_sizes[3] = 1; + + DSizes<Index, 5> right_strides(ComputeStrides<Layout, 5>(right_sizes)); + right_strides[1] = 0; + right_strides[3] = 0; + + // Generate random data. + float* left_data = GenerateRandomData<float>(left_sizes.TotalSize()); + float* right_data = GenerateRandomData<float>(right_sizes.TotalSize()); + + DSizes<Index, 5> output_sizes = rnd_dims; + DSizes<Index, 5> output_strides(ComputeStrides<Layout, 5>(output_sizes)); + + const Index output_total_size = output_sizes.TotalSize(); + float* output_data = new float[output_total_size]; + + BinaryFunctor functor; + TensorBlockCwiseBinaryIO::Run(functor, output_sizes, output_strides, + output_data, left_strides, left_data, + right_strides, right_data); + for (int i = 0; i < rnd_dims[0]; ++i) { + for (int j = 0; j < rnd_dims[1]; ++j) { + for (int k = 0; k < rnd_dims[2]; ++k) { + for (int l = 0; l < rnd_dims[3]; ++l) { + for (int m = 0; m < rnd_dims[4]; ++m) { + Index output_index = i * output_strides[0] + j * output_strides[1] + + k * output_strides[2] + l * output_strides[3] + + m * output_strides[4]; + Index left_index = i * left_strides[0] + j * left_strides[1] + + k * left_strides[2] + l * left_strides[3] + + m * left_strides[4]; + Index right_index = i * right_strides[0] + j * right_strides[1] + + k * right_strides[2] + l * right_strides[3] + + m * right_strides[4]; + VERIFY_IS_EQUAL( + output_data[output_index], + functor(left_data[left_index], right_data[right_index])); + } + } + } + } + } + + delete[] left_data; + delete[] right_data; + delete[] output_data; +} + +template <int Layout> +static void test_uniform_block_shape() +{ + typedef internal::TensorBlock<int, Index, 5, Layout> TensorBlock; + typedef internal::TensorBlockMapper<int, Index, 5, Layout> TensorBlockMapper; + + { + // Test shape 'UniformAllDims' with uniform 'max_coeff count'. + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 5 * 5 * 5 * 5 * 5; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + for (int i = 0; i < 5; ++i) { + VERIFY_IS_EQUAL(5, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills + // partially into first inner-most dimension. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 7 * 5 * 5 * 5 * 5; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[0]); + for (int i = 1; i < 5; ++i) { + VERIFY_IS_EQUAL(5, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 5 * 5 * 5 * 5 * 6; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(6, block.block_sizes()[4]); + for (int i = 3; i >= 0; --i) { + VERIFY_IS_EQUAL(5, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills + // fully into first inner-most dimension. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 5 * 5 * 5 * 5; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + for (int i = 1; i < 5; ++i) { + VERIFY_IS_EQUAL(5, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 5 * 5 * 5 * 5 * 7; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + for (int i = 3; i >= 0; --i) { + VERIFY_IS_EQUAL(5, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills + // fully into first few inner-most dimensions. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(7, 5, 6, 17, 7); + const Index max_coeff_count = 7 * 5 * 6 * 7 * 5; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[0]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(6, block.block_sizes()[2]); + VERIFY_IS_EQUAL(7, block.block_sizes()[3]); + VERIFY_IS_EQUAL(5, block.block_sizes()[4]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(7, 5, 6, 9, 7); + const Index max_coeff_count = 5 * 5 * 5 * 6 * 7; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY_IS_EQUAL(6, block.block_sizes()[3]); + VERIFY_IS_EQUAL(5, block.block_sizes()[2]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(5, block.block_sizes()[0]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'UniformAllDims' with full allocation to all dims. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(7, 5, 6, 17, 7); + const Index max_coeff_count = 7 * 5 * 6 * 17 * 7; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[0]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(6, block.block_sizes()[2]); + VERIFY_IS_EQUAL(17, block.block_sizes()[3]); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(7, 5, 6, 9, 7); + const Index max_coeff_count = 7 * 5 * 6 * 9 * 7; + TensorBlockMapper block_mapper(dims, internal::kUniformAllDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY_IS_EQUAL(9, block.block_sizes()[3]); + VERIFY_IS_EQUAL(6, block.block_sizes()[2]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(7, block.block_sizes()[0]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } +} + +template <int Layout> +static void test_skewed_inner_dim_block_shape() +{ + typedef internal::TensorBlock<int, Index, 5, Layout> TensorBlock; + typedef internal::TensorBlockMapper<int, Index, 5, Layout> TensorBlockMapper; + + // Test shape 'SkewedInnerDims' with partial allocation to inner-most dim. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 10 * 1 * 1 * 1 * 1; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(10, block.block_sizes()[0]); + for (int i = 1; i < 5; ++i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 1 * 1 * 1 * 1 * 6; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(6, block.block_sizes()[4]); + for (int i = 3; i >= 0; --i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'SkewedInnerDims' with full allocation to inner-most dim. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 1 * 1 * 1 * 1; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + for (int i = 1; i < 5; ++i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 1 * 1 * 1 * 1 * 7; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + for (int i = 3; i >= 0; --i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, + // and partial allocation to second inner-dim. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 3 * 1 * 1 * 1; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + VERIFY_IS_EQUAL(3, block.block_sizes()[1]); + for (int i = 2; i < 5; ++i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 1 * 1 * 1 * 15 * 7; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY_IS_EQUAL(15, block.block_sizes()[3]); + for (int i = 2; i >= 0; --i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, + // and partial allocation to third inner-dim. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 5 * 5 * 1 * 1; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(5, block.block_sizes()[2]); + for (int i = 3; i < 5; ++i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 1 * 1 * 5 * 17 * 7; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY_IS_EQUAL(17, block.block_sizes()[3]); + VERIFY_IS_EQUAL(5, block.block_sizes()[2]); + for (int i = 1; i >= 0; --i) { + VERIFY_IS_EQUAL(1, block.block_sizes()[i]); + } + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } + + // Test shape 'SkewedInnerDims' with full allocation to all dims. + if (Layout == ColMajor) { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(6, block.block_sizes()[2]); + VERIFY_IS_EQUAL(17, block.block_sizes()[3]); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } else { + DSizes<Index, 5> dims(11, 5, 6, 17, 7); + const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; + TensorBlockMapper block_mapper(dims, internal::kSkewedInnerDims, + max_coeff_count); + TensorBlock block = block_mapper.GetBlockForIndex(0, NULL); + VERIFY_IS_EQUAL(7, block.block_sizes()[4]); + VERIFY_IS_EQUAL(17, block.block_sizes()[3]); + VERIFY_IS_EQUAL(6, block.block_sizes()[2]); + VERIFY_IS_EQUAL(5, block.block_sizes()[1]); + VERIFY_IS_EQUAL(11, block.block_sizes()[0]); + VERIFY(block.block_sizes().TotalSize() <= max_coeff_count); + } +} + +template <int Layout> +static void test_empty_dims(const internal::TensorBlockShapeType block_shape) +{ + // Test blocking of tensors with zero dimensions: + // - we must not crash on asserts and divisions by zero + // - we must not return block with zero dimensions + // (recipe for overflows/underflows, divisions by zero and NaNs later) + // - total block count must be zero + { + typedef internal::TensorBlockMapper<int, Index, 1, Layout> TensorBlockMapper; + DSizes<Index, 1> dims(0); + for (int max_coeff_count = 0; max_coeff_count < 2; ++max_coeff_count) { + TensorBlockMapper block_mapper(dims, block_shape, max_coeff_count); + VERIFY_IS_EQUAL(block_mapper.total_block_count(), 0); + VERIFY(block_mapper.block_dims_total_size() >= 1); + } + } + + { + typedef internal::TensorBlockMapper<int, Index, 2, Layout> TensorBlockMapper; + for (int dim1 = 0; dim1 < 3; ++dim1) { + for (int dim2 = 0; dim2 < 3; ++dim2) { + DSizes<Index, 2> dims(dim1, dim2); + for (int max_coeff_count = 0; max_coeff_count < 2; ++max_coeff_count) { + TensorBlockMapper block_mapper(dims, block_shape, max_coeff_count); + if (dim1 * dim2 == 0) { + VERIFY_IS_EQUAL(block_mapper.total_block_count(), 0); + } + VERIFY(block_mapper.block_dims_total_size() >= 1); + } + } + } + } +} + +#define TEST_LAYOUTS(NAME) \ + CALL_SUBTEST(NAME<ColMajor>()); \ + CALL_SUBTEST(NAME<RowMajor>()) + +#define TEST_LAYOUTS_AND_DIMS(TYPE, NAME) \ + CALL_SUBTEST((NAME<TYPE, 1, ColMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 1, RowMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 2, ColMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 2, RowMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 3, ColMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 3, RowMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 4, ColMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 4, RowMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 5, ColMajor>())); \ + CALL_SUBTEST((NAME<TYPE, 5, RowMajor>())) + +#define TEST_LAYOUTS_WITH_ARG(NAME, ARG) \ + CALL_SUBTEST(NAME<ColMajor>(ARG)); \ + CALL_SUBTEST(NAME<RowMajor>(ARG)) + +EIGEN_DECLARE_TEST(cxx11_tensor_block_access) { + TEST_LAYOUTS(test_block_mapper_sanity); + TEST_LAYOUTS_AND_DIMS(float, test_block_mapper_maps_every_element); + TEST_LAYOUTS_AND_DIMS(float, test_slice_block_mapper_maps_every_element); + TEST_LAYOUTS_AND_DIMS(float, test_block_io_copy_data_from_source_to_target); + TEST_LAYOUTS_AND_DIMS(Data, test_block_io_copy_data_from_source_to_target); + TEST_LAYOUTS_AND_DIMS(float, test_block_io_copy_using_reordered_dimensions); + TEST_LAYOUTS_AND_DIMS(Data, test_block_io_copy_using_reordered_dimensions); + TEST_LAYOUTS(test_block_io_copy_using_reordered_dimensions_do_not_squeeze); + TEST_LAYOUTS(test_block_io_copy_using_reordered_dimensions_squeeze); + TEST_LAYOUTS(test_block_io_zero_stride); + TEST_LAYOUTS(test_block_io_squeeze_ones); + TEST_LAYOUTS_AND_DIMS(float, test_block_cwise_unary_io_basic); + TEST_LAYOUTS(test_block_cwise_unary_io_squeeze_ones); + TEST_LAYOUTS(test_block_cwise_unary_io_zero_strides); + TEST_LAYOUTS_AND_DIMS(float, test_block_cwise_binary_io_basic); + TEST_LAYOUTS(test_block_cwise_binary_io_squeeze_ones); + TEST_LAYOUTS(test_block_cwise_binary_io_zero_strides); + TEST_LAYOUTS(test_uniform_block_shape); + TEST_LAYOUTS(test_skewed_inner_dim_block_shape); + TEST_LAYOUTS_WITH_ARG(test_empty_dims, internal::kUniformAllDims); + TEST_LAYOUTS_WITH_ARG(test_empty_dims, internal::kSkewedInnerDims); +} + +#undef TEST_LAYOUTS +#undef TEST_LAYOUTS_WITH_ARG
diff --git a/unsupported/test/cxx11_tensor_broadcast_sycl.cpp b/unsupported/test/cxx11_tensor_broadcast_sycl.cpp new file mode 100644 index 0000000..20f84b8 --- /dev/null +++ b/unsupported/test/cxx11_tensor_broadcast_sycl.cpp
@@ -0,0 +1,144 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_broadcast_sycl_fixed(const Eigen::SyclDevice &sycl_device){ + + // BROADCAST test: + IndexType inDim1=2; + IndexType inDim2=3; + IndexType inDim3=5; + IndexType inDim4=7; + IndexType bDim1=2; + IndexType bDim2=3; + IndexType bDim3=1; + IndexType bDim4=4; + array<IndexType, 4> in_range = {{inDim1, inDim2, inDim3, inDim4}}; + array<IndexType, 4> broadcasts = {{bDim1, bDim2, bDim3, bDim4}}; + array<IndexType, 4> out_range; // = in_range * broadcasts + for (size_t i = 0; i < out_range.size(); ++i) + out_range[i] = in_range[i] * broadcasts[i]; + + Tensor<DataType, 4, DataLayout, IndexType> input(in_range); + Tensor<DataType, 4, DataLayout, IndexType> out(out_range); + + for (size_t i = 0; i < in_range.size(); ++i) + VERIFY_IS_EQUAL(out.dimension(i), out_range[i]); + + + for (IndexType i = 0; i < input.size(); ++i) + input(i) = static_cast<DataType>(i); + + DataType * gpu_in_data = static_cast<DataType*>(sycl_device.allocate(input.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<TensorFixedSize<DataType, Sizes<2, 3, 5, 7>, DataLayout, IndexType>> gpu_in(gpu_in_data, in_range); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType>> gpu_out(gpu_out_data, out_range); + sycl_device.memcpyHostToDevice(gpu_in_data, input.data(),(input.dimensions().TotalSize())*sizeof(DataType)); + gpu_out.device(sycl_device) = gpu_in.broadcast(broadcasts); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.dimensions().TotalSize())*sizeof(DataType)); + + for (IndexType i = 0; i < inDim1*bDim1; ++i) { + for (IndexType j = 0; j < inDim2*bDim2; ++j) { + for (IndexType k = 0; k < inDim3*bDim3; ++k) { + for (IndexType l = 0; l < inDim4*bDim4; ++l) { + VERIFY_IS_APPROX(input(i%2,j%3,k%5,l%7), out(i,j,k,l)); + } + } + } + } + printf("Broadcast Test with fixed size Passed\n"); + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_broadcast_sycl(const Eigen::SyclDevice &sycl_device){ + + // BROADCAST test: + IndexType inDim1=2; + IndexType inDim2=3; + IndexType inDim3=5; + IndexType inDim4=7; + IndexType bDim1=2; + IndexType bDim2=3; + IndexType bDim3=1; + IndexType bDim4=4; + array<IndexType, 4> in_range = {{inDim1, inDim2, inDim3, inDim4}}; + array<IndexType, 4> broadcasts = {{bDim1, bDim2, bDim3, bDim4}}; + array<IndexType, 4> out_range; // = in_range * broadcasts + for (size_t i = 0; i < out_range.size(); ++i) + out_range[i] = in_range[i] * broadcasts[i]; + + Tensor<DataType, 4, DataLayout, IndexType> input(in_range); + Tensor<DataType, 4, DataLayout, IndexType> out(out_range); + + for (size_t i = 0; i < in_range.size(); ++i) + VERIFY_IS_EQUAL(out.dimension(i), out_range[i]); + + + for (IndexType i = 0; i < input.size(); ++i) + input(i) = static_cast<DataType>(i); + + DataType * gpu_in_data = static_cast<DataType*>(sycl_device.allocate(input.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 4, DataLayout, IndexType>> gpu_in(gpu_in_data, in_range); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType>> gpu_out(gpu_out_data, out_range); + sycl_device.memcpyHostToDevice(gpu_in_data, input.data(),(input.dimensions().TotalSize())*sizeof(DataType)); + gpu_out.device(sycl_device) = gpu_in.broadcast(broadcasts); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.dimensions().TotalSize())*sizeof(DataType)); + + for (IndexType i = 0; i < inDim1*bDim1; ++i) { + for (IndexType j = 0; j < inDim2*bDim2; ++j) { + for (IndexType k = 0; k < inDim3*bDim3; ++k) { + for (IndexType l = 0; l < inDim4*bDim4; ++l) { + VERIFY_IS_APPROX(input(i%inDim1,j%inDim2,k%inDim3,l%inDim4), out(i,j,k,l)); + } + } + } + } + printf("Broadcast Test Passed\n"); + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + +template<typename DataType> void sycl_broadcast_test_per_device(const cl::sycl::device& d){ + std::cout << "Running on " << d.template get_info<cl::sycl::info::device::name>() << std::endl; + QueueInterface queueInterface(d); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_broadcast_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_broadcast_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_broadcast_sycl_fixed<DataType, RowMajor, int64_t>(sycl_device); + test_broadcast_sycl_fixed<DataType, ColMajor, int64_t>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_broadcast_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_broadcast_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_broadcasting.cpp b/unsupported/test/cxx11_tensor_broadcasting.cpp index 2ddf472..7df5b53 100644 --- a/unsupported/test/cxx11_tensor_broadcasting.cpp +++ b/unsupported/test/cxx11_tensor_broadcasting.cpp
@@ -115,7 +115,7 @@ Tensor<float, 3, DataLayout> tensor(8,3,5); tensor.setRandom(); -#ifdef EIGEN_HAS_CONSTEXPR +#if defined(EIGEN_HAS_INDEX_LIST) Eigen::IndexList<Eigen::type2index<2>, Eigen::type2index<3>, Eigen::type2index<4>> broadcasts; #else Eigen::array<int, 3> broadcasts; @@ -180,8 +180,119 @@ #endif } +template <int DataLayout> +static void test_simple_broadcasting_one_by_n() +{ + Tensor<float, 4, DataLayout> tensor(1,13,5,7); + tensor.setRandom(); + array<ptrdiff_t, 4> broadcasts; + broadcasts[0] = 9; + broadcasts[1] = 1; + broadcasts[2] = 1; + broadcasts[3] = 1; + Tensor<float, 4, DataLayout> broadcast; + broadcast = tensor.broadcast(broadcasts); -void test_cxx11_tensor_broadcasting() + VERIFY_IS_EQUAL(broadcast.dimension(0), 9); + VERIFY_IS_EQUAL(broadcast.dimension(1), 13); + VERIFY_IS_EQUAL(broadcast.dimension(2), 5); + VERIFY_IS_EQUAL(broadcast.dimension(3), 7); + + for (int i = 0; i < 9; ++i) { + for (int j = 0; j < 13; ++j) { + for (int k = 0; k < 5; ++k) { + for (int l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i%1,j%13,k%5,l%7), broadcast(i,j,k,l)); + } + } + } + } +} + +template <int DataLayout> +static void test_simple_broadcasting_n_by_one() +{ + Tensor<float, 4, DataLayout> tensor(7,3,5,1); + tensor.setRandom(); + array<ptrdiff_t, 4> broadcasts; + broadcasts[0] = 1; + broadcasts[1] = 1; + broadcasts[2] = 1; + broadcasts[3] = 19; + Tensor<float, 4, DataLayout> broadcast; + broadcast = tensor.broadcast(broadcasts); + + VERIFY_IS_EQUAL(broadcast.dimension(0), 7); + VERIFY_IS_EQUAL(broadcast.dimension(1), 3); + VERIFY_IS_EQUAL(broadcast.dimension(2), 5); + VERIFY_IS_EQUAL(broadcast.dimension(3), 19); + + for (int i = 0; i < 7; ++i) { + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < 5; ++k) { + for (int l = 0; l < 19; ++l) { + VERIFY_IS_EQUAL(tensor(i%7,j%3,k%5,l%1), broadcast(i,j,k,l)); + } + } + } + } +} + +template <int DataLayout> +static void test_simple_broadcasting_one_by_n_by_one_1d() +{ + Tensor<float, 3, DataLayout> tensor(1,7,1); + tensor.setRandom(); + array<ptrdiff_t, 3> broadcasts; + broadcasts[0] = 5; + broadcasts[1] = 1; + broadcasts[2] = 13; + Tensor<float, 3, DataLayout> broadcasted; + broadcasted = tensor.broadcast(broadcasts); + + VERIFY_IS_EQUAL(broadcasted.dimension(0), 5); + VERIFY_IS_EQUAL(broadcasted.dimension(1), 7); + VERIFY_IS_EQUAL(broadcasted.dimension(2), 13); + + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 7; ++j) { + for (int k = 0; k < 13; ++k) { + VERIFY_IS_EQUAL(tensor(0,j%7,0), broadcasted(i,j,k)); + } + } + } +} + +template <int DataLayout> +static void test_simple_broadcasting_one_by_n_by_one_2d() +{ + Tensor<float, 4, DataLayout> tensor(1,7,13,1); + tensor.setRandom(); + array<ptrdiff_t, 4> broadcasts; + broadcasts[0] = 5; + broadcasts[1] = 1; + broadcasts[2] = 1; + broadcasts[3] = 19; + Tensor<float, 4, DataLayout> broadcast; + broadcast = tensor.broadcast(broadcasts); + + VERIFY_IS_EQUAL(broadcast.dimension(0), 5); + VERIFY_IS_EQUAL(broadcast.dimension(1), 7); + VERIFY_IS_EQUAL(broadcast.dimension(2), 13); + VERIFY_IS_EQUAL(broadcast.dimension(3), 19); + + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 7; ++j) { + for (int k = 0; k < 13; ++k) { + for (int l = 0; l < 19; ++l) { + VERIFY_IS_EQUAL(tensor(0,j%7,k%13,0), broadcast(i,j,k,l)); + } + } + } + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_broadcasting) { CALL_SUBTEST(test_simple_broadcasting<ColMajor>()); CALL_SUBTEST(test_simple_broadcasting<RowMajor>()); @@ -191,4 +302,12 @@ CALL_SUBTEST(test_static_broadcasting<RowMajor>()); CALL_SUBTEST(test_fixed_size_broadcasting<ColMajor>()); CALL_SUBTEST(test_fixed_size_broadcasting<RowMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n<RowMajor>()); + CALL_SUBTEST(test_simple_broadcasting_n_by_one<RowMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n<ColMajor>()); + CALL_SUBTEST(test_simple_broadcasting_n_by_one<ColMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n_by_one_1d<ColMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n_by_one_2d<ColMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n_by_one_1d<RowMajor>()); + CALL_SUBTEST(test_simple_broadcasting_one_by_n_by_one_2d<RowMajor>()); }
diff --git a/unsupported/test/cxx11_tensor_builtins_sycl.cpp b/unsupported/test/cxx11_tensor_builtins_sycl.cpp new file mode 100644 index 0000000..db29757 --- /dev/null +++ b/unsupported/test/cxx11_tensor_builtins_sycl.cpp
@@ -0,0 +1,267 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +namespace std { +template <typename T> T rsqrt(T x) { return 1 / std::sqrt(x); } +template <typename T> T square(T x) { return x * x; } +template <typename T> T cube(T x) { return x * x * x; } +template <typename T> T inverse(T x) { return 1 / x; } +} + +#define TEST_UNARY_BUILTINS_FOR_SCALAR(FUNC, SCALAR, OPERATOR, Layout) \ + { \ + /* out OPERATOR in.FUNC() */ \ + Tensor<SCALAR, 3, Layout, int64_t> in(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> out(tensorRange); \ + in = in.random() + static_cast<SCALAR>(0.01); \ + out = out.random() + static_cast<SCALAR>(0.01); \ + Tensor<SCALAR, 3, Layout, int64_t> reference(out); \ + SCALAR *gpu_data = static_cast<SCALAR *>( \ + sycl_device.allocate(in.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_out = static_cast<SCALAR *>( \ + sycl_device.allocate(out.size() * sizeof(SCALAR))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu(gpu_data, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data, in.data(), \ + (in.size()) * sizeof(SCALAR)); \ + sycl_device.memcpyHostToDevice(gpu_data_out, out.data(), \ + (out.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) OPERATOR gpu.FUNC(); \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(SCALAR)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + SCALAR ver = reference(i); \ + ver OPERATOR std::FUNC(in(i)); \ + VERIFY_IS_APPROX(out(i), ver); \ + } \ + sycl_device.deallocate(gpu_data); \ + sycl_device.deallocate(gpu_data_out); \ + } \ + { \ + /* out OPERATOR out.FUNC() */ \ + Tensor<SCALAR, 3, Layout, int64_t> out(tensorRange); \ + out = out.random() + static_cast<SCALAR>(0.01); \ + Tensor<SCALAR, 3, Layout, int64_t> reference(out); \ + SCALAR *gpu_data_out = static_cast<SCALAR *>( \ + sycl_device.allocate(out.size() * sizeof(SCALAR))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data_out, out.data(), \ + (out.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) OPERATOR gpu_out.FUNC(); \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(SCALAR)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + SCALAR ver = reference(i); \ + ver OPERATOR std::FUNC(reference(i)); \ + VERIFY_IS_APPROX(out(i), ver); \ + } \ + sycl_device.deallocate(gpu_data_out); \ + } + +#define TEST_UNARY_BUILTINS_OPERATOR(SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(abs, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(sqrt, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(rsqrt, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(square, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(cube, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(inverse, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(tanh, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(exp, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(expm1, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(log, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(abs, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(ceil, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(floor, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(round, SCALAR, OPERATOR , Layout) \ + TEST_UNARY_BUILTINS_FOR_SCALAR(log1p, SCALAR, OPERATOR , Layout) + +#define TEST_IS_THAT_RETURNS_BOOL(SCALAR, FUNC, Layout) \ + { \ + /* out = in.FUNC() */ \ + Tensor<SCALAR, 3, Layout, int64_t> in(tensorRange); \ + Tensor<bool, 3, Layout, int64_t> out(tensorRange); \ + in = in.random() + static_cast<SCALAR>(0.01); \ + SCALAR *gpu_data = static_cast<SCALAR *>( \ + sycl_device.allocate(in.size() * sizeof(SCALAR))); \ + bool *gpu_data_out = \ + static_cast<bool *>(sycl_device.allocate(out.size() * sizeof(bool))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu(gpu_data, tensorRange); \ + TensorMap<Tensor<bool, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data, in.data(), \ + (in.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) = gpu.FUNC(); \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(bool)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + VERIFY_IS_EQUAL(out(i), std::FUNC(in(i))); \ + } \ + sycl_device.deallocate(gpu_data); \ + sycl_device.deallocate(gpu_data_out); \ + } + +#define TEST_UNARY_BUILTINS(SCALAR, Layout) \ + TEST_UNARY_BUILTINS_OPERATOR(SCALAR, +=, Layout) \ + TEST_UNARY_BUILTINS_OPERATOR(SCALAR, =, Layout) \ + TEST_IS_THAT_RETURNS_BOOL(SCALAR, isnan, Layout) \ + TEST_IS_THAT_RETURNS_BOOL(SCALAR, isfinite, Layout) \ + TEST_IS_THAT_RETURNS_BOOL(SCALAR, isinf, Layout) + +static void test_builtin_unary_sycl(const Eigen::SyclDevice &sycl_device) { + int64_t sizeDim1 = 10; + int64_t sizeDim2 = 10; + int64_t sizeDim3 = 10; + array<int64_t, 3> tensorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + + TEST_UNARY_BUILTINS(float, RowMajor) + TEST_UNARY_BUILTINS(float, ColMajor) +} + +namespace std { +template <typename T> T cwiseMax(T x, T y) { return std::max(x, y); } +template <typename T> T cwiseMin(T x, T y) { return std::min(x, y); } +} + +#define TEST_BINARY_BUILTINS_FUNC(SCALAR, FUNC, Layout) \ + { \ + /* out = in_1.FUNC(in_2) */ \ + Tensor<SCALAR, 3, Layout, int64_t> in_1(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> in_2(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> out(tensorRange); \ + in_1 = in_1.random() + static_cast<SCALAR>(0.01); \ + in_2 = in_2.random() + static_cast<SCALAR>(0.01); \ + Tensor<SCALAR, 3, Layout, int64_t> reference(out); \ + SCALAR *gpu_data_1 = static_cast<SCALAR *>( \ + sycl_device.allocate(in_1.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_2 = static_cast<SCALAR *>( \ + sycl_device.allocate(in_2.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_out = static_cast<SCALAR *>( \ + sycl_device.allocate(out.size() * sizeof(SCALAR))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_1(gpu_data_1, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_2(gpu_data_2, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data_1, in_1.data(), \ + (in_1.size()) * sizeof(SCALAR)); \ + sycl_device.memcpyHostToDevice(gpu_data_2, in_2.data(), \ + (in_2.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) = gpu_1.FUNC(gpu_2); \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(SCALAR)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + SCALAR ver = reference(i); \ + ver = std::FUNC(in_1(i), in_2(i)); \ + VERIFY_IS_APPROX(out(i), ver); \ + } \ + sycl_device.deallocate(gpu_data_1); \ + sycl_device.deallocate(gpu_data_2); \ + sycl_device.deallocate(gpu_data_out); \ + } + +#define TEST_BINARY_BUILTINS_OPERATORS(SCALAR, OPERATOR, Layout) \ + { \ + /* out = in_1 OPERATOR in_2 */ \ + Tensor<SCALAR, 3, Layout, int64_t> in_1(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> in_2(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> out(tensorRange); \ + in_1 = in_1.random() + static_cast<SCALAR>(0.01); \ + in_2 = in_2.random() + static_cast<SCALAR>(0.01); \ + Tensor<SCALAR, 3, Layout, int64_t> reference(out); \ + SCALAR *gpu_data_1 = static_cast<SCALAR *>( \ + sycl_device.allocate(in_1.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_2 = static_cast<SCALAR *>( \ + sycl_device.allocate(in_2.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_out = static_cast<SCALAR *>( \ + sycl_device.allocate(out.size() * sizeof(SCALAR))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_1(gpu_data_1, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_2(gpu_data_2, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data_1, in_1.data(), \ + (in_1.size()) * sizeof(SCALAR)); \ + sycl_device.memcpyHostToDevice(gpu_data_2, in_2.data(), \ + (in_2.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) = gpu_1 OPERATOR gpu_2; \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(SCALAR)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + VERIFY_IS_APPROX(out(i), in_1(i) OPERATOR in_2(i)); \ + } \ + sycl_device.deallocate(gpu_data_1); \ + sycl_device.deallocate(gpu_data_2); \ + sycl_device.deallocate(gpu_data_out); \ + } + +#define TEST_BINARY_BUILTINS_OPERATORS_THAT_TAKES_SCALAR(SCALAR, OPERATOR, Layout) \ + { \ + /* out = in_1 OPERATOR 2 */ \ + Tensor<SCALAR, 3, Layout, int64_t> in_1(tensorRange); \ + Tensor<SCALAR, 3, Layout, int64_t> out(tensorRange); \ + in_1 = in_1.random() + static_cast<SCALAR>(0.01); \ + Tensor<SCALAR, 3, Layout, int64_t> reference(out); \ + SCALAR *gpu_data_1 = static_cast<SCALAR *>( \ + sycl_device.allocate(in_1.size() * sizeof(SCALAR))); \ + SCALAR *gpu_data_out = static_cast<SCALAR *>( \ + sycl_device.allocate(out.size() * sizeof(SCALAR))); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_1(gpu_data_1, tensorRange); \ + TensorMap<Tensor<SCALAR, 3, Layout, int64_t>> gpu_out(gpu_data_out, tensorRange); \ + sycl_device.memcpyHostToDevice(gpu_data_1, in_1.data(), \ + (in_1.size()) * sizeof(SCALAR)); \ + gpu_out.device(sycl_device) = gpu_1 OPERATOR 2; \ + sycl_device.memcpyDeviceToHost(out.data(), gpu_data_out, \ + (out.size()) * sizeof(SCALAR)); \ + for (int64_t i = 0; i < out.size(); ++i) { \ + VERIFY_IS_APPROX(out(i), in_1(i) OPERATOR 2); \ + } \ + sycl_device.deallocate(gpu_data_1); \ + sycl_device.deallocate(gpu_data_out); \ + } + +#define TEST_BINARY_BUILTINS(SCALAR, Layout) \ + TEST_BINARY_BUILTINS_FUNC(SCALAR, cwiseMax , Layout) \ + TEST_BINARY_BUILTINS_FUNC(SCALAR, cwiseMin , Layout) \ + TEST_BINARY_BUILTINS_OPERATORS(SCALAR, + , Layout) \ + TEST_BINARY_BUILTINS_OPERATORS(SCALAR, - , Layout) \ + TEST_BINARY_BUILTINS_OPERATORS(SCALAR, * , Layout) \ + TEST_BINARY_BUILTINS_OPERATORS(SCALAR, / , Layout) + +static void test_builtin_binary_sycl(const Eigen::SyclDevice &sycl_device) { + int64_t sizeDim1 = 10; + int64_t sizeDim2 = 10; + int64_t sizeDim3 = 10; + array<int64_t, 3> tensorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + TEST_BINARY_BUILTINS(float, RowMajor) + TEST_BINARY_BUILTINS_OPERATORS_THAT_TAKES_SCALAR(int, %, RowMajor) + TEST_BINARY_BUILTINS(float, ColMajor) + TEST_BINARY_BUILTINS_OPERATORS_THAT_TAKES_SCALAR(int, %, ColMajor) +} + +EIGEN_DECLARE_TEST(cxx11_tensor_builtins_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + QueueInterface queueInterface(device); + Eigen::SyclDevice sycl_device(&queueInterface); + CALL_SUBTEST(test_builtin_unary_sycl(sycl_device)); + CALL_SUBTEST(test_builtin_binary_sycl(sycl_device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_cast_float16_cuda.cu b/unsupported/test/cxx11_tensor_cast_float16_gpu.cu similarity index 91% rename from unsupported/test/cxx11_tensor_cast_float16_cuda.cu rename to unsupported/test/cxx11_tensor_cast_float16_gpu.cu index f22b99d..97923d1 100644 --- a/unsupported/test/cxx11_tensor_cast_float16_cuda.cu +++ b/unsupported/test/cxx11_tensor_cast_float16_gpu.cu
@@ -9,18 +9,17 @@ #define EIGEN_TEST_NO_LONGDOUBLE #define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_cast_float16_cuda + #define EIGEN_DEFAULT_DENSE_INDEX_TYPE int #define EIGEN_USE_GPU - #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> using Eigen::Tensor; -void test_cuda_conversion() { - Eigen::CudaStreamDevice stream; +void test_gpu_conversion() { + Eigen::GpuStreamDevice stream; Eigen::GpuDevice gpu_device(&stream); int num_elem = 101; @@ -73,8 +72,8 @@ } -void test_cxx11_tensor_cast_float16_cuda() +EIGEN_DECLARE_TEST(cxx11_tensor_cast_float16_gpu) { - CALL_SUBTEST(test_cuda_conversion()); + CALL_SUBTEST(test_gpu_conversion()); CALL_SUBTEST(test_fallback_conversion()); }
diff --git a/unsupported/test/cxx11_tensor_casts.cpp b/unsupported/test/cxx11_tensor_casts.cpp index 3c6d0d2..c4fe9a7 100644 --- a/unsupported/test/cxx11_tensor_casts.cpp +++ b/unsupported/test/cxx11_tensor_casts.cpp
@@ -105,7 +105,7 @@ } -void test_cxx11_tensor_casts() +EIGEN_DECLARE_TEST(cxx11_tensor_casts) { CALL_SUBTEST(test_simple_cast()); CALL_SUBTEST(test_vectorized_cast());
diff --git a/unsupported/test/cxx11_tensor_chipping.cpp b/unsupported/test/cxx11_tensor_chipping.cpp index 1832dec..9222744 100644 --- a/unsupported/test/cxx11_tensor_chipping.cpp +++ b/unsupported/test/cxx11_tensor_chipping.cpp
@@ -43,7 +43,7 @@ VERIFY_IS_EQUAL(chip2.dimension(2), 7); VERIFY_IS_EQUAL(chip2.dimension(3), 11); for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { + for (int j = 0; j < 5; ++j) { for (int k = 0; k < 7; ++k) { for (int l = 0; l < 11; ++l) { VERIFY_IS_EQUAL(chip2(i,j,k,l), tensor(i,1,j,k,l)); @@ -75,7 +75,7 @@ for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 5; ++k) { - for (int l = 0; l < 7; ++l) { + for (int l = 0; l < 11; ++l) { VERIFY_IS_EQUAL(chip4(i,j,k,l), tensor(i,j,k,5,l)); } } @@ -126,7 +126,7 @@ VERIFY_IS_EQUAL(chip2.dimension(2), 7); VERIFY_IS_EQUAL(chip2.dimension(3), 11); for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { + for (int j = 0; j < 5; ++j) { for (int k = 0; k < 7; ++k) { for (int l = 0; l < 11; ++l) { VERIFY_IS_EQUAL(chip2(i,j,k,l), tensor(i,1,j,k,l)); @@ -158,7 +158,7 @@ for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 5; ++k) { - for (int l = 0; l < 7; ++l) { + for (int l = 0; l < 11; ++l) { VERIFY_IS_EQUAL(chip4(i,j,k,l), tensor(i,j,k,5,l)); } } @@ -410,7 +410,7 @@ VERIFY_IS_EQUAL(chip4.data(), static_cast<float*>(0)); } -void test_cxx11_tensor_chipping() +EIGEN_DECLARE_TEST(cxx11_tensor_chipping) { CALL_SUBTEST(test_simple_chip<ColMajor>()); CALL_SUBTEST(test_simple_chip<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_chipping_sycl.cpp b/unsupported/test/cxx11_tensor_chipping_sycl.cpp new file mode 100644 index 0000000..a91efe0 --- /dev/null +++ b/unsupported/test/cxx11_tensor_chipping_sycl.cpp
@@ -0,0 +1,622 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_static_chip_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + IndexType sizeDim5 = 11; + + array<IndexType, 5> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + array<IndexType, 4> chip1TensorRange = {{sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + + Tensor<DataType, 5, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout,IndexType> chip1(chip1TensorRange); + + tensor.setRandom(); + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + const size_t chip1TensorBuffSize =chip1.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_chip1 = static_cast<DataType*>(sycl_device.allocate(chip1TensorBuffSize)); + + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip1(gpu_data_chip1, chip1TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_tensor, tensor.data(), tensorBuffSize); + gpu_chip1.device(sycl_device)=gpu_tensor.template chip<0l>(1l); + sycl_device.memcpyDeviceToHost(chip1.data(), gpu_data_chip1, chip1TensorBuffSize); + + VERIFY_IS_EQUAL(chip1.dimension(0), sizeDim2); + VERIFY_IS_EQUAL(chip1.dimension(1), sizeDim3); + VERIFY_IS_EQUAL(chip1.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip1.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim2; ++i) { + for (IndexType j = 0; j < sizeDim3; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip1(i,j,k,l), tensor(1l,i,j,k,l)); + } + } + } + } + + array<IndexType, 4> chip2TensorRange = {{sizeDim1, sizeDim3, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip2(chip2TensorRange); + const size_t chip2TensorBuffSize =chip2.size()*sizeof(DataType); + DataType* gpu_data_chip2 = static_cast<DataType*>(sycl_device.allocate(chip2TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip2(gpu_data_chip2, chip2TensorRange); + + gpu_chip2.device(sycl_device)=gpu_tensor.template chip<1l>(1l); + sycl_device.memcpyDeviceToHost(chip2.data(), gpu_data_chip2, chip2TensorBuffSize); + + VERIFY_IS_EQUAL(chip2.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip2.dimension(1), sizeDim3); + VERIFY_IS_EQUAL(chip2.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip2.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim3; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip2(i,j,k,l), tensor(i,1l,j,k,l)); + } + } + } + } + + array<IndexType, 4> chip3TensorRange = {{sizeDim1, sizeDim2, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip3(chip3TensorRange); + const size_t chip3TensorBuffSize =chip3.size()*sizeof(DataType); + DataType* gpu_data_chip3 = static_cast<DataType*>(sycl_device.allocate(chip3TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip3(gpu_data_chip3, chip3TensorRange); + + gpu_chip3.device(sycl_device)=gpu_tensor.template chip<2l>(2l); + sycl_device.memcpyDeviceToHost(chip3.data(), gpu_data_chip3, chip3TensorBuffSize); + + VERIFY_IS_EQUAL(chip3.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip3.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip3.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip3.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip3(i,j,k,l), tensor(i,j,2l,k,l)); + } + } + } + } + + array<IndexType, 4> chip4TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip4(chip4TensorRange); + const size_t chip4TensorBuffSize =chip4.size()*sizeof(DataType); + DataType* gpu_data_chip4 = static_cast<DataType*>(sycl_device.allocate(chip4TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip4(gpu_data_chip4, chip4TensorRange); + + gpu_chip4.device(sycl_device)=gpu_tensor.template chip<3l>(5l); + sycl_device.memcpyDeviceToHost(chip4.data(), gpu_data_chip4, chip4TensorBuffSize); + + VERIFY_IS_EQUAL(chip4.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip4.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip4.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(chip4.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip4(i,j,k,l), tensor(i,j,k,5l,l)); + } + } + } + } + + + array<IndexType, 4> chip5TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> chip5(chip5TensorRange); + const size_t chip5TensorBuffSize =chip5.size()*sizeof(DataType); + DataType* gpu_data_chip5 = static_cast<DataType*>(sycl_device.allocate(chip5TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip5(gpu_data_chip5, chip5TensorRange); + + gpu_chip5.device(sycl_device)=gpu_tensor.template chip<4l>(7l); + sycl_device.memcpyDeviceToHost(chip5.data(), gpu_data_chip5, chip5TensorBuffSize); + + VERIFY_IS_EQUAL(chip5.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip5.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip5.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(chip5.dimension(3), sizeDim4); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim4; ++l) { + VERIFY_IS_EQUAL(chip5(i,j,k,l), tensor(i,j,k,l,7l)); + } + } + } + } + + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_chip1); + sycl_device.deallocate(gpu_data_chip2); + sycl_device.deallocate(gpu_data_chip3); + sycl_device.deallocate(gpu_data_chip4); + sycl_device.deallocate(gpu_data_chip5); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_dynamic_chip_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + IndexType sizeDim5 = 11; + + array<IndexType, 5> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + array<IndexType, 4> chip1TensorRange = {{sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + + Tensor<DataType, 5, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout,IndexType> chip1(chip1TensorRange); + + tensor.setRandom(); + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + const size_t chip1TensorBuffSize =chip1.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_chip1 = static_cast<DataType*>(sycl_device.allocate(chip1TensorBuffSize)); + + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip1(gpu_data_chip1, chip1TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_tensor, tensor.data(), tensorBuffSize); + gpu_chip1.device(sycl_device)=gpu_tensor.chip(1l,0l); + sycl_device.memcpyDeviceToHost(chip1.data(), gpu_data_chip1, chip1TensorBuffSize); + + VERIFY_IS_EQUAL(chip1.dimension(0), sizeDim2); + VERIFY_IS_EQUAL(chip1.dimension(1), sizeDim3); + VERIFY_IS_EQUAL(chip1.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip1.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim2; ++i) { + for (IndexType j = 0; j < sizeDim3; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip1(i,j,k,l), tensor(1l,i,j,k,l)); + } + } + } + } + + array<IndexType, 4> chip2TensorRange = {{sizeDim1, sizeDim3, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip2(chip2TensorRange); + const size_t chip2TensorBuffSize =chip2.size()*sizeof(DataType); + DataType* gpu_data_chip2 = static_cast<DataType*>(sycl_device.allocate(chip2TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip2(gpu_data_chip2, chip2TensorRange); + + gpu_chip2.device(sycl_device)=gpu_tensor.chip(1l,1l); + sycl_device.memcpyDeviceToHost(chip2.data(), gpu_data_chip2, chip2TensorBuffSize); + + VERIFY_IS_EQUAL(chip2.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip2.dimension(1), sizeDim3); + VERIFY_IS_EQUAL(chip2.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip2.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim3; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip2(i,j,k,l), tensor(i,1l,j,k,l)); + } + } + } + } + + array<IndexType, 4> chip3TensorRange = {{sizeDim1, sizeDim2, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip3(chip3TensorRange); + const size_t chip3TensorBuffSize =chip3.size()*sizeof(DataType); + DataType* gpu_data_chip3 = static_cast<DataType*>(sycl_device.allocate(chip3TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip3(gpu_data_chip3, chip3TensorRange); + + gpu_chip3.device(sycl_device)=gpu_tensor.chip(2l,2l); + sycl_device.memcpyDeviceToHost(chip3.data(), gpu_data_chip3, chip3TensorBuffSize); + + VERIFY_IS_EQUAL(chip3.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip3.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip3.dimension(2), sizeDim4); + VERIFY_IS_EQUAL(chip3.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim4; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip3(i,j,k,l), tensor(i,j,2l,k,l)); + } + } + } + } + + array<IndexType, 4> chip4TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> chip4(chip4TensorRange); + const size_t chip4TensorBuffSize =chip4.size()*sizeof(DataType); + DataType* gpu_data_chip4 = static_cast<DataType*>(sycl_device.allocate(chip4TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip4(gpu_data_chip4, chip4TensorRange); + + gpu_chip4.device(sycl_device)=gpu_tensor.chip(5l,3l); + sycl_device.memcpyDeviceToHost(chip4.data(), gpu_data_chip4, chip4TensorBuffSize); + + VERIFY_IS_EQUAL(chip4.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip4.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip4.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(chip4.dimension(3), sizeDim5); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim5; ++l) { + VERIFY_IS_EQUAL(chip4(i,j,k,l), tensor(i,j,k,5l,l)); + } + } + } + } + + + array<IndexType, 4> chip5TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> chip5(chip5TensorRange); + const size_t chip5TensorBuffSize =chip5.size()*sizeof(DataType); + DataType* gpu_data_chip5 = static_cast<DataType*>(sycl_device.allocate(chip5TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip5(gpu_data_chip5, chip5TensorRange); + + gpu_chip5.device(sycl_device)=gpu_tensor.chip(7l,4l); + sycl_device.memcpyDeviceToHost(chip5.data(), gpu_data_chip5, chip5TensorBuffSize); + + VERIFY_IS_EQUAL(chip5.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(chip5.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(chip5.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(chip5.dimension(3), sizeDim4); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim4; ++l) { + VERIFY_IS_EQUAL(chip5(i,j,k,l), tensor(i,j,k,l,7l)); + } + } + } + } + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_chip1); + sycl_device.deallocate(gpu_data_chip2); + sycl_device.deallocate(gpu_data_chip3); + sycl_device.deallocate(gpu_data_chip4); + sycl_device.deallocate(gpu_data_chip5); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_chip_in_expr(const Eigen::SyclDevice& sycl_device) { + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + IndexType sizeDim5 = 11; + + array<IndexType, 5> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + array<IndexType, 4> chip1TensorRange = {{sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + + Tensor<DataType, 5, DataLayout,IndexType> tensor(tensorRange); + + Tensor<DataType, 4, DataLayout,IndexType> chip1(chip1TensorRange); + Tensor<DataType, 4, DataLayout,IndexType> tensor1(chip1TensorRange); + tensor.setRandom(); + tensor1.setRandom(); + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + const size_t chip1TensorBuffSize =chip1.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_chip1 = static_cast<DataType*>(sycl_device.allocate(chip1TensorBuffSize)); + DataType* gpu_data_tensor1 = static_cast<DataType*>(sycl_device.allocate(chip1TensorBuffSize)); + + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_chip1(gpu_data_chip1, chip1TensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_tensor1(gpu_data_tensor1, chip1TensorRange); + + + sycl_device.memcpyHostToDevice(gpu_data_tensor, tensor.data(), tensorBuffSize); + sycl_device.memcpyHostToDevice(gpu_data_tensor1, tensor1.data(), chip1TensorBuffSize); + gpu_chip1.device(sycl_device)=gpu_tensor.template chip<0l>(0l) + gpu_tensor1; + sycl_device.memcpyDeviceToHost(chip1.data(), gpu_data_chip1, chip1TensorBuffSize); + + for (int i = 0; i < sizeDim2; ++i) { + for (int j = 0; j < sizeDim3; ++j) { + for (int k = 0; k < sizeDim4; ++k) { + for (int l = 0; l < sizeDim5; ++l) { + float expected = tensor(0l,i,j,k,l) + tensor1(i,j,k,l); + VERIFY_IS_EQUAL(chip1(i,j,k,l), expected); + } + } + } + } + + array<IndexType, 3> chip2TensorRange = {{sizeDim2, sizeDim4, sizeDim5}}; + Tensor<DataType, 3, DataLayout,IndexType> tensor2(chip2TensorRange); + Tensor<DataType, 3, DataLayout,IndexType> chip2(chip2TensorRange); + tensor2.setRandom(); + const size_t chip2TensorBuffSize =tensor2.size()*sizeof(DataType); + DataType* gpu_data_tensor2 = static_cast<DataType*>(sycl_device.allocate(chip2TensorBuffSize)); + DataType* gpu_data_chip2 = static_cast<DataType*>(sycl_device.allocate(chip2TensorBuffSize)); + TensorMap<Tensor<DataType, 3, DataLayout,IndexType>> gpu_tensor2(gpu_data_tensor2, chip2TensorRange); + TensorMap<Tensor<DataType, 3, DataLayout,IndexType>> gpu_chip2(gpu_data_chip2, chip2TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_tensor2, tensor2.data(), chip2TensorBuffSize); + gpu_chip2.device(sycl_device)=gpu_tensor.template chip<0l>(0l).template chip<1l>(2l) + gpu_tensor2; + sycl_device.memcpyDeviceToHost(chip2.data(), gpu_data_chip2, chip2TensorBuffSize); + + for (int i = 0; i < sizeDim2; ++i) { + for (int j = 0; j < sizeDim4; ++j) { + for (int k = 0; k < sizeDim5; ++k) { + float expected = tensor(0l,i,2l,j,k) + tensor2(i,j,k); + VERIFY_IS_EQUAL(chip2(i,j,k), expected); + } + } + } + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_tensor1); + sycl_device.deallocate(gpu_data_chip1); + sycl_device.deallocate(gpu_data_tensor2); + sycl_device.deallocate(gpu_data_chip2); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_chip_as_lvalue_sycl(const Eigen::SyclDevice& sycl_device) +{ + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + IndexType sizeDim5 = 11; + + array<IndexType, 5> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + array<IndexType, 4> input2TensorRange = {{sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + + Tensor<DataType, 5, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 5, DataLayout,IndexType> input1(tensorRange); + Tensor<DataType, 4, DataLayout,IndexType> input2(input2TensorRange); + input1.setRandom(); + input2.setRandom(); + + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + const size_t input2TensorBuffSize =input2.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_input1 = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_input2 = static_cast<DataType*>(sycl_device.allocate(input2TensorBuffSize)); + + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_input1(gpu_data_input1, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_input2(gpu_data_input2, input2TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input1, input1.data(), tensorBuffSize); + gpu_tensor.device(sycl_device)=gpu_input1; + sycl_device.memcpyHostToDevice(gpu_data_input2, input2.data(), input2TensorBuffSize); + gpu_tensor.template chip<0l>(1l).device(sycl_device)=gpu_input2; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k < sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (i != 1) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input2(j,k,l,m)); + } + } + } + } + } + } + + gpu_tensor.device(sycl_device)=gpu_input1; + array<IndexType, 4> input3TensorRange = {{sizeDim1, sizeDim3, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> input3(input3TensorRange); + input3.setRandom(); + + const size_t input3TensorBuffSize =input3.size()*sizeof(DataType); + DataType* gpu_data_input3 = static_cast<DataType*>(sycl_device.allocate(input3TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_input3(gpu_data_input3, input3TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input3, input3.data(), input3TensorBuffSize); + gpu_tensor.template chip<1l>(1l).device(sycl_device)=gpu_input3; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k <sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (j != 1) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input3(i,k,l,m)); + } + } + } + } + } + } + + gpu_tensor.device(sycl_device)=gpu_input1; + array<IndexType, 4> input4TensorRange = {{sizeDim1, sizeDim2, sizeDim4, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> input4(input4TensorRange); + input4.setRandom(); + + const size_t input4TensorBuffSize =input4.size()*sizeof(DataType); + DataType* gpu_data_input4 = static_cast<DataType*>(sycl_device.allocate(input4TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_input4(gpu_data_input4, input4TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input4, input4.data(), input4TensorBuffSize); + gpu_tensor.template chip<2l>(3l).device(sycl_device)=gpu_input4; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k <sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (k != 3) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input4(i,j,l,m)); + } + } + } + } + } + } + + gpu_tensor.device(sycl_device)=gpu_input1; + array<IndexType, 4> input5TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim5}}; + Tensor<DataType, 4, DataLayout,IndexType> input5(input5TensorRange); + input5.setRandom(); + + const size_t input5TensorBuffSize =input5.size()*sizeof(DataType); + DataType* gpu_data_input5 = static_cast<DataType*>(sycl_device.allocate(input5TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_input5(gpu_data_input5, input5TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input5, input5.data(), input5TensorBuffSize); + gpu_tensor.template chip<3l>(4l).device(sycl_device)=gpu_input5; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k <sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (l != 4) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input5(i,j,k,m)); + } + } + } + } + } + } + gpu_tensor.device(sycl_device)=gpu_input1; + array<IndexType, 4> input6TensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> input6(input6TensorRange); + input6.setRandom(); + + const size_t input6TensorBuffSize =input6.size()*sizeof(DataType); + DataType* gpu_data_input6 = static_cast<DataType*>(sycl_device.allocate(input6TensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_input6(gpu_data_input6, input6TensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input6, input6.data(), input6TensorBuffSize); + gpu_tensor.template chip<4l>(5l).device(sycl_device)=gpu_input6; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k <sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (m != 5) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input6(i,j,k,l)); + } + } + } + } + } + } + + + gpu_tensor.device(sycl_device)=gpu_input1; + Tensor<DataType, 5, DataLayout,IndexType> input7(tensorRange); + input7.setRandom(); + + DataType* gpu_data_input7 = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_input7(gpu_data_input7, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_input7, input7.data(), tensorBuffSize); + gpu_tensor.chip(0l,0l).device(sycl_device)=gpu_input7.chip(0l,0l); + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data_tensor, tensorBuffSize); + + for (int i = 0; i < sizeDim1; ++i) { + for (int j = 0; j < sizeDim2; ++j) { + for (int k = 0; k <sizeDim3; ++k) { + for (int l = 0; l < sizeDim4; ++l) { + for (int m = 0; m < sizeDim5; ++m) { + if (i != 0) { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input1(i,j,k,l,m)); + } else { + VERIFY_IS_EQUAL(tensor(i,j,k,l,m), input7(i,j,k,l,m)); + } + } + } + } + } + } + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_input1); + sycl_device.deallocate(gpu_data_input2); + sycl_device.deallocate(gpu_data_input3); + sycl_device.deallocate(gpu_data_input4); + sycl_device.deallocate(gpu_data_input5); + sycl_device.deallocate(gpu_data_input6); + sycl_device.deallocate(gpu_data_input7); + +} + +template<typename DataType, typename dev_Selector> void sycl_chipping_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_static_chip_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_static_chip_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_dynamic_chip_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_dynamic_chip_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_chip_in_expr<DataType, RowMajor, int64_t>(sycl_device); + test_chip_in_expr<DataType, ColMajor, int64_t>(sycl_device); + test_chip_as_lvalue_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_chip_as_lvalue_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_chipping_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_chipping_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_comparisons.cpp b/unsupported/test/cxx11_tensor_comparisons.cpp index b1ff8ae..1a18e07 100644 --- a/unsupported/test/cxx11_tensor_comparisons.cpp +++ b/unsupported/test/cxx11_tensor_comparisons.cpp
@@ -77,7 +77,7 @@ } -void test_cxx11_tensor_comparisons() +EIGEN_DECLARE_TEST(cxx11_tensor_comparisons) { CALL_SUBTEST(test_orderings()); CALL_SUBTEST(test_equality());
diff --git a/unsupported/test/cxx11_tensor_complex_cwise_ops_gpu.cu b/unsupported/test/cxx11_tensor_complex_cwise_ops_gpu.cu new file mode 100644 index 0000000..f2a2a6c --- /dev/null +++ b/unsupported/test/cxx11_tensor_complex_cwise_ops_gpu.cu
@@ -0,0 +1,100 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE + +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template<typename T> +void test_cuda_complex_cwise_ops() { + const int kNumItems = 2; + std::size_t complex_bytes = kNumItems * sizeof(std::complex<T>); + + std::complex<T>* d_in1; + std::complex<T>* d_in2; + std::complex<T>* d_out; + cudaMalloc((void**)(&d_in1), complex_bytes); + cudaMalloc((void**)(&d_in2), complex_bytes); + cudaMalloc((void**)(&d_out), complex_bytes); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_in1( + d_in1, kNumItems); + Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_in2( + d_in2, kNumItems); + Eigen::TensorMap<Eigen::Tensor<std::complex<T>, 1, 0, int>, Eigen::Aligned> gpu_out( + d_out, kNumItems); + + const std::complex<T> a(3.14f, 2.7f); + const std::complex<T> b(-10.6f, 1.4f); + + gpu_in1.device(gpu_device) = gpu_in1.constant(a); + gpu_in2.device(gpu_device) = gpu_in2.constant(b); + + enum CwiseOp { + Add = 0, + Sub, + Mul, + Div, + Neg, + NbOps + }; + + Tensor<std::complex<T>, 1, 0, int> actual(kNumItems); + for (int op = Add; op < NbOps; op++) { + std::complex<T> expected; + switch (static_cast<CwiseOp>(op)) { + case Add: + gpu_out.device(gpu_device) = gpu_in1 + gpu_in2; + expected = a + b; + break; + case Sub: + gpu_out.device(gpu_device) = gpu_in1 - gpu_in2; + expected = a - b; + break; + case Mul: + gpu_out.device(gpu_device) = gpu_in1 * gpu_in2; + expected = a * b; + break; + case Div: + gpu_out.device(gpu_device) = gpu_in1 / gpu_in2; + expected = a / b; + break; + case Neg: + gpu_out.device(gpu_device) = -gpu_in1; + expected = -a; + break; + } + assert(cudaMemcpyAsync(actual.data(), d_out, complex_bytes, cudaMemcpyDeviceToHost, + gpu_device.stream()) == cudaSuccess); + assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + + for (int i = 0; i < kNumItems; ++i) { + VERIFY_IS_APPROX(actual(i), expected); + } + } + + cudaFree(d_in1); + cudaFree(d_in2); + cudaFree(d_out); +} + + +EIGEN_DECLARE_TEST(test_cxx11_tensor_complex_cwise_ops) +{ + CALL_SUBTEST(test_cuda_complex_cwise_ops<float>()); + CALL_SUBTEST(test_cuda_complex_cwise_ops<double>()); +}
diff --git a/unsupported/test/cxx11_tensor_complex_gpu.cu b/unsupported/test/cxx11_tensor_complex_gpu.cu new file mode 100644 index 0000000..f8b8ae7 --- /dev/null +++ b/unsupported/test/cxx11_tensor_complex_gpu.cu
@@ -0,0 +1,186 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE + +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +void test_cuda_nullary() { + Tensor<std::complex<float>, 1, 0, int> in1(2); + Tensor<std::complex<float>, 1, 0, int> in2(2); + in1.setRandom(); + in2.setRandom(); + + std::size_t float_bytes = in1.size() * sizeof(float); + std::size_t complex_bytes = in1.size() * sizeof(std::complex<float>); + + std::complex<float>* d_in1; + std::complex<float>* d_in2; + float* d_out2; + cudaMalloc((void**)(&d_in1), complex_bytes); + cudaMalloc((void**)(&d_in2), complex_bytes); + cudaMalloc((void**)(&d_out2), float_bytes); + cudaMemcpy(d_in1, in1.data(), complex_bytes, cudaMemcpyHostToDevice); + cudaMemcpy(d_in2, in2.data(), complex_bytes, cudaMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<std::complex<float>, 1, 0, int>, Eigen::Aligned> gpu_in1( + d_in1, 2); + Eigen::TensorMap<Eigen::Tensor<std::complex<float>, 1, 0, int>, Eigen::Aligned> gpu_in2( + d_in2, 2); + Eigen::TensorMap<Eigen::Tensor<float, 1, 0, int>, Eigen::Aligned> gpu_out2( + d_out2, 2); + + gpu_in1.device(gpu_device) = gpu_in1.constant(std::complex<float>(3.14f, 2.7f)); + gpu_out2.device(gpu_device) = gpu_in2.abs(); + + Tensor<std::complex<float>, 1, 0, int> new1(2); + Tensor<float, 1, 0, int> new2(2); + + assert(cudaMemcpyAsync(new1.data(), d_in1, complex_bytes, cudaMemcpyDeviceToHost, + gpu_device.stream()) == cudaSuccess); + assert(cudaMemcpyAsync(new2.data(), d_out2, float_bytes, cudaMemcpyDeviceToHost, + gpu_device.stream()) == cudaSuccess); + + assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); + + for (int i = 0; i < 2; ++i) { + VERIFY_IS_APPROX(new1(i), std::complex<float>(3.14f, 2.7f)); + VERIFY_IS_APPROX(new2(i), std::abs(in2(i))); + } + + cudaFree(d_in1); + cudaFree(d_in2); + cudaFree(d_out2); +} + + +static void test_cuda_sum_reductions() { + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + const int num_rows = internal::random<int>(1024, 5*1024); + const int num_cols = internal::random<int>(1024, 5*1024); + + Tensor<std::complex<float>, 2> in(num_rows, num_cols); + in.setRandom(); + + Tensor<std::complex<float>, 0> full_redux; + full_redux = in.sum(); + + std::size_t in_bytes = in.size() * sizeof(std::complex<float>); + std::size_t out_bytes = full_redux.size() * sizeof(std::complex<float>); + std::complex<float>* gpu_in_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(in_bytes)); + std::complex<float>* gpu_out_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(out_bytes)); + gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes); + + TensorMap<Tensor<std::complex<float>, 2> > in_gpu(gpu_in_ptr, num_rows, num_cols); + TensorMap<Tensor<std::complex<float>, 0> > out_gpu(gpu_out_ptr); + + out_gpu.device(gpu_device) = in_gpu.sum(); + + Tensor<std::complex<float>, 0> full_redux_gpu; + gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes); + gpu_device.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux(), full_redux_gpu()); + + gpu_device.deallocate(gpu_in_ptr); + gpu_device.deallocate(gpu_out_ptr); +} + +static void test_cuda_mean_reductions() { + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + const int num_rows = internal::random<int>(1024, 5*1024); + const int num_cols = internal::random<int>(1024, 5*1024); + + Tensor<std::complex<float>, 2> in(num_rows, num_cols); + in.setRandom(); + + Tensor<std::complex<float>, 0> full_redux; + full_redux = in.mean(); + + std::size_t in_bytes = in.size() * sizeof(std::complex<float>); + std::size_t out_bytes = full_redux.size() * sizeof(std::complex<float>); + std::complex<float>* gpu_in_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(in_bytes)); + std::complex<float>* gpu_out_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(out_bytes)); + gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes); + + TensorMap<Tensor<std::complex<float>, 2> > in_gpu(gpu_in_ptr, num_rows, num_cols); + TensorMap<Tensor<std::complex<float>, 0> > out_gpu(gpu_out_ptr); + + out_gpu.device(gpu_device) = in_gpu.mean(); + + Tensor<std::complex<float>, 0> full_redux_gpu; + gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes); + gpu_device.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux(), full_redux_gpu()); + + gpu_device.deallocate(gpu_in_ptr); + gpu_device.deallocate(gpu_out_ptr); +} + +static void test_cuda_product_reductions() { + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + const int num_rows = internal::random<int>(1024, 5*1024); + const int num_cols = internal::random<int>(1024, 5*1024); + + Tensor<std::complex<float>, 2> in(num_rows, num_cols); + in.setRandom(); + + Tensor<std::complex<float>, 0> full_redux; + full_redux = in.prod(); + + std::size_t in_bytes = in.size() * sizeof(std::complex<float>); + std::size_t out_bytes = full_redux.size() * sizeof(std::complex<float>); + std::complex<float>* gpu_in_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(in_bytes)); + std::complex<float>* gpu_out_ptr = static_cast<std::complex<float>*>(gpu_device.allocate(out_bytes)); + gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes); + + TensorMap<Tensor<std::complex<float>, 2> > in_gpu(gpu_in_ptr, num_rows, num_cols); + TensorMap<Tensor<std::complex<float>, 0> > out_gpu(gpu_out_ptr); + + out_gpu.device(gpu_device) = in_gpu.prod(); + + Tensor<std::complex<float>, 0> full_redux_gpu; + gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes); + gpu_device.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux(), full_redux_gpu()); + + gpu_device.deallocate(gpu_in_ptr); + gpu_device.deallocate(gpu_out_ptr); +} + + +EIGEN_DECLARE_TEST(test_cxx11_tensor_complex) +{ + CALL_SUBTEST(test_cuda_nullary()); + CALL_SUBTEST(test_cuda_sum_reductions()); + CALL_SUBTEST(test_cuda_mean_reductions()); + CALL_SUBTEST(test_cuda_product_reductions()); +}
diff --git a/unsupported/test/cxx11_tensor_concatenation.cpp b/unsupported/test/cxx11_tensor_concatenation.cpp index 03ef12e..bb9418d 100644 --- a/unsupported/test/cxx11_tensor_concatenation.cpp +++ b/unsupported/test/cxx11_tensor_concatenation.cpp
@@ -50,7 +50,13 @@ .reshape(Tensor<int, 3>::Dimensions(2, 3, 1)) .concatenate(right, 0); Tensor<int, 2, DataLayout> alternative = left - .concatenate(right.reshape(Tensor<int, 2>::Dimensions{{{2, 3}}}), 0); + // Clang compiler break with {{{}}} with an ambiguous error on copy constructor + // the variadic DSize constructor added for #ifndef EIGEN_EMULATE_CXX11_META_H. + // Solution: + // either the code should change to + // Tensor<int, 2>::Dimensions{{2, 3}} + // or Tensor<int, 2>::Dimensions{Tensor<int, 2>::Dimensions{{2, 3}}} + .concatenate(right.reshape(Tensor<int, 2>::Dimensions(2, 3)), 0); } template<int DataLayout> @@ -123,7 +129,7 @@ } -void test_cxx11_tensor_concatenation() +EIGEN_DECLARE_TEST(cxx11_tensor_concatenation) { CALL_SUBTEST(test_dimension_failures<ColMajor>()); CALL_SUBTEST(test_dimension_failures<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_concatenation_sycl.cpp b/unsupported/test/cxx11_tensor_concatenation_sycl.cpp new file mode 100644 index 0000000..765991b --- /dev/null +++ b/unsupported/test/cxx11_tensor_concatenation_sycl.cpp
@@ -0,0 +1,180 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template<typename DataType, int DataLayout, typename IndexType> +static void test_simple_concatenation(const Eigen::SyclDevice& sycl_device) +{ + IndexType leftDim1 = 2; + IndexType leftDim2 = 3; + IndexType leftDim3 = 1; + Eigen::array<IndexType, 3> leftRange = {{leftDim1, leftDim2, leftDim3}}; + IndexType rightDim1 = 2; + IndexType rightDim2 = 3; + IndexType rightDim3 = 1; + Eigen::array<IndexType, 3> rightRange = {{rightDim1, rightDim2, rightDim3}}; + + //IndexType concatDim1 = 3; +// IndexType concatDim2 = 3; +// IndexType concatDim3 = 1; + //Eigen::array<IndexType, 3> concatRange = {{concatDim1, concatDim2, concatDim3}}; + + Tensor<DataType, 3, DataLayout, IndexType> left(leftRange); + Tensor<DataType, 3, DataLayout, IndexType> right(rightRange); + left.setRandom(); + right.setRandom(); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(left.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_in2_data = static_cast<DataType*>(sycl_device.allocate(right.dimensions().TotalSize()*sizeof(DataType))); + + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_in1(gpu_in1_data, leftRange); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_in2(gpu_in2_data, rightRange); + sycl_device.memcpyHostToDevice(gpu_in1_data, left.data(),(left.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_in2_data, right.data(),(right.dimensions().TotalSize())*sizeof(DataType)); + /// + Tensor<DataType, 3, DataLayout, IndexType> concatenation1(leftDim1+rightDim1, leftDim2, leftDim3); + DataType * gpu_out_data1 = static_cast<DataType*>(sycl_device.allocate(concatenation1.dimensions().TotalSize()*sizeof(DataType))); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_out1(gpu_out_data1, concatenation1.dimensions()); + + //concatenation = left.concatenate(right, 0); + gpu_out1.device(sycl_device) =gpu_in1.concatenate(gpu_in2, 0); + sycl_device.memcpyDeviceToHost(concatenation1.data(), gpu_out_data1,(concatenation1.dimensions().TotalSize())*sizeof(DataType)); + + VERIFY_IS_EQUAL(concatenation1.dimension(0), 4); + VERIFY_IS_EQUAL(concatenation1.dimension(1), 3); + VERIFY_IS_EQUAL(concatenation1.dimension(2), 1); + for (IndexType j = 0; j < 3; ++j) { + for (IndexType i = 0; i < 2; ++i) { + VERIFY_IS_EQUAL(concatenation1(i, j, 0), left(i, j, 0)); + } + for (IndexType i = 2; i < 4; ++i) { + VERIFY_IS_EQUAL(concatenation1(i, j, 0), right(i - 2, j, 0)); + } + } + + sycl_device.deallocate(gpu_out_data1); + Tensor<DataType, 3, DataLayout, IndexType> concatenation2(leftDim1, leftDim2 +rightDim2, leftDim3); + DataType * gpu_out_data2 = static_cast<DataType*>(sycl_device.allocate(concatenation2.dimensions().TotalSize()*sizeof(DataType))); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_out2(gpu_out_data2, concatenation2.dimensions()); + gpu_out2.device(sycl_device) =gpu_in1.concatenate(gpu_in2, 1); + sycl_device.memcpyDeviceToHost(concatenation2.data(), gpu_out_data2,(concatenation2.dimensions().TotalSize())*sizeof(DataType)); + + //concatenation = left.concatenate(right, 1); + VERIFY_IS_EQUAL(concatenation2.dimension(0), 2); + VERIFY_IS_EQUAL(concatenation2.dimension(1), 6); + VERIFY_IS_EQUAL(concatenation2.dimension(2), 1); + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + VERIFY_IS_EQUAL(concatenation2(i, j, 0), left(i, j, 0)); + } + for (IndexType j = 3; j < 6; ++j) { + VERIFY_IS_EQUAL(concatenation2(i, j, 0), right(i, j - 3, 0)); + } + } + sycl_device.deallocate(gpu_out_data2); + Tensor<DataType, 3, DataLayout, IndexType> concatenation3(leftDim1, leftDim2, leftDim3+rightDim3); + DataType * gpu_out_data3 = static_cast<DataType*>(sycl_device.allocate(concatenation3.dimensions().TotalSize()*sizeof(DataType))); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_out3(gpu_out_data3, concatenation3.dimensions()); + gpu_out3.device(sycl_device) =gpu_in1.concatenate(gpu_in2, 2); + sycl_device.memcpyDeviceToHost(concatenation3.data(), gpu_out_data3,(concatenation3.dimensions().TotalSize())*sizeof(DataType)); + + //concatenation = left.concatenate(right, 2); + VERIFY_IS_EQUAL(concatenation3.dimension(0), 2); + VERIFY_IS_EQUAL(concatenation3.dimension(1), 3); + VERIFY_IS_EQUAL(concatenation3.dimension(2), 2); + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + VERIFY_IS_EQUAL(concatenation3(i, j, 0), left(i, j, 0)); + VERIFY_IS_EQUAL(concatenation3(i, j, 1), right(i, j, 0)); + } + } + sycl_device.deallocate(gpu_out_data3); + sycl_device.deallocate(gpu_in1_data); + sycl_device.deallocate(gpu_in2_data); +} +template<typename DataType, int DataLayout, typename IndexType> +static void test_concatenation_as_lvalue(const Eigen::SyclDevice& sycl_device) +{ + + IndexType leftDim1 = 2; + IndexType leftDim2 = 3; + Eigen::array<IndexType, 2> leftRange = {{leftDim1, leftDim2}}; + + IndexType rightDim1 = 2; + IndexType rightDim2 = 3; + Eigen::array<IndexType, 2> rightRange = {{rightDim1, rightDim2}}; + + IndexType concatDim1 = 4; + IndexType concatDim2 = 3; + Eigen::array<IndexType, 2> resRange = {{concatDim1, concatDim2}}; + + Tensor<DataType, 2, DataLayout, IndexType> left(leftRange); + Tensor<DataType, 2, DataLayout, IndexType> right(rightRange); + Tensor<DataType, 2, DataLayout, IndexType> result(resRange); + + left.setRandom(); + right.setRandom(); + result.setRandom(); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(left.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_in2_data = static_cast<DataType*>(sycl_device.allocate(right.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(result.dimensions().TotalSize()*sizeof(DataType))); + + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType>> gpu_in1(gpu_in1_data, leftRange); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType>> gpu_in2(gpu_in2_data, rightRange); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType>> gpu_out(gpu_out_data, resRange); + + sycl_device.memcpyHostToDevice(gpu_in1_data, left.data(),(left.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_in2_data, right.data(),(right.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_out_data, result.data(),(result.dimensions().TotalSize())*sizeof(DataType)); + +// t1.concatenate(t2, 0) = result; + gpu_in1.concatenate(gpu_in2, 0).device(sycl_device) =gpu_out; + sycl_device.memcpyDeviceToHost(left.data(), gpu_in1_data,(left.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(right.data(), gpu_in2_data,(right.dimensions().TotalSize())*sizeof(DataType)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + VERIFY_IS_EQUAL(left(i, j), result(i, j)); + VERIFY_IS_EQUAL(right(i, j), result(i+2, j)); + } + } + sycl_device.deallocate(gpu_in1_data); + sycl_device.deallocate(gpu_in2_data); + sycl_device.deallocate(gpu_out_data); +} + + +template <typename DataType, typename Dev_selector> void tensorConcat_perDevice(Dev_selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_concatenation<DataType, RowMajor, int64_t>(sycl_device); + test_simple_concatenation<DataType, ColMajor, int64_t>(sycl_device); + test_concatenation_as_lvalue<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_concatenation_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(tensorConcat_perDevice<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_const.cpp b/unsupported/test/cxx11_tensor_const.cpp index ad9c9da..9d806ee 100644 --- a/unsupported/test/cxx11_tensor_const.cpp +++ b/unsupported/test/cxx11_tensor_const.cpp
@@ -55,7 +55,7 @@ } -void test_cxx11_tensor_const() +EIGEN_DECLARE_TEST(cxx11_tensor_const) { CALL_SUBTEST(test_simple_assign()); CALL_SUBTEST(test_assign_of_const_tensor());
diff --git a/unsupported/test/cxx11_tensor_contract_cuda.cu b/unsupported/test/cxx11_tensor_contract_cuda.cu deleted file mode 100644 index 6d1ef07..0000000 --- a/unsupported/test/cxx11_tensor_contract_cuda.cu +++ /dev/null
@@ -1,152 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// Copyright (C) 2014 Navdeep Jaitly <ndjaitly@google.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_cuda -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int -#define EIGEN_USE_GPU - - -#include "main.h" -#include <unsupported/Eigen/CXX11/Tensor> - -using Eigen::Tensor; -typedef Tensor<float, 1>::DimensionPair DimPair; - -template<int DataLayout> -void test_cuda_contraction(int m_size, int k_size, int n_size) -{ - std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; - // with these dimensions, the output has 300 * 140 elements, which is - // more than 30 * 1024, which is the number of threads in blocks on - // a 15 SM GK110 GPU - Tensor<float, 2, DataLayout> t_left(m_size, k_size); - Tensor<float, 2, DataLayout> t_right(k_size, n_size); - Tensor<float, 2, DataLayout> t_result(m_size, n_size); - Tensor<float, 2, DataLayout> t_result_gpu(m_size, n_size); - Eigen::array<DimPair, 1> dims(DimPair(1, 0)); - - t_left.setRandom(); - t_right.setRandom(); - - std::size_t t_left_bytes = t_left.size() * sizeof(float); - std::size_t t_right_bytes = t_right.size() * sizeof(float); - std::size_t t_result_bytes = t_result.size() * sizeof(float); - - float* d_t_left; - float* d_t_right; - float* d_t_result; - - cudaMalloc((void**)(&d_t_left), t_left_bytes); - cudaMalloc((void**)(&d_t_right), t_right_bytes); - cudaMalloc((void**)(&d_t_result), t_result_bytes); - - cudaMemcpy(d_t_left, t_left.data(), t_left_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_t_right, t_right.data(), t_right_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > - gpu_t_left(d_t_left, Eigen::array<int, 2>(m_size, k_size)); - Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > - gpu_t_right(d_t_right, Eigen::array<int, 2>(k_size, n_size)); - Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > - gpu_t_result(d_t_result, Eigen::array<int, 2>(m_size, n_size)); - - - gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims); - t_result = t_left.contract(t_right, dims); - - cudaMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, cudaMemcpyDeviceToHost); - for (size_t i = 0; i < t_result.size(); i++) { - if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) { - continue; - } - if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) { - continue; - } - std::cout << "mismatch detected at index " << i << ": " << t_result(i) - << " vs " << t_result_gpu(i) << std::endl; - assert(false); - } - - cudaFree((void*)d_t_left); - cudaFree((void*)d_t_right); - cudaFree((void*)d_t_result); -} - -template<int DataLayout> -void test_cuda_contraction_m() { - for (int k = 32; k < 256; k++) { - test_cuda_contraction<ColMajor>(k, 128, 128); - test_cuda_contraction<RowMajor>(k, 128, 128); - } -} - -template<int DataLayout> -void test_cuda_contraction_k() { - for (int k = 32; k < 256; k++) { - test_cuda_contraction<ColMajor>(128, k, 128); - test_cuda_contraction<RowMajor>(128, k, 128); - } -} - -template<int DataLayout> -void test_cuda_contraction_n() { - for (int k = 32; k < 256; k++) { - test_cuda_contraction<ColMajor>(128, 128, k); - test_cuda_contraction<RowMajor>(128, 128, k); - } -} - - -template<int DataLayout> -void test_cuda_contraction_sizes() { - int m_sizes[] = { 31, 39, 63, 64, 65, - 127, 129, 255, 257 , 511, - 512, 513, 1023, 1024, 1025}; - - int n_sizes[] = { 31, 39, 63, 64, 65, - 127, 129, 255, 257, 511, - 512, 513, 1023, 1024, 1025}; - - int k_sizes[] = { 31, 39, 63, 64, 65, - 95, 96, 127, 129, 255, - 257, 511, 512, 513, 1023, - 1024, 1025}; - - for (int i = 0; i < 15; i++) { - for (int j = 0; j < 15; j++) { - for (int k = 0; k < 17; k++) { - test_cuda_contraction<DataLayout>(m_sizes[i], n_sizes[j], k_sizes[k]); - } - } - } -} - -void test_cxx11_tensor_cuda() -{ - CALL_SUBTEST_1(test_cuda_contraction<ColMajor>(128, 128, 128)); - CALL_SUBTEST_1(test_cuda_contraction<RowMajor>(128, 128, 128)); - - CALL_SUBTEST_2(test_cuda_contraction_m<ColMajor>()); - CALL_SUBTEST_3(test_cuda_contraction_m<RowMajor>()); - - CALL_SUBTEST_4(test_cuda_contraction_k<ColMajor>()); - CALL_SUBTEST_5(test_cuda_contraction_k<RowMajor>()); - - CALL_SUBTEST_6(test_cuda_contraction_n<ColMajor>()); - CALL_SUBTEST_7(test_cuda_contraction_n<RowMajor>()); - - CALL_SUBTEST_8(test_cuda_contraction_sizes<ColMajor>()); - CALL_SUBTEST_9(test_cuda_contraction_sizes<RowMajor>()); -}
diff --git a/unsupported/test/cxx11_tensor_contract_gpu.cu b/unsupported/test/cxx11_tensor_contract_gpu.cu new file mode 100644 index 0000000..575bdc1 --- /dev/null +++ b/unsupported/test/cxx11_tensor_contract_gpu.cu
@@ -0,0 +1,218 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// Copyright (C) 2014 Navdeep Jaitly <ndjaitly@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + +using Eigen::Tensor; +typedef Tensor<float, 1>::DimensionPair DimPair; + +template<int DataLayout> +void test_gpu_contraction(int m_size, int k_size, int n_size) +{ + std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; + // with these dimensions, the output has 300 * 140 elements, which is + // more than 30 * 1024, which is the number of threads in blocks on + // a 15 SM GK110 GPU + Tensor<float, 2, DataLayout> t_left(m_size, k_size); + Tensor<float, 2, DataLayout> t_right(k_size, n_size); + Tensor<float, 2, DataLayout> t_result(m_size, n_size); + Tensor<float, 2, DataLayout> t_result_gpu(m_size, n_size); + Eigen::array<DimPair, 1> dims(DimPair(1, 0)); + + t_left.setRandom(); + t_right.setRandom(); + + std::size_t t_left_bytes = t_left.size() * sizeof(float); + std::size_t t_right_bytes = t_right.size() * sizeof(float); + std::size_t t_result_bytes = t_result.size() * sizeof(float); + + float* d_t_left; + float* d_t_right; + float* d_t_result; + + gpuMalloc((void**)(&d_t_left), t_left_bytes); + gpuMalloc((void**)(&d_t_right), t_right_bytes); + gpuMalloc((void**)(&d_t_result), t_result_bytes); + + gpuMemcpy(d_t_left, t_left.data(), t_left_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_t_right, t_right.data(), t_right_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > + gpu_t_left(d_t_left, Eigen::array<int, 2>(m_size, k_size)); + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > + gpu_t_right(d_t_right, Eigen::array<int, 2>(k_size, n_size)); + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > + gpu_t_result(d_t_result, Eigen::array<int, 2>(m_size, n_size)); + + + gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims); + t_result = t_left.contract(t_right, dims); + + gpuMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, gpuMemcpyDeviceToHost); + for (DenseIndex i = 0; i < t_result.size(); i++) { + if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) { + continue; + } + if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) { + continue; + } + std::cout << "mismatch detected at index " << i << ": " << t_result(i) + << " vs " << t_result_gpu(i) << std::endl; + assert(false); + } + + gpuFree((void*)d_t_left); + gpuFree((void*)d_t_right); + gpuFree((void*)d_t_result); +} + + +template<int DataLayout> +void test_scalar(int m_size, int k_size, int n_size) +{ + std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; + // with these dimensions, the output has 300 * 140 elements, which is + // more than 30 * 1024, which is the number of threads in blocks on + // a 15 SM GK110 GPU + Tensor<float, 2, DataLayout> t_left(m_size, k_size); + Tensor<float, 2, DataLayout> t_right(k_size, n_size); + Tensor<float, 0, DataLayout> t_result; + Tensor<float, 0, DataLayout> t_result_gpu; + Eigen::array<DimPair, 2> dims(DimPair(0, 0), DimPair(1, 1)); + + t_left.setRandom(); + t_right.setRandom(); + + std::size_t t_left_bytes = t_left.size() * sizeof(float); + std::size_t t_right_bytes = t_right.size() * sizeof(float); + std::size_t t_result_bytes = sizeof(float); + + float* d_t_left; + float* d_t_right; + float* d_t_result; + + gpuMalloc((void**)(&d_t_left), t_left_bytes); + gpuMalloc((void**)(&d_t_right), t_right_bytes); + gpuMalloc((void**)(&d_t_result), t_result_bytes); + + gpuMemcpy(d_t_left, t_left.data(), t_left_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_t_right, t_right.data(), t_right_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > + gpu_t_left(d_t_left, m_size, k_size); + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > + gpu_t_right(d_t_right, k_size, n_size); + Eigen::TensorMap<Eigen::Tensor<float, 0, DataLayout> > + gpu_t_result(d_t_result); + + gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims); + t_result = t_left.contract(t_right, dims); + + gpuMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, gpuMemcpyDeviceToHost); + if (fabs(t_result() - t_result_gpu()) > 1e-4f && + !Eigen::internal::isApprox(t_result(), t_result_gpu(), 1e-4f)) { + std::cout << "mismatch detected: " << t_result() + << " vs " << t_result_gpu() << std::endl; + assert(false); + } + + gpuFree((void*)d_t_left); + gpuFree((void*)d_t_right); + gpuFree((void*)d_t_result); +} + + +template<int DataLayout> +void test_gpu_contraction_m() { + for (int k = 32; k < 256; k++) { + test_gpu_contraction<ColMajor>(k, 128, 128); + test_gpu_contraction<RowMajor>(k, 128, 128); + } +} + +template<int DataLayout> +void test_gpu_contraction_k() { + for (int k = 32; k < 256; k++) { + test_gpu_contraction<ColMajor>(128, k, 128); + test_gpu_contraction<RowMajor>(128, k, 128); + } +} + +template<int DataLayout> +void test_gpu_contraction_n() { + for (int k = 32; k < 256; k++) { + test_gpu_contraction<ColMajor>(128, 128, k); + test_gpu_contraction<RowMajor>(128, 128, k); + } +} + + +template<int DataLayout> +void test_gpu_contraction_sizes() { + int m_sizes[] = { 31, 39, 63, 64, 65, + 127, 129, 255, 257 , 511, + 512, 513, 1023, 1024, 1025}; + + int n_sizes[] = { 31, 39, 63, 64, 65, + 127, 129, 255, 257, 511, + 512, 513, 1023, 1024, 1025}; + + int k_sizes[] = { 31, 39, 63, 64, 65, + 95, 96, 127, 129, 255, + 257, 511, 512, 513, 1023, + 1024, 1025}; + + for (int i = 0; i < 15; i++) { + for (int j = 0; j < 15; j++) { + for (int k = 0; k < 17; k++) { + test_gpu_contraction<DataLayout>(m_sizes[i], n_sizes[j], k_sizes[k]); + } + } + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_contract_gpu) +{ + CALL_SUBTEST_1(test_gpu_contraction<ColMajor>(128, 128, 128)); + CALL_SUBTEST_1(test_gpu_contraction<RowMajor>(128, 128, 128)); + + CALL_SUBTEST_1(test_scalar<ColMajor>(128, 128, 128)); + CALL_SUBTEST_1(test_scalar<RowMajor>(128, 128, 128)); + + CALL_SUBTEST_2(test_gpu_contraction_m<ColMajor>()); + CALL_SUBTEST_3(test_gpu_contraction_m<RowMajor>()); + + CALL_SUBTEST_4(test_gpu_contraction_k<ColMajor>()); + CALL_SUBTEST_5(test_gpu_contraction_k<RowMajor>()); + + CALL_SUBTEST_6(test_gpu_contraction_n<ColMajor>()); + CALL_SUBTEST_7(test_gpu_contraction_n<RowMajor>()); + +#if !defined(EIGEN_USE_HIP) +// disable these subtests for HIP + CALL_SUBTEST_8(test_gpu_contraction_sizes<ColMajor>()); + CALL_SUBTEST_9(test_gpu_contraction_sizes<RowMajor>()); +#endif +}
diff --git a/unsupported/test/cxx11_tensor_contract_sycl.cpp b/unsupported/test/cxx11_tensor_contract_sycl.cpp new file mode 100644 index 0000000..c8e86e6 --- /dev/null +++ b/unsupported/test/cxx11_tensor_contract_sycl.cpp
@@ -0,0 +1,290 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include <iostream> +#include <chrono> +#include <ctime> + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void static test_sycl_contraction(const Device& sycl_device, IndexType m_size, IndexType k_size, IndexType n_size) +{ + typedef typename Tensor<DataType, 1, DataLayout, IndexType>::DimensionPair DimPair; + static const DataType error_threshold =1e-4f; +// std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; + // with these dimensions, the output has 300 * 140 elements, which is + // more than 30 * 1024, which is the number of threads in blocks on + // a 15 SM GK110 GPU + Tensor<DataType, 2, DataLayout, IndexType> t_left(m_size, k_size); + Tensor<DataType, 2, DataLayout, IndexType> t_right(k_size, n_size); + Tensor<DataType, 2, DataLayout, IndexType> t_result(m_size, n_size); + Tensor<DataType, 2, DataLayout, IndexType> t_result_gpu(m_size, n_size); +// Eigen::array<DimPair, 1> dims(DimPair(1, 0)); + Eigen::array<DimPair, 1> dims = {{DimPair(1, 0)}}; + Eigen::array<IndexType, 2> left_dims = {{m_size, k_size}}; + Eigen::array<IndexType, 2> right_dims = {{k_size, n_size}}; + Eigen::array<IndexType, 2> result_dims = {{m_size, n_size}}; + + t_left.setRandom(); + t_right.setRandom(); + + std::size_t t_left_bytes = t_left.size() * sizeof(DataType); + std::size_t t_right_bytes = t_right.size() * sizeof(DataType); + std::size_t t_result_bytes = t_result.size() * sizeof(DataType); + + DataType * d_t_left = static_cast<DataType*>(sycl_device.allocate(t_left_bytes)); + DataType * d_t_right = static_cast<DataType*>(sycl_device.allocate(t_right_bytes)); + DataType * d_t_result = static_cast<DataType*>(sycl_device.allocate(t_result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_left(d_t_left, left_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_right(d_t_right, right_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_result(d_t_result, result_dims); + + sycl_device.memcpyHostToDevice(d_t_left, t_left.data(),t_left_bytes); + sycl_device.memcpyHostToDevice(d_t_right, t_right.data(),t_right_bytes); + + gpu_t_result.device(sycl_device) = gpu_t_left.contract(gpu_t_right, dims); + sycl_device.memcpyDeviceToHost(t_result_gpu.data(), d_t_result, t_result_bytes); + + t_result = t_left.contract(t_right, dims); + + for (IndexType i = 0; i < t_result.size(); i++) { + if (static_cast<DataType>(fabs(t_result(i) - t_result_gpu(i))) < error_threshold) { + continue; + } + if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), error_threshold)) { + continue; + } + std::cout << "mismatch detected at IndexType " << i << ": " << t_result(i) + << " vs " << t_result_gpu(i) << std::endl; + assert(false); + } + sycl_device.deallocate(d_t_left); + sycl_device.deallocate(d_t_right); + sycl_device.deallocate(d_t_result); +} + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_TF(const Device& sycl_device) +{ + typedef typename Tensor<DataType, 1, DataLayout, IndexType>::DimensionPair DimPair; + static const DataType error_threshold =1e-4f; + Eigen::array<IndexType, 2> left_dims = {{2, 3}}; + Eigen::array<IndexType, 2> right_dims = {{3, 1}}; + Eigen::array<IndexType, 2> res_dims = {{2, 1}}; + Eigen::array<DimPair, 1> dims = {{DimPair(1, 0)}}; + + + Tensor<DataType, 2, DataLayout, IndexType> t_left(left_dims); + Tensor<DataType, 2, DataLayout, IndexType> t_right(right_dims); + Tensor<DataType, 2, DataLayout, IndexType> t_result_gpu(res_dims); + Tensor<DataType, 2, DataLayout, IndexType> t_result(res_dims); + + t_left.data()[0] = 1.0f; + t_left.data()[1] = 2.0f; + t_left.data()[2] = 3.0f; + t_left.data()[3] = 4.0f; + t_left.data()[4] = 5.0f; + t_left.data()[5] = 6.0f; + + t_right.data()[0] = -1.0f; + t_right.data()[1] = 0.5f; + t_right.data()[2] = 2.0f; + + std::size_t t_left_bytes = t_left.size() * sizeof(DataType); + std::size_t t_right_bytes = t_right.size() * sizeof(DataType); + std::size_t t_result_bytes = t_result.size()*sizeof(DataType); + + + DataType * d_t_left = static_cast<DataType*>(sycl_device.allocate(t_left_bytes)); + DataType * d_t_right = static_cast<DataType*>(sycl_device.allocate(t_right_bytes)); + DataType * d_t_result = static_cast<DataType*>(sycl_device.allocate(t_result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_left(d_t_left, left_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_right(d_t_right, right_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_result(d_t_result, res_dims); + + sycl_device.memcpyHostToDevice(d_t_left, t_left.data(),t_left_bytes); + sycl_device.memcpyHostToDevice(d_t_right, t_right.data(),t_right_bytes); + + gpu_t_result.device(sycl_device) = gpu_t_left.contract(gpu_t_right, dims); + sycl_device.memcpyDeviceToHost(t_result_gpu.data(), d_t_result, t_result_bytes); + + t_result = t_left.contract(t_right, dims); + + for (IndexType i = 0; i < t_result.size(); i++) { + if (static_cast<DataType>(fabs(t_result(i) - t_result_gpu(i))) < error_threshold) { + continue; + } + if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), error_threshold)) { + continue; + } + std::cout << "mismatch detected at IndexType " << i << ": " << t_result(i) + << " vs " << t_result_gpu(i) << std::endl; + assert(false); + } + sycl_device.deallocate(d_t_left); + sycl_device.deallocate(d_t_right); + sycl_device.deallocate(d_t_result); + + +} + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_scalar(const Device& sycl_device, IndexType m_size, IndexType k_size, IndexType n_size) +{ + //std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; + // with these dimensions, the output has 300 * 140 elements, which is + // more than 30 * 1024, which is the number of threads in blocks on + // a 15 SM GK110 GPU + typedef typename Tensor<DataType, 1, DataLayout, IndexType>::DimensionPair DimPair; + static const DataType error_threshold =1e-4f; + Tensor<DataType, 2, DataLayout, IndexType> t_left(m_size, k_size); + Tensor<DataType, 2, DataLayout, IndexType> t_right(k_size, n_size); + Tensor<DataType, 0, DataLayout, IndexType> t_result; + Tensor<DataType, 0, DataLayout, IndexType> t_result_gpu; + Eigen::array<DimPair, 2> dims = {{DimPair(0, 0), DimPair(1, 1)}}; + Eigen::array<IndexType, 2> left_dims = {{m_size, k_size}}; + Eigen::array<IndexType, 2> right_dims = {{k_size, n_size}}; + t_left.setRandom(); + t_right.setRandom(); + + std::size_t t_left_bytes = t_left.size() * sizeof(DataType); + std::size_t t_right_bytes = t_right.size() * sizeof(DataType); + std::size_t t_result_bytes = sizeof(DataType); + + + DataType * d_t_left = static_cast<DataType*>(sycl_device.allocate(t_left_bytes)); + DataType * d_t_right = static_cast<DataType*>(sycl_device.allocate(t_right_bytes)); + DataType * d_t_result = static_cast<DataType*>(sycl_device.allocate(t_result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_left(d_t_left, left_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_t_right(d_t_right, right_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 0, DataLayout, IndexType> > gpu_t_result(d_t_result); + + sycl_device.memcpyHostToDevice(d_t_left, t_left.data(),t_left_bytes); + sycl_device.memcpyHostToDevice(d_t_right, t_right.data(),t_right_bytes); + + gpu_t_result.device(sycl_device) = gpu_t_left.contract(gpu_t_right, dims); + sycl_device.memcpyDeviceToHost(t_result_gpu.data(), d_t_result, t_result_bytes); + + t_result = t_left.contract(t_right, dims); + + if (static_cast<DataType>(fabs(t_result() - t_result_gpu())) > error_threshold && + !Eigen::internal::isApprox(t_result(), t_result_gpu(), error_threshold)) { + std::cout << "mismatch detected: " << t_result() + << " vs " << t_result_gpu() << std::endl; + assert(false); + } + + sycl_device.deallocate(d_t_left); + sycl_device.deallocate(d_t_right); + sycl_device.deallocate(d_t_result); +} + + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_sycl_contraction_m(const Device& sycl_device) { + for (IndexType k = 32; k < 256; k++) { + test_sycl_contraction<DataLayout, DataType, IndexType>(sycl_device, k, 128, 128); + } +} + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_sycl_contraction_k(const Device& sycl_device) { + for (IndexType k = 32; k < 256; k++) { + test_sycl_contraction<DataLayout, DataType, IndexType>(sycl_device, 128, k, 128); + } +} + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_sycl_contraction_n(const Device& sycl_device) { + for (IndexType k = 32; k < 256; k++) { + test_sycl_contraction<DataLayout, DataType, IndexType>(sycl_device, 128, 128, k); + } +} + + +template<int DataLayout, typename DataType, typename IndexType, typename Device> +void test_sycl_contraction_sizes(const Device& sycl_device) { + IndexType m_sizes[] = { 31, 39, 63, 64, 65, + 127, 129, 255, 257 , 511, + 512, 513, 1023, 1024, 1025}; + + IndexType n_sizes[] = { 31, 39, 63, 64, 65, + 127, 129, 255, 257, 511, + 512, 513, 1023, 1024, 1025}; + + IndexType k_sizes[] = { 31, 39, 63, 64, 65, + 95, 96, 127, 129, 255, + 257, 511, 512, 513, 1023, + 1024, 1025}; + + for (IndexType i = 0; i < 15; i++) { + for (IndexType j = 0; j < 15; j++) { + for (IndexType k = 0; k < 17; k++) { + test_sycl_contraction<DataLayout, DataType,IndexType>(sycl_device, m_sizes[i], n_sizes[j], k_sizes[k]); + } + } + } +} + +template <typename Dev_selector> void tensorContractionPerDevice(Dev_selector& s){ + QueueInterface queueInterface(s); + auto sycl_device=Eigen::SyclDevice(&queueInterface); + test_sycl_contraction<ColMajor, float,int64_t>(sycl_device, 32, 32, 32); + test_sycl_contraction<RowMajor,float,int64_t>(sycl_device, 32, 32, 32); + test_scalar<ColMajor,float,int64_t>(sycl_device, 32, 32, 32); + test_scalar<RowMajor,float,int64_t>(sycl_device, 32, 32, 32); + std::chrono::time_point<std::chrono::system_clock> start, end; + start = std::chrono::system_clock::now(); + test_sycl_contraction<ColMajor,float,int64_t>(sycl_device, 128, 128, 128); + test_sycl_contraction<RowMajor,float,int64_t>(sycl_device, 128, 128, 128); + test_scalar<ColMajor,float,int64_t>(sycl_device, 128, 128, 128); + test_scalar<RowMajor,float,int64_t>(sycl_device, 128, 128, 128); + test_sycl_contraction_m<ColMajor, float, int64_t>(sycl_device); + test_sycl_contraction_m<RowMajor, float, int64_t>(sycl_device); + test_sycl_contraction_n<ColMajor, float, int64_t>(sycl_device); + test_sycl_contraction_n<RowMajor, float, int64_t>(sycl_device); + test_sycl_contraction_k<ColMajor, float, int64_t>(sycl_device); + test_sycl_contraction_k<RowMajor, float, int64_t>(sycl_device); + test_sycl_contraction_sizes<ColMajor, float, int64_t>(sycl_device); + test_sycl_contraction_sizes<RowMajor, float, int64_t>(sycl_device); + test_TF<RowMajor, float, int64_t>(sycl_device); + test_TF<ColMajor, float, int64_t>(sycl_device); + + end = std::chrono::system_clock::now(); + std::chrono::duration<double> elapsed_seconds = end-start; + std::time_t end_time = std::chrono::system_clock::to_time_t(end); + std::cout << "finished computation at " << std::ctime(&end_time) + << "elapsed time: " << elapsed_seconds.count() << "s\n"; + +} + +EIGEN_DECLARE_TEST(cxx11_tensor_contract_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(tensorContractionPerDevice(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_contraction.cpp b/unsupported/test/cxx11_tensor_contraction.cpp index 0e16308..2fd1281 100644 --- a/unsupported/test/cxx11_tensor_contraction.cpp +++ b/unsupported/test/cxx11_tensor_contraction.cpp
@@ -87,19 +87,14 @@ vec1.setRandom(); vec2.setRandom(); - Tensor<float, 1, DataLayout> scalar(1); - scalar.setZero(); Eigen::array<DimPair, 1> dims = {{DimPair(0, 0)}}; - typedef TensorEvaluator<decltype(vec1.contract(vec2, dims)), DefaultDevice> Evaluator; - Evaluator eval(vec1.contract(vec2, dims), DefaultDevice()); - eval.evalTo(scalar.data()); - EIGEN_STATIC_ASSERT(Evaluator::NumDims==1ul, YOU_MADE_A_PROGRAMMING_MISTAKE); + Tensor<float, 0, DataLayout> scalar = vec1.contract(vec2, dims); float expected = 0.0f; for (int i = 0; i < 6; ++i) { expected += vec1(i) * vec2(i); } - VERIFY_IS_APPROX(scalar(0), expected); + VERIFY_IS_APPROX(scalar(), expected); } template<int DataLayout> @@ -476,7 +471,8 @@ mat1.setRandom(); mat2.setRandom(); - Tensor<float, 4, DataLayout> result = mat1.contract(mat2, Eigen::array<DimPair, 0>{{}}); + Eigen::array<DimPair, 0> dims; + Tensor<float, 4, DataLayout> result = mat1.contract(mat2, dims); VERIFY_IS_EQUAL(result.dimension(0), 2); VERIFY_IS_EQUAL(result.dimension(1), 3); @@ -494,7 +490,77 @@ } -void test_cxx11_tensor_contraction() +template<int DataLayout> +static void test_const_inputs() +{ + Tensor<float, 2, DataLayout> in1(2, 3); + Tensor<float, 2, DataLayout> in2(3, 2); + in1.setRandom(); + in2.setRandom(); + + TensorMap<Tensor<const float, 2, DataLayout> > mat1(in1.data(), 2, 3); + TensorMap<Tensor<const float, 2, DataLayout> > mat2(in2.data(), 3, 2); + Tensor<float, 2, DataLayout> mat3(2,2); + + Eigen::array<DimPair, 1> dims = {{DimPair(1, 0)}}; + mat3 = mat1.contract(mat2, dims); + + VERIFY_IS_APPROX(mat3(0,0), mat1(0,0)*mat2(0,0) + mat1(0,1)*mat2(1,0) + mat1(0,2)*mat2(2,0)); + VERIFY_IS_APPROX(mat3(0,1), mat1(0,0)*mat2(0,1) + mat1(0,1)*mat2(1,1) + mat1(0,2)*mat2(2,1)); + VERIFY_IS_APPROX(mat3(1,0), mat1(1,0)*mat2(0,0) + mat1(1,1)*mat2(1,0) + mat1(1,2)*mat2(2,0)); + VERIFY_IS_APPROX(mat3(1,1), mat1(1,0)*mat2(0,1) + mat1(1,1)*mat2(1,1) + mat1(1,2)*mat2(2,1)); +} + +// Apply Sqrt to all output elements. +struct SqrtOutputKernel { + template <typename Index, typename Scalar> + EIGEN_ALWAYS_INLINE void operator()( + const internal::blas_data_mapper<Scalar, Index, ColMajor>& output_mapper, + const TensorContractionParams&, Index, Index, Index num_rows, + Index num_cols) const { + for (int i = 0; i < num_rows; ++i) { + for (int j = 0; j < num_cols; ++j) { + output_mapper(i, j) = std::sqrt(output_mapper(i, j)); + } + } + } +}; + +template <int DataLayout> +static void test_large_contraction_with_output_kernel() { + Tensor<float, 4, DataLayout> t_left(30, 50, 8, 31); + Tensor<float, 5, DataLayout> t_right(8, 31, 7, 20, 10); + Tensor<float, 5, DataLayout> t_result(30, 50, 7, 20, 10); + + t_left.setRandom(); + t_right.setRandom(); + // Put trash in mat4 to verify contraction clears output memory. + t_result.setRandom(); + + // Add a little offset so that the results won't be close to zero. + t_left += t_left.constant(1.0f); + t_right += t_right.constant(1.0f); + + typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; + MapXf m_left(t_left.data(), 1500, 248); + MapXf m_right(t_right.data(), 248, 1400); + Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(1500, 1400); + + // this contraction should be equivalent to a single matrix multiplication + Eigen::array<DimPair, 2> dims({{DimPair(2, 0), DimPair(3, 1)}}); + + // compute results by separate methods + t_result = t_left.contract(t_right, dims, SqrtOutputKernel()); + + m_result = m_left * m_right; + + for (std::ptrdiff_t i = 0; i < t_result.dimensions().TotalSize(); i++) { + VERIFY(&t_result.data()[i] != &m_result.data()[i]); + VERIFY_IS_APPROX(t_result.data()[i], std::sqrt(m_result.data()[i])); + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_contraction) { CALL_SUBTEST(test_evals<ColMajor>()); CALL_SUBTEST(test_evals<RowMajor>()); @@ -524,4 +590,8 @@ CALL_SUBTEST(test_small_blocking_factors<RowMajor>()); CALL_SUBTEST(test_tensor_product<ColMajor>()); CALL_SUBTEST(test_tensor_product<RowMajor>()); + CALL_SUBTEST(test_const_inputs<ColMajor>()); + CALL_SUBTEST(test_const_inputs<RowMajor>()); + CALL_SUBTEST(test_large_contraction_with_output_kernel<ColMajor>()); + CALL_SUBTEST(test_large_contraction_with_output_kernel<RowMajor>()); }
diff --git a/unsupported/test/cxx11_tensor_convolution.cpp b/unsupported/test/cxx11_tensor_convolution.cpp index e3d4675..c3688f6 100644 --- a/unsupported/test/cxx11_tensor_convolution.cpp +++ b/unsupported/test/cxx11_tensor_convolution.cpp
@@ -25,7 +25,8 @@ Tensor<float, 2, DataLayout> result(2,3); result.setZero(); - Eigen::array<Tensor<float, 2>::Index, 1> dims3{{0}}; + Eigen::array<Tensor<float, 2>::Index, 1> dims3; + dims3[0] = 0; typedef TensorEvaluator<decltype(input.convolve(kernel, dims3)), DefaultDevice> Evaluator; Evaluator eval(input.convolve(kernel, dims3), DefaultDevice()); @@ -136,7 +137,7 @@ input(12)*kernel(2))); } -void test_cxx11_tensor_convolution() +EIGEN_DECLARE_TEST(cxx11_tensor_convolution) { CALL_SUBTEST(test_evals<ColMajor>()); CALL_SUBTEST(test_evals<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_convolution_sycl.cpp b/unsupported/test/cxx11_tensor_convolution_sycl.cpp new file mode 100644 index 0000000..3954c8a --- /dev/null +++ b/unsupported/test/cxx11_tensor_convolution_sycl.cpp
@@ -0,0 +1,469 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include <iostream> +#include <chrono> +#include <ctime> + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> +#include <iomanip> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; +static const float error_threshold =1e-4f; + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_larg_expr1D(const Eigen::SyclDevice& sycl_device) +{ + IndexType indim0 =53; + IndexType indim1= 55; + IndexType indim2= 51; + IndexType outdim0=50; + IndexType outdim1=55; + IndexType outdim2=51; + Eigen::array<IndexType, 3> input_dims = {{indim0, indim1, indim2}}; + Eigen::array<IndexType, 1> kernel_dims = {{4}}; + Eigen::array<IndexType, 3> result_dims = {{outdim0, outdim1, outdim2}}; + + Tensor<DataType, 3, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 1, DataLayout,IndexType> kernel(kernel_dims); + Tensor<DataType, 3, DataLayout,IndexType> result(result_dims); + Tensor<DataType, 3, DataLayout,IndexType> result_host(result_dims); + + Eigen::array<IndexType, 1> dims3{{0}}; + + input.setRandom(); + kernel.setRandom(); + result.setZero(); + result_host.setZero(); + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout, IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_result(d_result, result_dims); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.convolve(gpu_kernel, dims3); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + result_host=input.convolve(kernel, dims3); + +for(IndexType i=0; i< outdim0; i++ ){ + for(IndexType j=0; j< outdim1; j++ ){ + for(IndexType k=0; k< outdim2; k++ ){ + if (!(Eigen::internal::isApprox(result(i,j,k), result_host(i,j,k), error_threshold))) { + std::cout <<std::setprecision(16)<< "mismatch detected at index ( "<< i << " , " << j << ", " << k << " ) " << " \t " << result(i,j,k) << " vs "<< result_host(i,j,k) << std::endl; + assert(false); + } + } + } +} + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_result); + +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_larg_expr2D(const Eigen::SyclDevice& sycl_device) +{ + IndexType indim0 =53; + IndexType indim1= 55; + IndexType indim2= 51; + IndexType outdim0=50; + IndexType outdim1=51; + IndexType outdim2=51; + Eigen::array<IndexType, 3> input_dims = {{indim0, indim1, indim2}}; + Eigen::array<IndexType, 2> kernel_dims = {{4,5}}; + Eigen::array<IndexType, 3> result_dims = {{outdim0, outdim1, outdim2}}; + + Tensor<DataType, 3, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 2, DataLayout,IndexType> kernel(kernel_dims); + Tensor<DataType, 3, DataLayout,IndexType> result(result_dims); + Tensor<DataType, 3, DataLayout,IndexType> result_host(result_dims); + + Eigen::array<IndexType, 2> dims3{{0,1}}; + + input.setRandom(); + kernel.setRandom(); + result.setZero(); + result_host.setZero(); + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_result(d_result, result_dims); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.convolve(gpu_kernel, dims3); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + result_host=input.convolve(kernel, dims3); + +for(IndexType i=0; i< outdim0; i++ ){ + for(IndexType j=0; j< outdim1; j++ ){ + for(IndexType k=0; k< outdim2; k++ ){ + if (!(Eigen::internal::isApprox(result(i,j,k), result_host(i,j,k), error_threshold))) { + std::cout <<std::setprecision(16)<< "mismatch detected at index ( "<< i << " , " << j << ", " << k << " ) " << " \t " << result(i,j,k) << " vs "<< result_host(i,j,k) << std::endl; + assert(false); + } + } + } +} + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_result); + +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_larg_expr3D(const Eigen::SyclDevice& sycl_device) +{ + IndexType indim0 =53; + IndexType indim1= 55; + IndexType indim2= 51; + IndexType outdim0=50; + IndexType outdim1=51; + IndexType outdim2=49; + Eigen::array<IndexType, 3> input_dims = {{indim0, indim1, indim2}}; + Eigen::array<IndexType, 3> kernel_dims = {{4,5,3}}; + Eigen::array<IndexType, 3> result_dims = {{outdim0, outdim1, outdim2}}; + + Tensor<DataType, 3, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 3, DataLayout,IndexType> kernel(kernel_dims); + Tensor<DataType, 3, DataLayout,IndexType> result(result_dims); + Tensor<DataType, 3, DataLayout,IndexType> result_host(result_dims); + + Eigen::array<IndexType, 3> dims3{{0,1,2}}; + + input.setRandom(); + kernel.setRandom(); + result.setZero(); + result_host.setZero(); + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > gpu_result(d_result, result_dims); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.convolve(gpu_kernel, dims3); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + result_host=input.convolve(kernel, dims3); + +for(IndexType i=0; i< outdim0; i++ ){ + for(IndexType j=0; j< outdim1; j++ ){ + for(IndexType k=0; k< outdim2; k++ ){ + if (!(Eigen::internal::isApprox(result(i,j,k), result_host(i,j,k), error_threshold))) { + std::cout <<std::setprecision(16)<< "mismatch detected at index ( "<< i << " , " << j << ", " << k << " ) " << " \t " << result(i,j,k) << " vs "<< result_host(i,j,k) << std::endl; + assert(false); + } + } + } +} + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_result); + +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_evals(const Eigen::SyclDevice& sycl_device) +{ + Eigen::array<IndexType, 2> input_dims = {{3, 3}}; + Eigen::array<IndexType, 1> kernel_dims = {{2}}; + Eigen::array<IndexType, 2> result_dims = {{2, 3}}; + + Tensor<DataType, 2, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 1, DataLayout,IndexType> kernel(kernel_dims); + Tensor<DataType, 2, DataLayout,IndexType> result(result_dims); + + Eigen::array<IndexType, 1> dims3{{0}}; + + input.setRandom(); + kernel.setRandom(); + result.setZero(); + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout, IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > gpu_result(d_result, result_dims); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.convolve(gpu_kernel, dims3); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + VERIFY_IS_APPROX(result(0,0), input(0,0)*kernel(0) + input(1,0)*kernel(1)); // index 0 + VERIFY_IS_APPROX(result(0,1), input(0,1)*kernel(0) + input(1,1)*kernel(1)); // index 2 + VERIFY_IS_APPROX(result(0,2), input(0,2)*kernel(0) + input(1,2)*kernel(1)); // index 4 + VERIFY_IS_APPROX(result(1,0), input(1,0)*kernel(0) + input(2,0)*kernel(1)); // index 1 + VERIFY_IS_APPROX(result(1,1), input(1,1)*kernel(0) + input(2,1)*kernel(1)); // index 3 + VERIFY_IS_APPROX(result(1,2), input(1,2)*kernel(0) + input(2,2)*kernel(1)); // index 5 + + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_result); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_expr(const Eigen::SyclDevice& sycl_device) +{ + Eigen::array<IndexType, 2> input_dims = {{3, 3}}; + Eigen::array<IndexType, 2> kernel_dims = {{2, 2}}; + Eigen::array<IndexType, 2> result_dims = {{2, 2}}; + + Tensor<DataType, 2, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 2, DataLayout, IndexType> kernel(kernel_dims); + Tensor<DataType, 2, DataLayout, IndexType> result(result_dims); + + input.setRandom(); + kernel.setRandom(); + Eigen::array<IndexType, 2> dims; + dims[0] = 0; + dims[1] = 1; + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout,IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout,IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout,IndexType> > gpu_result(d_result, result_dims); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.convolve(gpu_kernel, dims); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + VERIFY_IS_APPROX(result(0,0), input(0,0)*kernel(0,0) + input(0,1)*kernel(0,1) + + input(1,0)*kernel(1,0) + input(1,1)*kernel(1,1)); + VERIFY_IS_APPROX(result(0,1), input(0,1)*kernel(0,0) + input(0,2)*kernel(0,1) + + input(1,1)*kernel(1,0) + input(1,2)*kernel(1,1)); + VERIFY_IS_APPROX(result(1,0), input(1,0)*kernel(0,0) + input(1,1)*kernel(0,1) + + input(2,0)*kernel(1,0) + input(2,1)*kernel(1,1)); + VERIFY_IS_APPROX(result(1,1), input(1,1)*kernel(0,0) + input(1,2)*kernel(0,1) + + input(2,1)*kernel(1,0) + input(2,2)*kernel(1,1)); + + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_result); +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_modes(const Eigen::SyclDevice& sycl_device){ + +Eigen::array<IndexType, 1> input_dims = {{3}}; +Eigen::array<IndexType, 1> kernel_dims = {{3}}; + +Tensor<DataType, 1, DataLayout, IndexType> input(input_dims); +Tensor<DataType, 1, DataLayout, IndexType> kernel(kernel_dims); + +input.setRandom(); +kernel.setRandom(); +Eigen::array<IndexType, 1> dims; +dims[0] = 0; + + input(0) = 1.0f; + input(1) = 2.0f; + input(2) = 3.0f; + kernel(0) = 0.5f; + kernel(1) = 1.0f; + kernel(2) = 0.0f; + + Eigen::array<std::pair<IndexType, IndexType>, 1> padding; + + // Emulate VALID mode (as defined in + // http://docs.scipy.org/doc/numpy/reference/generated/numpy.convolve.html). + padding[0] = std::make_pair(0, 0); + Tensor<DataType, 1, DataLayout, IndexType> valid(1); + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t valid_bytes = valid.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_valid = static_cast<DataType*>(sycl_device.allocate(valid_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_valid(d_valid, valid.dimensions()); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_valid.device(sycl_device)=gpu_input.pad(padding).convolve(gpu_kernel, dims); + sycl_device.memcpyDeviceToHost(valid.data(), d_valid, valid_bytes); + + VERIFY_IS_EQUAL(valid.dimension(0), 1); + VERIFY_IS_APPROX(valid(0), 2.5f); + + // Emulate SAME mode (as defined in + // http://docs.scipy.org/doc/numpy/reference/generated/numpy.convolve.html). + padding[0] = std::make_pair(1, 1); + Tensor<DataType, 1, DataLayout, IndexType> same(3); + std::size_t same_bytes = same.size() * sizeof(DataType); + DataType * d_same = static_cast<DataType*>(sycl_device.allocate(same_bytes)); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_same(d_same, same.dimensions()); + gpu_same.device(sycl_device)=gpu_input.pad(padding).convolve(gpu_kernel, dims); + sycl_device.memcpyDeviceToHost(same.data(), d_same, same_bytes); + + VERIFY_IS_EQUAL(same.dimension(0), 3); + VERIFY_IS_APPROX(same(0), 1.0f); + VERIFY_IS_APPROX(same(1), 2.5f); + VERIFY_IS_APPROX(same(2), 4.0f); + + // Emulate FULL mode (as defined in + // http://docs.scipy.org/doc/numpy/reference/generated/numpy.convolve.html). + padding[0] = std::make_pair(2, 2); + + Tensor<DataType, 1, DataLayout, IndexType> full(5); + std::size_t full_bytes = full.size() * sizeof(DataType); + DataType * d_full = static_cast<DataType*>(sycl_device.allocate(full_bytes)); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_full(d_full, full.dimensions()); + gpu_full.device(sycl_device)=gpu_input.pad(padding).convolve(gpu_kernel, dims); + sycl_device.memcpyDeviceToHost(full.data(), d_full, full_bytes); + + VERIFY_IS_EQUAL(full.dimension(0), 5); + VERIFY_IS_APPROX(full(0), 0.0f); + VERIFY_IS_APPROX(full(1), 1.0f); + VERIFY_IS_APPROX(full(2), 2.5f); + VERIFY_IS_APPROX(full(3), 4.0f); + VERIFY_IS_APPROX(full(4), 1.5f); + + sycl_device.deallocate(d_input); + sycl_device.deallocate(d_kernel); + sycl_device.deallocate(d_valid); + sycl_device.deallocate(d_same); + sycl_device.deallocate(d_full); + +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_strides(const Eigen::SyclDevice& sycl_device){ + + Eigen::array<IndexType, 1> input_dims = {{13}}; + Eigen::array<IndexType, 1> kernel_dims = {{3}}; + + Tensor<DataType, 1, DataLayout, IndexType> input(input_dims); + Tensor<DataType, 1, DataLayout, IndexType> kernel(kernel_dims); + Tensor<DataType, 1, DataLayout, IndexType> result(2); + + input.setRandom(); + kernel.setRandom(); + Eigen::array<IndexType, 1> dims; + dims[0] = 0; + + Eigen::array<IndexType, 1> stride_of_3; + stride_of_3[0] = 3; + Eigen::array<IndexType, 1> stride_of_2; + stride_of_2[0] = 2; + + std::size_t input_bytes = input.size() * sizeof(DataType); + std::size_t kernel_bytes = kernel.size() * sizeof(DataType); + std::size_t result_bytes = result.size() * sizeof(DataType); + + DataType * d_input = static_cast<DataType*>(sycl_device.allocate(input_bytes)); + DataType * d_kernel = static_cast<DataType*>(sycl_device.allocate(kernel_bytes)); + DataType * d_result = static_cast<DataType*>(sycl_device.allocate(result_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_input(d_input, input_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_kernel(d_kernel, kernel_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 1, DataLayout,IndexType> > gpu_result(d_result, result.dimensions()); + sycl_device.memcpyHostToDevice(d_input, input.data(), input_bytes); + sycl_device.memcpyHostToDevice(d_kernel, kernel.data(), kernel_bytes); + + gpu_result.device(sycl_device)=gpu_input.stride(stride_of_3).convolve(gpu_kernel, dims).stride(stride_of_2); + sycl_device.memcpyDeviceToHost(result.data(), d_result, result_bytes); + + VERIFY_IS_EQUAL(result.dimension(0), 2); + VERIFY_IS_APPROX(result(0), (input(0)*kernel(0) + input(3)*kernel(1) + + input(6)*kernel(2))); + VERIFY_IS_APPROX(result(1), (input(6)*kernel(0) + input(9)*kernel(1) + + input(12)*kernel(2))); +} + +template <typename Dev_selector> void tensorConvolutionPerDevice(Dev_selector& s){ + QueueInterface queueInterface(s); + auto sycl_device=Eigen::SyclDevice(&queueInterface); + test_larg_expr1D<float, RowMajor, int64_t>(sycl_device); + test_larg_expr1D<float, ColMajor, int64_t>(sycl_device); + test_larg_expr2D<float, RowMajor, int64_t>(sycl_device); + test_larg_expr2D<float, ColMajor, int64_t>(sycl_device); + test_larg_expr3D<float, RowMajor, int64_t>(sycl_device); + test_larg_expr3D<float, ColMajor, int64_t>(sycl_device); + test_evals<float, ColMajor, int64_t>(sycl_device); + test_evals<float, RowMajor, int64_t>(sycl_device); + test_expr<float, ColMajor, int64_t>(sycl_device); + test_expr<float, RowMajor, int64_t>(sycl_device); + test_modes<float, ColMajor, int64_t>(sycl_device); + test_modes<float, RowMajor, int64_t>(sycl_device); + test_strides<float, ColMajor, int64_t>(sycl_device); + test_strides<float, RowMajor, int64_t>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_convolution_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(tensorConvolutionPerDevice(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_cuda.cu b/unsupported/test/cxx11_tensor_cuda.cu deleted file mode 100644 index db2d8ee..0000000 --- a/unsupported/test/cxx11_tensor_cuda.cu +++ /dev/null
@@ -1,1071 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_cuda -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int -#define EIGEN_USE_GPU - - -#include "main.h" -#include <unsupported/Eigen/CXX11/Tensor> - -using Eigen::Tensor; - -void test_cuda_elementwise_small() { - Tensor<float, 1> in1(Eigen::array<int, 1>(2)); - Tensor<float, 1> in2(Eigen::array<int, 1>(2)); - Tensor<float, 1> out(Eigen::array<int, 1>(2)); - in1.setRandom(); - in2.setRandom(); - - std::size_t in1_bytes = in1.size() * sizeof(float); - std::size_t in2_bytes = in2.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_in1; - float* d_in2; - float* d_out; - cudaMalloc((void**)(&d_in1), in1_bytes); - cudaMalloc((void**)(&d_in2), in2_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_in1, in1.data(), in1_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in2, in2.data(), in2_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1( - d_in1, Eigen::array<int, 1>(2)); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in2( - d_in2, Eigen::array<int, 1>(2)); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_out( - d_out, Eigen::array<int, 1>(2)); - - gpu_out.device(gpu_device) = gpu_in1 + gpu_in2; - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, - gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 2; ++i) { - VERIFY_IS_APPROX( - out(Eigen::array<int, 1>(i)), - in1(Eigen::array<int, 1>(i)) + in2(Eigen::array<int, 1>(i))); - } - - cudaFree(d_in1); - cudaFree(d_in2); - cudaFree(d_out); -} - -void test_cuda_elementwise() -{ - Tensor<float, 3> in1(Eigen::array<int, 3>(72,53,97)); - Tensor<float, 3> in2(Eigen::array<int, 3>(72,53,97)); - Tensor<float, 3> in3(Eigen::array<int, 3>(72,53,97)); - Tensor<float, 3> out(Eigen::array<int, 3>(72,53,97)); - in1.setRandom(); - in2.setRandom(); - in3.setRandom(); - - std::size_t in1_bytes = in1.size() * sizeof(float); - std::size_t in2_bytes = in2.size() * sizeof(float); - std::size_t in3_bytes = in3.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_in1; - float* d_in2; - float* d_in3; - float* d_out; - cudaMalloc((void**)(&d_in1), in1_bytes); - cudaMalloc((void**)(&d_in2), in2_bytes); - cudaMalloc((void**)(&d_in3), in3_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_in1, in1.data(), in1_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in2, in2.data(), in2_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in3, in3.data(), in3_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in1(d_in1, Eigen::array<int, 3>(72,53,97)); - Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in2(d_in2, Eigen::array<int, 3>(72,53,97)); - Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in3(d_in3, Eigen::array<int, 3>(72,53,97)); - Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_out(d_out, Eigen::array<int, 3>(72,53,97)); - - gpu_out.device(gpu_device) = gpu_in1 + gpu_in2 * gpu_in3; - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 72; ++i) { - for (int j = 0; j < 53; ++j) { - for (int k = 0; k < 97; ++k) { - VERIFY_IS_APPROX(out(Eigen::array<int, 3>(i,j,k)), in1(Eigen::array<int, 3>(i,j,k)) + in2(Eigen::array<int, 3>(i,j,k)) * in3(Eigen::array<int, 3>(i,j,k))); - } - } - } - - cudaFree(d_in1); - cudaFree(d_in2); - cudaFree(d_in3); - cudaFree(d_out); -} - -void test_cuda_props() { - Tensor<float, 1> in1(200); - Tensor<bool, 1> out(200); - in1.setRandom(); - - std::size_t in1_bytes = in1.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(bool); - - float* d_in1; - bool* d_out; - cudaMalloc((void**)(&d_in1), in1_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_in1, in1.data(), in1_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1( - d_in1, 200); - Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_out( - d_out, 200); - - gpu_out.device(gpu_device) = (gpu_in1.isnan)(); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, - gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 200; ++i) { - VERIFY_IS_EQUAL(out(i), (std::isnan)(in1(i))); - } - - cudaFree(d_in1); - cudaFree(d_out); -} - -void test_cuda_reduction() -{ - Tensor<float, 4> in1(72,53,97,113); - Tensor<float, 2> out(72,97); - in1.setRandom(); - - std::size_t in1_bytes = in1.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_in1; - float* d_out; - cudaMalloc((void**)(&d_in1), in1_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_in1, in1.data(), in1_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4> > gpu_in1(d_in1, 72,53,97,113); - Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97); - - array<int, 2> reduction_axis; - reduction_axis[0] = 1; - reduction_axis[1] = 3; - - gpu_out.device(gpu_device) = gpu_in1.maximum(reduction_axis); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 72; ++i) { - for (int j = 0; j < 97; ++j) { - float expected = 0; - for (int k = 0; k < 53; ++k) { - for (int l = 0; l < 113; ++l) { - expected = - std::max<float>(expected, in1(i, k, j, l)); - } - } - VERIFY_IS_APPROX(out(i,j), expected); - } - } - - cudaFree(d_in1); - cudaFree(d_out); -} - -template<int DataLayout> -void test_cuda_contraction() -{ - // with these dimensions, the output has 300 * 140 elements, which is - // more than 30 * 1024, which is the number of threads in blocks on - // a 15 SM GK110 GPU - Tensor<float, 4, DataLayout> t_left(6, 50, 3, 31); - Tensor<float, 5, DataLayout> t_right(Eigen::array<int, 5>(3, 31, 7, 20, 1)); - Tensor<float, 5, DataLayout> t_result(Eigen::array<int, 5>(6, 50, 7, 20, 1)); - - t_left.setRandom(); - t_right.setRandom(); - - std::size_t t_left_bytes = t_left.size() * sizeof(float); - std::size_t t_right_bytes = t_right.size() * sizeof(float); - std::size_t t_result_bytes = t_result.size() * sizeof(float); - - float* d_t_left; - float* d_t_right; - float* d_t_result; - - cudaMalloc((void**)(&d_t_left), t_left_bytes); - cudaMalloc((void**)(&d_t_right), t_right_bytes); - cudaMalloc((void**)(&d_t_result), t_result_bytes); - - cudaMemcpy(d_t_left, t_left.data(), t_left_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_t_right, t_right.data(), t_right_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_t_left(d_t_left, 6, 50, 3, 31); - Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_right(d_t_right, 3, 31, 7, 20, 1); - Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_result(d_t_result, 6, 50, 7, 20, 1); - - typedef Eigen::Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> > MapXf; - MapXf m_left(t_left.data(), 300, 93); - MapXf m_right(t_right.data(), 93, 140); - Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(300, 140); - - typedef Tensor<float, 1>::DimensionPair DimPair; - Eigen::array<DimPair, 2> dims; - dims[0] = DimPair(2, 0); - dims[1] = DimPair(3, 1); - - m_result = m_left * m_right; - gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims); - - cudaMemcpy(t_result.data(), d_t_result, t_result_bytes, cudaMemcpyDeviceToHost); - - for (size_t i = 0; i < t_result.dimensions().TotalSize(); i++) { - if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) { - std::cout << "mismatch detected at index " << i << ": " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; - assert(false); - } - } - - cudaFree(d_t_left); - cudaFree(d_t_right); - cudaFree(d_t_result); -} - -template<int DataLayout> -void test_cuda_convolution_1d() -{ - Tensor<float, 4, DataLayout> input(74,37,11,137); - Tensor<float, 1, DataLayout> kernel(4); - Tensor<float, 4, DataLayout> out(74,34,11,137); - input = input.constant(10.0f) + input.random(); - kernel = kernel.constant(7.0f) + kernel.random(); - - std::size_t input_bytes = input.size() * sizeof(float); - std::size_t kernel_bytes = kernel.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_input; - float* d_kernel; - float* d_out; - cudaMalloc((void**)(&d_input), input_bytes); - cudaMalloc((void**)(&d_kernel), kernel_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_input, input.data(), input_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_kernel, kernel.data(), kernel_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input, 74,37,11,137); - Eigen::TensorMap<Eigen::Tensor<float, 1, DataLayout> > gpu_kernel(d_kernel, 4); - Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out, 74,34,11,137); - - Eigen::array<int, 1> dims(1); - gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 74; ++i) { - for (int j = 0; j < 34; ++j) { - for (int k = 0; k < 11; ++k) { - for (int l = 0; l < 137; ++l) { - const float result = out(i,j,k,l); - const float expected = input(i,j+0,k,l) * kernel(0) + input(i,j+1,k,l) * kernel(1) + - input(i,j+2,k,l) * kernel(2) + input(i,j+3,k,l) * kernel(3); - VERIFY_IS_APPROX(result, expected); - } - } - } - } - - cudaFree(d_input); - cudaFree(d_kernel); - cudaFree(d_out); -} - -void test_cuda_convolution_inner_dim_col_major_1d() -{ - Tensor<float, 4, ColMajor> input(74,9,11,7); - Tensor<float, 1, ColMajor> kernel(4); - Tensor<float, 4, ColMajor> out(71,9,11,7); - input = input.constant(10.0f) + input.random(); - kernel = kernel.constant(7.0f) + kernel.random(); - - std::size_t input_bytes = input.size() * sizeof(float); - std::size_t kernel_bytes = kernel.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_input; - float* d_kernel; - float* d_out; - cudaMalloc((void**)(&d_input), input_bytes); - cudaMalloc((void**)(&d_kernel), kernel_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_input, input.data(), input_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_kernel, kernel.data(), kernel_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_input(d_input,74,9,11,7); - Eigen::TensorMap<Eigen::Tensor<float, 1, ColMajor> > gpu_kernel(d_kernel,4); - Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_out(d_out,71,9,11,7); - - Eigen::array<int, 1> dims(0); - gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 71; ++i) { - for (int j = 0; j < 9; ++j) { - for (int k = 0; k < 11; ++k) { - for (int l = 0; l < 7; ++l) { - const float result = out(i,j,k,l); - const float expected = input(i+0,j,k,l) * kernel(0) + input(i+1,j,k,l) * kernel(1) + - input(i+2,j,k,l) * kernel(2) + input(i+3,j,k,l) * kernel(3); - VERIFY_IS_APPROX(result, expected); - } - } - } - } - - cudaFree(d_input); - cudaFree(d_kernel); - cudaFree(d_out); -} - -void test_cuda_convolution_inner_dim_row_major_1d() -{ - Tensor<float, 4, RowMajor> input(7,9,11,74); - Tensor<float, 1, RowMajor> kernel(4); - Tensor<float, 4, RowMajor> out(7,9,11,71); - input = input.constant(10.0f) + input.random(); - kernel = kernel.constant(7.0f) + kernel.random(); - - std::size_t input_bytes = input.size() * sizeof(float); - std::size_t kernel_bytes = kernel.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_input; - float* d_kernel; - float* d_out; - cudaMalloc((void**)(&d_input), input_bytes); - cudaMalloc((void**)(&d_kernel), kernel_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_input, input.data(), input_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_kernel, kernel.data(), kernel_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_input(d_input, 7,9,11,74); - Eigen::TensorMap<Eigen::Tensor<float, 1, RowMajor> > gpu_kernel(d_kernel, 4); - Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_out(d_out, 7,9,11,71); - - Eigen::array<int, 1> dims(3); - gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 7; ++i) { - for (int j = 0; j < 9; ++j) { - for (int k = 0; k < 11; ++k) { - for (int l = 0; l < 71; ++l) { - const float result = out(i,j,k,l); - const float expected = input(i,j,k,l+0) * kernel(0) + input(i,j,k,l+1) * kernel(1) + - input(i,j,k,l+2) * kernel(2) + input(i,j,k,l+3) * kernel(3); - VERIFY_IS_APPROX(result, expected); - } - } - } - } - - cudaFree(d_input); - cudaFree(d_kernel); - cudaFree(d_out); -} - -template<int DataLayout> -void test_cuda_convolution_2d() -{ - Tensor<float, 4, DataLayout> input(74,37,11,137); - Tensor<float, 2, DataLayout> kernel(3,4); - Tensor<float, 4, DataLayout> out(74,35,8,137); - input = input.constant(10.0f) + input.random(); - kernel = kernel.constant(7.0f) + kernel.random(); - - std::size_t input_bytes = input.size() * sizeof(float); - std::size_t kernel_bytes = kernel.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_input; - float* d_kernel; - float* d_out; - cudaMalloc((void**)(&d_input), input_bytes); - cudaMalloc((void**)(&d_kernel), kernel_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_input, input.data(), input_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_kernel, kernel.data(), kernel_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input,74,37,11,137); - Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > gpu_kernel(d_kernel,3,4); - Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out,74,35,8,137); - - Eigen::array<int, 2> dims(1,2); - gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 74; ++i) { - for (int j = 0; j < 35; ++j) { - for (int k = 0; k < 8; ++k) { - for (int l = 0; l < 137; ++l) { - const float result = out(i,j,k,l); - const float expected = input(i,j+0,k+0,l) * kernel(0,0) + - input(i,j+1,k+0,l) * kernel(1,0) + - input(i,j+2,k+0,l) * kernel(2,0) + - input(i,j+0,k+1,l) * kernel(0,1) + - input(i,j+1,k+1,l) * kernel(1,1) + - input(i,j+2,k+1,l) * kernel(2,1) + - input(i,j+0,k+2,l) * kernel(0,2) + - input(i,j+1,k+2,l) * kernel(1,2) + - input(i,j+2,k+2,l) * kernel(2,2) + - input(i,j+0,k+3,l) * kernel(0,3) + - input(i,j+1,k+3,l) * kernel(1,3) + - input(i,j+2,k+3,l) * kernel(2,3); - VERIFY_IS_APPROX(result, expected); - } - } - } - } - - cudaFree(d_input); - cudaFree(d_kernel); - cudaFree(d_out); -} - -template<int DataLayout> -void test_cuda_convolution_3d() -{ - Tensor<float, 5, DataLayout> input(Eigen::array<int, 5>(74,37,11,137,17)); - Tensor<float, 3, DataLayout> kernel(3,4,2); - Tensor<float, 5, DataLayout> out(Eigen::array<int, 5>(74,35,8,136,17)); - input = input.constant(10.0f) + input.random(); - kernel = kernel.constant(7.0f) + kernel.random(); - - std::size_t input_bytes = input.size() * sizeof(float); - std::size_t kernel_bytes = kernel.size() * sizeof(float); - std::size_t out_bytes = out.size() * sizeof(float); - - float* d_input; - float* d_kernel; - float* d_out; - cudaMalloc((void**)(&d_input), input_bytes); - cudaMalloc((void**)(&d_kernel), kernel_bytes); - cudaMalloc((void**)(&d_out), out_bytes); - - cudaMemcpy(d_input, input.data(), input_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_kernel, kernel.data(), kernel_bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_input(d_input,74,37,11,137,17); - Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> > gpu_kernel(d_kernel,3,4,2); - Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_out(d_out,74,35,8,136,17); - - Eigen::array<int, 3> dims(1,2,3); - gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); - - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 74; ++i) { - for (int j = 0; j < 35; ++j) { - for (int k = 0; k < 8; ++k) { - for (int l = 0; l < 136; ++l) { - for (int m = 0; m < 17; ++m) { - const float result = out(i,j,k,l,m); - const float expected = input(i,j+0,k+0,l+0,m) * kernel(0,0,0) + - input(i,j+1,k+0,l+0,m) * kernel(1,0,0) + - input(i,j+2,k+0,l+0,m) * kernel(2,0,0) + - input(i,j+0,k+1,l+0,m) * kernel(0,1,0) + - input(i,j+1,k+1,l+0,m) * kernel(1,1,0) + - input(i,j+2,k+1,l+0,m) * kernel(2,1,0) + - input(i,j+0,k+2,l+0,m) * kernel(0,2,0) + - input(i,j+1,k+2,l+0,m) * kernel(1,2,0) + - input(i,j+2,k+2,l+0,m) * kernel(2,2,0) + - input(i,j+0,k+3,l+0,m) * kernel(0,3,0) + - input(i,j+1,k+3,l+0,m) * kernel(1,3,0) + - input(i,j+2,k+3,l+0,m) * kernel(2,3,0) + - input(i,j+0,k+0,l+1,m) * kernel(0,0,1) + - input(i,j+1,k+0,l+1,m) * kernel(1,0,1) + - input(i,j+2,k+0,l+1,m) * kernel(2,0,1) + - input(i,j+0,k+1,l+1,m) * kernel(0,1,1) + - input(i,j+1,k+1,l+1,m) * kernel(1,1,1) + - input(i,j+2,k+1,l+1,m) * kernel(2,1,1) + - input(i,j+0,k+2,l+1,m) * kernel(0,2,1) + - input(i,j+1,k+2,l+1,m) * kernel(1,2,1) + - input(i,j+2,k+2,l+1,m) * kernel(2,2,1) + - input(i,j+0,k+3,l+1,m) * kernel(0,3,1) + - input(i,j+1,k+3,l+1,m) * kernel(1,3,1) + - input(i,j+2,k+3,l+1,m) * kernel(2,3,1); - VERIFY_IS_APPROX(result, expected); - } - } - } - } - } - - cudaFree(d_input); - cudaFree(d_kernel); - cudaFree(d_out); -} - - -template <typename Scalar> -void test_cuda_lgamma(const Scalar stddev) -{ - Tensor<Scalar, 2> in(72,97); - in.setRandom(); - in *= in.constant(stddev); - Tensor<Scalar, 2> out(72,97); - out.setZero(); - - std::size_t bytes = in.size() * sizeof(Scalar); - - Scalar* d_in; - Scalar* d_out; - cudaMalloc((void**)(&d_in), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in, in.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); - - gpu_out.device(gpu_device) = gpu_in.lgamma(); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 72; ++i) { - for (int j = 0; j < 97; ++j) { - VERIFY_IS_APPROX(out(i,j), (std::lgamma)(in(i,j))); - } - } - - cudaFree(d_in); - cudaFree(d_out); -} - -template <typename Scalar> -void test_cuda_digamma() -{ - Tensor<Scalar, 1> in(7); - Tensor<Scalar, 1> out(7); - Tensor<Scalar, 1> expected_out(7); - out.setZero(); - - in(0) = Scalar(1); - in(1) = Scalar(1.5); - in(2) = Scalar(4); - in(3) = Scalar(-10.5); - in(4) = Scalar(10000.5); - in(5) = Scalar(0); - in(6) = Scalar(-1); - - expected_out(0) = Scalar(-0.5772156649015329); - expected_out(1) = Scalar(0.03648997397857645); - expected_out(2) = Scalar(1.2561176684318); - expected_out(3) = Scalar(2.398239129535781); - expected_out(4) = Scalar(9.210340372392849); - expected_out(5) = std::numeric_limits<Scalar>::infinity(); - expected_out(6) = std::numeric_limits<Scalar>::infinity(); - - std::size_t bytes = in.size() * sizeof(Scalar); - - Scalar* d_in; - Scalar* d_out; - cudaMalloc((void**)(&d_in), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in, in.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in(d_in, 7); - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7); - - gpu_out.device(gpu_device) = gpu_in.digamma(); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 5; ++i) { - VERIFY_IS_APPROX(out(i), expected_out(i)); - } - for (int i = 5; i < 7; ++i) { - VERIFY_IS_EQUAL(out(i), expected_out(i)); - } -} - -template <typename Scalar> -void test_cuda_zeta() -{ - Tensor<Scalar, 1> in_x(6); - Tensor<Scalar, 1> in_q(6); - Tensor<Scalar, 1> out(6); - Tensor<Scalar, 1> expected_out(6); - out.setZero(); - - in_x(0) = Scalar(1); - in_x(1) = Scalar(1.5); - in_x(2) = Scalar(4); - in_x(3) = Scalar(-10.5); - in_x(4) = Scalar(10000.5); - in_x(5) = Scalar(3); - - in_q(0) = Scalar(1.2345); - in_q(1) = Scalar(2); - in_q(2) = Scalar(1.5); - in_q(3) = Scalar(3); - in_q(4) = Scalar(1.0001); - in_q(5) = Scalar(-2.5); - - expected_out(0) = std::numeric_limits<Scalar>::infinity(); - expected_out(1) = Scalar(1.61237534869); - expected_out(2) = Scalar(0.234848505667); - expected_out(3) = Scalar(1.03086757337e-5); - expected_out(4) = Scalar(0.367879440865); - expected_out(5) = Scalar(0.054102025820864097); - - std::size_t bytes = in_x.size() * sizeof(Scalar); - - Scalar* d_in_x; - Scalar* d_in_q; - Scalar* d_out; - cudaMalloc((void**)(&d_in_x), bytes); - cudaMalloc((void**)(&d_in_q), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in_x, in_x.data(), bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in_q, in_q.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_q(d_in_q, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 6); - - gpu_out.device(gpu_device) = gpu_in_x.zeta(gpu_in_q); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - VERIFY_IS_EQUAL(out(0), expected_out(0)); - VERIFY((std::isnan)(out(3))); - - for (int i = 1; i < 6; ++i) { - if (i != 3) { - VERIFY_IS_APPROX(out(i), expected_out(i)); - } - } -} - -template <typename Scalar> -void test_cuda_polygamma() -{ - Tensor<Scalar, 1> in_x(7); - Tensor<Scalar, 1> in_n(7); - Tensor<Scalar, 1> out(7); - Tensor<Scalar, 1> expected_out(7); - out.setZero(); - - in_n(0) = Scalar(1); - in_n(1) = Scalar(1); - in_n(2) = Scalar(1); - in_n(3) = Scalar(17); - in_n(4) = Scalar(31); - in_n(5) = Scalar(28); - in_n(6) = Scalar(8); - - in_x(0) = Scalar(2); - in_x(1) = Scalar(3); - in_x(2) = Scalar(25.5); - in_x(3) = Scalar(4.7); - in_x(4) = Scalar(11.8); - in_x(5) = Scalar(17.7); - in_x(6) = Scalar(30.2); - - expected_out(0) = Scalar(0.644934066848); - expected_out(1) = Scalar(0.394934066848); - expected_out(2) = Scalar(0.0399946696496); - expected_out(3) = Scalar(293.334565435); - expected_out(4) = Scalar(0.445487887616); - expected_out(5) = Scalar(-2.47810300902e-07); - expected_out(6) = Scalar(-8.29668781082e-09); - - std::size_t bytes = in_x.size() * sizeof(Scalar); - - Scalar* d_in_x; - Scalar* d_in_n; - Scalar* d_out; - cudaMalloc((void**)(&d_in_x), bytes); - cudaMalloc((void**)(&d_in_n), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in_x, in_x.data(), bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in_n, in_n.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 7); - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_n(d_in_n, 7); - Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7); - - gpu_out.device(gpu_device) = gpu_in_n.polygamma(gpu_in_x); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 7; ++i) { - VERIFY_IS_APPROX(out(i), expected_out(i)); - } -} - -template <typename Scalar> -void test_cuda_igamma() -{ - Tensor<Scalar, 2> a(6, 6); - Tensor<Scalar, 2> x(6, 6); - Tensor<Scalar, 2> out(6, 6); - out.setZero(); - - Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - - for (int i = 0; i < 6; ++i) { - for (int j = 0; j < 6; ++j) { - a(i, j) = a_s[i]; - x(i, j) = x_s[j]; - } - } - - Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); - Scalar igamma_s[][6] = {{0.0, nan, nan, nan, nan, nan}, - {0.0, 0.6321205588285578, 0.7768698398515702, - 0.9816843611112658, 9.999500016666262e-05, 1.0}, - {0.0, 0.4275932955291202, 0.608374823728911, - 0.9539882943107686, 7.522076445089201e-07, 1.0}, - {0.0, 0.01898815687615381, 0.06564245437845008, - 0.5665298796332909, 4.166333347221828e-18, 1.0}, - {0.0, 0.9999780593618628, 0.9999899967080838, - 0.9999996219837988, 0.9991370418689945, 1.0}, - {0.0, 0.0, 0.0, 0.0, 0.0, 0.5042041932513908}}; - - - - std::size_t bytes = a.size() * sizeof(Scalar); - - Scalar* d_a; - Scalar* d_x; - Scalar* d_out; - cudaMalloc((void**)(&d_a), bytes); - cudaMalloc((void**)(&d_x), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_a, a.data(), bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_x, x.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6); - - gpu_out.device(gpu_device) = gpu_a.igamma(gpu_x); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 6; ++i) { - for (int j = 0; j < 6; ++j) { - if ((std::isnan)(igamma_s[i][j])) { - VERIFY((std::isnan)(out(i, j))); - } else { - VERIFY_IS_APPROX(out(i, j), igamma_s[i][j]); - } - } - } -} - -template <typename Scalar> -void test_cuda_igammac() -{ - Tensor<Scalar, 2> a(6, 6); - Tensor<Scalar, 2> x(6, 6); - Tensor<Scalar, 2> out(6, 6); - out.setZero(); - - Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; - - for (int i = 0; i < 6; ++i) { - for (int j = 0; j < 6; ++j) { - a(i, j) = a_s[i]; - x(i, j) = x_s[j]; - } - } - - Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); - Scalar igammac_s[][6] = {{nan, nan, nan, nan, nan, nan}, - {1.0, 0.36787944117144233, 0.22313016014842982, - 0.018315638888734182, 0.9999000049998333, 0.0}, - {1.0, 0.5724067044708798, 0.3916251762710878, - 0.04601170568923136, 0.9999992477923555, 0.0}, - {1.0, 0.9810118431238462, 0.9343575456215499, - 0.4334701203667089, 1.0, 0.0}, - {1.0, 2.1940638138146658e-05, 1.0003291916285e-05, - 3.7801620118431334e-07, 0.0008629581310054535, - 0.0}, - {1.0, 1.0, 1.0, 1.0, 1.0, 0.49579580674813944}}; - - std::size_t bytes = a.size() * sizeof(Scalar); - - Scalar* d_a; - Scalar* d_x; - Scalar* d_out; - cudaMalloc((void**)(&d_a), bytes); - cudaMalloc((void**)(&d_x), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_a, a.data(), bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_x, x.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6); - - gpu_out.device(gpu_device) = gpu_a.igammac(gpu_x); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 6; ++i) { - for (int j = 0; j < 6; ++j) { - if ((std::isnan)(igammac_s[i][j])) { - VERIFY((std::isnan)(out(i, j))); - } else { - VERIFY_IS_APPROX(out(i, j), igammac_s[i][j]); - } - } - } -} - -template <typename Scalar> -void test_cuda_erf(const Scalar stddev) -{ - Tensor<Scalar, 2> in(72,97); - in.setRandom(); - in *= in.constant(stddev); - Tensor<Scalar, 2> out(72,97); - out.setZero(); - - std::size_t bytes = in.size() * sizeof(Scalar); - - Scalar* d_in; - Scalar* d_out; - cudaMalloc((void**)(&d_in), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in, in.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); - - gpu_out.device(gpu_device) = gpu_in.erf(); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 72; ++i) { - for (int j = 0; j < 97; ++j) { - VERIFY_IS_APPROX(out(i,j), (std::erf)(in(i,j))); - } - } - - cudaFree(d_in); - cudaFree(d_out); -} - -template <typename Scalar> -void test_cuda_erfc(const Scalar stddev) -{ - Tensor<Scalar, 2> in(72,97); - in.setRandom(); - in *= in.constant(stddev); - Tensor<Scalar, 2> out(72,97); - out.setZero(); - - std::size_t bytes = in.size() * sizeof(Scalar); - - Scalar* d_in; - Scalar* d_out; - cudaMalloc((void**)(&d_in), bytes); - cudaMalloc((void**)(&d_out), bytes); - - cudaMemcpy(d_in, in.data(), bytes, cudaMemcpyHostToDevice); - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); - Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); - - gpu_out.device(gpu_device) = gpu_in.erfc(); - - assert(cudaMemcpyAsync(out.data(), d_out, bytes, cudaMemcpyDeviceToHost, gpu_device.stream()) == cudaSuccess); - assert(cudaStreamSynchronize(gpu_device.stream()) == cudaSuccess); - - for (int i = 0; i < 72; ++i) { - for (int j = 0; j < 97; ++j) { - VERIFY_IS_APPROX(out(i,j), (std::erfc)(in(i,j))); - } - } - - cudaFree(d_in); - cudaFree(d_out); -} - -void test_cxx11_tensor_cuda() -{ - CALL_SUBTEST_1(test_cuda_elementwise_small()); - CALL_SUBTEST_1(test_cuda_elementwise()); - CALL_SUBTEST_1(test_cuda_props()); - CALL_SUBTEST_1(test_cuda_reduction()); - CALL_SUBTEST_2(test_cuda_contraction<ColMajor>()); - CALL_SUBTEST_2(test_cuda_contraction<RowMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_1d<ColMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_1d<RowMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_inner_dim_col_major_1d()); - CALL_SUBTEST_3(test_cuda_convolution_inner_dim_row_major_1d()); - CALL_SUBTEST_3(test_cuda_convolution_2d<ColMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_2d<RowMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_3d<ColMajor>()); - CALL_SUBTEST_3(test_cuda_convolution_3d<RowMajor>()); - -#if __cplusplus > 199711L - // std::erf, std::erfc, and so on where only added in c++11. We use them - // as a golden reference to validate the results produced by Eigen. Therefore - // we can only run these tests if we use a c++11 compiler. - CALL_SUBTEST_4(test_cuda_lgamma<float>(1.0f)); - CALL_SUBTEST_4(test_cuda_lgamma<float>(100.0f)); - CALL_SUBTEST_4(test_cuda_lgamma<float>(0.01f)); - CALL_SUBTEST_4(test_cuda_lgamma<float>(0.001f)); - - CALL_SUBTEST_4(test_cuda_lgamma<double>(1.0)); - CALL_SUBTEST_4(test_cuda_lgamma<double>(100.0)); - CALL_SUBTEST_4(test_cuda_lgamma<double>(0.01)); - CALL_SUBTEST_4(test_cuda_lgamma<double>(0.001)); - - CALL_SUBTEST_4(test_cuda_erf<float>(1.0f)); - CALL_SUBTEST_4(test_cuda_erf<float>(100.0f)); - CALL_SUBTEST_4(test_cuda_erf<float>(0.01f)); - CALL_SUBTEST_4(test_cuda_erf<float>(0.001f)); - - CALL_SUBTEST_4(test_cuda_erfc<float>(1.0f)); - // CALL_SUBTEST(test_cuda_erfc<float>(100.0f)); - CALL_SUBTEST_4(test_cuda_erfc<float>(5.0f)); // CUDA erfc lacks precision for large inputs - CALL_SUBTEST_4(test_cuda_erfc<float>(0.01f)); - CALL_SUBTEST_4(test_cuda_erfc<float>(0.001f)); - - CALL_SUBTEST_4(test_cuda_erf<double>(1.0)); - CALL_SUBTEST_4(test_cuda_erf<double>(100.0)); - CALL_SUBTEST_4(test_cuda_erf<double>(0.01)); - CALL_SUBTEST_4(test_cuda_erf<double>(0.001)); - - CALL_SUBTEST_4(test_cuda_erfc<double>(1.0)); - // CALL_SUBTEST(test_cuda_erfc<double>(100.0)); - CALL_SUBTEST_4(test_cuda_erfc<double>(5.0)); // CUDA erfc lacks precision for large inputs - CALL_SUBTEST_4(test_cuda_erfc<double>(0.01)); - CALL_SUBTEST_4(test_cuda_erfc<double>(0.001)); - - CALL_SUBTEST_5(test_cuda_digamma<float>()); - CALL_SUBTEST_5(test_cuda_digamma<double>()); - - CALL_SUBTEST_5(test_cuda_polygamma<float>()); - CALL_SUBTEST_5(test_cuda_polygamma<double>()); - - CALL_SUBTEST_5(test_cuda_zeta<float>()); - CALL_SUBTEST_5(test_cuda_zeta<double>()); - - CALL_SUBTEST_5(test_cuda_igamma<float>()); - CALL_SUBTEST_5(test_cuda_igammac<float>()); - - CALL_SUBTEST_5(test_cuda_igamma<double>()); - CALL_SUBTEST_5(test_cuda_igammac<double>()); -#endif -}
diff --git a/unsupported/test/cxx11_tensor_custom_index.cpp b/unsupported/test/cxx11_tensor_custom_index.cpp index 4528cc1..b5dbc97 100644 --- a/unsupported/test/cxx11_tensor_custom_index.cpp +++ b/unsupported/test/cxx11_tensor_custom_index.cpp
@@ -88,7 +88,7 @@ } -void test_cxx11_tensor_custom_index() { +EIGEN_DECLARE_TEST(cxx11_tensor_custom_index) { test_map_as_index<ColMajor>(); test_map_as_index<RowMajor>(); test_matrix_as_index<ColMajor>();
diff --git a/unsupported/test/cxx11_tensor_custom_op.cpp b/unsupported/test/cxx11_tensor_custom_op.cpp index 8baa477..875ea57 100644 --- a/unsupported/test/cxx11_tensor_custom_op.cpp +++ b/unsupported/test/cxx11_tensor_custom_op.cpp
@@ -104,7 +104,7 @@ } -void test_cxx11_tensor_custom_op() +EIGEN_DECLARE_TEST(cxx11_tensor_custom_op) { CALL_SUBTEST(test_custom_unary_op()); CALL_SUBTEST(test_custom_binary_op());
diff --git a/unsupported/test/cxx11_tensor_custom_op_sycl.cpp b/unsupported/test/cxx11_tensor_custom_op_sycl.cpp new file mode 100644 index 0000000..cc3b024 --- /dev/null +++ b/unsupported/test/cxx11_tensor_custom_op_sycl.cpp
@@ -0,0 +1,165 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; +template<typename TensorType> +struct InsertZeros { + DSizes<DenseIndex, 2> dimensions(const TensorType& input) const { + DSizes<DenseIndex, 2> result; + result[0] = input.dimension(0) * 2; + result[1] = input.dimension(1) * 2; + return result; + } + + template <typename Output, typename Device> + void eval(const TensorType& input, Output& output, const Device& device) const + { + array<DenseIndex, 2> strides; + strides[0] = 2; + strides[1] = 2; + output.stride(strides).device(device) = input; + + Eigen::DSizes<DenseIndex, 2> offsets(1,1); + Eigen::DSizes<DenseIndex, 2> extents(output.dimension(0)-1, output.dimension(1)-1); + output.slice(offsets, extents).stride(strides).device(device) = input.constant(0.0f); + } +}; + +template<typename DataType, int DataLayout, typename IndexType> +static void test_custom_unary_op_sycl(const Eigen::SyclDevice &sycl_device) +{ + IndexType sizeDim1 = 3; + IndexType sizeDim2 = 5; + Eigen::array<IndexType, 2> tensorRange = {{sizeDim1, sizeDim2}}; + Eigen::array<IndexType, 2> tensorResultRange = {{6, 10}}; + + Eigen::Tensor<DataType, 2, DataLayout, IndexType> in1(tensorRange); + Eigen::Tensor<DataType, 2, DataLayout, IndexType> out(tensorResultRange); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(in1.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.dimensions().TotalSize()*sizeof(DataType))); + + typedef Eigen::TensorMap<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > TensorType; + TensorType gpu_in1(gpu_in1_data, tensorRange); + TensorType gpu_out(gpu_out_data, tensorResultRange); + + in1.setRandom(); + sycl_device.memcpyHostToDevice(gpu_in1_data, in1.data(),(in1.dimensions().TotalSize())*sizeof(DataType)); + gpu_out.device(sycl_device) = gpu_in1.customOp(InsertZeros<TensorType>()); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.dimensions().TotalSize())*sizeof(DataType)); + + VERIFY_IS_EQUAL(out.dimension(0), 6); + VERIFY_IS_EQUAL(out.dimension(1), 10); + + for (int i = 0; i < 6; i+=2) { + for (int j = 0; j < 10; j+=2) { + VERIFY_IS_EQUAL(out(i, j), in1(i/2, j/2)); + } + } + for (int i = 1; i < 6; i+=2) { + for (int j = 1; j < 10; j+=2) { + VERIFY_IS_EQUAL(out(i, j), 0); + } + } +} + +template<typename TensorType> +struct BatchMatMul { + DSizes<DenseIndex, 3> dimensions(const TensorType& input1, const TensorType& input2) const { + DSizes<DenseIndex, 3> result; + result[0] = input1.dimension(0); + result[1] = input2.dimension(1); + result[2] = input2.dimension(2); + return result; + } + + template <typename Output, typename Device> + void eval(const TensorType& input1, const TensorType& input2, + Output& output, const Device& device) const + { + typedef typename TensorType::DimensionPair DimPair; + array<DimPair, 1> dims; + dims[0] = DimPair(1, 0); + for (int64_t i = 0; i < output.dimension(2); ++i) { + output.template chip<2>(i).device(device) = input1.template chip<2>(i).contract(input2.template chip<2>(i), dims); + } + } +}; + +template<typename DataType, int DataLayout, typename IndexType> +static void test_custom_binary_op_sycl(const Eigen::SyclDevice &sycl_device) +{ + + Eigen::array<IndexType, 3> tensorRange1 = {{2, 3, 5}}; + Eigen::array<IndexType, 3> tensorRange2 = {{3,7,5}}; + Eigen::array<IndexType, 3> tensorResultRange = {{2, 7, 5}}; + + Eigen::Tensor<DataType, 3, DataLayout, IndexType> in1(tensorRange1); + Eigen::Tensor<DataType, 3, DataLayout, IndexType> in2(tensorRange2); + Eigen::Tensor<DataType, 3, DataLayout, IndexType> out(tensorResultRange); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(in1.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_in2_data = static_cast<DataType*>(sycl_device.allocate(in2.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.dimensions().TotalSize()*sizeof(DataType))); + + typedef Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType> > TensorType; + TensorType gpu_in1(gpu_in1_data, tensorRange1); + TensorType gpu_in2(gpu_in2_data, tensorRange2); + TensorType gpu_out(gpu_out_data, tensorResultRange); + + in1.setRandom(); + in2.setRandom(); + + sycl_device.memcpyHostToDevice(gpu_in1_data, in1.data(),(in1.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_in2_data, in2.data(),(in2.dimensions().TotalSize())*sizeof(DataType)); + + gpu_out.device(sycl_device) = gpu_in1.customOp(gpu_in2, BatchMatMul<TensorType>()); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.dimensions().TotalSize())*sizeof(DataType)); + + for (IndexType i = 0; i < 5; ++i) { + typedef typename Eigen::Tensor<DataType, 3, DataLayout, IndexType>::DimensionPair DimPair; + array<DimPair, 1> dims; + dims[0] = DimPair(1, 0); + Eigen::Tensor<DataType, 2, DataLayout, IndexType> reference = in1.template chip<2>(i).contract(in2.template chip<2>(i), dims); + TensorRef<Eigen::Tensor<DataType, 2, DataLayout, IndexType> > val = out.template chip<2>(i); + for (IndexType j = 0; j < 2; ++j) { + for (IndexType k = 0; k < 7; ++k) { + VERIFY_IS_APPROX(val(j, k), reference(j, k)); + } + } + } +} + +template <typename DataType, typename Dev_selector> void custom_op_perDevice(Dev_selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_custom_unary_op_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_custom_unary_op_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_custom_binary_op_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_custom_binary_op_sycl<DataType, RowMajor, int64_t>(sycl_device); + +} +EIGEN_DECLARE_TEST(cxx11_tensor_custom_op_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(custom_op_perDevice<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_device.cu b/unsupported/test/cxx11_tensor_device.cu index cbe9e64..c9f78d2 100644 --- a/unsupported/test/cxx11_tensor_device.cu +++ b/unsupported/test/cxx11_tensor_device.cu
@@ -9,14 +9,15 @@ #define EIGEN_TEST_NO_LONGDOUBLE #define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_device + #define EIGEN_DEFAULT_DENSE_INDEX_TYPE int #define EIGEN_USE_GPU - #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> +#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + using Eigen::Tensor; using Eigen::RowMajor; @@ -66,22 +67,22 @@ // Context for evaluation on GPU struct GPUContext { GPUContext(const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in1, Eigen::TensorMap<Eigen::Tensor<float, 3> >& in2, Eigen::TensorMap<Eigen::Tensor<float, 3> >& out) : in1_(in1), in2_(in2), out_(out), gpu_device_(&stream_) { - assert(cudaMalloc((void**)(&kernel_1d_), 2*sizeof(float)) == cudaSuccess); + assert(gpuMalloc((void**)(&kernel_1d_), 2*sizeof(float)) == gpuSuccess); float kernel_1d_val[] = {3.14f, 2.7f}; - assert(cudaMemcpy(kernel_1d_, kernel_1d_val, 2*sizeof(float), cudaMemcpyHostToDevice) == cudaSuccess); + assert(gpuMemcpy(kernel_1d_, kernel_1d_val, 2*sizeof(float), gpuMemcpyHostToDevice) == gpuSuccess); - assert(cudaMalloc((void**)(&kernel_2d_), 4*sizeof(float)) == cudaSuccess); + assert(gpuMalloc((void**)(&kernel_2d_), 4*sizeof(float)) == gpuSuccess); float kernel_2d_val[] = {3.14f, 2.7f, 0.2f, 7.0f}; - assert(cudaMemcpy(kernel_2d_, kernel_2d_val, 4*sizeof(float), cudaMemcpyHostToDevice) == cudaSuccess); + assert(gpuMemcpy(kernel_2d_, kernel_2d_val, 4*sizeof(float), gpuMemcpyHostToDevice) == gpuSuccess); - assert(cudaMalloc((void**)(&kernel_3d_), 8*sizeof(float)) == cudaSuccess); + assert(gpuMalloc((void**)(&kernel_3d_), 8*sizeof(float)) == gpuSuccess); float kernel_3d_val[] = {3.14f, -1.0f, 2.7f, -0.3f, 0.2f, -0.7f, 7.0f, -0.5f}; - assert(cudaMemcpy(kernel_3d_, kernel_3d_val, 8*sizeof(float), cudaMemcpyHostToDevice) == cudaSuccess); + assert(gpuMemcpy(kernel_3d_, kernel_3d_val, 8*sizeof(float), gpuMemcpyHostToDevice) == gpuSuccess); } ~GPUContext() { - assert(cudaFree(kernel_1d_) == cudaSuccess); - assert(cudaFree(kernel_2d_) == cudaSuccess); - assert(cudaFree(kernel_3d_) == cudaSuccess); + assert(gpuFree(kernel_1d_) == gpuSuccess); + assert(gpuFree(kernel_2d_) == gpuSuccess); + assert(gpuFree(kernel_3d_) == gpuSuccess); } const Eigen::GpuDevice& device() const { return gpu_device_; } @@ -102,7 +103,7 @@ float* kernel_2d_; float* kernel_3d_; - Eigen::CudaStreamDevice stream_; + Eigen::GpuStreamDevice stream_; Eigen::GpuDevice gpu_device_; }; @@ -241,7 +242,7 @@ const float result = out(i,j,k); const float expected = (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f) + (in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f); - if (fabs(expected) < 1e-4 && fabs(result) < 1e-4) { + if (fabs(expected) < 1e-4f && fabs(result) < 1e-4f) { continue; } VERIFY_IS_APPROX(expected, result); @@ -258,7 +259,7 @@ in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f) + (in1(i+1,j,k) * -1.0f + in1(i+1,j+1,k) * -0.3f + in1(i+1,j,k+1) * -0.7f + in1(i+1,j+1,k+1) * -0.5f); - if (fabs(expected) < 1e-4 && fabs(result) < 1e-4) { + if (fabs(expected) < 1e-4f && fabs(result) < 1e-4f) { continue; } VERIFY_IS_APPROX(expected, result); @@ -281,12 +282,12 @@ float* d_in1; float* d_in2; float* d_out; - cudaMalloc((void**)(&d_in1), in1_bytes); - cudaMalloc((void**)(&d_in2), in2_bytes); - cudaMalloc((void**)(&d_out), out_bytes); + gpuMalloc((void**)(&d_in1), in1_bytes); + gpuMalloc((void**)(&d_in2), in2_bytes); + gpuMalloc((void**)(&d_out), out_bytes); - cudaMemcpy(d_in1, in1.data(), in1_bytes, cudaMemcpyHostToDevice); - cudaMemcpy(d_in2, in2.data(), in2_bytes, cudaMemcpyHostToDevice); + gpuMemcpy(d_in1, in1.data(), in1_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in2, in2.data(), in2_bytes, gpuMemcpyHostToDevice); Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in1(d_in1, 40,50,70); Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in2(d_in2, 40,50,70); @@ -294,7 +295,7 @@ GPUContext context(gpu_in1, gpu_in2, gpu_out); test_contextual_eval(&context); - assert(cudaMemcpy(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost) == cudaSuccess); + assert(gpuMemcpy(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 50; ++j) { for (int k = 0; k < 70; ++k) { @@ -304,7 +305,7 @@ } test_forced_contextual_eval(&context); - assert(cudaMemcpy(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost) == cudaSuccess); + assert(gpuMemcpy(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 50; ++j) { for (int k = 0; k < 70; ++k) { @@ -314,7 +315,7 @@ } test_compound_assignment(&context); - assert(cudaMemcpy(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost) == cudaSuccess); + assert(gpuMemcpy(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 50; ++j) { for (int k = 0; k < 70; ++k) { @@ -324,7 +325,7 @@ } test_contraction(&context); - assert(cudaMemcpy(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost) == cudaSuccess); + assert(gpuMemcpy(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 40; ++j) { const float result = out(i,j,0); @@ -339,8 +340,8 @@ } test_1d_convolution(&context); - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, context.device().stream()) == cudaSuccess); - assert(cudaStreamSynchronize(context.device().stream()) == cudaSuccess); + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, context.device().stream()) == gpuSuccess); + assert(gpuStreamSynchronize(context.device().stream()) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 49; ++j) { for (int k = 0; k < 70; ++k) { @@ -350,8 +351,8 @@ } test_2d_convolution(&context); - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, context.device().stream()) == cudaSuccess); - assert(cudaStreamSynchronize(context.device().stream()) == cudaSuccess); + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, context.device().stream()) == gpuSuccess); + assert(gpuStreamSynchronize(context.device().stream()) == gpuSuccess); for (int i = 0; i < 40; ++i) { for (int j = 0; j < 49; ++j) { for (int k = 0; k < 69; ++k) { @@ -363,9 +364,13 @@ } } +#if !defined(EIGEN_USE_HIP) +// disable this test on the HIP platform +// 3D tensor convolutions seem to hang on the HIP platform + test_3d_convolution(&context); - assert(cudaMemcpyAsync(out.data(), d_out, out_bytes, cudaMemcpyDeviceToHost, context.device().stream()) == cudaSuccess); - assert(cudaStreamSynchronize(context.device().stream()) == cudaSuccess); + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, context.device().stream()) == gpuSuccess); + assert(gpuStreamSynchronize(context.device().stream()) == gpuSuccess); for (int i = 0; i < 39; ++i) { for (int j = 0; j < 49; ++j) { for (int k = 0; k < 69; ++k) { @@ -378,10 +383,13 @@ } } } + +#endif + } -void test_cxx11_tensor_device() +EIGEN_DECLARE_TEST(cxx11_tensor_device) { CALL_SUBTEST_1(test_cpu()); CALL_SUBTEST_2(test_gpu());
diff --git a/unsupported/test/cxx11_tensor_device_sycl.cpp b/unsupported/test/cxx11_tensor_device_sycl.cpp new file mode 100644 index 0000000..5095cb0 --- /dev/null +++ b/unsupported/test/cxx11_tensor_device_sycl.cpp
@@ -0,0 +1,77 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> +#include <stdint.h> +#include <iostream> + +template <typename DataType, int DataLayout, typename IndexType> +void test_device_memory(const Eigen::SyclDevice &sycl_device) { + std::cout << "Running on : " + << sycl_device.sycl_queue().get_device(). template get_info<cl::sycl::info::device::name>() + <<std::endl; + IndexType sizeDim1 = 100; + array<IndexType, 1> tensorRange = {{sizeDim1}}; + Tensor<DataType, 1, DataLayout,IndexType> in(tensorRange); + Tensor<DataType, 1, DataLayout,IndexType> in1(tensorRange); + memset(in1.data(), 1, in1.size() * sizeof(DataType)); + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(in.size()*sizeof(DataType))); + sycl_device.memset(gpu_in_data, 1, in.size()*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(in.data(), gpu_in_data, in.size()*sizeof(DataType)); + for (IndexType i=0; i<in.size(); i++) { + VERIFY_IS_EQUAL(in(i), in1(i)); + } + sycl_device.deallocate(gpu_in_data); +} + +template <typename DataType, int DataLayout, typename IndexType> +void test_device_exceptions(const Eigen::SyclDevice &sycl_device) { + VERIFY(sycl_device.ok()); + IndexType sizeDim1 = 100; + array<IndexType, 1> tensorDims = {{sizeDim1}}; + DataType* gpu_data = static_cast<DataType*>(sycl_device.allocate(sizeDim1*sizeof(DataType))); + sycl_device.memset(gpu_data, 1, sizeDim1*sizeof(DataType)); + + TensorMap<Tensor<DataType, 1, DataLayout,IndexType>> in(gpu_data, tensorDims); + TensorMap<Tensor<DataType, 1, DataLayout,IndexType>> out(gpu_data, tensorDims); + out.device(sycl_device) = in / in.constant(0); + + sycl_device.synchronize(); + VERIFY(!sycl_device.ok()); + sycl_device.deallocate(gpu_data); +} + +template<typename DataType> void sycl_device_test_per_device(const cl::sycl::device& d){ + std::cout << "Running on " << d.template get_info<cl::sycl::info::device::name>() << std::endl; + QueueInterface queueInterface(d); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_device_memory<DataType, RowMajor, int64_t>(sycl_device); + test_device_memory<DataType, ColMajor, int64_t>(sycl_device); + /// this test throw an exception. enable it if you want to see the exception + //test_device_exceptions<DataType, RowMajor>(sycl_device); + /// this test throw an exception. enable it if you want to see the exception + //test_device_exceptions<DataType, ColMajor>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_device_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_device_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_dimension.cpp b/unsupported/test/cxx11_tensor_dimension.cpp index ce78efe..ee416e1 100644 --- a/unsupported/test/cxx11_tensor_dimension.cpp +++ b/unsupported/test/cxx11_tensor_dimension.cpp
@@ -21,7 +21,7 @@ VERIFY_IS_EQUAL((int)Eigen::internal::array_get<0>(dimensions), 2); VERIFY_IS_EQUAL((int)Eigen::internal::array_get<1>(dimensions), 3); VERIFY_IS_EQUAL((int)Eigen::internal::array_get<2>(dimensions), 7); - VERIFY_IS_EQUAL(dimensions.TotalSize(), 2*3*7); + VERIFY_IS_EQUAL((int)dimensions.TotalSize(), 2*3*7); VERIFY_IS_EQUAL((int)dimensions[0], 2); VERIFY_IS_EQUAL((int)dimensions[1], 3); VERIFY_IS_EQUAL((int)dimensions[2], 7); @@ -34,13 +34,12 @@ VERIFY_IS_EQUAL((int)Eigen::internal::array_get<0>(dimensions), 2); VERIFY_IS_EQUAL((int)Eigen::internal::array_get<1>(dimensions), 3); VERIFY_IS_EQUAL((int)Eigen::internal::array_get<2>(dimensions), 7); - VERIFY_IS_EQUAL(dimensions.TotalSize(), 2*3*7); + VERIFY_IS_EQUAL((int)dimensions.TotalSize(), 2*3*7); } - static void test_match() { - Eigen::DSizes<int, 3> dyn(2,3,7); + Eigen::DSizes<unsigned int, 3> dyn((unsigned int)2,(unsigned int)3,(unsigned int)7); Eigen::Sizes<2,3,7> stat; VERIFY_IS_EQUAL(Eigen::dimensions_match(dyn, stat), true); @@ -49,10 +48,41 @@ VERIFY_IS_EQUAL(Eigen::dimensions_match(dyn1, dyn2), false); } +static void test_rank_zero() +{ + Eigen::Sizes<> scalar; + VERIFY_IS_EQUAL((int)scalar.TotalSize(), 1); + VERIFY_IS_EQUAL((int)scalar.rank(), 0); + VERIFY_IS_EQUAL((int)internal::array_prod(scalar), 1); -void test_cxx11_tensor_dimension() + Eigen::DSizes<ptrdiff_t, 0> dscalar; + VERIFY_IS_EQUAL((int)dscalar.TotalSize(), 1); + VERIFY_IS_EQUAL((int)dscalar.rank(), 0); +} + +static void test_index_type_promotion() { + Eigen::DSizes<int, 3> src0(1, 2, 3); + Eigen::array<int, 3> src1; + src1[0] = 4; + src1[1] = 5; + src1[2] = 6; + + Eigen::DSizes<long, 3> dst0(src0); + Eigen::DSizes<long, 3> dst1(src1); + + VERIFY_IS_EQUAL(dst0[0], 1L); + VERIFY_IS_EQUAL(dst0[1], 2L); + VERIFY_IS_EQUAL(dst0[2], 3L); + VERIFY_IS_EQUAL(dst1[0], 4L); + VERIFY_IS_EQUAL(dst1[1], 5L); + VERIFY_IS_EQUAL(dst1[2], 6L); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_dimension) { CALL_SUBTEST(test_dynamic_size()); CALL_SUBTEST(test_fixed_size()); CALL_SUBTEST(test_match()); + CALL_SUBTEST(test_rank_zero()); + CALL_SUBTEST(test_index_type_promotion()); }
diff --git a/unsupported/test/cxx11_tensor_empty.cpp b/unsupported/test/cxx11_tensor_empty.cpp index 9130fff..fd889c4 100644 --- a/unsupported/test/cxx11_tensor_empty.cpp +++ b/unsupported/test/cxx11_tensor_empty.cpp
@@ -24,16 +24,16 @@ static void test_empty_fixed_size_tensor() { - TensorFixedSize<float, Sizes<0>> source; - TensorFixedSize<float, Sizes<0>> tgt1 = source; - TensorFixedSize<float, Sizes<0>> tgt2(source); - TensorFixedSize<float, Sizes<0>> tgt3; + TensorFixedSize<float, Sizes<0> > source; + TensorFixedSize<float, Sizes<0> > tgt1 = source; + TensorFixedSize<float, Sizes<0> > tgt2(source); + TensorFixedSize<float, Sizes<0> > tgt3; tgt3 = tgt1; tgt3 = tgt2; } -void test_cxx11_tensor_empty() +EIGEN_DECLARE_TEST(cxx11_tensor_empty) { CALL_SUBTEST(test_empty_tensor()); CALL_SUBTEST(test_empty_fixed_size_tensor());
diff --git a/unsupported/test/cxx11_tensor_executor.cpp b/unsupported/test/cxx11_tensor_executor.cpp new file mode 100644 index 0000000..18c87b3 --- /dev/null +++ b/unsupported/test/cxx11_tensor_executor.cpp
@@ -0,0 +1,535 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2018 Eugene Zhulenev <ezhulenev@google.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_USE_THREADS + +#include "main.h" + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; +using Eigen::RowMajor; +using Eigen::ColMajor; + +// A set of tests to verify that different TensorExecutor strategies yields the +// same results for all the ops, supporting tiled evaluation. + +template <int NumDims> +static array<Index, NumDims> RandomDims(int min_dim = 1, int max_dim = 20) { + array<Index, NumDims> dims; + for (int i = 0; i < NumDims; ++i) { + dims[i] = internal::random<int>(min_dim, max_dim); + } + return dims; +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_unary_expr(Device d) +{ + static constexpr int Options = 0 | Layout; + + // Pick a large enough tensor size to bypass small tensor block evaluation + // optimization. + auto dims = RandomDims<NumDims>(50 / NumDims, 100 / NumDims); + + Tensor<T, NumDims, Options, Index> src(dims); + Tensor<T, NumDims, Options, Index> dst(dims); + + src.setRandom(); + const auto expr = src.square(); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + T square = src.coeff(i) * src.coeff(i); + VERIFY_IS_EQUAL(square, dst.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_binary_expr(Device d) +{ + static constexpr int Options = 0 | Layout; + + // Pick a large enough tensor size to bypass small tensor block evaluation + // optimization. + auto dims = RandomDims<NumDims>(50 / NumDims, 100 / NumDims); + + Tensor<T, NumDims, Options, Index> lhs(dims); + Tensor<T, NumDims, Options, Index> rhs(dims); + Tensor<T, NumDims, Options, Index> dst(dims); + + lhs.setRandom(); + rhs.setRandom(); + + const auto expr = lhs + rhs; + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + T sum = lhs.coeff(i) + rhs.coeff(i); + VERIFY_IS_EQUAL(sum, dst.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_broadcasting(Device d) +{ + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(1, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + const auto broadcasts = RandomDims<NumDims>(1, 7); + const auto expr = src.broadcast(broadcasts); + + // We assume that broadcasting on a default device is tested and correct, so + // we can rely on it to verify correctness of tensor executor and tiling. + Tensor<T, NumDims, Options, Index> golden; + golden = expr; + + // Now do the broadcasting using configured tensor executor. + Tensor<T, NumDims, Options, Index> dst(golden.dimensions()); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_chipping_rvalue(Device d) +{ + auto dims = RandomDims<NumDims>(1, 10); + Tensor<T, NumDims, Layout, Index> src(dims); + src.setRandom(); + +#define TEST_CHIPPING(CHIP_DIM) \ + if (NumDims > (CHIP_DIM)) { \ + const auto offset = internal::random<Index>(0, dims[(CHIP_DIM)] - 1); \ + const auto expr = src.template chip<(CHIP_DIM)>(offset); \ + \ + Tensor<T, NumDims - 1, Layout, Index> golden; \ + golden = expr; \ + \ + Tensor<T, NumDims - 1, Layout, Index> dst(golden.dimensions()); \ + \ + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; \ + using Executor = internal::TensorExecutor<const Assign, Device, \ + Vectorizable, Tileable>; \ + \ + Executor::run(Assign(dst, expr), d); \ + \ + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { \ + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); \ + } \ + } + + TEST_CHIPPING(0) + TEST_CHIPPING(1) + TEST_CHIPPING(2) + TEST_CHIPPING(3) + TEST_CHIPPING(4) + TEST_CHIPPING(5) + +#undef TEST_CHIPPING +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_chipping_lvalue(Device d) +{ + auto dims = RandomDims<NumDims>(1, 10); + +#define TEST_CHIPPING(CHIP_DIM) \ + if (NumDims > (CHIP_DIM)) { \ + /* Generate random data that we'll assign to the chipped tensor dim. */ \ + array<Index, NumDims - 1> src_dims; \ + for (int i = 0; i < NumDims - 1; ++i) { \ + int dim = i < (CHIP_DIM) ? i : i + 1; \ + src_dims[i] = dims[dim]; \ + } \ + \ + Tensor<T, NumDims - 1, Layout, Index> src(src_dims); \ + src.setRandom(); \ + \ + const auto offset = internal::random<Index>(0, dims[(CHIP_DIM)] - 1); \ + \ + /* Generate random data to fill non-chipped dimensions*/ \ + Tensor<T, NumDims, Layout, Index> random(dims); \ + random.setRandom(); \ + \ + Tensor<T, NumDims, Layout, Index> golden(dims); \ + golden = random; \ + golden.template chip<(CHIP_DIM)>(offset) = src; \ + \ + Tensor<T, NumDims, Layout, Index> dst(dims); \ + dst = random; \ + auto expr = dst.template chip<(CHIP_DIM)>(offset); \ + \ + using Assign = TensorAssignOp<decltype(expr), const decltype(src)>; \ + using Executor = internal::TensorExecutor<const Assign, Device, \ + Vectorizable, Tileable>; \ + \ + Executor::run(Assign(expr, src), d); \ + \ + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { \ + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); \ + } \ + } + + TEST_CHIPPING(0) + TEST_CHIPPING(1) + TEST_CHIPPING(2) + TEST_CHIPPING(3) + TEST_CHIPPING(4) + TEST_CHIPPING(5) + +#undef TEST_CHIPPING +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_shuffle_rvalue(Device d) +{ + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(1, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Create a random dimension re-ordering/shuffle. + std::vector<Index> shuffle; + for (int i = 0; i < NumDims; ++i) shuffle.push_back(i); + std::shuffle(shuffle.begin(), shuffle.end(), std::mt19937()); + + const auto expr = src.shuffle(shuffle); + + // We assume that shuffling on a default device is tested and correct, so + // we can rely on it to verify correctness of tensor executor and tiling. + Tensor<T, NumDims, Options, Index> golden; + golden = expr; + + // Now do the shuffling using configured tensor executor. + Tensor<T, NumDims, Options, Index> dst(golden.dimensions()); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_shuffle_lvalue(Device d) +{ + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(5, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Create a random dimension re-ordering/shuffle. + std::vector<Index> shuffle; + for (int i = 0; i < NumDims; ++i) shuffle.push_back(i); + std::shuffle(shuffle.begin(), shuffle.end(), std::mt19937()); + + array<Index, NumDims> shuffled_dims; + for (int i = 0; i < NumDims; ++i) shuffled_dims[shuffle[i]] = dims[i]; + + // We assume that shuffling on a default device is tested and correct, so + // we can rely on it to verify correctness of tensor executor and tiling. + Tensor<T, NumDims, Options, Index> golden(shuffled_dims); + golden.shuffle(shuffle) = src; + + // Now do the shuffling using configured tensor executor. + Tensor<T, NumDims, Options, Index> dst(shuffled_dims); + + auto expr = dst.shuffle(shuffle); + + using Assign = TensorAssignOp<decltype(expr), const decltype(src)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(expr, src), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_reduction(Device d) +{ + static_assert(NumDims >= 2, "NumDims must be greater or equal than 2"); + + static constexpr int ReducedDims = NumDims - 2; + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(5, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Pick two random and unique reduction dimensions. + int reduction0 = internal::random<int>(0, NumDims - 1); + int reduction1 = internal::random<int>(0, NumDims - 1); + while (reduction0 == reduction1) { + reduction1 = internal::random<int>(0, NumDims - 1); + } + + DSizes<Index, 2> reduction_axis; + reduction_axis[0] = reduction0; + reduction_axis[1] = reduction1; + + Tensor<T, ReducedDims, Options, Index> golden = src.sum(reduction_axis); + + // Now do the reduction using configured tensor executor. + Tensor<T, ReducedDims, Options, Index> dst(golden.dimensions()); + + auto expr = src.sum(reduction_axis); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_reshape(Device d) +{ + static_assert(NumDims >= 2, "NumDims must be greater or equal than 2"); + + static constexpr int ReshapedDims = NumDims - 1; + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(5, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Multiple 0th dimension and then shuffle. + std::vector<Index> shuffle; + for (int i = 0; i < ReshapedDims; ++i) shuffle.push_back(i); + std::shuffle(shuffle.begin(), shuffle.end(), std::mt19937()); + + DSizes<Index, ReshapedDims> reshaped_dims; + reshaped_dims[shuffle[0]] = dims[0] * dims[1]; + for (int i = 1; i < ReshapedDims; ++i) reshaped_dims[shuffle[i]] = dims[i + 1]; + + Tensor<T, ReshapedDims, Options, Index> golden = src.reshape(reshaped_dims); + + // Now reshape using configured tensor executor. + Tensor<T, ReshapedDims, Options, Index> dst(golden.dimensions()); + + auto expr = src.reshape(reshaped_dims); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_slice_rvalue(Device d) +{ + static_assert(NumDims >= 2, "NumDims must be greater or equal than 2"); + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(5, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Pick a random slice of src tensor. + auto slice_start = DSizes<Index, NumDims>(RandomDims<NumDims>()); + auto slice_size = DSizes<Index, NumDims>(RandomDims<NumDims>()); + + // Make sure that slice start + size do not overflow tensor dims. + for (int i = 0; i < NumDims; ++i) { + slice_start[i] = numext::mini(dims[i] - 1, slice_start[i]); + slice_size[i] = numext::mini(slice_size[i], dims[i] - slice_start[i]); + } + + Tensor<T, NumDims, Options, Index> golden = + src.slice(slice_start, slice_size); + + // Now reshape using configured tensor executor. + Tensor<T, NumDims, Options, Index> dst(golden.dimensions()); + + auto expr = src.slice(slice_start, slice_size); + + using Assign = TensorAssignOp<decltype(dst), const decltype(expr)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(dst, expr), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +template <typename T, int NumDims, typename Device, bool Vectorizable, + bool Tileable, int Layout> +static void test_execute_slice_lvalue(Device d) +{ + static_assert(NumDims >= 2, "NumDims must be greater or equal than 2"); + static constexpr int Options = 0 | Layout; + + auto dims = RandomDims<NumDims>(5, 10); + Tensor<T, NumDims, Options, Index> src(dims); + src.setRandom(); + + // Pick a random slice of src tensor. + auto slice_start = DSizes<Index, NumDims>(RandomDims<NumDims>(1, 10)); + auto slice_size = DSizes<Index, NumDims>(RandomDims<NumDims>(1, 10)); + + // Make sure that slice start + size do not overflow tensor dims. + for (int i = 0; i < NumDims; ++i) { + slice_start[i] = numext::mini(dims[i] - 1, slice_start[i]); + slice_size[i] = numext::mini(slice_size[i], dims[i] - slice_start[i]); + } + + Tensor<T, NumDims, Options, Index> slice(slice_size); + slice.setRandom(); + + // Assign a slice using default executor. + Tensor<T, NumDims, Options, Index> golden = src; + golden.slice(slice_start, slice_size) = slice; + + // And using configured execution strategy. + Tensor<T, NumDims, Options, Index> dst = src; + auto expr = dst.slice(slice_start, slice_size); + + using Assign = TensorAssignOp<decltype(expr), const decltype(slice)>; + using Executor = + internal::TensorExecutor<const Assign, Device, Vectorizable, Tileable>; + + Executor::run(Assign(expr, slice), d); + + for (Index i = 0; i < dst.dimensions().TotalSize(); ++i) { + VERIFY_IS_EQUAL(dst.coeff(i), golden.coeff(i)); + } +} + +#define CALL_SUBTEST_PART(PART) \ + CALL_SUBTEST_##PART + +#define CALL_SUBTEST_COMBINATIONS(PART, NAME, T, NUM_DIMS) \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, false, false, ColMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, false, true, ColMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, true, false, ColMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, true, true, ColMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, false, false, RowMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, false, true, RowMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, true, false, RowMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, DefaultDevice, true, true, RowMajor>(default_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, false, false, ColMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, false, true, ColMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, true, false, ColMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, true, true, ColMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, false, false, RowMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, false, true, RowMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, true, false, RowMajor>(tp_device))); \ + CALL_SUBTEST_PART(PART)((NAME<T, NUM_DIMS, ThreadPoolDevice, true, true, RowMajor>(tp_device))) + +EIGEN_DECLARE_TEST(cxx11_tensor_executor) { + Eigen::DefaultDevice default_device; + + const auto num_threads = internal::random<int>(1, 24); + Eigen::ThreadPool tp(num_threads); + Eigen::ThreadPoolDevice tp_device(&tp, num_threads); + + CALL_SUBTEST_COMBINATIONS(1, test_execute_unary_expr, float, 3); + CALL_SUBTEST_COMBINATIONS(1, test_execute_unary_expr, float, 4); + CALL_SUBTEST_COMBINATIONS(1, test_execute_unary_expr, float, 5); + + CALL_SUBTEST_COMBINATIONS(2, test_execute_binary_expr, float, 3); + CALL_SUBTEST_COMBINATIONS(2, test_execute_binary_expr, float, 4); + CALL_SUBTEST_COMBINATIONS(2, test_execute_binary_expr, float, 5); + + CALL_SUBTEST_COMBINATIONS(3, test_execute_broadcasting, float, 3); + CALL_SUBTEST_COMBINATIONS(3, test_execute_broadcasting, float, 4); + CALL_SUBTEST_COMBINATIONS(3, test_execute_broadcasting, float, 5); + + CALL_SUBTEST_COMBINATIONS(4, test_execute_chipping_rvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(4, test_execute_chipping_rvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(4, test_execute_chipping_rvalue, float, 5); + + CALL_SUBTEST_COMBINATIONS(5, test_execute_chipping_lvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(5, test_execute_chipping_lvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(5, test_execute_chipping_lvalue, float, 5); + + CALL_SUBTEST_COMBINATIONS(6, test_execute_shuffle_rvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(6, test_execute_shuffle_rvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(6, test_execute_shuffle_rvalue, float, 5); + + CALL_SUBTEST_COMBINATIONS(7, test_execute_shuffle_lvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(7, test_execute_shuffle_lvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(7, test_execute_shuffle_lvalue, float, 5); + + CALL_SUBTEST_COMBINATIONS(8, test_execute_reduction, float, 2); + CALL_SUBTEST_COMBINATIONS(8, test_execute_reduction, float, 3); + CALL_SUBTEST_COMBINATIONS(8, test_execute_reduction, float, 4); + CALL_SUBTEST_COMBINATIONS(8, test_execute_reduction, float, 5); + + CALL_SUBTEST_COMBINATIONS(9, test_execute_reshape, float, 2); + CALL_SUBTEST_COMBINATIONS(9, test_execute_reshape, float, 3); + CALL_SUBTEST_COMBINATIONS(9, test_execute_reshape, float, 4); + CALL_SUBTEST_COMBINATIONS(9, test_execute_reshape, float, 5); + + CALL_SUBTEST_COMBINATIONS(10, test_execute_slice_rvalue, float, 2); + CALL_SUBTEST_COMBINATIONS(10, test_execute_slice_rvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(10, test_execute_slice_rvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(10, test_execute_slice_rvalue, float, 5); + + CALL_SUBTEST_COMBINATIONS(11, test_execute_slice_lvalue, float, 2); + CALL_SUBTEST_COMBINATIONS(11, test_execute_slice_lvalue, float, 3); + CALL_SUBTEST_COMBINATIONS(11, test_execute_slice_lvalue, float, 4); + CALL_SUBTEST_COMBINATIONS(11, test_execute_slice_lvalue, float, 5); + + // Force CMake to split this test. + // EIGEN_SUFFIXES;1;2;3;4;5;6;7;8;9;10;11 +} + +#undef CALL_SUBTEST_COMBINATIONS
diff --git a/unsupported/test/cxx11_tensor_expr.cpp b/unsupported/test/cxx11_tensor_expr.cpp index 8389e98..30924b6 100644 --- a/unsupported/test/cxx11_tensor_expr.cpp +++ b/unsupported/test/cxx11_tensor_expr.cpp
@@ -16,8 +16,8 @@ static void test_1d() { - Tensor<float, 1> vec1({6}); - Tensor<float, 1, RowMajor> vec2({6}); + Tensor<float, 1> vec1(6); + Tensor<float, 1, RowMajor> vec2(6); vec1(0) = 4.0; vec2(0) = 0.0; vec1(1) = 8.0; vec2(1) = 1.0; @@ -112,13 +112,13 @@ Tensor<float, 3> mat1(2,3,7); Tensor<float, 3, RowMajor> mat2(2,3,7); - float val = 1.0; + float val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { mat1(i,j,k) = val; mat2(i,j,k) = val; - val += 1.0; + val += 1.0f; } } } @@ -142,7 +142,7 @@ Tensor<float, 3, RowMajor> mat11(2,3,7); mat11 = mat2 / 3.14f; - val = 1.0; + val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { @@ -155,7 +155,7 @@ VERIFY_IS_APPROX(mat9(i,j,k), val + 3.14f); VERIFY_IS_APPROX(mat10(i,j,k), val - 3.14f); VERIFY_IS_APPROX(mat11(i,j,k), val / 3.14f); - val += 1.0; + val += 1.0f; } } } @@ -167,25 +167,25 @@ Tensor<float, 3> mat2(2,3,7); Tensor<float, 3> mat3(2,3,7); - float val = 1.0; + float val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { mat1(i,j,k) = val; - val += 1.0; + val += 1.0f; } } } mat2 = mat1.constant(3.14f); mat3 = mat1.cwiseMax(7.3f).exp(); - val = 1.0; + val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { VERIFY_IS_APPROX(mat2(i,j,k), 3.14f); VERIFY_IS_APPROX(mat3(i,j,k), expf((std::max)(val, 7.3f))); - val += 1.0; + val += 1.0f; } } } @@ -228,25 +228,25 @@ Tensor<float, 3> mat2(2,3,7); Tensor<float, 3> mat3(2,3,7); - float val = 1.0; + float val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { mat1(i,j,k) = val; - val += 1.0; + val += 1.0f; } } } mat2 = mat1.inverse().unaryExpr(&asinf); mat3 = mat1.unaryExpr(&tanhf); - val = 1.0; + val = 1.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { VERIFY_IS_APPROX(mat2(i,j,k), asinf(1.0f / mat1(i,j,k))); VERIFY_IS_APPROX(mat3(i,j,k), tanhf(mat1(i,j,k))); - val += 1.0; + val += 1.0f; } } } @@ -300,8 +300,73 @@ } } +template <typename Scalar> +void test_minmax_nan_propagation_templ() { + for (int size = 1; size < 17; ++size) { + const Scalar kNan = std::numeric_limits<Scalar>::quiet_NaN(); + Tensor<Scalar, 1> vec_nan(size); + Tensor<Scalar, 1> vec_zero(size); + Tensor<Scalar, 1> vec_res(size); + vec_nan.setConstant(kNan); + vec_zero.setZero(); + vec_res.setZero(); -void test_cxx11_tensor_expr() + // Test that we propagate NaNs in the tensor when applying the + // cwiseMax(scalar) operator, which is used for the Relu operator. + vec_res = vec_nan.cwiseMax(Scalar(0)); + for (int i = 0; i < size; ++i) { + VERIFY((numext::isnan)(vec_res(i))); + } + + // Test that NaNs do not propagate if we reverse the arguments. + vec_res = vec_zero.cwiseMax(kNan); + for (int i = 0; i < size; ++i) { + VERIFY_IS_EQUAL(vec_res(i), Scalar(0)); + } + + // Test that we propagate NaNs in the tensor when applying the + // cwiseMin(scalar) operator. + vec_res.setZero(); + vec_res = vec_nan.cwiseMin(Scalar(0)); + for (int i = 0; i < size; ++i) { + VERIFY((numext::isnan)(vec_res(i))); + } + + // Test that NaNs do not propagate if we reverse the arguments. + vec_res = vec_zero.cwiseMin(kNan); + for (int i = 0; i < size; ++i) { + VERIFY_IS_EQUAL(vec_res(i), Scalar(0)); + } + } +} + +static void test_clip() +{ + Tensor<float, 1> vec(6); + vec(0) = 4.0; + vec(1) = 8.0; + vec(2) = 15.0; + vec(3) = 16.0; + vec(4) = 23.0; + vec(5) = 42.0; + + float kMin = 20; + float kMax = 30; + + Tensor<float, 1> vec_clipped(6); + vec_clipped = vec.clip(kMin, kMax); + for (int i = 0; i < 6; ++i) { + VERIFY_IS_EQUAL(vec_clipped(i), numext::mini(numext::maxi(vec(i), kMin), kMax)); + } +} + +static void test_minmax_nan_propagation() +{ + test_minmax_nan_propagation_templ<float>(); + test_minmax_nan_propagation_templ<double>(); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_expr) { CALL_SUBTEST(test_1d()); CALL_SUBTEST(test_2d()); @@ -311,4 +376,6 @@ CALL_SUBTEST(test_functors()); CALL_SUBTEST(test_type_casting()); CALL_SUBTEST(test_select()); + CALL_SUBTEST(test_clip()); + CALL_SUBTEST(test_minmax_nan_propagation()); }
diff --git a/unsupported/test/cxx11_tensor_fft.cpp b/unsupported/test/cxx11_tensor_fft.cpp index 8987434..641486a 100644 --- a/unsupported/test/cxx11_tensor_fft.cpp +++ b/unsupported/test/cxx11_tensor_fft.cpp
@@ -205,15 +205,15 @@ VERIFY_IS_EQUAL(output.dimension(i), input.dimension(i)); } - float energy_original = 0.0; - float energy_after_fft = 0.0; + RealScalar energy_original = 0.0; + RealScalar energy_after_fft = 0.0; for (int i = 0; i < total_size; ++i) { - energy_original += pow(std::abs(input(i)), 2); + energy_original += numext::abs2(input(i)); } for (int i = 0; i < total_size; ++i) { - energy_after_fft += pow(std::abs(output(i)), 2); + energy_after_fft += numext::abs2(output(i)); } if(FFTDirection == FFT_FORWARD) { @@ -224,7 +224,35 @@ } } -void test_cxx11_tensor_fft() { +template <typename RealScalar> +static void test_fft_non_power_of_2_round_trip(int exponent) { + int n = (1 << exponent) + 1; + + Eigen::DSizes<std::int64_t, 1> dimensions; + dimensions[0] = n; + const DSizes<std::int64_t, 1> arr = dimensions; + Tensor<RealScalar, 1, ColMajor, std::int64_t> input; + + input.resize(arr); + input.setRandom(); + + array<int, 1> fft; + fft[0] = 0; + + Tensor<std::complex<RealScalar>, 1, ColMajor> forward = + input.template fft<BothParts, FFT_FORWARD>(fft); + + Tensor<RealScalar, 1, ColMajor, std::int64_t> output = + forward.template fft<RealPart, FFT_REVERSE>(fft); + + for (int i = 0; i < n; ++i) { + RealScalar tol = test_precision<RealScalar>() * + (std::abs(input[i]) + std::abs(output[i]) + 1); + VERIFY_IS_APPROX_OR_LESS_THAN(std::abs(input[i] - output[i]), tol); + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_fft) { test_fft_complex_input_golden(); test_fft_real_input_golden(); @@ -270,4 +298,7 @@ test_fft_real_input_energy<RowMajor, double, true, Eigen::BothParts, FFT_FORWARD, 4>(); test_fft_real_input_energy<RowMajor, float, false, Eigen::BothParts, FFT_FORWARD, 4>(); test_fft_real_input_energy<RowMajor, double, false, Eigen::BothParts, FFT_FORWARD, 4>(); + + test_fft_non_power_of_2_round_trip<float>(7); + test_fft_non_power_of_2_round_trip<double>(7); }
diff --git a/unsupported/test/cxx11_tensor_fixed_size.cpp b/unsupported/test/cxx11_tensor_fixed_size.cpp index 5fe1648..456ce6b 100644 --- a/unsupported/test/cxx11_tensor_fixed_size.cpp +++ b/unsupported/test/cxx11_tensor_fixed_size.cpp
@@ -21,7 +21,7 @@ TensorFixedSize<float, Sizes<>, RowMajor> scalar2; VERIFY_IS_EQUAL(scalar1.rank(), 0); VERIFY_IS_EQUAL(scalar1.size(), 1); - VERIFY_IS_EQUAL(array_prod(scalar1.dimensions()), 1); + VERIFY_IS_EQUAL(internal::array_prod(scalar1.dimensions()), 1); scalar1() = 7.0; scalar2() = 13.0; @@ -130,9 +130,9 @@ static void test_2d() { float data1[6]; - TensorMap<TensorFixedSize<float, Sizes<2, 3> >> mat1(data1,2,3); + TensorMap<TensorFixedSize<float, Sizes<2, 3> > > mat1(data1,2,3); float data2[6]; - TensorMap<TensorFixedSize<float, Sizes<2, 3>, RowMajor>> mat2(data2,2,3); + TensorMap<TensorFixedSize<float, Sizes<2, 3>, RowMajor> > mat2(data2,2,3); VERIFY_IS_EQUAL((mat1.size()), 2*3); VERIFY_IS_EQUAL(mat1.rank(), 2); @@ -153,7 +153,7 @@ mat2(1,1) = -4.0; mat2(1,2) = -5.0; - TensorFixedSize<float, Sizes<2, 3>> mat3; + TensorFixedSize<float, Sizes<2, 3> > mat3; TensorFixedSize<float, Sizes<2, 3>, RowMajor> mat4; mat3 = mat1.abs(); mat4 = mat2.abs(); @@ -188,13 +188,13 @@ // VERIFY_IS_EQUAL((mat1.dimension(1)), 3); // VERIFY_IS_EQUAL((mat1.dimension(2)), 7); - float val = 0.0; + float val = 0.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { mat1(i,j,k) = val; mat2(i,j,k) = val; - val += 1.0; + val += 1.0f; } } } @@ -210,13 +210,13 @@ // VERIFY_IS_EQUAL((mat3.dimension(2)), 7); - val = 0.0; + val = 0.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { VERIFY_IS_APPROX(mat3(i,j,k), sqrtf(val)); VERIFY_IS_APPROX(mat4(i,j,k), sqrtf(val)); - val += 1.0; + val += 1.0f; } } } @@ -226,12 +226,12 @@ static void test_array() { TensorFixedSize<float, Sizes<2, 3, 7> > mat1; - float val = 0.0; + float val = 0.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { mat1(i,j,k) = val; - val += 1.0; + val += 1.0f; } } } @@ -239,18 +239,18 @@ TensorFixedSize<float, Sizes<2, 3, 7> > mat3; mat3 = mat1.pow(3.5f); - val = 0.0; + val = 0.0f; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { VERIFY_IS_APPROX(mat3(i,j,k), powf(val, 3.5f)); - val += 1.0; + val += 1.0f; } } } } -void test_cxx11_tensor_fixed_size() +EIGEN_DECLARE_TEST(cxx11_tensor_fixed_size) { CALL_SUBTEST(test_0d()); CALL_SUBTEST(test_1d());
diff --git a/unsupported/test/cxx11_tensor_forced_eval.cpp b/unsupported/test/cxx11_tensor_forced_eval.cpp index ad9de86..f76e2ea 100644 --- a/unsupported/test/cxx11_tensor_forced_eval.cpp +++ b/unsupported/test/cxx11_tensor_forced_eval.cpp
@@ -22,14 +22,15 @@ m1.setRandom(); m2.setRandom(); - TensorMap<Tensor<float, 2>> mat1(m1.data(), 3,3); - TensorMap<Tensor<float, 2>> mat2(m2.data(), 3,3); + TensorMap<Tensor<float, 2> > mat1(m1.data(), 3,3); + TensorMap<Tensor<float, 2> > mat2(m2.data(), 3,3); Tensor<float, 2> mat3(3,3); mat3 = mat1; typedef Tensor<float, 1>::DimensionPair DimPair; - Eigen::array<DimPair, 1> dims({{DimPair(1, 0)}}); + Eigen::array<DimPair, 1> dims; + dims[0] = DimPair(1, 0); mat3 = mat3.contract(mat2, dims).eval(); @@ -60,7 +61,7 @@ Eigen::array<int, 2> bcast; bcast[0] = 3; bcast[1] = 1; - const TensorMap<Tensor<const float, 2>> input_tensor(input.data(), 3, 3); + const TensorMap<Tensor<const float, 2> > input_tensor(input.data(), 3, 3); Tensor<float, 2> output_tensor= (input_tensor - input_tensor.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)); for (int i = 0; i < 3; ++i) { @@ -71,7 +72,7 @@ } -void test_cxx11_tensor_forced_eval() +EIGEN_DECLARE_TEST(cxx11_tensor_forced_eval) { CALL_SUBTEST(test_simple()); CALL_SUBTEST(test_const());
diff --git a/unsupported/test/cxx11_tensor_forced_eval_sycl.cpp b/unsupported/test/cxx11_tensor_forced_eval_sycl.cpp new file mode 100644 index 0000000..74d38a6 --- /dev/null +++ b/unsupported/test/cxx11_tensor_forced_eval_sycl.cpp
@@ -0,0 +1,76 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; +template <typename DataType, int DataLayout, typename IndexType> +void test_forced_eval_sycl(const Eigen::SyclDevice &sycl_device) { + + IndexType sizeDim1 = 100; + IndexType sizeDim2 = 20; + IndexType sizeDim3 = 20; + Eigen::array<IndexType, 3> tensorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + Eigen::Tensor<DataType, 3, DataLayout, IndexType> in1(tensorRange); + Eigen::Tensor<DataType, 3, DataLayout, IndexType> in2(tensorRange); + Eigen::Tensor<DataType, 3, DataLayout, IndexType> out(tensorRange); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(in1.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_in2_data = static_cast<DataType*>(sycl_device.allocate(in2.dimensions().TotalSize()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.dimensions().TotalSize()*sizeof(DataType))); + + in1 = in1.random() + in1.constant(10.0f); + in2 = in2.random() + in2.constant(10.0f); + + // creating TensorMap from tensor + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_in1(gpu_in1_data, tensorRange); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_in2(gpu_in2_data, tensorRange); + Eigen::TensorMap<Eigen::Tensor<DataType, 3, DataLayout, IndexType>> gpu_out(gpu_out_data, tensorRange); + sycl_device.memcpyHostToDevice(gpu_in1_data, in1.data(),(in1.dimensions().TotalSize())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_in2_data, in2.data(),(in2.dimensions().TotalSize())*sizeof(DataType)); + /// c=(a+b)*b + gpu_out.device(sycl_device) =(gpu_in1 + gpu_in2).eval() * gpu_in2; + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.dimensions().TotalSize())*sizeof(DataType)); + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i, j, k), + (in1(i, j, k) + in2(i, j, k)) * in2(i, j, k)); + } + } + } + printf("(a+b)*b Test Passed\n"); + sycl_device.deallocate(gpu_in1_data); + sycl_device.deallocate(gpu_in2_data); + sycl_device.deallocate(gpu_out_data); + +} + +template <typename DataType, typename Dev_selector> void tensorForced_evalperDevice(Dev_selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_forced_eval_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_forced_eval_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_forced_eval_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(tensorForced_evalperDevice<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_generator.cpp b/unsupported/test/cxx11_tensor_generator.cpp index dcb9287..ee5e29b 100644 --- a/unsupported/test/cxx11_tensor_generator.cpp +++ b/unsupported/test/cxx11_tensor_generator.cpp
@@ -80,7 +80,7 @@ } -void test_cxx11_tensor_generator() +EIGEN_DECLARE_TEST(cxx11_tensor_generator) { CALL_SUBTEST(test_1D<ColMajor>()); CALL_SUBTEST(test_1D<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_generator_sycl.cpp b/unsupported/test/cxx11_tensor_generator_sycl.cpp new file mode 100644 index 0000000..fb6e3d9 --- /dev/null +++ b/unsupported/test/cxx11_tensor_generator_sycl.cpp
@@ -0,0 +1,147 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL +static const float error_threshold =1e-8f; + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; +struct Generator1D { + Generator1D() { } + + float operator()(const array<Eigen::DenseIndex, 1>& coordinates) const { + return coordinates[0]; + } +}; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_1D_sycl(const Eigen::SyclDevice& sycl_device) +{ + + IndexType sizeDim1 = 6; + array<IndexType, 1> tensorRange = {{sizeDim1}}; + Tensor<DataType, 1, DataLayout,IndexType> vec(tensorRange); + Tensor<DataType, 1, DataLayout,IndexType> result(tensorRange); + + const size_t tensorBuffSize =vec.size()*sizeof(DataType); + DataType* gpu_data_vec = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_result = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + + TensorMap<Tensor<DataType, 1, DataLayout,IndexType>> gpu_vec(gpu_data_vec, tensorRange); + TensorMap<Tensor<DataType, 1, DataLayout,IndexType>> gpu_result(gpu_data_result, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_vec, vec.data(), tensorBuffSize); + gpu_result.device(sycl_device)=gpu_vec.generate(Generator1D()); + sycl_device.memcpyDeviceToHost(result.data(), gpu_data_result, tensorBuffSize); + + for (IndexType i = 0; i < 6; ++i) { + VERIFY_IS_EQUAL(result(i), i); + } +} + + +struct Generator2D { + Generator2D() { } + + float operator()(const array<Eigen::DenseIndex, 2>& coordinates) const { + return 3 * coordinates[0] + 11 * coordinates[1]; + } +}; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_2D_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 5; + IndexType sizeDim2 = 7; + array<IndexType, 2> tensorRange = {{sizeDim1, sizeDim2}}; + Tensor<DataType, 2, DataLayout,IndexType> matrix(tensorRange); + Tensor<DataType, 2, DataLayout,IndexType> result(tensorRange); + + const size_t tensorBuffSize =matrix.size()*sizeof(DataType); + DataType* gpu_data_matrix = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_result = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + + TensorMap<Tensor<DataType, 2, DataLayout,IndexType>> gpu_matrix(gpu_data_matrix, tensorRange); + TensorMap<Tensor<DataType, 2, DataLayout,IndexType>> gpu_result(gpu_data_result, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_matrix, matrix.data(), tensorBuffSize); + gpu_result.device(sycl_device)=gpu_matrix.generate(Generator2D()); + sycl_device.memcpyDeviceToHost(result.data(), gpu_data_result, tensorBuffSize); + + for (IndexType i = 0; i < 5; ++i) { + for (IndexType j = 0; j < 5; ++j) { + VERIFY_IS_EQUAL(result(i, j), 3*i + 11*j); + } + } +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_gaussian_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType rows = 32; + IndexType cols = 48; + array<DataType, 2> means; + means[0] = rows / 2.0f; + means[1] = cols / 2.0f; + array<DataType, 2> std_devs; + std_devs[0] = 3.14f; + std_devs[1] = 2.7f; + internal::GaussianGenerator<DataType, Eigen::DenseIndex, 2> gaussian_gen(means, std_devs); + + array<IndexType, 2> tensorRange = {{rows, cols}}; + Tensor<DataType, 2, DataLayout,IndexType> matrix(tensorRange); + Tensor<DataType, 2, DataLayout,IndexType> result(tensorRange); + + const size_t tensorBuffSize =matrix.size()*sizeof(DataType); + DataType* gpu_data_matrix = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_result = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + + TensorMap<Tensor<DataType, 2, DataLayout,IndexType>> gpu_matrix(gpu_data_matrix, tensorRange); + TensorMap<Tensor<DataType, 2, DataLayout,IndexType>> gpu_result(gpu_data_result, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_matrix, matrix.data(), tensorBuffSize); + gpu_result.device(sycl_device)=gpu_matrix.generate(gaussian_gen); + sycl_device.memcpyDeviceToHost(result.data(), gpu_data_result, tensorBuffSize); + + for (IndexType i = 0; i < rows; ++i) { + for (IndexType j = 0; j < cols; ++j) { + DataType g_rows = powf(rows/2.0f - i, 2) / (3.14f * 3.14f) * 0.5f; + DataType g_cols = powf(cols/2.0f - j, 2) / (2.7f * 2.7f) * 0.5f; + DataType gaussian = expf(-g_rows - g_cols); + Eigen::internal::isApprox(result(i, j), gaussian, error_threshold); + } + } +} + +template<typename DataType, typename dev_Selector> void sycl_generator_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_1D_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_1D_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_2D_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_2D_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_gaussian_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_gaussian_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_generator_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_generator_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_gpu.cu b/unsupported/test/cxx11_tensor_gpu.cu new file mode 100644 index 0000000..94625e6 --- /dev/null +++ b/unsupported/test/cxx11_tensor_gpu.cu
@@ -0,0 +1,1579 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + +#define EIGEN_GPU_TEST_C99_MATH EIGEN_HAS_CXX11 + +using Eigen::Tensor; + +void test_gpu_nullary() { + Tensor<float, 1, 0, int> in1(2); + Tensor<float, 1, 0, int> in2(2); + in1.setRandom(); + in2.setRandom(); + + std::size_t tensor_bytes = in1.size() * sizeof(float); + + float* d_in1; + float* d_in2; + gpuMalloc((void**)(&d_in1), tensor_bytes); + gpuMalloc((void**)(&d_in2), tensor_bytes); + gpuMemcpy(d_in1, in1.data(), tensor_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in2, in2.data(), tensor_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 1, 0, int>, Eigen::Aligned> gpu_in1( + d_in1, 2); + Eigen::TensorMap<Eigen::Tensor<float, 1, 0, int>, Eigen::Aligned> gpu_in2( + d_in2, 2); + + gpu_in1.device(gpu_device) = gpu_in1.constant(3.14f); + gpu_in2.device(gpu_device) = gpu_in2.random(); + + Tensor<float, 1, 0, int> new1(2); + Tensor<float, 1, 0, int> new2(2); + + assert(gpuMemcpyAsync(new1.data(), d_in1, tensor_bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuMemcpyAsync(new2.data(), d_in2, tensor_bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 2; ++i) { + VERIFY_IS_APPROX(new1(i), 3.14f); + VERIFY_IS_NOT_EQUAL(new2(i), in2(i)); + } + + gpuFree(d_in1); + gpuFree(d_in2); +} + +void test_gpu_elementwise_small() { + Tensor<float, 1> in1(Eigen::array<Eigen::DenseIndex, 1>(2)); + Tensor<float, 1> in2(Eigen::array<Eigen::DenseIndex, 1>(2)); + Tensor<float, 1> out(Eigen::array<Eigen::DenseIndex, 1>(2)); + in1.setRandom(); + in2.setRandom(); + + std::size_t in1_bytes = in1.size() * sizeof(float); + std::size_t in2_bytes = in2.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_in1; + float* d_in2; + float* d_out; + gpuMalloc((void**)(&d_in1), in1_bytes); + gpuMalloc((void**)(&d_in2), in2_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_in1, in1.data(), in1_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in2, in2.data(), in2_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1( + d_in1, Eigen::array<Eigen::DenseIndex, 1>(2)); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in2( + d_in2, Eigen::array<Eigen::DenseIndex, 1>(2)); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_out( + d_out, Eigen::array<Eigen::DenseIndex, 1>(2)); + + gpu_out.device(gpu_device) = gpu_in1 + gpu_in2; + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 2; ++i) { + VERIFY_IS_APPROX( + out(Eigen::array<Eigen::DenseIndex, 1>(i)), + in1(Eigen::array<Eigen::DenseIndex, 1>(i)) + in2(Eigen::array<Eigen::DenseIndex, 1>(i))); + } + + gpuFree(d_in1); + gpuFree(d_in2); + gpuFree(d_out); +} + +void test_gpu_elementwise() +{ + Tensor<float, 3> in1(Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Tensor<float, 3> in2(Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Tensor<float, 3> in3(Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Tensor<float, 3> out(Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + in1.setRandom(); + in2.setRandom(); + in3.setRandom(); + + std::size_t in1_bytes = in1.size() * sizeof(float); + std::size_t in2_bytes = in2.size() * sizeof(float); + std::size_t in3_bytes = in3.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_in1; + float* d_in2; + float* d_in3; + float* d_out; + gpuMalloc((void**)(&d_in1), in1_bytes); + gpuMalloc((void**)(&d_in2), in2_bytes); + gpuMalloc((void**)(&d_in3), in3_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_in1, in1.data(), in1_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in2, in2.data(), in2_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in3, in3.data(), in3_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in1(d_in1, Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in2(d_in2, Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in3(d_in3, Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_out(d_out, Eigen::array<Eigen::DenseIndex, 3>(72,53,97)); + + gpu_out.device(gpu_device) = gpu_in1 + gpu_in2 * gpu_in3; + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 72; ++i) { + for (int j = 0; j < 53; ++j) { + for (int k = 0; k < 97; ++k) { + VERIFY_IS_APPROX(out(Eigen::array<Eigen::DenseIndex, 3>(i,j,k)), in1(Eigen::array<Eigen::DenseIndex, 3>(i,j,k)) + in2(Eigen::array<Eigen::DenseIndex, 3>(i,j,k)) * in3(Eigen::array<Eigen::DenseIndex, 3>(i,j,k))); + } + } + } + + gpuFree(d_in1); + gpuFree(d_in2); + gpuFree(d_in3); + gpuFree(d_out); +} + +void test_gpu_props() { + Tensor<float, 1> in1(200); + Tensor<bool, 1> out(200); + in1.setRandom(); + + std::size_t in1_bytes = in1.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(bool); + + float* d_in1; + bool* d_out; + gpuMalloc((void**)(&d_in1), in1_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_in1, in1.data(), in1_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1( + d_in1, 200); + Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_out( + d_out, 200); + + gpu_out.device(gpu_device) = (gpu_in1.isnan)(); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 200; ++i) { + VERIFY_IS_EQUAL(out(i), (std::isnan)(in1(i))); + } + + gpuFree(d_in1); + gpuFree(d_out); +} + +void test_gpu_reduction() +{ + Tensor<float, 4> in1(72,53,97,113); + Tensor<float, 2> out(72,97); + in1.setRandom(); + + std::size_t in1_bytes = in1.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_in1; + float* d_out; + gpuMalloc((void**)(&d_in1), in1_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_in1, in1.data(), in1_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4> > gpu_in1(d_in1, 72,53,97,113); + Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97); + + array<Eigen::DenseIndex, 2> reduction_axis; + reduction_axis[0] = 1; + reduction_axis[1] = 3; + + gpu_out.device(gpu_device) = gpu_in1.maximum(reduction_axis); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 72; ++i) { + for (int j = 0; j < 97; ++j) { + float expected = 0; + for (int k = 0; k < 53; ++k) { + for (int l = 0; l < 113; ++l) { + expected = + std::max<float>(expected, in1(i, k, j, l)); + } + } + VERIFY_IS_APPROX(out(i,j), expected); + } + } + + gpuFree(d_in1); + gpuFree(d_out); +} + +template<int DataLayout> +void test_gpu_contraction() +{ + // with these dimensions, the output has 300 * 140 elements, which is + // more than 30 * 1024, which is the number of threads in blocks on + // a 15 SM GK110 GPU + Tensor<float, 4, DataLayout> t_left(6, 50, 3, 31); + Tensor<float, 5, DataLayout> t_right(Eigen::array<Eigen::DenseIndex, 5>(3, 31, 7, 20, 1)); + Tensor<float, 5, DataLayout> t_result(Eigen::array<Eigen::DenseIndex, 5>(6, 50, 7, 20, 1)); + + t_left.setRandom(); + t_right.setRandom(); + + std::size_t t_left_bytes = t_left.size() * sizeof(float); + std::size_t t_right_bytes = t_right.size() * sizeof(float); + std::size_t t_result_bytes = t_result.size() * sizeof(float); + + float* d_t_left; + float* d_t_right; + float* d_t_result; + + gpuMalloc((void**)(&d_t_left), t_left_bytes); + gpuMalloc((void**)(&d_t_right), t_right_bytes); + gpuMalloc((void**)(&d_t_result), t_result_bytes); + + gpuMemcpy(d_t_left, t_left.data(), t_left_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_t_right, t_right.data(), t_right_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_t_left(d_t_left, 6, 50, 3, 31); + Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_right(d_t_right, 3, 31, 7, 20, 1); + Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_result(d_t_result, 6, 50, 7, 20, 1); + + typedef Eigen::Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> > MapXf; + MapXf m_left(t_left.data(), 300, 93); + MapXf m_right(t_right.data(), 93, 140); + Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(300, 140); + + typedef Tensor<float, 1>::DimensionPair DimPair; + Eigen::array<DimPair, 2> dims; + dims[0] = DimPair(2, 0); + dims[1] = DimPair(3, 1); + + m_result = m_left * m_right; + gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims); + + gpuMemcpy(t_result.data(), d_t_result, t_result_bytes, gpuMemcpyDeviceToHost); + + for (DenseIndex i = 0; i < t_result.size(); i++) { + if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) { + std::cout << "mismatch detected at index " << i << ": " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; + assert(false); + } + } + + gpuFree(d_t_left); + gpuFree(d_t_right); + gpuFree(d_t_result); +} + +template<int DataLayout> +void test_gpu_convolution_1d() +{ + Tensor<float, 4, DataLayout> input(74,37,11,137); + Tensor<float, 1, DataLayout> kernel(4); + Tensor<float, 4, DataLayout> out(74,34,11,137); + input = input.constant(10.0f) + input.random(); + kernel = kernel.constant(7.0f) + kernel.random(); + + std::size_t input_bytes = input.size() * sizeof(float); + std::size_t kernel_bytes = kernel.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_input; + float* d_kernel; + float* d_out; + gpuMalloc((void**)(&d_input), input_bytes); + gpuMalloc((void**)(&d_kernel), kernel_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_input, input.data(), input_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_kernel, kernel.data(), kernel_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input, 74,37,11,137); + Eigen::TensorMap<Eigen::Tensor<float, 1, DataLayout> > gpu_kernel(d_kernel, 4); + Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out, 74,34,11,137); + + Eigen::array<Eigen::DenseIndex, 1> dims(1); + gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 74; ++i) { + for (int j = 0; j < 34; ++j) { + for (int k = 0; k < 11; ++k) { + for (int l = 0; l < 137; ++l) { + const float result = out(i,j,k,l); + const float expected = input(i,j+0,k,l) * kernel(0) + input(i,j+1,k,l) * kernel(1) + + input(i,j+2,k,l) * kernel(2) + input(i,j+3,k,l) * kernel(3); + VERIFY_IS_APPROX(result, expected); + } + } + } + } + + gpuFree(d_input); + gpuFree(d_kernel); + gpuFree(d_out); +} + +void test_gpu_convolution_inner_dim_col_major_1d() +{ + Tensor<float, 4, ColMajor> input(74,9,11,7); + Tensor<float, 1, ColMajor> kernel(4); + Tensor<float, 4, ColMajor> out(71,9,11,7); + input = input.constant(10.0f) + input.random(); + kernel = kernel.constant(7.0f) + kernel.random(); + + std::size_t input_bytes = input.size() * sizeof(float); + std::size_t kernel_bytes = kernel.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_input; + float* d_kernel; + float* d_out; + gpuMalloc((void**)(&d_input), input_bytes); + gpuMalloc((void**)(&d_kernel), kernel_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_input, input.data(), input_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_kernel, kernel.data(), kernel_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_input(d_input,74,9,11,7); + Eigen::TensorMap<Eigen::Tensor<float, 1, ColMajor> > gpu_kernel(d_kernel,4); + Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_out(d_out,71,9,11,7); + + Eigen::array<Eigen::DenseIndex, 1> dims(0); + gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 71; ++i) { + for (int j = 0; j < 9; ++j) { + for (int k = 0; k < 11; ++k) { + for (int l = 0; l < 7; ++l) { + const float result = out(i,j,k,l); + const float expected = input(i+0,j,k,l) * kernel(0) + input(i+1,j,k,l) * kernel(1) + + input(i+2,j,k,l) * kernel(2) + input(i+3,j,k,l) * kernel(3); + VERIFY_IS_APPROX(result, expected); + } + } + } + } + + gpuFree(d_input); + gpuFree(d_kernel); + gpuFree(d_out); +} + +void test_gpu_convolution_inner_dim_row_major_1d() +{ + Tensor<float, 4, RowMajor> input(7,9,11,74); + Tensor<float, 1, RowMajor> kernel(4); + Tensor<float, 4, RowMajor> out(7,9,11,71); + input = input.constant(10.0f) + input.random(); + kernel = kernel.constant(7.0f) + kernel.random(); + + std::size_t input_bytes = input.size() * sizeof(float); + std::size_t kernel_bytes = kernel.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_input; + float* d_kernel; + float* d_out; + gpuMalloc((void**)(&d_input), input_bytes); + gpuMalloc((void**)(&d_kernel), kernel_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_input, input.data(), input_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_kernel, kernel.data(), kernel_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_input(d_input, 7,9,11,74); + Eigen::TensorMap<Eigen::Tensor<float, 1, RowMajor> > gpu_kernel(d_kernel, 4); + Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_out(d_out, 7,9,11,71); + + Eigen::array<Eigen::DenseIndex, 1> dims(3); + gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 7; ++i) { + for (int j = 0; j < 9; ++j) { + for (int k = 0; k < 11; ++k) { + for (int l = 0; l < 71; ++l) { + const float result = out(i,j,k,l); + const float expected = input(i,j,k,l+0) * kernel(0) + input(i,j,k,l+1) * kernel(1) + + input(i,j,k,l+2) * kernel(2) + input(i,j,k,l+3) * kernel(3); + VERIFY_IS_APPROX(result, expected); + } + } + } + } + + gpuFree(d_input); + gpuFree(d_kernel); + gpuFree(d_out); +} + +template<int DataLayout> +void test_gpu_convolution_2d() +{ + Tensor<float, 4, DataLayout> input(74,37,11,137); + Tensor<float, 2, DataLayout> kernel(3,4); + Tensor<float, 4, DataLayout> out(74,35,8,137); + input = input.constant(10.0f) + input.random(); + kernel = kernel.constant(7.0f) + kernel.random(); + + std::size_t input_bytes = input.size() * sizeof(float); + std::size_t kernel_bytes = kernel.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_input; + float* d_kernel; + float* d_out; + gpuMalloc((void**)(&d_input), input_bytes); + gpuMalloc((void**)(&d_kernel), kernel_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_input, input.data(), input_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_kernel, kernel.data(), kernel_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input,74,37,11,137); + Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > gpu_kernel(d_kernel,3,4); + Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out,74,35,8,137); + + Eigen::array<Eigen::DenseIndex, 2> dims(1,2); + gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 74; ++i) { + for (int j = 0; j < 35; ++j) { + for (int k = 0; k < 8; ++k) { + for (int l = 0; l < 137; ++l) { + const float result = out(i,j,k,l); + const float expected = input(i,j+0,k+0,l) * kernel(0,0) + + input(i,j+1,k+0,l) * kernel(1,0) + + input(i,j+2,k+0,l) * kernel(2,0) + + input(i,j+0,k+1,l) * kernel(0,1) + + input(i,j+1,k+1,l) * kernel(1,1) + + input(i,j+2,k+1,l) * kernel(2,1) + + input(i,j+0,k+2,l) * kernel(0,2) + + input(i,j+1,k+2,l) * kernel(1,2) + + input(i,j+2,k+2,l) * kernel(2,2) + + input(i,j+0,k+3,l) * kernel(0,3) + + input(i,j+1,k+3,l) * kernel(1,3) + + input(i,j+2,k+3,l) * kernel(2,3); + VERIFY_IS_APPROX(result, expected); + } + } + } + } + + gpuFree(d_input); + gpuFree(d_kernel); + gpuFree(d_out); +} + +template<int DataLayout> +void test_gpu_convolution_3d() +{ + Tensor<float, 5, DataLayout> input(Eigen::array<Eigen::DenseIndex, 5>(74,37,11,137,17)); + Tensor<float, 3, DataLayout> kernel(3,4,2); + Tensor<float, 5, DataLayout> out(Eigen::array<Eigen::DenseIndex, 5>(74,35,8,136,17)); + input = input.constant(10.0f) + input.random(); + kernel = kernel.constant(7.0f) + kernel.random(); + + std::size_t input_bytes = input.size() * sizeof(float); + std::size_t kernel_bytes = kernel.size() * sizeof(float); + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_input; + float* d_kernel; + float* d_out; + gpuMalloc((void**)(&d_input), input_bytes); + gpuMalloc((void**)(&d_kernel), kernel_bytes); + gpuMalloc((void**)(&d_out), out_bytes); + + gpuMemcpy(d_input, input.data(), input_bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_kernel, kernel.data(), kernel_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_input(d_input,74,37,11,137,17); + Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> > gpu_kernel(d_kernel,3,4,2); + Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_out(d_out,74,35,8,136,17); + + Eigen::array<Eigen::DenseIndex, 3> dims(1,2,3); + gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 74; ++i) { + for (int j = 0; j < 35; ++j) { + for (int k = 0; k < 8; ++k) { + for (int l = 0; l < 136; ++l) { + for (int m = 0; m < 17; ++m) { + const float result = out(i,j,k,l,m); + const float expected = input(i,j+0,k+0,l+0,m) * kernel(0,0,0) + + input(i,j+1,k+0,l+0,m) * kernel(1,0,0) + + input(i,j+2,k+0,l+0,m) * kernel(2,0,0) + + input(i,j+0,k+1,l+0,m) * kernel(0,1,0) + + input(i,j+1,k+1,l+0,m) * kernel(1,1,0) + + input(i,j+2,k+1,l+0,m) * kernel(2,1,0) + + input(i,j+0,k+2,l+0,m) * kernel(0,2,0) + + input(i,j+1,k+2,l+0,m) * kernel(1,2,0) + + input(i,j+2,k+2,l+0,m) * kernel(2,2,0) + + input(i,j+0,k+3,l+0,m) * kernel(0,3,0) + + input(i,j+1,k+3,l+0,m) * kernel(1,3,0) + + input(i,j+2,k+3,l+0,m) * kernel(2,3,0) + + input(i,j+0,k+0,l+1,m) * kernel(0,0,1) + + input(i,j+1,k+0,l+1,m) * kernel(1,0,1) + + input(i,j+2,k+0,l+1,m) * kernel(2,0,1) + + input(i,j+0,k+1,l+1,m) * kernel(0,1,1) + + input(i,j+1,k+1,l+1,m) * kernel(1,1,1) + + input(i,j+2,k+1,l+1,m) * kernel(2,1,1) + + input(i,j+0,k+2,l+1,m) * kernel(0,2,1) + + input(i,j+1,k+2,l+1,m) * kernel(1,2,1) + + input(i,j+2,k+2,l+1,m) * kernel(2,2,1) + + input(i,j+0,k+3,l+1,m) * kernel(0,3,1) + + input(i,j+1,k+3,l+1,m) * kernel(1,3,1) + + input(i,j+2,k+3,l+1,m) * kernel(2,3,1); + VERIFY_IS_APPROX(result, expected); + } + } + } + } + } + + gpuFree(d_input); + gpuFree(d_kernel); + gpuFree(d_out); +} + + +#if EIGEN_GPU_TEST_C99_MATH +template <typename Scalar> +void test_gpu_lgamma(const Scalar stddev) +{ + Tensor<Scalar, 2> in(72,97); + in.setRandom(); + in *= in.constant(stddev); + Tensor<Scalar, 2> out(72,97); + out.setZero(); + + std::size_t bytes = in.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in, in.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); + + gpu_out.device(gpu_device) = gpu_in.lgamma(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 72; ++i) { + for (int j = 0; j < 97; ++j) { + VERIFY_IS_APPROX(out(i,j), (std::lgamma)(in(i,j))); + } + } + + gpuFree(d_in); + gpuFree(d_out); +} +#endif + +template <typename Scalar> +void test_gpu_digamma() +{ + Tensor<Scalar, 1> in(7); + Tensor<Scalar, 1> out(7); + Tensor<Scalar, 1> expected_out(7); + out.setZero(); + + in(0) = Scalar(1); + in(1) = Scalar(1.5); + in(2) = Scalar(4); + in(3) = Scalar(-10.5); + in(4) = Scalar(10000.5); + in(5) = Scalar(0); + in(6) = Scalar(-1); + + expected_out(0) = Scalar(-0.5772156649015329); + expected_out(1) = Scalar(0.03648997397857645); + expected_out(2) = Scalar(1.2561176684318); + expected_out(3) = Scalar(2.398239129535781); + expected_out(4) = Scalar(9.210340372392849); + expected_out(5) = std::numeric_limits<Scalar>::infinity(); + expected_out(6) = std::numeric_limits<Scalar>::infinity(); + + std::size_t bytes = in.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in, in.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in(d_in, 7); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7); + + gpu_out.device(gpu_device) = gpu_in.digamma(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 5; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + for (int i = 5; i < 7; ++i) { + VERIFY_IS_EQUAL(out(i), expected_out(i)); + } + + gpuFree(d_in); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_zeta() +{ + Tensor<Scalar, 1> in_x(6); + Tensor<Scalar, 1> in_q(6); + Tensor<Scalar, 1> out(6); + Tensor<Scalar, 1> expected_out(6); + out.setZero(); + + in_x(0) = Scalar(1); + in_x(1) = Scalar(1.5); + in_x(2) = Scalar(4); + in_x(3) = Scalar(-10.5); + in_x(4) = Scalar(10000.5); + in_x(5) = Scalar(3); + + in_q(0) = Scalar(1.2345); + in_q(1) = Scalar(2); + in_q(2) = Scalar(1.5); + in_q(3) = Scalar(3); + in_q(4) = Scalar(1.0001); + in_q(5) = Scalar(-2.5); + + expected_out(0) = std::numeric_limits<Scalar>::infinity(); + expected_out(1) = Scalar(1.61237534869); + expected_out(2) = Scalar(0.234848505667); + expected_out(3) = Scalar(1.03086757337e-5); + expected_out(4) = Scalar(0.367879440865); + expected_out(5) = Scalar(0.054102025820864097); + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_in_x; + Scalar* d_in_q; + Scalar* d_out; + gpuMalloc((void**)(&d_in_x), bytes); + gpuMalloc((void**)(&d_in_q), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in_x, in_x.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in_q, in_q.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_q(d_in_q, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 6); + + gpu_out.device(gpu_device) = gpu_in_x.zeta(gpu_in_q); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + VERIFY_IS_EQUAL(out(0), expected_out(0)); + VERIFY((std::isnan)(out(3))); + + for (int i = 1; i < 6; ++i) { + if (i != 3) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + } + + gpuFree(d_in_x); + gpuFree(d_in_q); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_polygamma() +{ + Tensor<Scalar, 1> in_x(7); + Tensor<Scalar, 1> in_n(7); + Tensor<Scalar, 1> out(7); + Tensor<Scalar, 1> expected_out(7); + out.setZero(); + + in_n(0) = Scalar(1); + in_n(1) = Scalar(1); + in_n(2) = Scalar(1); + in_n(3) = Scalar(17); + in_n(4) = Scalar(31); + in_n(5) = Scalar(28); + in_n(6) = Scalar(8); + + in_x(0) = Scalar(2); + in_x(1) = Scalar(3); + in_x(2) = Scalar(25.5); + in_x(3) = Scalar(4.7); + in_x(4) = Scalar(11.8); + in_x(5) = Scalar(17.7); + in_x(6) = Scalar(30.2); + + expected_out(0) = Scalar(0.644934066848); + expected_out(1) = Scalar(0.394934066848); + expected_out(2) = Scalar(0.0399946696496); + expected_out(3) = Scalar(293.334565435); + expected_out(4) = Scalar(0.445487887616); + expected_out(5) = Scalar(-2.47810300902e-07); + expected_out(6) = Scalar(-8.29668781082e-09); + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_in_x; + Scalar* d_in_n; + Scalar* d_out; + gpuMalloc((void**)(&d_in_x), bytes); + gpuMalloc((void**)(&d_in_n), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in_x, in_x.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in_n, in_n.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 7); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_n(d_in_n, 7); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7); + + gpu_out.device(gpu_device) = gpu_in_n.polygamma(gpu_in_x); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 7; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + + gpuFree(d_in_x); + gpuFree(d_in_n); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_igamma() +{ + Tensor<Scalar, 2> a(6, 6); + Tensor<Scalar, 2> x(6, 6); + Tensor<Scalar, 2> out(6, 6); + out.setZero(); + + Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 6; ++j) { + a(i, j) = a_s[i]; + x(i, j) = x_s[j]; + } + } + + Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); + Scalar igamma_s[][6] = {{0.0, nan, nan, nan, nan, nan}, + {0.0, 0.6321205588285578, 0.7768698398515702, + 0.9816843611112658, 9.999500016666262e-05, 1.0}, + {0.0, 0.4275932955291202, 0.608374823728911, + 0.9539882943107686, 7.522076445089201e-07, 1.0}, + {0.0, 0.01898815687615381, 0.06564245437845008, + 0.5665298796332909, 4.166333347221828e-18, 1.0}, + {0.0, 0.9999780593618628, 0.9999899967080838, + 0.9999996219837988, 0.9991370418689945, 1.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.5042041932513908}}; + + + + std::size_t bytes = a.size() * sizeof(Scalar); + + Scalar* d_a; + Scalar* d_x; + Scalar* d_out; + assert(gpuMalloc((void**)(&d_a), bytes) == gpuSuccess); + assert(gpuMalloc((void**)(&d_x), bytes) == gpuSuccess); + assert(gpuMalloc((void**)(&d_out), bytes) == gpuSuccess); + + gpuMemcpy(d_a, a.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_x, x.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6); + + gpu_out.device(gpu_device) = gpu_a.igamma(gpu_x); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 6; ++j) { + if ((std::isnan)(igamma_s[i][j])) { + VERIFY((std::isnan)(out(i, j))); + } else { + VERIFY_IS_APPROX(out(i, j), igamma_s[i][j]); + } + } + } + + gpuFree(d_a); + gpuFree(d_x); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_igammac() +{ + Tensor<Scalar, 2> a(6, 6); + Tensor<Scalar, 2> x(6, 6); + Tensor<Scalar, 2> out(6, 6); + out.setZero(); + + Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 6; ++j) { + a(i, j) = a_s[i]; + x(i, j) = x_s[j]; + } + } + + Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); + Scalar igammac_s[][6] = {{nan, nan, nan, nan, nan, nan}, + {1.0, 0.36787944117144233, 0.22313016014842982, + 0.018315638888734182, 0.9999000049998333, 0.0}, + {1.0, 0.5724067044708798, 0.3916251762710878, + 0.04601170568923136, 0.9999992477923555, 0.0}, + {1.0, 0.9810118431238462, 0.9343575456215499, + 0.4334701203667089, 1.0, 0.0}, + {1.0, 2.1940638138146658e-05, 1.0003291916285e-05, + 3.7801620118431334e-07, 0.0008629581310054535, + 0.0}, + {1.0, 1.0, 1.0, 1.0, 1.0, 0.49579580674813944}}; + + std::size_t bytes = a.size() * sizeof(Scalar); + + Scalar* d_a; + Scalar* d_x; + Scalar* d_out; + gpuMalloc((void**)(&d_a), bytes); + gpuMalloc((void**)(&d_x), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_a, a.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_x, x.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6); + + gpu_out.device(gpu_device) = gpu_a.igammac(gpu_x); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 6; ++j) { + if ((std::isnan)(igammac_s[i][j])) { + VERIFY((std::isnan)(out(i, j))); + } else { + VERIFY_IS_APPROX(out(i, j), igammac_s[i][j]); + } + } + } + + gpuFree(d_a); + gpuFree(d_x); + gpuFree(d_out); +} + +#if EIGEN_GPU_TEST_C99_MATH +template <typename Scalar> +void test_gpu_erf(const Scalar stddev) +{ + Tensor<Scalar, 2> in(72,97); + in.setRandom(); + in *= in.constant(stddev); + Tensor<Scalar, 2> out(72,97); + out.setZero(); + + std::size_t bytes = in.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + assert(gpuMalloc((void**)(&d_in), bytes) == gpuSuccess); + assert(gpuMalloc((void**)(&d_out), bytes) == gpuSuccess); + + gpuMemcpy(d_in, in.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); + + gpu_out.device(gpu_device) = gpu_in.erf(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 72; ++i) { + for (int j = 0; j < 97; ++j) { + VERIFY_IS_APPROX(out(i,j), (std::erf)(in(i,j))); + } + } + + gpuFree(d_in); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_erfc(const Scalar stddev) +{ + Tensor<Scalar, 2> in(72,97); + in.setRandom(); + in *= in.constant(stddev); + Tensor<Scalar, 2> out(72,97); + out.setZero(); + + std::size_t bytes = in.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in, in.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97); + Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97); + + gpu_out.device(gpu_device) = gpu_in.erfc(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 72; ++i) { + for (int j = 0; j < 97; ++j) { + VERIFY_IS_APPROX(out(i,j), (std::erfc)(in(i,j))); + } + } + + gpuFree(d_in); + gpuFree(d_out); +} +#endif + +template <typename Scalar> +void test_gpu_betainc() +{ + Tensor<Scalar, 1> in_x(125); + Tensor<Scalar, 1> in_a(125); + Tensor<Scalar, 1> in_b(125); + Tensor<Scalar, 1> out(125); + Tensor<Scalar, 1> expected_out(125); + out.setZero(); + + Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); + + Array<Scalar, 1, Dynamic> x(125); + Array<Scalar, 1, Dynamic> a(125); + Array<Scalar, 1, Dynamic> b(125); + Array<Scalar, 1, Dynamic> v(125); + + a << 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, + 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, + 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999, + 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, + 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, + 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999; + + b << 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999, + 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999, + 999.999, 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 999.999, 999.999, 999.999, 999.999, 999.999, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999, + 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999, + 999.999, 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 999.999, 999.999, 999.999, 999.999, 999.999, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999, + 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999, + 999.999, 999.999, 999.999; + + x << -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, + 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, + 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, + 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, + 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, + -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, + 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, + 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, + 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1; + + v << nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, + nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, + nan, nan, 0.47972119876364683, 0.5, 0.5202788012363533, nan, nan, + 0.9518683957740043, 0.9789663010413743, 0.9931729188073435, nan, nan, + 0.999995949033062, 0.9999999999993698, 0.9999999999999999, nan, nan, + 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, nan, nan, nan, + nan, nan, nan, nan, 0.006827081192655869, 0.0210336989586256, + 0.04813160422599567, nan, nan, 0.20014344256217678, 0.5000000000000001, + 0.7998565574378232, nan, nan, 0.9991401428435834, 0.999999999698403, + 0.9999999999999999, nan, nan, 0.9999999999999999, 0.9999999999999999, + 0.9999999999999999, nan, nan, nan, nan, nan, nan, nan, + 1.0646600232370887e-25, 6.301722877826246e-13, 4.050966937974938e-06, nan, + nan, 7.864342668429763e-23, 3.015969667594166e-10, 0.0008598571564165444, + nan, nan, 6.031987710123844e-08, 0.5000000000000007, 0.9999999396801229, + nan, nan, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, nan, + nan, nan, nan, nan, nan, nan, 0.0, 7.029920380986636e-306, + 2.2450728208591345e-101, nan, nan, 0.0, 9.275871147869727e-302, + 1.2232913026152827e-97, nan, nan, 0.0, 3.0891393081932924e-252, + 2.9303043666183996e-60, nan, nan, 2.248913486879199e-196, + 0.5000000000004947, 0.9999999999999999, nan; + + for (int i = 0; i < 125; ++i) { + in_x(i) = x(i); + in_a(i) = a(i); + in_b(i) = b(i); + expected_out(i) = v(i); + } + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_in_x; + Scalar* d_in_a; + Scalar* d_in_b; + Scalar* d_out; + gpuMalloc((void**)(&d_in_x), bytes); + gpuMalloc((void**)(&d_in_a), bytes); + gpuMalloc((void**)(&d_in_b), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in_x, in_x.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in_a, in_a.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_in_b, in_b.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 125); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_a(d_in_a, 125); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_b(d_in_b, 125); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 125); + + gpu_out.device(gpu_device) = betainc(gpu_in_a, gpu_in_b, gpu_in_x); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 1; i < 125; ++i) { + if ((std::isnan)(expected_out(i))) { + VERIFY((std::isnan)(out(i))); + } else { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + } + + gpuFree(d_in_x); + gpuFree(d_in_a); + gpuFree(d_in_b); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_i0e() +{ + Tensor<Scalar, 1> in_x(21); + Tensor<Scalar, 1> out(21); + Tensor<Scalar, 1> expected_out(21); + out.setZero(); + + Array<Scalar, 1, Dynamic> in_x_array(21); + Array<Scalar, 1, Dynamic> expected_out_array(21); + + in_x_array << -20.0, -18.0, -16.0, -14.0, -12.0, -10.0, -8.0, -6.0, -4.0, + -2.0, 0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 20.0; + + expected_out_array << 0.0897803118848, 0.0947062952128, 0.100544127361, + 0.107615251671, 0.116426221213, 0.127833337163, 0.143431781857, + 0.16665743264, 0.207001921224, 0.308508322554, 1.0, 0.308508322554, + 0.207001921224, 0.16665743264, 0.143431781857, 0.127833337163, + 0.116426221213, 0.107615251671, 0.100544127361, 0.0947062952128, + 0.0897803118848; + + for (int i = 0; i < 21; ++i) { + in_x(i) = in_x_array(i); + expected_out(i) = expected_out_array(i); + } + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in, in_x.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in(d_in, 21); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 21); + + gpu_out.device(gpu_device) = gpu_in.i0e(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 21; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + + gpuFree(d_in); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_i1e() +{ + Tensor<Scalar, 1> in_x(21); + Tensor<Scalar, 1> out(21); + Tensor<Scalar, 1> expected_out(21); + out.setZero(); + + Array<Scalar, 1, Dynamic> in_x_array(21); + Array<Scalar, 1, Dynamic> expected_out_array(21); + + in_x_array << -20.0, -18.0, -16.0, -14.0, -12.0, -10.0, -8.0, -6.0, -4.0, + -2.0, 0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 20.0; + + expected_out_array << -0.0875062221833, -0.092036796872, -0.0973496147565, + -0.103697667463, -0.11146429929, -0.121262681384, -0.134142493293, + -0.152051459309, -0.178750839502, -0.215269289249, 0.0, 0.215269289249, + 0.178750839502, 0.152051459309, 0.134142493293, 0.121262681384, + 0.11146429929, 0.103697667463, 0.0973496147565, 0.092036796872, + 0.0875062221833; + + for (int i = 0; i < 21; ++i) { + in_x(i) = in_x_array(i); + expected_out(i) = expected_out_array(i); + } + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_in; + Scalar* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_in, in_x.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in(d_in, 21); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 21); + + gpu_out.device(gpu_device) = gpu_in.i1e(); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 21; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + + gpuFree(d_in); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_igamma_der_a() +{ + Tensor<Scalar, 1> in_x(30); + Tensor<Scalar, 1> in_a(30); + Tensor<Scalar, 1> out(30); + Tensor<Scalar, 1> expected_out(30); + out.setZero(); + + Array<Scalar, 1, Dynamic> in_a_array(30); + Array<Scalar, 1, Dynamic> in_x_array(30); + Array<Scalar, 1, Dynamic> expected_out_array(30); + + // See special_functions.cpp for the Python code that generates the test data. + + in_a_array << 0.01, 0.01, 0.01, 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, + 1.0, 1.0, 1.0, 10.0, 10.0, 10.0, 10.0, 10.0, 100.0, 100.0, 100.0, 100.0, + 100.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0; + + in_x_array << 1.25668890405e-26, 1.17549435082e-38, 1.20938905072e-05, + 1.17549435082e-38, 1.17549435082e-38, 5.66572070696e-16, 0.0132865061065, + 0.0200034203853, 6.29263709118e-17, 1.37160367764e-06, 0.333412038288, + 1.18135687766, 0.580629033777, 0.170631439426, 0.786686768458, + 7.63873279537, 13.1944344379, 11.896042354, 10.5830172417, 10.5020942233, + 92.8918587747, 95.003720371, 86.3715926467, 96.0330217672, 82.6389930677, + 968.702906754, 969.463546828, 1001.79726022, 955.047416547, 1044.27458568; + + expected_out_array << -32.7256441441, -36.4394150514, -9.66467612263, + -36.4394150514, -36.4394150514, -1.0891900302, -2.66351229645, + -2.48666868596, -0.929700494428, -3.56327722764, -0.455320135314, + -0.391437214323, -0.491352055991, -0.350454834292, -0.471773162921, + -0.104084440522, -0.0723646747909, -0.0992828975532, -0.121638215446, + -0.122619605294, -0.0317670267286, -0.0359974812869, -0.0154359225363, + -0.0375775365921, -0.00794899153653, -0.00777303219211, -0.00796085782042, + -0.0125850719397, -0.00455500206958, -0.00476436993148; + + for (int i = 0; i < 30; ++i) { + in_x(i) = in_x_array(i); + in_a(i) = in_a_array(i); + expected_out(i) = expected_out_array(i); + } + + std::size_t bytes = in_x.size() * sizeof(Scalar); + + Scalar* d_a; + Scalar* d_x; + Scalar* d_out; + gpuMalloc((void**)(&d_a), bytes); + gpuMalloc((void**)(&d_x), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_a, in_a.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_x, in_x.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_a(d_a, 30); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_x(d_x, 30); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 30); + + gpu_out.device(gpu_device) = gpu_a.igamma_der_a(gpu_x); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 30; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + + gpuFree(d_a); + gpuFree(d_x); + gpuFree(d_out); +} + +template <typename Scalar> +void test_gpu_gamma_sample_der_alpha() +{ + Tensor<Scalar, 1> in_alpha(30); + Tensor<Scalar, 1> in_sample(30); + Tensor<Scalar, 1> out(30); + Tensor<Scalar, 1> expected_out(30); + out.setZero(); + + Array<Scalar, 1, Dynamic> in_alpha_array(30); + Array<Scalar, 1, Dynamic> in_sample_array(30); + Array<Scalar, 1, Dynamic> expected_out_array(30); + + // See special_functions.cpp for the Python code that generates the test data. + + in_alpha_array << 0.01, 0.01, 0.01, 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, + 1.0, 1.0, 1.0, 1.0, 10.0, 10.0, 10.0, 10.0, 10.0, 100.0, 100.0, 100.0, + 100.0, 100.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0; + + in_sample_array << 1.25668890405e-26, 1.17549435082e-38, 1.20938905072e-05, + 1.17549435082e-38, 1.17549435082e-38, 5.66572070696e-16, 0.0132865061065, + 0.0200034203853, 6.29263709118e-17, 1.37160367764e-06, 0.333412038288, + 1.18135687766, 0.580629033777, 0.170631439426, 0.786686768458, + 7.63873279537, 13.1944344379, 11.896042354, 10.5830172417, 10.5020942233, + 92.8918587747, 95.003720371, 86.3715926467, 96.0330217672, 82.6389930677, + 968.702906754, 969.463546828, 1001.79726022, 955.047416547, 1044.27458568; + + expected_out_array << 7.42424742367e-23, 1.02004297287e-34, 0.0130155240738, + 1.02004297287e-34, 1.02004297287e-34, 1.96505168277e-13, 0.525575786243, + 0.713903991771, 2.32077561808e-14, 0.000179348049886, 0.635500453302, + 1.27561284917, 0.878125852156, 0.41565819538, 1.03606488534, + 0.885964824887, 1.16424049334, 1.10764479598, 1.04590810812, + 1.04193666963, 0.965193152414, 0.976217589464, 0.93008035061, + 0.98153216096, 0.909196397698, 0.98434963993, 0.984738050206, + 1.00106492525, 0.97734200649, 1.02198794179; + + for (int i = 0; i < 30; ++i) { + in_alpha(i) = in_alpha_array(i); + in_sample(i) = in_sample_array(i); + expected_out(i) = expected_out_array(i); + } + + std::size_t bytes = in_alpha.size() * sizeof(Scalar); + + Scalar* d_alpha; + Scalar* d_sample; + Scalar* d_out; + gpuMalloc((void**)(&d_alpha), bytes); + gpuMalloc((void**)(&d_sample), bytes); + gpuMalloc((void**)(&d_out), bytes); + + gpuMemcpy(d_alpha, in_alpha.data(), bytes, gpuMemcpyHostToDevice); + gpuMemcpy(d_sample, in_sample.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_alpha(d_alpha, 30); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_sample(d_sample, 30); + Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 30); + + gpu_out.device(gpu_device) = gpu_alpha.gamma_sample_der_alpha(gpu_sample); + + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, + gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + for (int i = 0; i < 30; ++i) { + VERIFY_IS_APPROX(out(i), expected_out(i)); + } + + gpuFree(d_alpha); + gpuFree(d_sample); + gpuFree(d_out); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_gpu) +{ + CALL_SUBTEST_1(test_gpu_nullary()); + CALL_SUBTEST_1(test_gpu_elementwise_small()); + CALL_SUBTEST_1(test_gpu_elementwise()); + CALL_SUBTEST_1(test_gpu_props()); + CALL_SUBTEST_1(test_gpu_reduction()); + CALL_SUBTEST_2(test_gpu_contraction<ColMajor>()); + CALL_SUBTEST_2(test_gpu_contraction<RowMajor>()); + CALL_SUBTEST_3(test_gpu_convolution_1d<ColMajor>()); + CALL_SUBTEST_3(test_gpu_convolution_1d<RowMajor>()); + CALL_SUBTEST_3(test_gpu_convolution_inner_dim_col_major_1d()); + CALL_SUBTEST_3(test_gpu_convolution_inner_dim_row_major_1d()); + CALL_SUBTEST_3(test_gpu_convolution_2d<ColMajor>()); + CALL_SUBTEST_3(test_gpu_convolution_2d<RowMajor>()); +#if !defined(EIGEN_USE_HIP) +// disable these tests on HIP for now. +// they hang..need to investigate and fix + CALL_SUBTEST_3(test_gpu_convolution_3d<ColMajor>()); + CALL_SUBTEST_3(test_gpu_convolution_3d<RowMajor>()); +#endif + +#if EIGEN_GPU_TEST_C99_MATH + // std::erf, std::erfc, and so on where only added in c++11. We use them + // as a golden reference to validate the results produced by Eigen. Therefore + // we can only run these tests if we use a c++11 compiler. + CALL_SUBTEST_4(test_gpu_lgamma<float>(1.0f)); + CALL_SUBTEST_4(test_gpu_lgamma<float>(100.0f)); + CALL_SUBTEST_4(test_gpu_lgamma<float>(0.01f)); + CALL_SUBTEST_4(test_gpu_lgamma<float>(0.001f)); + + CALL_SUBTEST_4(test_gpu_lgamma<double>(1.0)); + CALL_SUBTEST_4(test_gpu_lgamma<double>(100.0)); + CALL_SUBTEST_4(test_gpu_lgamma<double>(0.01)); + CALL_SUBTEST_4(test_gpu_lgamma<double>(0.001)); + + CALL_SUBTEST_4(test_gpu_erf<float>(1.0f)); + CALL_SUBTEST_4(test_gpu_erf<float>(100.0f)); + CALL_SUBTEST_4(test_gpu_erf<float>(0.01f)); + CALL_SUBTEST_4(test_gpu_erf<float>(0.001f)); + + CALL_SUBTEST_4(test_gpu_erfc<float>(1.0f)); + // CALL_SUBTEST(test_gpu_erfc<float>(100.0f)); + CALL_SUBTEST_4(test_gpu_erfc<float>(5.0f)); // GPU erfc lacks precision for large inputs + CALL_SUBTEST_4(test_gpu_erfc<float>(0.01f)); + CALL_SUBTEST_4(test_gpu_erfc<float>(0.001f)); + + CALL_SUBTEST_4(test_gpu_erf<double>(1.0)); + CALL_SUBTEST_4(test_gpu_erf<double>(100.0)); + CALL_SUBTEST_4(test_gpu_erf<double>(0.01)); + CALL_SUBTEST_4(test_gpu_erf<double>(0.001)); + + CALL_SUBTEST_4(test_gpu_erfc<double>(1.0)); + // CALL_SUBTEST(test_gpu_erfc<double>(100.0)); + CALL_SUBTEST_4(test_gpu_erfc<double>(5.0)); // GPU erfc lacks precision for large inputs + CALL_SUBTEST_4(test_gpu_erfc<double>(0.01)); + CALL_SUBTEST_4(test_gpu_erfc<double>(0.001)); + +#if !defined(EIGEN_USE_HIP) +// disable these tests on HIP for now. + CALL_SUBTEST_5(test_gpu_digamma<float>()); + CALL_SUBTEST_5(test_gpu_digamma<double>()); + + CALL_SUBTEST_5(test_gpu_polygamma<float>()); + CALL_SUBTEST_5(test_gpu_polygamma<double>()); + + CALL_SUBTEST_5(test_gpu_zeta<float>()); + CALL_SUBTEST_5(test_gpu_zeta<double>()); +#endif + + CALL_SUBTEST_5(test_gpu_igamma<float>()); + CALL_SUBTEST_5(test_gpu_igammac<float>()); + + CALL_SUBTEST_5(test_gpu_igamma<double>()); + CALL_SUBTEST_5(test_gpu_igammac<double>()); + +#if !defined(EIGEN_USE_HIP) +// disable these tests on HIP for now. + CALL_SUBTEST_6(test_gpu_betainc<float>()); + CALL_SUBTEST_6(test_gpu_betainc<double>()); + + CALL_SUBTEST_6(test_gpu_i0e<float>()); + CALL_SUBTEST_6(test_gpu_i0e<double>()); + + CALL_SUBTEST_6(test_gpu_i1e<float>()); + CALL_SUBTEST_6(test_gpu_i1e<double>()); + + CALL_SUBTEST_6(test_gpu_i1e<float>()); + CALL_SUBTEST_6(test_gpu_i1e<double>()); + + CALL_SUBTEST_6(test_gpu_igamma_der_a<float>()); + CALL_SUBTEST_6(test_gpu_igamma_der_a<double>()); + + CALL_SUBTEST_6(test_gpu_gamma_sample_der_alpha<float>()); + CALL_SUBTEST_6(test_gpu_gamma_sample_der_alpha<double>()); +#endif + +#endif +}
diff --git a/unsupported/test/cxx11_tensor_ifft.cpp b/unsupported/test/cxx11_tensor_ifft.cpp index 5fd88fa..c20edd9 100644 --- a/unsupported/test/cxx11_tensor_ifft.cpp +++ b/unsupported/test/cxx11_tensor_ifft.cpp
@@ -131,7 +131,7 @@ } } -void test_cxx11_tensor_ifft() { +EIGEN_DECLARE_TEST(cxx11_tensor_ifft) { CALL_SUBTEST(test_1D_fft_ifft_invariant<ColMajor>(4)); CALL_SUBTEST(test_1D_fft_ifft_invariant<ColMajor>(16)); CALL_SUBTEST(test_1D_fft_ifft_invariant<ColMajor>(32));
diff --git a/unsupported/test/cxx11_tensor_image_patch.cpp b/unsupported/test/cxx11_tensor_image_patch.cpp index 5d6a491..862f1f7 100644 --- a/unsupported/test/cxx11_tensor_image_patch.cpp +++ b/unsupported/test/cxx11_tensor_image_patch.cpp
@@ -13,7 +13,7 @@ using Eigen::Tensor; -static void test_simple_patch() +void test_simple_patch() { Tensor<float, 4> tensor(2,3,5,7); tensor.setRandom(); @@ -180,7 +180,7 @@ } // Verifies VALID padding (no padding) with incrementing values. -static void test_patch_padding_valid() +void test_patch_padding_valid() { int input_depth = 3; int input_rows = 3; @@ -256,7 +256,7 @@ } // Verifies VALID padding (no padding) with the same value. -static void test_patch_padding_valid_same_value() +void test_patch_padding_valid_same_value() { int input_depth = 1; int input_rows = 5; @@ -329,7 +329,7 @@ } // Verifies SAME padding. -static void test_patch_padding_same() +void test_patch_padding_same() { int input_depth = 3; int input_rows = 4; @@ -405,7 +405,58 @@ } } -static void test_patch_no_extra_dim() +// Verifies that SAME padding, when computed as negative values, will be clipped +// to zero. +void test_patch_padding_same_negative_padding_clip_to_zero() { + int input_depth = 1; + int input_rows = 15; + int input_cols = 1; + int input_batches = 1; + int ksize = 1; // Corresponds to the Rows and Cols for + // tensor.extract_image_patches<>. + int row_stride = 5; + int col_stride = 1; + // ColMajor + Tensor<float, 4> tensor(input_depth, input_rows, input_cols, input_batches); + // Initializes tensor with incrementing numbers. + for (int i = 0; i < tensor.size(); ++i) { + tensor.data()[i] = i + 1; + } + Tensor<float, 5> result = tensor.extract_image_patches( + ksize, ksize, row_stride, col_stride, 1, 1, PADDING_SAME); + // row padding will be computed as -2 originally and then be clipped to 0. + VERIFY_IS_EQUAL(result.coeff(0), 1.0f); + VERIFY_IS_EQUAL(result.coeff(1), 6.0f); + VERIFY_IS_EQUAL(result.coeff(2), 11.0f); + + VERIFY_IS_EQUAL(result.dimension(0), input_depth); // depth + VERIFY_IS_EQUAL(result.dimension(1), ksize); // kernel rows + VERIFY_IS_EQUAL(result.dimension(2), ksize); // kernel cols + VERIFY_IS_EQUAL(result.dimension(3), 3); // number of patches + VERIFY_IS_EQUAL(result.dimension(4), input_batches); // number of batches + + // RowMajor + Tensor<float, 4, RowMajor> tensor_row_major = tensor.swap_layout(); + VERIFY_IS_EQUAL(tensor.dimension(0), tensor_row_major.dimension(3)); + VERIFY_IS_EQUAL(tensor.dimension(1), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor.dimension(2), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor.dimension(3), tensor_row_major.dimension(0)); + + Tensor<float, 5, RowMajor> result_row_major = + tensor_row_major.extract_image_patches(ksize, ksize, row_stride, + col_stride, 1, 1, PADDING_SAME); + VERIFY_IS_EQUAL(result_row_major.coeff(0), 1.0f); + VERIFY_IS_EQUAL(result_row_major.coeff(1), 6.0f); + VERIFY_IS_EQUAL(result_row_major.coeff(2), 11.0f); + + VERIFY_IS_EQUAL(result.dimension(0), result_row_major.dimension(4)); + VERIFY_IS_EQUAL(result.dimension(1), result_row_major.dimension(3)); + VERIFY_IS_EQUAL(result.dimension(2), result_row_major.dimension(2)); + VERIFY_IS_EQUAL(result.dimension(3), result_row_major.dimension(1)); + VERIFY_IS_EQUAL(result.dimension(4), result_row_major.dimension(0)); +} + +void test_patch_no_extra_dim() { Tensor<float, 3> tensor(2,3,5); tensor.setRandom(); @@ -553,7 +604,7 @@ } } -static void test_imagenet_patches() +void test_imagenet_patches() { // Test the code on typical configurations used by the 'imagenet' benchmarks at // https://github.com/soumith/convnet-benchmarks @@ -568,13 +619,7 @@ VERIFY_IS_EQUAL(l_out.dimension(4), 16); // RowMajor - Tensor<float, 4, RowMajor> l_in_row_major = l_in.swap_layout(); - VERIFY_IS_EQUAL(l_in.dimension(0), l_in_row_major.dimension(3)); - VERIFY_IS_EQUAL(l_in.dimension(1), l_in_row_major.dimension(2)); - VERIFY_IS_EQUAL(l_in.dimension(2), l_in_row_major.dimension(1)); - VERIFY_IS_EQUAL(l_in.dimension(3), l_in_row_major.dimension(0)); - - Tensor<float, 5, RowMajor> l_out_row_major = l_in_row_major.extract_image_patches(11, 11); + Tensor<float, 5, RowMajor> l_out_row_major = l_in.swap_layout().extract_image_patches(11, 11); VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 16); VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 128*128); VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 11); @@ -589,10 +634,8 @@ for (int r = 0; r < 11; ++r) { for (int d = 0; d < 3; ++d) { float expected = 0.0f; - float expected_row_major = 0.0f; if (r-5+i >= 0 && c-5+j >= 0 && r-5+i < 128 && c-5+j < 128) { expected = l_in(d, r-5+i, c-5+j, b); - expected_row_major = l_in_row_major(b, c-5+j, r-5+i, d); } // ColMajor if (l_out(d, r, c, patchId, b) != expected) { @@ -601,15 +644,13 @@ VERIFY_IS_EQUAL(l_out(d, r, c, patchId, b), expected); // RowMajor if (l_out_row_major(b, patchId, c, r, d) != - expected_row_major) { + expected) { std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; } VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), - expected_row_major); - // Check that ColMajor and RowMajor agree. - VERIFY_IS_EQUAL(expected, expected_row_major); + expected); } } } @@ -628,8 +669,7 @@ VERIFY_IS_EQUAL(l_out.dimension(4), 32); // RowMajor - l_in_row_major = l_in.swap_layout(); - l_out_row_major = l_in_row_major.extract_image_patches(9, 9); + l_out_row_major = l_in.swap_layout().extract_image_patches(9, 9); VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 64*64); VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 9); @@ -644,10 +684,8 @@ for (int r = 0; r < 9; ++r) { for (int d = 0; d < 16; ++d) { float expected = 0.0f; - float expected_row_major = 0.0f; if (r-4+i >= 0 && c-4+j >= 0 && r-4+i < 64 && c-4+j < 64) { expected = l_in(d, r-4+i, c-4+j, b); - expected_row_major = l_in_row_major(b, c-4+j, r-4+i, d); } // ColMajor if (l_out(d, r, c, patchId, b) != expected) { @@ -655,12 +693,10 @@ } VERIFY_IS_EQUAL(l_out(d, r, c, patchId, b), expected); // RowMajor - if (l_out_row_major(b, patchId, c, r, d) != expected_row_major) { + if (l_out_row_major(b, patchId, c, r, d) != expected) { std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; } - VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected_row_major); - // Check that ColMajor and RowMajor agree. - VERIFY_IS_EQUAL(expected, expected_row_major); + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); } } } @@ -679,8 +715,7 @@ VERIFY_IS_EQUAL(l_out.dimension(4), 32); // RowMajor - l_in_row_major = l_in.swap_layout(); - l_out_row_major = l_in_row_major.extract_image_patches(7, 7); + l_out_row_major = l_in.swap_layout().extract_image_patches(7, 7); VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 16*16); VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 7); @@ -695,10 +730,8 @@ for (int r = 0; r < 7; ++r) { for (int d = 0; d < 32; ++d) { float expected = 0.0f; - float expected_row_major = 0.0f; if (r-3+i >= 0 && c-3+j >= 0 && r-3+i < 16 && c-3+j < 16) { expected = l_in(d, r-3+i, c-3+j, b); - expected_row_major = l_in_row_major(b, c-3+j, r-3+i, d); } // ColMajor if (l_out(d, r, c, patchId, b) != expected) { @@ -706,12 +739,10 @@ } VERIFY_IS_EQUAL(l_out(d, r, c, patchId, b), expected); // RowMajor - if (l_out_row_major(b, patchId, c, r, d) != expected_row_major) { + if (l_out_row_major(b, patchId, c, r, d) != expected) { std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; } - VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected_row_major); - // Check that ColMajor and RowMajor agree. - VERIFY_IS_EQUAL(expected, expected_row_major); + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); } } } @@ -730,8 +761,7 @@ VERIFY_IS_EQUAL(l_out.dimension(4), 32); // RowMajor - l_in_row_major = l_in.swap_layout(); - l_out_row_major = l_in_row_major.extract_image_patches(3, 3); + l_out_row_major = l_in.swap_layout().extract_image_patches(3, 3); VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 13*13); VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 3); @@ -746,10 +776,8 @@ for (int r = 0; r < 3; ++r) { for (int d = 0; d < 64; ++d) { float expected = 0.0f; - float expected_row_major = 0.0f; if (r-1+i >= 0 && c-1+j >= 0 && r-1+i < 13 && c-1+j < 13) { expected = l_in(d, r-1+i, c-1+j, b); - expected_row_major = l_in_row_major(b, c-1+j, r-1+i, d); } // ColMajor if (l_out(d, r, c, patchId, b) != expected) { @@ -757,12 +785,10 @@ } VERIFY_IS_EQUAL(l_out(d, r, c, patchId, b), expected); // RowMajor - if (l_out_row_major(b, patchId, c, r, d) != expected_row_major) { + if (l_out_row_major(b, patchId, c, r, d) != expected) { std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; } - VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected_row_major); - // Check that ColMajor and RowMajor agree. - VERIFY_IS_EQUAL(expected, expected_row_major); + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); } } } @@ -771,12 +797,13 @@ } } -void test_cxx11_tensor_image_patch() +EIGEN_DECLARE_TEST(cxx11_tensor_image_patch) { - CALL_SUBTEST(test_simple_patch()); - CALL_SUBTEST(test_patch_no_extra_dim()); - CALL_SUBTEST(test_patch_padding_valid()); - CALL_SUBTEST(test_patch_padding_valid_same_value()); - CALL_SUBTEST(test_patch_padding_same()); - CALL_SUBTEST(test_imagenet_patches()); + CALL_SUBTEST_1(test_simple_patch()); + CALL_SUBTEST_2(test_patch_no_extra_dim()); + CALL_SUBTEST_3(test_patch_padding_valid()); + CALL_SUBTEST_4(test_patch_padding_valid_same_value()); + CALL_SUBTEST_5(test_patch_padding_same()); + CALL_SUBTEST_6(test_imagenet_patches()); + CALL_SUBTEST_7(test_patch_padding_same_negative_padding_clip_to_zero()); }
diff --git a/unsupported/test/cxx11_tensor_image_patch_sycl.cpp b/unsupported/test/cxx11_tensor_image_patch_sycl.cpp new file mode 100644 index 0000000..c1828a0 --- /dev/null +++ b/unsupported/test/cxx11_tensor_image_patch_sycl.cpp
@@ -0,0 +1,1092 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; +static const int DataLayout = ColMajor; + +template <typename DataType, typename IndexType> +static void test_simple_image_patch_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + array<IndexType, 4> tensorRowMajorRange = {{sizeDim4, sizeDim3, sizeDim2, sizeDim1}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + Tensor<DataType, 4, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + tensor_col_major.setRandom(); + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 4, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_col_major.size())*sizeof(DataType)); + + VERIFY_IS_EQUAL(tensor_col_major.dimension(0), tensor_row_major.dimension(3)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(1), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(2), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(3), tensor_row_major.dimension(0)); + + // Single pixel patch: ColMajor + array<IndexType, 5> patchColMajorTensorRange={{sizeDim1, 1, 1, sizeDim2*sizeDim3, sizeDim4}}; + Tensor<DataType, 5, DataLayout,IndexType> single_patch_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =single_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_single_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_single_patch_col_major(gpu_data_single_patch_col_major, patchColMajorTensorRange); + gpu_single_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(1, 1); + sycl_device.memcpyDeviceToHost(single_patch_col_major.data(), gpu_data_single_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(single_patch_col_major.dimension(0), 2); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(1), 1); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(3), 3*5); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(4), 7); + + // Single pixel patch: RowMajor + array<IndexType, 5> patchRowMajorTensorRange={{sizeDim4, sizeDim2*sizeDim3, 1, 1, sizeDim1}}; + Tensor<DataType, 5, RowMajor,IndexType> single_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =single_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_single_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_single_patch_row_major(gpu_data_single_patch_row_major, patchRowMajorTensorRange); + gpu_single_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(1, 1); + sycl_device.memcpyDeviceToHost(single_patch_row_major.data(), gpu_data_single_patch_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(single_patch_row_major.dimension(0), 7); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(1), 3*5); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(3), 1); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(4), 2); + + for (IndexType i = 0; i < tensor_col_major.size(); ++i) { + // ColMajor + if (tensor_col_major.data()[i] != single_patch_col_major.data()[i]) { + std::cout << "Mismatch detected at index colmajor " << i << " : " + << tensor_col_major.data()[i] << " vs " << single_patch_col_major.data()[i] + << std::endl; + } + VERIFY_IS_EQUAL(single_patch_col_major.data()[i], tensor_col_major.data()[i]); + // RowMajor + if (tensor_row_major.data()[i] != single_patch_row_major.data()[i]) { + std::cout << "Mismatch detected at index row major" << i << " : " + << tensor_row_major.data()[i] << " vs " + << single_patch_row_major.data()[i] << std::endl; + } + VERIFY_IS_EQUAL(single_patch_row_major.data()[i], + tensor_row_major.data()[i]); + VERIFY_IS_EQUAL(tensor_col_major.data()[i], tensor_row_major.data()[i]); + VERIFY_IS_EQUAL(single_patch_col_major.data()[i], + single_patch_row_major.data()[i]); + } + + + // Entire image patch: ColMajor + patchColMajorTensorRange={{sizeDim1, sizeDim2, sizeDim3, sizeDim2*sizeDim3, sizeDim4}}; + Tensor<DataType, 5, DataLayout,IndexType> entire_image_patch_col_major(patchColMajorTensorRange); + patchTensorBuffSize =entire_image_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_entire_image_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_entire_image_patch_col_major(gpu_data_entire_image_patch_col_major, patchColMajorTensorRange); + gpu_entire_image_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(3, 5); + sycl_device.memcpyDeviceToHost(entire_image_patch_col_major.data(), gpu_data_entire_image_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(0), 2); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(1), 3); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(2), 5); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(3), 3*5); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(4), 7); + + // Entire image patch: RowMajor + patchRowMajorTensorRange={{sizeDim4, sizeDim2*sizeDim3, sizeDim3, sizeDim2, sizeDim1}}; + Tensor<DataType, 5, RowMajor,IndexType> entire_image_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =entire_image_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_entire_image_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_entire_image_patch_row_major(gpu_data_entire_image_patch_row_major, patchRowMajorTensorRange); + gpu_entire_image_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(3, 5); + sycl_device.memcpyDeviceToHost(entire_image_patch_row_major.data(), gpu_data_entire_image_patch_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(0), 7); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(1), 3*5); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(2), 5); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(3), 3); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(4), 2); + + for (IndexType i = 0; i < 3; ++i) { + for (IndexType j = 0; j < 5; ++j) { + IndexType patchId = i+3*j; + for (IndexType r = 0; r < 3; ++r) { + for (IndexType c = 0; c < 5; ++c) { + for (IndexType d = 0; d < 2; ++d) { + for (IndexType b = 0; b < 7; ++b) { + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + if (r-1+i >= 0 && c-2+j >= 0 && r-1+i < 3 && c-2+j < 5) { + expected_col_major = tensor_col_major(d, r-1+i, c-2+j, b); + expected_row_major = tensor_row_major(b, c-2+j, r-1+i, d); + } + // ColMajor + if (entire_image_patch_col_major(d, r, c, patchId, b) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(entire_image_patch_col_major(d, r, c, patchId, b), expected_col_major); + // RowMajor + if (entire_image_patch_row_major(b, patchId, c, r, d) != + expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j + << " r=" << r << " c=" << c << " d=" << d << " b=" << b + << std::endl; + } + VERIFY_IS_EQUAL(entire_image_patch_row_major(b, patchId, c, r, d), + expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + } + + // 2D patch: ColMajor + patchColMajorTensorRange={{sizeDim1, 2, 2, sizeDim2*sizeDim3, sizeDim4}}; + Tensor<DataType, 5, DataLayout,IndexType> twod_patch_col_major(patchColMajorTensorRange); + patchTensorBuffSize =twod_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_twod_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_twod_patch_col_major(gpu_data_twod_patch_col_major, patchColMajorTensorRange); + gpu_twod_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(2, 2); + sycl_device.memcpyDeviceToHost(twod_patch_col_major.data(), gpu_data_twod_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(0), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(1), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(3), 3*5); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(4), 7); + + // 2D patch: RowMajor + patchRowMajorTensorRange={{sizeDim4, sizeDim2*sizeDim3, 2, 2, sizeDim1}}; + Tensor<DataType, 5, RowMajor,IndexType> twod_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =twod_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_twod_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_twod_patch_row_major(gpu_data_twod_patch_row_major, patchRowMajorTensorRange); + gpu_twod_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(2, 2); + sycl_device.memcpyDeviceToHost(twod_patch_row_major.data(), gpu_data_twod_patch_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(0), 7); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(1), 3*5); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(3), 2); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(4), 2); + + + // Based on the calculation described in TensorTraits.h, padding happens to be 0. + IndexType row_padding = 0; + IndexType col_padding = 0; + IndexType stride = 1; + + for (IndexType i = 0; i < 3; ++i) { + for (IndexType j = 0; j < 5; ++j) { + IndexType patchId = i+3*j; + for (IndexType r = 0; r < 2; ++r) { + for (IndexType c = 0; c < 2; ++c) { + for (IndexType d = 0; d < 2; ++d) { + for (IndexType b = 0; b < 7; ++b) { + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + IndexType row_offset = r*stride + i - row_padding; + IndexType col_offset = c*stride + j - col_padding; + // ColMajor + if (row_offset >= 0 && col_offset >= 0 && row_offset < tensor_col_major.dimension(1) && col_offset < tensor_col_major.dimension(2)) { + expected_col_major = tensor_col_major(d, row_offset, col_offset, b); + } + if (twod_patch_col_major(d, r, c, patchId, b) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(twod_patch_col_major(d, r, c, patchId, b), expected_col_major); + + // RowMajor + if (row_offset >= 0 && col_offset >= 0 && row_offset < tensor_row_major.dimension(2) && col_offset < tensor_row_major.dimension(1)) { + expected_row_major = tensor_row_major(b, col_offset, row_offset, d); + + } + if (twod_patch_row_major(b, patchId, c, r, d) != expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(twod_patch_row_major(b, patchId, c, r, d), expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + } + + sycl_device.deallocate(gpu_data_col_major); + sycl_device.deallocate(gpu_data_row_major); + sycl_device.deallocate(gpu_data_single_patch_col_major); + sycl_device.deallocate(gpu_data_single_patch_row_major); + sycl_device.deallocate(gpu_data_entire_image_patch_col_major); + sycl_device.deallocate(gpu_data_entire_image_patch_row_major); + sycl_device.deallocate(gpu_data_twod_patch_col_major); + sycl_device.deallocate(gpu_data_twod_patch_row_major); + +} + + +// Verifies VALID padding (no padding) with incrementing values. +template <typename DataType, typename IndexType> +static void test_patch_padding_valid_sycl(const Eigen::SyclDevice& sycl_device){ + IndexType input_depth = 3; + IndexType input_rows = 3; + IndexType input_cols = 3; + IndexType input_batches = 1; + IndexType ksize = 2; // Corresponds to the Rows and Cols for tensor.extract_image_patches<>. + IndexType stride = 2; // Only same stride is supported. + + array<IndexType, 4> tensorColMajorRange = {{input_depth, input_rows, input_cols, input_batches}}; + array<IndexType, 4> tensorRowMajorRange = {{input_batches, input_cols, input_rows, input_depth}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + Tensor<DataType, 4, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 4, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_col_major.size())*sizeof(DataType)); + + VERIFY_IS_EQUAL(tensor_col_major.dimension(0), tensor_row_major.dimension(3)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(1), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(2), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(3), tensor_row_major.dimension(0)); + + // Initializes tensor with incrementing numbers. + for (IndexType i = 0; i < tensor_col_major.size(); ++i) { + tensor_col_major.data()[i] = i + 1; + } + // ColMajor + array<IndexType, 5> patchColMajorTensorRange={{input_depth, ksize, ksize, 1, input_batches}}; + Tensor<DataType, 5, DataLayout,IndexType> result_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =result_col_major.size()*sizeof(DataType); + DataType* gpu_data_result_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_result_col_major(gpu_data_result_col_major, patchColMajorTensorRange); + gpu_result_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(ksize, ksize, stride, stride, 1, 1, PADDING_VALID); + sycl_device.memcpyDeviceToHost(result_col_major.data(), gpu_data_result_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(result_col_major.dimension(0), input_depth); // depth + VERIFY_IS_EQUAL(result_col_major.dimension(1), ksize); // kernel rows + VERIFY_IS_EQUAL(result_col_major.dimension(2), ksize); // kernel cols + VERIFY_IS_EQUAL(result_col_major.dimension(3), 1); // number of patches + VERIFY_IS_EQUAL(result_col_major.dimension(4), input_batches); // number of batches + + // RowMajor + array<IndexType, 5> patchRowMajorTensorRange={{input_batches, 1, ksize, ksize, input_depth }}; + Tensor<DataType, 5, RowMajor,IndexType> result_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =result_row_major.size()*sizeof(DataType); + DataType* gpu_data_result_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_result_row_major(gpu_data_result_row_major, patchRowMajorTensorRange); + gpu_result_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(ksize, ksize, stride, stride, 1, 1, PADDING_VALID); + sycl_device.memcpyDeviceToHost(result_row_major.data(), gpu_data_result_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(result_col_major.dimension(0), result_row_major.dimension(4)); + VERIFY_IS_EQUAL(result_col_major.dimension(1), result_row_major.dimension(3)); + VERIFY_IS_EQUAL(result_col_major.dimension(2), result_row_major.dimension(2)); + VERIFY_IS_EQUAL(result_col_major.dimension(3), result_row_major.dimension(1)); + VERIFY_IS_EQUAL(result_col_major.dimension(4), result_row_major.dimension(0)); + + // No padding is carried out. + IndexType row_padding = 0; + IndexType col_padding = 0; + + for (IndexType i = 0; (i+stride+ksize-1) < input_rows; i += stride) { // input rows + for (IndexType j = 0; (j+stride+ksize-1) < input_cols; j += stride) { // input cols + IndexType patchId = i+input_rows*j; + for (IndexType r = 0; r < ksize; ++r) { // patch rows + for (IndexType c = 0; c < ksize; ++c) { // patch cols + for (IndexType d = 0; d < input_depth; ++d) { // depth + for (IndexType b = 0; b < input_batches; ++b) { // batch + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + IndexType row_offset = r + i - row_padding; + IndexType col_offset = c + j - col_padding; + if (row_offset >= 0 && col_offset >= 0 && row_offset < input_rows && col_offset < input_cols) { + expected_col_major = tensor_col_major(d, row_offset, col_offset, b); + expected_row_major = tensor_row_major(b, col_offset, row_offset, d); + } + // ColMajor + if (result_col_major(d, r, c, patchId, b) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_col_major(d, r, c, patchId, b), expected_col_major); + // RowMajor + if (result_row_major(b, patchId, c, r, d) != expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_row_major(b, patchId, c, r, d), expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + } + sycl_device.deallocate(gpu_data_col_major); + sycl_device.deallocate(gpu_data_row_major); + sycl_device.deallocate(gpu_data_result_col_major); + sycl_device.deallocate(gpu_data_result_row_major); +} + +// Verifies VALID padding (no padding) with the same value. +template <typename DataType, typename IndexType> +static void test_patch_padding_valid_same_value_sycl(const Eigen::SyclDevice& sycl_device){ + IndexType input_depth = 1; + IndexType input_rows = 5; + IndexType input_cols = 5; + IndexType input_batches = 2; + IndexType ksize = 3; // Corresponds to the Rows and Cols for tensor.extract_image_patches<>. + IndexType stride = 2; // Only same stride is supported. + // ColMajor + + array<IndexType, 4> tensorColMajorRange = {{input_depth, input_rows, input_cols, input_batches}}; + array<IndexType, 4> tensorRowMajorRange = {{input_batches, input_cols, input_rows, input_depth}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + Tensor<DataType, 4, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 4, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + gpu_col_major.device(sycl_device)=gpu_col_major.constant(11.0f); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_col_major.data(), gpu_data_col_major, (tensor_col_major.size())*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_row_major.size())*sizeof(DataType)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(0), tensor_row_major.dimension(3)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(1), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(2), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(3), tensor_row_major.dimension(0)); + + array<IndexType, 5> patchColMajorTensorRange={{input_depth, ksize, ksize, 4, input_batches}}; + Tensor<DataType, 5, DataLayout,IndexType> result_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =result_col_major.size()*sizeof(DataType); + DataType* gpu_data_result_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_result_col_major(gpu_data_result_col_major, patchColMajorTensorRange); + gpu_result_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(ksize, ksize, stride, stride, 1, 1, PADDING_VALID); + sycl_device.memcpyDeviceToHost(result_col_major.data(), gpu_data_result_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(result_col_major.dimension(0), input_depth); // depth + VERIFY_IS_EQUAL(result_col_major.dimension(1), ksize); // kernel rows + VERIFY_IS_EQUAL(result_col_major.dimension(2), ksize); // kernel cols + VERIFY_IS_EQUAL(result_col_major.dimension(3), 4); // number of patches + VERIFY_IS_EQUAL(result_col_major.dimension(4), input_batches); // number of batches + + // RowMajor + array<IndexType, 5> patchRowMajorTensorRange={{input_batches, 4, ksize, ksize, input_depth }}; + Tensor<DataType, 5, RowMajor,IndexType> result_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =result_row_major.size()*sizeof(DataType); + DataType* gpu_data_result_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_result_row_major(gpu_data_result_row_major, patchRowMajorTensorRange); + gpu_result_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(ksize, ksize, stride, stride, 1, 1, PADDING_VALID); + sycl_device.memcpyDeviceToHost(result_row_major.data(), gpu_data_result_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(result_col_major.dimension(0), result_row_major.dimension(4)); + VERIFY_IS_EQUAL(result_col_major.dimension(1), result_row_major.dimension(3)); + VERIFY_IS_EQUAL(result_col_major.dimension(2), result_row_major.dimension(2)); + VERIFY_IS_EQUAL(result_col_major.dimension(3), result_row_major.dimension(1)); + VERIFY_IS_EQUAL(result_col_major.dimension(4), result_row_major.dimension(0)); + + // No padding is carried out. + IndexType row_padding = 0; + IndexType col_padding = 0; + + for (IndexType i = 0; (i+stride+ksize-1) <= input_rows; i += stride) { // input rows + for (IndexType j = 0; (j+stride+ksize-1) <= input_cols; j += stride) { // input cols + IndexType patchId = i+input_rows*j; + for (IndexType r = 0; r < ksize; ++r) { // patch rows + for (IndexType c = 0; c < ksize; ++c) { // patch cols + for (IndexType d = 0; d < input_depth; ++d) { // depth + for (IndexType b = 0; b < input_batches; ++b) { // batch + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + IndexType row_offset = r + i - row_padding; + IndexType col_offset = c + j - col_padding; + if (row_offset >= 0 && col_offset >= 0 && row_offset < input_rows && col_offset < input_cols) { + expected_col_major = tensor_col_major(d, row_offset, col_offset, b); + expected_row_major = tensor_row_major(b, col_offset, row_offset, d); + } + // ColMajor + if (result_col_major(d, r, c, patchId, b) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_col_major(d, r, c, patchId, b), expected_col_major); + // RowMajor + if (result_row_major(b, patchId, c, r, d) != expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_row_major(b, patchId, c, r, d), expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + } +} + +// Verifies SAME padding. +template <typename DataType, typename IndexType> +static void test_patch_padding_same_sycl(const Eigen::SyclDevice& sycl_device){ + IndexType input_depth = 3; + IndexType input_rows = 4; + IndexType input_cols = 2; + IndexType input_batches = 1; + IndexType ksize = 2; // Corresponds to the Rows and Cols for tensor.extract_image_patches<>. + IndexType stride = 2; // Only same stride is supported. + + // ColMajor + array<IndexType, 4> tensorColMajorRange = {{input_depth, input_rows, input_cols, input_batches}}; + array<IndexType, 4> tensorRowMajorRange = {{input_batches, input_cols, input_rows, input_depth}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + Tensor<DataType, 4, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 4, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_col_major.size())*sizeof(DataType)); + + VERIFY_IS_EQUAL(tensor_col_major.dimension(0), tensor_row_major.dimension(3)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(1), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(2), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(3), tensor_row_major.dimension(0)); + + // Initializes tensor with incrementing numbers. + for (IndexType i = 0; i < tensor_col_major.size(); ++i) { + tensor_col_major.data()[i] = i + 1; + } + +array<IndexType, 5> patchColMajorTensorRange={{input_depth, ksize, ksize, 2, input_batches}}; +Tensor<DataType, 5, DataLayout,IndexType> result_col_major(patchColMajorTensorRange); +size_t patchTensorBuffSize =result_col_major.size()*sizeof(DataType); +DataType* gpu_data_result_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); +TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_result_col_major(gpu_data_result_col_major, patchColMajorTensorRange); +gpu_result_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(ksize, ksize, stride, stride, PADDING_SAME); +sycl_device.memcpyDeviceToHost(result_col_major.data(), gpu_data_result_col_major, patchTensorBuffSize); + + + VERIFY_IS_EQUAL(result_col_major.dimension(0), input_depth); // depth + VERIFY_IS_EQUAL(result_col_major.dimension(1), ksize); // kernel rows + VERIFY_IS_EQUAL(result_col_major.dimension(2), ksize); // kernel cols + VERIFY_IS_EQUAL(result_col_major.dimension(3), 2); // number of patches + VERIFY_IS_EQUAL(result_col_major.dimension(4), input_batches); // number of batches + + // RowMajor + + array<IndexType, 5> patchRowMajorTensorRange={{input_batches, 2, ksize, ksize, input_depth }}; + Tensor<DataType, 5, RowMajor,IndexType> result_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =result_row_major.size()*sizeof(DataType); + DataType* gpu_data_result_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_result_row_major(gpu_data_result_row_major, patchRowMajorTensorRange); + gpu_result_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(ksize, ksize, stride, stride, PADDING_SAME); + sycl_device.memcpyDeviceToHost(result_row_major.data(), gpu_data_result_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(result_col_major.dimension(0), result_row_major.dimension(4)); + VERIFY_IS_EQUAL(result_col_major.dimension(1), result_row_major.dimension(3)); + VERIFY_IS_EQUAL(result_col_major.dimension(2), result_row_major.dimension(2)); + VERIFY_IS_EQUAL(result_col_major.dimension(3), result_row_major.dimension(1)); + VERIFY_IS_EQUAL(result_col_major.dimension(4), result_row_major.dimension(0)); + + // Based on the calculation described in TensorTraits.h, padding happens to be 0. + IndexType row_padding = 0; + IndexType col_padding = 0; + + for (IndexType i = 0; (i+stride+ksize-1) <= input_rows; i += stride) { // input rows + for (IndexType j = 0; (j+stride+ksize-1) <= input_cols; j += stride) { // input cols + IndexType patchId = i+input_rows*j; + for (IndexType r = 0; r < ksize; ++r) { // patch rows + for (IndexType c = 0; c < ksize; ++c) { // patch cols + for (IndexType d = 0; d < input_depth; ++d) { // depth + for (IndexType b = 0; b < input_batches; ++b) { // batch + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + IndexType row_offset = r*stride + i - row_padding; + IndexType col_offset = c*stride + j - col_padding; + if (row_offset >= 0 && col_offset >= 0 && row_offset < input_rows && col_offset < input_cols) { + expected_col_major = tensor_col_major(d, row_offset, col_offset, b); + expected_row_major = tensor_row_major(b, col_offset, row_offset, d); + } + // ColMajor + if (result_col_major(d, r, c, patchId, b) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_col_major(d, r, c, patchId, b), expected_col_major); + // RowMajor + if (result_row_major(b, patchId, c, r, d) != expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(result_row_major(b, patchId, c, r, d), expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + } +} + + +template <typename DataType, typename IndexType> +static void test_patch_no_extra_dim_sycl(const Eigen::SyclDevice& sycl_device){ + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + + // ColMajor + array<IndexType, 3> tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + array<IndexType, 3> tensorRowMajorRange = {{sizeDim3, sizeDim2, sizeDim1}}; + Tensor<DataType, 3, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + tensor_col_major.setRandom(); + Tensor<DataType, 3, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 3, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 3, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_row_major.size())*sizeof(DataType)); + + VERIFY_IS_EQUAL(tensor_col_major.dimension(0), tensor_row_major.dimension(2)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(1), tensor_row_major.dimension(1)); + VERIFY_IS_EQUAL(tensor_col_major.dimension(2), tensor_row_major.dimension(0)); + + + // Single pixel patch: ColMajor + array<IndexType, 4> patchColMajorTensorRange={{sizeDim1, 1, 1, sizeDim2*sizeDim3}}; + Tensor<DataType, 4, DataLayout,IndexType> single_patch_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =single_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_single_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_single_patch_col_major(gpu_data_single_patch_col_major, patchColMajorTensorRange); + gpu_single_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(1, 1); + sycl_device.memcpyDeviceToHost(single_patch_col_major.data(), gpu_data_single_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(single_patch_col_major.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(1), 1); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_patch_col_major.dimension(3), sizeDim2*sizeDim3); + + // Single pixel patch: RowMajor + array<IndexType, 4> patchRowMajorTensorRange={{sizeDim2*sizeDim3, 1, 1, sizeDim1}}; + Tensor<DataType, 4, RowMajor,IndexType> single_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =single_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_single_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 4, RowMajor,IndexType>> gpu_single_patch_row_major(gpu_data_single_patch_row_major, patchRowMajorTensorRange); + gpu_single_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(1, 1); + sycl_device.memcpyDeviceToHost(single_patch_row_major.data(), gpu_data_single_patch_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(single_patch_row_major.dimension(0), sizeDim2*sizeDim3); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(1), 1); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_patch_row_major.dimension(3), sizeDim1); + + for (IndexType i = 0; i < tensor_col_major.size(); ++i) { + // ColMajor + if (tensor_col_major.data()[i] != single_patch_col_major.data()[i]) { + std::cout << "Mismatch detected at index " << i << " : " << tensor_col_major.data()[i] << " vs " << single_patch_col_major.data()[i] << std::endl; + } + VERIFY_IS_EQUAL(single_patch_col_major.data()[i], tensor_col_major.data()[i]); + // RowMajor + if (tensor_row_major.data()[i] != single_patch_row_major.data()[i]) { + std::cout << "Mismatch detected at index " << i << " : " + << tensor_col_major.data()[i] << " vs " + << single_patch_row_major.data()[i] << std::endl; + } + VERIFY_IS_EQUAL(single_patch_row_major.data()[i], + tensor_row_major.data()[i]); + VERIFY_IS_EQUAL(tensor_col_major.data()[i], tensor_row_major.data()[i]); + VERIFY_IS_EQUAL(single_patch_col_major.data()[i], + single_patch_row_major.data()[i]); + } + + // Entire image patch: ColMajor + patchColMajorTensorRange={{sizeDim1, sizeDim2, sizeDim3, sizeDim2*sizeDim3}}; + Tensor<DataType, 4, DataLayout,IndexType> entire_image_patch_col_major(patchColMajorTensorRange); + patchTensorBuffSize =entire_image_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_entire_image_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_entire_image_patch_col_major(gpu_data_entire_image_patch_col_major, patchColMajorTensorRange); + gpu_entire_image_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(3, 5); + sycl_device.memcpyDeviceToHost(entire_image_patch_col_major.data(), gpu_data_entire_image_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(0), 2); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(1), 3); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(2), 5); + VERIFY_IS_EQUAL(entire_image_patch_col_major.dimension(3), 3*5); + + // Entire image patch: RowMajor +patchRowMajorTensorRange={{sizeDim2*sizeDim3, sizeDim3, sizeDim2, sizeDim1}}; +Tensor<DataType, 4, RowMajor,IndexType> entire_image_patch_row_major(patchRowMajorTensorRange); +patchTensorBuffSize =entire_image_patch_row_major.size()*sizeof(DataType); +DataType* gpu_data_entire_image_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); +TensorMap<Tensor<DataType, 4, RowMajor,IndexType>> gpu_entire_image_patch_row_major(gpu_data_entire_image_patch_row_major, patchRowMajorTensorRange); +gpu_entire_image_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(3, 5); +sycl_device.memcpyDeviceToHost(entire_image_patch_row_major.data(), gpu_data_entire_image_patch_row_major, patchTensorBuffSize); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(0), 3*5); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(1), 5); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(2), 3); + VERIFY_IS_EQUAL(entire_image_patch_row_major.dimension(3), 2); + + for (IndexType i = 0; i < 3; ++i) { + for (IndexType j = 0; j < 5; ++j) { + IndexType patchId = i+3*j; + for (IndexType r = 0; r < 3; ++r) { + for (IndexType c = 0; c < 5; ++c) { + for (IndexType d = 0; d < 2; ++d) { + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + if (r-1+i >= 0 && c-2+j >= 0 && r-1+i < 3 && c-2+j < 5) { + expected_col_major = tensor_col_major(d, r-1+i, c-2+j); + expected_row_major = tensor_row_major(c-2+j, r-1+i, d); + } + // ColMajor + if (entire_image_patch_col_major(d, r, c, patchId) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << std::endl; + } + VERIFY_IS_EQUAL(entire_image_patch_col_major(d, r, c, patchId), expected_col_major); + // RowMajor + if (entire_image_patch_row_major(patchId, c, r, d) != + expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << std::endl; + } + VERIFY_IS_EQUAL(entire_image_patch_row_major(patchId, c, r, d), + expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + + // 2D patch: ColMajor + patchColMajorTensorRange={{sizeDim1, 2, 2, sizeDim2*sizeDim3}}; + Tensor<DataType, 4, DataLayout,IndexType> twod_patch_col_major(patchColMajorTensorRange); + patchTensorBuffSize =twod_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_twod_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_twod_patch_col_major(gpu_data_twod_patch_col_major, patchColMajorTensorRange); + gpu_twod_patch_col_major.device(sycl_device)=gpu_col_major.extract_image_patches(2, 2); + sycl_device.memcpyDeviceToHost(twod_patch_col_major.data(), gpu_data_twod_patch_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(0), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(1), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch_col_major.dimension(3), 3*5); + + // 2D patch: RowMajor + patchRowMajorTensorRange={{sizeDim2*sizeDim3, 2, 2, sizeDim1}}; + Tensor<DataType, 4, RowMajor,IndexType> twod_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =twod_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_twod_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 4, RowMajor,IndexType>> gpu_twod_patch_row_major(gpu_data_twod_patch_row_major, patchRowMajorTensorRange); + gpu_twod_patch_row_major.device(sycl_device)=gpu_row_major.extract_image_patches(2, 2); + sycl_device.memcpyDeviceToHost(twod_patch_row_major.data(), gpu_data_twod_patch_row_major, patchTensorBuffSize); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(0), 3*5); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(1), 2); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch_row_major.dimension(3), 2); + + // Based on the calculation described in TensorTraits.h, padding happens to be 0. + IndexType row_padding = 0; + IndexType col_padding = 0; + IndexType stride = 1; + + for (IndexType i = 0; i < 3; ++i) { + for (IndexType j = 0; j < 5; ++j) { + IndexType patchId = i+3*j; + for (IndexType r = 0; r < 2; ++r) { + for (IndexType c = 0; c < 2; ++c) { + for (IndexType d = 0; d < 2; ++d) { + DataType expected_col_major = 0.0f; + DataType expected_row_major = 0.0f; + IndexType row_offset = r*stride + i - row_padding; + IndexType col_offset = c*stride + j - col_padding; + // ColMajor + if (row_offset >= 0 && col_offset >= 0 && row_offset < tensor_col_major.dimension(1) && col_offset < tensor_col_major.dimension(2)) { + expected_col_major = tensor_col_major(d, row_offset, col_offset); + } + if (twod_patch_col_major(d, r, c, patchId) != expected_col_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << std::endl; + } + VERIFY_IS_EQUAL(twod_patch_col_major(d, r, c, patchId), expected_col_major); + // RowMajor + if (row_offset >= 0 && col_offset >= 0 && row_offset < tensor_row_major.dimension(1) && col_offset < tensor_row_major.dimension(0)) { + expected_row_major = tensor_row_major(col_offset, row_offset, d); + } + if (twod_patch_row_major(patchId, c, r, d) != expected_row_major) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << std::endl; + } + VERIFY_IS_EQUAL(twod_patch_row_major(patchId, c, r, d), expected_row_major); + // Check that ColMajor and RowMajor agree. + VERIFY_IS_EQUAL(expected_col_major, expected_row_major); + } + } + } + } + } + + sycl_device.deallocate(gpu_data_col_major); + sycl_device.deallocate(gpu_data_row_major); + sycl_device.deallocate(gpu_data_single_patch_col_major); + sycl_device.deallocate(gpu_data_single_patch_row_major); + sycl_device.deallocate(gpu_data_entire_image_patch_col_major); + sycl_device.deallocate(gpu_data_entire_image_patch_row_major); + sycl_device.deallocate(gpu_data_twod_patch_col_major); + sycl_device.deallocate(gpu_data_twod_patch_row_major); +} + +template <typename DataType, typename IndexType> +static void test_imagenet_patches_sycl(const Eigen::SyclDevice& sycl_device) +{ + // Test the code on typical configurations used by the 'imagenet' benchmarks at + // https://github.com/soumith/convnet-benchmarks + // ColMajor + IndexType sizeDim1 = 3; + IndexType sizeDim2 = 128; + IndexType sizeDim3 = 128; + IndexType sizeDim4 = 16; + array<IndexType, 4> tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> l_in_col_major(tensorColMajorRange); + l_in_col_major.setRandom(); + + DataType* gpu_data_l_in_col_major = static_cast<DataType*>(sycl_device.allocate(l_in_col_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>> gpu_l_in_col_major(gpu_data_l_in_col_major, tensorColMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_l_in_col_major, l_in_col_major.data(),(l_in_col_major.size())*sizeof(DataType)); + + array<IndexType, 5> patchTensorRange={{sizeDim1, 11, 11, sizeDim2*sizeDim3, sizeDim4}}; + Tensor<DataType, 5, DataLayout,IndexType> l_out_col_major(patchTensorRange); + size_t patchTensorBuffSize =l_out_col_major.size()*sizeof(DataType); + DataType* gpu_data_l_out_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_l_out_col_major(gpu_data_l_out_col_major, patchTensorRange); + gpu_l_out_col_major.device(sycl_device)=gpu_l_in_col_major.extract_image_patches(11, 11); + sycl_device.memcpyDeviceToHost(l_out_col_major.data(), gpu_data_l_out_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_col_major.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(l_out_col_major.dimension(1), 11); + VERIFY_IS_EQUAL(l_out_col_major.dimension(2), 11); + VERIFY_IS_EQUAL(l_out_col_major.dimension(3), sizeDim2*sizeDim3); + VERIFY_IS_EQUAL(l_out_col_major.dimension(4), sizeDim4); + + // RowMajor + patchTensorRange={{sizeDim4, sizeDim2*sizeDim3, 11, 11, sizeDim1}}; + Tensor<DataType, 5, RowMajor,IndexType> l_out_row_major(patchTensorRange); + patchTensorBuffSize =l_out_row_major.size()*sizeof(DataType); + DataType* gpu_data_l_out_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>> gpu_l_out_row_major(gpu_data_l_out_row_major, patchTensorRange); + gpu_l_out_row_major.device(sycl_device)=gpu_l_in_col_major.swap_layout().extract_image_patches(11, 11); + sycl_device.memcpyDeviceToHost(l_out_row_major.data(), gpu_data_l_out_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_row_major.dimension(0), sizeDim4); + VERIFY_IS_EQUAL(l_out_row_major.dimension(1), sizeDim2*sizeDim3); + VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 11); + VERIFY_IS_EQUAL(l_out_row_major.dimension(3), 11); + VERIFY_IS_EQUAL(l_out_row_major.dimension(4), sizeDim1); + + for (IndexType b = 0; b < 16; ++b) { + for (IndexType i = 0; i < 128; ++i) { + for (IndexType j = 0; j < 128; ++j) { + IndexType patchId = i+128*j; + for (IndexType c = 0; c < 11; ++c) { + for (IndexType r = 0; r < 11; ++r) { + for (IndexType d = 0; d < 3; ++d) { + DataType expected = 0.0f; + if (r-5+i >= 0 && c-5+j >= 0 && r-5+i < 128 && c-5+j < 128) { + expected = l_in_col_major(d, r-5+i, c-5+j, b); + } + // ColMajor + if (l_out_col_major(d, r, c, patchId, b) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_col_major(d, r, c, patchId, b), expected); + // RowMajor + if (l_out_row_major(b, patchId, c, r, d) != + expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j + << " r=" << r << " c=" << c << " d=" << d << " b=" << b + << std::endl; + } + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), + expected); + } + } + } + } + } + } + + // ColMajor + sycl_device.deallocate(gpu_data_l_in_col_major); + sycl_device.deallocate(gpu_data_l_out_col_major); + sizeDim1 = 16; + sizeDim2 = 64; + sizeDim3 = 64; + sizeDim4 = 32; + tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + l_in_col_major.resize(tensorColMajorRange); + l_in_col_major.setRandom(); + gpu_data_l_in_col_major = static_cast<DataType*>(sycl_device.allocate(l_in_col_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>>gpu_l_in_col_major_resize1(gpu_data_l_in_col_major, tensorColMajorRange); + + patchTensorRange={{sizeDim1, 9, 9, sizeDim2*sizeDim3, sizeDim4}}; + l_out_col_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_col_major.size()*sizeof(DataType); + gpu_data_l_out_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>>gpu_l_out_col_major_resize1(gpu_data_l_out_col_major, patchTensorRange); + sycl_device.memcpyHostToDevice(gpu_data_l_in_col_major, l_in_col_major.data(),(l_in_col_major.size())*sizeof(DataType)); + gpu_l_out_col_major_resize1.device(sycl_device)=gpu_l_in_col_major_resize1.extract_image_patches(9, 9); + sycl_device.memcpyDeviceToHost(l_out_col_major.data(), gpu_data_l_out_col_major, patchTensorBuffSize); + VERIFY_IS_EQUAL(l_out_col_major.dimension(0), 16); + VERIFY_IS_EQUAL(l_out_col_major.dimension(1), 9); + VERIFY_IS_EQUAL(l_out_col_major.dimension(2), 9); + VERIFY_IS_EQUAL(l_out_col_major.dimension(3), 64*64); + VERIFY_IS_EQUAL(l_out_col_major.dimension(4), 32); + +// RowMajor + sycl_device.deallocate(gpu_data_l_out_row_major); + patchTensorRange={{sizeDim4, sizeDim2*sizeDim3, 9, 9 ,sizeDim1}}; + l_out_row_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_row_major.size()*sizeof(DataType); + gpu_data_l_out_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>>gpu_l_out_row_major_resize1(gpu_data_l_out_row_major, patchTensorRange); + gpu_l_out_row_major_resize1.device(sycl_device)=gpu_l_in_col_major_resize1.swap_layout().extract_image_patches(9, 9); + sycl_device.memcpyDeviceToHost(l_out_row_major.data(), gpu_data_l_out_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); + VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 64*64); + VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 9); + VERIFY_IS_EQUAL(l_out_row_major.dimension(3), 9); + VERIFY_IS_EQUAL(l_out_row_major.dimension(4), 16); + + for (IndexType b = 0; b < 32; ++b) { + for (IndexType i = 0; i < 64; ++i) { + for (IndexType j = 0; j < 64; ++j) { + IndexType patchId = i+64*j; + for (IndexType c = 0; c < 9; ++c) { + for (IndexType r = 0; r < 9; ++r) { + for (IndexType d = 0; d < 16; ++d) { + DataType expected = 0.0f; + if (r-4+i >= 0 && c-4+j >= 0 && r-4+i < 64 && c-4+j < 64) { + expected = l_in_col_major(d, r-4+i, c-4+j, b); + } + // ColMajor + if (l_out_col_major(d, r, c, patchId, b) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_col_major(d, r, c, patchId, b), expected); + // RowMajor + if (l_out_row_major(b, patchId, c, r, d) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); + } + } + } + } + } + } + + // ColMajor + + sycl_device.deallocate(gpu_data_l_in_col_major); + sycl_device.deallocate(gpu_data_l_out_col_major); + sizeDim1 = 32; + sizeDim2 = 16; + sizeDim3 = 16; + sizeDim4 = 32; + tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + l_in_col_major.resize(tensorColMajorRange); + l_in_col_major.setRandom(); + gpu_data_l_in_col_major = static_cast<DataType*>(sycl_device.allocate(l_in_col_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>>gpu_l_in_col_major_resize2(gpu_data_l_in_col_major, tensorColMajorRange); + + patchTensorRange={{sizeDim1, 7, 7, sizeDim2*sizeDim3, sizeDim4}}; + l_out_col_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_col_major.size()*sizeof(DataType); + gpu_data_l_out_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>>gpu_l_out_col_major_resize2(gpu_data_l_out_col_major, patchTensorRange); + sycl_device.memcpyHostToDevice(gpu_data_l_in_col_major, l_in_col_major.data(),(l_in_col_major.size())*sizeof(DataType)); + gpu_l_out_col_major_resize2.device(sycl_device)=gpu_l_in_col_major_resize2.extract_image_patches(7, 7); + sycl_device.memcpyDeviceToHost(l_out_col_major.data(), gpu_data_l_out_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_col_major.dimension(0), 32); + VERIFY_IS_EQUAL(l_out_col_major.dimension(1), 7); + VERIFY_IS_EQUAL(l_out_col_major.dimension(2), 7); + VERIFY_IS_EQUAL(l_out_col_major.dimension(3), 16*16); + VERIFY_IS_EQUAL(l_out_col_major.dimension(4), 32); + + // RowMajor + sycl_device.deallocate(gpu_data_l_out_row_major); + patchTensorRange={{sizeDim4, sizeDim2*sizeDim3, 7, 7 ,sizeDim1}}; + l_out_row_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_row_major.size()*sizeof(DataType); + gpu_data_l_out_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>>gpu_l_out_row_major_resize2(gpu_data_l_out_row_major, patchTensorRange); + gpu_l_out_row_major_resize2.device(sycl_device)=gpu_l_in_col_major_resize2.swap_layout().extract_image_patches(7, 7); + sycl_device.memcpyDeviceToHost(l_out_row_major.data(), gpu_data_l_out_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); + VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 16*16); + VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 7); + VERIFY_IS_EQUAL(l_out_row_major.dimension(3), 7); + VERIFY_IS_EQUAL(l_out_row_major.dimension(4), 32); + + for (IndexType b = 0; b < 32; ++b) { + for (IndexType i = 0; i < 16; ++i) { + for (IndexType j = 0; j < 16; ++j) { + IndexType patchId = i+16*j; + for (IndexType c = 0; c < 7; ++c) { + for (IndexType r = 0; r < 7; ++r) { + for (IndexType d = 0; d < 32; ++d) { + DataType expected = 0.0f; + if (r-3+i >= 0 && c-3+j >= 0 && r-3+i < 16 && c-3+j < 16) { + expected = l_in_col_major(d, r-3+i, c-3+j, b); + } + // ColMajor + if (l_out_col_major(d, r, c, patchId, b) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_col_major(d, r, c, patchId, b), expected); + // RowMajor + if (l_out_row_major(b, patchId, c, r, d) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); + } + } + } + } + } + } + + // ColMajor + sycl_device.deallocate(gpu_data_l_in_col_major); + sycl_device.deallocate(gpu_data_l_out_col_major); + sizeDim1 = 64; + sizeDim2 = 13; + sizeDim3 = 13; + sizeDim4 = 32; + tensorColMajorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + l_in_col_major.resize(tensorColMajorRange); + l_in_col_major.setRandom(); + gpu_data_l_in_col_major = static_cast<DataType*>(sycl_device.allocate(l_in_col_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4, ColMajor, IndexType>>gpu_l_in_col_major_resize3(gpu_data_l_in_col_major, tensorColMajorRange); + + patchTensorRange={{sizeDim1, 3, 3, sizeDim2*sizeDim3, sizeDim4}}; + l_out_col_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_col_major.size()*sizeof(DataType); + gpu_data_l_out_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>>gpu_l_out_col_major_resize3(gpu_data_l_out_col_major, patchTensorRange); + sycl_device.memcpyHostToDevice(gpu_data_l_in_col_major, l_in_col_major.data(),(l_in_col_major.size())*sizeof(DataType)); + gpu_l_out_col_major_resize3.device(sycl_device)=gpu_l_in_col_major_resize3.extract_image_patches(3, 3); + sycl_device.memcpyDeviceToHost(l_out_col_major.data(), gpu_data_l_out_col_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_col_major.dimension(0), 64); + VERIFY_IS_EQUAL(l_out_col_major.dimension(1), 3); + VERIFY_IS_EQUAL(l_out_col_major.dimension(2), 3); + VERIFY_IS_EQUAL(l_out_col_major.dimension(3), 13*13); + VERIFY_IS_EQUAL(l_out_col_major.dimension(4), 32); + + // RowMajor + sycl_device.deallocate(gpu_data_l_out_row_major); + patchTensorRange={{sizeDim4, sizeDim2*sizeDim3, 3, 3 ,sizeDim1}}; + l_out_row_major.resize(patchTensorRange); + patchTensorBuffSize =l_out_row_major.size()*sizeof(DataType); + gpu_data_l_out_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, RowMajor,IndexType>>gpu_l_out_row_major_resize3(gpu_data_l_out_row_major, patchTensorRange); + gpu_l_out_row_major_resize3.device(sycl_device)=gpu_l_in_col_major_resize3.swap_layout().extract_image_patches(3, 3); + sycl_device.memcpyDeviceToHost(l_out_row_major.data(), gpu_data_l_out_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(l_out_row_major.dimension(0), 32); + VERIFY_IS_EQUAL(l_out_row_major.dimension(1), 13*13); + VERIFY_IS_EQUAL(l_out_row_major.dimension(2), 3); + VERIFY_IS_EQUAL(l_out_row_major.dimension(3), 3); + VERIFY_IS_EQUAL(l_out_row_major.dimension(4), 64); + + for (IndexType b = 0; b < 32; ++b) { + for (IndexType i = 0; i < 13; ++i) { + for (IndexType j = 0; j < 13; ++j) { + IndexType patchId = i+13*j; + for (IndexType c = 0; c < 3; ++c) { + for (IndexType r = 0; r < 3; ++r) { + for (IndexType d = 0; d < 64; ++d) { + DataType expected = 0.0f; + if (r-1+i >= 0 && c-1+j >= 0 && r-1+i < 13 && c-1+j < 13) { + expected = l_in_col_major(d, r-1+i, c-1+j, b); + } + // ColMajor + if (l_out_col_major(d, r, c, patchId, b) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_col_major(d, r, c, patchId, b), expected); + // RowMajor + if (l_out_row_major(b, patchId, c, r, d) != expected) { + std::cout << "Mismatch detected at index i=" << i << " j=" << j << " r=" << r << " c=" << c << " d=" << d << " b=" << b << std::endl; + } + VERIFY_IS_EQUAL(l_out_row_major(b, patchId, c, r, d), expected); + } + } + } + } + } + } + sycl_device.deallocate(gpu_data_l_in_col_major); + sycl_device.deallocate(gpu_data_l_out_col_major); + sycl_device.deallocate(gpu_data_l_out_row_major); +} + + +template<typename DataType, typename dev_Selector> void sycl_tensor_image_patch_test_per_device(dev_Selector s){ +QueueInterface queueInterface(s); +auto sycl_device = Eigen::SyclDevice(&queueInterface); +test_simple_image_patch_sycl<DataType, int64_t>(sycl_device); +test_patch_padding_valid_sycl<DataType, int64_t>(sycl_device); +test_patch_padding_valid_same_value_sycl<DataType, int64_t>(sycl_device); +test_patch_padding_same_sycl<DataType, int64_t>(sycl_device); +test_patch_no_extra_dim_sycl<DataType, int64_t>(sycl_device); +test_imagenet_patches_sycl<DataType, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_image_patch_sycl) +{ +for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_tensor_image_patch_test_per_device<float>(device)); +} +}
diff --git a/unsupported/test/cxx11_tensor_index_list.cpp b/unsupported/test/cxx11_tensor_index_list.cpp index 4ce8dea..2166532 100644 --- a/unsupported/test/cxx11_tensor_index_list.cpp +++ b/unsupported/test/cxx11_tensor_index_list.cpp
@@ -22,9 +22,9 @@ VERIFY_IS_EQUAL(internal::array_get<0>(reduction_axis), 0); VERIFY_IS_EQUAL(internal::array_get<1>(reduction_axis), 1); VERIFY_IS_EQUAL(internal::array_get<2>(reduction_axis), 2); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[0]), 0); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[1]), 1); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[2]), 2); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[0]), 0); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[1]), 1); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[2]), 2); EIGEN_STATIC_ASSERT((internal::array_get<0>(reduction_axis) == 0), YOU_MADE_A_PROGRAMMING_MISTAKE); EIGEN_STATIC_ASSERT((internal::array_get<1>(reduction_axis) == 1), YOU_MADE_A_PROGRAMMING_MISTAKE); @@ -159,6 +159,110 @@ } +static void test_type2indexpair_list() +{ + Tensor<float, 5> tensor(2,3,5,7,11); + tensor.setRandom(); + tensor += tensor.constant(10.0f); + + typedef Eigen::IndexPairList<Eigen::type2indexpair<0,10>> Dims0; + typedef Eigen::IndexPairList<Eigen::type2indexpair<0,10>, Eigen::type2indexpair<1,11>, Eigen::type2indexpair<2,12>> Dims2_a; + typedef Eigen::IndexPairList<Eigen::type2indexpair<0,10>, Eigen::IndexPair<Index>, Eigen::type2indexpair<2,12>> Dims2_b; + typedef Eigen::IndexPairList<Eigen::IndexPair<Index>, Eigen::type2indexpair<1,11>, Eigen::IndexPair<Index>> Dims2_c; + + Dims2_a d2_a; + + Dims2_b d2_b; + d2_b.set(1, Eigen::IndexPair<Index>(1,11)); + + Dims2_c d2_c; + d2_c.set(0, Eigen::IndexPair<Index>(Eigen::IndexPair<Index>(0,10))); + d2_c.set(1, Eigen::IndexPair<Index>(1,11)); // setting type2indexpair to correct value. + d2_c.set(2, Eigen::IndexPair<Index>(2,12)); + + VERIFY_IS_EQUAL(d2_a[0].first, 0); + VERIFY_IS_EQUAL(d2_a[0].second, 10); + VERIFY_IS_EQUAL(d2_a[1].first, 1); + VERIFY_IS_EQUAL(d2_a[1].second, 11); + VERIFY_IS_EQUAL(d2_a[2].first, 2); + VERIFY_IS_EQUAL(d2_a[2].second, 12); + + VERIFY_IS_EQUAL(d2_b[0].first, 0); + VERIFY_IS_EQUAL(d2_b[0].second, 10); + VERIFY_IS_EQUAL(d2_b[1].first, 1); + VERIFY_IS_EQUAL(d2_b[1].second, 11); + VERIFY_IS_EQUAL(d2_b[2].first, 2); + VERIFY_IS_EQUAL(d2_b[2].second, 12); + + VERIFY_IS_EQUAL(d2_c[0].first, 0); + VERIFY_IS_EQUAL(d2_c[0].second, 10); + VERIFY_IS_EQUAL(d2_c[1].first, 1); + VERIFY_IS_EQUAL(d2_c[1].second, 11); + VERIFY_IS_EQUAL(d2_c[2].first, 2); + VERIFY_IS_EQUAL(d2_c[2].second, 12); + + EIGEN_STATIC_ASSERT((d2_a.value_known_statically(0) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_a.value_known_statically(1) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_a.value_known_statically(2) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((d2_b.value_known_statically(0) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_b.value_known_statically(1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_b.value_known_statically(2) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((d2_c.value_known_statically(0) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_c.value_known_statically(1) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((d2_c.value_known_statically(2) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims0>(0, 0) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims0>(0, 1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(0, 0) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(0, 1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(1, 1) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(1, 2) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(2, 2) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_a>(2, 3) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(0, 0) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(0, 1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(1, 1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(1, 2) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(2, 2) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_b>(2, 3) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(0, 0) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(0, 1) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(1, 1) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(1, 2) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(2, 2) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_first_statically_eq<Dims2_c>(2, 3) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims0>(0, 10) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims0>(0, 11) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(0, 10) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(0, 11) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(1, 11) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(1, 12) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(2, 12) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_a>(2, 13) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(0, 10) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(0, 11) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(1, 11) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(1, 12) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(2, 12) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_b>(2, 13) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(0, 10) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(0, 11) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(1, 11) == true), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(1, 12) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(2, 12) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((Eigen::internal::index_pair_second_statically_eq<Dims2_c>(2, 13) == false), YOU_MADE_A_PROGRAMMING_MISTAKE); +} + + static void test_dynamic_index_list() { Tensor<float, 4> tensor(2,3,5,7); @@ -173,9 +277,9 @@ VERIFY_IS_EQUAL(internal::array_get<0>(reduction_axis), 2); VERIFY_IS_EQUAL(internal::array_get<1>(reduction_axis), 1); VERIFY_IS_EQUAL(internal::array_get<2>(reduction_axis), 0); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[0]), 2); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[1]), 1); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[2]), 0); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[0]), 2); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[1]), 1); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[2]), 0); Tensor<float, 1> result = tensor.sum(reduction_axis); for (int i = 0; i < result.size(); ++i) { @@ -205,10 +309,10 @@ VERIFY_IS_EQUAL(internal::array_get<1>(reduction_axis), 1); VERIFY_IS_EQUAL(internal::array_get<2>(reduction_axis), 2); VERIFY_IS_EQUAL(internal::array_get<3>(reduction_axis), 3); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[0]), 0); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[1]), 1); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[2]), 2); - VERIFY_IS_EQUAL(static_cast<DenseIndex>(reduction_axis[3]), 3); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[0]), 0); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[1]), 1); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[2]), 2); + VERIFY_IS_EQUAL(static_cast<Index>(reduction_axis[3]), 3); typedef IndexList<type2index<0>, int, type2index<2>, int> ReductionIndices; ReductionIndices reduction_indices; @@ -268,11 +372,12 @@ #endif -void test_cxx11_tensor_index_list() +EIGEN_DECLARE_TEST(cxx11_tensor_index_list) { #ifdef EIGEN_HAS_INDEX_LIST CALL_SUBTEST(test_static_index_list()); CALL_SUBTEST(test_type2index_list()); + CALL_SUBTEST(test_type2indexpair_list()); CALL_SUBTEST(test_dynamic_index_list()); CALL_SUBTEST(test_mixed_index_list()); CALL_SUBTEST(test_dim_check());
diff --git a/unsupported/test/cxx11_tensor_inflation.cpp b/unsupported/test/cxx11_tensor_inflation.cpp index 4997935..75089e8 100644 --- a/unsupported/test/cxx11_tensor_inflation.cpp +++ b/unsupported/test/cxx11_tensor_inflation.cpp
@@ -74,7 +74,7 @@ } } -void test_cxx11_tensor_inflation() +EIGEN_DECLARE_TEST(cxx11_tensor_inflation) { CALL_SUBTEST(test_simple_inflation<ColMajor>()); CALL_SUBTEST(test_simple_inflation<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_inflation_sycl.cpp b/unsupported/test/cxx11_tensor_inflation_sycl.cpp new file mode 100644 index 0000000..521ae0c --- /dev/null +++ b/unsupported/test/cxx11_tensor_inflation_sycl.cpp
@@ -0,0 +1,136 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +// Inflation Definition for each dimension the inflated val would be +//((dim-1)*strid[dim] +1) + +// for 1 dimension vector of size 3 with value (4,4,4) with the inflated stride value of 3 would be changed to +// tensor of size (2*3) +1 = 7 with the value of +// (4, 0, 0, 4, 0, 0, 4). + +template <typename DataType, int DataLayout, typename IndexType> +void test_simple_inflation_sycl(const Eigen::SyclDevice &sycl_device) { + + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout,IndexType> no_stride(tensorRange); + tensor.setRandom(); + + array<IndexType, 4> strides; + strides[0] = 1; + strides[1] = 1; + strides[2] = 1; + strides[3] = 1; + + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_no_stride = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_no_stride(gpu_data_no_stride, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_tensor, tensor.data(), tensorBuffSize); + gpu_no_stride.device(sycl_device)=gpu_tensor.inflate(strides); + sycl_device.memcpyDeviceToHost(no_stride.data(), gpu_data_no_stride, tensorBuffSize); + + VERIFY_IS_EQUAL(no_stride.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(no_stride.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(no_stride.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(no_stride.dimension(3), sizeDim4); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), no_stride(i,j,k,l)); + } + } + } + } + + + strides[0] = 2; + strides[1] = 4; + strides[2] = 2; + strides[3] = 3; + + IndexType inflatedSizeDim1 = 3; + IndexType inflatedSizeDim2 = 9; + IndexType inflatedSizeDim3 = 9; + IndexType inflatedSizeDim4 = 19; + array<IndexType, 4> inflatedTensorRange = {{inflatedSizeDim1, inflatedSizeDim2, inflatedSizeDim3, inflatedSizeDim4}}; + + Tensor<DataType, 4, DataLayout, IndexType> inflated(inflatedTensorRange); + + const size_t inflatedTensorBuffSize =inflated.size()*sizeof(DataType); + DataType* gpu_data_inflated = static_cast<DataType*>(sycl_device.allocate(inflatedTensorBuffSize)); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType>> gpu_inflated(gpu_data_inflated, inflatedTensorRange); + gpu_inflated.device(sycl_device)=gpu_tensor.inflate(strides); + sycl_device.memcpyDeviceToHost(inflated.data(), gpu_data_inflated, inflatedTensorBuffSize); + + VERIFY_IS_EQUAL(inflated.dimension(0), inflatedSizeDim1); + VERIFY_IS_EQUAL(inflated.dimension(1), inflatedSizeDim2); + VERIFY_IS_EQUAL(inflated.dimension(2), inflatedSizeDim3); + VERIFY_IS_EQUAL(inflated.dimension(3), inflatedSizeDim4); + + for (IndexType i = 0; i < inflatedSizeDim1; ++i) { + for (IndexType j = 0; j < inflatedSizeDim2; ++j) { + for (IndexType k = 0; k < inflatedSizeDim3; ++k) { + for (IndexType l = 0; l < inflatedSizeDim4; ++l) { + if (i % strides[0] == 0 && + j % strides[1] == 0 && + k % strides[2] == 0 && + l % strides[3] == 0) { + VERIFY_IS_EQUAL(inflated(i,j,k,l), + tensor(i/strides[0], j/strides[1], k/strides[2], l/strides[3])); + } else { + VERIFY_IS_EQUAL(0, inflated(i,j,k,l)); + } + } + } + } + } + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_no_stride); + sycl_device.deallocate(gpu_data_inflated); +} + +template<typename DataType, typename dev_Selector> void sycl_inflation_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_inflation_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_simple_inflation_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_inflation_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_inflation_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_intdiv.cpp b/unsupported/test/cxx11_tensor_intdiv.cpp index 48aa6d3..d18a05e 100644 --- a/unsupported/test/cxx11_tensor_intdiv.cpp +++ b/unsupported/test/cxx11_tensor_intdiv.cpp
@@ -128,14 +128,14 @@ void test_specific() { // A particular combination that was previously failing int64_t div = 209715200; - int64_t num = 3238002688; + int64_t num = 3238002688ll; Eigen::internal::TensorIntDivisor<int64_t> divider(div); int64_t result = num/div; int64_t result_op = divider.divide(num); VERIFY_IS_EQUAL(result, result_op); } -void test_cxx11_tensor_intdiv() +EIGEN_DECLARE_TEST(cxx11_tensor_intdiv) { CALL_SUBTEST_1(test_signed_32bit()); CALL_SUBTEST_2(test_unsigned_32bit());
diff --git a/unsupported/test/cxx11_tensor_io.cpp b/unsupported/test/cxx11_tensor_io.cpp index 8bbcf70..2c638f9 100644 --- a/unsupported/test/cxx11_tensor_io.cpp +++ b/unsupported/test/cxx11_tensor_io.cpp
@@ -14,6 +14,20 @@ template<int DataLayout> +static void test_output_0d() +{ + Tensor<int, 0, DataLayout> tensor; + tensor() = 123; + + std::stringstream os; + os << tensor; + + std::string expected("123"); + VERIFY_IS_EQUAL(std::string(os.str()), expected); +} + + +template<int DataLayout> static void test_output_1d() { Tensor<int, 1, DataLayout> tensor(5); @@ -26,6 +40,12 @@ std::string expected("0\n1\n2\n3\n4"); VERIFY_IS_EQUAL(std::string(os.str()), expected); + + Eigen::Tensor<double,1,DataLayout> empty_tensor(0); + std::stringstream empty_os; + empty_os << empty_tensor; + std::string empty_string; + VERIFY_IS_EQUAL(std::string(empty_os.str()), empty_string); } @@ -99,8 +119,10 @@ } -void test_cxx11_tensor_io() +EIGEN_DECLARE_TEST(cxx11_tensor_io) { + CALL_SUBTEST(test_output_0d<ColMajor>()); + CALL_SUBTEST(test_output_0d<RowMajor>()); CALL_SUBTEST(test_output_1d<ColMajor>()); CALL_SUBTEST(test_output_1d<RowMajor>()); CALL_SUBTEST(test_output_2d<ColMajor>());
diff --git a/unsupported/test/cxx11_tensor_layout_swap.cpp b/unsupported/test/cxx11_tensor_layout_swap.cpp index ae297a9..efb3333 100644 --- a/unsupported/test/cxx11_tensor_layout_swap.cpp +++ b/unsupported/test/cxx11_tensor_layout_swap.cpp
@@ -54,7 +54,7 @@ } -void test_cxx11_tensor_layout_swap() +EIGEN_DECLARE_TEST(cxx11_tensor_layout_swap) { CALL_SUBTEST(test_simple_swap()); CALL_SUBTEST(test_swap_as_lvalue());
diff --git a/unsupported/test/cxx11_tensor_layout_swap_sycl.cpp b/unsupported/test/cxx11_tensor_layout_swap_sycl.cpp new file mode 100644 index 0000000..9546b91 --- /dev/null +++ b/unsupported/test/cxx11_tensor_layout_swap_sycl.cpp
@@ -0,0 +1,126 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template <typename DataType, typename IndexType> +static void test_simple_swap_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 7; + array<IndexType, 3> tensorColRange = {{sizeDim1, sizeDim2, sizeDim3}}; + array<IndexType, 3> tensorRowRange = {{sizeDim3, sizeDim2, sizeDim1}}; + + + Tensor<DataType, 3, ColMajor, IndexType> tensor1(tensorColRange); + Tensor<DataType, 3, RowMajor, IndexType> tensor2(tensorRowRange); + tensor1.setRandom(); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor1.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(tensor2.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 3, ColMajor, IndexType>> gpu1(gpu_data1, tensorColRange); + TensorMap<Tensor<DataType, 3, RowMajor, IndexType>> gpu2(gpu_data2, tensorRowRange); + + sycl_device.memcpyHostToDevice(gpu_data1, tensor1.data(),(tensor1.size())*sizeof(DataType)); + gpu2.device(sycl_device)=gpu1.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor2.data(), gpu_data2,(tensor2.size())*sizeof(DataType)); + + +// Tensor<float, 3, ColMajor> tensor(2,3,7); + //tensor.setRandom(); + +// Tensor<float, 3, RowMajor> tensor2 = tensor.swap_layout(); + VERIFY_IS_EQUAL(tensor1.dimension(0), tensor2.dimension(2)); + VERIFY_IS_EQUAL(tensor1.dimension(1), tensor2.dimension(1)); + VERIFY_IS_EQUAL(tensor1.dimension(2), tensor2.dimension(0)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 7; ++k) { + VERIFY_IS_EQUAL(tensor1(i,j,k), tensor2(k,j,i)); + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); +} + +template <typename DataType, typename IndexType> +static void test_swap_as_lvalue_sycl(const Eigen::SyclDevice& sycl_device) +{ + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 7; + array<IndexType, 3> tensorColRange = {{sizeDim1, sizeDim2, sizeDim3}}; + array<IndexType, 3> tensorRowRange = {{sizeDim3, sizeDim2, sizeDim1}}; + + Tensor<DataType, 3, ColMajor, IndexType> tensor1(tensorColRange); + Tensor<DataType, 3, RowMajor, IndexType> tensor2(tensorRowRange); + tensor1.setRandom(); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor1.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(tensor2.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 3, ColMajor, IndexType>> gpu1(gpu_data1, tensorColRange); + TensorMap<Tensor<DataType, 3, RowMajor, IndexType>> gpu2(gpu_data2, tensorRowRange); + + sycl_device.memcpyHostToDevice(gpu_data1, tensor1.data(),(tensor1.size())*sizeof(DataType)); + gpu2.swap_layout().device(sycl_device)=gpu1; + sycl_device.memcpyDeviceToHost(tensor2.data(), gpu_data2,(tensor2.size())*sizeof(DataType)); + + +// Tensor<float, 3, ColMajor> tensor(2,3,7); +// tensor.setRandom(); + + //Tensor<float, 3, RowMajor> tensor2(7,3,2); +// tensor2.swap_layout() = tensor; + VERIFY_IS_EQUAL(tensor1.dimension(0), tensor2.dimension(2)); + VERIFY_IS_EQUAL(tensor1.dimension(1), tensor2.dimension(1)); + VERIFY_IS_EQUAL(tensor1.dimension(2), tensor2.dimension(0)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 7; ++k) { + VERIFY_IS_EQUAL(tensor1(i,j,k), tensor2(k,j,i)); + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); +} + + +template<typename DataType, typename dev_Selector> void sycl_tensor_layout_swap_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_swap_sycl<DataType, int64_t>(sycl_device); + test_swap_as_lvalue_sycl<DataType, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_layout_swap_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_tensor_layout_swap_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_lvalue.cpp b/unsupported/test/cxx11_tensor_lvalue.cpp index 071f5b4..6ba9a21 100644 --- a/unsupported/test/cxx11_tensor_lvalue.cpp +++ b/unsupported/test/cxx11_tensor_lvalue.cpp
@@ -36,7 +36,7 @@ } -void test_cxx11_tensor_lvalue() +EIGEN_DECLARE_TEST(cxx11_tensor_lvalue) { CALL_SUBTEST(test_compound_assignment()); }
diff --git a/unsupported/test/cxx11_tensor_map.cpp b/unsupported/test/cxx11_tensor_map.cpp index a8a095e..ce608ac 100644 --- a/unsupported/test/cxx11_tensor_map.cpp +++ b/unsupported/test/cxx11_tensor_map.cpp
@@ -19,8 +19,8 @@ Tensor<int, 0> scalar1; Tensor<int, 0, RowMajor> scalar2; - TensorMap<Tensor<const int, 0>> scalar3(scalar1.data()); - TensorMap<Tensor<const int, 0, RowMajor>> scalar4(scalar2.data()); + TensorMap<Tensor<const int, 0> > scalar3(scalar1.data()); + TensorMap<Tensor<const int, 0, RowMajor> > scalar4(scalar2.data()); scalar1() = 7; scalar2() = 13; @@ -37,8 +37,8 @@ Tensor<int, 1> vec1(6); Tensor<int, 1, RowMajor> vec2(6); - TensorMap<Tensor<const int, 1>> vec3(vec1.data(), 6); - TensorMap<Tensor<const int, 1, RowMajor>> vec4(vec2.data(), 6); + TensorMap<Tensor<const int, 1> > vec3(vec1.data(), 6); + TensorMap<Tensor<const int, 1, RowMajor> > vec4(vec2.data(), 6); vec1(0) = 4; vec2(0) = 0; vec1(1) = 8; vec2(1) = 1; @@ -85,8 +85,8 @@ mat2(1,1) = 4; mat2(1,2) = 5; - TensorMap<Tensor<const int, 2>> mat3(mat1.data(), 2, 3); - TensorMap<Tensor<const int, 2, RowMajor>> mat4(mat2.data(), 2, 3); + TensorMap<Tensor<const int, 2> > mat3(mat1.data(), 2, 3); + TensorMap<Tensor<const int, 2, RowMajor> > mat4(mat2.data(), 2, 3); VERIFY_IS_EQUAL(mat3.rank(), 2); VERIFY_IS_EQUAL(mat3.size(), 6); @@ -129,8 +129,8 @@ } } - TensorMap<Tensor<const int, 3>> mat3(mat1.data(), 2, 3, 7); - TensorMap<Tensor<const int, 3, RowMajor>> mat4(mat2.data(), array<DenseIndex, 3>{{2, 3, 7}}); + TensorMap<Tensor<const int, 3> > mat3(mat1.data(), 2, 3, 7); + TensorMap<Tensor<const int, 3, RowMajor> > mat4(mat2.data(), 2, 3, 7); VERIFY_IS_EQUAL(mat3.rank(), 3); VERIFY_IS_EQUAL(mat3.size(), 2*3*7); @@ -173,8 +173,8 @@ } } - TensorMap<Tensor<int, 3>> mat3(mat1); - TensorMap<Tensor<int, 3, RowMajor>> mat4(mat2); + TensorMap<Tensor<int, 3> > mat3(mat1); + TensorMap<Tensor<int, 3, RowMajor> > mat4(mat2); VERIFY_IS_EQUAL(mat3.rank(), 3); VERIFY_IS_EQUAL(mat3.size(), 2*3*7); @@ -199,19 +199,23 @@ } } - TensorFixedSize<int, Sizes<2,3,7>> mat5; + TensorFixedSize<int, Sizes<2,3,7> > mat5; val = 0; for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < 7; ++k) { - mat5(i,j,k) = val; + array<ptrdiff_t, 3> coords; + coords[0] = i; + coords[1] = j; + coords[2] = k; + mat5(coords) = val; val++; } } } - TensorMap<TensorFixedSize<int, Sizes<2,3,7>>> mat6(mat5); + TensorMap<TensorFixedSize<int, Sizes<2,3,7> > > mat6(mat5); VERIFY_IS_EQUAL(mat6.rank(), 3); VERIFY_IS_EQUAL(mat6.size(), 2*3*7); @@ -233,8 +237,8 @@ static int f(const TensorMap<Tensor<int, 3> >& tensor) { // Size<0> empty; - EIGEN_STATIC_ASSERT((internal::array_size<Sizes<>>::value == 0), YOU_MADE_A_PROGRAMMING_MISTAKE); - EIGEN_STATIC_ASSERT((internal::array_size<DSizes<int, 0>>::value == 0), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((internal::array_size<Sizes<> >::value == 0), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT((internal::array_size<DSizes<int, 0> >::value == 0), YOU_MADE_A_PROGRAMMING_MISTAKE); Tensor<int, 0> result = tensor.sum(); return result(); } @@ -253,7 +257,7 @@ } } - TensorMap<Tensor<int, 3>> map(tensor); + TensorMap<Tensor<int, 3> > map(tensor); int sum1 = f(map); int sum2 = f(tensor); @@ -261,7 +265,7 @@ VERIFY_IS_EQUAL(sum1, 861); } -void test_cxx11_tensor_map() +EIGEN_DECLARE_TEST(cxx11_tensor_map) { CALL_SUBTEST(test_0d()); CALL_SUBTEST(test_1d());
diff --git a/unsupported/test/cxx11_tensor_math.cpp b/unsupported/test/cxx11_tensor_math.cpp index 61c742a..82a1a26 100644 --- a/unsupported/test/cxx11_tensor_math.cpp +++ b/unsupported/test/cxx11_tensor_math.cpp
@@ -39,7 +39,7 @@ } -void test_cxx11_tensor_math() +EIGEN_DECLARE_TEST(cxx11_tensor_math) { CALL_SUBTEST(test_tanh()); CALL_SUBTEST(test_sigmoid());
diff --git a/unsupported/test/cxx11_tensor_mixed_indices.cpp b/unsupported/test/cxx11_tensor_mixed_indices.cpp index 4fba6fd..ee2616f 100644 --- a/unsupported/test/cxx11_tensor_mixed_indices.cpp +++ b/unsupported/test/cxx11_tensor_mixed_indices.cpp
@@ -47,7 +47,7 @@ } -void test_cxx11_tensor_mixed_indices() +EIGEN_DECLARE_TEST(cxx11_tensor_mixed_indices) { CALL_SUBTEST(test_simple()); }
diff --git a/unsupported/test/cxx11_tensor_morphing.cpp b/unsupported/test/cxx11_tensor_morphing.cpp index eb3b891..4cbe15b 100644 --- a/unsupported/test/cxx11_tensor_morphing.cpp +++ b/unsupported/test/cxx11_tensor_morphing.cpp
@@ -13,6 +13,7 @@ using Eigen::Tensor; +template<typename> static void test_simple_reshape() { Tensor<float, 5> tensor1(2,3,1,7,1); @@ -40,7 +41,29 @@ } } +template <typename> +static void test_static_reshape() { +#if defined(EIGEN_HAS_INDEX_LIST) + using Eigen::type2index; + Tensor<float, 5> tensor(2, 3, 1, 7, 1); + tensor.setRandom(); + + // New dimensions: [2, 3, 7] + Eigen::IndexList<type2index<2>, type2index<3>, type2index<7>> dim; + Tensor<float, 3> reshaped = tensor.reshape(dim); + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < 7; ++k) { + VERIFY_IS_EQUAL(tensor(i, j, 0, k, 0), reshaped(i, j, k)); + } + } + } +#endif +} + +template<typename> static void test_reshape_in_expr() { MatrixXf m1(2,3*5*7*11); MatrixXf m2(3*5*7*11,13); @@ -65,7 +88,7 @@ } } - +template<typename> static void test_reshape_as_lvalue() { Tensor<float, 3> tensor(2,3,7); @@ -114,6 +137,7 @@ } } +template<typename=void> static void test_const_slice() { const float b[1] = {42}; @@ -315,6 +339,128 @@ VERIFY_IS_EQUAL(slice6.data(), tensor.data()); } + +template<int DataLayout> +static void test_strided_slice() +{ + typedef Tensor<float, 5, DataLayout> Tensor5f; + typedef Eigen::DSizes<Eigen::DenseIndex, 5> Index5; + typedef Tensor<float, 2, DataLayout> Tensor2f; + typedef Eigen::DSizes<Eigen::DenseIndex, 2> Index2; + Tensor<float, 5, DataLayout> tensor(2,3,5,7,11); + Tensor<float, 2, DataLayout> tensor2(7,11); + tensor.setRandom(); + tensor2.setRandom(); + + if (true) { + Tensor2f slice(2,3); + Index2 strides(-2,-1); + Index2 indicesStart(5,7); + Index2 indicesStop(0,4); + slice = tensor2.stridedSlice(indicesStart, indicesStop, strides); + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 3; ++k) { + VERIFY_IS_EQUAL(slice(j,k), tensor2(5-2*j,7-k)); + } + } + } + + if(true) { + Tensor2f slice(0,1); + Index2 strides(1,1); + Index2 indicesStart(5,4); + Index2 indicesStop(5,5); + slice = tensor2.stridedSlice(indicesStart, indicesStop, strides); + } + + if(true) { // test clamped degenerate interavls + Tensor2f slice(7,11); + Index2 strides(1,-1); + Index2 indicesStart(-3,20); // should become 0,10 + Index2 indicesStop(20,-11); // should become 11, -1 + slice = tensor2.stridedSlice(indicesStart, indicesStop, strides); + for (int j = 0; j < 7; ++j) { + for (int k = 0; k < 11; ++k) { + VERIFY_IS_EQUAL(slice(j,k), tensor2(j,10-k)); + } + } + } + + if(true) { + Tensor5f slice1(1,1,1,1,1); + Eigen::DSizes<Eigen::DenseIndex, 5> indicesStart(1, 2, 3, 4, 5); + Eigen::DSizes<Eigen::DenseIndex, 5> indicesStop(2, 3, 4, 5, 6); + Eigen::DSizes<Eigen::DenseIndex, 5> strides(1, 1, 1, 1, 1); + slice1 = tensor.stridedSlice(indicesStart, indicesStop, strides); + VERIFY_IS_EQUAL(slice1(0,0,0,0,0), tensor(1,2,3,4,5)); + } + + if(true) { + Tensor5f slice(1,1,2,2,3); + Index5 start(1, 1, 3, 4, 5); + Index5 stop(2, 2, 5, 6, 8); + Index5 strides(1, 1, 1, 1, 1); + slice = tensor.stridedSlice(start, stop, strides); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 3; ++k) { + VERIFY_IS_EQUAL(slice(0,0,i,j,k), tensor(1,1,3+i,4+j,5+k)); + } + } + } + } + + if(true) { + Tensor5f slice(1,1,2,2,3); + Index5 strides3(1, 1, -2, 1, -1); + Index5 indices3Start(1, 1, 4, 4, 7); + Index5 indices3Stop(2, 2, 0, 6, 4); + slice = tensor.stridedSlice(indices3Start, indices3Stop, strides3); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 3; ++k) { + VERIFY_IS_EQUAL(slice(0,0,i,j,k), tensor(1,1,4-2*i,4+j,7-k)); + } + } + } + } + + if(false) { // tests degenerate interval + Tensor5f slice(1,1,2,2,3); + Index5 strides3(1, 1, 2, 1, 1); + Index5 indices3Start(1, 1, 4, 4, 7); + Index5 indices3Stop(2, 2, 0, 6, 4); + slice = tensor.stridedSlice(indices3Start, indices3Stop, strides3); + } +} + +template<int DataLayout> +static void test_strided_slice_write() +{ + typedef Tensor<float, 2, DataLayout> Tensor2f; + typedef Eigen::DSizes<Eigen::DenseIndex, 2> Index2; + + Tensor<float, 2, DataLayout> tensor(7,11),tensor2(7,11); + tensor.setRandom(); + tensor2=tensor; + Tensor2f slice(2,3); + + slice.setRandom(); + + Index2 strides(1,1); + Index2 indicesStart(3,4); + Index2 indicesStop(5,7); + Index2 lengths(2,3); + + tensor.slice(indicesStart,lengths)=slice; + tensor2.stridedSlice(indicesStart,indicesStop,strides)=slice; + + for(int i=0;i<7;i++) for(int j=0;j<11;j++){ + VERIFY_IS_EQUAL(tensor(i,j), tensor2(i,j)); + } +} + + template<int DataLayout> static void test_composition() { @@ -335,22 +481,28 @@ } -void test_cxx11_tensor_morphing() +EIGEN_DECLARE_TEST(cxx11_tensor_morphing) { - CALL_SUBTEST(test_simple_reshape()); - CALL_SUBTEST(test_reshape_in_expr()); - CALL_SUBTEST(test_reshape_as_lvalue()); + CALL_SUBTEST_1(test_simple_reshape<void>()); + CALL_SUBTEST_1(test_static_reshape<void>()); + CALL_SUBTEST_1(test_reshape_in_expr<void>()); + CALL_SUBTEST_1(test_reshape_as_lvalue<void>()); - CALL_SUBTEST(test_simple_slice<ColMajor>()); - CALL_SUBTEST(test_simple_slice<RowMajor>()); - CALL_SUBTEST(test_const_slice()); - CALL_SUBTEST(test_slice_in_expr<ColMajor>()); - CALL_SUBTEST(test_slice_in_expr<RowMajor>()); - CALL_SUBTEST(test_slice_as_lvalue<ColMajor>()); - CALL_SUBTEST(test_slice_as_lvalue<RowMajor>()); - CALL_SUBTEST(test_slice_raw_data<ColMajor>()); - CALL_SUBTEST(test_slice_raw_data<RowMajor>()); + CALL_SUBTEST_1(test_simple_slice<ColMajor>()); + CALL_SUBTEST_1(test_simple_slice<RowMajor>()); + CALL_SUBTEST_1(test_const_slice()); + CALL_SUBTEST_2(test_slice_in_expr<ColMajor>()); + CALL_SUBTEST_3(test_slice_in_expr<RowMajor>()); + CALL_SUBTEST_4(test_slice_as_lvalue<ColMajor>()); + CALL_SUBTEST_4(test_slice_as_lvalue<RowMajor>()); + CALL_SUBTEST_5(test_slice_raw_data<ColMajor>()); + CALL_SUBTEST_5(test_slice_raw_data<RowMajor>()); - CALL_SUBTEST(test_composition<ColMajor>()); - CALL_SUBTEST(test_composition<RowMajor>()); + CALL_SUBTEST_6(test_strided_slice_write<ColMajor>()); + CALL_SUBTEST_6(test_strided_slice<ColMajor>()); + CALL_SUBTEST_6(test_strided_slice_write<RowMajor>()); + CALL_SUBTEST_6(test_strided_slice<RowMajor>()); + + CALL_SUBTEST_7(test_composition<ColMajor>()); + CALL_SUBTEST_7(test_composition<RowMajor>()); }
diff --git a/unsupported/test/cxx11_tensor_morphing_sycl.cpp b/unsupported/test/cxx11_tensor_morphing_sycl.cpp new file mode 100644 index 0000000..93dabe3 --- /dev/null +++ b/unsupported/test/cxx11_tensor_morphing_sycl.cpp
@@ -0,0 +1,248 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_reshape(const Eigen::SyclDevice& sycl_device) +{ + typename Tensor<DataType, 5 ,DataLayout, IndexType>::Dimensions dim1(2,3,1,7,1); + typename Tensor<DataType, 3 ,DataLayout, IndexType>::Dimensions dim2(2,3,7); + typename Tensor<DataType, 2 ,DataLayout, IndexType>::Dimensions dim3(6,7); + typename Tensor<DataType, 2 ,DataLayout, IndexType>::Dimensions dim4(2,21); + + Tensor<DataType, 5, DataLayout, IndexType> tensor1(dim1); + Tensor<DataType, 3, DataLayout, IndexType> tensor2(dim2); + Tensor<DataType, 2, DataLayout, IndexType> tensor3(dim3); + Tensor<DataType, 2, DataLayout, IndexType> tensor4(dim4); + + tensor1.setRandom(); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor1.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(tensor2.size()*sizeof(DataType))); + DataType* gpu_data3 = static_cast<DataType*>(sycl_device.allocate(tensor3.size()*sizeof(DataType))); + DataType* gpu_data4 = static_cast<DataType*>(sycl_device.allocate(tensor4.size()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 5,DataLayout, IndexType>> gpu1(gpu_data1, dim1); + TensorMap<Tensor<DataType, 3,DataLayout, IndexType>> gpu2(gpu_data2, dim2); + TensorMap<Tensor<DataType, 2,DataLayout, IndexType>> gpu3(gpu_data3, dim3); + TensorMap<Tensor<DataType, 2,DataLayout, IndexType>> gpu4(gpu_data4, dim4); + + sycl_device.memcpyHostToDevice(gpu_data1, tensor1.data(),(tensor1.size())*sizeof(DataType)); + + gpu2.device(sycl_device)=gpu1.reshape(dim2); + sycl_device.memcpyDeviceToHost(tensor2.data(), gpu_data2,(tensor1.size())*sizeof(DataType)); + + gpu3.device(sycl_device)=gpu1.reshape(dim3); + sycl_device.memcpyDeviceToHost(tensor3.data(), gpu_data3,(tensor3.size())*sizeof(DataType)); + + gpu4.device(sycl_device)=gpu1.reshape(dim2).reshape(dim4); + sycl_device.memcpyDeviceToHost(tensor4.data(), gpu_data4,(tensor4.size())*sizeof(DataType)); + for (IndexType i = 0; i < 2; ++i){ + for (IndexType j = 0; j < 3; ++j){ + for (IndexType k = 0; k < 7; ++k){ + VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor2(i,j,k)); ///ColMajor + if (static_cast<int>(DataLayout) == static_cast<int>(ColMajor)) { + VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor3(i+2*j,k)); ///ColMajor + VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor4(i,j+3*k)); ///ColMajor + } + else{ + //VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor2(i,j,k)); /// RowMajor + VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor4(i,j*7 +k)); /// RowMajor + VERIFY_IS_EQUAL(tensor1(i,j,0,k,0), tensor3(i*3 +j,k)); /// RowMajor + } + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); + sycl_device.deallocate(gpu_data3); + sycl_device.deallocate(gpu_data4); +} + + +template<typename DataType, int DataLayout, typename IndexType> +static void test_reshape_as_lvalue(const Eigen::SyclDevice& sycl_device) +{ + typename Tensor<DataType, 3, DataLayout, IndexType>::Dimensions dim1(2,3,7); + typename Tensor<DataType, 2, DataLayout, IndexType>::Dimensions dim2(6,7); + typename Tensor<DataType, 5, DataLayout, IndexType>::Dimensions dim3(2,3,1,7,1); + Tensor<DataType, 3, DataLayout, IndexType> tensor(dim1); + Tensor<DataType, 2, DataLayout, IndexType> tensor2d(dim2); + Tensor<DataType, 5, DataLayout, IndexType> tensor5d(dim3); + + tensor.setRandom(); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(tensor2d.size()*sizeof(DataType))); + DataType* gpu_data3 = static_cast<DataType*>(sycl_device.allocate(tensor5d.size()*sizeof(DataType))); + + TensorMap< Tensor<DataType, 3, DataLayout, IndexType> > gpu1(gpu_data1, dim1); + TensorMap< Tensor<DataType, 2, DataLayout, IndexType> > gpu2(gpu_data2, dim2); + TensorMap< Tensor<DataType, 5, DataLayout, IndexType> > gpu3(gpu_data3, dim3); + + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(),(tensor.size())*sizeof(DataType)); + + gpu2.reshape(dim1).device(sycl_device)=gpu1; + sycl_device.memcpyDeviceToHost(tensor2d.data(), gpu_data2,(tensor2d.size())*sizeof(DataType)); + + gpu3.reshape(dim1).device(sycl_device)=gpu1; + sycl_device.memcpyDeviceToHost(tensor5d.data(), gpu_data3,(tensor5d.size())*sizeof(DataType)); + + + for (IndexType i = 0; i < 2; ++i){ + for (IndexType j = 0; j < 3; ++j){ + for (IndexType k = 0; k < 7; ++k){ + VERIFY_IS_EQUAL(tensor5d(i,j,0,k,0), tensor(i,j,k)); + if (static_cast<int>(DataLayout) == static_cast<int>(ColMajor)) { + VERIFY_IS_EQUAL(tensor2d(i+2*j,k), tensor(i,j,k)); ///ColMajor + } + else{ + VERIFY_IS_EQUAL(tensor2d(i*3 +j,k),tensor(i,j,k)); /// RowMajor + } + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); + sycl_device.deallocate(gpu_data3); +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_slice(const Eigen::SyclDevice &sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + IndexType sizeDim5 = 11; + array<IndexType, 5> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4, sizeDim5}}; + Tensor<DataType, 5,DataLayout, IndexType> tensor(tensorRange); + tensor.setRandom(); + array<IndexType, 5> slice1_range ={{1, 1, 1, 1, 1}}; + Tensor<DataType, 5,DataLayout, IndexType> slice1(slice1_range); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(slice1.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 5,DataLayout, IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 5,DataLayout, IndexType>> gpu2(gpu_data2, slice1_range); + Eigen::DSizes<IndexType, 5> indices(1,2,3,4,5); + Eigen::DSizes<IndexType, 5> sizes(1,1,1,1,1); + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(),(tensor.size())*sizeof(DataType)); + gpu2.device(sycl_device)=gpu1.slice(indices, sizes); + sycl_device.memcpyDeviceToHost(slice1.data(), gpu_data2,(slice1.size())*sizeof(DataType)); + VERIFY_IS_EQUAL(slice1(0,0,0,0,0), tensor(1,2,3,4,5)); + + + array<IndexType, 5> slice2_range ={{1,1,2,2,3}}; + Tensor<DataType, 5,DataLayout, IndexType> slice2(slice2_range); + DataType* gpu_data3 = static_cast<DataType*>(sycl_device.allocate(slice2.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 5,DataLayout, IndexType>> gpu3(gpu_data3, slice2_range); + Eigen::DSizes<IndexType, 5> indices2(1,1,3,4,5); + Eigen::DSizes<IndexType, 5> sizes2(1,1,2,2,3); + gpu3.device(sycl_device)=gpu1.slice(indices2, sizes2); + sycl_device.memcpyDeviceToHost(slice2.data(), gpu_data3,(slice2.size())*sizeof(DataType)); + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 2; ++j) { + for (IndexType k = 0; k < 3; ++k) { + VERIFY_IS_EQUAL(slice2(0,0,i,j,k), tensor(1,1,3+i,4+j,5+k)); + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); + sycl_device.deallocate(gpu_data3); +} + +template<typename DataType, int DataLayout, typename IndexType> +static void test_strided_slice_write_sycl(const Eigen::SyclDevice& sycl_device) +{ + typedef Tensor<DataType, 2, DataLayout, IndexType> Tensor2f; + typedef Eigen::DSizes<IndexType, 2> Index2; + IndexType sizeDim1 = 7L; + IndexType sizeDim2 = 11L; + array<IndexType, 2> tensorRange = {{sizeDim1, sizeDim2}}; + Tensor<DataType, 2, DataLayout, IndexType> tensor(tensorRange),tensor2(tensorRange); + IndexType sliceDim1 = 2; + IndexType sliceDim2 = 3; + array<IndexType, 2> sliceRange = {{sliceDim1, sliceDim2}}; + Tensor2f slice(sliceRange); + Index2 strides(1L,1L); + Index2 indicesStart(3L,4L); + Index2 indicesStop(5L,7L); + Index2 lengths(2L,3L); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(tensor2.size()*sizeof(DataType))); + DataType* gpu_data3 = static_cast<DataType*>(sycl_device.allocate(slice.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 2,DataLayout,IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 2,DataLayout,IndexType>> gpu2(gpu_data2, tensorRange); + TensorMap<Tensor<DataType, 2,DataLayout,IndexType>> gpu3(gpu_data3, sliceRange); + + + tensor.setRandom(); + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(),(tensor.size())*sizeof(DataType)); + gpu2.device(sycl_device)=gpu1; + + slice.setRandom(); + sycl_device.memcpyHostToDevice(gpu_data3, slice.data(),(slice.size())*sizeof(DataType)); + + + gpu1.slice(indicesStart,lengths).device(sycl_device)=gpu3; + gpu2.stridedSlice(indicesStart,indicesStop,strides).device(sycl_device)=gpu3; + sycl_device.memcpyDeviceToHost(tensor.data(), gpu_data1,(tensor.size())*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(tensor2.data(), gpu_data2,(tensor2.size())*sizeof(DataType)); + + for(IndexType i=0;i<sizeDim1;i++) + for(IndexType j=0;j<sizeDim2;j++){ + VERIFY_IS_EQUAL(tensor(i,j), tensor2(i,j)); + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); + sycl_device.deallocate(gpu_data3); +} + +template<typename DataType, typename dev_Selector> void sycl_morphing_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_slice<DataType, RowMajor, int64_t>(sycl_device); + test_simple_slice<DataType, ColMajor, int64_t>(sycl_device); + test_simple_reshape<DataType, RowMajor, int64_t>(sycl_device); + test_simple_reshape<DataType, ColMajor, int64_t>(sycl_device); + test_reshape_as_lvalue<DataType, RowMajor, int64_t>(sycl_device); + test_reshape_as_lvalue<DataType, ColMajor, int64_t>(sycl_device); + test_strided_slice_write_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_strided_slice_write_sycl<DataType, RowMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_morphing_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_morphing_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_move.cpp b/unsupported/test/cxx11_tensor_move.cpp new file mode 100644 index 0000000..0ab2b77 --- /dev/null +++ b/unsupported/test/cxx11_tensor_move.cpp
@@ -0,0 +1,81 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Viktor Csomor <viktor.csomor@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +#include <Eigen/CXX11/Tensor> +#include <utility> + +using Eigen::Tensor; +using Eigen::RowMajor; + +static void calc_indices(int i, int& x, int& y, int& z) +{ + x = i / 4; + y = (i % 4) / 2; + z = i % 2; +} + +static void test_move() +{ + int x; + int y; + int z; + + Tensor<int,3> tensor1(2, 2, 2); + Tensor<int,3,RowMajor> tensor2(2, 2, 2); + + for (int i = 0; i < 8; i++) + { + calc_indices(i, x, y, z); + tensor1(x,y,z) = i; + tensor2(x,y,z) = 2 * i; + } + + // Invokes the move constructor. + Tensor<int,3> moved_tensor1 = std::move(tensor1); + Tensor<int,3,RowMajor> moved_tensor2 = std::move(tensor2); + + VERIFY_IS_EQUAL(tensor1.size(), 0); + VERIFY_IS_EQUAL(tensor2.size(), 0); + + for (int i = 0; i < 8; i++) + { + calc_indices(i, x, y, z); + VERIFY_IS_EQUAL(moved_tensor1(x,y,z), i); + VERIFY_IS_EQUAL(moved_tensor2(x,y,z), 2 * i); + } + + Tensor<int,3> moved_tensor3(2,2,2); + Tensor<int,3,RowMajor> moved_tensor4(2,2,2); + + moved_tensor3.setZero(); + moved_tensor4.setZero(); + + // Invokes the move assignment operator. + moved_tensor3 = std::move(moved_tensor1); + moved_tensor4 = std::move(moved_tensor2); + + VERIFY_IS_EQUAL(moved_tensor1.size(), 8); + VERIFY_IS_EQUAL(moved_tensor2.size(), 8); + + for (int i = 0; i < 8; i++) + { + calc_indices(i, x, y, z); + VERIFY_IS_EQUAL(moved_tensor1(x,y,z), 0); + VERIFY_IS_EQUAL(moved_tensor2(x,y,z), 0); + VERIFY_IS_EQUAL(moved_tensor3(x,y,z), i); + VERIFY_IS_EQUAL(moved_tensor4(x,y,z), 2 * i); + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_move) +{ + CALL_SUBTEST(test_move()); +}
diff --git a/unsupported/test/cxx11_tensor_notification.cpp b/unsupported/test/cxx11_tensor_notification.cpp index c946007..f63fee0 100644 --- a/unsupported/test/cxx11_tensor_notification.cpp +++ b/unsupported/test/cxx11_tensor_notification.cpp
@@ -13,15 +13,6 @@ #include "main.h" #include <Eigen/CXX11/Tensor> -#if EIGEN_OS_WIN || EIGEN_OS_WIN64 -#include <windows.h> -void sleep(int seconds) { - Sleep(seconds*1000); -} -#else -#include <unistd.h> -#endif - namespace { @@ -40,7 +31,7 @@ Eigen::Notification n; std::function<void()> func = std::bind(&WaitAndAdd, &n, &counter); thread_pool.Schedule(func); - sleep(1); + EIGEN_SLEEP(1000); // The thread should be waiting for the notification. VERIFY_IS_EQUAL(counter, 0); @@ -48,7 +39,7 @@ // Unblock the thread n.Notify(); - sleep(1); + EIGEN_SLEEP(1000); // Verify the counter has been incremented VERIFY_IS_EQUAL(counter, 1); @@ -67,14 +58,14 @@ thread_pool.Schedule(func); thread_pool.Schedule(func); thread_pool.Schedule(func); - sleep(1); + EIGEN_SLEEP(1000); VERIFY_IS_EQUAL(counter, 0); n.Notify(); - sleep(1); + EIGEN_SLEEP(1000); VERIFY_IS_EQUAL(counter, 4); } -void test_cxx11_tensor_notification() +EIGEN_DECLARE_TEST(cxx11_tensor_notification) { CALL_SUBTEST(test_notification_single()); CALL_SUBTEST(test_notification_multiple());
diff --git a/unsupported/test/cxx11_tensor_of_complex.cpp b/unsupported/test/cxx11_tensor_of_complex.cpp index e9d1b2d..99e18076 100644 --- a/unsupported/test/cxx11_tensor_of_complex.cpp +++ b/unsupported/test/cxx11_tensor_of_complex.cpp
@@ -94,7 +94,7 @@ } -void test_cxx11_tensor_of_complex() +EIGEN_DECLARE_TEST(cxx11_tensor_of_complex) { CALL_SUBTEST(test_additions()); CALL_SUBTEST(test_abs());
diff --git a/unsupported/test/cxx11_tensor_of_const_values.cpp b/unsupported/test/cxx11_tensor_of_const_values.cpp index f179a0c..344d678 100644 --- a/unsupported/test/cxx11_tensor_of_const_values.cpp +++ b/unsupported/test/cxx11_tensor_of_const_values.cpp
@@ -97,7 +97,7 @@ } -void test_cxx11_tensor_of_const_values() +EIGEN_DECLARE_TEST(cxx11_tensor_of_const_values) { CALL_SUBTEST(test_assign()); CALL_SUBTEST(test_plus());
diff --git a/unsupported/test/cxx11_tensor_of_float16_cuda.cu b/unsupported/test/cxx11_tensor_of_float16_cuda.cu deleted file mode 100644 index cb917bb..0000000 --- a/unsupported/test/cxx11_tensor_of_float16_cuda.cu +++ /dev/null
@@ -1,256 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_of_float16_cuda -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int -#define EIGEN_USE_GPU - - -#include "main.h" -#include <unsupported/Eigen/CXX11/Tensor> - -using Eigen::Tensor; - -#ifdef EIGEN_HAS_CUDA_FP16 - -void test_cuda_conversion() { - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - int num_elem = 101; - - float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); - Eigen::half* d_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); - float* d_conv = (float*)gpu_device.allocate(num_elem * sizeof(float)); - - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( - d_float, num_elem); - Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_half( - d_half, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_conv( - d_conv, num_elem); - - gpu_float.device(gpu_device) = gpu_float.random(); - gpu_half.device(gpu_device) = gpu_float.cast<Eigen::half>(); - gpu_conv.device(gpu_device) = gpu_half.cast<float>(); - - Tensor<float, 1> initial(num_elem); - Tensor<float, 1> final(num_elem); - gpu_device.memcpyDeviceToHost(initial.data(), d_float, num_elem*sizeof(float)); - gpu_device.memcpyDeviceToHost(final.data(), d_conv, num_elem*sizeof(float)); - - for (int i = 0; i < num_elem; ++i) { - VERIFY_IS_APPROX(initial(i), final(i)); - } - - gpu_device.deallocate(d_float); - gpu_device.deallocate(d_half); - gpu_device.deallocate(d_conv); -} - - -void test_cuda_unary() { - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - int num_elem = 101; - - float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); - - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( - d_float, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half( - d_res_half, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( - d_res_float, num_elem); - - gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f); - gpu_res_float.device(gpu_device) = gpu_float.abs(); - gpu_res_half.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().cast<float>(); - - Tensor<float, 1> half_prec(num_elem); - Tensor<float, 1> full_prec(num_elem); - gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float)); - gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); - gpu_device.synchronize(); - - for (int i = 0; i < num_elem; ++i) { - std::cout << "Checking unary " << i << std::endl; - VERIFY_IS_APPROX(full_prec(i), half_prec(i)); - } - - gpu_device.deallocate(d_float); - gpu_device.deallocate(d_res_half); - gpu_device.deallocate(d_res_float); -} - - -void test_cuda_elementwise() { - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - int num_elem = 101; - - float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); - - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float1( - d_float1, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float2( - d_float2, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half( - d_res_half, num_elem); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( - d_res_float, num_elem); - - gpu_float1.device(gpu_device) = gpu_float1.random(); - gpu_float2.device(gpu_device) = gpu_float2.random(); - gpu_res_float.device(gpu_device) = (gpu_float1 + gpu_float2) * gpu_float1; - gpu_res_half.device(gpu_device) = ((gpu_float1.cast<Eigen::half>() + gpu_float2.cast<Eigen::half>()) * gpu_float1.cast<Eigen::half>()).cast<float>(); - - Tensor<float, 1> half_prec(num_elem); - Tensor<float, 1> full_prec(num_elem); - gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float)); - gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); - gpu_device.synchronize(); - - for (int i = 0; i < num_elem; ++i) { - std::cout << "Checking elemwise " << i << std::endl; - VERIFY_IS_APPROX(full_prec(i), half_prec(i)); - } - - gpu_device.deallocate(d_float1); - gpu_device.deallocate(d_float2); - gpu_device.deallocate(d_res_half); - gpu_device.deallocate(d_res_float); -} - - -void test_cuda_contractions() { - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - int rows = 23; - int cols = 23; - int num_elem = rows*cols; - - float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); - - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1( - d_float1, rows, cols); - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2( - d_float2, rows, cols); - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_res_half( - d_res_half, rows, cols); - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_res_float( - d_res_float, rows, cols); - - gpu_float1.device(gpu_device) = gpu_float1.random() - gpu_float1.constant(0.5f); - gpu_float2.device(gpu_device) = gpu_float2.random() - gpu_float1.constant(0.5f); - - typedef Tensor<float, 2>::DimensionPair DimPair; - Eigen::array<DimPair, 1> dims(DimPair(1, 0)); - gpu_res_float.device(gpu_device) = gpu_float1.contract(gpu_float2, dims); - gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().contract(gpu_float2.cast<Eigen::half>(), dims).cast<float>(); - - Tensor<float, 2> half_prec(rows, cols); - Tensor<float, 2> full_prec(rows, cols); - gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float)); - gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); - gpu_device.synchronize(); - - for (int i = 0; i < rows; ++i) { - for (int j = 0; j < cols; ++j) { - std::cout << "Checking contract " << i << " " << j << std::endl; - VERIFY_IS_APPROX(full_prec(i, j), half_prec(i, j)); - } - } - - gpu_device.deallocate(d_float1); - gpu_device.deallocate(d_float2); - gpu_device.deallocate(d_res_half); - gpu_device.deallocate(d_res_float); -} - - -void test_cuda_reductions() { - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - int size = 13; - int num_elem = size*size; - - float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); - float* d_res_half = (float*)gpu_device.allocate(size * sizeof(float)); - float* d_res_float = (float*)gpu_device.allocate(size * sizeof(float)); - - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1( - d_float1, size, size); - Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2( - d_float2, size, size); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half( - d_res_half, size); - Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( - d_res_float, size); - - gpu_float1.device(gpu_device) = gpu_float1.random(); - gpu_float2.device(gpu_device) = gpu_float2.random(); - - Eigen::array<int, 1> redux_dim = {{0}}; - gpu_res_float.device(gpu_device) = gpu_float1.sum(redux_dim); - gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().sum(redux_dim).cast<float>(); - - Tensor<float, 1> half_prec(size); - Tensor<float, 1> full_prec(size); - gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, size*sizeof(float)); - gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, size*sizeof(float)); - gpu_device.synchronize(); - - for (int i = 0; i < size; ++i) { - std::cout << "Checking redux " << i << std::endl; - VERIFY_IS_APPROX(full_prec(i), half_prec(i)); - } - - gpu_device.deallocate(d_float1); - gpu_device.deallocate(d_float2); - gpu_device.deallocate(d_res_half); - gpu_device.deallocate(d_res_float); -} - - -#endif - - -void test_cxx11_tensor_of_float16_cuda() -{ -#ifdef EIGEN_HAS_CUDA_FP16 - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice device(&stream); - if (device.majorDeviceVersion() > 5 || - (device.majorDeviceVersion() == 5 && device.minorDeviceVersion() >= 3)) { - std::cout << "Running test on device with capability " << device.majorDeviceVersion() << "." << device.minorDeviceVersion() << std::endl; - - CALL_SUBTEST_1(test_cuda_conversion()); - CALL_SUBTEST_1(test_cuda_unary()); - CALL_SUBTEST_1(test_cuda_elementwise()); - CALL_SUBTEST_2(test_cuda_contractions()); - CALL_SUBTEST_3(test_cuda_reductions()); - } - else { - std::cout << "Half floats require compute capability of at least 5.3. This device only supports " << device.majorDeviceVersion() << "." << device.minorDeviceVersion() << ". Skipping the test" << std::endl; - } -#else - std::cout << "Half floats are not supported by this version of cuda: skipping the test" << std::endl; -#endif -}
diff --git a/unsupported/test/cxx11_tensor_of_float16_gpu.cu b/unsupported/test/cxx11_tensor_of_float16_gpu.cu new file mode 100644 index 0000000..4d74e61 --- /dev/null +++ b/unsupported/test/cxx11_tensor_of_float16_gpu.cu
@@ -0,0 +1,498 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + + +using Eigen::Tensor; + +template<typename> +void test_gpu_numext() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + bool* d_res_half = (bool*)gpu_device.allocate(num_elem * sizeof(bool)); + bool* d_res_float = (bool*)gpu_device.allocate(num_elem * sizeof(bool)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( + d_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_res_half( + d_res_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_res_float( + d_res_float, num_elem); + + gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f); + gpu_res_float.device(gpu_device) = gpu_float.unaryExpr(Eigen::internal::scalar_isnan_op<float>()); + gpu_res_half.device(gpu_device) = gpu_float.cast<Eigen::half>().unaryExpr(Eigen::internal::scalar_isnan_op<Eigen::half>()); + + Tensor<bool, 1> half_prec(num_elem); + Tensor<bool, 1> full_prec(num_elem); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(bool)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(bool)); + gpu_device.synchronize(); + + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking numext " << i << std::endl; + VERIFY_IS_EQUAL(full_prec(i), half_prec(i)); + } + + gpu_device.deallocate(d_float); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + + +#ifdef EIGEN_HAS_GPU_FP16 + +template<typename> +void test_gpu_conversion() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + Eigen::half* d_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + float* d_conv = (float*)gpu_device.allocate(num_elem * sizeof(float)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( + d_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_half( + d_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_conv( + d_conv, num_elem); + + gpu_float.device(gpu_device) = gpu_float.random(); + gpu_half.device(gpu_device) = gpu_float.cast<Eigen::half>(); + gpu_conv.device(gpu_device) = gpu_half.cast<float>(); + + Tensor<float, 1> initial(num_elem); + Tensor<float, 1> final(num_elem); + gpu_device.memcpyDeviceToHost(initial.data(), d_float, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(final.data(), d_conv, num_elem*sizeof(float)); + + for (int i = 0; i < num_elem; ++i) { + VERIFY_IS_APPROX(initial(i), final(i)); + } + + gpu_device.deallocate(d_float); + gpu_device.deallocate(d_half); + gpu_device.deallocate(d_conv); +} + +template<typename> +void test_gpu_unary() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( + d_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half( + d_res_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( + d_res_float, num_elem); + + gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f); + gpu_res_float.device(gpu_device) = gpu_float.abs(); + gpu_res_half.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().cast<float>(); + + Tensor<float, 1> half_prec(num_elem); + Tensor<float, 1> full_prec(num_elem); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); + gpu_device.synchronize(); + + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking unary " << i << std::endl; + VERIFY_IS_APPROX(full_prec(i), half_prec(i)); + } + + gpu_device.deallocate(d_float); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + +template<typename> +void test_gpu_elementwise() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float1( + d_float1, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float2( + d_float2, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half( + d_res_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( + d_res_float, num_elem); + + gpu_float1.device(gpu_device) = gpu_float1.random(); + gpu_float2.device(gpu_device) = gpu_float2.random(); + gpu_res_float.device(gpu_device) = (gpu_float1 + gpu_float2) * gpu_float1; + gpu_res_half.device(gpu_device) = ((gpu_float1.cast<Eigen::half>() + gpu_float2.cast<Eigen::half>()) * gpu_float1.cast<Eigen::half>()).cast<float>(); + + Tensor<float, 1> half_prec(num_elem); + Tensor<float, 1> full_prec(num_elem); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); + gpu_device.synchronize(); + + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking elemwise " << i << ": full prec = " << full_prec(i) << " vs half prec = " << half_prec(i) << std::endl; + VERIFY_IS_APPROX(static_cast<Eigen::half>(full_prec(i)), static_cast<Eigen::half>(half_prec(i))); + } + + gpu_device.deallocate(d_float1); + gpu_device.deallocate(d_float2); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + +template<typename> +void test_gpu_trancendental() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float3 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + Eigen::half* d_res1_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res1_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res2_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res2_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res3_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res3_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float1(d_float1, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float2(d_float2, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float3(d_float3, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res1_half(d_res1_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res1_float(d_res1_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res2_half(d_res2_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res2_float(d_res2_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res3_half(d_res3_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res3_float(d_res3_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res4_half(d_res3_half, num_elem); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res4_float(d_res3_float, num_elem); + + gpu_float1.device(gpu_device) = gpu_float1.random() - gpu_float1.constant(0.5f); + gpu_float2.device(gpu_device) = gpu_float2.random() + gpu_float1.constant(0.5f); + gpu_float3.device(gpu_device) = gpu_float3.random(); + gpu_res1_float.device(gpu_device) = gpu_float1.exp().cast<Eigen::half>(); + gpu_res2_float.device(gpu_device) = gpu_float2.log().cast<Eigen::half>(); + gpu_res3_float.device(gpu_device) = gpu_float3.log1p().cast<Eigen::half>(); + gpu_res4_float.device(gpu_device) = gpu_float3.expm1().cast<Eigen::half>(); + + gpu_res1_half.device(gpu_device) = gpu_float1.cast<Eigen::half>(); + gpu_res1_half.device(gpu_device) = gpu_res1_half.exp(); + + gpu_res2_half.device(gpu_device) = gpu_float2.cast<Eigen::half>(); + gpu_res2_half.device(gpu_device) = gpu_res2_half.log(); + + gpu_res3_half.device(gpu_device) = gpu_float3.cast<Eigen::half>(); + gpu_res3_half.device(gpu_device) = gpu_res3_half.log1p(); + + gpu_res3_half.device(gpu_device) = gpu_float3.cast<Eigen::half>(); + gpu_res3_half.device(gpu_device) = gpu_res3_half.expm1(); + + Tensor<float, 1> input1(num_elem); + Tensor<Eigen::half, 1> half_prec1(num_elem); + Tensor<Eigen::half, 1> full_prec1(num_elem); + Tensor<float, 1> input2(num_elem); + Tensor<Eigen::half, 1> half_prec2(num_elem); + Tensor<Eigen::half, 1> full_prec2(num_elem); + Tensor<float, 1> input3(num_elem); + Tensor<Eigen::half, 1> half_prec3(num_elem); + Tensor<Eigen::half, 1> full_prec3(num_elem); + gpu_device.memcpyDeviceToHost(input1.data(), d_float1, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(input2.data(), d_float2, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(input3.data(), d_float3, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(half_prec1.data(), d_res1_half, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec1.data(), d_res1_float, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(half_prec2.data(), d_res2_half, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec2.data(), d_res2_float, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(half_prec3.data(), d_res3_half, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec3.data(), d_res3_float, num_elem*sizeof(Eigen::half)); + gpu_device.synchronize(); + + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking elemwise exp " << i << " input = " << input1(i) << " full = " << full_prec1(i) << " half = " << half_prec1(i) << std::endl; + VERIFY_IS_APPROX(full_prec1(i), half_prec1(i)); + } + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking elemwise log " << i << " input = " << input2(i) << " full = " << full_prec2(i) << " half = " << half_prec2(i) << std::endl; + if(std::abs(input2(i)-1.f)<0.05f) // log lacks accuracy nearby 1 + VERIFY_IS_APPROX(full_prec2(i)+Eigen::half(0.1f), half_prec2(i)+Eigen::half(0.1f)); + else + VERIFY_IS_APPROX(full_prec2(i), half_prec2(i)); + } + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking elemwise plog1 " << i << " input = " << input3(i) << " full = " << full_prec3(i) << " half = " << half_prec3(i) << std::endl; + VERIFY_IS_APPROX(full_prec3(i), half_prec3(i)); + } + gpu_device.deallocate(d_float1); + gpu_device.deallocate(d_float2); + gpu_device.deallocate(d_float3); + gpu_device.deallocate(d_res1_half); + gpu_device.deallocate(d_res1_float); + gpu_device.deallocate(d_res2_half); + gpu_device.deallocate(d_res2_float); + gpu_device.deallocate(d_res3_float); + gpu_device.deallocate(d_res3_half); +} + +template<typename> +void test_gpu_contractions() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int rows = 23; + int cols = 23; + int num_elem = rows*cols; + + float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half)); + + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1( + d_float1, rows, cols); + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2( + d_float2, rows, cols); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 2>, Eigen::Aligned> gpu_res_half( + d_res_half, rows, cols); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 2>, Eigen::Aligned> gpu_res_float( + d_res_float, rows, cols); + + gpu_float1.device(gpu_device) = gpu_float1.random() - gpu_float1.constant(0.5f); + gpu_float2.device(gpu_device) = gpu_float2.random() - gpu_float2.constant(0.5f); + + typedef Tensor<float, 2>::DimensionPair DimPair; + Eigen::array<DimPair, 1> dims(DimPair(1, 0)); + gpu_res_float.device(gpu_device) = gpu_float1.contract(gpu_float2, dims).cast<Eigen::half>(); + gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().contract(gpu_float2.cast<Eigen::half>(), dims); + + Tensor<Eigen::half, 2> half_prec(rows, cols); + Tensor<Eigen::half, 2> full_prec(rows, cols); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(Eigen::half)); + gpu_device.synchronize(); + + for (int i = 0; i < rows; ++i) { + for (int j = 0; j < cols; ++j) { + std::cout << "Checking contract " << i << " " << j << full_prec(i, j) << " " << half_prec(i, j) << std::endl; + if (numext::abs(full_prec(i, j) - half_prec(i, j)) > Eigen::half(1e-2f)) { + VERIFY_IS_APPROX(full_prec(i, j), half_prec(i, j)); + } + } + } + + gpu_device.deallocate(d_float1); + gpu_device.deallocate(d_float2); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + +template<typename> +void test_gpu_reductions(int size1, int size2, int redux) { + + std::cout << "Reducing " << size1 << " by " << size2 + << " tensor along dim " << redux << std::endl; + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = size1*size2; + int result_size = (redux == 1 ? size1 : size2); + + float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(result_size * sizeof(Eigen::half)); + Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(result_size * sizeof(Eigen::half)); + + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1( + d_float1, size1, size2); + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2( + d_float2, size1, size2); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res_half( + d_res_half, result_size); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res_float( + d_res_float, result_size); + + gpu_float1.device(gpu_device) = gpu_float1.random() * 2.0f; + gpu_float2.device(gpu_device) = gpu_float2.random() * 2.0f; + + Eigen::array<int, 1> redux_dim = {{redux}}; + gpu_res_float.device(gpu_device) = gpu_float1.sum(redux_dim).cast<Eigen::half>(); + gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().sum(redux_dim); + + Tensor<Eigen::half, 1> half_prec(result_size); + Tensor<Eigen::half, 1> full_prec(result_size); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, result_size*sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, result_size*sizeof(Eigen::half)); + gpu_device.synchronize(); + + for (int i = 0; i < result_size; ++i) { + std::cout << "EXPECTED " << full_prec(i) << " GOT " << half_prec(i) << std::endl; + VERIFY_IS_APPROX(full_prec(i), half_prec(i)); + } + + gpu_device.deallocate(d_float1); + gpu_device.deallocate(d_float2); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + +template<typename> +void test_gpu_reductions() { + test_gpu_reductions<void>(13, 13, 0); + test_gpu_reductions<void>(13, 13, 1); + + test_gpu_reductions<void>(35, 36, 0); + test_gpu_reductions<void>(35, 36, 1); + + test_gpu_reductions<void>(36, 35, 0); + test_gpu_reductions<void>(36, 35, 1); +} + +template<typename> +void test_gpu_full_reductions() { + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int size = 13; + int num_elem = size*size; + + float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(1 * sizeof(Eigen::half)); + Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(1 * sizeof(Eigen::half)); + + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1( + d_float1, size, size); + Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2( + d_float2, size, size); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 0>, Eigen::Aligned> gpu_res_half( + d_res_half); + Eigen::TensorMap<Eigen::Tensor<Eigen::half, 0>, Eigen::Aligned> gpu_res_float( + d_res_float); + + gpu_float1.device(gpu_device) = gpu_float1.random(); + gpu_float2.device(gpu_device) = gpu_float2.random(); + + gpu_res_float.device(gpu_device) = gpu_float1.sum().cast<Eigen::half>(); + gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().sum(); + + Tensor<Eigen::half, 0> half_prec; + Tensor<Eigen::half, 0> full_prec; + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, sizeof(Eigen::half)); + gpu_device.synchronize(); + + VERIFY_IS_APPROX(full_prec(), half_prec()); + + gpu_res_float.device(gpu_device) = gpu_float1.maximum().cast<Eigen::half>(); + gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().maximum(); + gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, sizeof(Eigen::half)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, sizeof(Eigen::half)); + gpu_device.synchronize(); + + VERIFY_IS_APPROX(full_prec(), half_prec()); + + gpu_device.deallocate(d_float1); + gpu_device.deallocate(d_float2); + gpu_device.deallocate(d_res_half); + gpu_device.deallocate(d_res_float); +} + +template<typename> +void test_gpu_forced_evals() { + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + int num_elem = 101; + + float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_half1 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_half2 = (float*)gpu_device.allocate(num_elem * sizeof(float)); + float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float)); + + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float( + d_float, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half1( + d_res_half1, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Unaligned> gpu_res_half2( + d_res_half2, num_elem); + Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float( + d_res_float, num_elem); + + Eigen::array<int, 1> no_bcast; + no_bcast[0] = 1; + + gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f); + gpu_res_float.device(gpu_device) = gpu_float.abs(); + gpu_res_half1.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().eval().cast<float>(); + gpu_res_half2.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().broadcast(no_bcast).eval().cast<float>(); + + Tensor<float, 1> half_prec1(num_elem); + Tensor<float, 1> half_prec2(num_elem); + Tensor<float, 1> full_prec(num_elem); + gpu_device.memcpyDeviceToHost(half_prec1.data(), d_res_half1, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(half_prec2.data(), d_res_half1, num_elem*sizeof(float)); + gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float)); + gpu_device.synchronize(); + + for (int i = 0; i < num_elem; ++i) { + std::cout << "Checking forced eval " << i << full_prec(i) << " vs " << half_prec1(i) << " vs " << half_prec2(i) << std::endl; + VERIFY_IS_APPROX(full_prec(i), half_prec1(i)); + VERIFY_IS_APPROX(full_prec(i), half_prec2(i)); + } + + gpu_device.deallocate(d_float); + gpu_device.deallocate(d_res_half1); + gpu_device.deallocate(d_res_half2); + gpu_device.deallocate(d_res_float); +} +#endif + + +EIGEN_DECLARE_TEST(cxx11_tensor_of_float16_gpu) +{ + CALL_SUBTEST_1(test_gpu_numext<void>()); + +#ifdef EIGEN_HAS_GPU_FP16 + CALL_SUBTEST_1(test_gpu_conversion<void>()); + CALL_SUBTEST_1(test_gpu_unary<void>()); + CALL_SUBTEST_1(test_gpu_elementwise<void>()); + CALL_SUBTEST_1(test_gpu_trancendental<void>()); + CALL_SUBTEST_2(test_gpu_contractions<void>()); + CALL_SUBTEST_3(test_gpu_reductions<void>()); + CALL_SUBTEST_4(test_gpu_full_reductions<void>()); + CALL_SUBTEST_5(test_gpu_forced_evals<void>()); +#else + std::cout << "Half floats are not supported by this version of gpu: skipping the test" << std::endl; +#endif +}
diff --git a/unsupported/test/cxx11_tensor_of_strings.cpp b/unsupported/test/cxx11_tensor_of_strings.cpp index 4ef9aed..1596562 100644 --- a/unsupported/test/cxx11_tensor_of_strings.cpp +++ b/unsupported/test/cxx11_tensor_of_strings.cpp
@@ -141,7 +141,7 @@ } -void test_cxx11_tensor_of_strings() +EIGEN_DECLARE_TEST(cxx11_tensor_of_strings) { // Beware: none of this is likely to ever work on a GPU. CALL_SUBTEST(test_assign());
diff --git a/unsupported/test/cxx11_tensor_padding.cpp b/unsupported/test/cxx11_tensor_padding.cpp index ffa1989..b8a329d 100644 --- a/unsupported/test/cxx11_tensor_padding.cpp +++ b/unsupported/test/cxx11_tensor_padding.cpp
@@ -84,7 +84,7 @@ } } -void test_cxx11_tensor_padding() +EIGEN_DECLARE_TEST(cxx11_tensor_padding) { CALL_SUBTEST(test_simple_padding<ColMajor>()); CALL_SUBTEST(test_simple_padding<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_padding_sycl.cpp b/unsupported/test/cxx11_tensor_padding_sycl.cpp new file mode 100644 index 0000000..727a9ff --- /dev/null +++ b/unsupported/test/cxx11_tensor_padding_sycl.cpp
@@ -0,0 +1,157 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + + +template<typename DataType, int DataLayout, typename IndexType> +static void test_simple_padding(const Eigen::SyclDevice& sycl_device) +{ + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensorRange); + tensor.setRandom(); + + array<std::pair<IndexType, IndexType>, 4> paddings; + paddings[0] = std::make_pair(0, 0); + paddings[1] = std::make_pair(2, 1); + paddings[2] = std::make_pair(3, 4); + paddings[3] = std::make_pair(0, 0); + + IndexType padedSizeDim1 = 2; + IndexType padedSizeDim2 = 6; + IndexType padedSizeDim3 = 12; + IndexType padedSizeDim4 = 7; + array<IndexType, 4> padedtensorRange = {{padedSizeDim1, padedSizeDim2, padedSizeDim3, padedSizeDim4}}; + + Tensor<DataType, 4, DataLayout, IndexType> padded(padedtensorRange); + + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(padded.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4,DataLayout,IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 4,DataLayout,IndexType>> gpu2(gpu_data2, padedtensorRange); + + VERIFY_IS_EQUAL(padded.dimension(0), 2+0); + VERIFY_IS_EQUAL(padded.dimension(1), 3+3); + VERIFY_IS_EQUAL(padded.dimension(2), 5+7); + VERIFY_IS_EQUAL(padded.dimension(3), 7+0); + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(),(tensor.size())*sizeof(DataType)); + gpu2.device(sycl_device)=gpu1.pad(paddings); + sycl_device.memcpyDeviceToHost(padded.data(), gpu_data2,(padded.size())*sizeof(DataType)); + for (IndexType i = 0; i < padedSizeDim1; ++i) { + for (IndexType j = 0; j < padedSizeDim2; ++j) { + for (IndexType k = 0; k < padedSizeDim3; ++k) { + for (IndexType l = 0; l < padedSizeDim4; ++l) { + if (j >= 2 && j < 5 && k >= 3 && k < 8) { + VERIFY_IS_EQUAL(padded(i,j,k,l), tensor(i,j-2,k-3,l)); + } else { + VERIFY_IS_EQUAL(padded(i,j,k,l), 0.0f); + } + } + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); +} + +template<typename DataType, int DataLayout, typename IndexType> +static void test_padded_expr(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensorRange); + tensor.setRandom(); + + array<std::pair<IndexType, IndexType>, 4> paddings; + paddings[0] = std::make_pair(0, 0); + paddings[1] = std::make_pair(2, 1); + paddings[2] = std::make_pair(3, 4); + paddings[3] = std::make_pair(0, 0); + + Eigen::DSizes<IndexType, 2> reshape_dims; + reshape_dims[0] = 12; + reshape_dims[1] = 84; + + + Tensor<DataType, 2, DataLayout, IndexType> result(reshape_dims); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(tensor.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(result.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 4,DataLayout,IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 2,DataLayout,IndexType>> gpu2(gpu_data2, reshape_dims); + + + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(),(tensor.size())*sizeof(DataType)); + gpu2.device(sycl_device)=gpu1.pad(paddings).reshape(reshape_dims); + sycl_device.memcpyDeviceToHost(result.data(), gpu_data2,(result.size())*sizeof(DataType)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 6; ++j) { + for (IndexType k = 0; k < 12; ++k) { + for (IndexType l = 0; l < 7; ++l) { + const float result_value = DataLayout == ColMajor ? + result(i+2*j,k+12*l) : result(j+6*i,l+7*k); + if (j >= 2 && j < 5 && k >= 3 && k < 8) { + VERIFY_IS_EQUAL(result_value, tensor(i,j-2,k-3,l)); + } else { + VERIFY_IS_EQUAL(result_value, 0.0f); + } + } + } + } + } + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); +} + +template<typename DataType, typename dev_Selector> void sycl_padding_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_padding<DataType, RowMajor, int64_t>(sycl_device); + test_simple_padding<DataType, ColMajor, int64_t>(sycl_device); + test_padded_expr<DataType, RowMajor, int64_t>(sycl_device); + test_padded_expr<DataType, ColMajor, int64_t>(sycl_device); + +} +EIGEN_DECLARE_TEST(cxx11_tensor_padding_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_padding_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_patch.cpp b/unsupported/test/cxx11_tensor_patch.cpp index 4343597..498ab8c 100644 --- a/unsupported/test/cxx11_tensor_patch.cpp +++ b/unsupported/test/cxx11_tensor_patch.cpp
@@ -164,7 +164,7 @@ } } -void test_cxx11_tensor_patch() +EIGEN_DECLARE_TEST(cxx11_tensor_patch) { CALL_SUBTEST(test_simple_patch<ColMajor>()); CALL_SUBTEST(test_simple_patch<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_patch_sycl.cpp b/unsupported/test/cxx11_tensor_patch_sycl.cpp new file mode 100644 index 0000000..7f92bec --- /dev/null +++ b/unsupported/test/cxx11_tensor_patch_sycl.cpp
@@ -0,0 +1,249 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_patch_sycl(const Eigen::SyclDevice& sycl_device){ + + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + array<IndexType, 5> patchTensorRange; + if (DataLayout == ColMajor) { + patchTensorRange = {{1, 1, 1, 1, sizeDim1*sizeDim2*sizeDim3*sizeDim4}}; + }else{ + patchTensorRange = {{sizeDim1*sizeDim2*sizeDim3*sizeDim4,1, 1, 1, 1}}; + } + + Tensor<DataType, 4, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 5, DataLayout,IndexType> no_patch(patchTensorRange); + + tensor.setRandom(); + + array<ptrdiff_t, 4> patch_dims; + patch_dims[0] = 1; + patch_dims[1] = 1; + patch_dims[2] = 1; + patch_dims[3] = 1; + + const size_t tensorBuffSize =tensor.size()*sizeof(DataType); + size_t patchTensorBuffSize =no_patch.size()*sizeof(DataType); + DataType* gpu_data_tensor = static_cast<DataType*>(sycl_device.allocate(tensorBuffSize)); + DataType* gpu_data_no_patch = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu_tensor(gpu_data_tensor, tensorRange); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_no_patch(gpu_data_no_patch, patchTensorRange); + + sycl_device.memcpyHostToDevice(gpu_data_tensor, tensor.data(), tensorBuffSize); + gpu_no_patch.device(sycl_device)=gpu_tensor.extract_patches(patch_dims); + sycl_device.memcpyDeviceToHost(no_patch.data(), gpu_data_no_patch, patchTensorBuffSize); + + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(no_patch.dimension(0), 1); + VERIFY_IS_EQUAL(no_patch.dimension(1), 1); + VERIFY_IS_EQUAL(no_patch.dimension(2), 1); + VERIFY_IS_EQUAL(no_patch.dimension(3), 1); + VERIFY_IS_EQUAL(no_patch.dimension(4), tensor.size()); + } else { + VERIFY_IS_EQUAL(no_patch.dimension(0), tensor.size()); + VERIFY_IS_EQUAL(no_patch.dimension(1), 1); + VERIFY_IS_EQUAL(no_patch.dimension(2), 1); + VERIFY_IS_EQUAL(no_patch.dimension(3), 1); + VERIFY_IS_EQUAL(no_patch.dimension(4), 1); + } + + for (int i = 0; i < tensor.size(); ++i) { + VERIFY_IS_EQUAL(tensor.data()[i], no_patch.data()[i]); + } + + patch_dims[0] = 2; + patch_dims[1] = 3; + patch_dims[2] = 5; + patch_dims[3] = 7; + + if (DataLayout == ColMajor) { + patchTensorRange = {{sizeDim1,sizeDim2,sizeDim3,sizeDim4,1}}; + }else{ + patchTensorRange = {{1,sizeDim1,sizeDim2,sizeDim3,sizeDim4}}; + } + Tensor<DataType, 5, DataLayout,IndexType> single_patch(patchTensorRange); + patchTensorBuffSize =single_patch.size()*sizeof(DataType); + DataType* gpu_data_single_patch = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_single_patch(gpu_data_single_patch, patchTensorRange); + + gpu_single_patch.device(sycl_device)=gpu_tensor.extract_patches(patch_dims); + sycl_device.memcpyDeviceToHost(single_patch.data(), gpu_data_single_patch, patchTensorBuffSize); + + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(single_patch.dimension(0), 2); + VERIFY_IS_EQUAL(single_patch.dimension(1), 3); + VERIFY_IS_EQUAL(single_patch.dimension(2), 5); + VERIFY_IS_EQUAL(single_patch.dimension(3), 7); + VERIFY_IS_EQUAL(single_patch.dimension(4), 1); + } else { + VERIFY_IS_EQUAL(single_patch.dimension(0), 1); + VERIFY_IS_EQUAL(single_patch.dimension(1), 2); + VERIFY_IS_EQUAL(single_patch.dimension(2), 3); + VERIFY_IS_EQUAL(single_patch.dimension(3), 5); + VERIFY_IS_EQUAL(single_patch.dimension(4), 7); + } + + for (int i = 0; i < tensor.size(); ++i) { + VERIFY_IS_EQUAL(tensor.data()[i], single_patch.data()[i]); + } + patch_dims[0] = 1; + patch_dims[1] = 2; + patch_dims[2] = 2; + patch_dims[3] = 1; + + if (DataLayout == ColMajor) { + patchTensorRange = {{1,2,2,1,2*2*4*7}}; + }else{ + patchTensorRange = {{2*2*4*7, 1, 2,2,1}}; + } + Tensor<DataType, 5, DataLayout,IndexType> twod_patch(patchTensorRange); + patchTensorBuffSize =twod_patch.size()*sizeof(DataType); + DataType* gpu_data_twod_patch = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_twod_patch(gpu_data_twod_patch, patchTensorRange); + + gpu_twod_patch.device(sycl_device)=gpu_tensor.extract_patches(patch_dims); + sycl_device.memcpyDeviceToHost(twod_patch.data(), gpu_data_twod_patch, patchTensorBuffSize); + + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(twod_patch.dimension(0), 1); + VERIFY_IS_EQUAL(twod_patch.dimension(1), 2); + VERIFY_IS_EQUAL(twod_patch.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch.dimension(3), 1); + VERIFY_IS_EQUAL(twod_patch.dimension(4), 2*2*4*7); + } else { + VERIFY_IS_EQUAL(twod_patch.dimension(0), 2*2*4*7); + VERIFY_IS_EQUAL(twod_patch.dimension(1), 1); + VERIFY_IS_EQUAL(twod_patch.dimension(2), 2); + VERIFY_IS_EQUAL(twod_patch.dimension(3), 2); + VERIFY_IS_EQUAL(twod_patch.dimension(4), 1); + } + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 4; ++k) { + for (int l = 0; l < 7; ++l) { + int patch_loc; + if (DataLayout == ColMajor) { + patch_loc = i + 2 * (j + 2 * (k + 4 * l)); + } else { + patch_loc = l + 7 * (k + 4 * (j + 2 * i)); + } + for (int x = 0; x < 2; ++x) { + for (int y = 0; y < 2; ++y) { + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(tensor(i,j+x,k+y,l), twod_patch(0,x,y,0,patch_loc)); + } else { + VERIFY_IS_EQUAL(tensor(i,j+x,k+y,l), twod_patch(patch_loc,0,x,y,0)); + } + } + } + } + } + } + } + + patch_dims[0] = 1; + patch_dims[1] = 2; + patch_dims[2] = 3; + patch_dims[3] = 5; + + if (DataLayout == ColMajor) { + patchTensorRange = {{1,2,3,5,2*2*3*3}}; + }else{ + patchTensorRange = {{2*2*3*3, 1, 2,3,5}}; + } + Tensor<DataType, 5, DataLayout,IndexType> threed_patch(patchTensorRange); + patchTensorBuffSize =threed_patch.size()*sizeof(DataType); + DataType* gpu_data_threed_patch = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 5, DataLayout,IndexType>> gpu_threed_patch(gpu_data_threed_patch, patchTensorRange); + + gpu_threed_patch.device(sycl_device)=gpu_tensor.extract_patches(patch_dims); + sycl_device.memcpyDeviceToHost(threed_patch.data(), gpu_data_threed_patch, patchTensorBuffSize); + + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(threed_patch.dimension(0), 1); + VERIFY_IS_EQUAL(threed_patch.dimension(1), 2); + VERIFY_IS_EQUAL(threed_patch.dimension(2), 3); + VERIFY_IS_EQUAL(threed_patch.dimension(3), 5); + VERIFY_IS_EQUAL(threed_patch.dimension(4), 2*2*3*3); + } else { + VERIFY_IS_EQUAL(threed_patch.dimension(0), 2*2*3*3); + VERIFY_IS_EQUAL(threed_patch.dimension(1), 1); + VERIFY_IS_EQUAL(threed_patch.dimension(2), 2); + VERIFY_IS_EQUAL(threed_patch.dimension(3), 3); + VERIFY_IS_EQUAL(threed_patch.dimension(4), 5); + } + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 3; ++k) { + for (int l = 0; l < 3; ++l) { + int patch_loc; + if (DataLayout == ColMajor) { + patch_loc = i + 2 * (j + 2 * (k + 3 * l)); + } else { + patch_loc = l + 3 * (k + 3 * (j + 2 * i)); + } + for (int x = 0; x < 2; ++x) { + for (int y = 0; y < 3; ++y) { + for (int z = 0; z < 5; ++z) { + if (DataLayout == ColMajor) { + VERIFY_IS_EQUAL(tensor(i,j+x,k+y,l+z), threed_patch(0,x,y,z,patch_loc)); + } else { + VERIFY_IS_EQUAL(tensor(i,j+x,k+y,l+z), threed_patch(patch_loc,0,x,y,z)); + } + } + } + } + } + } + } + } + sycl_device.deallocate(gpu_data_tensor); + sycl_device.deallocate(gpu_data_no_patch); + sycl_device.deallocate(gpu_data_single_patch); + sycl_device.deallocate(gpu_data_twod_patch); + sycl_device.deallocate(gpu_data_threed_patch); +} + +template<typename DataType, typename dev_Selector> void sycl_tensor_patch_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_patch_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_simple_patch_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_patch_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_tensor_patch_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_random.cpp b/unsupported/test/cxx11_tensor_random.cpp index 0f3dc57..4740d58 100644 --- a/unsupported/test/cxx11_tensor_random.cpp +++ b/unsupported/test/cxx11_tensor_random.cpp
@@ -70,7 +70,7 @@ } } -void test_cxx11_tensor_random() +EIGEN_DECLARE_TEST(cxx11_tensor_random) { CALL_SUBTEST(test_default()); CALL_SUBTEST(test_normal());
diff --git a/unsupported/test/cxx11_tensor_random_cuda.cu b/unsupported/test/cxx11_tensor_random_cuda.cu deleted file mode 100644 index 5d091de..0000000 --- a/unsupported/test/cxx11_tensor_random_cuda.cu +++ /dev/null
@@ -1,35 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_random_cuda -#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int -#define EIGEN_USE_GPU - -#include "main.h" -#include <Eigen/CXX11/Tensor> - -static void test_default() -{ - Tensor<std::complex<float>, 1> vec(6); - vec.setRandom(); - - // Fixme: we should check that the generated numbers follow a uniform - // distribution instead. - for (int i = 1; i < 6; ++i) { - VERIFY_IS_NOT_EQUAL(vec(i), vec(i-1)); - } -} - - -void test_cxx11_tensor_random_cuda() -{ - CALL_SUBTEST(test_default()); -}
diff --git a/unsupported/test/cxx11_tensor_random_gpu.cu b/unsupported/test/cxx11_tensor_random_gpu.cu new file mode 100644 index 0000000..090986e --- /dev/null +++ b/unsupported/test/cxx11_tensor_random_gpu.cu
@@ -0,0 +1,86 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int +#define EIGEN_USE_GPU + +#include "main.h" +#include <Eigen/CXX11/Tensor> + +#include <Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + +void test_gpu_random_uniform() +{ + Tensor<float, 2> out(72,97); + out.setZero(); + + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_out; + gpuMalloc((void**)(&d_out), out_bytes); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97); + + gpu_out.device(gpu_device) = gpu_out.random(); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + + // For now we just check this code doesn't crash. + // TODO: come up with a valid test of randomness +} + + +void test_gpu_random_normal() +{ + Tensor<float, 2> out(72,97); + out.setZero(); + + std::size_t out_bytes = out.size() * sizeof(float); + + float* d_out; + gpuMalloc((void**)(&d_out), out_bytes); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97); + + Eigen::internal::NormalRandomGenerator<float> gen(true); + gpu_out.device(gpu_device) = gpu_out.random(gen); + + assert(gpuMemcpyAsync(out.data(), d_out, out_bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); +} + +static void test_complex() +{ + Tensor<std::complex<float>, 1> vec(6); + vec.setRandom(); + + // Fixme: we should check that the generated numbers follow a uniform + // distribution instead. + for (int i = 1; i < 6; ++i) { + VERIFY_IS_NOT_EQUAL(vec(i), vec(i-1)); + } +} + + +EIGEN_DECLARE_TEST(cxx11_tensor_random_gpu) +{ + CALL_SUBTEST(test_gpu_random_uniform()); + CALL_SUBTEST(test_gpu_random_normal()); + CALL_SUBTEST(test_complex()); +}
diff --git a/unsupported/test/cxx11_tensor_reduction.cpp b/unsupported/test/cxx11_tensor_reduction.cpp index 6a12890..996dba8 100644 --- a/unsupported/test/cxx11_tensor_reduction.cpp +++ b/unsupported/test/cxx11_tensor_reduction.cpp
@@ -53,20 +53,22 @@ } } -template <int DataLayout> +template <typename Scalar,int DataLayout> static void test_simple_reductions() { - Tensor<float, 4, DataLayout> tensor(2, 3, 5, 7); + Tensor<Scalar, 4, DataLayout> tensor(2, 3, 5, 7); tensor.setRandom(); + // Add a little offset so that the product reductions won't be close to zero. + tensor += tensor.constant(Scalar(0.5f)); array<ptrdiff_t, 2> reduction_axis2; reduction_axis2[0] = 1; reduction_axis2[1] = 3; - Tensor<float, 2, DataLayout> result = tensor.sum(reduction_axis2); + Tensor<Scalar, 2, DataLayout> result = tensor.sum(reduction_axis2); VERIFY_IS_EQUAL(result.dimension(0), 2); VERIFY_IS_EQUAL(result.dimension(1), 5); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 5; ++j) { - float sum = 0.0f; + Scalar sum = Scalar(0.0f); for (int k = 0; k < 3; ++k) { for (int l = 0; l < 7; ++l) { sum += tensor(i, k, j, l); @@ -77,7 +79,7 @@ } { - Tensor<float, 0, DataLayout> sum1 = tensor.sum(); + Tensor<Scalar, 0, DataLayout> sum1 = tensor.sum(); VERIFY_IS_EQUAL(sum1.rank(), 0); array<ptrdiff_t, 4> reduction_axis4; @@ -85,7 +87,7 @@ reduction_axis4[1] = 1; reduction_axis4[2] = 2; reduction_axis4[3] = 3; - Tensor<float, 0, DataLayout> sum2 = tensor.sum(reduction_axis4); + Tensor<Scalar, 0, DataLayout> sum2 = tensor.sum(reduction_axis4); VERIFY_IS_EQUAL(sum2.rank(), 0); VERIFY_IS_APPROX(sum1(), sum2()); @@ -98,7 +100,7 @@ VERIFY_IS_EQUAL(result.dimension(1), 7); for (int i = 0; i < 3; ++i) { for (int j = 0; j < 7; ++j) { - float prod = 1.0f; + Scalar prod = Scalar(1.0f); for (int k = 0; k < 2; ++k) { for (int l = 0; l < 5; ++l) { prod *= tensor(k, i, l, j); @@ -109,7 +111,7 @@ } { - Tensor<float, 0, DataLayout> prod1 = tensor.prod(); + Tensor<Scalar, 0, DataLayout> prod1 = tensor.prod(); VERIFY_IS_EQUAL(prod1.rank(), 0); array<ptrdiff_t, 4> reduction_axis4; @@ -117,7 +119,7 @@ reduction_axis4[1] = 1; reduction_axis4[2] = 2; reduction_axis4[3] = 3; - Tensor<float, 0, DataLayout> prod2 = tensor.prod(reduction_axis4); + Tensor<Scalar, 0, DataLayout> prod2 = tensor.prod(reduction_axis4); VERIFY_IS_EQUAL(prod2.rank(), 0); VERIFY_IS_APPROX(prod1(), prod2()); @@ -130,7 +132,7 @@ VERIFY_IS_EQUAL(result.dimension(1), 7); for (int i = 0; i < 3; ++i) { for (int j = 0; j < 7; ++j) { - float max_val = std::numeric_limits<float>::lowest(); + Scalar max_val = std::numeric_limits<Scalar>::lowest(); for (int k = 0; k < 2; ++k) { for (int l = 0; l < 5; ++l) { max_val = (std::max)(max_val, tensor(k, i, l, j)); @@ -141,7 +143,7 @@ } { - Tensor<float, 0, DataLayout> max1 = tensor.maximum(); + Tensor<Scalar, 0, DataLayout> max1 = tensor.maximum(); VERIFY_IS_EQUAL(max1.rank(), 0); array<ptrdiff_t, 4> reduction_axis4; @@ -149,7 +151,7 @@ reduction_axis4[1] = 1; reduction_axis4[2] = 2; reduction_axis4[3] = 3; - Tensor<float, 0, DataLayout> max2 = tensor.maximum(reduction_axis4); + Tensor<Scalar, 0, DataLayout> max2 = tensor.maximum(reduction_axis4); VERIFY_IS_EQUAL(max2.rank(), 0); VERIFY_IS_APPROX(max1(), max2()); @@ -162,7 +164,7 @@ VERIFY_IS_EQUAL(result.dimension(1), 7); for (int i = 0; i < 5; ++i) { for (int j = 0; j < 7; ++j) { - float min_val = (std::numeric_limits<float>::max)(); + Scalar min_val = (std::numeric_limits<Scalar>::max)(); for (int k = 0; k < 2; ++k) { for (int l = 0; l < 3; ++l) { min_val = (std::min)(min_val, tensor(k, l, i, j)); @@ -173,7 +175,7 @@ } { - Tensor<float, 0, DataLayout> min1 = tensor.minimum(); + Tensor<Scalar, 0, DataLayout> min1 = tensor.minimum(); VERIFY_IS_EQUAL(min1.rank(), 0); array<ptrdiff_t, 4> reduction_axis4; @@ -181,7 +183,7 @@ reduction_axis4[1] = 1; reduction_axis4[2] = 2; reduction_axis4[3] = 3; - Tensor<float, 0, DataLayout> min2 = tensor.minimum(reduction_axis4); + Tensor<Scalar, 0, DataLayout> min2 = tensor.minimum(reduction_axis4); VERIFY_IS_EQUAL(min2.rank(), 0); VERIFY_IS_APPROX(min1(), min2()); @@ -194,7 +196,7 @@ VERIFY_IS_EQUAL(result.dimension(1), 7); for (int i = 0; i < 5; ++i) { for (int j = 0; j < 7; ++j) { - float sum = 0.0f; + Scalar sum = Scalar(0.0f); int count = 0; for (int k = 0; k < 2; ++k) { for (int l = 0; l < 3; ++l) { @@ -207,7 +209,7 @@ } { - Tensor<float, 0, DataLayout> mean1 = tensor.mean(); + Tensor<Scalar, 0, DataLayout> mean1 = tensor.mean(); VERIFY_IS_EQUAL(mean1.rank(), 0); array<ptrdiff_t, 4> reduction_axis4; @@ -215,7 +217,7 @@ reduction_axis4[1] = 1; reduction_axis4[2] = 2; reduction_axis4[3] = 3; - Tensor<float, 0, DataLayout> mean2 = tensor.mean(reduction_axis4); + Tensor<Scalar, 0, DataLayout> mean2 = tensor.mean(reduction_axis4); VERIFY_IS_EQUAL(mean2.rank(), 0); VERIFY_IS_APPROX(mean1(), mean2()); @@ -225,11 +227,11 @@ Tensor<int, 1> ints(10); std::iota(ints.data(), ints.data() + ints.dimension(0), 0); - TensorFixedSize<bool, Sizes<> > all; - all = ints.all(); - VERIFY(!all()); - all = (ints >= ints.constant(0)).all(); - VERIFY(all()); + TensorFixedSize<bool, Sizes<> > all_; + all_ = ints.all(); + VERIFY(!all_()); + all_ = (ints >= ints.constant(0)).all(); + VERIFY(all_()); TensorFixedSize<bool, Sizes<> > any; any = (ints > ints.constant(10)).any(); @@ -239,6 +241,33 @@ } } + +template <int DataLayout> +static void test_reductions_in_expr() { + Tensor<float, 4, DataLayout> tensor(2, 3, 5, 7); + tensor.setRandom(); + array<ptrdiff_t, 2> reduction_axis2; + reduction_axis2[0] = 1; + reduction_axis2[1] = 3; + + Tensor<float, 2, DataLayout> result(2, 5); + result = result.constant(1.0f) - tensor.sum(reduction_axis2); + VERIFY_IS_EQUAL(result.dimension(0), 2); + VERIFY_IS_EQUAL(result.dimension(1), 5); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 5; ++j) { + float sum = 0.0f; + for (int k = 0; k < 3; ++k) { + for (int l = 0; l < 7; ++l) { + sum += tensor(i, k, j, l); + } + } + VERIFY_IS_APPROX(result(i, j), 1.0f - sum); + } + } +} + + template <int DataLayout> static void test_full_reductions() { Tensor<float, 2, DataLayout> tensor(2, 3); @@ -341,7 +370,7 @@ Tensor<float, 2, DataLayout> out(72, 97); in.setRandom(); -#ifndef EIGEN_HAS_CONSTEXPR +#if !EIGEN_HAS_CONSTEXPR array<int, 2> reduction_axis; reduction_axis[0] = 1; reduction_axis[1] = 3; @@ -359,7 +388,7 @@ expected = (std::max)(expected, in(i, k, j, l)); } } - VERIFY_IS_APPROX(out(i, j), expected); + VERIFY_IS_EQUAL(out(i, j), expected); } } } @@ -371,7 +400,7 @@ in.setRandom(); // Reduce on the innermost dimensions. -#ifndef EIGEN_HAS_CONSTEXPR +#if !EIGEN_HAS_CONSTEXPR array<int, 2> reduction_axis; reduction_axis[0] = 0; reduction_axis[1] = 1; @@ -390,7 +419,7 @@ expected = (std::max)(expected, in(l, k, i, j)); } } - VERIFY_IS_APPROX(out(i, j), expected); + VERIFY_IS_EQUAL(out(i, j), expected); } } } @@ -402,7 +431,7 @@ in.setRandom(); // Reduce on the innermost dimensions. -#ifndef EIGEN_HAS_CONSTEXPR +#if !EIGEN_HAS_CONSTEXPR array<int, 2> reduction_axis; reduction_axis[0] = 2; reduction_axis[1] = 3; @@ -421,7 +450,7 @@ expected = (std::max)(expected, in(i, j, k, l)); } } - VERIFY_IS_APPROX(out(i, j), expected); + VERIFY_IS_EQUAL(out(i, j), expected); } } } @@ -433,7 +462,7 @@ in.setRandom(); // Reduce on the innermost dimensions. -#ifndef EIGEN_HAS_CONSTEXPR +#if !EIGEN_HAS_CONSTEXPR array<int, 2> reduction_axis; reduction_axis[0] = 1; reduction_axis[1] = 2; @@ -452,16 +481,38 @@ expected = (std::max)(expected, in(i, k, l, j)); } } - VERIFY_IS_APPROX(out(i, j), expected); + VERIFY_IS_EQUAL(out(i, j), expected); } } } -void test_cxx11_tensor_reduction() { +static void test_sum_accuracy() { + Tensor<float, 3> tensor(101, 101, 101); + for (float prescribed_mean : {1.0f, 10.0f, 100.0f, 1000.0f, 10000.0f}) { + tensor.setRandom(); + tensor += tensor.constant(prescribed_mean); + + Tensor<float, 0> sum = tensor.sum(); + double expected_sum = 0.0; + for (int i = 0; i < 101; ++i) { + for (int j = 0; j < 101; ++j) { + for (int k = 0; k < 101; ++k) { + expected_sum += static_cast<double>(tensor(i, j, k)); + } + } + } + VERIFY_IS_APPROX(sum(), static_cast<float>(expected_sum)); + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_reduction) { CALL_SUBTEST(test_trivial_reductions<ColMajor>()); CALL_SUBTEST(test_trivial_reductions<RowMajor>()); - CALL_SUBTEST(test_simple_reductions<ColMajor>()); - CALL_SUBTEST(test_simple_reductions<RowMajor>()); + CALL_SUBTEST(( test_simple_reductions<float,ColMajor>() )); + CALL_SUBTEST(( test_simple_reductions<float,RowMajor>() )); + CALL_SUBTEST(( test_simple_reductions<Eigen::half,ColMajor>() )); + CALL_SUBTEST(test_reductions_in_expr<ColMajor>()); + CALL_SUBTEST(test_reductions_in_expr<RowMajor>()); CALL_SUBTEST(test_full_reductions<ColMajor>()); CALL_SUBTEST(test_full_reductions<RowMajor>()); CALL_SUBTEST(test_user_defined_reductions<ColMajor>()); @@ -476,4 +527,5 @@ CALL_SUBTEST(test_innermost_first_dims<RowMajor>()); CALL_SUBTEST(test_reduce_middle_dims<ColMajor>()); CALL_SUBTEST(test_reduce_middle_dims<RowMajor>()); + CALL_SUBTEST(test_sum_accuracy()); }
diff --git a/unsupported/test/cxx11_tensor_reduction_cuda.cu b/unsupported/test/cxx11_tensor_reduction_cuda.cu deleted file mode 100644 index cad0c08..0000000 --- a/unsupported/test/cxx11_tensor_reduction_cuda.cu +++ /dev/null
@@ -1,59 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2015 Benoit Steiner <benoit.steiner.goog@gmail.com> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#define EIGEN_TEST_NO_LONGDOUBLE -#define EIGEN_TEST_NO_COMPLEX -#define EIGEN_TEST_FUNC cxx11_tensor_reduction_cuda -#define EIGEN_USE_GPU - -#include "main.h" -#include <unsupported/Eigen/CXX11/Tensor> - - -template<int DataLayout> -static void test_full_reductions() { - - Eigen::CudaStreamDevice stream; - Eigen::GpuDevice gpu_device(&stream); - - const int num_rows = internal::random<int>(1024, 5*1024); - const int num_cols = internal::random<int>(1024, 5*1024); - - Tensor<float, 2, DataLayout> in(num_rows, num_cols); - in.setRandom(); - - Tensor<float, 0, DataLayout> full_redux; - full_redux = in.sum(); - - std::size_t in_bytes = in.size() * sizeof(float); - std::size_t out_bytes = full_redux.size() * sizeof(float); - float* gpu_in_ptr = static_cast<float*>(gpu_device.allocate(in_bytes)); - float* gpu_out_ptr = static_cast<float*>(gpu_device.allocate(out_bytes)); - gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes); - - TensorMap<Tensor<float, 2, DataLayout> > in_gpu(gpu_in_ptr, num_rows, num_cols); - TensorMap<Tensor<float, 0, DataLayout> > out_gpu(gpu_out_ptr); - - out_gpu.device(gpu_device) = in_gpu.sum(); - - Tensor<float, 0, DataLayout> full_redux_gpu; - gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes); - gpu_device.synchronize(); - - // Check that the CPU and GPU reductions return the same result. - VERIFY_IS_APPROX(full_redux(), full_redux_gpu()); - - gpu_device.deallocate(gpu_in_ptr); - gpu_device.deallocate(gpu_out_ptr); -} - -void test_cxx11_tensor_reduction_cuda() { - CALL_SUBTEST_1(test_full_reductions<ColMajor>()); - CALL_SUBTEST_2(test_full_reductions<RowMajor>()); -}
diff --git a/unsupported/test/cxx11_tensor_reduction_gpu.cu b/unsupported/test/cxx11_tensor_reduction_gpu.cu new file mode 100644 index 0000000..122ac94 --- /dev/null +++ b/unsupported/test/cxx11_tensor_reduction_gpu.cu
@@ -0,0 +1,154 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + + +template<typename Type, int DataLayout> +static void test_full_reductions() { + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + const int num_rows = internal::random<int>(1024, 5*1024); + const int num_cols = internal::random<int>(1024, 5*1024); + + Tensor<Type, 2, DataLayout> in(num_rows, num_cols); + in.setRandom(); + + Tensor<Type, 0, DataLayout> full_redux; + full_redux = in.sum(); + + std::size_t in_bytes = in.size() * sizeof(Type); + std::size_t out_bytes = full_redux.size() * sizeof(Type); + Type* gpu_in_ptr = static_cast<Type*>(gpu_device.allocate(in_bytes)); + Type* gpu_out_ptr = static_cast<Type*>(gpu_device.allocate(out_bytes)); + gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes); + + TensorMap<Tensor<Type, 2, DataLayout> > in_gpu(gpu_in_ptr, num_rows, num_cols); + TensorMap<Tensor<Type, 0, DataLayout> > out_gpu(gpu_out_ptr); + + out_gpu.device(gpu_device) = in_gpu.sum(); + + Tensor<Type, 0, DataLayout> full_redux_gpu; + gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes); + gpu_device.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux(), full_redux_gpu()); + + gpu_device.deallocate(gpu_in_ptr); + gpu_device.deallocate(gpu_out_ptr); +} + +template<typename Type, int DataLayout> +static void test_first_dim_reductions() { + int dim_x = 33; + int dim_y = 1; + int dim_z = 128; + + Tensor<Type, 3, DataLayout> in(dim_x, dim_y, dim_z); + in.setRandom(); + + Eigen::array<int, 1> red_axis; + red_axis[0] = 0; + Tensor<Type, 2, DataLayout> redux = in.sum(red_axis); + + // Create device + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice dev(&stream); + + // Create data(T) + Type* in_data = (Type*)dev.allocate(dim_x*dim_y*dim_z*sizeof(Type)); + Type* out_data = (Type*)dev.allocate(dim_z*dim_y*sizeof(Type)); + Eigen::TensorMap<Eigen::Tensor<Type, 3, DataLayout> > gpu_in(in_data, dim_x, dim_y, dim_z); + Eigen::TensorMap<Eigen::Tensor<Type, 2, DataLayout> > gpu_out(out_data, dim_y, dim_z); + + // Perform operation + dev.memcpyHostToDevice(in_data, in.data(), in.size()*sizeof(Type)); + gpu_out.device(dev) = gpu_in.sum(red_axis); + gpu_out.device(dev) += gpu_in.sum(red_axis); + Tensor<Type, 2, DataLayout> redux_gpu(dim_y, dim_z); + dev.memcpyDeviceToHost(redux_gpu.data(), out_data, gpu_out.size()*sizeof(Type)); + dev.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + for (int i = 0; i < gpu_out.size(); ++i) { + VERIFY_IS_APPROX(2*redux(i), redux_gpu(i)); + } + + dev.deallocate(in_data); + dev.deallocate(out_data); +} + +template<typename Type, int DataLayout> +static void test_last_dim_reductions() { + int dim_x = 128; + int dim_y = 1; + int dim_z = 33; + + Tensor<Type, 3, DataLayout> in(dim_x, dim_y, dim_z); + in.setRandom(); + + Eigen::array<int, 1> red_axis; + red_axis[0] = 2; + Tensor<Type, 2, DataLayout> redux = in.sum(red_axis); + + // Create device + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice dev(&stream); + + // Create data + Type* in_data = (Type*)dev.allocate(dim_x*dim_y*dim_z*sizeof(Type)); + Type* out_data = (Type*)dev.allocate(dim_x*dim_y*sizeof(Type)); + Eigen::TensorMap<Eigen::Tensor<Type, 3, DataLayout> > gpu_in(in_data, dim_x, dim_y, dim_z); + Eigen::TensorMap<Eigen::Tensor<Type, 2, DataLayout> > gpu_out(out_data, dim_x, dim_y); + + // Perform operation + dev.memcpyHostToDevice(in_data, in.data(), in.size()*sizeof(Type)); + gpu_out.device(dev) = gpu_in.sum(red_axis); + gpu_out.device(dev) += gpu_in.sum(red_axis); + Tensor<Type, 2, DataLayout> redux_gpu(dim_x, dim_y); + dev.memcpyDeviceToHost(redux_gpu.data(), out_data, gpu_out.size()*sizeof(Type)); + dev.synchronize(); + + // Check that the CPU and GPU reductions return the same result. + for (int i = 0; i < gpu_out.size(); ++i) { + VERIFY_IS_APPROX(2*redux(i), redux_gpu(i)); + } + + dev.deallocate(in_data); + dev.deallocate(out_data); +} + + +EIGEN_DECLARE_TEST(cxx11_tensor_reduction_gpu) { + CALL_SUBTEST_1((test_full_reductions<float, ColMajor>())); + CALL_SUBTEST_1((test_full_reductions<double, ColMajor>())); + CALL_SUBTEST_2((test_full_reductions<float, RowMajor>())); + CALL_SUBTEST_2((test_full_reductions<double, RowMajor>())); + + CALL_SUBTEST_3((test_first_dim_reductions<float, ColMajor>())); + CALL_SUBTEST_3((test_first_dim_reductions<double, ColMajor>())); + CALL_SUBTEST_4((test_first_dim_reductions<float, RowMajor>())); +// Outer reductions of doubles aren't supported just yet. +// CALL_SUBTEST_4((test_first_dim_reductions<double, RowMajor>())) + + CALL_SUBTEST_5((test_last_dim_reductions<float, ColMajor>())); +// Outer reductions of doubles aren't supported just yet. +// CALL_SUBTEST_5((test_last_dim_reductions<double, ColMajor>())); + CALL_SUBTEST_6((test_last_dim_reductions<float, RowMajor>())); + CALL_SUBTEST_6((test_last_dim_reductions<double, RowMajor>())); +}
diff --git a/unsupported/test/cxx11_tensor_reduction_sycl.cpp b/unsupported/test/cxx11_tensor_reduction_sycl.cpp new file mode 100644 index 0000000..f526299 --- /dev/null +++ b/unsupported/test/cxx11_tensor_reduction_sycl.cpp
@@ -0,0 +1,181 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_full_reductions_mean_sycl(const Eigen::SyclDevice& sycl_device) { + + const IndexType num_rows = 452; + const IndexType num_cols = 765; + array<IndexType, 2> tensorRange = {{num_rows, num_cols}}; + + Tensor<DataType, 2, DataLayout, IndexType> in(tensorRange); + Tensor<DataType, 0, DataLayout, IndexType> full_redux; + Tensor<DataType, 0, DataLayout, IndexType> full_redux_gpu; + + in.setRandom(); + + full_redux = in.mean(); + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(in.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data =(DataType*)sycl_device.allocate(sizeof(DataType)); + + TensorMap<Tensor<DataType, 2, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 0, DataLayout, IndexType> > out_gpu(gpu_out_data); + + sycl_device.memcpyHostToDevice(gpu_in_data, in.data(),(in.dimensions().TotalSize())*sizeof(DataType)); + out_gpu.device(sycl_device) = in_gpu.mean(); + sycl_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_data, sizeof(DataType)); + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux_gpu(), full_redux()); + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_full_reductions_min_sycl(const Eigen::SyclDevice& sycl_device) { + + const IndexType num_rows = 876; + const IndexType num_cols = 953; + array<IndexType, 2> tensorRange = {{num_rows, num_cols}}; + + Tensor<DataType, 2, DataLayout, IndexType> in(tensorRange); + Tensor<DataType, 0, DataLayout, IndexType> full_redux; + Tensor<DataType, 0, DataLayout, IndexType> full_redux_gpu; + + in.setRandom(); + + full_redux = in.minimum(); + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(in.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data =(DataType*)sycl_device.allocate(sizeof(DataType)); + + TensorMap<Tensor<DataType, 2, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 0, DataLayout, IndexType> > out_gpu(gpu_out_data); + + sycl_device.memcpyHostToDevice(gpu_in_data, in.data(),(in.dimensions().TotalSize())*sizeof(DataType)); + out_gpu.device(sycl_device) = in_gpu.minimum(); + sycl_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_data, sizeof(DataType)); + // Check that the CPU and GPU reductions return the same result. + VERIFY_IS_APPROX(full_redux_gpu(), full_redux()); + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_first_dim_reductions_max_sycl(const Eigen::SyclDevice& sycl_device) { + + IndexType dim_x = 145; + IndexType dim_y = 1; + IndexType dim_z = 67; + + array<IndexType, 3> tensorRange = {{dim_x, dim_y, dim_z}}; + Eigen::array<IndexType, 1> red_axis; + red_axis[0] = 0; + array<IndexType, 2> reduced_tensorRange = {{dim_y, dim_z}}; + + Tensor<DataType, 3, DataLayout, IndexType> in(tensorRange); + Tensor<DataType, 2, DataLayout, IndexType> redux(reduced_tensorRange); + Tensor<DataType, 2, DataLayout, IndexType> redux_gpu(reduced_tensorRange); + + in.setRandom(); + + redux= in.maximum(red_axis); + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(in.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data = static_cast<DataType*>(sycl_device.allocate(redux_gpu.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 3, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 2, DataLayout, IndexType> > out_gpu(gpu_out_data, reduced_tensorRange); + + sycl_device.memcpyHostToDevice(gpu_in_data, in.data(),(in.dimensions().TotalSize())*sizeof(DataType)); + out_gpu.device(sycl_device) = in_gpu.maximum(red_axis); + sycl_device.memcpyDeviceToHost(redux_gpu.data(), gpu_out_data, redux_gpu.dimensions().TotalSize()*sizeof(DataType)); + + // Check that the CPU and GPU reductions return the same result. + for(IndexType j=0; j<reduced_tensorRange[0]; j++ ) + for(IndexType k=0; k<reduced_tensorRange[1]; k++ ) + VERIFY_IS_APPROX(redux_gpu(j,k), redux(j,k)); + + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_last_dim_reductions_sum_sycl(const Eigen::SyclDevice &sycl_device) { + + IndexType dim_x = 567; + IndexType dim_y = 1; + IndexType dim_z = 47; + + array<IndexType, 3> tensorRange = {{dim_x, dim_y, dim_z}}; + Eigen::array<IndexType, 1> red_axis; + red_axis[0] = 2; + array<IndexType, 2> reduced_tensorRange = {{dim_x, dim_y}}; + + Tensor<DataType, 3, DataLayout, IndexType> in(tensorRange); + Tensor<DataType, 2, DataLayout, IndexType> redux(reduced_tensorRange); + Tensor<DataType, 2, DataLayout, IndexType> redux_gpu(reduced_tensorRange); + + in.setRandom(); + + redux= in.sum(red_axis); + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(in.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data = static_cast<DataType*>(sycl_device.allocate(redux_gpu.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 3, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 2, DataLayout, IndexType> > out_gpu(gpu_out_data, reduced_tensorRange); + + sycl_device.memcpyHostToDevice(gpu_in_data, in.data(),(in.dimensions().TotalSize())*sizeof(DataType)); + out_gpu.device(sycl_device) = in_gpu.sum(red_axis); + sycl_device.memcpyDeviceToHost(redux_gpu.data(), gpu_out_data, redux_gpu.dimensions().TotalSize()*sizeof(DataType)); + // Check that the CPU and GPU reductions return the same result. + for(IndexType j=0; j<reduced_tensorRange[0]; j++ ) + for(IndexType k=0; k<reduced_tensorRange[1]; k++ ) + VERIFY_IS_APPROX(redux_gpu(j,k), redux(j,k)); + + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); + +} +template<typename DataType> void sycl_reduction_test_per_device(const cl::sycl::device& d){ + std::cout << "Running on " << d.template get_info<cl::sycl::info::device::name>() << std::endl; + QueueInterface queueInterface(d); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + + test_full_reductions_mean_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_full_reductions_min_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_first_dim_reductions_max_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_last_dim_reductions_sum_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_full_reductions_mean_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_full_reductions_min_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_first_dim_reductions_max_sycl<DataType, ColMajor, int64_t>(sycl_device); + test_last_dim_reductions_sum_sycl<DataType, ColMajor, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_reduction_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_reduction_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_ref.cpp b/unsupported/test/cxx11_tensor_ref.cpp index c8f105e..7dbd047 100644 --- a/unsupported/test/cxx11_tensor_ref.cpp +++ b/unsupported/test/cxx11_tensor_ref.cpp
@@ -235,7 +235,7 @@ } -void test_cxx11_tensor_ref() +EIGEN_DECLARE_TEST(cxx11_tensor_ref) { CALL_SUBTEST(test_simple_lvalue_ref()); CALL_SUBTEST(test_simple_rvalue_ref());
diff --git a/unsupported/test/cxx11_tensor_reverse.cpp b/unsupported/test/cxx11_tensor_reverse.cpp index b35b8d2..5e44ec0 100644 --- a/unsupported/test/cxx11_tensor_reverse.cpp +++ b/unsupported/test/cxx11_tensor_reverse.cpp
@@ -179,7 +179,7 @@ } -void test_cxx11_tensor_reverse() +EIGEN_DECLARE_TEST(cxx11_tensor_reverse) { CALL_SUBTEST(test_simple_reverse<ColMajor>()); CALL_SUBTEST(test_simple_reverse<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_reverse_sycl.cpp b/unsupported/test/cxx11_tensor_reverse_sycl.cpp new file mode 100644 index 0000000..77c2235 --- /dev/null +++ b/unsupported/test/cxx11_tensor_reverse_sycl.cpp
@@ -0,0 +1,221 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2015 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_reverse(const Eigen::SyclDevice& sycl_device) { + + IndexType dim1 = 2; + IndexType dim2 = 3; + IndexType dim3 = 5; + IndexType dim4 = 7; + + array<IndexType, 4> tensorRange = {{dim1, dim2, dim3, dim4}}; + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout, IndexType> reversed_tensor(tensorRange); + tensor.setRandom(); + + array<bool, 4> dim_rev; + dim_rev[0] = false; + dim_rev[1] = true; + dim_rev[2] = true; + dim_rev[3] = false; + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(tensor.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data =static_cast<DataType*>(sycl_device.allocate(reversed_tensor.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 4, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType> > out_gpu(gpu_out_data, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_in_data, tensor.data(),(tensor.dimensions().TotalSize())*sizeof(DataType)); + out_gpu.device(sycl_device) = in_gpu.reverse(dim_rev); + sycl_device.memcpyDeviceToHost(reversed_tensor.data(), gpu_out_data, reversed_tensor.dimensions().TotalSize()*sizeof(DataType)); + // Check that the CPU and GPU reductions return the same result. + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), reversed_tensor(i,2-j,4-k,l)); + } + } + } + } + dim_rev[0] = true; + dim_rev[1] = false; + dim_rev[2] = false; + dim_rev[3] = false; + + out_gpu.device(sycl_device) = in_gpu.reverse(dim_rev); + sycl_device.memcpyDeviceToHost(reversed_tensor.data(), gpu_out_data, reversed_tensor.dimensions().TotalSize()*sizeof(DataType)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), reversed_tensor(1-i,j,k,l)); + } + } + } + } + + dim_rev[0] = true; + dim_rev[1] = false; + dim_rev[2] = false; + dim_rev[3] = true; + out_gpu.device(sycl_device) = in_gpu.reverse(dim_rev); + sycl_device.memcpyDeviceToHost(reversed_tensor.data(), gpu_out_data, reversed_tensor.dimensions().TotalSize()*sizeof(DataType)); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), reversed_tensor(1-i,j,k,6-l)); + } + } + } + } + + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} + + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_expr_reverse(const Eigen::SyclDevice& sycl_device, bool LValue) +{ + IndexType dim1 = 2; + IndexType dim2 = 3; + IndexType dim3 = 5; + IndexType dim4 = 7; + + array<IndexType, 4> tensorRange = {{dim1, dim2, dim3, dim4}}; + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout, IndexType> expected(tensorRange); + Tensor<DataType, 4, DataLayout, IndexType> result(tensorRange); + tensor.setRandom(); + + array<bool, 4> dim_rev; + dim_rev[0] = false; + dim_rev[1] = true; + dim_rev[2] = false; + dim_rev[3] = true; + + DataType* gpu_in_data = static_cast<DataType*>(sycl_device.allocate(tensor.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data_expected =static_cast<DataType*>(sycl_device.allocate(expected.dimensions().TotalSize()*sizeof(DataType))); + DataType* gpu_out_data_result =static_cast<DataType*>(sycl_device.allocate(result.dimensions().TotalSize()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 4, DataLayout, IndexType> > in_gpu(gpu_in_data, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType> > out_gpu_expected(gpu_out_data_expected, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout, IndexType> > out_gpu_result(gpu_out_data_result, tensorRange); + + + sycl_device.memcpyHostToDevice(gpu_in_data, tensor.data(),(tensor.dimensions().TotalSize())*sizeof(DataType)); + + if (LValue) { + out_gpu_expected.reverse(dim_rev).device(sycl_device) = in_gpu; + } else { + out_gpu_expected.device(sycl_device) = in_gpu.reverse(dim_rev); + } + sycl_device.memcpyDeviceToHost(expected.data(), gpu_out_data_expected, expected.dimensions().TotalSize()*sizeof(DataType)); + + + array<IndexType, 4> src_slice_dim; + src_slice_dim[0] = 2; + src_slice_dim[1] = 3; + src_slice_dim[2] = 1; + src_slice_dim[3] = 7; + array<IndexType, 4> src_slice_start; + src_slice_start[0] = 0; + src_slice_start[1] = 0; + src_slice_start[2] = 0; + src_slice_start[3] = 0; + array<IndexType, 4> dst_slice_dim = src_slice_dim; + array<IndexType, 4> dst_slice_start = src_slice_start; + + for (IndexType i = 0; i < 5; ++i) { + if (LValue) { + out_gpu_result.slice(dst_slice_start, dst_slice_dim).reverse(dim_rev).device(sycl_device) = + in_gpu.slice(src_slice_start, src_slice_dim); + } else { + out_gpu_result.slice(dst_slice_start, dst_slice_dim).device(sycl_device) = + in_gpu.slice(src_slice_start, src_slice_dim).reverse(dim_rev); + } + src_slice_start[2] += 1; + dst_slice_start[2] += 1; + } + sycl_device.memcpyDeviceToHost(result.data(), gpu_out_data_result, result.dimensions().TotalSize()*sizeof(DataType)); + + for (IndexType i = 0; i < expected.dimension(0); ++i) { + for (IndexType j = 0; j < expected.dimension(1); ++j) { + for (IndexType k = 0; k < expected.dimension(2); ++k) { + for (IndexType l = 0; l < expected.dimension(3); ++l) { + VERIFY_IS_EQUAL(result(i,j,k,l), expected(i,j,k,l)); + } + } + } + } + + dst_slice_start[2] = 0; + result.setRandom(); + sycl_device.memcpyHostToDevice(gpu_out_data_result, result.data(),(result.dimensions().TotalSize())*sizeof(DataType)); + for (IndexType i = 0; i < 5; ++i) { + if (LValue) { + out_gpu_result.slice(dst_slice_start, dst_slice_dim).reverse(dim_rev).device(sycl_device) = + in_gpu.slice(dst_slice_start, dst_slice_dim); + } else { + out_gpu_result.slice(dst_slice_start, dst_slice_dim).device(sycl_device) = + in_gpu.reverse(dim_rev).slice(dst_slice_start, dst_slice_dim); + } + dst_slice_start[2] += 1; + } + sycl_device.memcpyDeviceToHost(result.data(), gpu_out_data_result, result.dimensions().TotalSize()*sizeof(DataType)); + + for (IndexType i = 0; i < expected.dimension(0); ++i) { + for (IndexType j = 0; j < expected.dimension(1); ++j) { + for (IndexType k = 0; k < expected.dimension(2); ++k) { + for (IndexType l = 0; l < expected.dimension(3); ++l) { + VERIFY_IS_EQUAL(result(i,j,k,l), expected(i,j,k,l)); + } + } + } + } +} + + + +template<typename DataType> void sycl_reverse_test_per_device(const cl::sycl::device& d){ + std::cout << "Running on " << d.template get_info<cl::sycl::info::device::name>() << std::endl; + QueueInterface queueInterface(d); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_reverse<DataType, RowMajor, int64_t>(sycl_device); + test_simple_reverse<DataType, ColMajor, int64_t>(sycl_device); + test_expr_reverse<DataType, RowMajor, int64_t>(sycl_device, false); + test_expr_reverse<DataType, ColMajor, int64_t>(sycl_device, false); + test_expr_reverse<DataType, RowMajor, int64_t>(sycl_device, true); + test_expr_reverse<DataType, ColMajor, int64_t>(sycl_device, true); +} +EIGEN_DECLARE_TEST(cxx11_tensor_reverse_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_reverse_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_roundings.cpp b/unsupported/test/cxx11_tensor_roundings.cpp index 2c26151..83b5923 100644 --- a/unsupported/test/cxx11_tensor_roundings.cpp +++ b/unsupported/test/cxx11_tensor_roundings.cpp
@@ -54,7 +54,7 @@ } } -void test_cxx11_tensor_roundings() +EIGEN_DECLARE_TEST(cxx11_tensor_roundings) { CALL_SUBTEST(test_float_rounding()); CALL_SUBTEST(test_float_ceiling());
diff --git a/unsupported/test/cxx11_tensor_scan.cpp b/unsupported/test/cxx11_tensor_scan.cpp new file mode 100644 index 0000000..dccee9e --- /dev/null +++ b/unsupported/test/cxx11_tensor_scan.cpp
@@ -0,0 +1,110 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Igor Babuschkin <igor@babuschk.in> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" +#include <limits> +#include <numeric> +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; + +template <int DataLayout, typename Type=float, bool Exclusive = false> +static void test_1d_scan() +{ + int size = 50; + Tensor<Type, 1, DataLayout> tensor(size); + tensor.setRandom(); + Tensor<Type, 1, DataLayout> result = tensor.cumsum(0, Exclusive); + + VERIFY_IS_EQUAL(tensor.dimension(0), result.dimension(0)); + + float accum = 0; + for (int i = 0; i < size; i++) { + if (Exclusive) { + VERIFY_IS_EQUAL(result(i), accum); + accum += tensor(i); + } else { + accum += tensor(i); + VERIFY_IS_EQUAL(result(i), accum); + } + } + + accum = 1; + result = tensor.cumprod(0, Exclusive); + for (int i = 0; i < size; i++) { + if (Exclusive) { + VERIFY_IS_EQUAL(result(i), accum); + accum *= tensor(i); + } else { + accum *= tensor(i); + VERIFY_IS_EQUAL(result(i), accum); + } + } +} + +template <int DataLayout, typename Type=float> +static void test_4d_scan() +{ + int size = 5; + Tensor<Type, 4, DataLayout> tensor(size, size, size, size); + tensor.setRandom(); + + Tensor<Type, 4, DataLayout> result(size, size, size, size); + + result = tensor.cumsum(0); + float accum = 0; + for (int i = 0; i < size; i++) { + accum += tensor(i, 1, 2, 3); + VERIFY_IS_EQUAL(result(i, 1, 2, 3), accum); + } + result = tensor.cumsum(1); + accum = 0; + for (int i = 0; i < size; i++) { + accum += tensor(1, i, 2, 3); + VERIFY_IS_EQUAL(result(1, i, 2, 3), accum); + } + result = tensor.cumsum(2); + accum = 0; + for (int i = 0; i < size; i++) { + accum += tensor(1, 2, i, 3); + VERIFY_IS_EQUAL(result(1, 2, i, 3), accum); + } + result = tensor.cumsum(3); + accum = 0; + for (int i = 0; i < size; i++) { + accum += tensor(1, 2, 3, i); + VERIFY_IS_EQUAL(result(1, 2, 3, i), accum); + } +} + +template <int DataLayout> +static void test_tensor_maps() { + int inputs[20]; + TensorMap<Tensor<int, 1, DataLayout> > tensor_map(inputs, 20); + tensor_map.setRandom(); + + Tensor<int, 1, DataLayout> result = tensor_map.cumsum(0); + + int accum = 0; + for (int i = 0; i < 20; ++i) { + accum += tensor_map(i); + VERIFY_IS_EQUAL(result(i), accum); + } +} + +EIGEN_DECLARE_TEST(cxx11_tensor_scan) { + CALL_SUBTEST((test_1d_scan<ColMajor, float, true>())); + CALL_SUBTEST((test_1d_scan<ColMajor, float, false>())); + CALL_SUBTEST((test_1d_scan<RowMajor, float, true>())); + CALL_SUBTEST((test_1d_scan<RowMajor, float, false>())); + CALL_SUBTEST(test_4d_scan<ColMajor>()); + CALL_SUBTEST(test_4d_scan<RowMajor>()); + CALL_SUBTEST(test_tensor_maps<ColMajor>()); + CALL_SUBTEST(test_tensor_maps<RowMajor>()); +}
diff --git a/unsupported/test/cxx11_tensor_scan_gpu.cu b/unsupported/test/cxx11_tensor_scan_gpu.cu new file mode 100644 index 0000000..770a144 --- /dev/null +++ b/unsupported/test/cxx11_tensor_scan_gpu.cu
@@ -0,0 +1,78 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int +#define EIGEN_USE_GPU + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +#include <Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> + +using Eigen::Tensor; +typedef Tensor<float, 1>::DimensionPair DimPair; + +template<int DataLayout> +void test_gpu_cumsum(int m_size, int k_size, int n_size) +{ + std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl; + Tensor<float, 3, DataLayout> t_input(m_size, k_size, n_size); + Tensor<float, 3, DataLayout> t_result(m_size, k_size, n_size); + Tensor<float, 3, DataLayout> t_result_gpu(m_size, k_size, n_size); + + t_input.setRandom(); + + std::size_t t_input_bytes = t_input.size() * sizeof(float); + std::size_t t_result_bytes = t_result.size() * sizeof(float); + + float* d_t_input; + float* d_t_result; + + gpuMalloc((void**)(&d_t_input), t_input_bytes); + gpuMalloc((void**)(&d_t_result), t_result_bytes); + + gpuMemcpy(d_t_input, t_input.data(), t_input_bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + + Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> > + gpu_t_input(d_t_input, Eigen::array<int, 3>(m_size, k_size, n_size)); + Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> > + gpu_t_result(d_t_result, Eigen::array<int, 3>(m_size, k_size, n_size)); + + gpu_t_result.device(gpu_device) = gpu_t_input.cumsum(1); + t_result = t_input.cumsum(1); + + gpuMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, gpuMemcpyDeviceToHost); + for (DenseIndex i = 0; i < t_result.size(); i++) { + if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) { + continue; + } + if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) { + continue; + } + std::cout << "mismatch detected at index " << i << ": " << t_result(i) + << " vs " << t_result_gpu(i) << std::endl; + assert(false); + } + + gpuFree((void*)d_t_input); + gpuFree((void*)d_t_result); +} + + +EIGEN_DECLARE_TEST(cxx11_tensor_scan_gpu) +{ + CALL_SUBTEST_1(test_gpu_cumsum<ColMajor>(128, 128, 128)); + CALL_SUBTEST_2(test_gpu_cumsum<RowMajor>(128, 128, 128)); +}
diff --git a/unsupported/test/cxx11_tensor_shuffling.cpp b/unsupported/test/cxx11_tensor_shuffling.cpp index d11444a..2ec85d2 100644 --- a/unsupported/test/cxx11_tensor_shuffling.cpp +++ b/unsupported/test/cxx11_tensor_shuffling.cpp
@@ -81,12 +81,12 @@ Tensor<float, 4, DataLayout> expected; expected = tensor.shuffle(shuffles); - Tensor<float, 4, DataLayout> result(5,7,3,2); + Tensor<float, 4, DataLayout> result(5, 7, 3, 2); - array<int, 4> src_slice_dim{{2,3,1,7}}; - array<int, 4> src_slice_start{{0,0,0,0}}; - array<int, 4> dst_slice_dim{{1,7,3,2}}; - array<int, 4> dst_slice_start{{0,0,0,0}}; + array<ptrdiff_t, 4> src_slice_dim{{2, 3, 1, 7}}; + array<ptrdiff_t, 4> src_slice_start{{0, 0, 0, 0}}; + array<ptrdiff_t, 4> dst_slice_dim{{1, 7, 3, 2}}; + array<ptrdiff_t, 4> dst_slice_start{{0, 0, 0, 0}}; for (int i = 0; i < 5; ++i) { result.slice(dst_slice_start, dst_slice_dim) = @@ -215,7 +215,7 @@ } -void test_cxx11_tensor_shuffling() +EIGEN_DECLARE_TEST(cxx11_tensor_shuffling) { CALL_SUBTEST(test_simple_shuffling<ColMajor>()); CALL_SUBTEST(test_simple_shuffling<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_shuffling_sycl.cpp b/unsupported/test/cxx11_tensor_shuffling_sycl.cpp new file mode 100644 index 0000000..0e8cc3b --- /dev/null +++ b/unsupported/test/cxx11_tensor_shuffling_sycl.cpp
@@ -0,0 +1,119 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_shuffling_sycl(const Eigen::SyclDevice& sycl_device) +{ + IndexType sizeDim1 = 2; + IndexType sizeDim2 = 3; + IndexType sizeDim3 = 5; + IndexType sizeDim4 = 7; + array<IndexType, 4> tensorRange = {{sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; + Tensor<DataType, 4, DataLayout,IndexType> tensor(tensorRange); + Tensor<DataType, 4, DataLayout,IndexType> no_shuffle(tensorRange); + tensor.setRandom(); + + const size_t buffSize =tensor.size()*sizeof(DataType); + array<IndexType, 4> shuffles; + shuffles[0] = 0; + shuffles[1] = 1; + shuffles[2] = 2; + shuffles[3] = 3; + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(buffSize)); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(buffSize)); + + + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 4, DataLayout,IndexType>> gpu2(gpu_data2, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data1, tensor.data(), buffSize); + + gpu2.device(sycl_device)=gpu1.shuffle(shuffles); + sycl_device.memcpyDeviceToHost(no_shuffle.data(), gpu_data2, buffSize); + sycl_device.synchronize(); + + VERIFY_IS_EQUAL(no_shuffle.dimension(0), sizeDim1); + VERIFY_IS_EQUAL(no_shuffle.dimension(1), sizeDim2); + VERIFY_IS_EQUAL(no_shuffle.dimension(2), sizeDim3); + VERIFY_IS_EQUAL(no_shuffle.dimension(3), sizeDim4); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim4; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), no_shuffle(i,j,k,l)); + } + } + } + } + + shuffles[0] = 2; + shuffles[1] = 3; + shuffles[2] = 1; + shuffles[3] = 0; + array<IndexType, 4> tensorrangeShuffle = {{sizeDim3, sizeDim4, sizeDim2, sizeDim1}}; + Tensor<DataType, 4, DataLayout,IndexType> shuffle(tensorrangeShuffle); + DataType* gpu_data3 = static_cast<DataType*>(sycl_device.allocate(buffSize)); + TensorMap<Tensor<DataType, 4,DataLayout,IndexType>> gpu3(gpu_data3, tensorrangeShuffle); + + gpu3.device(sycl_device)=gpu1.shuffle(shuffles); + sycl_device.memcpyDeviceToHost(shuffle.data(), gpu_data3, buffSize); + sycl_device.synchronize(); + + VERIFY_IS_EQUAL(shuffle.dimension(0), sizeDim3); + VERIFY_IS_EQUAL(shuffle.dimension(1), sizeDim4); + VERIFY_IS_EQUAL(shuffle.dimension(2), sizeDim2); + VERIFY_IS_EQUAL(shuffle.dimension(3), sizeDim1); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + for (IndexType l = 0; l < sizeDim4; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), shuffle(k,l,j,i)); + } + } + } + } +} + + +template<typename DataType, typename dev_Selector> void sycl_shuffling_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_simple_shuffling_sycl<DataType, RowMajor, int64_t>(sycl_device); + test_simple_shuffling_sycl<DataType, ColMajor, int64_t>(sycl_device); + +} +EIGEN_DECLARE_TEST(cxx11_tensor_shuffling_sycl) +{ + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_shuffling_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_simple.cpp b/unsupported/test/cxx11_tensor_simple.cpp index 47d4d86..6d70f54 100644 --- a/unsupported/test/cxx11_tensor_simple.cpp +++ b/unsupported/test/cxx11_tensor_simple.cpp
@@ -195,7 +195,10 @@ VERIFY_IS_EQUAL((epsilon(0,2,1)), -1); VERIFY_IS_EQUAL((epsilon(1,0,2)), -1); - array<Eigen::DenseIndex, 3> dims{{2,3,4}}; + array<Eigen::DenseIndex, 3> dims; + dims[0] = 2; + dims[1] = 3; + dims[2] = 4; Tensor<int, 3> t1(dims); Tensor<int, 3, RowMajor> t2(dims); @@ -296,25 +299,24 @@ VERIFY_IS_EQUAL(epsilon.dimension(0), 2); VERIFY_IS_EQUAL(epsilon.dimension(1), 3); VERIFY_IS_EQUAL(epsilon.dimension(2), 7); - VERIFY_IS_EQUAL(epsilon.dimensions().TotalSize(), 2*3*7); + VERIFY_IS_EQUAL(epsilon.size(), 2*3*7); const int* old_data = epsilon.data(); epsilon.resize(3,2,7); VERIFY_IS_EQUAL(epsilon.dimension(0), 3); VERIFY_IS_EQUAL(epsilon.dimension(1), 2); VERIFY_IS_EQUAL(epsilon.dimension(2), 7); - VERIFY_IS_EQUAL(epsilon.dimensions().TotalSize(), 2*3*7); + VERIFY_IS_EQUAL(epsilon.size(), 2*3*7); VERIFY_IS_EQUAL(epsilon.data(), old_data); epsilon.resize(3,5,7); VERIFY_IS_EQUAL(epsilon.dimension(0), 3); VERIFY_IS_EQUAL(epsilon.dimension(1), 5); VERIFY_IS_EQUAL(epsilon.dimension(2), 7); - VERIFY_IS_EQUAL(epsilon.dimensions().TotalSize(), 3*5*7); - VERIFY_IS_NOT_EQUAL(epsilon.data(), old_data); + VERIFY_IS_EQUAL(epsilon.size(), 3*5*7); } -void test_cxx11_tensor_simple() +EIGEN_DECLARE_TEST(cxx11_tensor_simple) { CALL_SUBTEST(test_0d()); CALL_SUBTEST(test_1d());
diff --git a/unsupported/test/cxx11_tensor_striding.cpp b/unsupported/test/cxx11_tensor_striding.cpp index 935b908..aefdfa9 100644 --- a/unsupported/test/cxx11_tensor_striding.cpp +++ b/unsupported/test/cxx11_tensor_striding.cpp
@@ -110,7 +110,7 @@ } -void test_cxx11_tensor_striding() +EIGEN_DECLARE_TEST(cxx11_tensor_striding) { CALL_SUBTEST(test_simple_striding<ColMajor>()); CALL_SUBTEST(test_simple_striding<RowMajor>());
diff --git a/unsupported/test/cxx11_tensor_striding_sycl.cpp b/unsupported/test/cxx11_tensor_striding_sycl.cpp new file mode 100644 index 0000000..d3b1fa7 --- /dev/null +++ b/unsupported/test/cxx11_tensor_striding_sycl.cpp
@@ -0,0 +1,203 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include <iostream> +#include <chrono> +#include <ctime> + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + + +template <typename DataType, int DataLayout, typename IndexType> +static void test_simple_striding(const Eigen::SyclDevice& sycl_device) +{ + + Eigen::array<IndexType, 4> tensor_dims = {{2,3,5,7}}; + Eigen::array<IndexType, 4> stride_dims = {{1,1,3,3}}; + + + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensor_dims); + Tensor<DataType, 4, DataLayout,IndexType> no_stride(tensor_dims); + Tensor<DataType, 4, DataLayout,IndexType> stride(stride_dims); + + + std::size_t tensor_bytes = tensor.size() * sizeof(DataType); + std::size_t no_stride_bytes = no_stride.size() * sizeof(DataType); + std::size_t stride_bytes = stride.size() * sizeof(DataType); + DataType * d_tensor = static_cast<DataType*>(sycl_device.allocate(tensor_bytes)); + DataType * d_no_stride = static_cast<DataType*>(sycl_device.allocate(no_stride_bytes)); + DataType * d_stride = static_cast<DataType*>(sycl_device.allocate(stride_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_tensor(d_tensor, tensor_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_no_stride(d_no_stride, tensor_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_stride(d_stride, stride_dims); + + + tensor.setRandom(); + array<IndexType, 4> strides; + strides[0] = 1; + strides[1] = 1; + strides[2] = 1; + strides[3] = 1; + sycl_device.memcpyHostToDevice(d_tensor, tensor.data(), tensor_bytes); + gpu_no_stride.device(sycl_device)=gpu_tensor.stride(strides); + sycl_device.memcpyDeviceToHost(no_stride.data(), d_no_stride, no_stride_bytes); + + //no_stride = tensor.stride(strides); + + VERIFY_IS_EQUAL(no_stride.dimension(0), 2); + VERIFY_IS_EQUAL(no_stride.dimension(1), 3); + VERIFY_IS_EQUAL(no_stride.dimension(2), 5); + VERIFY_IS_EQUAL(no_stride.dimension(3), 7); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), no_stride(i,j,k,l)); + } + } + } + } + + strides[0] = 2; + strides[1] = 4; + strides[2] = 2; + strides[3] = 3; +//Tensor<float, 4, DataLayout> stride; +// stride = tensor.stride(strides); + + gpu_stride.device(sycl_device)=gpu_tensor.stride(strides); + sycl_device.memcpyDeviceToHost(stride.data(), d_stride, stride_bytes); + + VERIFY_IS_EQUAL(stride.dimension(0), 1); + VERIFY_IS_EQUAL(stride.dimension(1), 1); + VERIFY_IS_EQUAL(stride.dimension(2), 3); + VERIFY_IS_EQUAL(stride.dimension(3), 3); + + for (IndexType i = 0; i < 1; ++i) { + for (IndexType j = 0; j < 1; ++j) { + for (IndexType k = 0; k < 3; ++k) { + for (IndexType l = 0; l < 3; ++l) { + VERIFY_IS_EQUAL(tensor(2*i,4*j,2*k,3*l), stride(i,j,k,l)); + } + } + } + } + + sycl_device.deallocate(d_tensor); + sycl_device.deallocate(d_no_stride); + sycl_device.deallocate(d_stride); +} + +template <typename DataType, int DataLayout, typename IndexType> +static void test_striding_as_lvalue(const Eigen::SyclDevice& sycl_device) +{ + + Eigen::array<IndexType, 4> tensor_dims = {{2,3,5,7}}; + Eigen::array<IndexType, 4> stride_dims = {{3,12,10,21}}; + + + Tensor<DataType, 4, DataLayout, IndexType> tensor(tensor_dims); + Tensor<DataType, 4, DataLayout,IndexType> no_stride(stride_dims); + Tensor<DataType, 4, DataLayout,IndexType> stride(stride_dims); + + + std::size_t tensor_bytes = tensor.size() * sizeof(DataType); + std::size_t no_stride_bytes = no_stride.size() * sizeof(DataType); + std::size_t stride_bytes = stride.size() * sizeof(DataType); + + DataType * d_tensor = static_cast<DataType*>(sycl_device.allocate(tensor_bytes)); + DataType * d_no_stride = static_cast<DataType*>(sycl_device.allocate(no_stride_bytes)); + DataType * d_stride = static_cast<DataType*>(sycl_device.allocate(stride_bytes)); + + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_tensor(d_tensor, tensor_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_no_stride(d_no_stride, stride_dims); + Eigen::TensorMap<Eigen::Tensor<DataType, 4, DataLayout, IndexType> > gpu_stride(d_stride, stride_dims); + + //Tensor<float, 4, DataLayout> tensor(2,3,5,7); + tensor.setRandom(); + array<IndexType, 4> strides; + strides[0] = 2; + strides[1] = 4; + strides[2] = 2; + strides[3] = 3; + +// Tensor<float, 4, DataLayout> result(3, 12, 10, 21); +// result.stride(strides) = tensor; + sycl_device.memcpyHostToDevice(d_tensor, tensor.data(), tensor_bytes); + gpu_stride.stride(strides).device(sycl_device)=gpu_tensor; + sycl_device.memcpyDeviceToHost(stride.data(), d_stride, stride_bytes); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), stride(2*i,4*j,2*k,3*l)); + } + } + } + } + + array<IndexType, 4> no_strides; + no_strides[0] = 1; + no_strides[1] = 1; + no_strides[2] = 1; + no_strides[3] = 1; +// Tensor<float, 4, DataLayout> result2(3, 12, 10, 21); +// result2.stride(strides) = tensor.stride(no_strides); + + gpu_no_stride.stride(strides).device(sycl_device)=gpu_tensor.stride(no_strides); + sycl_device.memcpyDeviceToHost(no_stride.data(), d_no_stride, no_stride_bytes); + + for (IndexType i = 0; i < 2; ++i) { + for (IndexType j = 0; j < 3; ++j) { + for (IndexType k = 0; k < 5; ++k) { + for (IndexType l = 0; l < 7; ++l) { + VERIFY_IS_EQUAL(tensor(i,j,k,l), no_stride(2*i,4*j,2*k,3*l)); + } + } + } + } + sycl_device.deallocate(d_tensor); + sycl_device.deallocate(d_no_stride); + sycl_device.deallocate(d_stride); +} + + +template <typename Dev_selector> void tensorStridingPerDevice(Dev_selector& s){ + QueueInterface queueInterface(s); + auto sycl_device=Eigen::SyclDevice(&queueInterface); + test_simple_striding<float, ColMajor, int64_t>(sycl_device); + test_simple_striding<float, RowMajor, int64_t>(sycl_device); + test_striding_as_lvalue<float, ColMajor, int64_t>(sycl_device); + test_striding_as_lvalue<float, RowMajor, int64_t>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_striding_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(tensorStridingPerDevice(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_sugar.cpp b/unsupported/test/cxx11_tensor_sugar.cpp index a03f75c..2ca5c47 100644 --- a/unsupported/test/cxx11_tensor_sugar.cpp +++ b/unsupported/test/cxx11_tensor_sugar.cpp
@@ -33,7 +33,7 @@ } -static void test_scalar_sugar() { +static void test_scalar_sugar_add_mul() { Tensor<float, 3> A(6, 7, 5); Tensor<float, 3> B(6, 7, 5); A.setRandom(); @@ -41,21 +41,41 @@ const float alpha = 0.43f; const float beta = 0.21f; + const float gamma = 0.14f; - Tensor<float, 3> R = A * A.constant(alpha) + B * B.constant(beta); - Tensor<float, 3> S = A * alpha + B * beta; + Tensor<float, 3> R = A.constant(gamma) + A * A.constant(alpha) + B * B.constant(beta); + Tensor<float, 3> S = A * alpha + B * beta + gamma; + Tensor<float, 3> T = gamma + alpha * A + beta * B; - // TODO: add enough syntactic sugar to support this - // Tensor<float, 3> T = alpha * A + beta * B; + for (int i = 0; i < 6*7*5; ++i) { + VERIFY_IS_APPROX(R(i), S(i)); + VERIFY_IS_APPROX(R(i), T(i)); + } +} + +static void test_scalar_sugar_sub_div() { + Tensor<float, 3> A(6, 7, 5); + Tensor<float, 3> B(6, 7, 5); + A.setRandom(); + B.setRandom(); + + const float alpha = 0.43f; + const float beta = 0.21f; + const float gamma = 0.14f; + const float delta = 0.32f; + + Tensor<float, 3> R = A.constant(gamma) - A / A.constant(alpha) + - B.constant(beta) / B - A.constant(delta); + Tensor<float, 3> S = gamma - A / alpha - beta / B - delta; for (int i = 0; i < 6*7*5; ++i) { VERIFY_IS_APPROX(R(i), S(i)); } } - -void test_cxx11_tensor_sugar() +EIGEN_DECLARE_TEST(cxx11_tensor_sugar) { CALL_SUBTEST(test_comparison_sugar()); - CALL_SUBTEST(test_scalar_sugar()); + CALL_SUBTEST(test_scalar_sugar_add_mul()); + CALL_SUBTEST(test_scalar_sugar_sub_div()); }
diff --git a/unsupported/test/cxx11_tensor_sycl.cpp b/unsupported/test/cxx11_tensor_sycl.cpp new file mode 100644 index 0000000..9357bed --- /dev/null +++ b/unsupported/test/cxx11_tensor_sycl.cpp
@@ -0,0 +1,276 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::array; +using Eigen::SyclDevice; +using Eigen::Tensor; +using Eigen::TensorMap; + +template <typename DataType, int DataLayout, typename IndexType> +void test_sycl_mem_transfers(const Eigen::SyclDevice &sycl_device) { + IndexType sizeDim1 = 100; + IndexType sizeDim2 = 10; + IndexType sizeDim3 = 20; + array<IndexType, 3> tensorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + Tensor<DataType, 3, DataLayout, IndexType> in1(tensorRange); + Tensor<DataType, 3, DataLayout, IndexType> out1(tensorRange); + Tensor<DataType, 3, DataLayout, IndexType> out2(tensorRange); + Tensor<DataType, 3, DataLayout, IndexType> out3(tensorRange); + + in1 = in1.random(); + + DataType* gpu_data1 = static_cast<DataType*>(sycl_device.allocate(in1.size()*sizeof(DataType))); + DataType* gpu_data2 = static_cast<DataType*>(sycl_device.allocate(out1.size()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu1(gpu_data1, tensorRange); + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu2(gpu_data2, tensorRange); + + sycl_device.memcpyHostToDevice(gpu_data1, in1.data(),(in1.size())*sizeof(DataType)); + sycl_device.memcpyHostToDevice(gpu_data2, in1.data(),(in1.size())*sizeof(DataType)); + gpu1.device(sycl_device) = gpu1 * 3.14f; + gpu2.device(sycl_device) = gpu2 * 2.7f; + sycl_device.memcpyDeviceToHost(out1.data(), gpu_data1,(out1.size())*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(out2.data(), gpu_data1,(out2.size())*sizeof(DataType)); + sycl_device.memcpyDeviceToHost(out3.data(), gpu_data2,(out3.size())*sizeof(DataType)); + sycl_device.synchronize(); + + for (IndexType i = 0; i < in1.size(); ++i) { + VERIFY_IS_APPROX(out1(i), in1(i) * 3.14f); + VERIFY_IS_APPROX(out2(i), in1(i) * 3.14f); + VERIFY_IS_APPROX(out3(i), in1(i) * 2.7f); + } + + sycl_device.deallocate(gpu_data1); + sycl_device.deallocate(gpu_data2); +} + +template <typename DataType, int DataLayout, typename IndexType> +void test_sycl_mem_sync(const Eigen::SyclDevice &sycl_device) { + IndexType size = 20; + array<IndexType, 1> tensorRange = {{size}}; + Tensor<DataType, 1, DataLayout, IndexType> in1(tensorRange); + Tensor<DataType, 1, DataLayout, IndexType> in2(tensorRange); + Tensor<DataType, 1, DataLayout, IndexType> out(tensorRange); + + in1 = in1.random(); + in2 = in1; + + DataType* gpu_data = static_cast<DataType*>(sycl_device.allocate(in1.size()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 1, DataLayout, IndexType>> gpu1(gpu_data, tensorRange); + sycl_device.memcpyHostToDevice(gpu_data, in1.data(),(in1.size())*sizeof(DataType)); + sycl_device.synchronize(); + in1.setZero(); + + sycl_device.memcpyDeviceToHost(out.data(), gpu_data, out.size()*sizeof(DataType)); + sycl_device.synchronize(); + + for (IndexType i = 0; i < in1.size(); ++i) { + VERIFY_IS_APPROX(out(i), in2(i)); + } + + sycl_device.deallocate(gpu_data); +} + +template <typename DataType, int DataLayout, typename IndexType> +void test_sycl_computations(const Eigen::SyclDevice &sycl_device) { + + IndexType sizeDim1 = 100; + IndexType sizeDim2 = 10; + IndexType sizeDim3 = 20; + array<IndexType, 3> tensorRange = {{sizeDim1, sizeDim2, sizeDim3}}; + Tensor<DataType, 3,DataLayout, IndexType> in1(tensorRange); + Tensor<DataType, 3,DataLayout, IndexType> in2(tensorRange); + Tensor<DataType, 3,DataLayout, IndexType> in3(tensorRange); + Tensor<DataType, 3,DataLayout, IndexType> out(tensorRange); + + in2 = in2.random(); + in3 = in3.random(); + + DataType * gpu_in1_data = static_cast<DataType*>(sycl_device.allocate(in1.size()*sizeof(DataType))); + DataType * gpu_in2_data = static_cast<DataType*>(sycl_device.allocate(in2.size()*sizeof(DataType))); + DataType * gpu_in3_data = static_cast<DataType*>(sycl_device.allocate(in3.size()*sizeof(DataType))); + DataType * gpu_out_data = static_cast<DataType*>(sycl_device.allocate(out.size()*sizeof(DataType))); + + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu_in1(gpu_in1_data, tensorRange); + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu_in2(gpu_in2_data, tensorRange); + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu_in3(gpu_in3_data, tensorRange); + TensorMap<Tensor<DataType, 3, DataLayout, IndexType>> gpu_out(gpu_out_data, tensorRange); + + /// a=1.2f + gpu_in1.device(sycl_device) = gpu_in1.constant(1.2f); + sycl_device.memcpyDeviceToHost(in1.data(), gpu_in1_data ,(in1.size())*sizeof(DataType)); + sycl_device.synchronize(); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(in1(i,j,k), 1.2f); + } + } + } + printf("a=1.2f Test passed\n"); + + /// a=b*1.2f + gpu_out.device(sycl_device) = gpu_in1 * 1.2f; + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data ,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i,j,k), + in1(i,j,k) * 1.2f); + } + } + } + printf("a=b*1.2f Test Passed\n"); + + /// c=a*b + sycl_device.memcpyHostToDevice(gpu_in2_data, in2.data(),(in2.size())*sizeof(DataType)); + gpu_out.device(sycl_device) = gpu_in1 * gpu_in2; + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i,j,k), + in1(i,j,k) * + in2(i,j,k)); + } + } + } + printf("c=a*b Test Passed\n"); + + /// c=a+b + gpu_out.device(sycl_device) = gpu_in1 + gpu_in2; + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i,j,k), + in1(i,j,k) + + in2(i,j,k)); + } + } + } + printf("c=a+b Test Passed\n"); + + /// c=a*a + gpu_out.device(sycl_device) = gpu_in1 * gpu_in1; + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i,j,k), + in1(i,j,k) * + in1(i,j,k)); + } + } + } + printf("c= a*a Test Passed\n"); + + //a*3.14f + b*2.7f + gpu_out.device(sycl_device) = gpu_in1 * gpu_in1.constant(3.14f) + gpu_in2 * gpu_in2.constant(2.7f); + sycl_device.memcpyDeviceToHost(out.data(),gpu_out_data,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i,j,k), + in1(i,j,k) * 3.14f + + in2(i,j,k) * 2.7f); + } + } + } + printf("a*3.14f + b*2.7f Test Passed\n"); + + ///d= (a>0.5? b:c) + sycl_device.memcpyHostToDevice(gpu_in3_data, in3.data(),(in3.size())*sizeof(DataType)); + gpu_out.device(sycl_device) =(gpu_in1 > gpu_in1.constant(0.5f)).select(gpu_in2, gpu_in3); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data,(out.size())*sizeof(DataType)); + sycl_device.synchronize(); + for (IndexType i = 0; i < sizeDim1; ++i) { + for (IndexType j = 0; j < sizeDim2; ++j) { + for (IndexType k = 0; k < sizeDim3; ++k) { + VERIFY_IS_APPROX(out(i, j, k), (in1(i, j, k) > 0.5f) + ? in2(i, j, k) + : in3(i, j, k)); + } + } + } + printf("d= (a>0.5? b:c) Test Passed\n"); + sycl_device.deallocate(gpu_in1_data); + sycl_device.deallocate(gpu_in2_data); + sycl_device.deallocate(gpu_in3_data); + sycl_device.deallocate(gpu_out_data); +} +template<typename Scalar1, typename Scalar2, int DataLayout, typename IndexType> +static void test_sycl_cast(const Eigen::SyclDevice& sycl_device){ + IndexType size = 20; + array<IndexType, 1> tensorRange = {{size}}; + Tensor<Scalar1, 1, DataLayout, IndexType> in(tensorRange); + Tensor<Scalar2, 1, DataLayout, IndexType> out(tensorRange); + Tensor<Scalar2, 1, DataLayout, IndexType> out_host(tensorRange); + + in = in.random(); + + Scalar1* gpu_in_data = static_cast<Scalar1*>(sycl_device.allocate(in.size()*sizeof(Scalar1))); + Scalar2 * gpu_out_data = static_cast<Scalar2*>(sycl_device.allocate(out.size()*sizeof(Scalar2))); + + TensorMap<Tensor<Scalar1, 1, DataLayout, IndexType>> gpu_in(gpu_in_data, tensorRange); + TensorMap<Tensor<Scalar2, 1, DataLayout, IndexType>> gpu_out(gpu_out_data, tensorRange); + sycl_device.memcpyHostToDevice(gpu_in_data, in.data(),(in.size())*sizeof(Scalar1)); + gpu_out.device(sycl_device) = gpu_in. template cast<Scalar2>(); + sycl_device.memcpyDeviceToHost(out.data(), gpu_out_data, out.size()*sizeof(Scalar2)); + out_host = in. template cast<Scalar2>(); + for(IndexType i=0; i< size; i++) + { + VERIFY_IS_APPROX(out(i), out_host(i)); + } + printf("cast Test Passed\n"); + sycl_device.deallocate(gpu_in_data); + sycl_device.deallocate(gpu_out_data); +} +template<typename DataType, typename dev_Selector> void sycl_computing_test_per_device(dev_Selector s){ + QueueInterface queueInterface(s); + auto sycl_device = Eigen::SyclDevice(&queueInterface); + test_sycl_mem_transfers<DataType, RowMajor, int64_t>(sycl_device); + test_sycl_computations<DataType, RowMajor, int64_t>(sycl_device); + test_sycl_mem_sync<DataType, RowMajor, int64_t>(sycl_device); + test_sycl_mem_transfers<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_computations<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_mem_sync<DataType, ColMajor, int64_t>(sycl_device); + test_sycl_cast<DataType, int, RowMajor, int64_t>(sycl_device); + test_sycl_cast<DataType, int, ColMajor, int64_t>(sycl_device); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_sycl) { + for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_computing_test_per_device<float>(device)); + } +}
diff --git a/unsupported/test/cxx11_tensor_symmetry.cpp b/unsupported/test/cxx11_tensor_symmetry.cpp index d680e9b..fed269a 100644 --- a/unsupported/test/cxx11_tensor_symmetry.cpp +++ b/unsupported/test/cxx11_tensor_symmetry.cpp
@@ -801,7 +801,7 @@ } } -void test_cxx11_tensor_symmetry() +EIGEN_DECLARE_TEST(cxx11_tensor_symmetry) { CALL_SUBTEST(test_symgroups_static()); CALL_SUBTEST(test_symgroups_dynamic());
diff --git a/unsupported/test/cxx11_tensor_thread_pool.cpp b/unsupported/test/cxx11_tensor_thread_pool.cpp index e461974..f8a7b36 100644 --- a/unsupported/test/cxx11_tensor_thread_pool.cpp +++ b/unsupported/test/cxx11_tensor_thread_pool.cpp
@@ -16,6 +16,25 @@ using Eigen::Tensor; +class TestAllocator : public Allocator { + public: + ~TestAllocator() EIGEN_OVERRIDE {} + EIGEN_DEVICE_FUNC void* allocate(size_t num_bytes) const EIGEN_OVERRIDE { + const_cast<TestAllocator*>(this)->alloc_count_++; + return internal::aligned_malloc(num_bytes); + } + EIGEN_DEVICE_FUNC void deallocate(void* buffer) const EIGEN_OVERRIDE { + const_cast<TestAllocator*>(this)->dealloc_count_++; + internal::aligned_free(buffer); + } + + int alloc_count() const { return alloc_count_; } + int dealloc_count() const { return dealloc_count_; } + + private: + int alloc_count_ = 0; + int dealloc_count_ = 0; +}; void test_multithread_elementwise() { @@ -91,7 +110,7 @@ for (ptrdiff_t i = 0; i < t_result.size(); i++) { VERIFY(&t_result.data()[i] != &m_result.data()[i]); - if (fabs(t_result(i) - m_result(i)) < 1e-4) { + if (fabsf(t_result(i) - m_result(i)) < 1e-4f) { continue; } if (Eigen::internal::isApprox(t_result(i), m_result(i), 1e-4f)) { @@ -132,7 +151,7 @@ for (ptrdiff_t i = 0; i < t_result.size(); i++) { assert(!(numext::isnan)(t_result.data()[i])); - if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) { + if (fabsf(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) { std::cout << "mismatch detected at index " << i << " : " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; assert(false); } @@ -147,7 +166,7 @@ m_result = m_left.transpose() * m_right; for (ptrdiff_t i = 0; i < t_result.size(); i++) { assert(!(numext::isnan)(t_result.data()[i])); - if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) { + if (fabsf(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) { std::cout << "mismatch detected: " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; assert(false); } @@ -165,7 +184,7 @@ m_result = m_left.transpose() * m_right; for (ptrdiff_t i = 0; i < t_result.size(); i++) { assert(!(numext::isnan)(t_result.data()[i])); - if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) { + if (fabsf(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) { std::cout << "mismatch detected: " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; assert(false); } @@ -183,7 +202,7 @@ m_result = m_left.transpose() * m_right; for (ptrdiff_t i = 0; i < t_result.size(); i++) { assert(!(numext::isnan)(t_result.data()[i])); - if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) { + if (fabsf(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) { std::cout << "mismatch detected: " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl; assert(false); } @@ -226,12 +245,182 @@ for (ptrdiff_t i = 0; i < st_result.size(); i++) { // if both of the values are very small, then do nothing (because the test will fail // due to numerical precision issues when values are small) - if (fabs(st_result.data()[i] - tp_result.data()[i]) >= 1e-4) { + if (numext::abs(st_result.data()[i] - tp_result.data()[i]) >= 1e-4f) { VERIFY_IS_APPROX(st_result.data()[i], tp_result.data()[i]); } } } +// Apply Sqrt to all output elements. +struct SqrtOutputKernel { + template <typename Index, typename Scalar> + EIGEN_ALWAYS_INLINE void operator()( + const internal::blas_data_mapper<Scalar, Index, ColMajor>& output_mapper, + const TensorContractionParams&, Index, Index, Index num_rows, + Index num_cols) const { + for (int i = 0; i < num_rows; ++i) { + for (int j = 0; j < num_cols; ++j) { + output_mapper(i, j) = std::sqrt(output_mapper(i, j)); + } + } + } +}; + +template <int DataLayout> +static void test_multithread_contraction_with_output_kernel() { + typedef Tensor<float, 1>::DimensionPair DimPair; + + const int num_threads = internal::random<int>(2, 11); + ThreadPool threads(num_threads); + Eigen::ThreadPoolDevice device(&threads, num_threads); + + Tensor<float, 4, DataLayout> t_left(30, 50, 8, 31); + Tensor<float, 5, DataLayout> t_right(8, 31, 7, 20, 10); + Tensor<float, 5, DataLayout> t_result(30, 50, 7, 20, 10); + + t_left.setRandom(); + t_right.setRandom(); + // Put trash in mat4 to verify contraction clears output memory. + t_result.setRandom(); + + // Add a little offset so that the results won't be close to zero. + t_left += t_left.constant(1.0f); + t_right += t_right.constant(1.0f); + + typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; + MapXf m_left(t_left.data(), 1500, 248); + MapXf m_right(t_right.data(), 248, 1400); + Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(1500, 1400); + + // this contraction should be equivalent to a single matrix multiplication + Eigen::array<DimPair, 2> dims({{DimPair(2, 0), DimPair(3, 1)}}); + + // compute results by separate methods + t_result.device(device) = t_left.contract(t_right, dims, SqrtOutputKernel()); + + m_result = m_left * m_right; + + for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) { + VERIFY(&t_result.data()[i] != &m_result.data()[i]); + VERIFY_IS_APPROX(t_result.data()[i], std::sqrt(m_result.data()[i])); + } +} + +// We are triggering 'evalShardedByInnerDim' optimization. +template <int DataLayout> +static void test_sharded_by_inner_dim_contraction() +{ + typedef Tensor<float, 1>::DimensionPair DimPair; + + const int num_threads = internal::random<int>(4, 16); + ThreadPool threads(num_threads); + Eigen::ThreadPoolDevice device(&threads, num_threads); + + Tensor<float, 2, DataLayout> t_left(2, 10000); + Tensor<float, 2, DataLayout> t_right(10000, 10); + Tensor<float, 2, DataLayout> t_result(2, 10); + + t_left.setRandom(); + t_right.setRandom(); + // Put trash in t_result to verify contraction clears output memory. + t_result.setRandom(); + + // Add a little offset so that the results won't be close to zero. + t_left += t_left.constant(1.0f); + t_right += t_right.constant(1.0f); + + typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; + MapXf m_left(t_left.data(), 2, 10000); + MapXf m_right(t_right.data(), 10000, 10); + Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); + + // this contraction should be equivalent to a single matrix multiplication + Eigen::array<DimPair, 1> dims({{DimPair(1, 0)}}); + + // compute results by separate methods + t_result.device(device) = t_left.contract(t_right, dims); + m_result = m_left * m_right; + + for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) { + VERIFY_IS_APPROX(t_result.data()[i], m_result.data()[i]); + } +} + +// We are triggering 'evalShardedByInnerDim' optimization with output kernel. +template <int DataLayout> +static void test_sharded_by_inner_dim_contraction_with_output_kernel() +{ + typedef Tensor<float, 1>::DimensionPair DimPair; + + const int num_threads = internal::random<int>(4, 16); + ThreadPool threads(num_threads); + Eigen::ThreadPoolDevice device(&threads, num_threads); + + Tensor<float, 2, DataLayout> t_left(2, 10000); + Tensor<float, 2, DataLayout> t_right(10000, 10); + Tensor<float, 2, DataLayout> t_result(2, 10); + + t_left.setRandom(); + t_right.setRandom(); + // Put trash in t_result to verify contraction clears output memory. + t_result.setRandom(); + + // Add a little offset so that the results won't be close to zero. + t_left += t_left.constant(1.0f); + t_right += t_right.constant(1.0f); + + typedef Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout>> MapXf; + MapXf m_left(t_left.data(), 2, 10000); + MapXf m_right(t_right.data(), 10000, 10); + Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(2, 10); + + // this contraction should be equivalent to a single matrix multiplication + Eigen::array<DimPair, 1> dims({{DimPair(1, 0)}}); + + // compute results by separate methods + t_result.device(device) = t_left.contract(t_right, dims, SqrtOutputKernel()); + m_result = m_left * m_right; + + for (Index i = 0; i < t_result.dimensions().TotalSize(); i++) { + VERIFY_IS_APPROX(t_result.data()[i], std::sqrt(m_result.data()[i])); + } +} + +template<int DataLayout> +void test_full_contraction() { + int contract_size1 = internal::random<int>(1, 500); + int contract_size2 = internal::random<int>(1, 500); + + Tensor<float, 2, DataLayout> left(contract_size1, + contract_size2); + Tensor<float, 2, DataLayout> right(contract_size1, + contract_size2); + left.setRandom(); + right.setRandom(); + + // add constants to shift values away from 0 for more precision + left += left.constant(1.5f); + right += right.constant(1.5f); + + typedef Tensor<float, 2>::DimensionPair DimPair; + Eigen::array<DimPair, 2> dims({{DimPair(0, 0), DimPair(1, 1)}}); + + Eigen::ThreadPool tp(internal::random<int>(2, 11)); + Eigen::ThreadPoolDevice thread_pool_device(&tp, internal::random<int>(2, 11)); + + Tensor<float, 0, DataLayout> st_result; + st_result = left.contract(right, dims); + + Tensor<float, 0, DataLayout> tp_result; + tp_result.device(thread_pool_device) = left.contract(right, dims); + + VERIFY(dimensions_match(st_result.dimensions(), tp_result.dimensions())); + // if both of the values are very small, then do nothing (because the test will fail + // due to numerical precision issues when values are small) + if (numext::abs(st_result() - tp_result()) >= 1e-4f) { + VERIFY_IS_APPROX(st_result(), tp_result()); + } +} template<int DataLayout> void test_multithreaded_reductions() { @@ -284,14 +473,14 @@ } template<int DataLayout> -void test_multithread_shuffle() +void test_multithread_shuffle(Allocator* allocator) { Tensor<float, 4, DataLayout> tensor(17,5,7,11); tensor.setRandom(); const int num_threads = internal::random<int>(2, 11); ThreadPool threads(num_threads); - Eigen::ThreadPoolDevice device(&threads, num_threads); + Eigen::ThreadPoolDevice device(&threads, num_threads, allocator); Tensor<float, 4, DataLayout> shuffle(7,5,11,17); array<ptrdiff_t, 4> shuffles = {{2,1,3,0}}; @@ -308,8 +497,23 @@ } } +void test_threadpool_allocate(TestAllocator* allocator) +{ + const int num_threads = internal::random<int>(2, 11); + const int num_allocs = internal::random<int>(2, 11); + ThreadPool threads(num_threads); + Eigen::ThreadPoolDevice device(&threads, num_threads, allocator); -void test_cxx11_tensor_thread_pool() + for (int a = 0; a < num_allocs; ++a) { + void* ptr = device.allocate(512); + device.deallocate(ptr); + } + VERIFY(allocator != NULL); + VERIFY_IS_EQUAL(allocator->alloc_count(), num_allocs); + VERIFY_IS_EQUAL(allocator->dealloc_count(), num_allocs); +} + +EIGEN_DECLARE_TEST(cxx11_tensor_thread_pool) { CALL_SUBTEST_1(test_multithread_elementwise()); CALL_SUBTEST_1(test_multithread_compound_assignment()); @@ -319,16 +523,29 @@ CALL_SUBTEST_3(test_multithread_contraction_agrees_with_singlethread<ColMajor>()); CALL_SUBTEST_3(test_multithread_contraction_agrees_with_singlethread<RowMajor>()); + CALL_SUBTEST_3(test_multithread_contraction_with_output_kernel<ColMajor>()); + CALL_SUBTEST_3(test_multithread_contraction_with_output_kernel<RowMajor>()); + + CALL_SUBTEST_4(test_sharded_by_inner_dim_contraction<ColMajor>()); + CALL_SUBTEST_4(test_sharded_by_inner_dim_contraction<RowMajor>()); + CALL_SUBTEST_4(test_sharded_by_inner_dim_contraction_with_output_kernel<ColMajor>()); + CALL_SUBTEST_4(test_sharded_by_inner_dim_contraction_with_output_kernel<RowMajor>()); // Exercise various cases that have been problematic in the past. - CALL_SUBTEST_4(test_contraction_corner_cases<ColMajor>()); - CALL_SUBTEST_4(test_contraction_corner_cases<RowMajor>()); + CALL_SUBTEST_5(test_contraction_corner_cases<ColMajor>()); + CALL_SUBTEST_5(test_contraction_corner_cases<RowMajor>()); - CALL_SUBTEST_5(test_multithreaded_reductions<ColMajor>()); - CALL_SUBTEST_5(test_multithreaded_reductions<RowMajor>()); + CALL_SUBTEST_6(test_full_contraction<ColMajor>()); + CALL_SUBTEST_6(test_full_contraction<RowMajor>()); - CALL_SUBTEST_6(test_memcpy()); - CALL_SUBTEST_6(test_multithread_random()); - CALL_SUBTEST_6(test_multithread_shuffle<ColMajor>()); - CALL_SUBTEST_6(test_multithread_shuffle<RowMajor>()); + CALL_SUBTEST_7(test_multithreaded_reductions<ColMajor>()); + CALL_SUBTEST_7(test_multithreaded_reductions<RowMajor>()); + + CALL_SUBTEST_7(test_memcpy()); + CALL_SUBTEST_7(test_multithread_random()); + + TestAllocator test_allocator; + CALL_SUBTEST_7(test_multithread_shuffle<ColMajor>(NULL)); + CALL_SUBTEST_7(test_multithread_shuffle<RowMajor>(&test_allocator)); + CALL_SUBTEST_7(test_threadpool_allocate(&test_allocator)); }
diff --git a/unsupported/test/cxx11_tensor_trace.cpp b/unsupported/test/cxx11_tensor_trace.cpp new file mode 100644 index 0000000..0cb2306 --- /dev/null +++ b/unsupported/test/cxx11_tensor_trace.cpp
@@ -0,0 +1,171 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2017 Gagan Goel <gagan.nith@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" + +#include <Eigen/CXX11/Tensor> + +using Eigen::Tensor; +using Eigen::array; + +template <int DataLayout> +static void test_0D_trace() { + Tensor<float, 0, DataLayout> tensor; + tensor.setRandom(); + array<ptrdiff_t, 0> dims; + Tensor<float, 0, DataLayout> result = tensor.trace(dims); + VERIFY_IS_EQUAL(result(), tensor()); +} + + +template <int DataLayout> +static void test_all_dimensions_trace() { + Tensor<float, 3, DataLayout> tensor1(5, 5, 5); + tensor1.setRandom(); + Tensor<float, 0, DataLayout> result1 = tensor1.trace(); + VERIFY_IS_EQUAL(result1.rank(), 0); + float sum = 0.0f; + for (int i = 0; i < 5; ++i) { + sum += tensor1(i, i, i); + } + VERIFY_IS_EQUAL(result1(), sum); + + Tensor<float, 5, DataLayout> tensor2(7, 7, 7, 7, 7); + array<ptrdiff_t, 5> dims = { { 2, 1, 0, 3, 4 } }; + Tensor<float, 0, DataLayout> result2 = tensor2.trace(dims); + VERIFY_IS_EQUAL(result2.rank(), 0); + sum = 0.0f; + for (int i = 0; i < 7; ++i) { + sum += tensor2(i, i, i, i, i); + } + VERIFY_IS_EQUAL(result2(), sum); +} + + +template <int DataLayout> +static void test_simple_trace() { + Tensor<float, 3, DataLayout> tensor1(3, 5, 3); + tensor1.setRandom(); + array<ptrdiff_t, 2> dims1 = { { 0, 2 } }; + Tensor<float, 1, DataLayout> result1 = tensor1.trace(dims1); + VERIFY_IS_EQUAL(result1.rank(), 1); + VERIFY_IS_EQUAL(result1.dimension(0), 5); + float sum = 0.0f; + for (int i = 0; i < 5; ++i) { + sum = 0.0f; + for (int j = 0; j < 3; ++j) { + sum += tensor1(j, i, j); + } + VERIFY_IS_EQUAL(result1(i), sum); + } + + Tensor<float, 4, DataLayout> tensor2(5, 5, 7, 7); + tensor2.setRandom(); + array<ptrdiff_t, 2> dims2 = { { 2, 3 } }; + Tensor<float, 2, DataLayout> result2 = tensor2.trace(dims2); + VERIFY_IS_EQUAL(result2.rank(), 2); + VERIFY_IS_EQUAL(result2.dimension(0), 5); + VERIFY_IS_EQUAL(result2.dimension(1), 5); + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 5; ++j) { + sum = 0.0f; + for (int k = 0; k < 7; ++k) { + sum += tensor2(i, j, k, k); + } + VERIFY_IS_EQUAL(result2(i, j), sum); + } + } + + array<ptrdiff_t, 2> dims3 = { { 1, 0 } }; + Tensor<float, 2, DataLayout> result3 = tensor2.trace(dims3); + VERIFY_IS_EQUAL(result3.rank(), 2); + VERIFY_IS_EQUAL(result3.dimension(0), 7); + VERIFY_IS_EQUAL(result3.dimension(1), 7); + for (int i = 0; i < 7; ++i) { + for (int j = 0; j < 7; ++j) { + sum = 0.0f; + for (int k = 0; k < 5; ++k) { + sum += tensor2(k, k, i, j); + } + VERIFY_IS_EQUAL(result3(i, j), sum); + } + } + + Tensor<float, 5, DataLayout> tensor3(3, 7, 3, 7, 3); + tensor3.setRandom(); + array<ptrdiff_t, 3> dims4 = { { 0, 2, 4 } }; + Tensor<float, 2, DataLayout> result4 = tensor3.trace(dims4); + VERIFY_IS_EQUAL(result4.rank(), 2); + VERIFY_IS_EQUAL(result4.dimension(0), 7); + VERIFY_IS_EQUAL(result4.dimension(1), 7); + for (int i = 0; i < 7; ++i) { + for (int j = 0; j < 7; ++j) { + sum = 0.0f; + for (int k = 0; k < 3; ++k) { + sum += tensor3(k, i, k, j, k); + } + VERIFY_IS_EQUAL(result4(i, j), sum); + } + } + + Tensor<float, 5, DataLayout> tensor4(3, 7, 4, 7, 5); + tensor4.setRandom(); + array<ptrdiff_t, 2> dims5 = { { 1, 3 } }; + Tensor<float, 3, DataLayout> result5 = tensor4.trace(dims5); + VERIFY_IS_EQUAL(result5.rank(), 3); + VERIFY_IS_EQUAL(result5.dimension(0), 3); + VERIFY_IS_EQUAL(result5.dimension(1), 4); + VERIFY_IS_EQUAL(result5.dimension(2), 5); + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 4; ++j) { + for (int k = 0; k < 5; ++k) { + sum = 0.0f; + for (int l = 0; l < 7; ++l) { + sum += tensor4(i, l, j, l, k); + } + VERIFY_IS_EQUAL(result5(i, j, k), sum); + } + } + } +} + + +template<int DataLayout> +static void test_trace_in_expr() { + Tensor<float, 4, DataLayout> tensor(2, 3, 5, 3); + tensor.setRandom(); + array<ptrdiff_t, 2> dims = { { 1, 3 } }; + Tensor<float, 2, DataLayout> result(2, 5); + result = result.constant(1.0f) - tensor.trace(dims); + VERIFY_IS_EQUAL(result.rank(), 2); + VERIFY_IS_EQUAL(result.dimension(0), 2); + VERIFY_IS_EQUAL(result.dimension(1), 5); + float sum = 0.0f; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 5; ++j) { + sum = 0.0f; + for (int k = 0; k < 3; ++k) { + sum += tensor(i, k, j, k); + } + VERIFY_IS_EQUAL(result(i, j), 1.0f - sum); + } + } +} + + +EIGEN_DECLARE_TEST(cxx11_tensor_trace) { + CALL_SUBTEST(test_0D_trace<ColMajor>()); + CALL_SUBTEST(test_0D_trace<RowMajor>()); + CALL_SUBTEST(test_all_dimensions_trace<ColMajor>()); + CALL_SUBTEST(test_all_dimensions_trace<RowMajor>()); + CALL_SUBTEST(test_simple_trace<ColMajor>()); + CALL_SUBTEST(test_simple_trace<RowMajor>()); + CALL_SUBTEST(test_trace_in_expr<ColMajor>()); + CALL_SUBTEST(test_trace_in_expr<RowMajor>()); +} \ No newline at end of file
diff --git a/unsupported/test/cxx11_tensor_uint128.cpp b/unsupported/test/cxx11_tensor_uint128.cpp index d2a1e86..07691df 100644 --- a/unsupported/test/cxx11_tensor_uint128.cpp +++ b/unsupported/test/cxx11_tensor_uint128.cpp
@@ -144,7 +144,7 @@ #endif -void test_cxx11_tensor_uint128() +EIGEN_DECLARE_TEST(cxx11_tensor_uint128) { #ifdef EIGEN_NO_INT128 // Skip the test on compilers that don't support 128bit integers natively
diff --git a/unsupported/test/cxx11_tensor_volume_patch.cpp b/unsupported/test/cxx11_tensor_volume_patch.cpp index ca6840f..3aa363e 100644 --- a/unsupported/test/cxx11_tensor_volume_patch.cpp +++ b/unsupported/test/cxx11_tensor_volume_patch.cpp
@@ -105,7 +105,7 @@ } } -void test_cxx11_tensor_volume_patch() +EIGEN_DECLARE_TEST(cxx11_tensor_volume_patch) { CALL_SUBTEST(test_single_voxel_patch()); CALL_SUBTEST(test_entire_volume_patch());
diff --git a/unsupported/test/cxx11_tensor_volume_patch_sycl.cpp b/unsupported/test/cxx11_tensor_volume_patch_sycl.cpp new file mode 100644 index 0000000..ca7994f --- /dev/null +++ b/unsupported/test/cxx11_tensor_volume_patch_sycl.cpp
@@ -0,0 +1,222 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 +// Mehdi Goli Codeplay Software Ltd. +// Ralph Potter Codeplay Software Ltd. +// Luke Iwanski Codeplay Software Ltd. +// Contact: <eigen@codeplay.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX + +#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int64_t +#define EIGEN_USE_SYCL + +#include "main.h" +#include <unsupported/Eigen/CXX11/Tensor> + +using Eigen::Tensor; +static const int DataLayout = ColMajor; + +template <typename DataType, typename IndexType> +static void test_single_voxel_patch_sycl(const Eigen::SyclDevice& sycl_device) +{ + +IndexType sizeDim0 = 4; +IndexType sizeDim1 = 2; +IndexType sizeDim2 = 3; +IndexType sizeDim3 = 5; +IndexType sizeDim4 = 7; +array<IndexType, 5> tensorColMajorRange = {{sizeDim0, sizeDim1, sizeDim2, sizeDim3, sizeDim4}}; +array<IndexType, 5> tensorRowMajorRange = {{sizeDim4, sizeDim3, sizeDim2, sizeDim1, sizeDim0}}; +Tensor<DataType, 5, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); +Tensor<DataType, 5, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); +tensor_col_major.setRandom(); + + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 5, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 5, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + + + // single volume patch: ColMajor + array<IndexType, 6> patchColMajorTensorRange={{sizeDim0,1, 1, 1, sizeDim1*sizeDim2*sizeDim3, sizeDim4}}; + Tensor<DataType, 6, DataLayout,IndexType> single_voxel_patch_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =single_voxel_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_single_voxel_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 6, DataLayout,IndexType>> gpu_single_voxel_patch_col_major(gpu_data_single_voxel_patch_col_major, patchColMajorTensorRange); + gpu_single_voxel_patch_col_major.device(sycl_device)=gpu_col_major.extract_volume_patches(1, 1, 1); + sycl_device.memcpyDeviceToHost(single_voxel_patch_col_major.data(), gpu_data_single_voxel_patch_col_major, patchTensorBuffSize); + + + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(0), 4); + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(1), 1); + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(3), 1); + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(4), 2 * 3 * 5); + VERIFY_IS_EQUAL(single_voxel_patch_col_major.dimension(5), 7); + + array<IndexType, 6> patchRowMajorTensorRange={{sizeDim4, sizeDim1*sizeDim2*sizeDim3, 1, 1, 1, sizeDim0}}; + Tensor<DataType, 6, RowMajor,IndexType> single_voxel_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =single_voxel_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_single_voxel_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 6, RowMajor,IndexType>> gpu_single_voxel_patch_row_major(gpu_data_single_voxel_patch_row_major, patchRowMajorTensorRange); + gpu_single_voxel_patch_row_major.device(sycl_device)=gpu_row_major.extract_volume_patches(1, 1, 1); + sycl_device.memcpyDeviceToHost(single_voxel_patch_row_major.data(), gpu_data_single_voxel_patch_row_major, patchTensorBuffSize); + + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(0), 7); + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(1), 2 * 3 * 5); + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(2), 1); + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(3), 1); + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(4), 1); + VERIFY_IS_EQUAL(single_voxel_patch_row_major.dimension(5), 4); + + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_col_major.size())*sizeof(DataType)); + for (IndexType i = 0; i < tensor_col_major.size(); ++i) { + VERIFY_IS_EQUAL(tensor_col_major.data()[i], single_voxel_patch_col_major.data()[i]); + VERIFY_IS_EQUAL(tensor_row_major.data()[i], single_voxel_patch_row_major.data()[i]); + VERIFY_IS_EQUAL(tensor_col_major.data()[i], tensor_row_major.data()[i]); + } + + + sycl_device.deallocate(gpu_data_col_major); + sycl_device.deallocate(gpu_data_row_major); + sycl_device.deallocate(gpu_data_single_voxel_patch_col_major); + sycl_device.deallocate(gpu_data_single_voxel_patch_row_major); +} + +template <typename DataType, typename IndexType> +static void test_entire_volume_patch_sycl(const Eigen::SyclDevice& sycl_device) +{ + const int depth = 4; + const int patch_z = 2; + const int patch_y = 3; + const int patch_x = 5; + const int batch = 7; + + array<IndexType, 5> tensorColMajorRange = {{depth, patch_z, patch_y, patch_x, batch}}; + array<IndexType, 5> tensorRowMajorRange = {{batch, patch_x, patch_y, patch_z, depth}}; + Tensor<DataType, 5, DataLayout,IndexType> tensor_col_major(tensorColMajorRange); + Tensor<DataType, 5, RowMajor,IndexType> tensor_row_major(tensorRowMajorRange); + tensor_col_major.setRandom(); + + + DataType* gpu_data_col_major = static_cast<DataType*>(sycl_device.allocate(tensor_col_major.size()*sizeof(DataType))); + DataType* gpu_data_row_major = static_cast<DataType*>(sycl_device.allocate(tensor_row_major.size()*sizeof(DataType))); + TensorMap<Tensor<DataType, 5, ColMajor, IndexType>> gpu_col_major(gpu_data_col_major, tensorColMajorRange); + TensorMap<Tensor<DataType, 5, RowMajor, IndexType>> gpu_row_major(gpu_data_row_major, tensorRowMajorRange); + + sycl_device.memcpyHostToDevice(gpu_data_col_major, tensor_col_major.data(),(tensor_col_major.size())*sizeof(DataType)); + gpu_row_major.device(sycl_device)=gpu_col_major.swap_layout(); + sycl_device.memcpyDeviceToHost(tensor_row_major.data(), gpu_data_row_major, (tensor_col_major.size())*sizeof(DataType)); + + + // single volume patch: ColMajor + array<IndexType, 6> patchColMajorTensorRange={{depth,patch_z, patch_y, patch_x, patch_z*patch_y*patch_x, batch}}; + Tensor<DataType, 6, DataLayout,IndexType> entire_volume_patch_col_major(patchColMajorTensorRange); + size_t patchTensorBuffSize =entire_volume_patch_col_major.size()*sizeof(DataType); + DataType* gpu_data_entire_volume_patch_col_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 6, DataLayout,IndexType>> gpu_entire_volume_patch_col_major(gpu_data_entire_volume_patch_col_major, patchColMajorTensorRange); + gpu_entire_volume_patch_col_major.device(sycl_device)=gpu_col_major.extract_volume_patches(patch_z, patch_y, patch_x); + sycl_device.memcpyDeviceToHost(entire_volume_patch_col_major.data(), gpu_data_entire_volume_patch_col_major, patchTensorBuffSize); + + +// Tensor<float, 5> tensor(depth, patch_z, patch_y, patch_x, batch); +// tensor.setRandom(); +// Tensor<float, 5, RowMajor> tensor_row_major = tensor.swap_layout(); + + //Tensor<float, 6> entire_volume_patch; + //entire_volume_patch = tensor.extract_volume_patches(patch_z, patch_y, patch_x); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(0), depth); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(1), patch_z); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(2), patch_y); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(3), patch_x); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(4), patch_z * patch_y * patch_x); + VERIFY_IS_EQUAL(entire_volume_patch_col_major.dimension(5), batch); + +// Tensor<float, 6, RowMajor> entire_volume_patch_row_major; + //entire_volume_patch_row_major = tensor_row_major.extract_volume_patches(patch_z, patch_y, patch_x); + + array<IndexType, 6> patchRowMajorTensorRange={{batch,patch_z*patch_y*patch_x, patch_x, patch_y, patch_z, depth}}; + Tensor<DataType, 6, RowMajor,IndexType> entire_volume_patch_row_major(patchRowMajorTensorRange); + patchTensorBuffSize =entire_volume_patch_row_major.size()*sizeof(DataType); + DataType* gpu_data_entire_volume_patch_row_major = static_cast<DataType*>(sycl_device.allocate(patchTensorBuffSize)); + TensorMap<Tensor<DataType, 6, RowMajor,IndexType>> gpu_entire_volume_patch_row_major(gpu_data_entire_volume_patch_row_major, patchRowMajorTensorRange); + gpu_entire_volume_patch_row_major.device(sycl_device)=gpu_row_major.extract_volume_patches(patch_z, patch_y, patch_x); + sycl_device.memcpyDeviceToHost(entire_volume_patch_row_major.data(), gpu_data_entire_volume_patch_row_major, patchTensorBuffSize); + + + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(0), batch); + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(1), patch_z * patch_y * patch_x); + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(2), patch_x); + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(3), patch_y); + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(4), patch_z); + VERIFY_IS_EQUAL(entire_volume_patch_row_major.dimension(5), depth); + + const int dz = patch_z - 1; + const int dy = patch_y - 1; + const int dx = patch_x - 1; + + const int forward_pad_z = dz - dz / 2; + const int forward_pad_y = dy - dy / 2; + const int forward_pad_x = dx - dx / 2; + + for (int pz = 0; pz < patch_z; pz++) { + for (int py = 0; py < patch_y; py++) { + for (int px = 0; px < patch_x; px++) { + const int patchId = pz + patch_z * (py + px * patch_y); + for (int z = 0; z < patch_z; z++) { + for (int y = 0; y < patch_y; y++) { + for (int x = 0; x < patch_x; x++) { + for (int b = 0; b < batch; b++) { + for (int d = 0; d < depth; d++) { + float expected = 0.0f; + float expected_row_major = 0.0f; + const int eff_z = z - forward_pad_z + pz; + const int eff_y = y - forward_pad_y + py; + const int eff_x = x - forward_pad_x + px; + if (eff_z >= 0 && eff_y >= 0 && eff_x >= 0 && + eff_z < patch_z && eff_y < patch_y && eff_x < patch_x) { + expected = tensor_col_major(d, eff_z, eff_y, eff_x, b); + expected_row_major = tensor_row_major(b, eff_x, eff_y, eff_z, d); + } + VERIFY_IS_EQUAL(entire_volume_patch_col_major(d, z, y, x, patchId, b), expected); + VERIFY_IS_EQUAL(entire_volume_patch_row_major(b, patchId, x, y, z, d), expected_row_major); + } + } + } + } + } + } + } + } + sycl_device.deallocate(gpu_data_col_major); + sycl_device.deallocate(gpu_data_row_major); + sycl_device.deallocate(gpu_data_entire_volume_patch_col_major); + sycl_device.deallocate(gpu_data_entire_volume_patch_row_major); +} + + + +template<typename DataType, typename dev_Selector> void sycl_tensor_volume_patch_test_per_device(dev_Selector s){ +QueueInterface queueInterface(s); +auto sycl_device = Eigen::SyclDevice(&queueInterface); +std::cout << "Running on " << s.template get_info<cl::sycl::info::device::name>() << std::endl; +test_single_voxel_patch_sycl<DataType, int64_t>(sycl_device); +test_entire_volume_patch_sycl<DataType, int64_t>(sycl_device); +} +EIGEN_DECLARE_TEST(cxx11_tensor_volume_patch_sycl) +{ +for (const auto& device :Eigen::get_sycl_supported_devices()) { + CALL_SUBTEST(sycl_tensor_volume_patch_test_per_device<float>(device)); +} +}
diff --git a/unsupported/test/dgmres.cpp b/unsupported/test/dgmres.cpp index 2b11807..5f63161 100644 --- a/unsupported/test/dgmres.cpp +++ b/unsupported/test/dgmres.cpp
@@ -9,7 +9,7 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #include "../../test/sparse_solver.h" -#include <Eigen/src/IterativeSolvers/DGMRES.h> +#include <unsupported/Eigen/IterativeSolvers> template<typename T> void test_dgmres_T() { @@ -24,7 +24,7 @@ //CALL_SUBTEST( check_sparse_square_solving(dgmres_colmajor_ssor) ); } -void test_dgmres() +EIGEN_DECLARE_TEST(dgmres) { CALL_SUBTEST_1(test_dgmres_T<double>()); CALL_SUBTEST_2(test_dgmres_T<std::complex<double> >());
diff --git a/unsupported/test/forward_adolc.cpp b/unsupported/test/forward_adolc.cpp index 866db8e..14a909d 100644 --- a/unsupported/test/forward_adolc.cpp +++ b/unsupported/test/forward_adolc.cpp
@@ -35,7 +35,7 @@ int m_inputs, m_values; TestFunc1() : m_inputs(InputsAtCompileTime), m_values(ValuesAtCompileTime) {} - TestFunc1(int inputs, int values) : m_inputs(inputs), m_values(values) {} + TestFunc1(int inputs_, int values_) : m_inputs(inputs_), m_values(values_) {} int inputs() const { return m_inputs; } int values() const { return m_values; } @@ -119,7 +119,7 @@ VERIFY_IS_APPROX(j, jref); } -void test_forward_adolc() +EIGEN_DECLARE_TEST(forward_adolc) { adtl::setNumDir(NUMBER_DIRECTIONS); @@ -132,7 +132,7 @@ } { - // simple instanciation tests + // simple instantiation tests Matrix<adtl::adouble,2,1> x; foo(x); Matrix<adtl::adouble,Dynamic,Dynamic> A(4,4);;
diff --git a/unsupported/test/gmres.cpp b/unsupported/test/gmres.cpp index f296911..8d2254b 100644 --- a/unsupported/test/gmres.cpp +++ b/unsupported/test/gmres.cpp
@@ -24,7 +24,7 @@ //CALL_SUBTEST( check_sparse_square_solving(gmres_colmajor_ssor) ); } -void test_gmres() +EIGEN_DECLARE_TEST(gmres) { CALL_SUBTEST_1(test_gmres_T<double>()); CALL_SUBTEST_2(test_gmres_T<std::complex<double> >());
diff --git a/unsupported/test/kronecker_product.cpp b/unsupported/test/kronecker_product.cpp index 02411a2..b5b764c 100644 --- a/unsupported/test/kronecker_product.cpp +++ b/unsupported/test/kronecker_product.cpp
@@ -10,11 +10,12 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +#ifdef EIGEN_TEST_PART_1 + #include "sparse.h" #include <Eigen/SparseExtra> #include <Eigen/KroneckerProduct> - template<typename MatrixType> void check_dimension(const MatrixType& ab, const int rows, const int cols) { @@ -83,7 +84,7 @@ } -void test_kronecker_product() +EIGEN_DECLARE_TEST(kronecker_product) { // DM = dense matrix; SM = sparse matrix @@ -95,7 +96,7 @@ SM_a.insert(1,0) = DM_a.coeffRef(1,0) = -0.9076572187376921; SM_a.insert(1,1) = DM_a.coeffRef(1,1) = 0.6469156566545853; SM_a.insert(1,2) = DM_a.coeffRef(1,2) = -0.3658010398782789; - + MatrixXd DM_b(3,2); SparseMatrix<double> SM_b(3,2); SM_b.insert(0,0) = DM_b.coeffRef(0,0) = 0.9004440976767099; @@ -165,7 +166,7 @@ SM_a.insert(0,3) = -0.2; SM_a.insert(2,4) = 0.3; SM_a.finalize(); - + SM_b.insert(0,0) = 0.4; SM_b.insert(2,1) = -0.5; SM_b.finalize(); @@ -183,7 +184,7 @@ DM_b2.resize(4,8); DM_ab2 = kroneckerProduct(DM_a2,DM_b2); CALL_SUBTEST(check_dimension(DM_ab2,10*4,9*8)); - + for(int i = 0; i < g_repeat; i++) { double density = Eigen::internal::random<double>(0.01,0.5); @@ -196,37 +197,56 @@ MatrixXf dA(ra,ca), dB(rb,cb), dC; initSparse(density, dA, sA); initSparse(density, dB, sB); - + sC = kroneckerProduct(sA,sB); dC = kroneckerProduct(dA,dB); VERIFY_IS_APPROX(MatrixXf(sC),dC); - + sC = kroneckerProduct(sA.transpose(),sB); dC = kroneckerProduct(dA.transpose(),dB); VERIFY_IS_APPROX(MatrixXf(sC),dC); - + sC = kroneckerProduct(sA.transpose(),sB.transpose()); dC = kroneckerProduct(dA.transpose(),dB.transpose()); VERIFY_IS_APPROX(MatrixXf(sC),dC); - + sC = kroneckerProduct(sA,sB.transpose()); dC = kroneckerProduct(dA,dB.transpose()); VERIFY_IS_APPROX(MatrixXf(sC),dC); - + sC2 = kroneckerProduct(sA,sB); dC = kroneckerProduct(dA,dB); VERIFY_IS_APPROX(MatrixXf(sC2),dC); - + sC2 = kroneckerProduct(dA,sB); dC = kroneckerProduct(dA,dB); VERIFY_IS_APPROX(MatrixXf(sC2),dC); - + sC2 = kroneckerProduct(sA,dB); dC = kroneckerProduct(dA,dB); VERIFY_IS_APPROX(MatrixXf(sC2),dC); - + sC2 = kroneckerProduct(2*sA,sB); dC = kroneckerProduct(2*dA,dB); VERIFY_IS_APPROX(MatrixXf(sC2),dC); } } + +#endif + +#ifdef EIGEN_TEST_PART_2 + +// simply check that for a dense kronecker product, sparse module is not needed +#include "main.h" +#include <Eigen/KroneckerProduct> + +EIGEN_DECLARE_TEST(kronecker_product) +{ + MatrixXd a(2,2), b(3,3), c; + a.setRandom(); + b.setRandom(); + c = kroneckerProduct(a,b); + VERIFY_IS_APPROX(c.block(3,3,3,3), a(1,1)*b); +} + +#endif
diff --git a/unsupported/test/levenberg_marquardt.cpp b/unsupported/test/levenberg_marquardt.cpp index 64f168c..7f9a81c 100644 --- a/unsupported/test/levenberg_marquardt.cpp +++ b/unsupported/test/levenberg_marquardt.cpp
@@ -1445,7 +1445,7 @@ VERIFY_IS_APPROX(x[2], 4.5154121844E+02); } -void test_levenberg_marquardt() +EIGEN_DECLARE_TEST(levenberg_marquardt) { // Tests using the examples provided by (c)minpack CALL_SUBTEST(testLmder1());
diff --git a/unsupported/test/matrix_exponential.cpp b/unsupported/test/matrix_exponential.cpp index 50dec08..b032cbf 100644 --- a/unsupported/test/matrix_exponential.cpp +++ b/unsupported/test/matrix_exponential.cpp
@@ -119,7 +119,7 @@ } } -void test_matrix_exponential() +EIGEN_DECLARE_TEST(matrix_exponential) { CALL_SUBTEST_2(test2dRotation<double>(1e-13)); CALL_SUBTEST_1(test2dRotation<float>(2e-5)); // was 1e-5, relaxed for clang 2.8 / linux / x86-64
diff --git a/unsupported/test/matrix_function.cpp b/unsupported/test/matrix_function.cpp index 9a995f9..2049b8b 100644 --- a/unsupported/test/matrix_function.cpp +++ b/unsupported/test/matrix_function.cpp
@@ -23,9 +23,8 @@ // Returns a matrix with eigenvalues clustered around 0, 1 and 2. template<typename MatrixType> -MatrixType randomMatrixWithRealEivals(const typename MatrixType::Index size) +MatrixType randomMatrixWithRealEivals(const Index size) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; MatrixType diag = MatrixType::Zero(size, size); @@ -42,16 +41,15 @@ struct randomMatrixWithImagEivals { // Returns a matrix with eigenvalues clustered around 0 and +/- i. - static MatrixType run(const typename MatrixType::Index size); + static MatrixType run(const Index size); }; // Partial specialization for real matrices template<typename MatrixType> struct randomMatrixWithImagEivals<MatrixType, 0> { - static MatrixType run(const typename MatrixType::Index size) + static MatrixType run(const Index size) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; MatrixType diag = MatrixType::Zero(size, size); Index i = 0; @@ -77,9 +75,8 @@ template<typename MatrixType> struct randomMatrixWithImagEivals<MatrixType, 1> { - static MatrixType run(const typename MatrixType::Index size) + static MatrixType run(const Index size) { - typedef typename MatrixType::Index Index; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::RealScalar RealScalar; const Scalar imagUnit(0, 1); @@ -113,8 +110,8 @@ MatrixType scaledA; RealScalar maxImagPartOfSpectrum = A.eigenvalues().imag().cwiseAbs().maxCoeff(); - if (maxImagPartOfSpectrum >= 0.9 * EIGEN_PI) - scaledA = A * 0.9 * EIGEN_PI / maxImagPartOfSpectrum; + if (maxImagPartOfSpectrum >= RealScalar(0.9L * EIGEN_PI)) + scaledA = A * RealScalar(0.9L * EIGEN_PI) / maxImagPartOfSpectrum; else scaledA = A; @@ -171,7 +168,6 @@ { // Matrices with clustered eigenvalue lead to different code paths // in MatrixFunction.h and are thus useful for testing. - typedef typename MatrixType::Index Index; const Index size = m.rows(); for (int i = 0; i < g_repeat; i++) { @@ -181,7 +177,7 @@ } } -void test_matrix_function() +EIGEN_DECLARE_TEST(matrix_function) { CALL_SUBTEST_1(testMatrixType(Matrix<float,1,1>())); CALL_SUBTEST_2(testMatrixType(Matrix3cf()));
diff --git a/unsupported/test/matrix_functions.h b/unsupported/test/matrix_functions.h index 150b4c0..4e26364 100644 --- a/unsupported/test/matrix_functions.h +++ b/unsupported/test/matrix_functions.h
@@ -61,7 +61,7 @@ }; template <typename Derived, typename OtherDerived> -double relerr(const MatrixBase<Derived>& A, const MatrixBase<OtherDerived>& B) +typename Derived::RealScalar relerr(const MatrixBase<Derived>& A, const MatrixBase<OtherDerived>& B) { return std::sqrt((A - B).cwiseAbs2().sum() / (std::min)(A.cwiseAbs2().sum(), B.cwiseAbs2().sum())); }
diff --git a/unsupported/test/matrix_power.cpp b/unsupported/test/matrix_power.cpp index 8e104ed..dbaf9db 100644 --- a/unsupported/test/matrix_power.cpp +++ b/unsupported/test/matrix_power.cpp
@@ -10,7 +10,7 @@ #include "matrix_functions.h" template<typename T> -void test2dRotation(double tol) +void test2dRotation(const T& tol) { Matrix<T,2,2> A, B, C; T angle, c, s; @@ -19,19 +19,19 @@ MatrixPower<Matrix<T,2,2> > Apow(A); for (int i=0; i<=20; ++i) { - angle = pow(10, (i-10) / 5.); + angle = std::pow(T(10), T(i-10) / T(5.)); c = std::cos(angle); s = std::sin(angle); B << c, s, -s, c; - C = Apow(std::ldexp(angle,1) / EIGEN_PI); + C = Apow(std::ldexp(angle,1) / T(EIGEN_PI)); std::cout << "test2dRotation: i = " << i << " error powerm = " << relerr(C,B) << '\n'; VERIFY(C.isApprox(B, tol)); } } template<typename T> -void test2dHyperbolicRotation(double tol) +void test2dHyperbolicRotation(const T& tol) { Matrix<std::complex<T>,2,2> A, B, C; T angle, ch = std::cosh((T)1); @@ -53,7 +53,7 @@ } template<typename T> -void test3dRotation(double tol) +void test3dRotation(const T& tol) { Matrix<T,3,1> v; T angle; @@ -61,13 +61,13 @@ for (int i=0; i<=20; ++i) { v = Matrix<T,3,1>::Random(); v.normalize(); - angle = pow(10, (i-10) / 5.); + angle = std::pow(T(10), T(i-10) / T(5.)); VERIFY(AngleAxis<T>(angle, v).matrix().isApprox(AngleAxis<T>(1,v).matrix().pow(angle), tol)); } } template<typename MatrixType> -void testGeneral(const MatrixType& m, double tol) +void testGeneral(const MatrixType& m, const typename MatrixType::RealScalar& tol) { typedef typename MatrixType::RealScalar RealScalar; MatrixType m1, m2, m3, m4, m5; @@ -97,7 +97,7 @@ } template<typename MatrixType> -void testSingular(const MatrixType& m_const, double tol) +void testSingular(const MatrixType& m_const, const typename MatrixType::RealScalar& tol) { // we need to pass by reference in order to prevent errors with // MSVC for aligned data types ... @@ -119,18 +119,18 @@ MatrixPower<MatrixType> mpow(m); T = T.sqrt(); - VERIFY(mpow(0.5).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); + VERIFY(mpow(0.5L).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); T = T.sqrt(); - VERIFY(mpow(0.25).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); + VERIFY(mpow(0.25L).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); T = T.sqrt(); - VERIFY(mpow(0.125).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); + VERIFY(mpow(0.125L).isApprox(U * (TriangularType(T) * U.adjoint()), tol)); } } template<typename MatrixType> -void testLogThenExp(const MatrixType& m_const, double tol) +void testLogThenExp(const MatrixType& m_const, const typename MatrixType::RealScalar& tol) { // we need to pass by reference in order to prevent errors with // MSVC for aligned data types ... @@ -150,55 +150,55 @@ typedef Matrix<long double,3,3> Matrix3e; typedef Matrix<long double,Dynamic,Dynamic> MatrixXe; -void test_matrix_power() +EIGEN_DECLARE_TEST(matrix_power) { CALL_SUBTEST_2(test2dRotation<double>(1e-13)); - CALL_SUBTEST_1(test2dRotation<float>(2e-5)); // was 1e-5, relaxed for clang 2.8 / linux / x86-64 - CALL_SUBTEST_9(test2dRotation<long double>(1e-13)); + CALL_SUBTEST_1(test2dRotation<float>(2e-5f)); // was 1e-5, relaxed for clang 2.8 / linux / x86-64 + CALL_SUBTEST_9(test2dRotation<long double>(1e-13L)); CALL_SUBTEST_2(test2dHyperbolicRotation<double>(1e-14)); - CALL_SUBTEST_1(test2dHyperbolicRotation<float>(1e-5)); - CALL_SUBTEST_9(test2dHyperbolicRotation<long double>(1e-14)); + CALL_SUBTEST_1(test2dHyperbolicRotation<float>(1e-5f)); + CALL_SUBTEST_9(test2dHyperbolicRotation<long double>(1e-14L)); CALL_SUBTEST_10(test3dRotation<double>(1e-13)); - CALL_SUBTEST_11(test3dRotation<float>(1e-5)); - CALL_SUBTEST_12(test3dRotation<long double>(1e-13)); + CALL_SUBTEST_11(test3dRotation<float>(1e-5f)); + CALL_SUBTEST_12(test3dRotation<long double>(1e-13L)); CALL_SUBTEST_2(testGeneral(Matrix2d(), 1e-13)); CALL_SUBTEST_7(testGeneral(Matrix3dRowMajor(), 1e-13)); CALL_SUBTEST_3(testGeneral(Matrix4cd(), 1e-13)); CALL_SUBTEST_4(testGeneral(MatrixXd(8,8), 2e-12)); - CALL_SUBTEST_1(testGeneral(Matrix2f(), 1e-4)); - CALL_SUBTEST_5(testGeneral(Matrix3cf(), 1e-4)); - CALL_SUBTEST_8(testGeneral(Matrix4f(), 1e-4)); - CALL_SUBTEST_6(testGeneral(MatrixXf(2,2), 1e-3)); // see bug 614 - CALL_SUBTEST_9(testGeneral(MatrixXe(7,7), 1e-13)); + CALL_SUBTEST_1(testGeneral(Matrix2f(), 1e-4f)); + CALL_SUBTEST_5(testGeneral(Matrix3cf(), 1e-4f)); + CALL_SUBTEST_8(testGeneral(Matrix4f(), 1e-4f)); + CALL_SUBTEST_6(testGeneral(MatrixXf(2,2), 1e-3f)); // see bug 614 + CALL_SUBTEST_9(testGeneral(MatrixXe(7,7), 1e-13L)); CALL_SUBTEST_10(testGeneral(Matrix3d(), 1e-13)); - CALL_SUBTEST_11(testGeneral(Matrix3f(), 1e-4)); - CALL_SUBTEST_12(testGeneral(Matrix3e(), 1e-13)); + CALL_SUBTEST_11(testGeneral(Matrix3f(), 1e-4f)); + CALL_SUBTEST_12(testGeneral(Matrix3e(), 1e-13L)); CALL_SUBTEST_2(testSingular(Matrix2d(), 1e-13)); CALL_SUBTEST_7(testSingular(Matrix3dRowMajor(), 1e-13)); CALL_SUBTEST_3(testSingular(Matrix4cd(), 1e-13)); CALL_SUBTEST_4(testSingular(MatrixXd(8,8), 2e-12)); - CALL_SUBTEST_1(testSingular(Matrix2f(), 1e-4)); - CALL_SUBTEST_5(testSingular(Matrix3cf(), 1e-4)); - CALL_SUBTEST_8(testSingular(Matrix4f(), 1e-4)); - CALL_SUBTEST_6(testSingular(MatrixXf(2,2), 1e-3)); - CALL_SUBTEST_9(testSingular(MatrixXe(7,7), 1e-13)); + CALL_SUBTEST_1(testSingular(Matrix2f(), 1e-4f)); + CALL_SUBTEST_5(testSingular(Matrix3cf(), 1e-4f)); + CALL_SUBTEST_8(testSingular(Matrix4f(), 1e-4f)); + CALL_SUBTEST_6(testSingular(MatrixXf(2,2), 1e-3f)); + CALL_SUBTEST_9(testSingular(MatrixXe(7,7), 1e-13L)); CALL_SUBTEST_10(testSingular(Matrix3d(), 1e-13)); - CALL_SUBTEST_11(testSingular(Matrix3f(), 1e-4)); - CALL_SUBTEST_12(testSingular(Matrix3e(), 1e-13)); + CALL_SUBTEST_11(testSingular(Matrix3f(), 1e-4f)); + CALL_SUBTEST_12(testSingular(Matrix3e(), 1e-13L)); CALL_SUBTEST_2(testLogThenExp(Matrix2d(), 1e-13)); CALL_SUBTEST_7(testLogThenExp(Matrix3dRowMajor(), 1e-13)); CALL_SUBTEST_3(testLogThenExp(Matrix4cd(), 1e-13)); CALL_SUBTEST_4(testLogThenExp(MatrixXd(8,8), 2e-12)); - CALL_SUBTEST_1(testLogThenExp(Matrix2f(), 1e-4)); - CALL_SUBTEST_5(testLogThenExp(Matrix3cf(), 1e-4)); - CALL_SUBTEST_8(testLogThenExp(Matrix4f(), 1e-4)); - CALL_SUBTEST_6(testLogThenExp(MatrixXf(2,2), 1e-3)); - CALL_SUBTEST_9(testLogThenExp(MatrixXe(7,7), 1e-13)); + CALL_SUBTEST_1(testLogThenExp(Matrix2f(), 1e-4f)); + CALL_SUBTEST_5(testLogThenExp(Matrix3cf(), 1e-4f)); + CALL_SUBTEST_8(testLogThenExp(Matrix4f(), 1e-4f)); + CALL_SUBTEST_6(testLogThenExp(MatrixXf(2,2), 1e-3f)); + CALL_SUBTEST_9(testLogThenExp(MatrixXe(7,7), 1e-13L)); CALL_SUBTEST_10(testLogThenExp(Matrix3d(), 1e-13)); - CALL_SUBTEST_11(testLogThenExp(Matrix3f(), 1e-4)); - CALL_SUBTEST_12(testLogThenExp(Matrix3e(), 1e-13)); + CALL_SUBTEST_11(testLogThenExp(Matrix3f(), 1e-4f)); + CALL_SUBTEST_12(testLogThenExp(Matrix3e(), 1e-13L)); }
diff --git a/unsupported/test/matrix_square_root.cpp b/unsupported/test/matrix_square_root.cpp index ea541e1..034f292 100644 --- a/unsupported/test/matrix_square_root.cpp +++ b/unsupported/test/matrix_square_root.cpp
@@ -18,7 +18,7 @@ VERIFY_IS_APPROX(sqrtA * sqrtA, A); } -void test_matrix_square_root() +EIGEN_DECLARE_TEST(matrix_square_root) { for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(testMatrixSqrt(Matrix3cf()));
diff --git a/unsupported/test/minres.cpp b/unsupported/test/minres.cpp index 8b300b7..2eb40fe 100644 --- a/unsupported/test/minres.cpp +++ b/unsupported/test/minres.cpp
@@ -36,7 +36,7 @@ } -void test_minres() +EIGEN_DECLARE_TEST(minres) { CALL_SUBTEST_1(test_minres_T<double>()); // CALL_SUBTEST_2(test_minres_T<std::compex<double> >());
diff --git a/unsupported/test/mpreal/mpreal.h b/unsupported/test/mpreal/mpreal.h index 9b0cf72..2b4f8e8 100644 --- a/unsupported/test/mpreal/mpreal.h +++ b/unsupported/test/mpreal/mpreal.h
@@ -1,34 +1,34 @@ /* - MPFR C++: Multi-precision floating point number class for C++. + MPFR C++: Multi-precision floating point number class for C++. Based on MPFR library: http://mpfr.org Project homepage: http://www.holoborodko.com/pavel/mpfr Contact e-mail: pavel@holoborodko.com - Copyright (c) 2008-2015 Pavel Holoborodko + Copyright (c) 2008-2016 Pavel Holoborodko Contributors: - Dmitriy Gubanov, Konstantin Holoborodko, Brian Gladman, - Helmut Jarausch, Fokko Beekhof, Ulrich Mutze, Heinz van Saanen, - Pere Constans, Peter van Hoof, Gael Guennebaud, Tsai Chia Cheng, + Dmitriy Gubanov, Konstantin Holoborodko, Brian Gladman, + Helmut Jarausch, Fokko Beekhof, Ulrich Mutze, Heinz van Saanen, + Pere Constans, Peter van Hoof, Gael Guennebaud, Tsai Chia Cheng, Alexei Zubanov, Jauhien Piatlicki, Victor Berger, John Westwood, Petr Aleksandrov, Orion Poplawski, Charles Karney, Arash Partow, - Rodney James, Jorge Leitao. + Rodney James, Jorge Leitao, Jerome Benoit. Licensing: (A) MPFR C++ is under GNU General Public License ("GPL"). - - (B) Non-free licenses may also be purchased from the author, for users who + + (B) Non-free licenses may also be purchased from the author, for users who do not want their programs protected by the GPL. - The non-free licenses are for users that wish to use MPFR C++ in - their products but are unwilling to release their software - under the GPL (which would require them to release source code + The non-free licenses are for users that wish to use MPFR C++ in + their products but are unwilling to release their software + under the GPL (which would require them to release source code and allow free redistribution). Such users can purchase an unlimited-use license from the author. Contact us for more details. - + GNU General Public License ("GPL") copyright permissions statement: ************************************************************************** This program is free software: you can redistribute it and/or modify @@ -58,6 +58,7 @@ #include <limits> #include <complex> #include <algorithm> +#include <stdint.h> // Options #define MPREAL_HAVE_MSVC_DEBUGVIEW // Enable Debugger Visualizer for "Debug" builds in MSVC. @@ -68,16 +69,18 @@ // Library version #define MPREAL_VERSION_MAJOR 3 #define MPREAL_VERSION_MINOR 6 -#define MPREAL_VERSION_PATCHLEVEL 2 -#define MPREAL_VERSION_STRING "3.6.2" +#define MPREAL_VERSION_PATCHLEVEL 5 +#define MPREAL_VERSION_STRING "3.6.5" // Detect compiler using signatures from http://predef.sourceforge.net/ -#if defined(__GNUC__) - #define IsInf(x) (isinf)(x) // GNU C++/Intel ICC compiler on Linux -#elif defined(_MSC_VER) // Microsoft Visual C++ - #define IsInf(x) (!_finite(x)) +#if defined(__GNUC__) && defined(__INTEL_COMPILER) + #define IsInf(x) isinf EIGEN_NOT_A_MACRO (x) // Intel ICC compiler on Linux + +#elif defined(_MSC_VER) // Microsoft Visual C++ + #define IsInf(x) (!_finite(x)) + #else - #define IsInf(x) (std::isinf)(x) // GNU C/C++ (and/or other compilers), just hope for C99 conformance + #define IsInf(x) std::isinf EIGEN_NOT_A_MACRO (x) // GNU C/C++ (and/or other compilers), just hope for C99 conformance #endif // A Clang feature extension to determine compiler features. @@ -85,22 +88,26 @@ #define __has_feature(x) 0 #endif -// Detect support for r-value references (move semantic). Borrowed from Eigen. +// Detect support for r-value references (move semantic). +// Move semantic should be enabled with great care in multi-threading environments, +// especially if MPFR uses custom memory allocators. +// Everything should be thread-safe and support passing ownership over thread boundary. #if (__has_feature(cxx_rvalue_references) || \ defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ - (defined(_MSC_VER) && _MSC_VER >= 1600)) + (defined(_MSC_VER) && _MSC_VER >= 1600) && !defined(MPREAL_DISABLE_MOVE_SEMANTIC)) #define MPREAL_HAVE_MOVE_SUPPORT - // Use fields in mpfr_t structure to check if it was initialized / set dummy initialization + // Use fields in mpfr_t structure to check if it was initialized / set dummy initialization #define mpfr_is_initialized(x) (0 != (x)->_mpfr_d) #define mpfr_set_uninitialized(x) ((x)->_mpfr_d = 0 ) #endif -// Detect support for explicit converters. +// Detect support for explicit converters. #if (__has_feature(cxx_explicit_conversions) || \ - (defined(__GXX_EXPERIMENTAL_CXX0X__) && __GNUC_MINOR >= 5) || __cplusplus >= 201103L || \ - (defined(_MSC_VER) && _MSC_VER >= 1800)) + (defined(__GXX_EXPERIMENTAL_CXX0X__) && __GNUC_MINOR__ >= 5) || __cplusplus >= 201103L || \ + (defined(_MSC_VER) && _MSC_VER >= 1800) || \ + (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1300)) #define MPREAL_HAVE_EXPLICIT_CONVERTERS #endif @@ -111,8 +118,8 @@ #define MPREAL_MSVC_DEBUGVIEW_CODE DebugView = toString(); #define MPREAL_MSVC_DEBUGVIEW_DATA std::string DebugView; #else - #define MPREAL_MSVC_DEBUGVIEW_CODE - #define MPREAL_MSVC_DEBUGVIEW_DATA + #define MPREAL_MSVC_DEBUGVIEW_CODE + #define MPREAL_MSVC_DEBUGVIEW_DATA #endif #include <mpfr.h> @@ -122,12 +129,12 @@ #endif // Less important options -#define MPREAL_DOUBLE_BITS_OVERFLOW -1 // Triggers overflow exception during conversion to double if mpreal +#define MPREAL_DOUBLE_BITS_OVERFLOW -1 // Triggers overflow exception during conversion to double if mpreal // cannot fit in MPREAL_DOUBLE_BITS_OVERFLOW bits // = -1 disables overflow checks (default) // Fast replacement for mpfr_set_zero(x, +1): -// (a) uses low-level data members, might not be compatible with new versions of MPFR +// (a) uses low-level data members, might not be forward compatible // (b) sign is not set, add (x)->_mpfr_sign = 1; #define mpfr_set_zero_fast(x) ((x)->_mpfr_exp = __MPFR_EXP_ZERO) @@ -142,19 +149,19 @@ class mpreal { private: mpfr_t mp; - + public: - + // Get default rounding mode & precision inline static mp_rnd_t get_default_rnd() { return (mp_rnd_t)(mpfr_get_default_rounding_mode()); } - inline static mp_prec_t get_default_prec() { return mpfr_get_default_prec(); } + inline static mp_prec_t get_default_prec() { return (mpfr_get_default_prec)(); } // Constructors && type conversions mpreal(); mpreal(const mpreal& u); - mpreal(const mpf_t u); - mpreal(const mpz_t u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); - mpreal(const mpq_t u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); + mpreal(const mpf_t u); + mpreal(const mpz_t u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); + mpreal(const mpq_t u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const double u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const long double u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const unsigned long long int u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); @@ -163,15 +170,15 @@ mpreal(const unsigned int u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const long int u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const int u, mp_prec_t prec = mpreal::get_default_prec(), mp_rnd_t mode = mpreal::get_default_rnd()); - + // Construct mpreal from mpfr_t structure. - // shared = true allows to avoid deep copy, so that mpreal and 'u' share the same data & pointers. - mpreal(const mpfr_t u, bool shared = false); + // shared = true allows to avoid deep copy, so that mpreal and 'u' share the same data & pointers. + mpreal(const mpfr_t u, bool shared = false); mpreal(const char* s, mp_prec_t prec = mpreal::get_default_prec(), int base = 10, mp_rnd_t mode = mpreal::get_default_rnd()); mpreal(const std::string& s, mp_prec_t prec = mpreal::get_default_prec(), int base = 10, mp_rnd_t mode = mpreal::get_default_rnd()); - ~mpreal(); + ~mpreal(); #ifdef MPREAL_HAVE_MOVE_SUPPORT mpreal& operator=(mpreal&& v); @@ -180,7 +187,7 @@ // Operations // = - // +, -, *, /, ++, --, <<, >> + // +, -, *, /, ++, --, <<, >> // *=, +=, -=, /=, // <, >, ==, <=, >= @@ -190,7 +197,7 @@ mpreal& operator=(const mpz_t v); mpreal& operator=(const mpq_t v); mpreal& operator=(const long double v); - mpreal& operator=(const double v); + mpreal& operator=(const double v); mpreal& operator=(const unsigned long int v); mpreal& operator=(const unsigned long long int v); mpreal& operator=(const long long int v); @@ -224,7 +231,7 @@ const mpreal operator+() const; mpreal& operator++ (); - const mpreal operator++ (int); + const mpreal operator++ (int); // - mpreal& operator-=(const mpreal& v); @@ -242,7 +249,7 @@ friend const mpreal operator-(const long int b, const mpreal& a); friend const mpreal operator-(const int b, const mpreal& a); friend const mpreal operator-(const double b, const mpreal& a); - mpreal& operator-- (); + mpreal& operator-- (); const mpreal operator-- (int); // * @@ -255,7 +262,7 @@ mpreal& operator*=(const unsigned int v); mpreal& operator*=(const long int v); mpreal& operator*=(const int v); - + // / mpreal& operator/=(const mpreal& v); mpreal& operator/=(const mpz_t v); @@ -343,17 +350,19 @@ friend inline const mpreal div_2ui(const mpreal& v, unsigned long int k, mp_rnd_t rnd_mode); friend inline const mpreal div_2si(const mpreal& v, long int k, mp_rnd_t rnd_mode); friend int cmpabs(const mpreal& a,const mpreal& b); - + friend const mpreal log (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal log2 (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal logb (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal log10(const mpreal& v, mp_rnd_t rnd_mode); - friend const mpreal exp (const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal exp (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal exp2 (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal exp10(const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal log1p(const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal expm1(const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal nextpow2(const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal cos(const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal sin(const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal tan(const mpreal& v, mp_rnd_t rnd_mode); @@ -395,17 +404,17 @@ friend const mpreal zeta (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal erf (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal erfc (const mpreal& v, mp_rnd_t rnd_mode); - friend const mpreal besselj0 (const mpreal& v, mp_rnd_t rnd_mode); - friend const mpreal besselj1 (const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal besselj0 (const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal besselj1 (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal besseljn (long n, const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal bessely0 (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal bessely1 (const mpreal& v, mp_rnd_t rnd_mode); - friend const mpreal besselyn (long n, const mpreal& v, mp_rnd_t rnd_mode); + friend const mpreal besselyn (long n, const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal fma (const mpreal& v1, const mpreal& v2, const mpreal& v3, mp_rnd_t rnd_mode); friend const mpreal fms (const mpreal& v1, const mpreal& v2, const mpreal& v3, mp_rnd_t rnd_mode); friend const mpreal agm (const mpreal& v1, const mpreal& v2, mp_rnd_t rnd_mode); friend const mpreal sum (const mpreal tab[], const unsigned long int n, int& status, mp_rnd_t rnd_mode); - friend int sgn(const mpreal& v); // returns -1 or +1 + friend int sgn (const mpreal& v); // MPFR 2.4.0 Specifics #if (MPFR_VERSION >= MPFR_VERSION_NUM(2,4,0)) @@ -438,7 +447,7 @@ // Splits mpreal value into fractional and integer parts. // Returns fractional part and stores integer part in n. - friend const mpreal modf(const mpreal& v, mpreal& n); + friend const mpreal modf(const mpreal& v, mpreal& n); // Constants // don't forget to call mpfr_free_cache() for every thread where you are using const-functions @@ -467,14 +476,14 @@ friend const mpreal frac (const mpreal& v, mp_rnd_t rnd_mode); friend const mpreal remainder ( const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode); friend const mpreal remquo (long* q, const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode); - + // Miscellaneous Functions friend const mpreal nexttoward (const mpreal& x, const mpreal& y); friend const mpreal nextabove (const mpreal& x); friend const mpreal nextbelow (const mpreal& x); // use gmp_randinit_default() to init state, gmp_randclear() to clear - friend const mpreal urandomb (gmp_randstate_t& state); + friend const mpreal urandomb (gmp_randstate_t& state); // MPFR < 2.4.2 Specifics #if (MPFR_VERSION <= MPFR_VERSION_NUM(2,4,2)) @@ -482,7 +491,7 @@ #endif // Instance Checkers - friend bool (isnan) (const mpreal& v); + friend bool isnan EIGEN_NOT_A_MACRO (const mpreal& v); friend bool (isinf) (const mpreal& v); friend bool (isfinite) (const mpreal& v); @@ -501,15 +510,15 @@ // Aliases for get_prec(), set_prec() - needed for compatibility with std::complex<mpreal> interface inline mpreal& setPrecision(int Precision, mp_rnd_t RoundingMode = get_default_rnd()); inline int getPrecision() const; - + // Set mpreal to +/- inf, NaN, +/-0 - mpreal& setInf (int Sign = +1); + mpreal& setInf (int Sign = +1); mpreal& setNan (); mpreal& setZero (int Sign = +1); mpreal& setSign (int Sign, mp_rnd_t RoundingMode = get_default_rnd()); //Exponent - mp_exp_t get_exp(); + mp_exp_t get_exp() const; int set_exp(mp_exp_t e); int check_range (int t, mp_rnd_t rnd_mode = get_default_rnd()); int subnormalize (int t, mp_rnd_t rnd_mode = get_default_rnd()); @@ -532,7 +541,7 @@ // Efficient swapping of two mpreal values - needed for std algorithms friend void swap(mpreal& x, mpreal& y); - + friend const mpreal fmax(const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode); friend const mpreal fmin(const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode); @@ -542,7 +551,7 @@ // // mpfr::mpreal=<DebugView> ; Show value only // mpfr::mpreal=<DebugView>, <mp[0]._mpfr_prec,u>bits ; Show value & precision - // + // // at the beginning of // [Visual Studio Installation Folder]\Common7\Packages\Debugger\autoexp.dat MPREAL_MSVC_DEBUGVIEW_DATA @@ -561,15 +570,15 @@ ////////////////////////////////////////////////////////////////////////// // Constructors & converters // Default constructor: creates mp number and initializes it to 0. -inline mpreal::mpreal() -{ - mpfr_init2(mpfr_ptr(), mpreal::get_default_prec()); +inline mpreal::mpreal() +{ + mpfr_init2(mpfr_ptr(), mpreal::get_default_prec()); mpfr_set_zero_fast(mpfr_ptr()); MPREAL_MSVC_DEBUGVIEW_CODE; } -inline mpreal::mpreal(const mpreal& u) +inline mpreal::mpreal(const mpreal& u) { mpfr_init2(mpfr_ptr(),mpfr_get_prec(u.mpfr_srcptr())); mpfr_set (mpfr_ptr(),u.mpfr_srcptr(),mpreal::get_default_rnd()); @@ -580,7 +589,7 @@ #ifdef MPREAL_HAVE_MOVE_SUPPORT inline mpreal::mpreal(mpreal&& other) { - mpfr_set_uninitialized(mpfr_ptr()); // make sure "other" holds no pointer to actual data + mpfr_set_uninitialized(mpfr_ptr()); // make sure "other" holds null-pointer (in uninitialized state) mpfr_swap(mpfr_ptr(), other.mpfr_ptr()); MPREAL_MSVC_DEBUGVIEW_CODE; @@ -588,9 +597,11 @@ inline mpreal& mpreal::operator=(mpreal&& other) { - mpfr_swap(mpfr_ptr(), other.mpfr_ptr()); - - MPREAL_MSVC_DEBUGVIEW_CODE; + if (this != &other) + { + mpfr_swap(mpfr_ptr(), other.mpfr_ptr()); // destructor for "other" will be called just afterwards + MPREAL_MSVC_DEBUGVIEW_CODE; + } return *this; } #endif @@ -639,20 +650,20 @@ mpfr_init2(mpfr_ptr(), prec); #if (MPREAL_DOUBLE_BITS_OVERFLOW > -1) - if(fits_in_bits(u, MPREAL_DOUBLE_BITS_OVERFLOW)) - { - mpfr_set_d(mpfr_ptr(), u, mode); - }else - throw conversion_overflow(); + if(fits_in_bits(u, MPREAL_DOUBLE_BITS_OVERFLOW)) + { + mpfr_set_d(mpfr_ptr(), u, mode); + }else + throw conversion_overflow(); #else - mpfr_set_d(mpfr_ptr(), u, mode); + mpfr_set_d(mpfr_ptr(), u, mode); #endif MPREAL_MSVC_DEBUGVIEW_CODE; } inline mpreal::mpreal(const long double u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_ld(mpfr_ptr(), u, mode); @@ -660,7 +671,7 @@ } inline mpreal::mpreal(const unsigned long long int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_uj(mpfr_ptr(), u, mode); @@ -668,7 +679,7 @@ } inline mpreal::mpreal(const long long int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_sj(mpfr_ptr(), u, mode); @@ -676,7 +687,7 @@ } inline mpreal::mpreal(const unsigned long int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_ui(mpfr_ptr(), u, mode); @@ -684,7 +695,7 @@ } inline mpreal::mpreal(const unsigned int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_ui(mpfr_ptr(), u, mode); @@ -692,7 +703,7 @@ } inline mpreal::mpreal(const long int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_si(mpfr_ptr(), u, mode); @@ -700,7 +711,7 @@ } inline mpreal::mpreal(const int u, mp_prec_t prec, mp_rnd_t mode) -{ +{ mpfr_init2 (mpfr_ptr(), prec); mpfr_set_si(mpfr_ptr(), u, mode); @@ -710,7 +721,7 @@ inline mpreal::mpreal(const char* s, mp_prec_t prec, int base, mp_rnd_t mode) { mpfr_init2 (mpfr_ptr(), prec); - mpfr_set_str(mpfr_ptr(), s, base, mode); + mpfr_set_str(mpfr_ptr(), s, base, mode); MPREAL_MSVC_DEBUGVIEW_CODE; } @@ -718,7 +729,7 @@ inline mpreal::mpreal(const std::string& s, mp_prec_t prec, int base, mp_rnd_t mode) { mpfr_init2 (mpfr_ptr(), prec); - mpfr_set_str(mpfr_ptr(), s.c_str(), base, mode); + mpfr_set_str(mpfr_ptr(), s.c_str(), base, mode); MPREAL_MSVC_DEBUGVIEW_CODE; } @@ -726,15 +737,15 @@ inline void mpreal::clear(::mpfr_ptr x) { #ifdef MPREAL_HAVE_MOVE_SUPPORT - if(mpfr_is_initialized(x)) + if(mpfr_is_initialized(x)) #endif mpfr_clear(x); } -inline mpreal::~mpreal() -{ +inline mpreal::~mpreal() +{ clear(mpfr_ptr()); -} +} // internal namespace needed for template magic namespace internal{ @@ -742,55 +753,55 @@ // Use SFINAE to restrict arithmetic operations instantiation only for numeric types // This is needed for smooth integration with libraries based on expression templates, like Eigen. // TODO: Do the same for boolean operators. - template <typename ArgumentType> struct result_type {}; - - template <> struct result_type<mpreal> {typedef mpreal type;}; - template <> struct result_type<mpz_t> {typedef mpreal type;}; - template <> struct result_type<mpq_t> {typedef mpreal type;}; - template <> struct result_type<long double> {typedef mpreal type;}; - template <> struct result_type<double> {typedef mpreal type;}; - template <> struct result_type<unsigned long int> {typedef mpreal type;}; - template <> struct result_type<unsigned int> {typedef mpreal type;}; - template <> struct result_type<long int> {typedef mpreal type;}; - template <> struct result_type<int> {typedef mpreal type;}; - template <> struct result_type<long long> {typedef mpreal type;}; - template <> struct result_type<unsigned long long> {typedef mpreal type;}; + template <typename ArgumentType> struct result_type {}; + + template <> struct result_type<mpreal> {typedef mpreal type;}; + template <> struct result_type<mpz_t> {typedef mpreal type;}; + template <> struct result_type<mpq_t> {typedef mpreal type;}; + template <> struct result_type<long double> {typedef mpreal type;}; + template <> struct result_type<double> {typedef mpreal type;}; + template <> struct result_type<unsigned long int> {typedef mpreal type;}; + template <> struct result_type<unsigned int> {typedef mpreal type;}; + template <> struct result_type<long int> {typedef mpreal type;}; + template <> struct result_type<int> {typedef mpreal type;}; + template <> struct result_type<long long> {typedef mpreal type;}; + template <> struct result_type<unsigned long long> {typedef mpreal type;}; } // + Addition -template <typename Rhs> -inline const typename internal::result_type<Rhs>::type +template <typename Rhs> +inline const typename internal::result_type<Rhs>::type operator+(const mpreal& lhs, const Rhs& rhs){ return mpreal(lhs) += rhs; } -template <typename Lhs> -inline const typename internal::result_type<Lhs>::type - operator+(const Lhs& lhs, const mpreal& rhs){ return mpreal(rhs) += lhs; } +template <typename Lhs> +inline const typename internal::result_type<Lhs>::type + operator+(const Lhs& lhs, const mpreal& rhs){ return mpreal(rhs) += lhs; } // - Subtraction -template <typename Rhs> -inline const typename internal::result_type<Rhs>::type +template <typename Rhs> +inline const typename internal::result_type<Rhs>::type operator-(const mpreal& lhs, const Rhs& rhs){ return mpreal(lhs) -= rhs; } -template <typename Lhs> -inline const typename internal::result_type<Lhs>::type +template <typename Lhs> +inline const typename internal::result_type<Lhs>::type operator-(const Lhs& lhs, const mpreal& rhs){ return mpreal(lhs) -= rhs; } // * Multiplication -template <typename Rhs> -inline const typename internal::result_type<Rhs>::type +template <typename Rhs> +inline const typename internal::result_type<Rhs>::type operator*(const mpreal& lhs, const Rhs& rhs){ return mpreal(lhs) *= rhs; } -template <typename Lhs> -inline const typename internal::result_type<Lhs>::type - operator*(const Lhs& lhs, const mpreal& rhs){ return mpreal(rhs) *= lhs; } +template <typename Lhs> +inline const typename internal::result_type<Lhs>::type + operator*(const Lhs& lhs, const mpreal& rhs){ return mpreal(rhs) *= lhs; } // / Division -template <typename Rhs> -inline const typename internal::result_type<Rhs>::type +template <typename Rhs> +inline const typename internal::result_type<Rhs>::type operator/(const mpreal& lhs, const Rhs& rhs){ return mpreal(lhs) /= rhs; } -template <typename Lhs> -inline const typename internal::result_type<Lhs>::type +template <typename Lhs> +inline const typename internal::result_type<Lhs>::type operator/(const Lhs& lhs, const mpreal& rhs){ return mpreal(lhs) /= rhs; } ////////////////////////////////////////////////////////////////////////// @@ -840,17 +851,17 @@ const mpreal pow(const int a, const unsigned long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const int a, const unsigned int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const int a, const long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); -const mpreal pow(const int a, const int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); +const mpreal pow(const int a, const int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const int a, const long double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); -const mpreal pow(const int a, const double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); +const mpreal pow(const int a, const double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); -const mpreal pow(const long double a, const long double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); +const mpreal pow(const long double a, const long double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const long double a, const unsigned long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const long double a, const unsigned int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const long double a, const long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const long double a, const int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); -const mpreal pow(const double a, const double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); +const mpreal pow(const double a, const double b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const double a, const unsigned long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const double a, const unsigned int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); const mpreal pow(const double a, const long int b, mp_rnd_t rnd_mode = mpreal::get_default_rnd()); @@ -867,9 +878,9 @@ inline mpreal machine_epsilon(mp_prec_t prec = mpreal::get_default_prec()); // Returns smallest eps such that x + eps != x (relative machine epsilon) -inline mpreal machine_epsilon(const mpreal& x); +inline mpreal machine_epsilon(const mpreal& x); -// Gives max & min values for the required precision, +// Gives max & min values for the required precision, // minval is 'safe' meaning 1 / minval does not overflow // maxval is 'safe' meaning 1 / maxval does not underflow inline mpreal minval(mp_prec_t prec = mpreal::get_default_prec()); @@ -882,7 +893,7 @@ inline bool isEqualFuzzy(const mpreal& a, const mpreal& b); // 'Bitwise' equality check -// maxUlps - a and b can be apart by maxUlps binary numbers. +// maxUlps - a and b can be apart by maxUlps binary numbers. inline bool isEqualUlps(const mpreal& a, const mpreal& b, int maxUlps); ////////////////////////////////////////////////////////////////////////// @@ -908,13 +919,13 @@ { if (this != &v) { - mp_prec_t tp = mpfr_get_prec( mpfr_srcptr()); - mp_prec_t vp = mpfr_get_prec(v.mpfr_srcptr()); + mp_prec_t tp = mpfr_get_prec( mpfr_srcptr()); + mp_prec_t vp = mpfr_get_prec(v.mpfr_srcptr()); - if(tp != vp){ - clear(mpfr_ptr()); - mpfr_init2(mpfr_ptr(), vp); - } + if(tp != vp){ + clear(mpfr_ptr()); + mpfr_init2(mpfr_ptr(), vp); + } mpfr_set(mpfr_ptr(), v.mpfr_srcptr(), mpreal::get_default_rnd()); @@ -926,7 +937,7 @@ inline mpreal& mpreal::operator=(const mpf_t v) { mpfr_set_f(mpfr_ptr(), v, mpreal::get_default_rnd()); - + MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } @@ -934,7 +945,7 @@ inline mpreal& mpreal::operator=(const mpz_t v) { mpfr_set_z(mpfr_ptr(), v, mpreal::get_default_rnd()); - + MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } @@ -947,73 +958,73 @@ return *this; } -inline mpreal& mpreal::operator=(const long double v) -{ +inline mpreal& mpreal::operator=(const long double v) +{ mpfr_set_ld(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const double v) -{ +inline mpreal& mpreal::operator=(const double v) +{ #if (MPREAL_DOUBLE_BITS_OVERFLOW > -1) - if(fits_in_bits(v, MPREAL_DOUBLE_BITS_OVERFLOW)) - { - mpfr_set_d(mpfr_ptr(),v,mpreal::get_default_rnd()); - }else - throw conversion_overflow(); + if(fits_in_bits(v, MPREAL_DOUBLE_BITS_OVERFLOW)) + { + mpfr_set_d(mpfr_ptr(),v,mpreal::get_default_rnd()); + }else + throw conversion_overflow(); #else - mpfr_set_d(mpfr_ptr(),v,mpreal::get_default_rnd()); + mpfr_set_d(mpfr_ptr(),v,mpreal::get_default_rnd()); #endif - MPREAL_MSVC_DEBUGVIEW_CODE; + MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const unsigned long int v) -{ - mpfr_set_ui(mpfr_ptr(), v, mpreal::get_default_rnd()); +inline mpreal& mpreal::operator=(const unsigned long int v) +{ + mpfr_set_ui(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const unsigned int v) -{ - mpfr_set_ui(mpfr_ptr(), v, mpreal::get_default_rnd()); +inline mpreal& mpreal::operator=(const unsigned int v) +{ + mpfr_set_ui(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const unsigned long long int v) -{ - mpfr_set_uj(mpfr_ptr(), v, mpreal::get_default_rnd()); +inline mpreal& mpreal::operator=(const unsigned long long int v) +{ + mpfr_set_uj(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const long long int v) -{ - mpfr_set_sj(mpfr_ptr(), v, mpreal::get_default_rnd()); +inline mpreal& mpreal::operator=(const long long int v) +{ + mpfr_set_sj(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } -inline mpreal& mpreal::operator=(const long int v) -{ - mpfr_set_si(mpfr_ptr(), v, mpreal::get_default_rnd()); +inline mpreal& mpreal::operator=(const long int v) +{ + mpfr_set_si(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } inline mpreal& mpreal::operator=(const int v) -{ - mpfr_set_si(mpfr_ptr(), v, mpreal::get_default_rnd()); +{ + mpfr_set_si(mpfr_ptr(), v, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; @@ -1034,7 +1045,7 @@ if(0 == mpfr_set_str(t, s, 10, mpreal::get_default_rnd())) { - mpfr_set(mpfr_ptr(), t, mpreal::get_default_rnd()); + mpfr_set(mpfr_ptr(), t, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; } @@ -1057,7 +1068,7 @@ if(0 == mpfr_set_str(t, s.c_str(), 10, mpreal::get_default_rnd())) { - mpfr_set(mpfr_ptr(), t, mpreal::get_default_rnd()); + mpfr_set(mpfr_ptr(), t, mpreal::get_default_rnd()); MPREAL_MSVC_DEBUGVIEW_CODE; } @@ -1065,7 +1076,7 @@ return *this; } -template <typename real_t> +template <typename real_t> inline mpreal& mpreal::operator= (const std::complex<real_t>& z) { return *this = z.real(); @@ -1103,9 +1114,9 @@ inline mpreal& mpreal::operator+= (const long double u) { - *this += mpreal(u); + *this += mpreal(u); MPREAL_MSVC_DEBUGVIEW_CODE; - return *this; + return *this; } inline mpreal& mpreal::operator+= (const double u) @@ -1161,12 +1172,12 @@ inline const mpreal operator+(const mpreal& a, const mpreal& b) { - mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); - mpfr_add(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); - return c; + mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); + mpfr_add(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); + return c; } -inline mpreal& mpreal::operator++() +inline mpreal& mpreal::operator++() { return *this += 1; } @@ -1178,7 +1189,7 @@ return x; } -inline mpreal& mpreal::operator--() +inline mpreal& mpreal::operator--() { return *this -= 1; } @@ -1215,9 +1226,9 @@ inline mpreal& mpreal::operator-=(const long double v) { - *this -= mpreal(v); + *this -= mpreal(v); MPREAL_MSVC_DEBUGVIEW_CODE; - return *this; + return *this; } inline mpreal& mpreal::operator-=(const double v) @@ -1225,7 +1236,7 @@ #if (MPFR_VERSION >= MPFR_VERSION_NUM(2,4,0)) mpfr_sub_d(mpfr_ptr(),mpfr_srcptr(),v,mpreal::get_default_rnd()); #else - *this -= mpreal(v); + *this -= mpreal(v); #endif MPREAL_MSVC_DEBUGVIEW_CODE; @@ -1269,9 +1280,9 @@ inline const mpreal operator-(const mpreal& a, const mpreal& b) { - mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); - mpfr_sub(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); - return c; + mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); + mpfr_sub(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); + return c; } inline const mpreal operator-(const double b, const mpreal& a) @@ -1340,9 +1351,9 @@ inline mpreal& mpreal::operator*=(const long double v) { - *this *= mpreal(v); + *this *= mpreal(v); MPREAL_MSVC_DEBUGVIEW_CODE; - return *this; + return *this; } inline mpreal& mpreal::operator*=(const double v) @@ -1350,7 +1361,7 @@ #if (MPFR_VERSION >= MPFR_VERSION_NUM(2,4,0)) mpfr_mul_d(mpfr_ptr(),mpfr_srcptr(),v,mpreal::get_default_rnd()); #else - *this *= mpreal(v); + *this *= mpreal(v); #endif MPREAL_MSVC_DEBUGVIEW_CODE; return *this; @@ -1386,9 +1397,9 @@ inline const mpreal operator*(const mpreal& a, const mpreal& b) { - mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); - mpfr_mul(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); - return c; + mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_ptr()), mpfr_get_prec(b.mpfr_ptr()))); + mpfr_mul(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); + return c; } ////////////////////////////////////////////////////////////////////////// @@ -1418,7 +1429,7 @@ { *this /= mpreal(v); MPREAL_MSVC_DEBUGVIEW_CODE; - return *this; + return *this; } inline mpreal& mpreal::operator/=(const double v) @@ -1426,7 +1437,7 @@ #if (MPFR_VERSION >= MPFR_VERSION_NUM(2,4,0)) mpfr_div_d(mpfr_ptr(),mpfr_srcptr(),v,mpreal::get_default_rnd()); #else - *this /= mpreal(v); + *this /= mpreal(v); #endif MPREAL_MSVC_DEBUGVIEW_CODE; return *this; @@ -1462,9 +1473,9 @@ inline const mpreal operator/(const mpreal& a, const mpreal& b) { - mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_srcptr()), mpfr_get_prec(b.mpfr_srcptr()))); - mpfr_div(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); - return c; + mpreal c(0, (std::max)(mpfr_get_prec(a.mpfr_srcptr()), mpfr_get_prec(b.mpfr_srcptr()))); + mpfr_div(c.mpfr_ptr(), a.mpfr_srcptr(), b.mpfr_srcptr(), mpreal::get_default_rnd()); + return c; } inline const mpreal operator/(const unsigned long int b, const mpreal& a) @@ -1639,9 +1650,9 @@ ////////////////////////////////////////////////////////////////////////// //Relational operators -// WARNING: +// WARNING: // -// Please note that following checks for double-NaN are guaranteed to work only in IEEE math mode: +// Please note that following checks for double-NaN are guaranteed to work only in IEEE math mode: // // isnan(b) = (b != b) // isnan(b) = !(b == b) (we use in code below) @@ -1659,7 +1670,7 @@ inline bool operator >= (const mpreal& a, const mpreal& b ){ return (mpfr_greaterequal_p(a.mpfr_srcptr(),b.mpfr_srcptr()) != 0 ); } inline bool operator >= (const mpreal& a, const unsigned long int b ){ return !isnan EIGEN_NOT_A_MACRO (a) && (mpfr_cmp_ui(a.mpfr_srcptr(),b) >= 0 ); } -// inline bool operator >= (const mpreal& a, const unsigned int b ){ return !isnan EIGEN_NOT_A_MACRO (isnan()a) && (mpfr_cmp_ui(a.mpfr_srcptr(),b) >= 0 ); } +inline bool operator >= (const mpreal& a, const unsigned int b ){ return !isnan EIGEN_NOT_A_MACRO (a) && (mpfr_cmp_ui(a.mpfr_srcptr(),b) >= 0 ); } inline bool operator >= (const mpreal& a, const long int b ){ return !isnan EIGEN_NOT_A_MACRO (a) && (mpfr_cmp_si(a.mpfr_srcptr(),b) >= 0 ); } inline bool operator >= (const mpreal& a, const int b ){ return !isnan EIGEN_NOT_A_MACRO (a) && (mpfr_cmp_si(a.mpfr_srcptr(),b) >= 0 ); } inline bool operator >= (const mpreal& a, const long double b ){ return !isnan EIGEN_NOT_A_MACRO (a) && (b == b) && (mpfr_cmp_ld(a.mpfr_srcptr(),b) >= 0 ); } @@ -1697,7 +1708,7 @@ inline bool operator != (const mpreal& a, const long double b ){ return !(a == b); } inline bool operator != (const mpreal& a, const double b ){ return !(a == b); } -inline bool (isnan) (const mpreal& op){ return (mpfr_nan_p (op.mpfr_srcptr()) != 0 ); } +inline bool isnan EIGEN_NOT_A_MACRO (const mpreal& op){ return (mpfr_nan_p (op.mpfr_srcptr()) != 0 ); } inline bool (isinf) (const mpreal& op){ return (mpfr_inf_p (op.mpfr_srcptr()) != 0 ); } inline bool (isfinite) (const mpreal& op){ return (mpfr_number_p (op.mpfr_srcptr()) != 0 ); } inline bool iszero (const mpreal& op){ return (mpfr_zero_p (op.mpfr_srcptr()) != 0 ); } @@ -1705,7 +1716,7 @@ #if (MPFR_VERSION >= MPFR_VERSION_NUM(3,0,0)) inline bool isregular(const mpreal& op){ return (mpfr_regular_p(op.mpfr_srcptr()));} -#endif +#endif ////////////////////////////////////////////////////////////////////////// // Type Converters @@ -1762,21 +1773,21 @@ std::ostringstream format; - int digits = (n >= 0) ? n : 1 + bits2digits(mpfr_get_prec(mpfr_srcptr())); - + int digits = (n >= 0) ? n : 2 + bits2digits(mpfr_get_prec(mpfr_srcptr())); + format << "%." << digits << "RNg"; return toString(format.str()); #else - char *s, *ns = NULL; + char *s, *ns = NULL; size_t slen, nslen; mp_exp_t exp; std::string out; if(mpfr_inf_p(mp)) - { + { if(mpfr_sgn(mp)>0) return "+Inf"; else return "-Inf"; } @@ -1791,7 +1802,7 @@ { slen = strlen(s); nslen = strlen(ns); - if(nslen<=slen) + if(nslen<=slen) { mpfr_free_str(s); s = ns; @@ -1808,7 +1819,7 @@ { // Remove zeros starting from right end char* ptr = s+slen-1; - while (*ptr=='0' && ptr>s+exp) ptr--; + while (*ptr=='0' && ptr>s+exp) ptr--; if(ptr==s+exp) out = std::string(s,exp+1); else out = std::string(s,exp+1)+'.'+std::string(s+exp+1,ptr-(s+exp+1)+1); @@ -1819,7 +1830,7 @@ { // Remove zeros starting from right end char* ptr = s+slen-1; - while (*ptr=='0' && ptr>s+exp-1) ptr--; + while (*ptr=='0' && ptr>s+exp-1) ptr--; if(ptr==s+exp-1) out = std::string(s,exp); else out = std::string(s,exp)+'.'+std::string(s+exp,ptr-(s+exp)+1); @@ -1832,7 +1843,7 @@ { // Remove zeros starting from right end char* ptr = s+slen-1; - while (*ptr=='0' && ptr>s+1) ptr--; + while (*ptr=='0' && ptr>s+1) ptr--; if(ptr==s+1) out = std::string(s,2); else out = std::string(s,2)+'.'+std::string(s+2,ptr-(s+2)+1); @@ -1843,7 +1854,7 @@ { // Remove zeros starting from right end char* ptr = s+slen-1; - while (*ptr=='0' && ptr>s) ptr--; + while (*ptr=='0' && ptr>s) ptr--; if(ptr==s) out = std::string(s,1); else out = std::string(s,1)+'.'+std::string(s+1,ptr-(s+1)+1); @@ -1870,7 +1881,7 @@ ////////////////////////////////////////////////////////////////////////// // I/O -inline std::ostream& mpreal::output(std::ostream& os) const +inline std::ostream& mpreal::output(std::ostream& os) const { std::ostringstream format; const std::ios::fmtflags flags = os.flags(); @@ -1931,14 +1942,9 @@ ////////////////////////////////////////////////////////////////////////// // Set/Get number properties -inline int sgn(const mpreal& op) -{ - return mpfr_sgn(op.mpfr_srcptr()); -} - inline mpreal& mpreal::setSign(int sign, mp_rnd_t RoundingMode) { - mpfr_setsign(mpfr_ptr(), mpfr_srcptr(), (sign < 0 ? 1 : 0), RoundingMode); + mpfr_setsign(mpfr_ptr(), mpfr_srcptr(), sign < 0, RoundingMode); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; } @@ -1955,14 +1961,14 @@ return *this; } -inline mpreal& mpreal::setInf(int sign) -{ +inline mpreal& mpreal::setInf(int sign) +{ mpfr_set_inf(mpfr_ptr(), sign); MPREAL_MSVC_DEBUGVIEW_CODE; return *this; -} +} -inline mpreal& mpreal::setNan() +inline mpreal& mpreal::setNan() { mpfr_set_nan(mpfr_ptr()); MPREAL_MSVC_DEBUGVIEW_CODE; @@ -1976,7 +1982,7 @@ #else mpfr_set_si(mpfr_ptr(), 0, (mpfr_get_default_rounding_mode)()); setSign(sign); -#endif +#endif MPREAL_MSVC_DEBUGVIEW_CODE; return *this; @@ -1993,7 +1999,7 @@ MPREAL_MSVC_DEBUGVIEW_CODE; } -inline mp_exp_t mpreal::get_exp () +inline mp_exp_t mpreal::get_exp () const { return mpfr_get_exp(mpfr_srcptr()); } @@ -2022,7 +2028,7 @@ mpreal x(v); // rounding is not important since we are just increasing the exponent (= exact operation) - mpfr_mul_2si(x.mpfr_ptr(), x.mpfr_srcptr(), exp, mpreal::get_default_rnd()); + mpfr_mul_2si(x.mpfr_ptr(), x.mpfr_srcptr(), exp, mpreal::get_default_rnd()); return x; } @@ -2038,7 +2044,7 @@ } inline mpreal machine_epsilon(const mpreal& x) -{ +{ /* the smallest eps such that x + eps != x */ if( x < 0) { @@ -2059,7 +2065,7 @@ inline mpreal maxval(mp_prec_t prec) { /* max = (1 - eps) * 2^emax, eps is machine epsilon */ - return (mpreal(1, prec) - machine_epsilon(prec)) << mpreal::get_emax(); + return (mpreal(1, prec) - machine_epsilon(prec)) << mpreal::get_emax(); } inline bool isEqualUlps(const mpreal& a, const mpreal& b, int maxUlps) @@ -2091,12 +2097,18 @@ return mpfr_signbit(x.mpfr_srcptr()); } +inline mpreal& setsignbit(mpreal& x, bool minus, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) +{ + mpfr_setsign(x.mpfr_ptr(), x.mpfr_srcptr(), minus, rnd_mode); + return x; +} + inline const mpreal modf(const mpreal& v, mpreal& n) { mpreal f(v); // rounding is not important since we are using the same number - mpfr_frac (f.mpfr_ptr(),f.mpfr_srcptr(),mpreal::get_default_rnd()); + mpfr_frac (f.mpfr_ptr(),f.mpfr_srcptr(),mpreal::get_default_rnd()); mpfr_trunc(n.mpfr_ptr(),v.mpfr_srcptr()); return f; } @@ -2159,7 +2171,7 @@ #define MPREAL_UNARY_MATH_FUNCTION_BODY(f) \ mpreal y(0, mpfr_get_prec(x.mpfr_srcptr())); \ mpfr_##f(y.mpfr_ptr(), x.mpfr_srcptr(), r); \ - return y; + return y; inline const mpreal sqr (const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) { MPREAL_UNARY_MATH_FUNCTION_BODY(sqr ); } @@ -2182,7 +2194,7 @@ inline const mpreal sqrt(const long int v, mp_rnd_t rnd_mode) { if (v>=0) return sqrt(static_cast<unsigned long int>(v),rnd_mode); - else return mpreal().setNan(); // NaN + else return mpreal().setNan(); // NaN } inline const mpreal sqrt(const int v, mp_rnd_t rnd_mode) @@ -2193,9 +2205,13 @@ inline const mpreal root(const mpreal& x, unsigned long int k, mp_rnd_t r = mpreal::get_default_rnd()) { - mpreal y(0, mpfr_get_prec(x.mpfr_srcptr())); + mpreal y(0, mpfr_get_prec(x.mpfr_srcptr())); + #if (MPFR_VERSION >= MPFR_VERSION_NUM(4,0,0)) + mpfr_rootn_ui(y.mpfr_ptr(), x.mpfr_srcptr(), k, r); + #else mpfr_root(y.mpfr_ptr(), x.mpfr_srcptr(), k, r); - return y; + #endif + return y; } inline const mpreal dim(const mpreal& a, const mpreal& b, mp_rnd_t r = mpreal::get_default_rnd()) @@ -2270,6 +2286,16 @@ inline const mpreal bessely0(const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) { MPREAL_UNARY_MATH_FUNCTION_BODY(y0 ); } inline const mpreal bessely1(const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) { MPREAL_UNARY_MATH_FUNCTION_BODY(y1 ); } +inline const mpreal nextpow2(const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) +{ + mpreal y(0, x.getPrecision()); + + if(!iszero(x)) + y = ceil(log2(abs(x,r),r)); + + return y; +} + inline const mpreal atan2 (const mpreal& y, const mpreal& x, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) { mpreal a(0,(std::max)(y.getPrecision(), x.getPrecision())); @@ -2284,8 +2310,46 @@ return a; } -inline const mpreal remainder (const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) +inline const mpreal hypot(const mpreal& a, const mpreal& b, const mpreal& c) { + if(isnan EIGEN_NOT_A_MACRO (a) || isnan EIGEN_NOT_A_MACRO (b) || isnan EIGEN_NOT_A_MACRO(c)) return mpreal().setNan(); + else + { + mpreal absa = abs(a), absb = abs(b), absc = abs(c); + mpreal w = (std::max)(absa, (std::max)(absb, absc)); + mpreal r; + + if (!iszero(w)) + { + mpreal iw = 1/w; + r = w * sqrt(sqr(absa*iw) + sqr(absb*iw) + sqr(absc*iw)); + } + + return r; + } +} + +inline const mpreal hypot(const mpreal& a, const mpreal& b, const mpreal& c, const mpreal& d) +{ + if(isnan EIGEN_NOT_A_MACRO (a) || isnan EIGEN_NOT_A_MACRO (b) || isnan EIGEN_NOT_A_MACRO (c) || isnan EIGEN_NOT_A_MACRO (d)) return mpreal().setNan(); + else + { + mpreal absa = abs(a), absb = abs(b), absc = abs(c), absd = abs(d); + mpreal w = (std::max)(absa, (std::max)(absb, (std::max)(absc, absd))); + mpreal r; + + if (!iszero(w)) + { + mpreal iw = 1/w; + r = w * sqrt(sqr(absa*iw) + sqr(absb*iw) + sqr(absc*iw) + sqr(absd*iw)); + } + + return r; + } +} + +inline const mpreal remainder (const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) +{ mpreal a(0,(std::max)(y.getPrecision(), x.getPrecision())); mpfr_remainder(a.mpfr_ptr(), x.mpfr_srcptr(), y.mpfr_srcptr(), rnd_mode); return a; @@ -2299,7 +2363,7 @@ } inline const mpreal fac_ui (unsigned long int v, mp_prec_t prec = mpreal::get_default_prec(), - mp_rnd_t rnd_mode = mpreal::get_default_rnd()) + mp_rnd_t rnd_mode = mpreal::get_default_rnd()) { mpreal x(0, prec); mpfr_fac_ui(x.mpfr_ptr(),v,rnd_mode); @@ -2338,9 +2402,9 @@ mpreal a; mp_prec_t p1, p2, p3; - p1 = v1.get_prec(); - p2 = v2.get_prec(); - p3 = v3.get_prec(); + p1 = v1.get_prec(); + p2 = v2.get_prec(); + p3 = v3.get_prec(); a.set_prec(p3>p2?(p3>p1?p3:p1):(p2>p1?p2:p1)); @@ -2353,9 +2417,9 @@ mpreal a; mp_prec_t p1, p2, p3; - p1 = v1.get_prec(); - p2 = v2.get_prec(); - p3 = v3.get_prec(); + p1 = v1.get_prec(); + p2 = v2.get_prec(); + p3 = v3.get_prec(); a.set_prec(p3>p2?(p3>p1?p3:p1):(p2>p1?p2:p1)); @@ -2368,8 +2432,8 @@ mpreal a; mp_prec_t p1, p2; - p1 = v1.get_prec(); - p2 = v2.get_prec(); + p1 = v1.get_prec(); + p2 = v2.get_prec(); a.set_prec(p1>p2?p1:p2); @@ -2387,7 +2451,7 @@ mpreal x; status = mpfr_sum(x.mpfr_ptr(), (mpfr_ptr*)p, n, mode); - + delete [] p; return x; } @@ -2401,9 +2465,9 @@ return mpfr_sinh_cosh(s.mp,c.mp,v.mp,rnd_mode); } -inline const mpreal li2 (const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) -{ - MPREAL_UNARY_MATH_FUNCTION_BODY(li2); +inline const mpreal li2 (const mpreal& x, mp_rnd_t r = mpreal::get_default_rnd()) +{ + MPREAL_UNARY_MATH_FUNCTION_BODY(li2); } inline const mpreal rem (const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) @@ -2415,16 +2479,16 @@ inline const mpreal mod (const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) { (void)rnd_mode; - - /* + + /* m = mod(x,y) if y != 0, returns x - n*y where n = floor(x/y) The following are true by convention: - mod(x,0) is x - mod(x,x) is 0 - - mod(x,y) for x != y and y != 0 has the same sign as y. - + - mod(x,y) for x != y and y != 0 has the same sign as y. + */ if(iszero(y)) return x; @@ -2432,9 +2496,7 @@ mpreal m = x - floor(x / y) * y; - m.setSign(sgn(y)); // make sure result has the same sign as Y - - return m; + return copysign(abs(m),y); // make sure result has the same sign as Y } inline const mpreal fmod (const mpreal& x, const mpreal& y, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) @@ -2442,8 +2504,8 @@ mpreal a; mp_prec_t yp, xp; - yp = y.get_prec(); - xp = x.get_prec(); + yp = y.get_prec(); + xp = x.get_prec(); a.set_prec(yp>xp?yp:xp); @@ -2543,7 +2605,16 @@ ////////////////////////////////////////////////////////////////////////// // Miscellaneous Functions -inline void swap (mpreal& a, mpreal& b) { mpfr_swap(a.mp,b.mp); } +inline int sgn(const mpreal& op) +{ + // Please note, this is classic signum function which ignores sign of zero. + // Use signbit if you need sign of zero. + return mpfr_sgn(op.mpfr_srcptr()); +} + +////////////////////////////////////////////////////////////////////////// +// Miscellaneous Functions +inline void swap (mpreal& a, mpreal& b) { mpfr_swap(a.mpfr_ptr(),b.mpfr_ptr()); } inline const mpreal (max)(const mpreal& x, const mpreal& y){ return (x>y?x:y); } inline const mpreal (min)(const mpreal& x, const mpreal& y){ return (x<y?x:y); } @@ -2634,14 +2705,23 @@ } -#if (MPFR_VERSION >= MPFR_VERSION_NUM(3,1,0)) +#if (MPFR_VERSION >= MPFR_VERSION_NUM(3,1,0) && MPFR_VERSION < MPFR_VERSION_NUM(4,0,0)) +// TODO: +// Use mpfr_nrandom since mpfr_grandom is deprecated +#if defined(_MSC_VER) +#pragma warning( push ) +#pragma warning( disable : 1478) +#endif inline const mpreal grandom (gmp_randstate_t& state, mp_rnd_t rnd_mode = mpreal::get_default_rnd()) { mpreal x; mpfr_grandom(x.mpfr_ptr(), NULL, state, rnd_mode); return x; } +#if defined(_MSC_VER) +#pragma warning( pop ) +#endif inline const mpreal grandom(unsigned int seed = 0) { @@ -2664,17 +2744,17 @@ ////////////////////////////////////////////////////////////////////////// // Set/Get global properties inline void mpreal::set_default_prec(mp_prec_t prec) -{ - mpfr_set_default_prec(prec); +{ + mpfr_set_default_prec(prec); } inline void mpreal::set_default_rnd(mp_rnd_t rnd_mode) -{ - mpfr_set_default_rounding_mode(rnd_mode); +{ + mpfr_set_default_rounding_mode(rnd_mode); } inline bool mpreal::fits_in_bits(double x, int n) -{ +{ int i; double t; return IsInf(x) || (std::modf ( std::ldexp ( std::frexp ( x, &i ), n ), &t ) == 0.0); @@ -2924,7 +3004,7 @@ else return pow(mpreal(a),mpreal(b),rnd_mode); //mpfr_pow } -// pow long double +// pow long double inline const mpreal pow(const long double a, const long double b, mp_rnd_t rnd_mode) { return pow(mpreal(a),mpreal(b),rnd_mode); @@ -2981,11 +3061,11 @@ // Non-throwing swap C++ idiom: http://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Non-throwing_swap namespace std { - // we are allowed to extend namespace std with specializations only + // we are allowed to extend namespace std with specializations only template <> - inline void swap(mpfr::mpreal& x, mpfr::mpreal& y) - { - return mpfr::swap(x, y); + inline void swap(mpfr::mpreal& x, mpfr::mpreal& y) + { + return mpfr::swap(x, y); } template<> @@ -2996,7 +3076,7 @@ static const bool is_signed = true; static const bool is_integer = false; static const bool is_exact = false; - static const int radix = 2; + static const int radix = 2; static const bool has_infinity = true; static const bool has_quiet_NaN = true; @@ -3016,7 +3096,7 @@ // Returns smallest eps such that 1 + eps != 1 (classic machine epsilon) inline static mpfr::mpreal epsilon(mp_prec_t precision = mpfr::mpreal::get_default_prec()) { return mpfr::machine_epsilon(precision); } - + // Returns smallest eps such that x + eps != x (relative machine epsilon) inline static mpfr::mpreal epsilon(const mpfr::mpreal& x) { return mpfr::machine_epsilon(x); } @@ -3024,8 +3104,8 @@ { mp_rnd_t r = mpfr::mpreal::get_default_rnd(); - if(r == GMP_RNDN) return mpfr::mpreal(0.5, precision); - else return mpfr::mpreal(1.0, precision); + if(r == GMP_RNDN) return mpfr::mpreal(0.5, precision); + else return mpfr::mpreal(1.0, precision); } inline static const mpfr::mpreal infinity() { return mpfr::const_infinity(); } @@ -3036,17 +3116,17 @@ // Please note, exponent range is not fixed in MPFR static const int min_exponent = MPFR_EMIN_DEFAULT; static const int max_exponent = MPFR_EMAX_DEFAULT; - MPREAL_PERMISSIVE_EXPR static const int min_exponent10 = (int) (MPFR_EMIN_DEFAULT * 0.3010299956639811); - MPREAL_PERMISSIVE_EXPR static const int max_exponent10 = (int) (MPFR_EMAX_DEFAULT * 0.3010299956639811); + MPREAL_PERMISSIVE_EXPR static const int min_exponent10 = (int) (MPFR_EMIN_DEFAULT * 0.3010299956639811); + MPREAL_PERMISSIVE_EXPR static const int max_exponent10 = (int) (MPFR_EMAX_DEFAULT * 0.3010299956639811); #ifdef MPREAL_HAVE_DYNAMIC_STD_NUMERIC_LIMITS // Following members should be constant according to standard, but they can be variable in MPFR - // So we define them as functions here. + // So we define them as functions here. // // This is preferable way for std::numeric_limits<mpfr::mpreal> specialization. - // But it is incompatible with standard std::numeric_limits and might not work with other libraries, e.g. boost. - // See below for compatible implementation. + // But it is incompatible with standard std::numeric_limits and might not work with other libraries, e.g. boost. + // See below for compatible implementation. inline static float_round_style round_style() { mp_rnd_t r = mpfr::mpreal::get_default_rnd(); @@ -3054,9 +3134,9 @@ switch (r) { case GMP_RNDN: return round_to_nearest; - case GMP_RNDZ: return round_toward_zero; - case GMP_RNDU: return round_toward_infinity; - case GMP_RNDD: return round_toward_neg_infinity; + case GMP_RNDZ: return round_toward_zero; + case GMP_RNDU: return round_toward_infinity; + case GMP_RNDD: return round_toward_neg_infinity; default: return round_indeterminate; } } @@ -3083,13 +3163,13 @@ // If possible, please use functions digits() and round_style() defined above. // // These (default) values are preserved for compatibility with existing libraries, e.g. boost. - // Change them accordingly to your application. + // Change them accordingly to your application. // // For example, if you use 256 bits of precision uniformly in your program, then: // digits = 256 - // digits10 = 77 + // digits10 = 77 // max_digits10 = 78 - // + // // Approximate formula for decimal digits is: digits10 = floor(log10(2) * digits). See bits2digits() for more details. static const std::float_round_style round_style = round_to_nearest;
diff --git a/unsupported/test/mpreal_support.cpp b/unsupported/test/mpreal_support.cpp index 1aa9e78..4a25e99 100644 --- a/unsupported/test/mpreal_support.cpp +++ b/unsupported/test/mpreal_support.cpp
@@ -7,16 +7,18 @@ using namespace mpfr; using namespace Eigen; -void test_mpreal_support() +EIGEN_DECLARE_TEST(mpreal_support) { // set precision to 256 bits (double has only 53 bits) mpreal::set_default_prec(256); typedef Matrix<mpreal,Eigen::Dynamic,Eigen::Dynamic> MatrixXmp; + typedef Matrix<std::complex<mpreal>,Eigen::Dynamic,Eigen::Dynamic> MatrixXcmp; std::cerr << "epsilon = " << NumTraits<mpreal>::epsilon() << "\n"; std::cerr << "dummy_precision = " << NumTraits<mpreal>::dummy_precision() << "\n"; std::cerr << "highest = " << NumTraits<mpreal>::highest() << "\n"; std::cerr << "lowest = " << NumTraits<mpreal>::lowest() << "\n"; + std::cerr << "digits10 = " << NumTraits<mpreal>::digits10() << "\n"; for(int i = 0; i < g_repeat; i++) { int s = Eigen::internal::random<int>(1,100); @@ -24,6 +26,10 @@ MatrixXmp B = MatrixXmp::Random(s,s); MatrixXmp S = A.adjoint() * A; MatrixXmp X; + MatrixXcmp Ac = MatrixXcmp::Random(s,s); + MatrixXcmp Bc = MatrixXcmp::Random(s,s); + MatrixXcmp Sc = Ac.adjoint() * Ac; + MatrixXcmp Xc; // Basic stuffs VERIFY_IS_APPROX(A.real(), A); @@ -36,6 +42,9 @@ // Cholesky X = S.selfadjointView<Lower>().llt().solve(B); VERIFY_IS_APPROX((S.selfadjointView<Lower>()*X).eval(),B); + + Xc = Sc.selfadjointView<Lower>().llt().solve(Bc); + VERIFY_IS_APPROX((Sc.selfadjointView<Lower>()*Xc).eval(),Bc); // partial LU X = A.lu().solve(B);
diff --git a/unsupported/test/openglsupport.cpp b/unsupported/test/openglsupport.cpp index 706a816..eadd7f9 100644 --- a/unsupported/test/openglsupport.cpp +++ b/unsupported/test/openglsupport.cpp
@@ -106,7 +106,7 @@ return prg_id; } -void test_openglsupport() +EIGEN_DECLARE_TEST(openglsupport) { int argc = 0; glutInit(&argc, 0); @@ -318,10 +318,6 @@ GLint prg_id = createShader(vtx,frg); - typedef Vector2d Vector2d; - typedef Vector3d Vector3d; - typedef Vector4d Vector4d; - VERIFY_UNIFORM(dv,v2d, Vector2d); VERIFY_UNIFORM(dv,v3d, Vector3d); VERIFY_UNIFORM(dv,v4d, Vector4d);
diff --git a/unsupported/test/polynomialsolver.cpp b/unsupported/test/polynomialsolver.cpp index 0c87478..4ff9bda 100644 --- a/unsupported/test/polynomialsolver.cpp +++ b/unsupported/test/polynomialsolver.cpp
@@ -26,15 +26,25 @@ } } +template<typename PolynomialType> +PolynomialType polyder(const PolynomialType& p) +{ + typedef typename PolynomialType::Scalar Scalar; + PolynomialType res(p.size()); + for(Index i=1; i<p.size(); ++i) + res[i-1] = p[i]*Scalar(i); + res[p.size()-1] = 0.; + return res; +} template<int Deg, typename POLYNOMIAL, typename SOLVER> bool aux_evalSolver( const POLYNOMIAL& pols, SOLVER& psolve ) { - typedef typename POLYNOMIAL::Index Index; typedef typename POLYNOMIAL::Scalar Scalar; + typedef typename POLYNOMIAL::RealScalar RealScalar; typedef typename SOLVER::RootsType RootsType; - typedef Matrix<Scalar,Deg,1> EvalRootsType; + typedef Matrix<RealScalar,Deg,1> EvalRootsType; const Index deg = pols.size()-1; @@ -44,10 +54,17 @@ psolve.compute( pols ); const RootsType& roots( psolve.roots() ); EvalRootsType evr( deg ); + POLYNOMIAL pols_der = polyder(pols); + EvalRootsType der( deg ); for( int i=0; i<roots.size(); ++i ){ - evr[i] = std::abs( poly_eval( pols, roots[i] ) ); } + evr[i] = std::abs( poly_eval( pols, roots[i] ) ); + der[i] = numext::maxi(RealScalar(1.), std::abs( poly_eval( pols_der, roots[i] ) )); + } - bool evalToZero = evr.isZero( test_precision<Scalar>() ); + // we need to divide by the magnitude of the derivative because + // with a high derivative is very small error in the value of the root + // yiels a very large error in the polynomial evaluation. + bool evalToZero = (evr.cwiseQuotient(der)).isZero( test_precision<Scalar>() ); if( !evalToZero ) { cerr << "WRONG root: " << endl; @@ -57,7 +74,7 @@ cerr << endl; } - std::vector<Scalar> rootModuli( roots.size() ); + std::vector<RealScalar> rootModuli( roots.size() ); Map< EvalRootsType > aux( &rootModuli[0], roots.size() ); aux = roots.array().abs(); std::sort( rootModuli.begin(), rootModuli.end() ); @@ -83,7 +100,7 @@ { typedef typename POLYNOMIAL::Scalar Scalar; - typedef PolynomialSolver<Scalar, Deg > PolynomialSolverType; + typedef PolynomialSolver<Scalar, Deg > PolynomialSolverType; PolynomialSolverType psolve; aux_evalSolver<Deg, POLYNOMIAL, PolynomialSolverType>( pols, psolve ); @@ -97,6 +114,7 @@ { using std::sqrt; typedef typename POLYNOMIAL::Scalar Scalar; + typedef typename POLYNOMIAL::RealScalar RealScalar; typedef PolynomialSolver<Scalar, Deg > PolynomialSolverType; @@ -107,15 +125,12 @@ // 1) the roots found are correct // 2) the roots have distinct moduli - typedef typename POLYNOMIAL::Scalar Scalar; - typedef typename REAL_ROOTS::Scalar Real; - //Test realRoots - std::vector< Real > calc_realRoots; - psolve.realRoots( calc_realRoots ); - VERIFY( calc_realRoots.size() == (size_t)real_roots.size() ); + std::vector< RealScalar > calc_realRoots; + psolve.realRoots( calc_realRoots, test_precision<RealScalar>()); + VERIFY_IS_EQUAL( calc_realRoots.size() , (size_t)real_roots.size() ); - const Scalar psPrec = sqrt( test_precision<Scalar>() ); + const RealScalar psPrec = sqrt( test_precision<RealScalar>() ); for( size_t i=0; i<calc_realRoots.size(); ++i ) { @@ -138,7 +153,7 @@ bool hasRealRoot; //Test absGreatestRealRoot - Real r = psolve.absGreatestRealRoot( hasRealRoot ); + RealScalar r = psolve.absGreatestRealRoot( hasRealRoot ); VERIFY( hasRealRoot == (real_roots.size() > 0 ) ); if( hasRealRoot ){ VERIFY( internal::isApprox( real_roots.array().abs().maxCoeff(), abs(r), psPrec ) ); } @@ -167,9 +182,11 @@ template<typename _Scalar, int _Deg> void polynomialsolver(int deg) { - typedef internal::increment_if_fixed_size<_Deg> Dim; + typedef typename NumTraits<_Scalar>::Real RealScalar; + typedef internal::increment_if_fixed_size<_Deg> Dim; typedef Matrix<_Scalar,Dim::ret,1> PolynomialType; typedef Matrix<_Scalar,_Deg,1> EvalRootsType; + typedef Matrix<RealScalar,_Deg,1> RealRootsType; cout << "Standard cases" << endl; PolynomialType pols = PolynomialType::Random(deg+1); @@ -182,19 +199,15 @@ evalSolver<_Deg,PolynomialType>( pols ); cout << "Test sugar" << endl; - EvalRootsType realRoots = EvalRootsType::Random(deg); + RealRootsType realRoots = RealRootsType::Random(deg); roots_to_monicPolynomial( realRoots, pols ); evalSolverSugarFunction<_Deg>( pols, - realRoots.template cast < - std::complex< - typename NumTraits<_Scalar>::Real - > - >(), + realRoots.template cast <std::complex<RealScalar> >().eval(), realRoots ); } -void test_polynomialsolver() +EIGEN_DECLARE_TEST(polynomialsolver) { for(int i = 0; i < g_repeat; i++) { @@ -214,5 +227,6 @@ internal::random<int>(9,13) )) ); CALL_SUBTEST_11((polynomialsolver<float,Dynamic>(1)) ); + CALL_SUBTEST_12((polynomialsolver<std::complex<double>,Dynamic>(internal::random<int>(2,13))) ); } }
diff --git a/unsupported/test/polynomialutils.cpp b/unsupported/test/polynomialutils.cpp index 5fc9684..8ff4519 100644 --- a/unsupported/test/polynomialutils.cpp +++ b/unsupported/test/polynomialutils.cpp
@@ -101,7 +101,7 @@ internal::random<int>(18,26) )) ); } -void test_polynomialutils() +EIGEN_DECLARE_TEST(polynomialutils) { for(int i = 0; i < g_repeat; i++) {
diff --git a/unsupported/test/sparse_extra.cpp b/unsupported/test/sparse_extra.cpp index a010ceb..b5d656f 100644 --- a/unsupported/test/sparse_extra.cpp +++ b/unsupported/test/sparse_extra.cpp
@@ -8,10 +8,26 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// import basic and product tests for deprectaed DynamicSparseMatrix +// import basic and product tests for deprecated DynamicSparseMatrix +#if 0 // sparse_basic(DynamicSparseMatrix) does not compile at all -> disabled +static long g_realloc_count = 0; +#define EIGEN_SPARSE_COMPRESSED_STORAGE_REALLOCATE_PLUGIN g_realloc_count++; + +static long g_dense_op_sparse_count = 0; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_DENSE_OP_SPARSE_PLUGIN g_dense_op_sparse_count++; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_ADD_DENSE_PLUGIN g_dense_op_sparse_count+=10; +#define EIGEN_SPARSE_ASSIGNMENT_FROM_SPARSE_SUB_DENSE_PLUGIN g_dense_op_sparse_count+=20; + +#define EIGEN_SPARSE_TEST_INCLUDED_FROM_SPARSE_EXTRA 1 +#endif + #define EIGEN_NO_DEPRECATED_WARNING -#include "sparse_basic.cpp" #include "sparse_product.cpp" + +#if 0 // sparse_basic(DynamicSparseMatrix) does not compile at all -> disabled +#include "sparse_basic.cpp" +#endif + #include <Eigen/SparseExtra> template<typename SetterType,typename DenseType, typename Scalar, int Options> @@ -129,7 +145,20 @@ } -void test_sparse_extra() +template<typename SparseMatrixType> +void check_marketio() +{ + typedef Matrix<typename SparseMatrixType::Scalar, Dynamic, Dynamic> DenseMatrix; + Index rows = internal::random<Index>(1,100); + Index cols = internal::random<Index>(1,100); + SparseMatrixType m1, m2; + m1 = DenseMatrix::Random(rows, cols).sparseView(); + saveMarket(m1, "sparse_extra.mtx"); + loadMarket(m2, "sparse_extra.mtx"); + VERIFY_IS_EQUAL(DenseMatrix(m1),DenseMatrix(m2)); +} + +EIGEN_DECLARE_TEST(sparse_extra) { for(int i = 0; i < g_repeat; i++) { int s = Eigen::internal::random<int>(1,50); @@ -143,5 +172,15 @@ CALL_SUBTEST_3( (sparse_product<DynamicSparseMatrix<float, ColMajor> >()) ); CALL_SUBTEST_3( (sparse_product<DynamicSparseMatrix<float, RowMajor> >()) ); + + CALL_SUBTEST_4( (check_marketio<SparseMatrix<float,ColMajor,int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<double,ColMajor,int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<std::complex<float>,ColMajor,int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<std::complex<double>,ColMajor,int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<float,ColMajor,long int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<double,ColMajor,long int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<std::complex<float>,ColMajor,long int> >()) ); + CALL_SUBTEST_4( (check_marketio<SparseMatrix<std::complex<double>,ColMajor,long int> >()) ); + TEST_SET_BUT_UNUSED_VARIABLE(s); } }
diff --git a/unsupported/test/special_functions.cpp b/unsupported/test/special_functions.cpp new file mode 100644 index 0000000..50dae6c --- /dev/null +++ b/unsupported/test/special_functions.cpp
@@ -0,0 +1,479 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include "main.h" +#include "../Eigen/SpecialFunctions" + +template<typename X, typename Y> +void verify_component_wise(const X& x, const Y& y) +{ + for(Index i=0; i<x.size(); ++i) + { + if((numext::isfinite)(y(i))) + VERIFY_IS_APPROX( x(i), y(i) ); + else if((numext::isnan)(y(i))) + VERIFY((numext::isnan)(x(i))); + else + VERIFY_IS_EQUAL( x(i), y(i) ); + } +} + +template<typename ArrayType> void array_special_functions() +{ + using std::abs; + using std::sqrt; + typedef typename ArrayType::Scalar Scalar; + typedef typename NumTraits<Scalar>::Real RealScalar; + + Scalar plusinf = std::numeric_limits<Scalar>::infinity(); + Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); + + Index rows = internal::random<Index>(1,30); + Index cols = 1; + + // API + { + ArrayType m1 = ArrayType::Random(rows,cols); +#if EIGEN_HAS_C99_MATH + VERIFY_IS_APPROX(m1.lgamma(), lgamma(m1)); + VERIFY_IS_APPROX(m1.digamma(), digamma(m1)); + VERIFY_IS_APPROX(m1.erf(), erf(m1)); + VERIFY_IS_APPROX(m1.erfc(), erfc(m1)); +#endif // EIGEN_HAS_C99_MATH + } + + +#if EIGEN_HAS_C99_MATH + // check special functions (comparing against numpy implementation) + if (!NumTraits<Scalar>::IsComplex) + { + + { + ArrayType m1 = ArrayType::Random(rows,cols); + ArrayType m2 = ArrayType::Random(rows,cols); + + // Test various propreties of igamma & igammac. These are normalized + // gamma integrals where + // igammac(a, x) = Gamma(a, x) / Gamma(a) + // igamma(a, x) = gamma(a, x) / Gamma(a) + // where Gamma and gamma are considered the standard unnormalized + // upper and lower incomplete gamma functions, respectively. + ArrayType a = m1.abs() + 2; + ArrayType x = m2.abs() + 2; + ArrayType zero = ArrayType::Zero(rows, cols); + ArrayType one = ArrayType::Constant(rows, cols, Scalar(1.0)); + ArrayType a_m1 = a - one; + ArrayType Gamma_a_x = Eigen::igammac(a, x) * a.lgamma().exp(); + ArrayType Gamma_a_m1_x = Eigen::igammac(a_m1, x) * a_m1.lgamma().exp(); + ArrayType gamma_a_x = Eigen::igamma(a, x) * a.lgamma().exp(); + ArrayType gamma_a_m1_x = Eigen::igamma(a_m1, x) * a_m1.lgamma().exp(); + + // Gamma(a, 0) == Gamma(a) + VERIFY_IS_APPROX(Eigen::igammac(a, zero), one); + + // Gamma(a, x) + gamma(a, x) == Gamma(a) + VERIFY_IS_APPROX(Gamma_a_x + gamma_a_x, a.lgamma().exp()); + + // Gamma(a, x) == (a - 1) * Gamma(a-1, x) + x^(a-1) * exp(-x) + VERIFY_IS_APPROX(Gamma_a_x, (a - 1) * Gamma_a_m1_x + x.pow(a-1) * (-x).exp()); + + // gamma(a, x) == (a - 1) * gamma(a-1, x) - x^(a-1) * exp(-x) + VERIFY_IS_APPROX(gamma_a_x, (a - 1) * gamma_a_m1_x - x.pow(a-1) * (-x).exp()); + } + + { + // Check exact values of igamma and igammac against a third party calculation. + Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)}; + + // location i*6+j corresponds to a_s[i], x_s[j]. + Scalar igamma_s[][6] = {{0.0, nan, nan, nan, nan, nan}, + {0.0, 0.6321205588285578, 0.7768698398515702, + 0.9816843611112658, 9.999500016666262e-05, 1.0}, + {0.0, 0.4275932955291202, 0.608374823728911, + 0.9539882943107686, 7.522076445089201e-07, 1.0}, + {0.0, 0.01898815687615381, 0.06564245437845008, + 0.5665298796332909, 4.166333347221828e-18, 1.0}, + {0.0, 0.9999780593618628, 0.9999899967080838, + 0.9999996219837988, 0.9991370418689945, 1.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.5042041932513908}}; + Scalar igammac_s[][6] = {{nan, nan, nan, nan, nan, nan}, + {1.0, 0.36787944117144233, 0.22313016014842982, + 0.018315638888734182, 0.9999000049998333, 0.0}, + {1.0, 0.5724067044708798, 0.3916251762710878, + 0.04601170568923136, 0.9999992477923555, 0.0}, + {1.0, 0.9810118431238462, 0.9343575456215499, + 0.4334701203667089, 1.0, 0.0}, + {1.0, 2.1940638138146658e-05, 1.0003291916285e-05, + 3.7801620118431334e-07, 0.0008629581310054535, + 0.0}, + {1.0, 1.0, 1.0, 1.0, 1.0, 0.49579580674813944}}; + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 6; ++j) { + if ((std::isnan)(igamma_s[i][j])) { + VERIFY((std::isnan)(numext::igamma(a_s[i], x_s[j]))); + } else { + VERIFY_IS_APPROX(numext::igamma(a_s[i], x_s[j]), igamma_s[i][j]); + } + + if ((std::isnan)(igammac_s[i][j])) { + VERIFY((std::isnan)(numext::igammac(a_s[i], x_s[j]))); + } else { + VERIFY_IS_APPROX(numext::igammac(a_s[i], x_s[j]), igammac_s[i][j]); + } + } + } + } + } +#endif // EIGEN_HAS_C99_MATH + + // Check the zeta function against scipy.special.zeta + { + ArrayType x(7), q(7), res(7), ref(7); + x << 1.5, 4, 10.5, 10000.5, 3, 1, 0.9; + q << 2, 1.5, 3, 1.0001, -2.5, 1.2345, 1.2345; + ref << 1.61237534869, 0.234848505667, 1.03086757337e-5, 0.367879440865, 0.054102025820864097, plusinf, nan; + CALL_SUBTEST( verify_component_wise(ref, ref); ); + CALL_SUBTEST( res = x.zeta(q); verify_component_wise(res, ref); ); + CALL_SUBTEST( res = zeta(x,q); verify_component_wise(res, ref); ); + } + + // digamma + { + ArrayType x(7), res(7), ref(7); + x << 1, 1.5, 4, -10.5, 10000.5, 0, -1; + ref << -0.5772156649015329, 0.03648997397857645, 1.2561176684318, 2.398239129535781, 9.210340372392849, plusinf, plusinf; + CALL_SUBTEST( verify_component_wise(ref, ref); ); + + CALL_SUBTEST( res = x.digamma(); verify_component_wise(res, ref); ); + CALL_SUBTEST( res = digamma(x); verify_component_wise(res, ref); ); + } + + +#if EIGEN_HAS_C99_MATH + { + ArrayType n(11), x(11), res(11), ref(11); + n << 1, 1, 1, 1.5, 17, 31, 28, 8, 42, 147, 170; + x << 2, 3, 25.5, 1.5, 4.7, 11.8, 17.7, 30.2, 15.8, 54.1, 64; + ref << 0.644934066848, 0.394934066848, 0.0399946696496, nan, 293.334565435, 0.445487887616, -2.47810300902e-07, -8.29668781082e-09, -0.434562276666, 0.567742190178, -0.0108615497927; + CALL_SUBTEST( verify_component_wise(ref, ref); ); + + if(sizeof(RealScalar)>=8) { // double + // Reason for commented line: http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1232 + // CALL_SUBTEST( res = x.polygamma(n); verify_component_wise(res, ref); ); + CALL_SUBTEST( res = polygamma(n,x); verify_component_wise(res, ref); ); + } + else { + // CALL_SUBTEST( res = x.polygamma(n); verify_component_wise(res.head(8), ref.head(8)); ); + CALL_SUBTEST( res = polygamma(n,x); verify_component_wise(res.head(8), ref.head(8)); ); + } + } +#endif + +#if EIGEN_HAS_C99_MATH + { + // Inputs and ground truth generated with scipy via: + // a = np.logspace(-3, 3, 5) - 1e-3 + // b = np.logspace(-3, 3, 5) - 1e-3 + // x = np.linspace(-0.1, 1.1, 5) + // (full_a, full_b, full_x) = np.vectorize(lambda a, b, x: (a, b, x))(*np.ix_(a, b, x)) + // full_a = full_a.flatten().tolist() # same for full_b, full_x + // v = scipy.special.betainc(full_a, full_b, full_x).flatten().tolist() + // + // Note in Eigen, we call betainc with arguments in the order (x, a, b). + ArrayType a(125); + ArrayType b(125); + ArrayType x(125); + ArrayType v(125); + ArrayType res(125); + + a << 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, + 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, + 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, + 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, + 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, + 999.999, 999.999, 999.999; + + b << 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999, + 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, + 999.999, 999.999, 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, + 0.999, 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 999.999, 999.999, 999.999, + 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 999.999, 999.999, 999.999, + 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 999.999, 999.999, 999.999, + 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, + 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, + 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, + 31.62177660168379, 31.62177660168379, 31.62177660168379, + 31.62177660168379, 31.62177660168379, 999.999, 999.999, 999.999, + 999.999, 999.999; + + x << -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, + 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, + 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, + 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, + -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, + 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, + 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, + 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, + 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, + 0.8, 1.1; + + v << nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, + nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, + nan, nan, nan, 0.47972119876364683, 0.5, 0.5202788012363533, nan, nan, + 0.9518683957740043, 0.9789663010413743, 0.9931729188073435, nan, nan, + 0.999995949033062, 0.9999999999993698, 0.9999999999999999, nan, nan, + 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, nan, nan, + nan, nan, nan, nan, nan, 0.006827081192655869, 0.0210336989586256, + 0.04813160422599567, nan, nan, 0.20014344256217678, 0.5000000000000001, + 0.7998565574378232, nan, nan, 0.9991401428435834, 0.999999999698403, + 0.9999999999999999, nan, nan, 0.9999999999999999, 0.9999999999999999, + 0.9999999999999999, nan, nan, nan, nan, nan, nan, nan, + 1.0646600232370887e-25, 6.301722877826246e-13, 4.050966937974938e-06, + nan, nan, 7.864342668429763e-23, 3.015969667594166e-10, + 0.0008598571564165444, nan, nan, 6.031987710123844e-08, + 0.5000000000000007, 0.9999999396801229, nan, nan, 0.9999999999999999, + 0.9999999999999999, 0.9999999999999999, nan, nan, nan, nan, nan, nan, + nan, 0.0, 7.029920380986636e-306, 2.2450728208591345e-101, nan, nan, + 0.0, 9.275871147869727e-302, 1.2232913026152827e-97, nan, nan, 0.0, + 3.0891393081932924e-252, 2.9303043666183996e-60, nan, nan, + 2.248913486879199e-196, 0.5000000000004947, 0.9999999999999999, nan; + + CALL_SUBTEST(res = betainc(a, b, x); + verify_component_wise(res, v);); + } + + // Test various properties of betainc + { + ArrayType m1 = ArrayType::Random(32); + ArrayType m2 = ArrayType::Random(32); + ArrayType m3 = ArrayType::Random(32); + ArrayType one = ArrayType::Constant(32, Scalar(1.0)); + const Scalar eps = std::numeric_limits<Scalar>::epsilon(); + ArrayType a = (m1 * 4.0).exp(); + ArrayType b = (m2 * 4.0).exp(); + ArrayType x = m3.abs(); + + // betainc(a, 1, x) == x**a + CALL_SUBTEST( + ArrayType test = betainc(a, one, x); + ArrayType expected = x.pow(a); + verify_component_wise(test, expected);); + + // betainc(1, b, x) == 1 - (1 - x)**b + CALL_SUBTEST( + ArrayType test = betainc(one, b, x); + ArrayType expected = one - (one - x).pow(b); + verify_component_wise(test, expected);); + + // betainc(a, b, x) == 1 - betainc(b, a, 1-x) + CALL_SUBTEST( + ArrayType test = betainc(a, b, x) + betainc(b, a, one - x); + ArrayType expected = one; + verify_component_wise(test, expected);); + + // betainc(a+1, b, x) = betainc(a, b, x) - x**a * (1 - x)**b / (a * beta(a, b)) + CALL_SUBTEST( + ArrayType num = x.pow(a) * (one - x).pow(b); + ArrayType denom = a * (a.lgamma() + b.lgamma() - (a + b).lgamma()).exp(); + // Add eps to rhs and lhs so that component-wise test doesn't result in + // nans when both outputs are zeros. + ArrayType expected = betainc(a, b, x) - num / denom + eps; + ArrayType test = betainc(a + one, b, x) + eps; + if (sizeof(Scalar) >= 8) { // double + verify_component_wise(test, expected); + } else { + // Reason for limited test: http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1232 + verify_component_wise(test.head(8), expected.head(8)); + }); + + // betainc(a, b+1, x) = betainc(a, b, x) + x**a * (1 - x)**b / (b * beta(a, b)) + CALL_SUBTEST( + // Add eps to rhs and lhs so that component-wise test doesn't result in + // nans when both outputs are zeros. + ArrayType num = x.pow(a) * (one - x).pow(b); + ArrayType denom = b * (a.lgamma() + b.lgamma() - (a + b).lgamma()).exp(); + ArrayType expected = betainc(a, b, x) + num / denom + eps; + ArrayType test = betainc(a, b + one, x) + eps; + verify_component_wise(test, expected);); + } +#endif // EIGEN_HAS_C99_MATH + + // Test Bessel function i0e. Reference results obtained with SciPy. + { + ArrayType x(21); + ArrayType expected(21); + ArrayType res(21); + + x << -20.0, -18.0, -16.0, -14.0, -12.0, -10.0, -8.0, -6.0, -4.0, -2.0, 0.0, + 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 20.0; + + expected << 0.0897803118848, 0.0947062952128, 0.100544127361, + 0.107615251671, 0.116426221213, 0.127833337163, 0.143431781857, + 0.16665743264, 0.207001921224, 0.308508322554, 1.0, 0.308508322554, + 0.207001921224, 0.16665743264, 0.143431781857, 0.127833337163, + 0.116426221213, 0.107615251671, 0.100544127361, 0.0947062952128, + 0.0897803118848; + + CALL_SUBTEST(res = i0e(x); + verify_component_wise(res, expected);); + } + + // Test Bessel function i1e. Reference results obtained with SciPy. + { + ArrayType x(21); + ArrayType expected(21); + ArrayType res(21); + + x << -20.0, -18.0, -16.0, -14.0, -12.0, -10.0, -8.0, -6.0, -4.0, -2.0, 0.0, + 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 20.0; + + expected << -0.0875062221833, -0.092036796872, -0.0973496147565, + -0.103697667463, -0.11146429929, -0.121262681384, -0.134142493293, + -0.152051459309, -0.178750839502, -0.215269289249, 0.0, 0.215269289249, + 0.178750839502, 0.152051459309, 0.134142493293, 0.121262681384, + 0.11146429929, 0.103697667463, 0.0973496147565, 0.092036796872, + 0.0875062221833; + + CALL_SUBTEST(res = i1e(x); + verify_component_wise(res, expected);); + } + + /* Code to generate the data for the following two test cases. + N = 5 + np.random.seed(3) + + a = np.logspace(-2, 3, 6) + a = np.ravel(np.tile(np.reshape(a, [-1, 1]), [1, N])) + x = np.random.gamma(a, 1.0) + x = np.maximum(x, np.finfo(np.float32).tiny) + + def igamma(a, x): + return mpmath.gammainc(a, 0, x, regularized=True) + + def igamma_der_a(a, x): + res = mpmath.diff(lambda a_prime: igamma(a_prime, x), a) + return np.float64(res) + + def gamma_sample_der_alpha(a, x): + igamma_x = igamma(a, x) + def igammainv_of_igamma(a_prime): + return mpmath.findroot(lambda x_prime: igamma(a_prime, x_prime) - + igamma_x, x, solver='newton') + return np.float64(mpmath.diff(igammainv_of_igamma, a)) + + v_igamma_der_a = np.vectorize(igamma_der_a)(a, x) + v_gamma_sample_der_alpha = np.vectorize(gamma_sample_der_alpha)(a, x) + */ + +#if EIGEN_HAS_C99_MATH + // Test igamma_der_a + { + ArrayType a(30); + ArrayType x(30); + ArrayType res(30); + ArrayType v(30); + + a << 0.01, 0.01, 0.01, 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, 1.0, + 1.0, 1.0, 10.0, 10.0, 10.0, 10.0, 10.0, 100.0, 100.0, 100.0, 100.0, + 100.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0; + + x << 1.25668890405e-26, 1.17549435082e-38, 1.20938905072e-05, + 1.17549435082e-38, 1.17549435082e-38, 5.66572070696e-16, + 0.0132865061065, 0.0200034203853, 6.29263709118e-17, 1.37160367764e-06, + 0.333412038288, 1.18135687766, 0.580629033777, 0.170631439426, + 0.786686768458, 7.63873279537, 13.1944344379, 11.896042354, + 10.5830172417, 10.5020942233, 92.8918587747, 95.003720371, + 86.3715926467, 96.0330217672, 82.6389930677, 968.702906754, + 969.463546828, 1001.79726022, 955.047416547, 1044.27458568; + + v << -32.7256441441, -36.4394150514, -9.66467612263, -36.4394150514, + -36.4394150514, -1.0891900302, -2.66351229645, -2.48666868596, + -0.929700494428, -3.56327722764, -0.455320135314, -0.391437214323, + -0.491352055991, -0.350454834292, -0.471773162921, -0.104084440522, + -0.0723646747909, -0.0992828975532, -0.121638215446, -0.122619605294, + -0.0317670267286, -0.0359974812869, -0.0154359225363, -0.0375775365921, + -0.00794899153653, -0.00777303219211, -0.00796085782042, + -0.0125850719397, -0.00455500206958, -0.00476436993148; + + CALL_SUBTEST(res = igamma_der_a(a, x); verify_component_wise(res, v);); + } + + // Test gamma_sample_der_alpha + { + ArrayType alpha(30); + ArrayType sample(30); + ArrayType res(30); + ArrayType v(30); + + alpha << 0.01, 0.01, 0.01, 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, + 1.0, 1.0, 1.0, 10.0, 10.0, 10.0, 10.0, 10.0, 100.0, 100.0, 100.0, 100.0, + 100.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0; + + sample << 1.25668890405e-26, 1.17549435082e-38, 1.20938905072e-05, + 1.17549435082e-38, 1.17549435082e-38, 5.66572070696e-16, + 0.0132865061065, 0.0200034203853, 6.29263709118e-17, 1.37160367764e-06, + 0.333412038288, 1.18135687766, 0.580629033777, 0.170631439426, + 0.786686768458, 7.63873279537, 13.1944344379, 11.896042354, + 10.5830172417, 10.5020942233, 92.8918587747, 95.003720371, + 86.3715926467, 96.0330217672, 82.6389930677, 968.702906754, + 969.463546828, 1001.79726022, 955.047416547, 1044.27458568; + + v << 7.42424742367e-23, 1.02004297287e-34, 0.0130155240738, + 1.02004297287e-34, 1.02004297287e-34, 1.96505168277e-13, 0.525575786243, + 0.713903991771, 2.32077561808e-14, 0.000179348049886, 0.635500453302, + 1.27561284917, 0.878125852156, 0.41565819538, 1.03606488534, + 0.885964824887, 1.16424049334, 1.10764479598, 1.04590810812, + 1.04193666963, 0.965193152414, 0.976217589464, 0.93008035061, + 0.98153216096, 0.909196397698, 0.98434963993, 0.984738050206, + 1.00106492525, 0.97734200649, 1.02198794179; + + CALL_SUBTEST(res = gamma_sample_der_alpha(alpha, sample); + verify_component_wise(res, v);); + } +#endif // EIGEN_HAS_C99_MATH +} + +EIGEN_DECLARE_TEST(special_functions) +{ + CALL_SUBTEST_1(array_special_functions<ArrayXf>()); + CALL_SUBTEST_2(array_special_functions<ArrayXd>()); +}
diff --git a/unsupported/test/splines.cpp b/unsupported/test/splines.cpp index 3be0204..88ec87b 100644 --- a/unsupported/test/splines.cpp +++ b/unsupported/test/splines.cpp
@@ -268,7 +268,7 @@ } } -void test_splines() +EIGEN_DECLARE_TEST(splines) { for (int i = 0; i < g_repeat; ++i) {
