diff --git a/CMakeLists.txt b/CMakeLists.txt index 9faf51a..3bcbe28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,11 +7,12 @@ project (graph_framework CXX) #------------------------------------------------------------------------------- option (USE_PCH "Enable the use of precompiled headers" ON) option (SAVE_KERNEL_SOURCE "Writes the kernel source code to a file." OFF) -option (USE_INPUT_CACHE "Cache the values kernel input values." OFF) -option (USE_CONSTANT_CACHE "Cache the value of constants in kernel registers." OFF) +option (USE_INPUT_CACHE "Cache the values kernel input values." ON) +option (USE_CONSTANT_CACHE "Cache the value of constants in kernel registers." ON) option (SHOW_USE_COUNT "Add a comment showing the use count in kernel sources." OFF) -option (USE_INDEX_CACHE "Cache index values instead of computing them every time." OFF) +option (USE_INDEX_CACHE "Cache index values instead of computing them every time." ON) option (USE_VERBOSE "Verbose jit option." OFF) +option (PROFILE_KERNELS "Display kernel timing information" OFF) option (BUILD_C_BINDING "Build C interface." OFF) option (BUILD_Fortran_BINDING "Build Fortran interface." OFF) @@ -207,6 +208,7 @@ FetchContent_GetProperties ( ) # Do not build llvm until pull command is finished. +add_dependencies (CASPluginTest_exports pull_llvm) add_dependencies (gpu-resource-headers pull_llvm) add_dependencies (llvm-offload-resource-headers pull_llvm) add_dependencies (LLVMDemangle pull_llvm) @@ -374,8 +376,12 @@ macro (add_tool_target target lang) graph_framework ) - if (${USE_PCH} AND ${BUILD_C_BINDING}) - target_precompile_headers (${target} REUSE_FROM graph_c) + if (${USE_PCH}) + if (${BUILD_C_BINDING}) + target_precompile_headers (${target} REUSE_FROM graph_c) + elseif (NOT ${target} MATCHES xrays) + target_precompile_headers (${target} REUSE_FROM xrays) + endif () endif () endmacro () @@ -404,4 +410,18 @@ macro (add_test_target target lang) endif () endmacro () +macro (add_compile_test target lang) + cmake_path (GET CMAKE_CXX_COMPILER FILENAME compiler_command) + add_test (NAME ${target}_off + COMMAND ${compiler_command} -std=c++23 -DUSE_VERBOSE=false -DCHECK_TEST -c ${CMAKE_CURRENT_SOURCE_DIR}/${target}.${lang} + ) + add_test (NAME ${target}_on + COMMAND ${compiler_command} -std=c++23 -DUSE_VERBOSE=false -c ${CMAKE_CURRENT_SOURCE_DIR}/${target}.${lang} + ) + set_tests_properties (${target}_on + PROPERTIES + WILL_FAIL true + ) +endmacro () + add_subdirectory (graph_tests) diff --git a/graph_c_binding/graph_c_binding.cpp b/graph_c_binding/graph_c_binding.cpp index 1076d09..f1c508d 100644 --- a/graph_c_binding/graph_c_binding.cpp +++ b/graph_c_binding/graph_c_binding.cpp @@ -19,7 +19,7 @@ template struct graph_c_context_type : public graph_c_context { /// Variables nodes. - std::map> nodes; + std::unordered_map> nodes; /// Workflow manager. workflow::manager work; @@ -1259,22 +1259,24 @@ extern "C" { /// @brief Construct a random state node. /// /// @param[in] c The graph C context. +/// @param[in] size The number of randoms needed. /// @param[in] seed Intial random seed. /// @returns A random state node. //------------------------------------------------------------------------------ graph_node graph_random_state(STRUCT_TAG graph_c_context *c, + const size_t size, const uint32_t seed) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); @@ -1283,13 +1285,13 @@ extern "C" { case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); @@ -1298,13 +1300,13 @@ extern "C" { case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - auto temp = graph::random_state, true> (jit::context, true>::random_state_size, + auto temp = graph::random_state, true> (jit::context, true>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast> *> (c); - auto temp = graph::random_state> (jit::context>::random_state_size, seed); + auto temp = graph::random_state> (jit::context>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } @@ -1312,13 +1314,13 @@ extern "C" { case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - auto temp = graph::random_state, true> (jit::context, true>::random_state_size, + auto temp = graph::random_state, true> (jit::context, true>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast> *> (c); - auto temp = graph::random_state> (jit::context>::random_state_size, seed); + auto temp = graph::random_state> (jit::context>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } @@ -1886,103 +1888,4213 @@ extern "C" { } } -//****************************************************************************** -// JIT -//****************************************************************************** //------------------------------------------------------------------------------ -/// @brief Create 2D piecewise node with complex arguments. +/// @brief Create an atomic accumulate 1D index. +/// +/// @param[in] c The graph C context. +/// @param[in] variable The variable to index. +/// @param[in] index The function argument. +/// @param[in] scale Scale factor argument. +/// @param[in] offset Offset factor argument. +/// @param[in] arg Argument. +/// @returns An atomic accumulate 1D node. +//---------------------------------------------- +//------------------------------------------------------------------------------ + graph_node graph_atomic_accumulate_1D(STRUCT_TAG graph_c_context *c, + graph_node variable, + graph_node index, + const double scale, + const double offset, + graph_node arg) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an index code. /// /// @param[in] c The graph C context. -/// @returns The number of concurrent devices. +/// @returns An index node. //------------------------------------------------------------------------------ - size_t graph_get_max_concurrency(graph_c_context *c) { + graph_node graph_index(STRUCT_TAG graph_c_context *c) { switch (c->type) { case FLOAT: if (c->safe_math) { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } case DOUBLE: if (c->safe_math) { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } case COMPLEX_FLOAT: if (c->safe_math) { - return jit::context, true>::max_concurrency(); + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::index, true> (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context>::max_concurrency(); + auto d = reinterpret_cast> *> (c); + auto temp = graph::index> (); + d->nodes[temp.get()] = temp; + return temp.get(); } case COMPLEX_DOUBLE: if (c->safe_math) { - return jit::context, true>::max_concurrency(); + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::index, true> (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context>::max_concurrency(); + auto d = reinterpret_cast> *> (c); + auto temp = graph::index> (); + d->nodes[temp.get()] = temp; + return temp.get(); } } } -//****************************************************************************** -// Workflows -//****************************************************************************** //------------------------------------------------------------------------------ -/// @brief Choose the device number. +/// @brief Create not node. /// /// @param[in] c The graph C context. -/// @param[in] num The device number. +/// @param[in] arg The function argument. +/// @returns !arg //------------------------------------------------------------------------------ - void graph_set_device_number(STRUCT_TAG graph_c_context *c, - const size_t num) { + graph_node graph_not(STRUCT_TAG graph_c_context *c, + graph_node arg) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left == right +//------------------------------------------------------------------------------ + graph_node graph_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work = workflow::manager, true> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast> *> (c); - d->work = workflow::manager> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work = workflow::manager, true> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast> *> (c); - d->work = workflow::manager> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a not equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left != right +//------------------------------------------------------------------------------ + graph_node graph_not_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a greater than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left > right +//------------------------------------------------------------------------------ + graph_node graph_greater_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left < right +//------------------------------------------------------------------------------ + graph_node graph_less_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a greater than equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left >= right +//------------------------------------------------------------------------------ + graph_node graph_greater_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left <= right +//------------------------------------------------------------------------------ + graph_node graph_less_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an and node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left && right +//------------------------------------------------------------------------------ + graph_node graph_and(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an or node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left || right +//------------------------------------------------------------------------------ + graph_node graph_or(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a if node. +/// +/// @param[in] c The graph C context. +/// @param[in] condition The logical condition. +/// @param[in] t The true case. +/// @param[in] f The false case. +/// @returns condiiton ? t : f +//------------------------------------------------------------------------------ + graph_node graph_if(STRUCT_TAG graph_c_context *c, + graph_node condition, + graph_node t, + graph_node f) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//****************************************************************************** +// JIT +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Create 2D piecewise node with complex arguments. +/// +/// @param[in] c The graph C context. +/// @returns The number of concurrent devices. +//------------------------------------------------------------------------------ + size_t graph_get_max_concurrency(graph_c_context *c) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + return jit::context::max_concurrency(); + } else { + return jit::context::max_concurrency(); + } + + case DOUBLE: + if (c->safe_math) { + return jit::context::max_concurrency(); + } else { + return jit::context::max_concurrency(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + return jit::context, true>::max_concurrency(); + } else { + return jit::context>::max_concurrency(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + return jit::context, true>::max_concurrency(); + } else { + return jit::context>::max_concurrency(); + } + } + } + +//****************************************************************************** +// Workflows +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Choose the device number. +/// +/// @param[in] c The graph C context. +/// @param[in] num The device number. +//------------------------------------------------------------------------------ + void graph_set_device_number(STRUCT_TAG graph_c_context *c, + const size_t num) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } else { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } else { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work = workflow::manager, true> (num); + } else { + auto d = reinterpret_cast> *> (c); + d->work = workflow::manager> (num); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work = workflow::manager, true> (num); + } else { + auto d = reinterpret_cast> *> (c); + d->work = workflow::manager> (num); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add pre workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_pre_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add pre workflow loop item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_pre_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add workflow loopitem. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add post workflow loopitem. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_post_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a pre converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_pre_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } } break; } } //------------------------------------------------------------------------------ -/// @brief Add pre workflow item. +/// @brief Add a post converge item. /// /// @param[in] c The graph C context. /// @param[in] inputs Array of input nodes. @@ -1992,18 +6104,25 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ - void graph_add_pre_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, - graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size) { + void graph_add_post_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { switch (c->type) { case FLOAT: if (c->safe_math) { @@ -2014,7 +6133,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2028,20 +6147,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } else { auto d = reinterpret_cast *> (c); @@ -2051,7 +6182,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2065,20 +6196,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } break; @@ -2092,7 +6235,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2106,20 +6249,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } else { auto d = reinterpret_cast *> (c); @@ -2129,7 +6284,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2143,20 +6298,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); } } break; @@ -2170,7 +6337,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2184,24 +6351,249 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + atom.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } - } else { - d->work.add_preitem(in, out, map, NULL, name, size); + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a pre zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_pre_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_zero_item (in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } } + + d->work.add_zero_item (in); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2211,31 +6603,40 @@ extern "C" { exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item (in); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_preitem(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2252,31 +6653,8 @@ extern "C" { exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_preitem(in, out, map, NULL, name, size); - } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2289,59 +6667,22 @@ extern "C" { exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_preitem(in, out, map, NULL, name, size); - } + + d->work.add_zero_item (in); } break; } } //------------------------------------------------------------------------------ -/// @brief Add workflow item. +/// @brief Add a copy item. /// -/// @param[in] c The graph C context. -/// @param[in] inputs Array of input nodes. -/// @param[in] num_inputs Number of inputs. -/// @param[in] outputs Array of output nodes. -/// @param[in] num_outputs Number of outputs. -/// @param[in] map_inputs Array of map input nodes. -/// @param[in] map_outputs Array of map output nodes. -/// @param[in] num_maps Number of maps. -/// @param[in] random_state Optional random state, can be NULL if not used. -/// @param[in] name Name for the kernel. -/// @param[in] size Number of elements to operate on. +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. //------------------------------------------------------------------------------ - void graph_add_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, - graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size) { + void graph_add_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { switch (c->type) { case FLOAT: if (c->safe_math) { @@ -2356,34 +6697,43 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item(in); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2393,38 +6743,47 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item(in); } break; - case DOUBLE: + case COMPLEX_DOUBLE: if (c->safe_math) { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2434,68 +6793,98 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a post zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_post_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item (in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2508,35 +6897,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2545,35 +6911,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2586,35 +6929,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2623,393 +6943,462 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; } } //------------------------------------------------------------------------------ -/// @brief Add a converge item. +/// @brief Add a pre copy item. /// -/// @param[in] c The graph C context. -/// @param[in] inputs Array of input nodes. -/// @param[in] num_inputs Number of inputs. -/// @param[in] outputs Array of output nodes. -/// @param[in] num_outputs Number of outputs. -/// @param[in] map_inputs Array of map input nodes. -/// @param[in] map_outputs Array of map output nodes. -/// @param[in] num_maps Number of maps. -/// @param[in] random_state Optional random state, can be NULL if not used. -/// @param[in] name Name for the kernel. -/// @param[in] size Number of elements to operate on. -/// @param[in] tol Tolerance to converge the function to. -/// @param[in] max_iter Maximum number of iterations before giving up. +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. //------------------------------------------------------------------------------ - void graph_add_converge_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, + void graph_add_pre_copy_item(STRUCT_TAG graph_c_context *c, graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size, - const double tol, - const size_t max_iter) { + graph_node *map_outputs, size_t num_maps) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } } - graph::map_nodes map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; + } + } - case DOUBLE: +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps) { + switch (c->type) { + case FLOAT: if (c->safe_math) { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + + d->work.add_copy_item(map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item(map); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item(map); } else { auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); - } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); - } + + d->work.add_copy_item(map); } break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - graph::input_nodes, true> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; + + d->work.add_copy_item(map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item(map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item(map); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; + + d->work.add_copy_item(map); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a post copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_post_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; - case COMPLEX_DOUBLE: + case DOUBLE: if (c->safe_math) { - auto d = reinterpret_cast, true> *> (c); - graph::input_nodes, true> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; } @@ -3074,40 +7463,40 @@ extern "C" { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } break; case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast> *> (c); - d->work.pre_run(); + d->work.run (); } break; case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast> *> (c); - d->work.pre_run(); + d->work.run (); } break; } @@ -3162,6 +7551,55 @@ extern "C" { } } +//------------------------------------------------------------------------------ +/// @brief Run post work items. +/// +/// @param[in] c The graph C context. +//------------------------------------------------------------------------------ + void graph_post_run(graph_c_context *c) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast *> (c); + d->work.run (); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast *> (c); + d->work.run (); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast> *> (c); + d->work.run (); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast> *> (c); + d->work.run (); + } + break; + } + } + //------------------------------------------------------------------------------ /// @brief Wait for work items to complete. /// diff --git a/graph_c_binding/graph_c_binding.h b/graph_c_binding/graph_c_binding.h index 49a27b5..3024076 100644 --- a/graph_c_binding/graph_c_binding.h +++ b/graph_c_binding/graph_c_binding.h @@ -385,10 +385,12 @@ extern "C" { /// @brief Construct a random state node. /// /// @param[in] c The graph C context. +/// @param[in] size The number of randoms needed. /// @param[in] seed Initial random seed. /// @returns A random state node. //------------------------------------------------------------------------------ graph_node graph_random_state(STRUCT_TAG graph_c_context *c, + const size_t size, const uint32_t seed); //------------------------------------------------------------------------------ @@ -485,6 +487,152 @@ extern "C" { const double y_scale, const double y_offset); +//------------------------------------------------------------------------------ +/// @brief Create an atomic accumulate 1D index. +/// +/// @param[in] c The graph C context. +/// @param[in] variable The variable to index. +/// @param[in] index The function argument. +/// @param[in] scale Scale factor argument. +/// @param[in] offset Offset factor argument. +/// @param[in] arg Argument. +/// @returns An atomic accumulate 1D node. +//------------------------------------------------------------------------------ + graph_node graph_atomic_accumulate_1D(STRUCT_TAG graph_c_context *c, + graph_node variable, + graph_node index, + const double scale, + const double offset, + graph_node arg); + +//------------------------------------------------------------------------------ +/// @brief Create an index code. +/// +/// @param[in] c The graph C context. +/// @returns An index node. +//------------------------------------------------------------------------------ + graph_node graph_index(STRUCT_TAG graph_c_context *c); + +//------------------------------------------------------------------------------ +/// @brief Create not node. +/// +/// @param[in] c The graph C context. +/// @param[in] arg The function argument. +/// @returns !arg +//------------------------------------------------------------------------------ + graph_node graph_not(STRUCT_TAG graph_c_context *c, + graph_node arg); + +//------------------------------------------------------------------------------ +/// @brief Create an equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left == right +//------------------------------------------------------------------------------ + graph_node graph_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a not equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left != right +//------------------------------------------------------------------------------ + graph_node graph_not_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a greater than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left > right +//------------------------------------------------------------------------------ + graph_node graph_greater_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left < right +//------------------------------------------------------------------------------ + graph_node graph_less_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a greater than equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left >= right +//------------------------------------------------------------------------------ + graph_node graph_greater_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left <= right +//------------------------------------------------------------------------------ + graph_node graph_less_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create an and node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left && right +//------------------------------------------------------------------------------ + graph_node graph_and(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create an or node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left || right +//------------------------------------------------------------------------------ + graph_node graph_or(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a if node. +/// +/// @param[in] c The graph C context. +/// @param[in] condition The logical condition. +/// @param[in] t The true case. +/// @param[in] f The false case. +/// @returns condiiton ? t : f +//------------------------------------------------------------------------------ + graph_node graph_if(STRUCT_TAG graph_c_context *c, + graph_node condition, + graph_node t, + graph_node f); + //------------------------------------------------------------------------------ /// @brief Create 2D piecewise node with complex arguments. /// @@ -513,6 +661,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -522,6 +672,7 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size); @@ -537,6 +688,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -546,10 +699,154 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size); +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size); + +//------------------------------------------------------------------------------ +/// @brief Add pre loop workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_pre_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations); + +//------------------------------------------------------------------------------ +/// @brief Add workflow loop item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations); + +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size); + +//------------------------------------------------------------------------------ +/// @brief Add a pre converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_pre_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter); + //------------------------------------------------------------------------------ /// @brief Add a converge item. /// @@ -561,6 +858,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -572,12 +871,110 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size, const double tol, const size_t max_iter); +//------------------------------------------------------------------------------ +/// @brief Add a post converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_post_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter); + +//------------------------------------------------------------------------------ +/// @brief Add a pre zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_pre_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a post zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_post_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a pre copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_pre_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + +//------------------------------------------------------------------------------ +/// @brief Add a post copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_post_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + //------------------------------------------------------------------------------ /// @brief Compile the work items. /// @@ -599,6 +996,13 @@ extern "C" { //------------------------------------------------------------------------------ void graph_run(STRUCT_TAG graph_c_context *c); +//------------------------------------------------------------------------------ +/// @brief Run post work items. +/// +/// @param[in] c The graph C context. +//------------------------------------------------------------------------------ + void graph_post_run(STRUCT_TAG graph_c_context *c); + //------------------------------------------------------------------------------ /// @brief Wait for work items to complete. /// diff --git a/graph_docs/code_performance.dox b/graph_docs/code_performance.dox index 0fc897e..391c45e 100644 --- a/graph_docs/code_performance.dox +++ b/graph_docs/code_performance.dox @@ -25,7 +25,7 @@ * * The figure above shows the advantage even a single GPU has over CPU * execution. In single precision, the M2's GPU is almost @f$100\times@f$ faster - * a single CPU core while the a single A100 has a nearly $800\times$ advantage. + * a single CPU core while the a single A100 has a nearly @f$800\times@f$ advantage. * An interesting thing to note is the M2 Max CPU show no advantage between * single and double precision execution. * @@ -112,7 +112,7 @@ for (size_t i = 0, ie = threads.size(); i < ie; i++) { {v_next->get_x(), graph::variable_cast(vx)}, {v_next->get_y(), graph::variable_cast(vy)}, {v_next->get_z(), graph::variable_cast(vz)} - }, NULL, "Lorentz_kernel", local_size); + }, {}, NULL, "Lorentz_kernel", local_size); work.compile(); time_steps.start_time(thread_number); diff --git a/graph_docs/discription.dox b/graph_docs/discription.dox index 1f48961..9ff9378 100644 --- a/graph_docs/discription.dox +++ b/graph_docs/discription.dox @@ -39,9 +39,9 @@ * expression nodes. The factory method checks a node_cache to avoid building * duplicate sub-graphs. Identification of duplicate graphs is performed by * computing a hash of the sub-graph. This hash can be rapidly checked if the - * same hash already exists in a std::map container. If the sub-graph - * already exists, the existing graph is returned otherwise a new sub-graph is - * registered in the node_cache. + * same hash already exists in a std::unordered_map container. If the + * sub-graph already exists, the existing graph is returned otherwise a new + * sub-graph is registered in the node_cache. * * Each time an expression is built, the reduce method is called to simplify the * graph. For instance, a graph consisting of constant added to a constant will diff --git a/graph_docs/kernel_optimization.dox b/graph_docs/kernel_optimization.dox index e4302fc..ac9641e 100644 --- a/graph_docs/kernel_optimization.dox +++ b/graph_docs/kernel_optimization.dox @@ -57,7 +57,7 @@ void field_solve_example() { + graph::exp(static_cast (-1)*arg*arg/static_cast (10)); } - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(num_particles), 0); auto random = graph::random (graph::random_state_cast(state)); const T max = 1.0; const T min = -1.0; @@ -68,7 +68,7 @@ void field_solve_example() { timing::measure_diagnostic compile("compile"); workflow::manager work(0); - work.add_preitem({ + work.add_item ({ graph::variable_cast(particle_positions) }, {}, { {random_real, variable_cast(particle_positions)} diff --git a/graph_docs/use_cases.dox b/graph_docs/use_cases.dox index f47fa27..1025793 100644 --- a/graph_docs/use_cases.dox +++ b/graph_docs/use_cases.dox @@ -125,7 +125,7 @@ * branch is absorbed in the upper hybrid resonance, @f$\omega_{h}@f$, while the * O-Mode branch can pass through it. * - * @subsubsection use_cases_rf_correctness Comparison to GENRAY + * @subsubsection use_cases_rf_correctness_genray Comparison to GENRAY * Genray is an RF-Ray tracing * code written in Fortran which operates in a cylindrical geometry. Toroidal * equilibria can be imported using the diff --git a/graph_driver/xrays.cpp b/graph_driver/xrays.cpp index cb8ec2d..e09c8b4 100644 --- a/graph_driver/xrays.cpp +++ b/graph_driver/xrays.cpp @@ -740,7 +740,7 @@ void bin_power(const commandline::parser &cl, {z, graph::variable_cast(z_last)}, {p_next, graph::variable_cast(power)}, {k_next, graph::variable_cast(k_sum)} - }, graph::shared_random_state (), "power", local_num_rays); + }, {}, NULL, "power", local_num_rays); work.compile(); output::result_file file(stream.str()); diff --git a/graph_fortran_binding/graph_fortran_binding.f90 b/graph_fortran_binding/graph_fortran_binding.f90 index bc5d7c6..184ee63 100644 --- a/graph_fortran_binding/graph_fortran_binding.f90 +++ b/graph_fortran_binding/graph_fortran_binding.f90 @@ -568,14 +568,16 @@ TYPE(C_PTR) FUNCTION graph_atan(c, left, right) & !> @brief Construct a random state node. !> !> @param[in] c The graph C context. +!> @param[in] size Number of randoms needed. !> @param[in] seed Initial random seed. !> @returns A random state node. !------------------------------------------------------------------------------- - TYPE(C_PTR) FUNCTION graph_random_state(c, seed) & + TYPE(C_PTR) FUNCTION graph_random_state(c, size, seed) & BIND(C, NAME='graph_random_state') USE, INTRINSIC :: ISO_C_BINDING IMPLICIT NONE TYPE(C_PTR), VALUE :: c + INTEGER(C_LONG), value :: size INTEGER(C_INT32_T), VALUE :: seed END FUNCTION @@ -742,6 +744,8 @@ SUBROUTINE graph_set_device_number(c, num) & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -749,6 +753,7 @@ SUBROUTINE graph_set_device_number(c, num) & SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles) & BIND(C, NAME='graph_add_pre_item') USE, INTRINSIC :: ISO_C_BINDING @@ -761,6 +766,8 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -777,6 +784,8 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -784,6 +793,7 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & SUBROUTINE graph_add_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles) & BIND(C, NAME='graph_add_item') USE, INTRINSIC :: ISO_C_BINDING @@ -796,6 +806,8 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -812,6 +824,8 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -821,6 +835,7 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & SUBROUTINE graph_add_converge_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles, & tol, max_iter) & BIND(C, NAME='graph_add_converge_item') @@ -834,6 +849,8 @@ SUBROUTINE graph_add_converge_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -1588,20 +1605,23 @@ FUNCTION graph_context_atan(this, left, right) !> @brief Get random size. !> !> @param[in,out] this @ref graph_context instance. +!> @param[in] size Number of random numbers needed. !> @param[in] seed Initial random seed. !> @returns The random size. !------------------------------------------------------------------------------- - FUNCTION graph_context_random_state(this, seed) + FUNCTION graph_context_random_state(this, size, seed) IMPLICIT NONE ! Declare Arguments TYPE(C_PTR) :: graph_context_random_state CLASS(graph_context), INTENT(INOUT) :: this + INTEGER(C_LONG), INTENT(IN) :: size INTEGER(C_INT32_T), INTENT(IN) :: seed ! Start of executable. - graph_context_random_state = graph_random_state(this%c_context, seed) + graph_context_random_state = graph_random_state(this%c_context, & + size, seed) END FUNCTION @@ -2030,13 +2050,15 @@ SUBROUTINE graph_context_set_device_number(this, num) !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, num_particles) + atomics, random_state, name, & + num_particles) IMPLICIT NONE @@ -2046,6 +2068,7 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2056,6 +2079,8 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & LOC(outputs), INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles) END SUBROUTINE @@ -2068,13 +2093,15 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, num_particles) + atomics, random_state, name, & + num_particles) IMPLICIT NONE @@ -2084,6 +2111,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2094,6 +2122,8 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & LOC(outputs), INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles) END SUBROUTINE @@ -2106,6 +2136,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -2114,7 +2145,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, & + atomics, random_state, name, & num_particles, tol, max_iter) IMPLICIT NONE @@ -2125,6 +2156,7 @@ SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2138,6 +2170,8 @@ SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles, & tol, max_iter) diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index e6213ad..c9c7a90 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -9,6 +9,8 @@ /* Begin PBXBuildFile section */ C70D93152A30FF4E006A4227 /* special_functions.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C70D93132A30FF4E006A4227 /* special_functions.hpp */; }; C713426A2947F39400672AD4 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C715C79B2FD09E7D003EEFF4 /* pic_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7C16FD02FCF488F008D4ABB /* pic_test.cpp */; }; + C715C79C2FD09FF3003EEFF4 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C7170CC02C66A228003274E2 /* efit_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7D3C5B02C654AD3008AD8C6 /* efit_test.cpp */; }; C7170CC12C66A238003274E2 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C73690382A38C958001733B0 /* node.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C79141AE22DA9C3000E0BA0D /* node.hpp */; }; @@ -41,15 +43,21 @@ C74F2ADD2F6D9B0D00B48216 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C74F2AEA2F6DE8E400B48216 /* workflow_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C74F2ADE2F6DC10E00B48216 /* workflow_test.cpp */; }; C74F2AEB2F6DE8EC00B48216 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C76263472FE0887300F283DF /* logical_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C762633B2FE0754F00F283DF /* logical_test.cpp */; }; + C76263482FE0891300F283DF /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C7DC9EEF2E397BE600524F6F /* Cocoa.framework */; }; C78F3D972DC41AF2002E3D94 /* random_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C78F3D8A2DC122C7002E3D94 /* random_test.cpp */; }; C78F3D982DC41B05002E3D94 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C78F3DA72DC41BB8002E3D94 /* xkorc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C78F3D882DC122B1002E3D94 /* xkorc.cpp */; }; C78F3DA82DC41BCA002E3D94 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C79141B622DAAD0C00E0BA0D /* xrays.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C79141B522DAAD0C00E0BA0D /* xrays.cpp */; }; C7B676082AA9023F005AB34C /* xrays_bench.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7B676072AA9023F005AB34C /* xrays_bench.cpp */; }; + C7C16FCF2FCF3EF2008D4ABB /* particle_in_cell.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */; }; C7D12D9A2DBAB31F00925420 /* random.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C7D12D992DBAB31F00925420 /* random.hpp */; }; C7D371132A0595A40074676E /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C7DC9EEC2E39790100524F6F /* graph_c_binding.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7DC9EE22E39768300524F6F /* graph_c_binding.cpp */; }; + C7DF2CC7305C635500E8491C /* c_binding_test.c in Sources */ = {isa = PBXBuildFile; fileRef = C7DC9EF12E3A688F00524F6F /* c_binding_test.c */; }; + C7DF2CC8305C636000E8491C /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C7DF2CCD305C63DE00E8491C /* libgraph_c.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C7DC9EE82E39789900524F6F /* libgraph_c.a */; }; C7E5644528A2A1AA000F31A2 /* backend_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E7328074F540033B488 /* backend_test.cpp */; }; C7E5645128A2A1DD000F31A2 /* dispersion_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E6B28073BCA0033B488 /* dispersion_test.cpp */; }; C7E5645D28A2A21D000F31A2 /* solver_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E6C28073BCA0033B488 /* solver_test.cpp */; }; @@ -161,9 +169,32 @@ remoteGlobalIDString = C79141A522DA9BF200E0BA0D; remoteInfo = graph_framework; }; + C7DF2CC9305C63A900E8491C /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = C791419E22DA9BF200E0BA0D /* Project object */; + proxyType = 1; + remoteGlobalIDString = C7DC9EE72E39789900524F6F; + remoteInfo = graph_c; + }; + C7DF2CCB305C63B100E8491C /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = C791419E22DA9BF200E0BA0D /* Project object */; + proxyType = 1; + remoteGlobalIDString = C79141A522DA9BF200E0BA0D; + remoteInfo = graph_framework; + }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ + C715C7922FD09E29003EEFF4 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C7170CB72C66A10D003274E2 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -236,6 +267,15 @@ ); runOnlyForDeploymentPostprocessing = 1; }; + C762633E2FE0882200F283DF /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C78F3D8D2DC41ACA002E3D94 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -263,6 +303,15 @@ ); runOnlyForDeploymentPostprocessing = 1; }; + C7DF2CBE305C632700E8491C /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C7E5643C28A2A16F000F31A2 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -355,6 +404,7 @@ C713425C2942665300672AD4 /* register.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = register.hpp; sourceTree = ""; }; C71342652947D57900672AD4 /* metal_context.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = metal_context.hpp; sourceTree = ""; }; C71342682947F36100672AD4 /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = System/Library/Frameworks/Metal.framework; sourceTree = SDKROOT; }; + C715C7942FD09E29003EEFF4 /* pic_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = pic_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7167B222AC5CE8500E03131 /* fix_NaN.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = fix_NaN.py; sourceTree = ""; }; C7170CB92C66A10D003274E2 /* efit_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = efit_test; sourceTree = BUILT_PRODUCTS_DIR; }; C717CB8D2A02E361008FBDD8 /* FindNetCDF.cmake */ = {isa = PBXFileReference; lastKnownFileType = text; path = FindNetCDF.cmake; sourceTree = ""; }; @@ -390,8 +440,11 @@ C75C42932E5CA60B00B0950B /* main.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = main.dox; sourceTree = ""; }; C75C42952E5CC80B00B0950B /* tutorial.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = tutorial.dox; sourceTree = ""; }; C760B1AB2BC6D760001737A3 /* get_includes.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = get_includes.py; sourceTree = ""; }; + C762633B2FE0754F00F283DF /* logical_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = logical_test.cpp; sourceTree = ""; }; + C76263402FE0882200F283DF /* logical_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = logical_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7678FBD2B45C2850025F37E /* bin.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = bin.py; sourceTree = ""; }; C77707F62F5F288B00BA4E87 /* kernel_optimization.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = kernel_optimization.dox; sourceTree = ""; }; + C77CA28F2FDB7CBA00D71BF6 /* no_derivative_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = no_derivative_test.cpp; sourceTree = ""; }; C77E6DF522DD64E700469621 /* trigonometry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = trigonometry.hpp; sourceTree = ""; }; C78F3D872DC122B1002E3D94 /* CMakeLists.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = CMakeLists.txt; sourceTree = ""; }; C78F3D882DC122B1002E3D94 /* xkorc.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = xkorc.cpp; sourceTree = ""; }; @@ -420,6 +473,8 @@ C7B676092AA90243005AB34C /* CMakeLists.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = CMakeLists.txt; sourceTree = ""; }; C7B677D829E45C9500D3ADC6 /* backend.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = backend.hpp; sourceTree = ""; }; C7B677DA29E464AE00D3ADC6 /* cpu_context.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = cpu_context.hpp; sourceTree = ""; }; + C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = particle_in_cell.hpp; sourceTree = ""; }; + C7C16FD02FCF488F008D4ABB /* pic_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = pic_test.cpp; sourceTree = ""; }; C7CEA0042948D02A00F61D09 /* timing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = timing.hpp; sourceTree = ""; }; C7CEA0052948EB0F00F61D09 /* cuda_context.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = cuda_context.hpp; sourceTree = ""; }; C7D12D992DBAB31F00925420 /* random.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = random.hpp; sourceTree = ""; }; @@ -433,6 +488,7 @@ C7DC9EF12E3A688F00524F6F /* c_binding_test.c */ = {isa = PBXFileReference; explicitFileType = sourcecode.c.objc; path = c_binding_test.c; sourceTree = ""; }; C7DD87D32E664B440058BA66 /* code_structure.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = code_structure.dox; sourceTree = ""; }; C7DD87D42E665E260058BA66 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; + C7DF2CC0305C632700E8491C /* c_binding_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = c_binding_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E134492A3CB3EC0083F6A7 /* output.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = output.hpp; sourceTree = ""; }; C7E5643E28A2A16F000F31A2 /* backend_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = backend_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E5644A28A2A1C5000F31A2 /* dispersion_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = dispersion_test; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -443,11 +499,20 @@ C7E5648628A2A324000F31A2 /* vector_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = vector_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E5649228A2A34A000F31A2 /* physics_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = physics_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E7D02F283565A200E09896 /* vector_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = vector_test.cpp; sourceTree = ""; }; + C7EBEA5A2FE0648C005C0463 /* logical.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = logical.hpp; sourceTree = ""; }; C7FA0DFD29590B7400A31E4D /* jit_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; fileEncoding = 4; path = jit_test.cpp; sourceTree = ""; }; C7FA0E0329590EF300A31E4D /* jit_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = jit_test; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + C715C7912FD09E29003EEFF4 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C715C79C2FD09FF3003EEFF4 /* Metal.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7170CB62C66A10D003274E2 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -510,6 +575,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C762633D2FE0882200F283DF /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C76263482FE0891300F283DF /* Cocoa.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C78F3D8C2DC41ACA002E3D94 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -548,6 +621,15 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C7DF2CBD305C632700E8491C /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C7DF2CCD305C63DE00E8491C /* libgraph_c.a in Frameworks */, + C7DF2CC8305C636000E8491C /* Metal.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7E5643B28A2A16F000F31A2 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -743,6 +825,9 @@ C7DC9EE82E39789900524F6F /* libgraph_c.a */, C74F2AD22F6D9A6E00B48216 /* graph_pic */, C74F2AE32F6DE8C500B48216 /* workflow_test */, + C715C7942FD09E29003EEFF4 /* pic_test */, + C76263402FE0882200F283DF /* logical_test */, + C7DF2CC0305C632700E8491C /* c_binding_test */, ); name = Products; sourceTree = ""; @@ -753,6 +838,7 @@ C7931E7028073BE70033B488 /* CMakeLists.txt */, C7D453872EBFD05D00A828DB /* graph_framework.hpp */, C79141AE22DA9C3000E0BA0D /* node.hpp */, + C7EBEA5A2FE0648C005C0463 /* logical.hpp */, C72358F52C4027A10084A489 /* commandline_parser.hpp */, C70B705629F4F86A00098AA0 /* piecewise.hpp */, C7922EEB29E0ABDF000BB9C7 /* workflow.hpp */, @@ -775,6 +861,7 @@ C7CEA0042948D02A00F61D09 /* timing.hpp */, C73BBE9629F8669F0027BB7F /* newton.hpp */, C7E134492A3CB3EC0083F6A7 /* output.hpp */, + C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */, ); path = graph_framework; sourceTree = ""; @@ -810,6 +897,9 @@ C7DC9EF12E3A688F00524F6F /* c_binding_test.c */, C7AE06662E3C2AEE00586BCD /* f_binding_test.f90 */, C74F2ADE2F6DC10E00B48216 /* workflow_test.cpp */, + C7C16FD02FCF488F008D4ABB /* pic_test.cpp */, + C77CA28F2FDB7CBA00D71BF6 /* no_derivative_test.cpp */, + C762633B2FE0754F00F283DF /* logical_test.cpp */, ); path = graph_tests; sourceTree = ""; @@ -850,6 +940,7 @@ C736903F2A38C958001733B0 /* dispersion.hpp in Headers */, C73690402A38C958001733B0 /* solver.hpp in Headers */, C7D12D9A2DBAB31F00925420 /* random.hpp in Headers */, + C7C16FCF2FCF3EF2008D4ABB /* particle_in_cell.hpp in Headers */, C73690412A38C958001733B0 /* backend.hpp in Headers */, C73690422A38C958001733B0 /* equilibrium.hpp in Headers */, C73690432A38C958001733B0 /* jit.hpp in Headers */, @@ -873,6 +964,25 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXNativeTarget section */ + C715C7932FD09E29003EEFF4 /* pic_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C715C79A2FD09E29003EEFF4 /* Build configuration list for PBXNativeTarget "pic_test" */; + buildPhases = ( + C715C7902FD09E29003EEFF4 /* Sources */, + C715C7912FD09E29003EEFF4 /* Frameworks */, + C715C7922FD09E29003EEFF4 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = pic_test; + packageProductDependencies = ( + ); + productName = pic_test; + productReference = C715C7942FD09E29003EEFF4 /* pic_test */; + productType = "com.apple.product-type.tool"; + }; C7170CB82C66A10D003274E2 /* efit_test */ = { isa = PBXNativeTarget; buildConfigurationList = C7170CBF2C66A10D003274E2 /* Build configuration list for PBXNativeTarget "efit_test" */; @@ -1018,6 +1128,25 @@ productReference = C74F2AE32F6DE8C500B48216 /* workflow_test */; productType = "com.apple.product-type.tool"; }; + C762633F2FE0882200F283DF /* logical_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C76263442FE0882200F283DF /* Build configuration list for PBXNativeTarget "logical_test" */; + buildPhases = ( + C762633C2FE0882200F283DF /* Sources */, + C762633D2FE0882200F283DF /* Frameworks */, + C762633E2FE0882200F283DF /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = logical_test; + packageProductDependencies = ( + ); + productName = logical_test; + productReference = C76263402FE0882200F283DF /* logical_test */; + productType = "com.apple.product-type.tool"; + }; C78F3D8E2DC41ACA002E3D94 /* random_test */ = { isa = PBXNativeTarget; buildConfigurationList = C78F3D932DC41ACA002E3D94 /* Build configuration list for PBXNativeTarget "random_test" */; @@ -1111,6 +1240,27 @@ productReference = C7DC9EE82E39789900524F6F /* libgraph_c.a */; productType = "com.apple.product-type.library.static"; }; + C7DF2CBF305C632700E8491C /* c_binding_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C7DF2CC6305C632700E8491C /* Build configuration list for PBXNativeTarget "c_binding_test" */; + buildPhases = ( + C7DF2CBC305C632700E8491C /* Sources */, + C7DF2CBD305C632700E8491C /* Frameworks */, + C7DF2CBE305C632700E8491C /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + C7DF2CCC305C63B100E8491C /* PBXTargetDependency */, + C7DF2CCA305C63A900E8491C /* PBXTargetDependency */, + ); + name = c_binding_test; + packageProductDependencies = ( + ); + productName = c_binding_test; + productReference = C7DF2CC0305C632700E8491C /* c_binding_test */; + productType = "com.apple.product-type.tool"; + }; C7E5643D28A2A16F000F31A2 /* backend_test */ = { isa = PBXNativeTarget; buildConfigurationList = C7E5644228A2A16F000F31A2 /* Build configuration list for PBXNativeTarget "backend_test" */; @@ -1280,9 +1430,12 @@ isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = YES; - LastUpgradeCheck = 2610; + LastUpgradeCheck = 2640; ORGANIZATIONNAME = "Cianciosa, Mark R."; TargetAttributes = { + C715C7932FD09E29003EEFF4 = { + CreatedOnToolsVersion = 26.4; + }; C7170CB82C66A10D003274E2 = { CreatedOnToolsVersion = 15.4; }; @@ -1307,6 +1460,9 @@ C74F2AE22F6DE8C500B48216 = { CreatedOnToolsVersion = 26.1; }; + C762633F2FE0882200F283DF = { + CreatedOnToolsVersion = 26.4; + }; C78F3D8E2DC41ACA002E3D94 = { CreatedOnToolsVersion = 16.3; }; @@ -1322,6 +1478,9 @@ C7DC9EE72E39789900524F6F = { CreatedOnToolsVersion = 16.4; }; + C7DF2CBF305C632700E8491C = { + CreatedOnToolsVersion = 27.0; + }; C7E5643D28A2A16F000F31A2 = { CreatedOnToolsVersion = 13.4; }; @@ -1386,11 +1545,22 @@ C7DC9EE72E39789900524F6F /* graph_c */, C74F2AD12F6D9A6E00B48216 /* graph_pic */, C74F2AE22F6DE8C500B48216 /* workflow_test */, + C715C7932FD09E29003EEFF4 /* pic_test */, + C762633F2FE0882200F283DF /* logical_test */, + C7DF2CBF305C632700E8491C /* c_binding_test */, ); }; /* End PBXProject section */ /* Begin PBXSourcesBuildPhase section */ + C715C7902FD09E29003EEFF4 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C715C79B2FD09E7D003EEFF4 /* pic_test.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7170CB52C66A10D003274E2 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1455,6 +1625,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C762633C2FE0882200F283DF /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C76263472FE0887300F283DF /* logical_test.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C78F3D8B2DC41ACA002E3D94 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1494,6 +1672,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C7DF2CBC305C632700E8491C /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C7DF2CC7305C635500E8491C /* c_binding_test.c in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7E5643A28A2A16F000F31A2 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1639,9 +1825,51 @@ target = C79141A522DA9BF200E0BA0D /* graph_framework */; targetProxy = C7DC9EED2E39791C00524F6F /* PBXContainerItemProxy */; }; + C7DF2CCA305C63A900E8491C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = C7DC9EE72E39789900524F6F /* graph_c */; + targetProxy = C7DF2CC9305C63A900E8491C /* PBXContainerItemProxy */; + }; + C7DF2CCC305C63B100E8491C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = C79141A522DA9BF200E0BA0D /* graph_framework */; + targetProxy = C7DF2CCB305C63B100E8491C /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + C715C7982FD09E29003EEFF4 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C715C7992FD09E29003EEFF4 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C7170CBD2C66A10D003274E2 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -1857,7 +2085,7 @@ "$(inherited)", ); LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.1; + MACOSX_DEPLOYMENT_TARGET = 26.0; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Debug; @@ -1871,7 +2099,7 @@ CODE_SIGN_STYLE = Automatic; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.1; + MACOSX_DEPLOYMENT_TARGET = 26.0; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Release; @@ -1908,6 +2136,38 @@ }; name = Release; }; + C76263452FE0882200F283DF /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C76263462FE0882200F283DF /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C78F3D942DC41ACA002E3D94 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -2031,6 +2291,7 @@ "\"CXX_ARGS=\\\"-I/Users/m4c/Projects/graph_framework/graph_framework -I/usr/local/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/v1 -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks -fgnuc-version=4.2.1 -std=gnu++2a\\\"\"", STATIC, "MACOS_LIB_RT=\\\"/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/lib/darwin/libclang_rt.osx.a\\\"", + USE_INPUT_CACHE, USE_INDEX_CACHE, "USE_VERBOSE=false", "$(inherited)", @@ -2049,7 +2310,7 @@ "build/_deps/llvm-build/lib", /usr/local/lib, ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -2143,6 +2404,7 @@ "-lclangParse", "-lclangAPINotes", "-lclangOptions", + "-lclangCodeGenUtils", "-lclangCodeGen", "-rpath", /usr/local/lib, @@ -2207,6 +2469,7 @@ USE_METAL, "MACOS_LIB_RT=\\\"/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/lib/darwin/libclang_rt.osx.a\\\"", "USE_VERBOSE=false", + USE_INPUT_CACHE, USE_INDEX_CACHE, "\"CXX_ARGS=\\\"-I/Users/m4c/Projects/graph_framework/graph_framework -I/usr/local/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/v1 -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks -fgnuc-version=4.2.1 -std=gnu++2a\\\"\"", "$(inherited)", @@ -2225,7 +2488,7 @@ "build/_deps/llvm-build/lib", /usr/local/lib, ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -2319,6 +2582,7 @@ "-lclangParse", "-lclangAPINotes", "-lclangOptions", + "-lclangCodeGenUtils", "-lclangCodeGen", "-rpath", /usr/local/lib, @@ -2345,7 +2609,15 @@ GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; MACOSX_DEPLOYMENT_TARGET = 15.0; OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; - OTHER_LDFLAGS = ""; + "OTHER_CPLUSPLUSFLAGS[arch=*]" = ( + "$(OTHER_CFLAGS)", + "-fsanitize=undefined", + "-fsanitize=float-divide-by-zero", + ); + OTHER_LDFLAGS = ( + "-fsanitize=float-divide-by-zero", + "-fsenitize=undefined", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SKIP_INSTALL = YES; @@ -2362,7 +2634,10 @@ GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; MACOSX_DEPLOYMENT_TARGET = 15.0; OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; - OTHER_LDFLAGS = ""; + OTHER_LDFLAGS = ( + "-fsanitize=float-divide-by-zero", + "-fsenitize=undefined", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SKIP_INSTALL = YES; @@ -2416,7 +2691,7 @@ "$(inherited)", ); LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.5; + MACOSX_DEPLOYMENT_TARGET = 26.6; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; }; @@ -2431,12 +2706,234 @@ EXECUTABLE_PREFIX = lib; GCC_C_LANGUAGE_STANDARD = gnu23; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.5; + MACOSX_DEPLOYMENT_TARGET = 26.6; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; }; name = Release; }; + C7DF2CC4305C632700E8491C /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.7; + OTHER_LDFLAGS = ( + "-lc++", + "-lnetcdf", + "-ld_classic", + "-L/Users/m4c/Projects/graph_framework/build/_deps/llvm-build/lib", + "-lz", + "-lLLVMCoverage", + "-lLLVMSupport", + "-lLLVMDebugInfoCodeView", + "-lLLVMRemarks", + "-lLLVMJITLink", + "-lLLVMLinker", + "-lLLVMTextAPI", + "-lLLVMRuntimeDyld", + "-lLLVMOrcShared", + "-lLLVMOrcDebugging", + "-lLLVMOrcTargetProcess", + "-lLLVMOrcJIT", + "-lLLVMHipStdPar", + "-lLLVMAggressiveInstCombine", + "-lLLVMVectorize", + "-lLLVMAsmParser", + "-lLLVMOption", + "-lLLVMLTO", + "-lLLVMObject", + "-lLLVMWindowsDriver", + "-lLLVMDemangle", + "-lLLVMIRReader", + "-lLLVMIRPrinter", + "-lLLVMInstCombine", + "-lLLVMBinaryFormat", + "-lLLVMCoroutines", + "-lLLVMBitstreamReader", + "-lLLVMBitReader", + "-lLLVMBitWriter", + "-lLLVMDebugInfoDWARF", + "-lLLVMInstrumentation", + "-lLLVMCFGuard", + "-lLLVMObjCARCOpts", + "-lLLVMipo", + "-lLLVMGlobalISel", + "-lLLVMExecutionEngine", + "-lLLVMFrontendDriver", + "-lLLVMFrontendHLSL", + "-lLLVMFrontendOpenMP", + "-lLLVMFrontendDirective", + "-lLLVMFrontendOffloading", + "-lLLVMSelectionDAG", + "-lLLVMProfileData", + "-lLLVMAnalysis", + "-lLLVMScalarOpts", + "-lLLVMCodeGenTypes", + "-lLLVMCodeGen", + "-lLLVMTargetParser", + "-lLLVMScalarOpts", + "-lLLVMTarget", + "-lLLVMTransformUtils", + "-lLLVMPasses", + "-lLLVMSupport", + "-lLLVMMCParser", + "-lLLVMMC", + "-lLLVMCore", + "-lLLVMAsmPrinter", + "-lLLVMAArch64Utils", + "-lLLVMAArch64Info", + "-lLLVMAArch64Desc", + "-lLLVMAArch64AsmParser", + "-lLLVMDebugInfoDWARFLowLevel", + "-lLLVMAArch64CodeGen", + "-lLLVMCGData", + "-lLLVMSandboxIR", + "-lLLVMObjectYAML", + "-lLLVMPlugins", + "-lLLVMABI", + "-lLLVMFrontendAtomic", + "-lclangFrontend", + "-lclangBasic", + "-lclangEdit", + "-lclangLex", + "-lclangDriver", + "-lclangSerialization", + "-lclangAST", + "-lclangSema", + "-lclangAnalysisLifetimeSafety", + "-lclangAnalysis", + "-lclangASTMatchers", + "-lclangSupport", + "-lclangParse", + "-lclangAPINotes", + "-lclangOptions", + "-lclangCodeGenUtils", + "-lclangCodeGen", + "-rpath", + /usr/local/lib, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C7DF2CC5305C632700E8491C /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.7; + OTHER_LDFLAGS = ( + "-lc++", + "-lnetcdf", + "-ld_classic", + "-L/Users/m4c/Projects/graph_framework/build/_deps/llvm-build/lib", + "-lz", + "-lLLVMCoverage", + "-lLLVMSupport", + "-lLLVMDebugInfoCodeView", + "-lLLVMRemarks", + "-lLLVMJITLink", + "-lLLVMLinker", + "-lLLVMTextAPI", + "-lLLVMRuntimeDyld", + "-lLLVMOrcShared", + "-lLLVMOrcDebugging", + "-lLLVMOrcTargetProcess", + "-lLLVMOrcJIT", + "-lLLVMHipStdPar", + "-lLLVMAggressiveInstCombine", + "-lLLVMVectorize", + "-lLLVMAsmParser", + "-lLLVMOption", + "-lLLVMLTO", + "-lLLVMObject", + "-lLLVMWindowsDriver", + "-lLLVMDemangle", + "-lLLVMIRReader", + "-lLLVMIRPrinter", + "-lLLVMInstCombine", + "-lLLVMBinaryFormat", + "-lLLVMCoroutines", + "-lLLVMBitstreamReader", + "-lLLVMBitReader", + "-lLLVMBitWriter", + "-lLLVMDebugInfoDWARF", + "-lLLVMInstrumentation", + "-lLLVMCFGuard", + "-lLLVMObjCARCOpts", + "-lLLVMipo", + "-lLLVMGlobalISel", + "-lLLVMExecutionEngine", + "-lLLVMFrontendDriver", + "-lLLVMFrontendHLSL", + "-lLLVMFrontendOpenMP", + "-lLLVMFrontendDirective", + "-lLLVMFrontendOffloading", + "-lLLVMSelectionDAG", + "-lLLVMProfileData", + "-lLLVMAnalysis", + "-lLLVMScalarOpts", + "-lLLVMCodeGenTypes", + "-lLLVMCodeGen", + "-lLLVMTargetParser", + "-lLLVMScalarOpts", + "-lLLVMTarget", + "-lLLVMTransformUtils", + "-lLLVMPasses", + "-lLLVMSupport", + "-lLLVMMCParser", + "-lLLVMMC", + "-lLLVMCore", + "-lLLVMAsmPrinter", + "-lLLVMAArch64Utils", + "-lLLVMAArch64Info", + "-lLLVMAArch64Desc", + "-lLLVMAArch64AsmParser", + "-lLLVMDebugInfoDWARFLowLevel", + "-lLLVMAArch64CodeGen", + "-lLLVMCGData", + "-lLLVMSandboxIR", + "-lLLVMObjectYAML", + "-lLLVMPlugins", + "-lLLVMABI", + "-lLLVMFrontendAtomic", + "-lclangFrontend", + "-lclangBasic", + "-lclangEdit", + "-lclangLex", + "-lclangDriver", + "-lclangSerialization", + "-lclangAST", + "-lclangSema", + "-lclangAnalysisLifetimeSafety", + "-lclangAnalysis", + "-lclangASTMatchers", + "-lclangSupport", + "-lclangParse", + "-lclangAPINotes", + "-lclangOptions", + "-lclangCodeGenUtils", + "-lclangCodeGen", + "-rpath", + /usr/local/lib, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C7E5644328A2A16F000F31A2 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -2691,6 +3188,15 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + C715C79A2FD09E29003EEFF4 /* Build configuration list for PBXNativeTarget "pic_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C715C7982FD09E29003EEFF4 /* Debug */, + C715C7992FD09E29003EEFF4 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C7170CBF2C66A10D003274E2 /* Build configuration list for PBXNativeTarget "efit_test" */ = { isa = XCConfigurationList; buildConfigurations = ( @@ -2763,6 +3269,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + C76263442FE0882200F283DF /* Build configuration list for PBXNativeTarget "logical_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C76263452FE0882200F283DF /* Debug */, + C76263462FE0882200F283DF /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C78F3D932DC41ACA002E3D94 /* Build configuration list for PBXNativeTarget "random_test" */ = { isa = XCConfigurationList; buildConfigurations = ( @@ -2817,6 +3332,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + C7DF2CC6305C632700E8491C /* Build configuration list for PBXNativeTarget "c_binding_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C7DF2CC4305C632700E8491C /* Debug */, + C7DF2CC5305C632700E8491C /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C7E5644228A2A16F000F31A2 /* Build configuration list for PBXNativeTarget "backend_test" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme b/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme index 1688675..b5495d8 100644 --- a/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme +++ b/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme @@ -1,6 +1,6 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 87ddff0..abd15ff 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -17,7 +17,7 @@ execute_process (COMMAND ${Python_EXECUTABLE} get_includes.py --compiler=${CMAKE target_compile_definitions (graph_framework INTERFACE $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -fgnuc-version=4.2.1 -std=gnu++2a"> - $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -std=gnu++2a -fno-use-cxa-atexit"> + $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -std=gnu++2a -fno-use-cxa-atexit -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -D__GCC_ATOMIC_POINTER_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR_LOCK_FREE=2 -D__GCC_ATOMIC_SHORT_LOCK_FREE=2 -D__GCC_ATOMIC_INT_LOCK_FREE=2 -D__GCC_ATOMIC_LONG_LOCK_FREE=2 -D__GCC_ATOMIC_LLONG_LOCK_FREE -D__GCC_ATOMIC_CHAR8_T_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR16_T_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR32_T_LOCK_FREE=2 -D__GCC_ATOMIC_BOOL_LOCK_FREE=2 -D__GCC_ATOMIC_WCHAR_T_LOCK_FREE=2"> EFIT_FILE="${CMAKE_CURRENT_SOURCE_DIR}/../graph_tests/efit.nc" VMEC_FILE="${CMAKE_CURRENT_SOURCE_DIR}/../graph_tests/vmec.nc" $<$:HEADER_DIR="$"> @@ -26,6 +26,7 @@ target_compile_definitions (graph_framework $<$:SHOW_USE_COUNT> $<$:USE_INDEX_CACHE> $,USE_VERBOSE=true,USE_VERBOSE=false> + $<$:PROFILE_KERNELS> ) target_include_directories (graph_framework @@ -66,4 +67,6 @@ target_precompile_headers (graph_framework $<$:$> $<$:$<$:$>> $<$:$<$:$>> + $<$:$> + $<$:$> ) diff --git a/graph_framework/absorption.hpp b/graph_framework/absorption.hpp index 2816851..1b48885 100644 --- a/graph_framework/absorption.hpp +++ b/graph_framework/absorption.hpp @@ -233,7 +233,7 @@ namespace absorption { {graph::zero (), graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, NULL, + work.add_item(inputs, {}, setters, {}, NULL, "root_find_init_kernel", inputs.back()->size()); inputs.push_back(graph::variable_cast(this->t)); @@ -245,8 +245,7 @@ namespace absorption { kvec + kamp_vec, x, y, z, t, eq); - solver::newton(work, {kamp}, inputs, {D}, - graph::shared_random_state ()); + solver::newton (work, {kamp}, inputs, {}, {D}, NULL); inputs = { graph::variable_cast(this->kamp), @@ -260,7 +259,7 @@ namespace absorption { setters = { {klen + kamp, graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, NULL, + work.add_item(inputs, {}, setters, {}, NULL, "final_kamp", inputs.back()->size()); } @@ -426,8 +425,7 @@ namespace absorption { {kamp1, graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, - graph::shared_random_state (), + work.add_item(inputs, {}, setters, {}, NULL, "weak_damping_kimg_kernel", inputs.back()->size()); } diff --git a/graph_framework/arithmetic.hpp b/graph_framework/arithmetic.hpp index 19734d1..efba008 100644 --- a/graph_framework/arithmetic.hpp +++ b/graph_framework/arithmetic.hpp @@ -193,11 +193,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -206,13 +204,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -221,29 +219,29 @@ namespace graph { result.add_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.add_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.add_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.add_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // Identity reductions. @@ -635,26 +633,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -682,12 +676,10 @@ namespace graph { auto x_cast = add_cast(x); if (x_cast.get()) { // Addition is commutative. - if ((this->left->is_match(x_cast->get_left()) && - this->right->is_match(x_cast->get_right())) || - (this->right->is_match(x_cast->get_left()) && - this->left->is_match(x_cast->get_right()))) { - return true; - } + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); } return false; @@ -949,11 +941,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -962,13 +952,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -976,30 +966,30 @@ namespace graph { backend::buffer result = pl1->evaluate(); result.subtract_row(pr2->evaluate()); return piecewise_2D(result, - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.subtract_col(pr2->evaluate()); return piecewise_2D(result, - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.subtract_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.subtract_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // (c1 + a) - c2 -> c3 + a // c1 - (c2 + a) -> c3 - a @@ -1069,6 +1059,24 @@ namespace graph { } } +// (a + b) - a -> b +// (a + b) - b -> a +// a - (a + b) -> -b +// b - (a + b) -> -a + if (la.get()) { + if (la->get_left()->is_match(this->right)) { + return la->get_right(); + } else if (la->get_right()->is_match(this->right)) { + return la->get_left(); + } + } else if (ra.get()) { + if (ra->get_left()->is_match(this->left)) { + return none ()*ra->get_right(); + } else if (ra->get_right()->is_match(this->left)) { + return none ()*ra->get_left(); + } + } + // Assume constants are on the left. // v1 - -c*v2 -> v1 + c*v2 if (rm.get() && @@ -1465,26 +1473,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -1901,11 +1905,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -1914,13 +1916,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -1929,29 +1931,29 @@ namespace graph { result.multiply_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.multiply_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.multiply_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.multiply_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // Move constants to the left. @@ -2472,6 +2474,96 @@ namespace graph { } } +// Sqrt(a)*Sqrt(b) -> Sqrt(a*b) + auto lsqr = sqrt_cast(this->left); + auto rsqr = sqrt_cast(this->right); + if (lsqr.get() && rsqr.get()) { + return sqrt(lsqr->get_arg()*rsqr->get_arg()); + } + + if constexpr (std::floating_point) { +// hypot(b,c)*Sqrt(a) -> Sqrt((b^2 + c^2)*a) +// Sqrt(a)*hypot(b,c) -> Sqrt(a*(b^2 + c^2)) + auto lhypot = hypot_cast(this->left); + auto rhypot = hypot_cast(this->right); + if (lhypot.get() && rsqr.get()) { + return sqrt((pow(lhypot->get_left(), static_cast (2)) + + pow(lhypot->get_right(), static_cast (2))) * + rsqr->get_arg()); + } else if (rhypot.get() && lsqr.get()) { + return sqrt((pow(rhypot->get_left(), static_cast (2)) + + pow(rhypot->get_right(), static_cast (2))) * + lsqr->get_arg()); + } + +// Sqrt(x^2)*copysign(1,x) -> x +// copysign(1,x)*Sqrt(x^2) -> x + auto lcsc = copysign_cast(this->left); + auto rcsc = copysign_cast(this->right); + if (rcsc.get() && lsqr.get()) { + auto rcsclc = constant_cast(rcsc->get_left()); + auto lsqrpc = pow_cast(lsqr->get_arg()); + if (lsqrpc.get() && rcsclc.get() && rcsclc->is(1) && + lsqrpc->get_left()->is_match(rcsc->get_right())) { + return rcsc->get_right(); + } + } else if (lcsc.get() && rsqr.get()) { + auto lcsclc = constant_cast(lcsc->get_left()); + auto rsqrpc = pow_cast(rsqr->get_arg()); + if (rsqrpc.get() && lcsclc.get() && lcsclc->is(1) && + rsqrpc->get_left()->is_match(lcsc->get_right())) { + return lcsc->get_right(); + } + } + } + +// (a + b/c)*c -> fma(a,c,b) +// c*(a + b/c) -> fma(a,c,b) +// (b/c + a)*c -> fma(a,c,b) +// c*(b/c + a) -> fma(a,c,b) + auto la = add_cast(this->left); + if (la.get()) { + auto lald = divide_cast(la->get_left()); + auto lard = divide_cast(la->get_right()); + if (lald.get() && lald->get_right()->is_match(this->right)) { + return fma(la->get_right(), this->right, lald->get_left()); + } else if (lard.get() && lard->get_right()->is_match(this->right)) { + return fma(la->get_left(), this->right, lard->get_left()); + } + } else if (ra.get()) { + auto rald = divide_cast(ra->get_left()); + auto rard = divide_cast(ra->get_right()); + if (rald.get() && rald->get_right()->is_match(this->left)) { + return fma(ra->get_right(), this->left, rald->get_left()); + } else if (rard.get() && rard->get_right()->is_match(this->left)) { + return fma(ra->get_left(), this->left, rard->get_left()); + } + } + +// (a - b/c)*c -> a*c - b +// c*(a - b/c) -> a*c - b +// (b/c - a)*c -> b - a*c +// c*(b/c - a) -> b - a*c + auto ls = subtract_cast(this->left); + auto rs = subtract_cast(this->right); + if (ls.get()) { + auto lsld = divide_cast(ls->get_left()); + auto lsrd = divide_cast(ls->get_right()); + if (lsld.get() && lsld->get_right()->is_match(this->right)) { + return lsld->get_left() - ls->get_right()*this->right; + } else if (lsrd.get() && lsrd->get_right()->is_match(this->right)) { + return ls->get_left()*this->right - lsrd->get_left(); + } + } else if (rs.get()) { + auto rsld = divide_cast(rs->get_left()); + auto rsrd = divide_cast(rs->get_right()); + if (rsld.get() && rsld->get_right()->is_match(this->left)) { + return rsld->get_right() - rs->get_right()*this->left; + } else if (rsrd.get() && rsrd->get_right()->is_match(this->left)) { + return rs->get_left()*this->left - rsld->get_left(); + } + } + // Cases like // (c/exp(a))*(exp(b)/d) -> (c/d)*(exp(b)/exp(a)) // (c/exp(a))*(d/exp(b)) -> (c*e)/(exp(b)*exp(a)) @@ -2506,26 +2598,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -2835,11 +2923,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -2848,13 +2934,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -2863,29 +2949,29 @@ namespace graph { result.divide_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.divide_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.divide_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.divide_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } if (this->left->is_match(this->right)) { @@ -3498,26 +3584,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -5036,6 +5118,49 @@ namespace graph { } } +// fma(sqrt(a),sqrt(b),c) -> sqrt(a*b) + c + auto lsqr = sqrt_cast(this->left); + auto msqr = sqrt_cast(this->middle); + if (lsqr.get() && msqr.get()) { + return sqrt(lsqr->get_arg()*msqr->get_arg()) + this->right; + } + + if constexpr (std::floating_point) { +// fma(hypot(b,c),Sqrt(a),d) -> Sqrt((b^2 + c^2)*a) + d +// fma(Sqrt(a),hypot(b,c),d) -> Sqrt(a*(b^2 + c^2)) + d + auto lhypot = hypot_cast(this->left); + auto mhypot = hypot_cast(this->middle); + if (lhypot.get() && msqr.get()) { + return sqrt((pow(lhypot->get_left(), static_cast (2)) + + pow(lhypot->get_right(), static_cast (2))) * + msqr->get_arg()) + this->right; + } else if (mhypot.get() && lsqr.get()) { + return sqrt((pow(mhypot->get_left(), static_cast (2)) + + pow(mhypot->get_right(), static_cast (2))) * + lsqr->get_arg()) + this->right; + } + +// fma(Sqrt(x^2),copysign(1,x),d) -> x +// fma(copysign(1,x),Sqrt(x^2),d) -> x + auto lcsc = copysign_cast(this->left); + auto mcsc = copysign_cast(this->middle); + if (mcsc.get() && lsqr.get()) { + auto mcsclc = constant_cast(mcsc->get_left()); + auto lsqrpc = pow_cast(lsqr->get_arg()); + if (lsqrpc.get() && mcsclc.get() && mcsclc->is(1) && + lsqrpc->get_left()->is_match(mcsc->get_right())) { + return mcsc->get_right() + this->right; + } + } else if (lcsc.get() && msqr.get()) { + auto lcsclc = constant_cast(lcsc->get_left()); + auto msqrpc = pow_cast(msqr->get_arg()); + if (msqrpc.get() && lcsclc.get() && lcsclc->is(1) && + msqrpc->get_left()->is_match(lcsc->get_right())) { + return lcsc->get_right() + this->right; + } + } + } + return this->shared_from_this(); } @@ -5069,30 +5194,24 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf m = this->middle->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto m = this->middle->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -5402,6 +5521,264 @@ namespace graph { shared_fma fma_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// Modulo node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An Modulo node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class modulo_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "%" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an modulo node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + modulo_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + modulo_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of modulo. +/// +/// result = l % r +/// +/// @returns The value of l % r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result % r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an modulo node. +/// +/// @returns A reduced modulo node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = fmod(" + << registers[l.get()] << "," + << registers[r.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\%"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"%\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf modulo(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(shared_leaf l, + shared_leaf r) { + return modulo (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(const L l, + shared_leaf r) { + return modulo (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(shared_leaf l, + const R r) { + return modulo (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared modulo nodes. + template + using shared_modulo = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a modulo node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_modulo modulo_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* arithmetic_h */ diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 39170d0..9246802 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -26,17 +26,17 @@ namespace backend { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template - class buffer { - private: -/// The data buffer to hold the data. - std::vector memory; - + class buffer : public std::vector { public: + using std::vector::size; + using std::vector::data; + using std::vector::assign; + //------------------------------------------------------------------------------ /// @brief Construct an empty buffer backend. //------------------------------------------------------------------------------ buffer() : - memory() {} + std::vector () {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend with a size. @@ -44,7 +44,7 @@ namespace backend { /// @param[in] s Size of he data buffer. //------------------------------------------------------------------------------ buffer(const size_t s) : - memory(s) {} + std::vector (s) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend with a size. @@ -53,7 +53,7 @@ namespace backend { /// @param[in] d Scalar data to initialize. //------------------------------------------------------------------------------ buffer(const size_t s, const T d) : - memory(s, d) {} + std::vector (s, d) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend from a vector. @@ -61,7 +61,7 @@ namespace backend { /// @param[in] d Array buffer. //------------------------------------------------------------------------------ buffer(const std::vector &d) : - memory(d) {} + std::vector (d) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend from a buffer backend. @@ -69,27 +69,19 @@ namespace backend { /// @param[in] d Backend buffer. //------------------------------------------------------------------------------ buffer(const buffer &d) : - memory(d.memory) {} - -//------------------------------------------------------------------------------ -/// @brief Index operator. -//------------------------------------------------------------------------------ - T &operator[] (const size_t index) { - return memory[index]; - } - -//------------------------------------------------------------------------------ -/// @brief Const index operator. -//------------------------------------------------------------------------------ - const T &operator[] (const size_t index) const { - return memory[index]; - } + std::vector (d) {} //------------------------------------------------------------------------------ -/// @brief Get value at. +/// @brief Construct a buffer backend linearly. +/// +/// @param[in] min Minimum value.. +/// @param[in] dx Step size. +/// @param[in] num Number of mesh points. //------------------------------------------------------------------------------ - const T at(const size_t index) const { - return memory.at(index); + buffer(const T min, const T dx, const size_t num) : std::vector (num) { + for (size_t i = 0; i < num; i++) { + (*this)[i] = dx*i + min; + } } //------------------------------------------------------------------------------ @@ -98,7 +90,7 @@ namespace backend { /// @param[in] d Scalar data to set. //------------------------------------------------------------------------------ void set(const T d) { - memory.assign(memory.size(), d); + assign(size(), d); } //------------------------------------------------------------------------------ @@ -107,14 +99,7 @@ namespace backend { /// @param[in] d Vector data to set. //------------------------------------------------------------------------------ void set(const std::vector &d) { - memory.assign(d.cbegin(), d.cend()); - } - -//------------------------------------------------------------------------------ -/// @brief Get size of the buffer. -//------------------------------------------------------------------------------ - size_t size() const { - return memory.size(); + assign(d.cbegin(), d.cend()); } //------------------------------------------------------------------------------ @@ -123,9 +108,9 @@ namespace backend { /// @returns Returns true if every element is the same. //------------------------------------------------------------------------------ bool is_same() const { - const T same = memory.at(0); - for (size_t i = 1, ie = memory.size(); i < ie; i++) { - if (memory.at(i) != same) { + const T same = (*this)[0]; + for (size_t i = 1, ie = size(); i < ie; i++) { + if ((*this)[i] != same) { return false; } } @@ -139,7 +124,7 @@ namespace backend { /// @returns Returns true if every element is zero. //------------------------------------------------------------------------------ bool is_zero() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d != static_cast (0.0)) { return false; } @@ -154,7 +139,7 @@ namespace backend { /// @returns Returns true if any element is zero. //------------------------------------------------------------------------------ bool has_zero() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d == static_cast (0.0)) { return true; } @@ -169,7 +154,7 @@ namespace backend { /// @returns Returns true if every element is negative. //------------------------------------------------------------------------------ bool is_negative() const { - for (const T &d : memory) { + for (const T &d : *this) { if (std::real(d) > std::real(static_cast (0.0))) { return false; } @@ -184,7 +169,7 @@ namespace backend { /// @returns Returns true if every element is negative. //------------------------------------------------------------------------------ bool is_even() const { - for (const T &d : memory) { + for (const T &d : *this) { if (std::fmod(std::real(d), std::real(static_cast (2.0)))) { return false; } @@ -199,7 +184,7 @@ namespace backend { /// @returns Returns true if every element is negative one. //------------------------------------------------------------------------------ bool is_none() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d != static_cast (-1.0)) { return false; } @@ -208,67 +193,70 @@ namespace backend { return true; } +//------------------------------------------------------------------------------ +/// @brief Applies an operation over all elements in the buffer. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define apply_op(op) \ +for (T &d : *this) { \ + d = op(d); \ +} + //------------------------------------------------------------------------------ /// @brief Take sqrt. //------------------------------------------------------------------------------ void sqrt() { - for (T &d : memory) { - d = std::sqrt(d); - } + apply_op(std::sqrt) } //------------------------------------------------------------------------------ /// @brief Take exp. //------------------------------------------------------------------------------ void exp() { - for (T &d : memory) { - d = std::exp(d); - } + apply_op(std::exp) } //------------------------------------------------------------------------------ /// @brief Take log. //------------------------------------------------------------------------------ void log() { - for (T &d : memory) { - d = std::log(d); - } + apply_op(std::log) } //------------------------------------------------------------------------------ /// @brief Take sin. //------------------------------------------------------------------------------ void sin() { - for (T &d : memory) { - d = std::sin(d); - } + apply_op(std::sin) } //------------------------------------------------------------------------------ /// @brief Take cos. //------------------------------------------------------------------------------ void cos() { - for (T &d : memory) { - d = std::cos(d); - } + apply_op(std::cos) } //------------------------------------------------------------------------------ -/// @brief Take erfi. +/// @brief Take cos. //------------------------------------------------------------------------------ - void erfi() requires(jit::complex_scalar) { - for (T &d : memory) { - d = special::erfi(d); - } + void real() { + apply_op(std::real) } //------------------------------------------------------------------------------ -/// @brief Get a pointer to the basic memory buffer. -/// -/// @returns The pointer to the buffer memory. +/// @brief Take erf. +//------------------------------------------------------------------------------ + void erf() requires(std::floating_point) { + apply_op(std::erf) + } + +//------------------------------------------------------------------------------ +/// @brief Take erfi. //------------------------------------------------------------------------------ - T *data() { - return memory.data(); + void erfi() requires(jit::complex_scalar) { + apply_op(special::erfi) } //------------------------------------------------------------------------------ @@ -277,7 +265,7 @@ namespace backend { /// @returns False if any NaN or Inf is found. //------------------------------------------------------------------------------ bool is_normal() const { - for (const T &x : memory) { + for (const T &x : *this) { if constexpr (jit::complex_scalar) { if (std::isnan(std::real(x)) || std::isinf(std::real(x)) || std::isnan(std::imag(x)) || std::isinf(std::imag(x))) { @@ -303,7 +291,7 @@ namespace backend { buffer b(num_columns); const size_t num_rows = size()/num_columns; for (size_t j = 0; j < num_columns; j++) { - b[j] = memory[index*num_rows + j]; + b[j] = (*this)[index*num_rows + j]; } return b; } @@ -319,11 +307,44 @@ namespace backend { const size_t num_rows = size()/num_columns; buffer b(num_rows); for (size_t i = 0; i < num_rows; i++) { - b[i] = memory[i*num_rows + index]; + b[i] = (*this)[i*num_rows + index]; } return b; } +//------------------------------------------------------------------------------ +/// @brief Applies an operator along a row. +/// +/// @param opp The operation to apply. +/// @param oppeq The assignment operator to apply. +//------------------------------------------------------------------------------ +#define row_op(opp, oppeq) \ +if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] oppeq x[i]; \ + } \ + } \ +} else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = (*this)[i] opp x[i*num_columns + j]; \ + } \ + } \ + *this = m; \ +} + //------------------------------------------------------------------------------ /// @brief Add row operation. /// @@ -333,33 +354,42 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void add_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_rows + j] += x[i]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i] + x[i*num_columns + j]; - } - } - memory = m; - } + row_op(+, +=) } +//------------------------------------------------------------------------------ +/// @brief Applies an operator along a column. +/// +/// @param opp The operation to apply. +/// @param oppeq The assignment operator to apply. +//------------------------------------------------------------------------------ +#define col_op(opp, oppeq) \ +if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] oppeq x[j]; \ + } \ + } \ +} else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = (*this)[j] opp x[i*num_columns + j]; \ + } \ + } \ + *this = m; \ +} + //------------------------------------------------------------------------------ /// @brief Add col operation. /// @@ -369,31 +399,7 @@ namespace backend { /// @param[in] x The other operand. //------------------------------------------------------------------------------ void add_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] += x[j]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j] + x[i*num_columns + j]; - } - } - memory = m; - } + col_op(+, +=) } //------------------------------------------------------------------------------ @@ -405,6 +411,166 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void subtract_row(const buffer &x) { + row_op(-, -=) + } + +//------------------------------------------------------------------------------ +/// @brief Subtract col operation. +/// +/// Subtracts m_ij - v_j or v_j - m_ij. This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void subtract_col(const buffer &x) { + col_op(-, -=) + } + +//------------------------------------------------------------------------------ +/// @brief Multiply row operation. +/// +/// Multiplies m_ij * v_i or v_i * m_ij. This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The right operand. +//------------------------------------------------------------------------------ + void multiply_row(const buffer &x) { + row_op(*, *=) + } + +//------------------------------------------------------------------------------ +/// @brief Multiply col operation. +/// +/// Multiplies m_ij * v_j or v_j * m_ij. This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void multiply_col(const buffer &x) { + col_op(*, *=) + } + +//------------------------------------------------------------------------------ +/// @brief Divide row operation. +/// +/// Divides m_ij / v_i or v_i / m_ij. This will resize the buffer if it needs +/// to be. +/// +/// @param[in] x The right operand. +//------------------------------------------------------------------------------ + void divide_row(const buffer &x) { + row_op(/, /=) + } + +//------------------------------------------------------------------------------ +/// @brief Divide col operation. +/// +/// Divides m_ij / v_j or v_j / m_ij. This will resize the buffer if it needs +/// to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void divide_col(const buffer &x) { + col_op(/, /=) + } + +//------------------------------------------------------------------------------ +/// @brief Applies a function along a row. +/// +/// @param fn The function to apply. +//------------------------------------------------------------------------------ + #define row_fn(fn) \ + if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] = fn((*this)[i*num_columns + j], x[i]); \ + } \ + } \ + } else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = fn((*this)[i], x[i*num_columns + j]); \ + } \ + } \ + *this = m; \ + } + +//------------------------------------------------------------------------------ +/// @brief Min row operation. +/// +/// Takes Min(m_ij, v_i) or Min(v_i, m_ij). This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void min_row(const buffer &x) { + row_fn(std::min) + } + +//------------------------------------------------------------------------------ +/// @brief Applies a function along a column. +/// +/// @param fn The function to apply. +//------------------------------------------------------------------------------ + #define col_fn(fn) \ + if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] = fn((*this)[i*num_columns + j], x[j]); \ + } \ + } \ + } else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = fn((*this)[j], x[i*num_columns + j]); \ + } \ + } \ + *this = m; \ + } + +//------------------------------------------------------------------------------ +/// @brief Min col operation. +/// +/// Takes Min(m_ij, v_j) or Min(v_j, m_ij). This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void min_col(const buffer &x) { + col_fn(std::min) + } + +//------------------------------------------------------------------------------ +/// @brief Atan row operation. +/// +/// Computes atan(m_ij, v_i) or atan(v_i, m_ij). This will resize the buffer if +/// it needs to be. +/// +/// @param[in] x The right operand. +//------------------------------------------------------------------------------ + void atan_row(const buffer &x) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -413,7 +579,11 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] -= x[i]; + if constexpr (jit::complex_scalar) { + (*this)[i*num_columns + j] = std::atan(x[i]/(*this)[i*num_columns + j]); + } else { + (*this)[i*num_columns + j] = std::atan2(x[i], (*this)[i*num_columns + j]); + } } } } else { @@ -423,33 +593,41 @@ namespace backend { std::vector m(x.size()); const size_t num_columns = x.size()/size(); const size_t num_rows = size(); - for (size_t i = 0; i < num_columns; i++) { - for (size_t j = 0; j < num_rows; j++) { - m[i*num_columns + j] = memory[i] - x[i*num_columns + j]; + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + if constexpr (jit::complex_scalar) { + m[i*num_columns + j] = std::atan(x[i*num_columns + j]/(*this)[i]); + } else { + m[i*num_columns + j] = std::atan2(x[i*num_columns + j], (*this)[i]); + } } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Subtract col operation. +/// @brief Atan col operation. /// -/// Subtracts m_ij - v_j or v_j - m_ij. This will resize the buffer if it -/// needs to be. +/// Computes atan(m_ij, v_j) or atan(v_j, m_ij). This will resize the buffer if +/// it needs to be. /// /// @param[in] x The other operand. //------------------------------------------------------------------------------ - void subtract_col(const buffer &x) { + void atan_col(const buffer &x) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); const size_t num_columns = size()/x.size(); const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] -= x[j]; + for (size_t i = 0; i < num_columns; i++) { + for (size_t j = 0; j < num_rows; j++) { + if constexpr (jit::complex_scalar) { + (*this)[i*num_columns + j] = std::atan(x[j]/(*this)[i*num_columns + j]); + } else { + (*this)[i*num_columns + j] = std::atan2(x[j], (*this)[i*num_columns + j]); + } } } } else { @@ -461,22 +639,26 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j] - x[i*num_columns + j]; + if constexpr (jit::complex_scalar) { + m[i*num_columns + j] = std::atan(x[i*num_columns + j]/(*this)[j]); + } else { + m[i*num_columns + j] = std::atan2(x[i*num_columns + j], (*this)[j]); + } } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Multiply row operation. +/// @brief Pow row operation. /// -/// Multiplies m_ij * v_i or v_i * m_ij. This will resize the buffer if it -/// needs to be. +/// Computes pow(m_ij, v_i) or pow(v_i, m_ij). This will resize the buffer if +/// it needs to be. /// /// @param[in] x The right operand. //------------------------------------------------------------------------------ - void multiply_row(const buffer &x) { + void pow_row(const buffer &x) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -485,7 +667,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] *= x[i]; + (*this)[i*num_columns + j] = std::pow((*this)[i*num_columns + j], x[i]); } } } else { @@ -495,24 +677,24 @@ namespace backend { std::vector m(x.size()); const size_t num_columns = x.size()/size(); const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i]*x[i*num_columns + j]; + for (size_t i = 0; i < num_columns; i++) { + for (size_t j = 0; j < num_rows; j++) { + m[i*num_columns + j] = std::pow((*this)[i], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Multiply col operation. +/// @brief Pow col operation. /// -/// Multiplies m_ij * v_j or v_j * m_ij. This will resize the buffer if it -/// needs to be. +/// Computes pow(m_ij, v_j) or pow(v_j, m_ij). This will resize the buffer if +/// it needs to be. /// /// @param[in] x The other operand. //------------------------------------------------------------------------------ - void multiply_col(const buffer &x) { + void pow_col(const buffer &x) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -521,7 +703,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] *= x[j]; + (*this)[i*num_columns + j] = std::pow((*this)[i*num_columns + j], x[j]); } } } else { @@ -533,22 +715,22 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j]*x[i*num_columns + j]; + m[i*num_columns + j] = std::pow((*this)[j], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Divide row operation. +/// @brief Hypot row operation. /// -/// Divides m_ij / v_i or v_i / m_ij. This will resize the buffer if it needs -/// to be. +/// Computes Hypot(m_ij, v_i) or Hypot(v_i, m_ij). This will resize the buffer +/// if it needs to be. /// /// @param[in] x The right operand. //------------------------------------------------------------------------------ - void divide_row(const buffer &x) { + void hypot_row(const buffer &x) requires(std::floating_point) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -557,7 +739,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] /= x[i]; + (*this)[i*num_columns + j] = std::hypot((*this)[i*num_columns + j], x[i]); } } } else { @@ -567,24 +749,24 @@ namespace backend { std::vector m(x.size()); const size_t num_columns = x.size()/size(); const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i]/x[i*num_columns + j]; + for (size_t i = 0; i < num_columns; i++) { + for (size_t j = 0; j < num_rows; j++) { + m[i*num_columns + j] = std::hypot((*this)[i], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Divide col operation. +/// @brief Hypot col operation. /// -/// Divides m_ij / v_j or v_j / m_ij. This will resize the buffer if it needs -/// to be. +/// Computes Hypot(m_ij, v_j) or Hypot(v_j, m_ij). This will resize the buffer +/// if it needs to be. /// /// @param[in] x The other operand. //------------------------------------------------------------------------------ - void divide_col(const buffer &x) { + void hypot_col(const buffer &x) requires(std::floating_point) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -593,7 +775,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] /= x[j]; + (*this)[i*num_columns + j] = std::hypot((*this)[i*num_columns + j], x[j]); } } } else { @@ -605,22 +787,22 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j]/x[i*num_columns + j]; + m[i*num_columns + j] = std::hypot((*this)[j], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Atan row operation. +/// @brief copysign row operation. /// -/// Computes atan(m_ij, v_i) or atan(v_i, m_ij). This will resize the buffer if -/// it needs to be. +/// Computes copysign(m_ij, v_i) or copysign(v_i, m_ij). This will resize the +/// buffer if it needs to be. /// /// @param[in] x The right operand. //------------------------------------------------------------------------------ - void atan_row(const buffer &x) { + void copysign_row(const buffer &x) requires(std::floating_point) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -629,11 +811,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - if constexpr (jit::complex_scalar) { - memory[i*num_columns + j] = std::atan(x[i]/memory[i*num_columns + j]); - } else { - memory[i*num_columns + j] = std::atan2(x[i], memory[i*num_columns + j]); - } + (*this)[i*num_columns + j] = std::copysign((*this)[i*num_columns + j], x[i]); } } } else { @@ -645,26 +823,22 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - if constexpr (jit::complex_scalar) { - m[i*num_columns + j] = std::atan(x[i*num_columns + j]/memory[i]); - } else { - m[i*num_columns + j] = std::atan2(x[i*num_columns + j], memory[i]); - } + m[i*num_columns + j] = std::copysign((*this)[i], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Atan col operation. +/// @brief copysign col operation. /// /// Computes atan(m_ij, v_j) or atan(v_j, m_ij). This will resize the buffer if /// it needs to be. /// /// @param[in] x The other operand. //------------------------------------------------------------------------------ - void atan_col(const buffer &x) { + void copysign_col(const buffer &x) { if (size() > x.size()) { assert(size()%x.size() == 0 && "Vector operand size is not a multiple of matrix operand size"); @@ -673,11 +847,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_columns; i++) { for (size_t j = 0; j < num_rows; j++) { - if constexpr (jit::complex_scalar) { - memory[i*num_columns + j] = std::atan(x[j]/memory[i*num_columns + j]); - } else { - memory[i*num_columns + j] = std::atan2(x[j], memory[i*num_columns + j]); - } + (*this)[i*num_columns + j] = std::copysign((*this)[i*num_columns + j], x[j]); } } } else { @@ -689,151 +859,269 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - if constexpr (jit::complex_scalar) { - m[i*num_columns + j] = std::atan(x[i*num_columns + j]/memory[j]); - } else { - m[i*num_columns + j] = std::atan2(x[i*num_columns + j], memory[j]); - } + m[i*num_columns + j] = std::copysign((*this)[j], x[i*num_columns + j]); } } - memory = m; + *this = m; } } //------------------------------------------------------------------------------ -/// @brief Pow row operation. -/// -/// Computes pow(m_ij, v_i) or pow(v_i, m_ij). This will resize the buffer if -/// it needs to be. +/// @brief Not operation. /// -/// @param[in] x The right operand. +/// @returns The negation of the buffer. //------------------------------------------------------------------------------ - void pow_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] = std::pow(memory[i*num_columns + j], x[i]); - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_columns; i++) { - for (size_t j = 0; j < num_rows; j++) { - m[i*num_columns + j] = std::pow(memory[i], x[i*num_columns + j]); - } + buffer operator!() { + for (T &d : *this) { + if constexpr (jit::complex_scalar) { + assert(d.imag() == 0.0 && "Imaginary part not zero."); + d = static_cast (!d.real()); + } else { + d = !d; } - memory = m; } + return *this; } //------------------------------------------------------------------------------ -/// @brief Pow col operation. -/// -/// Computes pow(m_ij, v_j) or pow(v_j, m_ij). This will resize the buffer if -/// it needs to be. +/// @brief Apply condition. /// -/// @param[in] x The other operand. +/// @params[in] t True condition. +/// @params[in] f False condition. //------------------------------------------------------------------------------ - void pow_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] = std::pow(memory[i*num_columns + j], x[j]); - } + buffer if_(const buffer &t, const buffer &f) { + if (size() == 1) { + if constexpr (std::floating_point) { + return (*this)[0] ? t : f; + } else { + return (*this)[0] != static_cast (0) ? t : f; } } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = std::pow(memory[j], x[i*num_columns + j]); + if (t.size() == 1) { + if (f.size() == 1) { + for (T &d : *this) { + if constexpr (std::floating_point) { + d = d ? t[0] : f[0]; + } else { + d = d != static_cast (0) ? t[0] : f[0]; + } + } + return *this; + } else { + assert(size() == f.size() && "Incompatable buffersize."); + for (size_t i = 0, ie = size(); i < ie; i++) { + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[0] : f[i]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[0] : f[i]; + } + } + return *this; + } + } else { + assert(size() == t.size() && "Incompatable buffersize."); + if (f.size() == 1) { + for (size_t i = 0, ie = size(); i < ie; i++) { + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[i] : f[0]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[i] : f[0]; + } + } + return *this; + } else { + assert(size() == f.size() && "Incompatable buffersize."); + for (size_t i = 0, ie = size(); i < ie; i++) { + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[i] : f[i]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[i] : f[i]; + } + } + return *this; } } - memory = m; } } +//------------------------------------------------------------------------------ +/// @brief Applies a logical is operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define logic_is(op) \ +for (T &d : *this) { \ + d = std::op(d); \ +} + +//------------------------------------------------------------------------------ +/// @brief isinf query. +//------------------------------------------------------------------------------ + void isinf() { + logic_is(isinf) + } + +//------------------------------------------------------------------------------ +/// @brief isnan query. +//------------------------------------------------------------------------------ + void isnan() { + logic_is(isnan) + } + /// Type def to retrieve the backend T type. typedef T base; }; //------------------------------------------------------------------------------ -/// @brief Add operation. +/// @brief Equal operation. /// /// @tparam T Base type of the calculation. /// /// @param[in] a Left operand. /// @param[in] b Right operand. -/// @returns a + b. +/// @returns a == b. //------------------------------------------------------------------------------ template - inline buffer operator+(buffer &a, - buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] += right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] += left; - } - return b; + inline bool operator==(const buffer &a, const buffer &b) { + if (a.size() != b.size()) { + return false; } - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] += b.at(i); + if (a[i] != b[i]) { + return false; + } } - return a; + return true; } //------------------------------------------------------------------------------ -/// @brief Equal operation. +/// @brief Applies an associative function. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define build_assoc_func(func) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l = func(std::real(l), \ + std::real(right)); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = func(std::real(r), \ + std::real(left)); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = func(std::real(a[i]), \ + std::real(b[i])); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Max operation. /// /// @tparam T Base type of the calculation. /// /// @param[in] a Left operand. /// @param[in] b Right operand. -/// @returns a == b. +/// @returns max(a, b). //------------------------------------------------------------------------------ template - inline bool operator==(const buffer &a, - const buffer &b) { - if (a.size() != b.size()) { - return false; - } + inline buffer max(buffer &a, buffer &b) { + build_assoc_func(std::max); + } - for (size_t i = 0, ie = a.size(); i < ie; i++) { - if (a.at(i) != b.at(i)) { - return false; - } - } - return true; +//------------------------------------------------------------------------------ +/// @brief Min operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns min(a, b). +//------------------------------------------------------------------------------ + template + inline buffer min(buffer &a, buffer &b) { + build_assoc_func(std::min); } +//------------------------------------------------------------------------------ +/// @brief Applies an associative operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define build_assoc_op(op) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l op right; \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r op left; \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] op b[i]; \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Add operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns a + b. +//------------------------------------------------------------------------------ + template + inline buffer operator+(buffer &a, buffer &b) { + build_assoc_op(+=) + } + +//------------------------------------------------------------------------------ +/// @brief Applies a non-associative operator. +/// +/// @param op The operation to apply. +/// @param opeq The assign operation to apply. +//------------------------------------------------------------------------------ +#define build_non_assoc_op(op, opeq) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l opeq right; \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = left op r; \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] opeq b[i]; \ +} \ +return a; + //------------------------------------------------------------------------------ /// @brief Subtract operation. /// @@ -844,28 +1132,8 @@ namespace backend { /// @returns a - b. //------------------------------------------------------------------------------ template - inline buffer operator-(buffer &a, - buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] -= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = left - b.at(i); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] -= b.at(i); - } - return a; + inline buffer operator-(buffer &a, buffer &b) { + build_non_assoc_op(-, -=) } //------------------------------------------------------------------------------ @@ -878,28 +1146,8 @@ namespace backend { /// @returns a * b. //------------------------------------------------------------------------------ template - inline buffer operator*(buffer &a, - buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] *= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] *= left; - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] *= b.at(i); - } - return a; + inline buffer operator*(buffer &a, buffer &b) { + build_assoc_op(*=) } //------------------------------------------------------------------------------ @@ -912,28 +1160,49 @@ namespace backend { /// @returns a / b. //------------------------------------------------------------------------------ template - inline buffer operator/(buffer &a, - buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] /= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = left/b.at(i); - } - return b; - } + inline buffer operator/(buffer &a, buffer &b) { + build_non_assoc_op(/, /=) + } - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] /= b.at(i); - } - return a; +//------------------------------------------------------------------------------ +/// @brief Applies an associative function. +/// +/// @param func The function to apply. +//------------------------------------------------------------------------------ +#define build_assoc_func(func) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l = func(l, right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = func(r, left); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = func(a[i], b[i]); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief hypot operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns hypot(a,b) +//------------------------------------------------------------------------------ + template + inline buffer hypot(buffer &a, buffer &b) { + build_assoc_func(std::hypot) } //------------------------------------------------------------------------------ @@ -947,9 +1216,7 @@ namespace backend { /// @returns a*b + c. //------------------------------------------------------------------------------ template - inline buffer fma(buffer &a, - buffer &b, - buffer &c) { + inline buffer fma(buffer &a, buffer &b, buffer &c) { constexpr bool use_fma = !jit::complex_scalar && #ifdef FP_FAST_FMA true; @@ -958,25 +1225,25 @@ namespace backend { #endif if (a.size() == 1) { - const T left = a.at(0); + const T left = a[0]; if (b.size() == 1) { - const T middle = b.at(0); + const T middle = b[0]; for (size_t i = 0, ie = c.size(); i < ie; i++) { if constexpr (use_fma) { - c[i] = std::fma(left, middle, c.at(i)); + c[i] = std::fma(left, middle, c[i]); } else { - c[i] = left*middle + c.at(i); + c[i] = left*middle + c[i]; } } return c; } else if (c.size() == 1) { - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = b.size(); i < ie; i++) { if constexpr (use_fma) { - b[i] = std::fma(left, b.at(i), right); + b[i] = std::fma(left, b[i], right); } else { - b[i] = left*b.at(i) + right; + b[i] = left*b[i] + right; } } return b; @@ -986,21 +1253,21 @@ namespace backend { "Size mismatch between middle and right."); for (size_t i = 0, ie = b.size(); i < ie; i++) { if constexpr (use_fma) { - b[i] = std::fma(left, b.at(i), c.at(i)); + b[i] = std::fma(left, b[i], c[i]); } else { - b[i] = left*b.at(i) + c.at(i); + b[i] = left*b[i] + c[i]; } } return b; } else if (b.size() == 1) { - const T middle = b.at(0); + const T middle = b[0]; if (c.size() == 1) { - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), middle, right); + a[i] = std::fma(a[i], middle, right); } else { - a[i] = a.at(i)*middle + right; + a[i] = a[i]*middle + right; } } return a; @@ -1010,21 +1277,21 @@ namespace backend { "Size mismatch between left and right."); for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), middle, c.at(i)); + a[i] = std::fma(a[i], middle, c[i]); } else { - a[i] = a.at(i)*middle + c.at(i); + a[i] = a[i]*middle + c[i]; } } return a; } else if (c.size() == 1) { assert(a.size() == b.size() && "Size mismatch between left and middle."); - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), b.at(i), right); + a[i] = std::fma(a[i], b[i], right); } else { - a[i] = a.at(i)*b.at(i) + right; + a[i] = a[i]*b[i] + right; } } return a; @@ -1036,14 +1303,227 @@ namespace backend { "Left, middle and right sizes are incompatible."); for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), b.at(i), c.at(i)); + a[i] = std::fma(a[i], b[i], c[i]); } else { - a[i] = a.at(i)*b.at(i) + c.at(i); + a[i] = a[i]*b[i] + c[i]; } } return a; } +//------------------------------------------------------------------------------ +/// @brief Modulo operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a % b. +//------------------------------------------------------------------------------ + template + inline buffer operator%(buffer &a, buffer &b) { + if (b.size() == 1) { + const T right = b[0]; + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = std::fmod(a[i], right); + } + return a; + } else if (a.size() == 1) { + const T left = a[0]; + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = std::fmod(left, b[i]); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = std::fmod(a[i], b[i]); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Applies a logical operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define logic_op(op) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = static_cast (a[i] op right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (size_t i = 0, ie = b.size(); i < ie; i++) { \ + b[i] = static_cast (left op b[i]); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = static_cast (a[i] op b[i]); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a == b. +//------------------------------------------------------------------------------ + template + inline buffer operator==(buffer &a, buffer &b) { + logic_op(==) + } + +//------------------------------------------------------------------------------ +/// @brief Not equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a == b. +//------------------------------------------------------------------------------ + template + inline buffer operator!=(buffer &a, buffer &b) { + logic_op(!=) + } + +//------------------------------------------------------------------------------ +/// @brief Greater than operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a > b. +//------------------------------------------------------------------------------ + template + inline buffer operator>(buffer &a, buffer &b) { + logic_op(>) + } + +//------------------------------------------------------------------------------ +/// @brief Less than operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a < b. +//------------------------------------------------------------------------------ + template + inline buffer operator<(buffer &a, buffer &b) { + logic_op(<) + } + +//------------------------------------------------------------------------------ +/// @brief Greater than equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a >= b. +//------------------------------------------------------------------------------ + template + inline buffer operator>=(buffer &a, buffer &b) { + logic_op(>=) + } + +//------------------------------------------------------------------------------ +/// @brief Less than equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a <= b. +//------------------------------------------------------------------------------ + template + inline buffer operator<=(buffer &a, buffer &b) { + logic_op(<=) + } + +//------------------------------------------------------------------------------ +/// @brief And operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a && b. +//------------------------------------------------------------------------------ + template + inline buffer operator&&(buffer &a, buffer &b) { + logic_op(&&) + } + +//------------------------------------------------------------------------------ +/// @brief Or operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a || b. +//------------------------------------------------------------------------------ + template + inline buffer operator||(buffer &a, buffer &b) { + logic_op(||) + } + +//------------------------------------------------------------------------------ +/// @brief Applies a function with two operands. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define branch_fn(fn) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = fn(a[i], right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (size_t i = 0, ie = b.size(); i < ie; i++) { \ + b[i] = fn(left, b[i]); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = fn(a[i], b[i]); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Min operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns min(a, b). +//------------------------------------------------------------------------------ + template + inline buffer min(buffer &a, buffer &b) { + branch_fn(std::min) + } + //------------------------------------------------------------------------------ /// @brief Take the power. /// @@ -1054,10 +1534,9 @@ namespace backend { /// @returns base^exponent. //------------------------------------------------------------------------------ template - inline buffer pow(buffer &base, - buffer &exponent) { + inline buffer pow(buffer &base, buffer &exponent) { if (exponent.size() == 1) { - const T right = exponent.at(0); + const T right = exponent[0]; if (std::imag(right) == 0) { const int64_t right_int = static_cast (std::real(right)); if (std::real(right) - right_int) { @@ -1067,14 +1546,14 @@ namespace backend { } for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), right); + base[i] = std::pow(base[i], right); } return base; } if (right_int > 0) { for (size_t i = 0, ie = base.size(); i < ie; i++) { - const T left = base.at(i); + const T left = base[i]; for (size_t j = 0, je = right_int - 1; j < je; j++) { base[i] *= left; } @@ -1087,7 +1566,7 @@ namespace backend { return base; } else { for (size_t i = 0, ie = base.size(); i < ie; i++) { - const T left = static_cast (1.0)/base.at(i); + const T left = static_cast (1.0)/base[i]; base[i] = left; for (size_t j = 0, je = std::abs(right_int) - 1; j < je; j++) { base[i] *= left; @@ -1097,14 +1576,14 @@ namespace backend { } } else { for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), right); + base[i] = std::pow(base[i], right); } return base; } } else if (base.size() == 1) { - const T left = base.at(0); + const T left = base[0]; for (size_t i = 0, ie = exponent.size(); i < ie; i++) { - exponent[i] = std::pow(left, exponent.at(i)); + exponent[i] = std::pow(left, exponent[i]); } return exponent; } @@ -1112,7 +1591,7 @@ namespace backend { assert(base.size() == exponent.size() && "Left and right sizes are incompatible."); for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), exponent.at(i)); + base[i] = std::pow(base[i], exponent[i]); } return base; } @@ -1127,10 +1606,9 @@ namespace backend { /// @returns atan2(y, x) //------------------------------------------------------------------------------ template - inline buffer atan(buffer &x, - buffer &y) { + inline buffer atan(buffer &x, buffer &y) { if (y.size() == 1) { - const T right = y.at(0); + const T right = y[0]; for (size_t i = 0, ie = x.size(); i < ie; i++) { if constexpr (jit::complex_scalar) { x[i] = std::atan(right/x[i]); @@ -1140,7 +1618,7 @@ namespace backend { } return x; } else if (x.size() == 1) { - const T left = x.at(0); + const T left = x[0]; for (size_t i = 0, ie = y.size(); i < ie; i++) { if constexpr (jit::complex_scalar) { y[i] = std::atan(y[i]/left); @@ -1162,6 +1640,40 @@ namespace backend { } return x; } + +//------------------------------------------------------------------------------ +/// @brief Copy the sign of x and apply it to y. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x X argument. +/// @param[in] y Y argument. +/// @returns copysign(x, y) +//------------------------------------------------------------------------------ + template + inline buffer copysign(buffer &x, + buffer &y) { + if (y.size() == 1) { + const T right = y[0]; + for (size_t i = 0, ie = x.size(); i < ie; i++) { + x[i] = std::copysign(x[i], right); + } + return x; + } else if (x.size() == 1) { + const T left = x[0]; + for (size_t i = 0, ie = y.size(); i < ie; i++) { + y[i] = std::copysign(left, y[i]); + } + return y; + } + + assert(x.size() == y.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = x.size(); i < ie; i++) { + x[i] = std::copysign(x[i], y[i]); + } + return x; + } } #endif /* backend_h */ diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index d0b48db..04fc05b 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -12,7 +12,6 @@ #include #include #include -#include // Clang headers will define IBAction and IBOutlet these so undefined them // here. @@ -38,6 +37,8 @@ #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "random.hpp" +#include "piecewise.hpp" +#include "timing.hpp" #ifndef NDEBUG //------------------------------------------------------------------------------ @@ -84,11 +85,11 @@ namespace gpu { /// Handle for the dynamic library. std::unique_ptr jit; /// Argument map. - std::map *, std::vector> kernel_arguments; + std::unordered_map *, std::vector> kernel_arguments; /// Host buffer map. - std::map *, std::vector> host_buffers; + std::unordered_map *, std::vector> host_buffers; /// Argument index map. - std::map *, size_t> arg_index; + std::unordered_map *, size_t> arg_index; public: /// Size of random state needed. @@ -224,6 +225,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @param[in] tex1d_list List of 1D textures. @@ -233,33 +235,43 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, const jit::texture2d_list &tex2d_list) { auto entry = std::move(jit->lookup(kernel_name)).get(); - std::map buffers; + std::unordered_map buffers; for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { - backend::buffer buffer = input->evaluate(); - std::vector arg(buffer.size()); - memcpy(arg.data(), buffer.data(), buffer.size()*sizeof(T)); + std::vector arg(input->size()); + memcpy(arg.data(), input->data(), input->size()*sizeof(T)); kernel_arguments[input.get()] = arg; } buffers[reinterpret_cast (input.get())] = kernel_arguments[input.get()].data(); } for (auto &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - std::vector arg(num_rays); - kernel_arguments[output.get()] = arg; + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + std::vector arg(num_rays); + kernel_arguments[output.get()] = arg; + } + buffers[reinterpret_cast (output.get())] = kernel_arguments[output.get()].data(); + } + } + for (auto &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + std::vector arg(atomic->size()); + memcpy(arg.data(), atomic->data(), atomic->size()*sizeof(T)); + kernel_arguments[atomic.get()] = arg; } - buffers[reinterpret_cast (output.get())] = kernel_arguments[output.get()].data(); + buffers[reinterpret_cast (atomic.get())] = kernel_arguments[atomic.get()].data(); } if (state.get()) { - auto kernel = entry.toPtr &, typename graph::random_state_node::mt_state *)> (); + auto kernel = entry.toPtr &, typename graph::random_state_node::mt_state *)> (); if (!kernel) { std::cerr << "Failed to load function. " << kernel_name @@ -273,11 +285,21 @@ namespace gpu { << std::endl; } - return [kernel, buffers, state] () mutable { + return [kernel, buffers, state +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer(kernel_name); +#endif kernel(buffers, state->data()); +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } else { - auto kernel = entry.toPtr &)> (); + auto kernel = entry.toPtr &)> (); if (!kernel) { std::cerr << "Failed to load function. " << kernel_name @@ -291,8 +313,18 @@ namespace gpu { << std::endl; } - return [kernel, buffers] () mutable { + return [kernel, buffers +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer(kernel_name); +#endif kernel(buffers); +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } } @@ -302,6 +334,7 @@ namespace gpu { /// /// @param[in] argument Node to reduce. /// @param[in] run Function to run before reduction. +/// @returns A lambda function to run the kernel. //------------------------------------------------------------------------------ std::function create_max_call(graph::shared_leaf &argument, std::function run) { @@ -321,6 +354,80 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will be memset a buffer to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector buffers; + std::vector sizes; + + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + std::vector arg(input->size()); + memcpy(arg.data(), input->data(), input->size()*sizeof(T)); + kernel_arguments[input.get()] = arg; + } + buffers.push_back(kernel_arguments[input.get()].data()); + sizes.push_back(input->size()*sizeof(T)); + } + + return [buffers, sizes] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("zero buffer"); +#endif + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + std::memset(buffers[i], 0, sizes[i]); + } +#ifdef PROFILE_KERNELS + timer.print(); +#endif + }; + } + +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector sources; + std::vector destinations; + std::vector sizes; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + std::vector arg(in->size()); + memcpy(arg.data(), in->data(), in->size()*sizeof(T)); + kernel_arguments[in.get()] = arg; + } + destinations.push_back(kernel_arguments[in.get()].data()); + sizes.push_back(in->size()*sizeof(T)); + + if (!kernel_arguments.contains(out.get())) { + std::vector arg(out->size()); + memcpy(arg.data(), out->data(), out->size()*sizeof(T)); + kernel_arguments[out.get()] = arg; + } + sources.push_back(kernel_arguments[out.get()].data()); + } + + return [sources, destinations, sizes] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("copy buffer"); +#endif + for (size_t i = 0, ie = sources.size(); i < ie; i++) { + std::memcpy(destinations[i], sources[i], sizes[i]); + } +#ifdef PROFILE_KERNELS + timer.print(); +#endif + }; + } + //------------------------------------------------------------------------------ /// @brief Hold the current thread until the command buffer has completed. /// @@ -335,6 +442,24 @@ namespace gpu { } } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { +#ifdef PROFILE_KERNELS + return [callback]() { + timing::measure_diagnostic timer("callback"); + callback(); + timer.print(); + }; +#else + return callback; +#endif + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// @@ -398,16 +523,18 @@ namespace gpu { /// @param[in,out] source_buffer Source buffer stream. //------------------------------------------------------------------------------ void create_header(std::ostringstream &source_buffer) { - source_buffer << "#include " << std::endl - << "#include " << std::endl - << "#include " << std::endl; + source_buffer << "#include " << std::endl + << "#include " << std::endl + << "#include " << std::endl + << "#include " << std::endl; if (jit::complex_scalar) { source_buffer << "#include " << std::endl; source_buffer << "#include " << std::endl; } else { source_buffer << "#include " << std::endl; } - source_buffer << "using namespace std;" << std::endl; + source_buffer << "#include " << std::endl + << "using namespace std;" << std::endl; } //------------------------------------------------------------------------------ @@ -417,6 +544,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -424,22 +552,29 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t size, const std::vector &is_constant, jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" void " << name << "(" << std::endl; - source_buffer << " map (source_buffer); source_buffer << " *> &args"; if (state.get()) { @@ -448,7 +583,7 @@ namespace gpu { } source_buffer << ") {" << std::endl; - std::unordered_set used_args; + jit::argument_set used_args; for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { source_buffer << " "; @@ -464,7 +599,8 @@ namespace gpu { } } for (auto &output : outputs) { - if (!used_args.contains(output.get())) { + if (!used_args.contains(output.get()) && + !graph::atomic_accumulate_1D_cast(output).get()) { source_buffer << " "; jit::add_type (source_buffer); source_buffer << " *" << jit::to_string('o', output.get()) @@ -474,17 +610,30 @@ namespace gpu { used_args.insert(output.get()); } } + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + source_buffer << " "; + jit::add_type (source_buffer); + source_buffer << " *" + << jit::to_string('v', atomics[i].get()) + << " = args[" + << reinterpret_cast (atomics[i].get()) + << "];" + << std::endl; + used_args.insert(atomics[i].get()); + } + } if (state.get()) { registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " mt_state &" << registers[state.get()] << " = " - << jit::to_string('s', state.get()) << "[0];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(state.get()) -#endif - << std::endl; + << jit::to_string('s', state.get()) << "[0]"; + state->endline(source_buffer, usage); } source_buffer << " for (size_t i = 0; i < " << size << "; i++) {" << std::endl; + if (iterations > 1) { + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } for (auto &input : inputs) { registers[input.get()] = jit::to_string('r', input.get()); @@ -492,11 +641,8 @@ namespace gpu { jit::add_type (source_buffer); source_buffer << " " << registers[input.get()] << " = " << jit::to_string('v', input.get()) - << "[i]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) -#endif - << std::endl; + << "[i]"; + input->endline(source_buffer, usage); } } @@ -508,39 +654,52 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, - const jit::register_usage &usage) { - std::unordered_set out_registers; + const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { + jit::argument_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('v', in.get()); source_buffer << "[i] = "; if constexpr (SAFE_MATH) { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] - << ";" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] + << ";" << std::endl; } } else { source_buffer << registers[a.get()] << ";" << std::endl; @@ -549,28 +708,38 @@ namespace gpu { } } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()); source_buffer << "[i] = "; if constexpr (SAFE_MATH) { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] - << ";" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] + << ";" << std::endl; } } else { source_buffer << registers[a.get()] << ";" << std::endl; @@ -579,8 +748,10 @@ namespace gpu { } } - source_buffer << " }" << std::endl; - source_buffer << "}" << std::endl; + if (iterations > 1) { + source_buffer << " }" << std::endl; + } + source_buffer << " }" << std::endl << "}" << std::endl; } //------------------------------------------------------------------------------ diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index d03b9f4..c2e36b8 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -8,7 +8,6 @@ #ifndef cuda_context_h #define cuda_context_h -#include #include #include @@ -16,9 +15,11 @@ #include #include "random.hpp" +#include "timing.hpp" +#include "piecewise.hpp" /// Maximum number of registers to use. -#define MAX_REG 128 +#define MAX_REG 256 namespace gpu { //------------------------------------------------------------------------------ @@ -79,10 +80,10 @@ namespace gpu { /// The cuda code library. CUmodule module; /// Argument map. - std::map *, CUdeviceptr> kernel_arguments; + std::unordered_map *, CUdeviceptr> kernel_arguments; #ifdef USE_CUDA_TEXTURES /// Textures. - std::map texture_arguments; + std::unordered_map texture_arguments; #endif /// Result buffer. CUdeviceptr result_buffer; @@ -107,8 +108,10 @@ namespace gpu { } public: +/// Random state size multiplyer. + constexpr static size_t random_state_scale = 3000; /// Size of random state needed. - constexpr static size_t random_state_size = 1024; + constexpr static size_t random_state_size = 1024*random_state_scale; /// Remaining constant memory in bytes. int remaining_const_memory; @@ -225,6 +228,19 @@ namespace gpu { if (jit::verbose) { std::cout << "CUDA GPU info." << std::endl; std::cout << " Major compute capability : " << compute_version << std::endl; + + int value; + check_error(cuDeviceGetAttribute(&value, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, + device), "cuDeviceGetAttribute"); + + std::cout << " Max shared memory : " << value << std::endl; + + check_error(cuDeviceGetAttribute(&value, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); + + std::cout << " Warp size : " << value << std::endl; } check_error(cuDeviceGetAttribute(&compute_version, @@ -307,6 +323,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace.' /// @param[in] tex1d_list List of 1D textures. @@ -316,6 +333,7 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, @@ -324,23 +342,20 @@ namespace gpu { check_error(cuModuleGetFunction(&function, module, kernel_name.c_str()), "cuModuleGetFunction"); std::vector buffers; - std::set *> needed_buffers; + std::unordered_set *> needed_buffers; const size_t buffer_element_size = sizeof(T); for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { kernel_arguments.try_emplace(input.get()); - const backend::buffer backend = input->evaluate(); check_error(cuMemAllocManaged(&kernel_arguments[input.get()], - backend.size()*sizeof(T), + input->size()*sizeof(T), CU_MEM_ATTACH_GLOBAL), "cuMemAllocManaged"); check_error(cuMemcpyHtoD(kernel_arguments[input.get()], - &backend[0], - backend.size()*sizeof(T)), + input->data(), + input->size()*sizeof(T)), "cuMemcpyHtoD"); - buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); - needed_buffers.insert(input.get()); } if (!needed_buffers.contains(input.get())) { buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); @@ -348,18 +363,35 @@ namespace gpu { } } for (auto &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - kernel_arguments.try_emplace(output.get()); - check_error(cuMemAllocManaged(&kernel_arguments[output.get()], - num_rays*sizeof(T), + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + kernel_arguments.try_emplace(output.get()); + check_error(cuMemAllocManaged(&kernel_arguments[output.get()], + num_rays*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + } + if (!needed_buffers.contains(output.get())) { + buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); + needed_buffers.insert(output.get()); + } + } + } + for (auto &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + kernel_arguments.try_emplace(atomic.get()); + check_error(cuMemAllocManaged(&kernel_arguments[atomic.get()], + atomic->size()*sizeof(T), CU_MEM_ATTACH_GLOBAL), "cuMemAllocManaged"); - buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); - needed_buffers.insert(output.get()); + check_error(cuMemcpyHtoD(kernel_arguments[atomic.get()], + atomic->data(), + atomic->size()*sizeof(T)), + "cuMemcpyHtoD"); } - if (!needed_buffers.contains(output.get())) { - buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); - needed_buffers.insert(output.get()); + if (!needed_buffers.contains(atomic.get())) { + buffers.push_back(reinterpret_cast (&kernel_arguments[atomic.get()])); + needed_buffers.insert(atomic.get()); } } @@ -490,42 +522,79 @@ namespace gpu { int value; check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); - unsigned int threads_per_group = value; - unsigned int thread_groups = num_rays/threads_per_group + (num_rays%threads_per_group ? 1 : 0); + int warp_size; + check_error(cuDeviceGetAttribute(&warp_size, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); + + unsigned int total_parallel = state.get() ? state->size() : num_rays; + unsigned int threads_per_group = total_parallel < 1024 ? warp_size : value; + unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); int min_grid; check_error(cuOccupancyMaxPotentialBlockSize(&min_grid, &value, function, 0, 0, 0), "cuOccupancyMaxPotentialBlockSize"); if (jit::verbose) { - std::cout << " Kernel name : " << kernel_name << std::endl; + std::cout << " Kernel name : " << kernel_name << std::endl; std::cout << " Threads per group : " << threads_per_group << std::endl; std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; + std::cout << " Total parallel : " << total_parallel << std::endl; std::cout << " Min grid size : " << min_grid << std::endl; std::cout << " Suggested Block size : " << value << std::endl; } - +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer(kernel_name); +#endif if (state.get()) { - return [this, num_rays, function, threads_per_group, buffers] () mutable { - for (uint32_t i = 0; i < num_rays; i += threads_per_group) { + return [this, num_rays, function, thread_groups, threads_per_group, buffers +#ifdef PROFILE_KERNELS + , timer +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); +#endif + for (uint32_t i = 0, ie = threads_per_group*thread_groups; i < num_rays; i += ie) { check_error_async(cuStreamWriteValue32(stream, offset_buffer, i, CU_STREAM_WRITE_VALUE_DEFAULT), "cuStreamWriteValue32"); check_error_async(cuLaunchKernel(function, - 1, 1, 1, + thread_groups, 1, 1, threads_per_group, 1, 1, 0, stream, buffers.data(), NULL), "cuLaunchKernel"); } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); +#endif }; } else { - return [this, function, thread_groups, threads_per_group, buffers] () mutable { + return [this, function, thread_groups, threads_per_group, buffers +#ifdef PROFILE_KERNELS + , timer +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); +#endif check_error_async(cuLaunchKernel(function, thread_groups, 1, 1, threads_per_group, 1, 1, 0, stream, buffers.data(), NULL), "cuLaunchKernel"); +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); +#endif }; } } @@ -575,6 +644,134 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will be memset a buffer to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector buffers; + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + kernel_arguments.try_emplace(input.get()); + check_error(cuMemAllocManaged(&kernel_arguments[input.get()], + input->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[input.get()], + input->data(), + input->size()*sizeof(T)), + "cuMemcpyHtoD"); + } + buffers.push_back(kernel_arguments[input.get()]); + } + + std::vector sizes; + for (CUdeviceptr &buffer : buffers) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); + sizes.push_back(size); + } +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("zero buffer"); +#endif + return [this, buffers, sizes +#ifdef PROFILE_KERNELS + , timer +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); +#endif + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + check_error_async(cuMemsetD8Async(buffers[i], 0, sizes[i], + stream), + "cuMemsetD8Async"); + } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); +#endif + }; + } + +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector sources; + std::vector destinations; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + kernel_arguments.try_emplace(in.get()); + check_error(cuMemAllocManaged(&kernel_arguments[in.get()], + in->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[in.get()], + in->data(), + in->size()*sizeof(T)), + "cuMemcpyHtoD"); + } + destinations.push_back(kernel_arguments[in.get()]); + + if (!kernel_arguments.contains(out.get())) { + kernel_arguments.try_emplace(out.get()); + check_error(cuMemAllocManaged(&kernel_arguments[out.get()], + out->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[out.get()], + out->data(), + out->size()*sizeof(T)), + "cuMemcpyHtoD"); + } + sources.push_back(kernel_arguments[out.get()]); + } + + std::vector sizes; + for (CUdeviceptr &buffer : sources) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); + sizes.push_back(size); + } +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("copy buffer"); +#endif + return [this, sources, destinations, sizes +#ifdef PROFILE_KERNELS + , timer +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); +#endif + for (size_t i = 0, ie = sources.size(); i < ie; i++) { + check_error_async(cuMemcpyDtoDAsync(destinations[i], + sources[i], + sizes[i], stream), + "cuMemcpyDtoDAsync"); + } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); +#endif + }; + } + //------------------------------------------------------------------------------ /// @brief Hold the current thread until the stream has completed. //------------------------------------------------------------------------------ @@ -583,6 +780,37 @@ namespace gpu { check_error(cuCtxSynchronize(), "cuCtxSynchronize"); } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("callback"); +#endif + return [this, callback +#ifdef PROFILE_KERNELS + , timer +#endif + ]() mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); +#endif + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast *> (arg)->operator()(); + }, &callback), "cuLaunchHostFunc"); +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); +#endif + }; + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// @@ -651,6 +879,7 @@ namespace gpu { source_buffer << "typedef unsigned int uint32_t;" << std::endl << "typedef unsigned short uint16_t;" << std::endl << "typedef short int16_t;" << std::endl + << "typedef unsigned char uint8_t;" << std::endl << "template" << std::endl << "class array {" << std::endl << "private:" << std::endl @@ -702,6 +931,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -709,24 +939,62 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t size, const std::vector &is_constant, jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" __global__ void " << name << "(" << std::endl; - std::unordered_set used_args; + int used_thread_mem = 0; + int max_shared_mem; + check_error(cuDeviceGetAttribute(&max_shared_mem, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, + device), "cuDeviceGetAttribute"); + + int warp_size; + check_error(cuDeviceGetAttribute(&warp_size, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); + + jit::argument_set used_args; if (inputs.size()) { +#ifdef USE_INPUT_CACHE + if (!is_constant[0] && iterations > 1) { + const size_t needed_mem = inputs[0]->size() > 1024 ? + 1024*sizeof(T) : + warp_size*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[0].get()); + } + } else if (is_constant[0] && + inputs[0]->size() < size && + inputs[0]->size() < 1024) { + const size_t needed_mem = inputs[0]->size()*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[0].get()); + thread_mem[inputs[0].get()] = jit::to_string('t', inputs[0].get()); + } + } +#endif source_buffer << " "; if (is_constant[0]) { source_buffer << "const "; @@ -737,14 +1005,28 @@ namespace gpu { used_args.insert(inputs[0].get()); } for (size_t i = 1, ie = inputs.size(); i < ie; i++) { - if (!used_args.contains(inputs[i].get())) { - source_buffer << ", // " << inputs[i - 1]->get_symbol() -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - << " used " << usage.at(inputs[i - 1].get()) -#endif +#ifdef USE_INPUT_CACHE + if (!is_constant[i] && iterations > 1) { + const size_t needed_mem = inputs[i]->size() > 1024 ? + 1024*sizeof(T) : + warp_size*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + } + } else if (is_constant[i] && + inputs[i]->size() < size && + inputs[i]->size() < 1024) { + const size_t needed_mem = inputs[i]->size()*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); + } + } #endif - << std::endl; + if (!used_args.contains(inputs[i].get())) { + inputs[i]->endline(source_buffer, usage, ','); source_buffer << " "; if (is_constant[i]) { source_buffer << "const "; @@ -756,18 +1038,12 @@ namespace gpu { } } for (size_t i = 0, ie = outputs.size(); i < ie; i++) { - if (!used_args.contains(outputs[i].get())) { + if (!used_args.contains(outputs[i].get()) && + !graph::atomic_accumulate_1D_cast(outputs[i]).get()) { if (i == 0) { if (inputs.size()) { - source_buffer << ", // " - << inputs[inputs.size() - 1]->get_symbol(); -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - source_buffer << " used " - << usage.at(inputs[inputs.size() - 1].get()); -#endif -#endif - source_buffer << std::endl; + inputs[inputs.size() - 1]->endline(source_buffer, + usage, ','); } } else { source_buffer << "," << std::endl; @@ -780,6 +1056,27 @@ namespace gpu { used_args.insert(outputs[i].get()); } } + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + if (i == 0) { + if (outputs.size()) { + outputs[outputs.size() - 1]->endline(source_buffer, + usage, ','); + } else if (inputs.size()) { + inputs[inputs.size() - 1]->endline(source_buffer, + usage, ','); + } + } else { + source_buffer << "," << std::endl; + } + + source_buffer << " "; + jit::add_type (source_buffer); + source_buffer << " * __restrict__ " + << jit::to_string('v', atomics[i].get()); + used_args.insert(atomics[i].get()); + } + } if (state.get()) { source_buffer << "," << std::endl << " mt_state * __restrict__ " @@ -800,50 +1097,141 @@ namespace gpu { << jit::to_string('a', key); } #endif - source_buffer << ") {" << std::endl - << " const int index = blockIdx.x*blockDim.x + threadIdx.x;" - << std::endl; - if (state.get()) { -#ifdef USE_INPUT_CACHE - registers[state.get()] = jit::to_string('r', state.get()); - source_buffer << " mt_state &" << registers[state.get()] << " = " - << jit::to_string('s', state.get()) - << "[threadIdx.x];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(state.get()) -#endif + source_buffer << ") {" << std::endl; + if (thread_shared.size()) { + source_buffer << " const int t_index = threadIdx.x;" << std::endl; -#else - registers[state.get()] = jit::to_string('s', state.get()) + "[threadIdx.x]"; -#endif } + source_buffer << " const int index = blockIdx.x*blockDim.x + threadIdx.x;" + << std::endl; + source_buffer << " if ("; if (state.get()) { source_buffer << "offset[0] + "; } source_buffer << "index < " << size << ") {" << std::endl; + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (is_constant[i]) { +#ifdef USE_INPUT_CACHE + if (usage.at(inputs[i].get()) && inputs[i]->size() == size) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } +#else + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + + "[" + + (state.get() ? "offset[0] + " : "") + + "index]"; +#endif + } + } - for (auto &input : inputs) { + if (thread_shared.size()) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " __shared__ "; + jit::add_type (source_buffer); + source_buffer << " " << jit::to_string('t', inputs[i].get()) + << "[" << inputs[i]->size() << "]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " for(int j = t_index; j < " + << inputs[i]->size() + << "; j += blockDim.x) {" << std::endl + << " " + << jit::to_string('t', inputs[i].get()) + << "[j] = " + << jit::to_string('v', inputs[i].get()) + << "[j]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " }" << std::endl; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " __syncthreads();" + << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + thread_shared.erase(inputs[i].get()); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) { + source_buffer << " __shared__ "; + jit::add_type (source_buffer); + source_buffer << " " << jit::to_string('t', inputs[i].get()) + << "[" + << (inputs[i]->size() > 1024 ? 1024 : warp_size) + << "]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } + } + } + + if (iterations > 1) { + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } + + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (!is_constant[i]) { #ifdef USE_INPUT_CACHE - if (usage.at(input.get())) { - registers[input.get()] = jit::to_string('r', input.get()); - source_buffer << " const "; - jit::add_type (source_buffer); - source_buffer << " " << registers[input.get()] << " = " - << jit::to_string('v', input.get()) - << "["; - if (state.get()) { - source_buffer << "offset[0] + "; + if (usage.at(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = "; + if (thread_shared.contains(inputs[i].get())) { + source_buffer << jit::to_string('t', inputs[i].get()) + << "[t_index]"; + } else { + source_buffer << jit::to_string('v', inputs[i].get()) + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index]"; + } + inputs[i]->endline(source_buffer, usage); + } +#else + if (thread_shared.contains(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; + } else { + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[" + + (state.get() ? "offset[0] + " : "") + + "index]"; } - source_buffer << "index]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) #endif - << std::endl; } + } + if (state.get()) { +#ifdef USE_INPUT_CACHE + registers[state.get()] = jit::to_string('r', state.get()); + source_buffer << " mt_state &" << registers[state.get()] << " = " + << jit::to_string('s', state.get()) + << "[index]"; + state->endline(source_buffer, usage); #else - registers[input.get()] = jit::to_string('v', input.get()) + "[index]"; + registers[state.get()] = jit::to_string('s', state.get()) + "[threadIdx.x]"; #endif } } @@ -856,45 +1244,61 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, - const jit::register_usage &usage) { + const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { std::unordered_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); - source_buffer << " " - << jit::to_string('v', in.get()) - << "["; - if (state.get()) { - source_buffer << "offset[0] + "; + auto a = out->compile(source_buffer, registers, + thread_mem, usage); + source_buffer << " "; + if (thread_shared.contains(in.get())) { + source_buffer << jit::to_string('t', in.get()) + << "[t_index] = "; + } else { + source_buffer << jit::to_string('v', in.get()) + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index] = "; } - source_buffer << "index] = "; if constexpr (SAFE_MATH) { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" - << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" - << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] << ";" << std::endl; } } else { @@ -905,12 +1309,11 @@ namespace gpu { } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) << "["; @@ -922,17 +1325,26 @@ namespace gpu { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" - << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" - << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] << ";" << std::endl; } } else { @@ -942,6 +1354,22 @@ namespace gpu { } } + if (iterations > 1) { + source_buffer << " }" << std::endl; + } + for (auto &[out, in] : setters) { + if (thread_shared.contains(in.get())) { + source_buffer << " " + << jit::to_string('v', in.get()) + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index] = " + << jit::to_string('t', in.get()) + << "[t_index];" << std::endl; + } + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/dispersion.hpp b/graph_framework/dispersion.hpp index f9d2526..5a73345 100644 --- a/graph_framework/dispersion.hpp +++ b/graph_framework/dispersion.hpp @@ -1461,10 +1461,11 @@ namespace dispersion { workflow::manager work(index); - solver::newton(work, {x}, inputs, this->D, - graph::shared_random_state (), - tolerance, max_iterations); + solver::newton (work, {x}, inputs, + {}, this->D, NULL, + tolerance, + max_iterations); work.compile(); work.run(); diff --git a/graph_framework/equilibrium.hpp b/graph_framework/equilibrium.hpp index 9d03531..d996989 100644 --- a/graph_framework/equilibrium.hpp +++ b/graph_framework/equilibrium.hpp @@ -1599,11 +1599,11 @@ namespace equilibrium { }; workflow::manager work(device_number); - solver::newton(work, { + solver::newton (work, { x_axis, z_axis - }, inputs, (psi_cache - psimin)/dpsi, graph::shared_random_state (), static_cast (1.0E-30), 1000, static_cast (0.1)); - work.add_item(inputs, {b_mod}, {}, - graph::shared_random_state (), + }, inputs, {}, (psi_cache - psimin)/dpsi, + NULL, static_cast (1.0E-30), 1000, static_cast (0.1)); + work.add_item(inputs, {b_mod}, {}, {}, NULL, "bmod_at_axis", inputs.back()->size()); work.compile(); work.run(); diff --git a/graph_framework/graph_framework.hpp b/graph_framework/graph_framework.hpp index 6139869..251b89e 100644 --- a/graph_framework/graph_framework.hpp +++ b/graph_framework/graph_framework.hpp @@ -27,6 +27,8 @@ #include "trigonometry.hpp" #include "vector.hpp" #include "workflow.hpp" +#include "particle_in_cell.hpp" +#include "logical.hpp" #ifdef USE_CUDA #include "cuda_context.hpp" diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index b73afaa..1338f25 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -52,12 +52,14 @@ namespace jit { std::ostringstream source_buffer; /// Nodes that have been jitted. register_map registers; +/// Prefunctions that have been defined. + preamble_map pre_funcs; /// Kernel names. std::vector kernel_names; /// Kernel textures. - std::map kernel_1dtextures; + std::unordered_map kernel_1dtextures; /// Kernel textures. - std::map kernel_2dtextures; + std::unordered_map kernel_2dtextures; /// Type for the GPU context. using gpu_context_type = typename std::conditional (), @@ -72,13 +74,21 @@ namespace jit { /// GPU Context. gpu_context_type gpu_context; -/// Used random. - bool used_random; public: /// Size of random state needed. constexpr static size_t random_state_size = gpu_context_type::random_state_size; +//------------------------------------------------------------------------------ +/// @brief Get the number of random states needed. +/// +/// @param[in] size Number of random numbers needed. +/// @returns The maximum number of random states needed. +//------------------------------------------------------------------------------ + static size_t max_random_state_size(const size_t size) { + return std::min(size, random_state_size); + } + //------------------------------------------------------------------------------ /// @brief Get the maximum number of concurrent instances. /// @@ -98,7 +108,7 @@ namespace jit { /// /// @param[in] index Concurrent index. Not used. //------------------------------------------------------------------------------ - context(const size_t index) : gpu_context(index), used_random(false) { + context(const size_t index) : gpu_context(index) { source_buffer << std::setprecision(max_digits10 ()); gpu_context.create_header(source_buffer); } @@ -108,27 +118,25 @@ namespace jit { /// /// Build the source code for a kernel graph. /// -/// @param[in] name Name to call the kernel. -/// @param[in] inputs Input variables of the kernel. -/// @param[in] outputs Output nodes of the graph to compute. -/// @param[in] setters Map outputs back to input values. -/// @param[in] state Random state node. -/// @param[in] size Size of the kernel. +/// @param[in] name Name to call the kernel. +/// @param[in] inputs Input variables of the kernel. +/// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] setters Map outputs back to input values. +/// @param[in] atomics Input variables for atomic operations. +/// @param[in] state Random state node. +/// @param[in] size Size of the kernel. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void add_kernel(const std::string name, - graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes setters, - graph::shared_random_state state, - const size_t size) { + graph::input_nodes &inputs, + graph::output_nodes &outputs, + graph::map_nodes &setters, + graph::input_nodes &atomics, + graph::shared_random_state &state, + const size_t size, + const size_t iterations=1) { kernel_names.push_back(name); - if (state.get() && !used_random) { - used_random = true; - graph::random_state_node::compile_random_state(source_buffer); - graph::random_node::compile_random(source_buffer); - } - std::vector is_constant(inputs.size(), true); visiter_map visited; register_usage usage; @@ -145,6 +153,7 @@ namespace jit { visited, usage, kernel_1dtextures[name], kernel_2dtextures[name], + pre_funcs, gpu_context.remaining_const_memory); } for (auto &out : outputs) { @@ -152,6 +161,7 @@ namespace jit { visited, usage, kernel_1dtextures[name], kernel_2dtextures[name], + pre_funcs, gpu_context.remaining_const_memory); } @@ -161,32 +171,48 @@ namespace jit { } } + argument_set thread_shared; + jit::register_map thread_mem; + gpu_context.create_kernel_prefix(source_buffer, - name, inputs, outputs, state, + name, inputs, outputs, + atomics, state, size, is_constant, registers, usage, kernel_1dtextures[name], - kernel_2dtextures[name]); + kernel_2dtextures[name], + thread_shared, + thread_mem, + iterations); - register_map indices; for (auto &[out, in] : setters) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, thread_mem, usage); } for (auto &out : outputs) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, thread_mem, usage); } gpu_context.create_kernel_postfix(source_buffer, outputs, setters, state, - registers, indices, usage); + registers, usage, + thread_shared, + thread_mem, + iterations); // Delete the registers so that they can be used again in other kernels. std::vector removed_elements; for (auto &[key, value] : registers) { - if (value[0] == 'r') { + if (value[0] == 'r' || + value[0] == 'l' || + value[0] == 'i') { removed_elements.push_back(key); } } + for (auto &out : outputs) { + if (graph::atomic_accumulate_1D_cast(out).get()) { + removed_elements.push_back(out.get()); + } + } for (auto &key : removed_elements) { registers.erase(key); @@ -202,6 +228,26 @@ namespace jit { gpu_context.create_reduction(source_buffer, size); } +//------------------------------------------------------------------------------ +/// @brief Add zero. +/// +/// @param[in] inputs Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes inputs) { + return gpu_context.create_zero_call(inputs); + } + +//------------------------------------------------------------------------------ +/// @brief Add copy. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes setters) { + return gpu_context.create_copy_call(setters); + } + //------------------------------------------------------------------------------ /// @brief Print the kernel source. //------------------------------------------------------------------------------ @@ -250,6 +296,7 @@ namespace jit { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @returns A lambda function to run the kernel. @@ -257,9 +304,11 @@ namespace jit { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays) { - return gpu_context.create_kernel_call(kernel_name, inputs, outputs, state, num_rays, + return gpu_context.create_kernel_call(kernel_name, inputs, outputs, + atomics, state, num_rays, kernel_1dtextures[kernel_name], kernel_2dtextures[kernel_name]); } @@ -306,6 +355,15 @@ namespace jit { gpu_context.wait(); } +//------------------------------------------------------------------------------ +/// @brief Run a function. +/// +/// @returns A lambda function to run run the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return gpu_context.run_function(callback); + } + //------------------------------------------------------------------------------ /// @brief Copy contexts of buffer to device. /// diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp new file mode 100644 index 0000000..97cc7af --- /dev/null +++ b/graph_framework/logical.hpp @@ -0,0 +1,3340 @@ +//------------------------------------------------------------------------------ +/// @file logical.hpp +/// @brief Nodes for boolean logic. +/// +/// Defines a tree of operations that allows automatic differentiation. +//------------------------------------------------------------------------------ +#ifndef logical_h +#define logical_h + +#include "node.hpp" + +/// Name space for graph nodes. +namespace graph { +/// Convenience type for true constant. + template + constexpr shared_leaf true_constant() { + return one (); + } + +/// Convenience type for false constant. + template + constexpr shared_leaf false_constant() { + return zero (); + } + +//****************************************************************************** +// IsInf node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief isinf node. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class isinf_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "isinf" + + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an isinf node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + isinf_node(shared_leaf arg) : + no_derivative> (arg, + isinf_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of isinf. +/// +/// result = isinf(a) +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + arg.isinf(); + return arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an isinf node. +/// +/// @returns A reduced isinf node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = isinf(" + << registers[arg.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "isinf\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build not node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf isinf(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared isinf nodes. + template + using shared_isinf = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a isinf node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_isinf isinf_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// IsNaN node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief isnan node. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class isnan_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "isnan" + + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an isnan node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + isnan_node(shared_leaf arg) : + no_derivative> (arg, + isnan_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of isnan. +/// +/// result = isnan(a) +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + arg.isnan(); + return arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an isnan node. +/// +/// @returns A reduced isnan node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = isnan(" + << registers[arg.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "isnan\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build isnan node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf isnan(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared isnan nodes. + template + using shared_isnan = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an isnan node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_isnan isnan_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Not node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Not node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class not_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "!" + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an not node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + not_node(shared_leaf arg) : + no_derivative> (arg, + not_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of not. +/// +/// result = !a +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + return !arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a not node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + +// !(a == b) -> a != b + auto equalc = equal_cast(this->arg); + if (equalc.get()) { + return equalc->get_left() != equalc->get_right(); + } + +// !(a != b) -> a == b + auto nequalc = not_equal_cast(this->arg); + if (nequalc.get()) { + return nequalc->get_left() == nequalc->get_right(); + } + + if constexpr (!jit::complex_scalar) { +// !(a < b) -> a >= b + auto ltc = less_than_cast(this->arg); + if (ltc.get()) { + return ltc->get_left() >= ltc->get_right(); + } + +// !(a <= b) -> a > b + auto lec = less_than_equal_cast(this->arg); + if (lec.get()) { + return lec->get_left() > lec->get_right(); + } + +// !(a > b) -> a <= b + auto gtc = greater_than_cast(this->arg); + if (gtc.get()) { + return gtc->get_left() <= gtc->get_right(); + } + +// !(a >= b) -> a < b + auto gec = greater_than_equal_cast(this->arg); + if (gec.get()) { + return gec->get_left() < gec->get_right(); + } + } + +// !!a -> a + auto n = not_cast(this->arg); + if (n.get()) { + return n->get_arg(); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool " + << registers[this] << " = !"; + if constexpr (jit::complex_scalar) { + stream << "real("; + } + stream << registers[arg.get()]; + if constexpr (jit::complex_scalar) { + stream << ")"; + } + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool arg_brackets = add_cast(this->arg).get() || + subtract_cast(this->arg).get(); + std::cout << "\neg"; + if (arg_brackets) { + std::cout << "\\left("; + } + this->arg->to_latex(); + if (arg_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build not node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf not_(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build not node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf operator!(shared_leaf arg) { + return not_ (arg); + } + +/// Convenience type alias for shared not nodes. + template + using shared_not = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a not node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_not not_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "==" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of equal. +/// +/// result = l == r +/// +/// @returns The value of l == r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result == r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an equal node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "==" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = equal_cast(x); + if (x_cast.get()) { +// equal is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "="; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"==\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(shared_leaf l, + shared_leaf r) { + return equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(const L l, + shared_leaf r) { + return equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(shared_leaf l, + const R r) { + return equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared equal nodes. + template + using shared_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_equal equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Not equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A not equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class not_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "!=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an not equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + not_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + not_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of not equal. +/// +/// result = l != r +/// +/// @returns The value of l != r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result != r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an not equal node. +/// +/// @returns A reduced not equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "!=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = not_equal_cast(x); + if (x_cast.get()) { +// equal is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\ne"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build note equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf not_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(shared_leaf l, + shared_leaf r) { + return not_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(const L l, + shared_leaf r) { + return not_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(shared_leaf l, + const R r) { + return not_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared not equal nodes. + template + using shared_not_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_not_equal not_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Greater than node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A greater than node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class greater_than_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + ">" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a greater than node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + greater_than_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + greater_than_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of greater than. +/// +/// result = l > r +/// +/// @returns The value of l > r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result > r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce greater than node. +/// +/// @returns A reduced not equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << ">" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << ">"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \">\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf greater_than(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(shared_leaf l, + shared_leaf r) { + return greater_than (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(const L l, + shared_leaf r) { + return greater_than (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(shared_leaf l, + const R r) { + return greater_than (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared greater than nodes. + template + using shared_greater_than = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a greater than node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_greater_than greater_than_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Less than node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A less than node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class less_than_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "<" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a less than node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + less_than_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + less_than_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than. +/// +/// result = l < r +/// +/// @returns The value of l < r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result < r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce less than node. +/// +/// @returns A reduced less than node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "<" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "<"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"<\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf less_than(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(shared_leaf l, + shared_leaf r) { + return less_than (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(const L l, + shared_leaf r) { + return less_than (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(shared_leaf l, + const R r) { + return less_than (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared less than nodes. + template + using shared_less_than = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a less than node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_less_than less_than_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Greater than equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A greater than equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class greater_than_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + ">=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a greater than equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + greater_than_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + greater_than_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of greater than equal. +/// +/// result = l >= r +/// +/// @returns The value of l >= r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result >= r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce greater than equal node. +/// +/// @returns A reduced greater than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << ">=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\ge"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \">=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf greater_than_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(shared_leaf l, + shared_leaf r) { + return greater_than_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(const L l, + shared_leaf r) { + return greater_than_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(shared_leaf l, + const R r) { + return greater_than_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared greater than equal nodes. + template + using shared_greater_than_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a greater than equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_greater_than_equal greater_than_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Less than equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A less than equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class less_than_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "<=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a less than equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + less_than_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + less_than_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l <= r +/// +/// @returns The value of l <= r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result <= r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce less than equal node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "<=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\le"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"<=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf less_than_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(shared_leaf l, + shared_leaf r) { + return less_than_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(const L l, + shared_leaf r) { + return less_than_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(shared_leaf l, + const R r) { + return less_than_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared less than equal nodes. + template + using shared_less_than_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a less than equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_less_than_equal less_than_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// And node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A and node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class and_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "&&" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an and node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + and_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + and_node::to_string(l.get(), r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l && r +/// +/// @returns The value of l && r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result && r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce and node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "&&" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = and_cast(x); + if (x_cast.get()) { +// and is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\land"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"&&\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf and_(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(shared_leaf l, + shared_leaf r) { + return and_ (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(const bool l, + shared_leaf r) { + return and_ (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(shared_leaf l, + const bool r) { + return and_ (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared and nodes. + template + using shared_and = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an and node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_and and_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Or node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A or node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class or_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "||" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a or node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + or_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + or_node::to_string(l.get(), r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l || r +/// +/// @returns The value of l || r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result || r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce or node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "||" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = or_cast(x); + if (x_cast.get()) { +// or is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\lor"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"||\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf or_(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(shared_leaf l, + shared_leaf r) { + return or_ (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(const bool l, + shared_leaf r) { + return or_ (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(shared_leaf l, + const bool r) { + return or_ (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared or nodes. + template + using shared_or = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a or node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_or or_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// If node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An If conditional node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class if_node final : public triple_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] c Condition node. +/// @param[in] t True condition. +/// @param[in] f False condition. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *c, + leaf_node *t, + leaf_node *f) { + return "if" + + jit::format_to_string(reinterpret_cast (c)) + + jit::format_to_string(reinterpret_cast (t)) + + jit::format_to_string(reinterpret_cast (f)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] c Condition node. +/// @param[in] t True condition branch. +/// @param[in] f False condition branch. +//------------------------------------------------------------------------------ + if_node(shared_leaf c, + shared_leaf t, + shared_leaf f) : + triple_node (c, t, f, + if_node::to_string(c.get(), + t.get(), + f.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of if. +/// +/// result = if(c, t, f) +/// +/// @returns The value of if(c, t, f). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer c_result = this->left->evaluate(); + backend::buffer t_result = this->middle->evaluate(); + backend::buffer f_result = this->right->evaluate(); + return c_result.if_(t_result, f_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an if node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto c = constant_cast(this->left); + auto t = constant_cast(this->middle); + auto f = constant_cast(this->right); + + if (c.get() && t.get() && f.get()) { + return constant (this->evaluate()); + } + +// If(c, a, a) -> a + if (this->middle->is_match(this->right)) { + return this->middle; + } + +// If(!a, b, c) -> If(a, c, b) + auto n = not_cast(this->left); + if (n.get()) { + return if_(n->get_arg(), this->right, this->middle); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d if(c,t,f)/dx = if(c,dt/dx,df/dx) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = if_ (this->left, + this->middle->df(x), + this->right->df(x)); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto c = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); + auto t = this->middle->compile(stream, registers, + thread_mem, usage); + auto f = this->right->compile(stream, registers, + thread_mem, usage); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = " + << registers[c.get()] << " ? " + << registers[t.get()] << " : " + << registers[f.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "if\\left("; + this->left->to_latex(); + std::cout << ","; + this->middle->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return if_ (this->left, + this->middle->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"if\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto c = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[c.get()] << ";" << std::endl; + auto t = this->middle->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[t.get()] << ";" << std::endl; + auto f = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[f.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build an if node from a condition and two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] c Condition branch. +/// @param[in] t True branch. +/// @param[in] f False branch. +/// @returns A reduced if node. +//------------------------------------------------------------------------------ + template + shared_leaf if_(shared_leaf c, + shared_leaf t, + shared_leaf f) { + auto temp = std::make_shared> (c, t, f)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared add nodes. + template + using shared_if = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an if node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_if if_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Min node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A Min node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class min_node final : public branch_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "min" + + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + min_node(shared_leaf l, + shared_leaf r) : + branch_node (l, r, + min_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of if. +/// +/// result = min(l, r) +/// +/// @returns The value of min(l, r). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::min(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a min node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (lc.get() && rc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.min_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.min_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.min_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.min_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d min(l,r)/dx = min(dl/dx,dr/dx) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = min (this->left->df(x), + this->right->df(x)); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); + + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = min(" + << registers[l.get()] << ", " + << registers[r.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = min_cast(x); + if (x_cast.get()) { +// Min is commutative. + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "min\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return min (this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"min\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build a max node from a condition and two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(const L l, + shared_leaf r) { + return min(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(shared_leaf l, + const R r) { + return min(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared min nodes. + template + using shared_min = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_min min_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } +} + +#endif /* logical_h */ diff --git a/graph_framework/math.hpp b/graph_framework/math.hpp index f22f3f6..5f9651f 100644 --- a/graph_framework/math.hpp +++ b/graph_framework/math.hpp @@ -7,6 +7,7 @@ #define math_h #include +#include #include "node.hpp" @@ -75,38 +76,49 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Handle cases like sqrt(c*x) where c is constant or cases like sqrt((x^a)*y). -// Note that we need to disable this reduction C is a negative real. +// Note that we need to disable this reduction if C is a negative real or +// a == 2. auto am = multiply_cast(this->arg); if (am.get()) { - if (pow_cast(am->get_left()).get() || - am->get_left()->is_constant() || - pow_cast(am->get_right()).get() || - am->get_right()->is_constant()) { + if (am->get_left()->is_constant()) { if constexpr (jit::complex_scalar) { return sqrt(am->get_left()) * sqrt(am->get_right()); } else { - if (am->get_left()->is_constant() && - !am->get_left()->evaluate().is_negative()) { + if (!am->get_left()->evaluate().is_negative()) { return sqrt(am->get_left()) * sqrt(am->get_right()); } } } + + auto amlp = pow_cast(am->get_left()); + auto amrp = pow_cast(am->get_right()); + if (amlp.get()) { + auto amlprc = constant_cast(amlp->get_right()); + if (amlprc.get() && !amlprc->is(2)) { + return sqrt(am->get_left()) * + sqrt(am->get_right()); + } + } else if (amrp.get()) { + auto amrprc = constant_cast(amrp->get_right()); + if (amrprc.get() && !amrprc->is(2)) { + return sqrt(am->get_left()) * + sqrt(am->get_right()); + } + } } auto ad = divide_cast(this->arg); @@ -156,22 +168,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -381,17 +391,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Reduce exp(log(x)) -> x @@ -426,22 +434,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -646,17 +652,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Reduce log(exp(x)) -> x @@ -691,22 +695,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -858,7 +860,7 @@ namespace graph { public: //------------------------------------------------------------------------------ -/// @brief Construct an power node. +/// @brief Construct a power node. /// /// @param[in] l Left branch. /// @param[in] r Right branch. @@ -869,7 +871,7 @@ namespace graph { r.get())) {} //------------------------------------------------------------------------------ -/// @brief Evaluate the results of addition. +/// @brief Evaluate the results of pow. /// /// result = l^r /// @@ -910,11 +912,9 @@ namespace graph { auto pl1 = piecewise_1D_cast(this->left); auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -922,13 +922,13 @@ namespace graph { if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -937,29 +937,29 @@ namespace graph { result.pow_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.pow_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.pow_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.pow_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } auto lp = pow_cast(this->left); @@ -1160,13 +1160,38 @@ namespace graph { return exp(this->right*temp->get_arg()); } + if constexpr (std::floating_point) { +// hypot(a,b)^2 -> a^2 + b^2 + auto lhp = hypot_cast(this->left); + if (lhp.get() && rc.get() && rc->is(2)) { + return pow(lhp->get_left(), this->right) + + pow(lhp->get_right(), this->right); + } + +// (a/hypot(b,c))^2 -> a^2/(b^2 + c^2) +// (hypot(b,c)/a)^2 -> (b^2 + c^2)/a^2 + if (ld.get() && rc.get() && rc->is(2)) { + auto ldlhp = hypot_cast(ld->get_left()); + auto ldrhp = hypot_cast(ld->get_right()); + if (ldlhp.get()) { + return (pow(ldlhp->get_left(), static_cast (2)) + + pow(ldlhp->get_right(), static_cast (2))) / + pow(ld->get_right(), static_cast (2)); + } else if (ldrhp.get()) { + return pow(ld->get_left(), static_cast (2)) / + (pow(ldrhp->get_left(), static_cast (2)) + + pow(ldrhp->get_right(), static_cast (2))); + } + } + } + return this->shared_from_this(); } //------------------------------------------------------------------------------ /// @brief Transform node to derivative. /// -/// d a^b dx = b*a^(b-1)*da/dx + ln(a)a^b*db/dx +/// d a^b/ dx = b*a^(b-1)*da/dx + ln(a)a^b*db/dx /// /// @param[in] x The variable to take the derivative to. /// @returns The derivative of the node. @@ -1189,26 +1214,25 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); shared_leaf r; auto temp = constant_cast(this->right); if (!temp.get() || !temp->is_integer()) { - r = this->right->compile(stream, registers, indices, usage); + r = this->right->compile(stream, registers, + thread_mem, usage); } registers[this] = jit::to_string('r', this); @@ -1425,6 +1449,241 @@ namespace graph { return std::dynamic_pointer_cast> (x); } +//****************************************************************************** +// Erf node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An error function node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// Note use templates here to defer this so it can use the operator functions. +//------------------------------------------------------------------------------ + template + class erf_node final : public straight_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] a Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *a) { + return "erf" + jit::format_to_string(reinterpret_cast (a)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a erf node. +/// +/// @param[in] x Argument. +//------------------------------------------------------------------------------ + erf_node(shared_leaf x) : + straight_node (x, erf_node::to_string(x.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of erf. +/// +/// result = erf(x) +/// +/// @returns The value of erf(x). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer result = this->arg->evaluate(); + result.erf(); + return result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the erf(x). +/// +/// @returns Reduced graph from erf. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + if (constant_cast(this->arg).get()) { + return constant (this->evaluate()); + } + + auto ap1 = piecewise_1D_cast(this->arg); + if (ap1.get()) { + return piecewise_1D(this->evaluate(), + ap1->get_arg()); + } + + auto ap2 = piecewise_2D_cast(this->arg); + if (ap2.get()) { + return piecewise_2D(this->evaluate(), + ap2->get_num_columns(), + ap2->get_left(), + ap2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d erf(y)/dx = 2/sqrt(pi)Exp(-y^2)*dy/dx +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = static_cast (2) + * std::numbers::inv_sqrtpi_v + * exp(this->arg*this->arg)*this->arg->df(x); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = erf(" + << registers[a.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = erf_cast(x); + if (x_cast.get()) { + return this->arg->is_match(x_cast->get_arg()); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "erf\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return erf(this->arg->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"erf\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto a = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[a.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Define erf convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Argument. +/// @returns A reduced exp node. +//------------------------------------------------------------------------------ + template + shared_leaf erf(shared_leaf x) { + auto temp = std::make_shared> (x)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared erf nodes. + template + using shared_erf = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a erf node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_erf erf_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //****************************************************************************** // Erfi node. //****************************************************************************** @@ -1451,7 +1710,7 @@ namespace graph { public: //------------------------------------------------------------------------------ -/// @brief Construct a exp node. +/// @brief Construct a erfi node. /// /// @param[in] x Argument. //------------------------------------------------------------------------------ @@ -1474,7 +1733,7 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Reduce the erfi(x). /// -/// @returns Reduced graph from exp. +/// @returns Reduced graph from erfi. //------------------------------------------------------------------------------ virtual shared_leaf reduce() { if (constant_cast(this->arg).get()) { @@ -1484,17 +1743,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } return this->shared_from_this(); @@ -1515,7 +1772,14 @@ namespace graph { const size_t hash = reinterpret_cast (x.get()); if (this->df_cache.find(hash) == this->df_cache.end()) { - this->df_cache[hash] = 2.0/std::sqrt(M_PI) + T invsqpi; + if constexpr(std::same_as>) { + invsqpi = std::numbers::inv_sqrtpi_v; + } else { + invsqpi = std::numbers::inv_sqrtpi_v; + } + this->df_cache[hash] = static_cast (2) + * invsqpi * exp(this->arg*this->arg)*this->arg->df(x); } return this->df_cache[hash]; @@ -1524,22 +1788,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -1645,12 +1907,12 @@ namespace graph { #endif } -/// Convenience type alias for shared exp nodes. +/// Convenience type alias for shared erfi nodes. template using shared_erfi = std::shared_ptr>; //------------------------------------------------------------------------------ -/// @brief Cast to a exp node. +/// @brief Cast to a erfi node. /// /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. @@ -1662,6 +1924,679 @@ namespace graph { shared_erfi erfi_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// Hypot node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A hypot node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class hypot_node final : public branch_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Argument node pointer. +/// @param[in] r Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "hypot" + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a hypot node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + hypot_node(shared_leaf l, + shared_leaf r) : + branch_node (l, r, hypot_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of hypot. +/// +/// result = hypot(l, r) +/// +/// @returns The value of hypot(l, r) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::hypot(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a hypot node. +/// +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (rc.get() && rc->is(0)) { + return sqrt(pow(this->left, static_cast (2))); + } else if (lc.get() && lc->is(0)) { + return sqrt(pow(this->right, static_cast (2))); + } else if (rc.get() && lc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.hypot_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.hypot_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.hypot_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.hypot_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + +// hypot(sqrt(a), sqrt(b)) -> sqrt(a + b) +// hypot(a, sqrt(b)) -> sqrt(a^2 + b) +// hypot(sqrt(a), b) -> sqrt(a + b^2) + auto sql = sqrt_cast(this->left); + auto sqr = sqrt_cast(this->right); + if (sql.get() && sqr.get()) { + return sqrt(sql->get_arg() + sqr->get_arg()); + } else if (sql.get()) { + return sqrt(sql->get_arg() + pow(this->right, + static_cast (2))); + } else if (sqr.get()) { + return sqrt(sqr->get_arg() + pow(this->left, + static_cast (2))); + } + +// hypoy(a,a) -> sqrt(2)sqrt(a^2) + if (this->left->is_match(this->right)) { + return std::numbers::sqrt2_v*sqrt(pow(this->left, + static_cast (2))); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d hypot(a,b)/ dx = (a*da/dx + b*db/dx)/hypot(a,b) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = (this->left*this->left->df(x) + + this->right*this->right->df(x)) + / this->shared_from_this(); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = "; + if constexpr (jit::use_metal ()) { + stream << "length({"; + } else { + stream << "hypot("; + } + stream << registers[l.get()] << ", " << registers[r.get()]; + if constexpr (jit::use_metal ()) { + stream << "})"; + } else { + stream << ")"; + } + + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = hypot_cast(x); + if (x_cast.get()) { +// Hypot is commutative. + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "hypot\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return hypot(this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"hypot\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(const L l, + shared_leaf r) { + return hypot(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(shared_leaf l, + const R r) { + return hypot(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared hypot nodes. + template + using shared_hypot = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a hypot node. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_hypot hypot_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// copysign node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A copysign node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class copysign_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Argument node pointer. +/// @param[in] r Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "copysign" + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a hypot node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + copysign_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + copysign_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of copysign. +/// +/// result = copysign(l, r) +/// +/// @returns The value of copysign(l, r) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::copysign(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a copysign node. +/// +/// @returns A reduced copysign node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (rc.get() && rc->is(0)) { + return sqrt(pow(this->left, static_cast (2))); + } else if (lc.get() && lc->is(0)) { + return sqrt(pow(this->right, static_cast (2))); + } else if (rc.get() && lc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.copysign_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.copysign_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.copysign_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.copysign_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = copysign(" + << registers[l.get()] << ", " + << registers[r.get()] << ")"; + + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = hypot_cast(x); + if (x_cast.get()) { + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "copysign\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return copysign(this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"copysign\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build copysign node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build copysign node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(const L l, + shared_leaf r) { + return copysign(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build power node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(shared_leaf l, + const R r) { + return copysign(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared copysign nodes. + template + using shared_copysign = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a copysign node. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_copysign copysign_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* math_h */ diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index b88c7ce..ff48d3f 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -8,11 +8,11 @@ #ifndef metal_context_h #define metal_context_h -#include - #import #include "random.hpp" +#include "timing.hpp" +#include "piecewise.hpp" /// Name space for GPU backends. namespace gpu { @@ -29,19 +29,21 @@ namespace gpu { /// The metal command queue. id queue; /// Argument map. - std::map *, id> kernel_arguments; + std::unordered_map *, id> kernel_arguments; /// Textures. - std::map> texture_arguments; + std::unordered_map> texture_arguments; /// Metal command buffer. id command_buffer; /// Metal library. id library; /// Buffer mutability descriptor. - std::map> bufferMutability; + std::unordered_map> bufferMutability; public: +/// Random state size multiplier. + constexpr static size_t random_state_scale = 3000; /// Size of random state needed. - constexpr static size_t random_state_size = 1024; + constexpr static size_t random_state_size = 1024*random_state_scale; /// Remaining constant memory in bytes. NOT USED. int remaining_const_memory; @@ -92,7 +94,13 @@ namespace gpu { } if (jit::verbose) { - std::cout << "Metal GPU info." << std::endl; + std::cout << "Metal GPU info." << std::endl + << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl + << " Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl + << " Device name : " << [device.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl + << " Architecture : " << [device.architecture.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl + << " Max buffer length : " << device.maxBufferLength << std::endl + << " Max working set : " << device.recommendedMaxWorkingSetSize << std::endl; } } @@ -102,6 +110,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @param[in] tex1d_list List of 1D textures. @@ -111,14 +120,20 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, const jit::texture2d_list &tex2d_list) { NSError *error; - id function = [library newFunctionWithName:[NSString stringWithCString:kernel_name.c_str() - encoding:NSUTF8StringEncoding]]; + MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new]; + funcDesc.options = MTLFunctionOptionNone; + funcDesc.name = [NSString stringWithCString:kernel_name.c_str() + encoding:NSUTF8StringEncoding]; + + id function = [library newFunctionWithDescriptor:funcDesc + error:&error]; MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new]; compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES; @@ -128,17 +143,17 @@ namespace gpu { compute.buffers[i].mutability = bufferMutability[kernel_name][i]; } - id pipline = [device newComputePipelineStateWithDescriptor:compute - options:MTLPipelineOptionNone - reflection:NULL - error:&error]; + id pipeline = [device newComputePipelineStateWithDescriptor:compute + options:MTLPipelineOptionNone + reflection:NULL + error:&error]; if (error) { NSLog(@"%@", error); } std::vector> buffers; - std::set *> needed_buffers; + std::unordered_set *> needed_buffers; const size_t buffer_element_size = sizeof(float); for (graph::shared_variable &input : inputs) { @@ -147,8 +162,6 @@ namespace gpu { kernel_arguments[input.get()] = [device newBufferWithBytes:buffer.data() length:buffer.size()*buffer_element_size options:MTLResourceStorageModeShared]; - buffers.push_back(kernel_arguments[input.get()]); - needed_buffers.insert(input.get()); } if (!needed_buffers.contains(input.get())) { buffers.push_back(kernel_arguments[input.get()]); @@ -156,15 +169,25 @@ namespace gpu { } } for (graph::shared_leaf &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float) + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float) + options:MTLResourceStorageModeShared]; + } + if (!needed_buffers.contains(output.get())) { + buffers.push_back(kernel_arguments[output.get()]); + needed_buffers.insert(output.get()); + } + } + } + for (graph::shared_variable &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + kernel_arguments[atomic.get()] = [device newBufferWithLength:atomic->size()*buffer_element_size options:MTLResourceStorageModeShared]; - buffers.push_back(kernel_arguments[output.get()]); - needed_buffers.insert(output.get()); } - if (!needed_buffers.contains(output.get())) { - buffers.push_back(kernel_arguments[output.get()]); - needed_buffers.insert(output.get()); + if (!needed_buffers.contains(atomic.get())) { + buffers.push_back(kernel_arguments[atomic.get()]); + needed_buffers.insert(atomic.get()); } } if (state.get()) { @@ -187,7 +210,7 @@ namespace gpu { descriptor.textureType = MTLTextureType1D; descriptor.pixelFormat = MTLPixelFormatR32Float; descriptor.width = size; - descriptor.storageMode = MTLStorageModeManaged; + descriptor.storageMode = MTLStorageModeShared; descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined; descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked; descriptor.usage = MTLTextureUsageShaderRead; @@ -208,7 +231,7 @@ namespace gpu { descriptor.pixelFormat = MTLPixelFormatR32Float; descriptor.width = size[1]; descriptor.height = size[0]; - descriptor.storageMode = MTLStorageModeManaged; + descriptor.storageMode = MTLStorageModeShared; descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined; descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked; descriptor.usage = MTLTextureUsageShaderRead; @@ -229,29 +252,35 @@ namespace gpu { NSRange range = NSMakeRange(0, buffers.size()); NSRange tex_range = NSMakeRange(0, textures.size()); - NSUInteger threads_per_group = pipline.maxTotalThreadsPerThreadgroup; - NSUInteger thread_width = pipline.threadExecutionWidth; - NSUInteger thread_groups = num_rays/threads_per_group + (num_rays%threads_per_group ? 1 : 0); + NSUInteger total_parallel = state.get() ? state->size() : num_rays; + NSUInteger thread_width = pipeline.threadExecutionWidth; + NSUInteger threads_per_group = total_parallel < pipeline.maxTotalThreadsPerThreadgroup ? + thread_width : + pipeline.maxTotalThreadsPerThreadgroup; + NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); if (jit::verbose) { - std::cout << " Kernel name : " << kernel_name << std::endl; - std::cout << " Thread execution width : " << thread_width << std::endl; - std::cout << " Threads per group : " << threads_per_group << std::endl; - std::cout << " Number of groups : " << thread_groups << std::endl; - std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; + std::cout << " Kernel name : " << kernel_name << std::endl + << " Thread execution width : " << thread_width << std::endl + << " Threads per group : " << threads_per_group << std::endl + << " Number of groups : " << thread_groups << std::endl + << " Total problem size : " << threads_per_group*thread_groups << std::endl + << " Total parallel size : " << total_parallel << std::endl + << " Current allocation size : " << device.currentAllocatedSize << std::endl; } if (state.get()) { - return [this, num_rays, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures] () mutable { + + return [this, num_rays, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { command_buffer = [queue commandBuffer]; - for (uint32_t i = 0; i < num_rays; i += threads_per_group) { + for (NSUInteger i = 0, ie = thread_groups*threads_per_group; i < num_rays; i += ie) { id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; - for (size_t j = 0, je = buffers.size() - 1; j < je; j++) { - offsets[j] = i*sizeof(float); - } - - [encoder setComputePipelineState:pipline]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffers:buffers.data() offsets:offsets.data() withRange:range]; @@ -261,19 +290,27 @@ namespace gpu { [encoder setTextures:textures.data() withRange:tex_range]; - [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) + [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; } - +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[kernel_name](id commandBuffer) { + std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; + }]; +#endif [command_buffer commit]; }; } else { - return [this, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures] () mutable { + return [this, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { command_buffer = [queue commandBuffer]; id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; - [encoder setComputePipelineState:pipline]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffers:buffers.data() offsets:offsets.data() withRange:range]; @@ -283,7 +320,11 @@ namespace gpu { [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; - +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[kernel_name](id commandBuffer) { + std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; + }]; +#endif [command_buffer commit]; }; } @@ -298,13 +339,21 @@ namespace gpu { //------------------------------------------------------------------------------ std::function create_max_call(graph::shared_leaf &argument, std::function run) { + NSError *error; + + MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new]; + funcDesc.options = MTLFunctionOptionNone; + funcDesc.name = @"max_reduction"; + + id function = [library newFunctionWithDescriptor:funcDesc + error:&error]; + MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new]; compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES; - compute.computeFunction = [library newFunctionWithName:@"max_reduction"]; + compute.computeFunction = function; compute.maxTotalThreadsPerThreadgroup = 1024; compute.buffers[0].mutability = MTLMutabilityImmutable; - NSError *error; id max_state = [device newComputePipelineStateWithDescriptor:compute options:MTLPipelineOptionNone reflection:NULL @@ -348,6 +397,94 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will be memset a buffer to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector> buffers; + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + kernel_arguments[input.get()] = [device newBufferWithBytes:input->data() + length:input->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + } + buffers.push_back(kernel_arguments[input.get()]); + } + + std::vector ranges; + for (id buffer : buffers) { + ranges.push_back(NSMakeRange(0, buffer.length)); + } + + return [this, buffers, ranges] () mutable { + command_buffer = [queue commandBuffer]; + id encoder = [command_buffer blitCommandEncoder]; + + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + [encoder fillBuffer:buffers[i] + range:ranges[i] + value:0]; + } + [encoder endEncoding]; +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[](id commandBuffer) { + std::cout << std::endl << " zero buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; + }]; +#endif + [command_buffer commit]; + }; + } + +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector> sources; + std::vector> destinations; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + kernel_arguments[in.get()] = [device newBufferWithBytes:in->data() + length:in->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + } + destinations.push_back(kernel_arguments[in.get()]); + + if (!kernel_arguments.contains(out.get())) { + kernel_arguments[out.get()] = [device newBufferWithBytes:out->data() + length:out->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + } + sources.push_back(kernel_arguments[out.get()]); + } + + return [this, sources, destinations] () mutable { + command_buffer = [queue commandBuffer]; + id encoder = [command_buffer blitCommandEncoder]; + + for (size_t i = 0, ie = sources.size(); i < ie; i++) { + [encoder copyFromBuffer:sources[i] + sourceOffset:0 + toBuffer:destinations[i] + destinationOffset:0 + size:sources[i].length]; + } + [encoder endEncoding]; +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[](id commandBuffer) { + std::cout << std::endl << " copy buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; + }]; +#endif + [command_buffer commit]; + }; + } + //------------------------------------------------------------------------------ /// @brief Get the compile options. //------------------------------------------------------------------------------ @@ -355,6 +492,8 @@ namespace gpu { MTLCompileOptions *options = [MTLCompileOptions new]; options.mathMode = MTLMathModeFast; options.mathFloatingPointFunctions = MTLMathFloatingPointFunctionsFast; + options.optimizationLevel = MTLLibraryOptimizationLevelDefault; + options.languageVersion = MTLLanguageVersion3_2; return options; } @@ -368,6 +507,30 @@ namespace gpu { [command_buffer waitUntilCompleted]; } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return [this, callback]() { + command_buffer = [queue commandBuffer]; + + [command_buffer addCompletedHandler:[callback](id commandBuffer) { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("callback"); +#endif + callback(); +#ifdef PROFILE_KERNELS + timer.print(); +#endif + }]; + + [command_buffer commit]; + }; + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// @@ -445,6 +608,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -452,44 +616,71 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, - graph::shared_random_state state, + graph::input_nodes atomics, + graph::shared_random_state &state, const size_t size, const std::vector &is_constant, jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "kernel void " << name << "(" << std::endl; bufferMutability[name] = std::vector (); + size_t used_thread_mem = 0; + size_t buffer_count = 0; - std::unordered_set used_args; + jit::argument_set used_args; for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { +#ifdef USE_INPUT_CACHE + if (!is_constant[i] && iterations > 1) { + const size_t needed_mem = inputs[i]->size() > 1024 ? + 1024*4 : + 32*4; + if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + } + } else if (is_constant[i] && + inputs[i]->size() < size && + inputs[i]->size() < 1024) { + const size_t needed_mem = inputs[i]->size()*4; + if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); + } + } +#endif bufferMutability[name].push_back(is_constant[i] ? MTLMutabilityMutable : MTLMutabilityImmutable); source_buffer << " " << (is_constant[i] ? "constant" : "device") << " float *" << jit::to_string('v', inputs[i].get()) - << " [[buffer(" << buffer_count++ << ")]], // " - << inputs[i]->get_symbol() -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - << " used " << usage.at(inputs[i].get()) -#endif -#endif - << std::endl; + << " [[buffer(" << buffer_count++ << ")]]"; + inputs[i]->endline(source_buffer, usage, ','); used_args.insert(inputs[i].get()); } } + assert(used_args.size() == inputs.size() && + "Kernel inputs contain duplicates."); + for (size_t i = 0, ie = outputs.size(); i < ie; i++) { - if (!used_args.contains(outputs[i].get())) { + if (!used_args.contains(outputs[i].get()) && + !graph::atomic_accumulate_1D_cast(outputs[i]).get()) { bufferMutability[name].push_back(MTLMutabilityMutable); source_buffer << " device float *" << jit::to_string('o', outputs[i].get()) @@ -498,6 +689,23 @@ namespace gpu { used_args.insert(outputs[i].get()); } } + assert(used_args.size() == inputs.size() + outputs.size() && + "Kernel outputs contain duplicates."); + + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + bufferMutability[name].push_back(MTLMutabilityMutable); + source_buffer << " device atomic_float *" + << jit::to_string('v', atomics[i].get()) + << " [[buffer(" << buffer_count++ << ")]]," + << std::endl; + used_args.insert(atomics[i].get()); + } + } + assert(used_args.size() == inputs.size() + outputs.size() + + atomics.size() && + "Kernel atomics contain duplicates."); + if (state.get()) { bufferMutability[name].push_back(MTLMutabilityMutable); source_buffer << " device mt_state *" @@ -521,47 +729,147 @@ namespace gpu { << " [[texture(" << index++ << ")]]," << std::endl; } - if (state.get()) { - source_buffer << " uint thread_index [[thread_index_in_threadgroup]]," - << std::endl; + if (thread_shared.size()) { + source_buffer << " ushort t_index [[thread_position_in_threadgroup]]," << std::endl; + source_buffer << " ushort t_total [[threads_per_threadgroup]]," << std::endl; } - source_buffer << " uint index [[thread_position_in_grid]]) {" << std::endl + source_buffer << " " + << jit::smallest_uint_type (size) + << " index [[thread_position_in_grid]]) {" << std::endl << " if ("; if (state.get()) { source_buffer << "offset + "; } source_buffer << "index < " << size << ") {" << std::endl; - for (auto &input : inputs) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (is_constant[i]) { #ifdef USE_INPUT_CACHE - if (usage.at(input.get())) { - registers[input.get()] = jit::to_string('r', input.get()); - source_buffer << " const "; - jit::add_type (source_buffer); - source_buffer << " " << registers[input.get()] << " = " - << jit::to_string('v', input.get()) - << "[index]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) + if (usage.at(inputs[i].get()) && inputs[i]->size() == size) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = " + << jit::to_string('v', inputs[i].get()) + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; + inputs[i]->endline(source_buffer, usage); + } +#else + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + + "[" + + (state.get() ? "offset + " : "") + + "index]"; #endif - << std::endl; } + } + + if (thread_shared.size()) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " threadgroup float " + << jit::to_string('t', inputs[i].get()) + << "[" << inputs[i]->size() << "]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " for(int j = t_index; j < " + << inputs[i]->size() + << "; j += t_total) {" << std::endl + << " " + << jit::to_string('t', inputs[i].get()) + << "[j] = " + << jit::to_string('v', inputs[i].get()) + << "[j]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " }" << std::endl; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " threadgroup_barrier(mem_flags::mem_threadgroup);" + << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + thread_shared.erase(inputs[i].get()); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) { + source_buffer << " threadgroup float " + << jit::to_string('t', inputs[i].get()) + << "[" + << (inputs[i]->size() > 1024 ? 1024 : 32) + << "]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; + inputs[i]->endline(source_buffer, usage); + } + } + } + + if (iterations > 1) { + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } + + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (!is_constant[i]) { +#ifdef USE_INPUT_CACHE + if (usage.at(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = "; + if (thread_shared.contains(inputs[i].get())) { + source_buffer << jit::to_string('t', inputs[i].get()) + << "[t_index]"; + } else { + source_buffer << jit::to_string('v', inputs[i].get()) + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; + } + inputs[i]->endline(source_buffer, usage); + } #else - registers[input.get()] = jit::to_string('v', input.get()) + "[index]"; + if (thread_shared.contains(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; + } else { + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[" + + (state.get() ? "offset + " : "") + + "index]"; + } #endif + } } if (state.get()) { #ifdef USE_INPUT_CACHE registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " device mt_state &" << registers[state.get()] << " = " << jit::to_string('s', state.get()) - << "[thread_index];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(input.get()) -#endif - << std::endl; + << "[index]"; + state->endline(source_buffer, usage); #else - registers[state.get()] = jit::to_string('s', state.get()) + "[thread_index]"; + registers[state.get()] = jit::to_string('s', state.get()) + + "[index]"; #endif } } @@ -574,26 +882,37 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, - const jit::register_usage &usage) { - std::unordered_set out_registers; + const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, + const size_t iterations=1) { + jit::argument_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); - source_buffer << " " - << jit::to_string('v', in.get()) - << "[index] = "; + auto a = out->compile(source_buffer, registers, + thread_mem, usage); + source_buffer << " "; + if (thread_shared.contains(in.get())) { + source_buffer << jit::to_string('t', in.get()) + << "[t_index] = "; + } else { + source_buffer << jit::to_string('v', in.get()) + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = "; + } if constexpr (SAFE_MATH) { source_buffer << "isnan(" << registers[a.get()] << ") ? 0.0 : "; @@ -604,23 +923,44 @@ namespace gpu { } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - indices, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) - << "[index] = "; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = "; if constexpr (SAFE_MATH) { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : "; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } } source_buffer << registers[a.get()] << ";" << std::endl; out_registers.insert(out.get()); } } + if (iterations > 1) { + source_buffer << " }" << std::endl; + } + for (auto &[out, in] : setters) { + if (thread_shared.contains(in.get())) { + source_buffer << " " + << jit::to_string('v', in.get()) + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = " + << jit::to_string('t', in.get()) + << "[t_index];" << std::endl; + } + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/newton.hpp b/graph_framework/newton.hpp index 62d5308..ca386eb 100644 --- a/graph_framework/newton.hpp +++ b/graph_framework/newton.hpp @@ -22,6 +22,7 @@ namespace solver { /// @param[in,out] work Workflow manager. /// @param[in] vars The unknowns to solver for. /// @param[in] inputs Inputs for jit compile. +/// @param[in] atomics Atomic inputs for jit compile. /// @param[in] func Function to find the root of. /// @param[in] state Random state node. /// @param[in] tolerance Tolerance to solve the dispersion function @@ -34,6 +35,7 @@ namespace solver { void newton(workflow::manager &work, graph::output_nodes vars, graph::input_nodes inputs, + graph::input_nodes atomics, graph::shared_leaf func, graph::shared_random_state state, const T tolerance = 1.0E-30, @@ -45,7 +47,7 @@ namespace solver { graph::variable_cast(x)}); } - work.add_converge_item(inputs, {func*func}, setters, state, + work.add_converge_item(inputs, {func*func}, setters, atomics, state, "loss_kernel", inputs.back()->size(), tolerance, max_iterations); } diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 7add76f..d085dad 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -194,20 +194,15 @@ /// jit::register_usage &usage, /// jit::texture1d_list &textures1d, /// jit::texture2d_list &textures2d, +/// jit::preamble_map &pre_funcs, /// int &avail_const_mem) { -/// if (visited.find(this) == visited.end()) { +/// if (!visited.contains(this)) { /// this->arg->compile_preamble(stream, registers, /// visited, usage, /// textures1d, textures2d, +/// pre_funcs, /// avail_const_mem); /// -/// jit::add_type (stream); -/// stream << " foo(const " -/// jit::add_type (stream); -/// stream << "x) {" -/// << " return 2*x;" -/// << "}"; -/// /// visited.insert(this); /// #ifdef SHOW_USE_COUNT /// usage[this] = 1; @@ -215,6 +210,17 @@ /// ++usage[this]; /// #endif /// } +/// +/// if (!pre_funcs.contains("foo")) { +/// visited.insert("foo"); +/// +/// jit::add_type (stream); +/// stream << " foo(const " +/// jit::add_type (stream); +/// stream << "x) {" << std::endl +/// << " return 2*x;" << std::endl +/// << "}" << std::endl; +/// } /// } /// @endcode /// The compile methods generate kernel source code. In this case we created a @@ -233,12 +239,12 @@ /// virtual shared_leaf /// compile(std::ostringstream &stream, /// jit::register_map ®isters, -/// jit::register_map &indices, +/// const jit::register_map &thread_mem, /// const jit::register_usage &usage) { /// if (registers.find(this) == registers.end()) { /// shared_leaf a = this->arg->compile(stream, /// registers, -/// indices, +/// thread_mem, /// usage); /// /// registers[this] = jit::to_string('r', this); @@ -346,6 +352,7 @@ #include #include #include +#include #include "backend.hpp" @@ -368,7 +375,7 @@ namespace graph { /// Graph complexity. const size_t complexity; /// Cache derivative terms. - std::map>> df_cache; + std::unordered_map>> df_cache; /// Node contains pseudo variables. const bool contains_pseudo; @@ -416,7 +423,9 @@ namespace graph { /// @returns The derivative of the node. //------------------------------------------------------------------------------ virtual std::shared_ptr> - df(std::shared_ptr> x) = 0; + df(std::shared_ptr> x) { + return std::shared_ptr> (); + }; //------------------------------------------------------------------------------ /// @brief Compile preamble. @@ -430,6 +439,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -438,6 +448,7 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { #ifdef SHOW_USE_COUNT if (usage.find(this) == usage.end()) { @@ -451,16 +462,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual std::shared_ptr> compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) = 0; //------------------------------------------------------------------------------ @@ -516,7 +527,7 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Convert the node to latex. //------------------------------------------------------------------------------ - virtual void to_latex() const = 0; + virtual void to_latex() const {}; //------------------------------------------------------------------------------ /// @brief Convert the node to vizgraph. @@ -560,7 +571,9 @@ namespace graph { /// /// @returns True if all the sub-nodes terminate in variables. //------------------------------------------------------------------------------ - virtual bool is_all_variables() const = 0; + virtual bool is_all_variables() const { + return false; + } //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. @@ -635,14 +648,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in] usage List of register usage count. +/// @param[in] end The end character. //------------------------------------------------------------------------------ virtual void endline(std::ostringstream &stream, - const jit::register_usage &usage) + const jit::register_usage &usage, + const char end=';') #ifndef SHOW_USE_COUNT const #endif - final { - stream << ";" + { + stream << end #ifdef SHOW_USE_COUNT << " // used " << usage.at(this) #endif @@ -659,9 +674,9 @@ namespace graph { //------------------------------------------------------------------------------ struct caches_t { /// Cache of node. - std::map>> nodes; + std::unordered_map>> nodes; /// Cache of backend buffers. - std::map> backends; + std::unordered_map> backends; }; /// A per thread instance of the cache structure. @@ -716,6 +731,165 @@ namespace graph { std::cout << stream.str() << std::endl; } +//****************************************************************************** +/// @brief Index node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing kernel thread index. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class index_node final : public leaf_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string() { + return "i"; + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a constant node from a vector. +//------------------------------------------------------------------------------ + index_node() : + leaf_node (index_node::to_string(), 1, false) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate method. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + return backend::buffer (); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + return this->is_match(x) ? one () : zero (); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual std::shared_ptr> + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + registers[this] = jit::to_string('i', this); + stream << " const "; + if constexpr (jit::use_cuda()) { + stream << "int " << registers[this] << " = index"; + } else if constexpr (jit::use_metal ()) { + stream << "uint " << registers[this] << " = index"; + } else { + stream << "size_t " << registers[this] << " = i"; + } + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "i"; + }; + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('i', this); + registers[this] = name; + stream << " " << name + << " [label = \"i\", shape = box, style = \"rounded,filled\", fillcolor = black, fontcolor = white];" << std::endl; + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the exponent of a power. +/// +/// @returns The exponent of a power like node. +//------------------------------------------------------------------------------ + virtual shared_leaf get_power_exponent() const { + return one (); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Construct an index. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @returns A reduced constant node. +//------------------------------------------------------------------------------ + template + shared_leaf index() { + auto temp = std::make_shared> (); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared index nodes. + template + using shared_index = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a index node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_index index_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //****************************************************************************** // Constant node. //****************************************************************************** @@ -727,6 +901,7 @@ namespace graph { //------------------------------------------------------------------------------ template class constant_node final : public leaf_node { + private: //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -737,7 +912,6 @@ namespace graph { return jit::format_to_string (d); } - private: /// Storage buffer for the data. const backend::buffer data; @@ -749,6 +923,9 @@ namespace graph { //------------------------------------------------------------------------------ constant_node(const backend::buffer &d) : leaf_node (constant_node::to_string(d.at(0)), 1, false), data(d) { + if constexpr (SAFE_MATH) { + assert(d.is_normal() && "Denormal encountered"); + } assert(d.size() == 1 && "Constants need to be scalar functions."); } @@ -774,16 +951,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { #ifdef USE_CONSTANT_CACHE @@ -893,15 +1070,6 @@ namespace graph { return data.has_zero(); } -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -1071,7 +1239,7 @@ namespace graph { /// @returns The evaluated value of the node. //------------------------------------------------------------------------------ virtual backend::buffer evaluate() { - return this->arg->evaluate(); + return arg->evaluate(); } //------------------------------------------------------------------------------ @@ -1083,6 +1251,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1091,12 +1260,13 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - this->arg->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); + if (!visited.contains(this)) { + arg->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1109,25 +1279,27 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { - return this->arg->compile(stream, registers, indices, usage); + return arg->compile(stream, registers, thread_mem, usage); } //------------------------------------------------------------------------------ /// @brief Get the argument. +/// +/// @returns The argument. //------------------------------------------------------------------------------ - shared_leaf get_arg() { - return this->arg; + shared_leaf get_arg() const { + return arg; } //------------------------------------------------------------------------------ @@ -1136,7 +1308,7 @@ namespace graph { /// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ virtual bool is_all_variables() const { - return this->arg->is_all_variables(); + return arg->is_all_variables(); } //------------------------------------------------------------------------------ @@ -1211,6 +1383,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1219,16 +1392,17 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - this->left->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); - this->right->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); + if (!visited.contains(this)) { + left->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + right->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1240,16 +1414,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Get the left branch. +/// +/// @returns The left argument. //------------------------------------------------------------------------------ - shared_leaf get_left() { - return this->left; + shared_leaf get_left() const { + return left; } //------------------------------------------------------------------------------ /// @brief Get the right branch. +/// +/// @returns The right argument. //------------------------------------------------------------------------------ - shared_leaf get_right() { - return this->right; + shared_leaf get_right() const { + return right; } //------------------------------------------------------------------------------ @@ -1258,8 +1436,8 @@ namespace graph { /// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ virtual bool is_all_variables() const { - return this->left->is_all_variables() && - this->right->is_all_variables(); + return left->is_all_variables() && + right->is_all_variables(); } //------------------------------------------------------------------------------ @@ -1292,7 +1470,6 @@ namespace graph { shared_leaf middle; public: - //------------------------------------------------------------------------------ /// @brief Reduces and assigns the left and right branches. /// @@ -1323,6 +1500,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1331,20 +1509,21 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - this->left->compile_preamble(stream, registers, + if (!visited.contains(this)) { + this->left->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->middle->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->right->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1355,10 +1534,12 @@ namespace graph { } //------------------------------------------------------------------------------ -/// @brief Get the right branch. +/// @brief Get the middle branch. +/// +/// @returns The middle branch. //------------------------------------------------------------------------------ - shared_leaf get_middle() { - return this->middle; + shared_leaf get_middle() const { + return middle; } //------------------------------------------------------------------------------ @@ -1368,11 +1549,245 @@ namespace graph { //------------------------------------------------------------------------------ virtual bool is_all_variables() const { return this->left->is_all_variables() && - this->middle->is_all_variables() && + middle->is_all_variables() && this->right->is_all_variables(); } }; +//****************************************************************************** +// Base N arg node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing a N branch node. +/// +/// @tparam N Number of branches of the node. +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// This ensures that the base leaf type has the common type between the two +/// template arguments. +//------------------------------------------------------------------------------ + template + class n_branch_node : public leaf_node { + protected: +/// Branches of the tree. + std::array, N> branches; + +//------------------------------------------------------------------------------ +/// @brief Check if any sub-node has a pseudo variable. +/// +/// @param[in] branches Array of branches. +/// @returns True if any branch contains pseudo. +//------------------------------------------------------------------------------ + bool any_has_pseudo(std::array, N> &branches) { + for (auto &b : branches) { + const bool test = b->has_pseudo(); + if (test) { + return test; + } + } + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Check if any sub-node has a pseudo variable. +/// +/// @param[in] branches Array of branches. +/// @returns True if any branch contains pseudo. +//------------------------------------------------------------------------------ + size_t total_complexity(std::array, N> &branches) { + size_t complexity = 1; + for (auto &b : branches) { + complexity += b->get_complexity(); + } + return complexity; + } + + public: +//------------------------------------------------------------------------------ +/// @brief Reduces and assigns the branches. +/// +/// @param[in] branches Array of branches. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + n_branch_node(std::array, N> branches, + const std::string s) : + leaf_node (s, n_branch_node::total_complexity(branches), + n_branch_node::any_has_pseudo(branches)), + branches(branches) {} + +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, + int &avail_const_mem) { + if (!visited.contains(this)) { + for (auto &b : branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + } + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Get the Nth arg. +/// +/// @param[in] index The argument index. +/// @returns The argument at the index. +//------------------------------------------------------------------------------ + shared_leaf get_arg(const size_t index) const { + return branches[index]; + } + +//------------------------------------------------------------------------------ +/// @brief Test if node acts like a variable. +/// +/// @returns True if the node acts like a variable. +//------------------------------------------------------------------------------ + virtual bool is_all_variables() const { + for (auto b : branches) { + const bool test = b->is_all_variables(); + if (!test) { + return test; + } + } + return true; + } + +//------------------------------------------------------------------------------ +/// @brief Get the exponent of a power. +/// +/// @returns Returns a power of one. +//------------------------------------------------------------------------------ + virtual std::shared_ptr> + get_power_exponent() const { + return one (); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Type trait for not having a valid derivative. +/// +/// @tparam T Type (Only used to compile time errors) +//------------------------------------------------------------------------------ + template + struct has_no_derivative : std::false_type {}; + +//------------------------------------------------------------------------------ +/// @brief Nodes without derivatives. +/// +/// Some functions have no derivative. This can be used as a base class to +/// case a compile error if a derivative node is attempted. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// @tparam BASE_NODE Base code to subclass from. +/// @tparam N Number of sub-nodes. +//------------------------------------------------------------------------------ + template, size_t N=1> + class no_derivative : public BASE_NODE { + public: + template + std::shared_ptr> + df(std::shared_ptr> x) requires(has_no_derivative::value); + +//------------------------------------------------------------------------------ +/// @brief Constructor for base leaf nodes base classes. +/// +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + leaf_node (s, 0, false) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for straight node base classes. +/// +/// @param[in] arg Node argument. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf arg, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + straight_node (arg, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base branch nodes base classes. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf l, + shared_leaf r, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + branch_node (l, r, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base triple nodes base classes. +/// +/// @param[in] l Left branch. +/// @param[in] m Middle branch. +/// @param[in] r Right branch. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf l, + shared_leaf m, + shared_leaf r, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + triple_node (l, m, r, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base n branch nodes base classes. +/// +/// @param[in] b Array of branches. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(std::array, N> b, + const std::string s) + requires(std::is_base_of_v, + no_derivative, + N>>) : + n_branch_node (b, s) {} + }; + //****************************************************************************** // Variable node. //****************************************************************************** @@ -1483,6 +1898,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1491,6 +1907,7 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { if (usage.find(this) == usage.end()) { usage[this] = 1; @@ -1504,16 +1921,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { return this->shared_from_this(); } @@ -1640,6 +2057,27 @@ namespace graph { virtual shared_leaf get_power_exponent() const { return one (); } + +//------------------------------------------------------------------------------ +/// @brief End a line in the kernel source. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in] usage List of register usage count. +/// @param[in] end The end character. +//------------------------------------------------------------------------------ + virtual void endline(std::ostringstream &stream, + const jit::register_usage &usage, + const char end=';') + #ifndef SHOW_USE_COUNT + const + #endif + { + stream << end << " // " << symbol + #ifdef SHOW_USE_COUNT + << " used " << usage.at(this) + #endif + << std::endl; + } }; //------------------------------------------------------------------------------ @@ -1713,6 +2151,10 @@ namespace graph { template using map_nodes = std::vector, shared_variable>>; +/// Convenience type alias for copying buffers. + template + using copy_nodes = std::vector, + shared_variable>>; //------------------------------------------------------------------------------ /// @brief Cast to a variable node. diff --git a/graph_framework/output.hpp b/graph_framework/output.hpp index 02b2f75..04c2fed 100644 --- a/graph_framework/output.hpp +++ b/graph_framework/output.hpp @@ -369,33 +369,17 @@ namespace output { for (variable &var : variables) { sync.lock(); if constexpr (jit::float_base) { - if constexpr (jit::complex_scalar) { - check_error(nc_put_vara_float(result.get_ncid(), - var.id, - start.data(), - count.data(), - reinterpret_cast (var.buffer))); - } else { - check_error(nc_put_vara_float(result.get_ncid(), - var.id, - start.data(), - count.data(), - var.buffer)); - } + check_error(nc_put_vara_float(result.get_ncid(), + var.id, + start.data(), + count.data(), + reinterpret_cast (var.buffer))); } else { - if constexpr (jit::complex_scalar) { - check_error(nc_put_vara_double(result.get_ncid(), - var.id, - start.data(), - count.data(), - reinterpret_cast (var.buffer))); - } else { - check_error(nc_put_vara_double(result.get_ncid(), - var.id, - start.data(), - count.data(), - var.buffer)); - } + check_error(nc_put_vara_double(result.get_ncid(), + var.id, + start.data(), + count.data(), + reinterpret_cast (var.buffer))); } sync.unlock(); } @@ -430,41 +414,21 @@ namespace output { sync.lock(); if constexpr (jit::float_base) { - if constexpr (jit::complex_scalar) { - check_error(nc_get_varm_float(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - reinterpret_cast (ref.buffer))); - } else { - check_error(nc_get_varm_float(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - ref.buffer)); - } + check_error(nc_get_varm_float(result.get_ncid(), + ref.id, + ref_start.data(), + ref_count.data(), + stride.data(), + map.data(), + reinterpret_cast (ref.buffer))); } else { - if constexpr (jit::complex_scalar) { - check_error(nc_get_varm_double(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - reinterpret_cast (ref.buffer))); - } else { - check_error(nc_get_varm_double(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - ref.buffer)); - } + check_error(nc_get_varm_double(result.get_ncid(), + ref.id, + ref_start.data(), + ref_count.data(), + stride.data(), + map.data(), + reinterpret_cast (ref.buffer))); } sync.unlock(); diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp new file mode 100644 index 0000000..1f2f659 --- /dev/null +++ b/graph_framework/particle_in_cell.hpp @@ -0,0 +1,1851 @@ +//------------------------------------------------------------------------------ +/// @file particle_in_cell.hpp +/// @brief Utilities needed for a particle in cell code. +/// +/// Defines graphs for use in Particle In Cell (PIC) codes. +//------------------------------------------------------------------------------ + +#ifndef particle_in_cell_h +#define particle_in_cell_h + +#include +#include + +#include "piecewise.hpp" +#include "workflow.hpp" +#include "random.hpp" +#include "logical.hpp" + +namespace graph { +//------------------------------------------------------------------------------ +/// @brief U Collision node. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class apply_u_node final : public no_derivative, + 7> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] branches Array of branches. +/// @return A string rep of a the node. +//------------------------------------------------------------------------------ + static std::string to_string(std::array, 7> branches) { + std::string s = "apply_u"; + for (auto &b : branches) { + s += jit::format_to_string(reinterpret_cast (b.get())); + } + return s; + } + +//------------------------------------------------------------------------------ +/// @brief Define a CPU evaluator. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +//------------------------------------------------------------------------------ + static void func(backend::buffer &x, + const backend::buffer &i, + const backend::buffer &rand, + const backend::buffer &mof, + const backend::buffer &tbnu_e_dt, + const backend::buffer &A, + const backend::buffer &B) { + const size_t size = x.size(); + assert(size == i.size() && + size == rand.size() && + size == mof.size() && + size == tbnu_e_dt.size() && + size == A.size() && + size == B.size() && + "Expected all arguments to have the same length."); + + for (size_t j = 0; j < size; j++) { + const T mof_j = mof[j]; + const T a_j = A[j]; + const T b_j = B[j]; + T temp_x = x[j]; + const T tbnu_e_dt_j = tbnu_e_dt[j]; + if constexpr (std::same_as) { + uint32_t rand_j = std::bit_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T E0 = mof_j*temp_x; + const int8_t rm = 4*(rand_j & 1) - 2; + const T C = rm*std::sqrt(tbnu_e_dt_j*E0); + temp_x = (E0*a_j + b_j + C)/mof_j; + } + } else { + uint64_t rand_j = std::bit_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T E0 = mof_j*temp_x; + const int8_t rm = 4*(rand_j & 1) - 2; + const T C = rm*std::sqrt(tbnu_e_dt_j*E0); + temp_x = (E0*a_j + b_j + C)/mof_j; + } + } + x[j] = temp_x; + } + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an apply_u_node. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +//------------------------------------------------------------------------------ + apply_u_node(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf mof, + shared_leaf tbnu_e_dt, + shared_leaf A, + shared_leaf B) : + no_derivative, 7> ({x, i, rand, mof, tbnu_e_dt, A, B}, + apply_u_node::to_string({x, i, rand, mof, tbnu_e_dt, A, B})) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of the applying the u operator. +/// +/// result = apply_u(x, i, rand, mof, tbnu_e_dt, A, B) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer x = this->branches[0]->evaluate(); + const backend::buffer i = this->branches[1]->evaluate(); + const backend::buffer rand = this->branches[2]->evaluate(); + const backend::buffer mof = this->branches[3]->evaluate(); + const backend::buffer tbnu_e_dt = this->branches[4]->evaluate(); + const backend::buffer A = this->branches[5]->evaluate(); + const backend::buffer B = this->branches[6]->evaluate(); + + apply_u_node::func(x, i, rand, mof, tbnu_e_dt, A, B); + return x; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the apply_u(x, i, rand, mof, tbnu_e_dt, A, B). +/// +/// @returns Reduced graph from apply_u. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, + int &avail_const_mem) { + if (!pre_funcs.contains("apply_u")) { + pre_funcs.insert("apply_u"); + + jit::add_type (stream); + stream << " apply_u(const "; + jit::add_type (stream); + stream << " x, const uint8_t i, uint32_t rand, const "; + jit::add_type (stream); + stream << " mof, const "; + jit::add_type (stream); + stream << " tbnu_e_dt, const "; + jit::add_type (stream); + stream << " A, const "; + jit::add_type (stream); + stream << " B) {" << std::endl + << " "; + jit::add_type (stream); + stream << " temp_x = x;" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " const "; + jit::add_type (stream); + stream << " E0 = mof*temp_x;" << std::endl + << " const uint8_t rm = 4*(rand & 1) - 2;" << std::endl + << " const "; + jit::add_type (stream); + stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl + << " temp_x = (E0*A + B + C)/mof;" << std::endl + << " }" << std::endl + << " return temp_x;" << std::endl + << "}" << std::endl; + } + + if (!visited.contains(this)) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + } + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); + auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); + auto rand = this->branches[2]->compile(stream, registers, thread_mem, usage); + auto mof = this->branches[3]->compile(stream, registers, thread_mem, usage); + auto tbnu_e_dt = this->branches[4]->compile(stream, registers, thread_mem, usage); + auto A = this->branches[5]->compile(stream, registers, thread_mem, usage); + auto B = this->branches[6]->compile(stream, registers, thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = apply_u(" + << registers[x.get()] << ", " + << registers[i.get()] << ", "; + stream << registers[rand.get()] << ", " + << registers[mof.get()] << ", " + << registers[tbnu_e_dt.get()] << ", " + << registers[A.get()] << ", " + << registers[B.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = apply_u_cast(x); + bool temp; + if (x_cast.get()) { + temp = this->branches[0]->is_match(x_cast->get_arg(0)); + for (size_t i = 1; i < 7 && temp; i++) { + temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); + } + } + + return temp; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "\\apply_u{\\left("; + this->branches[0]->to_latex(); + for (uint8_t i = 1; i < 7; i++) { + std::cout << ", "; + this->branches[i]->to_latex(); + } + std::cout << "\\right)}"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return apply_u(this->branches[0]->remove_pseudo(), + this->branches[1]->remove_pseudo(), + this->branches[2]->remove_pseudo(), + this->branches[3]->remove_pseudo(), + this->branches[4]->remove_pseudo(), + this->branches[5]->remove_pseudo(), + this->branches[6]->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"apply_u\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + for (auto &b : this->branches) { + auto temp = b->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[temp.get()] << ";" << std::endl; + } + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +/// @returns A reduced apply_u node. +//------------------------------------------------------------------------------ + template + shared_leaf apply_u(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf mof, + shared_leaf tbnu_e_dt, + shared_leaf A, + shared_leaf B) { + auto temp = std::make_shared> (x, i, rand, mof, + tbnu_e_dt, A, B)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared sqrt nodes. + template + using shared_apply_u = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_apply_u apply_u_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//------------------------------------------------------------------------------ +/// @brief Xi Collision node. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class apply_xi_node final : public no_derivative, + 4> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] branches Array of branches. +/// @return A string rep of a the node. +//------------------------------------------------------------------------------ + static std::string to_string(std::array, 4> branches) { + std::string s = "apply_xi"; + for (auto &b : branches) { + s += jit::format_to_string(reinterpret_cast (b.get())); + } + return s; + } + +//------------------------------------------------------------------------------ +/// @brief Define a CPU evaluator. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +//------------------------------------------------------------------------------ + static void func(backend::buffer &x, + const backend::buffer &i, + const backend::buffer &rand, + const backend::buffer &nu_D_dt) { + const size_t size = x.size(); + assert(size == i.size() && + size == rand.size() && + size == nu_D_dt.size() && + "Expected all arguments to have the same length."); + + for (size_t j = 0; j < size; j++) { + const T nu_D_dt_j = nu_D_dt[j]; + T temp_x = x[j]; + if constexpr (std::same_as) { + uint32_t rand_j = std::bit_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T A = -temp_x*nu_D_dt_j; + const int8_t rm = 2*(rand_j & 1) - 1; + const T C = rm*std::sqrt((1 - temp_x*temp_x)*nu_D_dt_j); + temp_x += A + C; + } + } else { + uint64_t rand_j = std::bit_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T A = -temp_x*nu_D_dt_j; + const int8_t rm = 2*(rand_j & 1) - 1; + const T C = rm*std::sqrt((1 - temp_x*temp_x)*nu_D_dt_j); + temp_x += A + C; + } + } + x[j] = temp_x; + } + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an apply_xi_node. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +//------------------------------------------------------------------------------ + apply_xi_node(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf nu_D_dt) : + no_derivative, 4> ({x, i, rand, nu_D_dt}, + apply_xi_node::to_string({x, i, rand, nu_D_dt})) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of the applying the xi operator. +/// +/// result = apply_xi(x, i, rand, nu_D_dt) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer x = this->branches[0]->evaluate(); + const backend::buffer i = this->branches[1]->evaluate(); + const backend::buffer rand = this->branches[2]->evaluate(); + const backend::buffer nu_D_dt = this->branches[3]->evaluate(); + + apply_xi_node::func(x, i, rand, nu_D_dt); + return x; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the apply_zi(x, i, rand, nu_D_dt). +/// +/// @returns Reduced graph from apply_xi. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, + int &avail_const_mem) { + if (!pre_funcs.contains("apply_xi")) { + pre_funcs.insert("apply_xi"); + + jit::add_type (stream); + stream << " apply_xi(const "; + jit::add_type (stream); + stream << " x, const uint8_t i, uint32_t rand, const "; + jit::add_type (stream); + stream << " nu_D_dt) {" << std::endl + << " "; + jit::add_type (stream); + stream << " temp_x = x;" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " const "; + jit::add_type (stream); + stream << " A = -temp_x*nu_D_dt;" << std::endl + << " const uint8_t rm = 2*(rand & 1) - 1;" << std::endl + << " const "; + jit::add_type (stream); + stream << " C = rm*sqrt((1 - temp_x*temp_x)*nu_D_dt);" << std::endl + << " temp_x += A + C;" << std::endl + << " }" << std::endl + << " return temp_x;" << std::endl + << "}" << std::endl; + } + + if (!visited.contains(this)) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + } + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); + auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); + auto rand = this->branches[2]->compile(stream, registers, thread_mem, usage); + auto nu_D_dt = this->branches[3]->compile(stream, registers, thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = apply_xi(" + << registers[x.get()] << ", " + << registers[i.get()] << ", "; + stream << registers[rand.get()] << ", " + << registers[nu_D_dt.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = apply_xi_cast(x); + bool temp; + if (x_cast.get()) { + temp = this->branches[0]->is_match(x_cast->get_arg(0)); + for (size_t i = 1; i < 4 && temp; i++) { + temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); + } + } + + return temp; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "\\apply_xi{\\left("; + this->branches[0]->to_latex(); + for (uint8_t i = 1; i < 4; i++) { + std::cout << ", "; + this->branches[i]->to_latex(); + } + std::cout << "\\right)}"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return apply_xi(this->branches[0]->remove_pseudo(), + this->branches[1]->remove_pseudo(), + this->branches[2]->remove_pseudo(), + this->branches[3]->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"apply_xi\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + for (auto &b : this->branches) { + auto temp = b->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[temp.get()] << ";" << std::endl; + } + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build apply_xi node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +/// @returns A reduced apply_xi node. +//------------------------------------------------------------------------------ + template + shared_leaf apply_xi(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf nu_D_dt) { + auto temp = std::make_shared> (x, i, rand, + nu_D_dt)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared sqrt nodes. + template + using shared_apply_xi = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_apply_xi apply_xi_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } +} + +namespace pic { +// FIXME: This should be in a separate file of physics constants. +/// Speed of light m/s. + template + constexpr T c = static_cast (299792458.0); +/// Vacuum permitivity F/m. + template + constexpr T epsilon0 = static_cast (8.8541878188E-12); +/// Fundamental charge coulombs. + template + constexpr T q = static_cast (1.602176634E-19); +/// Hydrogen mass kg. + template + constexpr T m_hydrogen = static_cast (1.67362192595E-27); +/// Atomic mass + template + constexpr T m_atomic = static_cast (1.66053906892E-27); +/// Electron mass kg. + template + constexpr T m_electron = static_cast (9.1093837139E-31); +/// Boltzman constant. + template + constexpr T kb = static_cast (1.380649E-23); + +//------------------------------------------------------------------------------ +/// @brief Characteristic factors. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class characteristics { + private: +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_masses Ion masses. +/// @returns (∑(m_i) + me)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_m(const std::vector &ion_masses) { + T total_m = static_cast (0); + for (const T &mass : ion_masses) { + total_m += mass; + } + return total_m/ion_masses.size(); + } + +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_zs Ion Z. +/// @returns (∑(Z_i)*q + q)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_q(const std::vector &ion_zs) { + T total_q = static_cast (0); + for (const uint8_t &z : ion_zs) { + total_q += z*pic::q; + } + return total_q/ion_zs.size(); + } + + public: +/// Mass + const T m; +/// Charge + const T q; +/// Electron density. + const T ne; +/// Plasma Frequency. + const T wpe; +/// Time. + const T t; +/// Length + const T l; +/// Velocity + const T v; +/// Electron temperature; + const T te; +/// Electric field; + const T efield; +/// Magnetic field; + const T bfield; + +//------------------------------------------------------------------------------ +/// @brief Construct the characteristics. +/// +/// @param[in] ion_masses Ion masses for all species. +/// @param[in] ion_zs Ion Z effective all species. +/// @param[in] ne Characteristic density. +//------------------------------------------------------------------------------ + characteristics(const std::vector &ion_masses, + const std::vector &ion_zs, + const T ne) : + m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), + wpe(std::sqrt(ne*q*q/(m*epsilon0))), + t(1/wpe), l(c/wpe), v(c), te(m*v*v/kb), efield(m*c/(q*t)), + bfield(efield/c) {} + +//------------------------------------------------------------------------------ +/// @brief Get the characteristic volume. +/// +/// @return l^3. +//------------------------------------------------------------------------------ + T get_volume() const { + return l*l*l; + } + }; + +//------------------------------------------------------------------------------ +/// @brief Parameter Class +//------------------------------------------------------------------------------ + template + class parameters { + public: +/// Initial magnetic field + const T b0; +/// Geometry + const T a0; +/// Filter Iterations. + const size_t filter_iterations; +/// Smoothing parameters. + const T smoothing; +/// Time step. + const T dt; +/// Parallel temperature. + const T t_para; +/// Perpendicular temperature. + const T t_perp; + +//------------------------------------------------------------------------------ +/// @brief Construct a parameters object. +/// +/// @param[in] b0 Initial magnetic field. +/// @param[in] r1 +/// @param[in] r2 +/// @param[in] filter_iterations Number of times to apply smoothing filter. +/// @param[in] smoothing Smoothing parameter. +/// @param[in] dt Time step. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + parameters(const T b0, const T r1, const T r2, + const size_t filter_iterations, + const T smoothing, const T dt, + const T t_para, const T t_perp, + const characteristics &norms) : + b0(b0/norms.bfield), + a0(std::numbers::pi_v*(r2*r2 - r1*r1)/(norms.l*norms.l)), + filter_iterations(filter_iterations), smoothing(smoothing), + dt(dt/norms.t), t_para(t_para), t_perp(t_perp) {} + }; + +//------------------------------------------------------------------------------ +/// @brief ion class. +/// +/// These values need to be initialized using normalized quantities. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class ion { + public: +/// Atomic number. + const uint8_t z; +/// Charge + const T charge; +/// Particle mass + const T mass; +/// Normalized Position + graph::shared_leaf x; +/// Normalized Parallel velocity. + graph::shared_leaf v_para; +/// Normalized Perpendicular velocity. + graph::shared_leaf v_perp; +/// Mesh Weights + std::array, 3> weights; +/// Number of real particles + const T num_real; + +//------------------------------------------------------------------------------ +/// @brief Construct an ion object. +/// +/// @param[in] mass Ion mass. +/// @param[in] z Ion Z. +/// @param[in] num_ions Number of ions. +/// @param[in] num_real Number of real particles. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + ion(const T mass, + const uint8_t z, + const size_t num_ions, + const T num_real, + const characteristics &norms) : + z(z), charge(z*pic::q/norms.q), + mass(mass), num_real(num_real), + x(graph::variable (num_ions, "x")), + v_para(graph::variable (num_ions, "v_{||}")), + v_perp(graph::variable (num_ions, "v_{\\perp}")), + weights({ + graph::variable (num_ions, "w_{0}"), + graph::variable (num_ions, "w_{1}"), + graph::variable (num_ions, "w_{2}") + }) {} + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_x() const { + return graph::variable_cast(x); + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + size_t size() const { + return get_x()->size(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for x. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *x_data() const { + return get_x()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_para() const { + return graph::variable_cast(v_para); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the parallel velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_para_data() const { + return get_v_para()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_perp() const { + return graph::variable_cast(v_perp); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the perpendicular velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_perp_data() const { + return graph::variable_cast(v_perp)->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Conversion factor from super particles to real particles. +/// +/// @returns The super to real conversion factor. +//------------------------------------------------------------------------------ + T super_to_real() const { + return num_real/size(); + } + +//------------------------------------------------------------------------------ +/// @brief Define variables. +/// +/// @param[in] file A @ref output::result_file object to define variables. +/// @param[in,out] data A @ref output::data_set object to create variable. +/// @param[in,out] work A @ref workflow::manager object where data was +/// computed. +/// @param[in] tag Unique identity for give the ion species. +//------------------------------------------------------------------------------ + void define_variables(const output::result_file &file, + output::data_set &data, + workflow::manager &work, + const std::string tag) { + data.create_variable(file, "x_" + tag, x, work.get_context()); + data.create_variable(file, "vpara_" + tag, v_para, + work.get_context()); + data.create_variable(file, "vperp_" + tag, v_perp, + work.get_context()); + } + +//------------------------------------------------------------------------------ +/// @brief Build a profile. +/// +/// @param[in] func The profile function. +/// @returns The parallel temperature profile. +//------------------------------------------------------------------------------ + graph::shared_leaf build_profile(std::function(graph::shared_leaf)> func) const { + return func(x); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Mesh class. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class mesh { + private: +//------------------------------------------------------------------------------ +/// @brief Build y index. +/// +/// @tparam I Mesh index. +/// +/// @param[in] x The x position. +/// @param[in] scale Scale factor. +/// @param[in] iterations Iterations. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_y_index(graph::shared_leaf x, + const T scale, + const size_t iterations=0) const { + auto low = iterations ? build_y_index (x - dx, scale, iterations - 1) : + graph::index_1D(y[I], x, dx, xmin + dx); + auto center = graph::index_1D(y[I], x, dx, xmin); + auto high = iterations ? build_y_index (x + dx, scale, iterations - 1) : + graph::index_1D(y[I], x, dx, xmin - dx); + + const T center_w = static_cast (0.5); + const T side_w = static_cast (0.25); + + auto b = center_w*center + side_w*low + side_w*high; + return (1 - scale)*center + scale*b; + } + + public: +/// Min x + const T xmin; +/// Max x + const T xmax; +/// Dx + const T dx; +/// Mesh y values. + std::array, 4> y; +/// Mesh point. + enum offset { +/// Lower index. + low, +/// Center index. + center, +/// Higher index. + high + }; + +//------------------------------------------------------------------------------ +/// @brief Construct a mesh object. +/// +/// @param[in] x_min Minimum X postion of mesh. +/// @param[in] x_max Maximum X position of mesh. +/// @param[in] num Number of mesh points. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + mesh(const T x_min, + const T x_max, + const size_t num, + const characteristics &norms) : + y({ + graph::variable (num, "y^{0}_{m}"), + graph::variable (num, "y^{1}_{m}"), + graph::variable (num, "y^{2}_{m}"), + graph::variable (num, "y^{3}_{m}") + }), xmin(x_min/norms.l), xmax(x_max/norms.l), + dx((xmax - xmin)/(num - 1)) {} + +//------------------------------------------------------------------------------ +/// @brief Build x index. +/// +/// @param[in] x The x position. +/// @returns The indexed mesh X position. +//------------------------------------------------------------------------------ + graph::shared_leaf build_x_index(graph::shared_leaf x) const { + return dx*graph::argument(x, dx, xmin, size()) + xmin; + } + +//------------------------------------------------------------------------------ +/// @brief Build i index. +/// +/// @param[in] x The x position. +/// @returns The indexed mesh X position. +//------------------------------------------------------------------------------ + graph::shared_leaf build_i_index(graph::shared_leaf x) const { + return graph::argument(x, dx, xmin, size()); + } + +//------------------------------------------------------------------------------ +/// @brief Build y index. +/// +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] params A @ref pic::parameters object. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_y_index(graph::shared_leaf x, + const parameters ¶ms) const { + if constexpr (O == low) { + return build_y_index (x - dx, params.smoothing, + params.filter_iterations - 1); + } else if constexpr (O == center) { + return build_y_index (x, params.smoothing, + params.filter_iterations - 1); + } else { + return build_y_index (x + dx, params.smoothing, + params.filter_iterations - 1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Build dy/dx index. +/// +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] params A @ref pic::parameters object. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_dydx_index(graph::shared_leaf x, + const parameters ¶ms) const { + const T two = 2; + if constexpr (O == low) { + auto low = build_y_index (x - two*dx, params.smoothing, + params.filter_iterations); + auto high = build_y_index (x, params.smoothing, + params.filter_iterations); + return (high - low)/two; + } else if constexpr (O == center) { + auto low = build_y_index (x - dx, params.smoothing, + params.filter_iterations); + auto high = build_y_index (x + dx, params.smoothing, + params.filter_iterations); + return (high - low)/two; + } else { + auto low = build_y_index (x, params.smoothing, + params.filter_iterations); + auto high = build_y_index (x + two*dx, params.smoothing, + params.filter_iterations); + return (high - low)/two; + } + } + +//------------------------------------------------------------------------------ +/// @brief Build mesh accumulation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @returns Expressions for mesh accumulation. +//------------------------------------------------------------------------------ + std::array, 3> build_mesh_solve(const ion &ion) const { + auto weights = build_weights(ion.x); + auto sum_low = graph::atomic_accumulate_1D(y[0], ion.x - dx, + dx, xmin, weights[0]); + auto sum = graph::atomic_accumulate_1D(y[0], ion.x, + dx, xmin, weights[1]); + auto sum_high = graph::atomic_accumulate_1D(y[0], ion.x + dx, + dx, xmin, weights[2]); + return {sum_low, sum, sum_high}; + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + size_t size() const { + return graph::variable_cast(y[0])->size(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. +/// +/// @tparam I Mesh index. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + template + T *data() const { + return graph::variable_cast(y[I])->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Define variables. +/// +/// @param[in] file A @ref output::result_file object to define variables. +/// @param[in,out] data A @ref output::data_set object to create variable. +/// @param[in] work A @ref workflow::manager object where data was +/// computed. +//------------------------------------------------------------------------------ + void define_variables(const output::result_file &file, + output::data_set &data, + workflow::manager &work) { + data.create_variable(file, "y_0", y[0], work.get_context()); + data.create_variable(file, "y_1", y[1], work.get_context()); + data.create_variable(file, "y_2", y[2], work.get_context()); + data.create_variable(file, "y_3", y[3], work.get_context()); + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation weights. +/// +/// @param[in] x The x position. +/// @returns The interpolated mesh weights. +//------------------------------------------------------------------------------ + std::array, 3> build_weights(graph::shared_leaf x) const { + auto x_off = build_x_index(x) - x; + auto xnorm1 = static_cast (1.5) + (x_off - dx)/dx; + auto xnorm2 = x_off/dx; + auto xnorm3 = static_cast (1.5) - (x_off + dx)/dx; + + auto w0 = static_cast (0.5)*xnorm1*xnorm1; + auto w1 = static_cast (0.75) - xnorm2*xnorm2; + auto w2 = static_cast (0.5)*xnorm3*xnorm3; + + return {w0, w1, w2}; + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @param[in] x The x position. +/// @returns The interpolated mesh quantity. +//------------------------------------------------------------------------------ + graph::shared_leaf build_interpolation(graph::shared_leaf x) const { + auto weights = build_weights(x); + + auto ymesh0 = graph::index_1D(y[0], x - dx, dx, xmin); + auto ymesh1 = graph::index_1D(y[0], x, dx, xmin); + auto ymesh2 = graph::index_1D(y[0], x + dx, dx, xmin); + + return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; + } + }; + +//------------------------------------------------------------------------------ +/// @brief Convert from cartesian to sphereical coordinates. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x +/// @param[in] y +/// @returns The coordinates as sphereical coordinates. +//------------------------------------------------------------------------------ + template + std::array, 3> cartesian_to_sphereical(graph::shared_leaf x, + graph::shared_leaf y) { + auto w = graph::hypot(x, y); + return { + w, x/w, + graph::none ()*graph::copysign(static_cast (1), y) + }; + } + +//------------------------------------------------------------------------------ +/// @brief Convert from sphereical to cartesian coordinates. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] w +/// @param[in] xi +/// @param[in] sinphi +/// @returns The coordinates as cartesian coordinates. +//------------------------------------------------------------------------------ + template + std::array, 2> sphereical_to_cartesian(graph::shared_leaf w, + graph::shared_leaf xi, + graph::shared_leaf sinphi) { + return { + w*xi, + graph::none ()*w*graph::sqrt(static_cast (1) - xi*xi)*sinphi + }; + } + +//------------------------------------------------------------------------------ +/// @brief Model for coulomb scattering operators. +//------------------------------------------------------------------------------ + enum model { + /// From Hinton 1983 EQ 92 and T.S. Chen 1988 EQ 50 + hinton, + /// From T.S. Chen 1988 Report EQ 57 commonly used for NBI + chen + }; + +//------------------------------------------------------------------------------ +/// @brief Calculate ν_{E}. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] xab +/// @param[in] mass_a Mass of particle a. +/// @param[in] mass_b Mass of particle b. +/// @param[in] gb +/// @param[in] nuab0 +/// @param[in] erfp_xab Derivaive of error function. +/// @returns ν_{E}. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_colision_rate(graph::shared_leaf xab, + const T mass_a, + const T mass_b, + graph::shared_leaf gb, + graph::shared_leaf nuab0, + graph::shared_leaf erfp_xab) { + auto mass_ratio = static_cast (2)*mass_a/mass_b*gb; + auto nu = nuab0/xab; + if constexpr (M == model::hinton) { + return nu*(mass_ratio - erfp_xab/xab); + } else { + return nu*mass_ratio; + } + } + +//------------------------------------------------------------------------------ +/// @brief Build ion ion collision. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] ion_b Ions for species b. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in,out] total_density Accumulated density. +/// @param[in,out] total_flux Accumulated total_flux. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_ion_ion_collision(const ion &ion_a, + const ion &ion_b, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + graph::shared_leaf total_density, + graph::shared_leaf total_flux, + const graph::shared_random_state state) { + const T dt = params.dt*norms.t; + + const T mass_b = ion_b.mass*norms.m; + const uint8_t zb2 = ion_b.z*ion_b.z; + + auto nb = build_density(ion_a.x, ion_b, mesh, norms, params)/norms.get_volume(); + auto tpara = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + }); + auto tperp = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + }); + auto tb = static_cast (0.5)*(tpara + tperp)*norms.te*kb/q; + auto nv = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + })*norms.v/norms.get_volume(); + auto uxb = nv/nb; + + total_density = total_density + nb; + total_flux = total_flux + nv; + + return build_common_collision (ion_a, norms, uxb, tb, nb, mass_b, + zb2, dt, state); + } + +//------------------------------------------------------------------------------ +/// @brief Build ion electron collision. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in] total_density Accumulated density. +/// @param[in] total_flux Accumulated total_flux. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_ion_electron_collision(const ion &ion_a, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + graph::shared_leaf total_density, + graph::shared_leaf total_flux, + const graph::shared_random_state state) { + const T dt = params.dt*norms.t; + + const T mass_b = m_electron; + const uint8_t zb2 = 1; + + auto nb = total_density; + auto tb = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + })*norms.te; + + auto uxb = total_flux/total_density; + + return build_common_collision (ion_a, norms, uxb, tb, nb, mass_b, + zb2, dt, state); + } + +//------------------------------------------------------------------------------ +/// @brief Build common collisions graphs for all species. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] uxb +/// @param[in] tb Temperature of species b. +/// @param[in] nb Density of species b. +/// @param[in] mass_b Mass of species b. +/// @param[in] zb2 Z effective squared. +/// @param[in] dt Time step. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_common_collision(const ion &ion_a, + const characteristics &norms, + graph::shared_leaf uxb, + graph::shared_leaf tb, + graph::shared_leaf nb, + const T mass_b, + const uint8_t zb2, + const T dt, + const graph::shared_random_state state) { + const T mass_a = ion_a.mass*norms.m; + const uint8_t za2 = ion_a.z*ion_a.z; + +// Convert to b frame. + auto wxa = ion_a.v_para*norms.v - uxb; + auto wya = ion_a.v_perp*norms.v; + +// Convert to sphereical. + auto sphere = cartesian_to_sphereical(wxa, wya); + +// Apply Monte-Carlo collision operator. + auto wtb = graph::sqrt(static_cast (2)*q*tb/mass_b); + auto xab = sphere[0]/wtb; + + auto erf_xab = graph::erf(xab); + auto erfp_xab = erf_xab->df(xab); + auto erfpp_xab = erfp_xab->df(xab); + + auto gb = (erf_xab - xab*erfp_xab)/(static_cast (2)*xab*xab); + auto logA = static_cast (30) - static_cast (0.5)*graph::log(nb/(tb*graph::sqrt(tb))); + auto nuab0 = nb*q*q*q*q*static_cast (za2*zb2)*logA + / (static_cast (2)*std::numbers::pi_v*mass_a*mass_a*epsilon0*epsilon0*wtb*wtb*wtb); + +// Velocity Scattering operator. + auto nu_e_dt = build_colision_rate (xab, mass_a, mass_b, gb, nuab0, erfp_xab)*dt; + auto steps = graph::min(nu_e_dt*2.5, static_cast (32)); + + nu_e_dt = nu_e_dt*steps; + + auto E_nuE_d_nu_E_dE = static_cast (0.5)*((static_cast (3.0)*(xab*erfp_xab - erf_xab) - xab*xab*erfpp_xab)/(erf_xab - xab*erfp_xab)); + + const T mof = mass_a/(static_cast (2)*q); + auto A = static_cast (1) - static_cast (2)*nu_e_dt; + auto tbnu_e_df = tb*nu_e_dt; + auto B = static_cast (2)*tbnu_e_df*(static_cast (1.5) + E_nuE_d_nu_E_dE); + + auto rand1 = graph::random (state); + auto u_op = graph::apply_u(sphere[0]*sphere[0], steps, rand1, + graph::constant(mof), tbnu_e_df, A, B); + sphere[0] = graph::sqrt(u_op); + + auto nu_D_dt = nuab0*(erf_xab - gb)/(xab*xab*xab)*dt; + steps = graph::min(nu_D_dt, static_cast (32)); + nu_D_dt = nu_D_dt/steps; + + auto rand2 = graph::random (state); + sphere[1] = graph::apply_xi(sphere[1], steps, rand2, nu_D_dt); + + sphere[1] = graph::if_(sphere[1]*sphere[1] > static_cast (1), + graph::copysign(static_cast (1), sphere[1]) - + sphere[1] % graph::copysign(static_cast (1), sphere[1]), + sphere[1]); + + auto cart = sphereical_to_cartesian(sphere[0], sphere[1], sphere[2]); + return {(cart[0] + uxb)/norms.v, cart[1]/norms.v}; + } + +//------------------------------------------------------------------------------ +/// @brief Build initialization. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in] state Random state node. +/// @returns Initialized normalized values for x, v||, and v⟂ +//------------------------------------------------------------------------------ + template + std::array,3> build_initialization(const pic::ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + const graph::shared_random_state state) { +// The mesh is already normalized so position_dist will be a normalized quantity. + auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, + state); + auto phi_dist = graph::uniform_random (static_cast (0), + static_cast (2)*std::numbers::pi_v, + state); + auto r_dist = graph::uniform_random (std::numeric_limits::min(), + static_cast (1), + state); + + const T vtpara = std::sqrt(2*params.t_para*q/ion.mass); + auto vpara = vtpara*graph::sqrt(-graph::log(r_dist)) + * graph::sin(phi_dist); + + phi_dist = graph::uniform_random (static_cast (0), + static_cast (2)*std::numbers::pi_v, + state); + r_dist = graph::uniform_random (std::numeric_limits::min(), + static_cast (1), + state); + + const T vtperp = std::sqrt(2*params.t_perp*q/ion.mass); + auto vperp1 = vtperp*graph::sqrt(-graph::log(r_dist)) + * graph::cos(phi_dist); + auto vperp2 = vtperp*graph::sqrt(-graph::log(r_dist)) + * graph::sin(phi_dist); + auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); + + return {position_dist, vpara/norms.v, vperp/norms.v}; + } + +//------------------------------------------------------------------------------ +/// @brief Build reinjected expressions. +/// +/// If the particles leave the mesh, reinitalize them using the same +/// initialization. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in] state Random state node. +/// @returns Reinjected values for x, v||, and v⟂ +//------------------------------------------------------------------------------ + template + std::array, 3> build_reinjection(const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + const graph::shared_random_state state) { + auto resampled = build_initialization(ion, mesh, norms, params, state); + auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; + + auto reinject_x = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[0], ion.x); + auto reinject_vpara = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[1], ion.v_para); + auto reinject_vperp = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[2], ion.v_perp); + return {reinject_x, reinject_vpara, reinject_vperp}; + } + +//------------------------------------------------------------------------------ +/// @brief Build a magnetic field. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The expression for the magnetic field. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_magnetic_field(graph::shared_leaf x, + const characteristics &norms, + const parameters ¶ms) { + return (x*x*(norms.l*norms.l) + static_cast (0.5))*params.b0; + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the density. +/// +/// @tparam T Base type of the calculation. +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The expression for the density. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_density(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto y = mesh.template build_y_index (x, params); + +// Compression factor. + auto cf = build_magnetic_field(x, norms, params)/params.b0; +// Scale factor. + const T sf = ion.super_to_real()/(params.a0*mesh.dx); + + return static_cast (ion.z)*y*cf*sf; + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the density gradient. +/// +/// @tparam T Base type of the calculation. +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The expression for the density. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_density_gradient(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto y = mesh.template build_dydx_index (x, params); + +// Compression factor. + auto cf = build_magnetic_field(x, norms, params)/params.b0; +// Scale factor. + const T sf = ion.super_to_real()/(params.a0*mesh.dx); + + return static_cast (ion.z)*y*cf*sf; + } + +//------------------------------------------------------------------------------ +/// @brief Build Expressions for Electron temperature. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] norms A @ref pic::characteristics object. +/// @returns The expressions for the electron temperature. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_electron_temperature(graph::shared_leaf x, + const characteristics &norms) { + return graph::constant (static_cast (2.5)*q/(norms.te*kb)); + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the electric field. +/// +/// @tparam T Base type of the calculation. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_electric_efield(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto n0 = build_density (x, ion, mesh, norms, params); + auto n1 = build_density (x, ion, mesh, norms, params); + auto n2 = build_density (x, ion, mesh, norms, params); + auto n3 = build_density (x, ion, mesh, norms, params); + + auto dn0dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn1dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn2dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn3dx = build_density_gradient (x, ion, mesh, norms, params); + + auto n = (n0 + n1 + n2 + n3)/static_cast (4); + auto dndx = (dn0dx + dn1dx + dn2dx + dn3dx)/static_cast (4); + + auto te = build_electron_temperature(x, norms); + const T scale = norms.q/q; + auto pressure = te*n*scale; + + return graph::none ()/n*(dndx*te*scale + pressure->df(x)); + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The interpolated mesh quantity. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_interpolate_efield(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto weights = mesh.build_weights(x); + + auto ymesh0 = build_electric_efield::low> (x, ion, mesh, norms, params); + auto ymesh1 = build_electric_efield::center> (x, ion, mesh, norms, params); + auto ymesh2 = build_electric_efield::high> (x, ion, mesh, norms, params); + + return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; + } + +//------------------------------------------------------------------------------ +/// @brief Build F expressions. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] z Runga Kutta substep. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns the Forces on the particles. +//------------------------------------------------------------------------------ + template + std::array, 3> build_F_expressions(const std::array, 3> z, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto bfield = build_magnetic_field (z[0], norms, params); + auto efield = build_interpolate_efield (z[0], ion, mesh, norms, params); + auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; + return { + z[1]*params.dt, + temp*params.dt, + (ion.charge/ion.mass*norms.m*efield - temp)*params.dt + }; + } + +//------------------------------------------------------------------------------ +/// @brief Build Runga Kutta step update. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns Step update expressions for x, v_para, and x_perp. +//------------------------------------------------------------------------------ + template + std::array, 3> build_rk4_step(const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { +// Step 1 + std::array, 3> Z1{ion.x, ion.v_para, ion.v_perp}; + std::array, 3> dZ1(build_F_expressions (Z1, ion, mesh, norms, params)); + +// Step 2 + std::array, 3> Z2{ + Z1[0] + dZ1[0]/static_cast (2), + Z1[1] + dZ1[1]/static_cast (2), + Z1[2] + dZ1[2]/static_cast (2) + }; + std::array, 3> dZ2(build_F_expressions (Z2, ion, mesh, norms, params)); + +// Step 3 + std::array, 3> Z3{ + Z1[0] + dZ2[0]/static_cast (2), + Z1[1] + dZ2[1]/static_cast (2), + Z1[2] + dZ2[2]/static_cast (2) + }; + std::array, 3> dZ3(build_F_expressions (Z3, ion, mesh, norms, params)); + +// Step 4 + std::array, 3> Z4{ + Z1[0] + dZ3[0], + Z1[1] + dZ3[1], + Z1[2] + dZ3[2] + }; + std::array, 3> dZ4(build_F_expressions (Z4, ion, mesh, norms, params)); + +// Rk4 Solution + return { + Z1[0] + (dZ1[0] + static_cast (2)*(dZ2[0] + dZ3[0]) + dZ4[0])/static_cast (6), + Z1[1] + (dZ1[1] + static_cast (2)*(dZ2[1] + dZ3[1]) + dZ4[1])/static_cast (6), + Z1[2] + (dZ1[2] + static_cast (2)*(dZ2[2] + dZ3[2]) + dZ4[2])/static_cast (6) + }; + } +} + +#endif /* particle_in_cell_h */ diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index c9fca38..d13c822 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -22,47 +22,330 @@ namespace graph { /// @param[in] scale Argument scale factor. /// @param[in] offset Argument offset factor. //------------------------------------------------------------------------------ -template -void compile_index(std::ostringstream &stream, - const std::string ®ister_name, - const size_t length, - const T scale, - const T offset) { - const std::string type = jit::type_to_string (); - stream << "(" << jit::smallest_uint_type (length) << ")min"; - if constexpr (!jit::use_metal ()) { - stream << "<" << type << ">"; + template + void compile_index(std::ostringstream &stream, + const std::string ®ister_name, + const size_t length, + const T scale, + const T offset) { + const std::string type = jit::type_to_string (); + stream << "(" << jit::smallest_uint_type (length) << ")min"; + if constexpr (!jit::use_metal () && + !jit::use_cuda()) { + stream << "<" << type << ">"; + } + stream << "(max"; + if constexpr (!jit::use_metal () && + !jit::use_cuda ()) { + stream << "<" << type << ">"; + } + stream << "("; + if constexpr (jit::complex_scalar) { + stream << "real("; + } + stream << "(" << register_name << " - "; + if constexpr (jit::complex_scalar) { + stream << jit::get_type_string (); + } + stream << offset << ")/"; + if constexpr (jit::complex_scalar) { + stream << jit::get_type_string (); + } + stream << scale; + if constexpr (jit::complex_scalar) { + stream << ")"; + } + stream << ","; + if constexpr (jit::use_metal () || + jit::use_cuda()) { + stream << "(" << type << ")"; + } + stream << "0),"; + if constexpr (jit::use_metal () || + jit::use_cuda()) { + stream << "(" << type << ")"; + } + stream << length - 1 << ")"; } - stream << "(max"; - if constexpr (!jit::use_metal ()) { - stream << "<" << type << ">"; - } - stream << "("; - if constexpr (jit::complex_scalar) { - stream << "real("; - } - stream << "(" << register_name << " - "; - if constexpr (jit::complex_scalar) { - stream << jit::get_type_string (); - } - stream << offset << ")/"; - if constexpr (jit::complex_scalar) { - stream << jit::get_type_string (); - } - stream << scale; - if constexpr (jit::complex_scalar) { - stream << ")"; + +//------------------------------------------------------------------------------ +/// @brief Compile an 2D index. +/// +/// 2D indicies are flattened to a single index. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in] x_register_name Register for the x argument. +/// @param[in] y_register_name Register for the x argument. +/// @param[in] num_columns The y index. +//------------------------------------------------------------------------------ + template + void compile_2D_index(std::ostringstream &stream, + const std::string &x_register_name, + const std::string &y_register_name, + const size_t num_columns) { + stream << x_register_name << "*" << num_columns << " + " + << y_register_name; } - stream << ","; - if constexpr (jit::use_metal ()) { - stream << "(" << type << ")"; + +//****************************************************************************** +// Argument node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Node class to contain the index argument. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class argument_node final : public no_derivative> { + private: +/// Scale factor. + const T scale; +/// Offset factor. + const T offset; +/// Length + const size_t length; + +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string with the argument. +/// +/// @param[in] x Argument. +/// @param[in] scale Scale factor for the argument. +/// @param[in] offset Offset factor for the argument. +/// @param[in] length Length of the array to index. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(shared_leaf x, + const T scale, + const T offset, + const size_t length) { + return jit::format_to_string(x->get_hash()) + + jit::format_to_string(scale) + + jit::format_to_string(offset) + + jit::format_to_string(length); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an Argument node. +/// +/// @param[in] x Argument. +/// @param[in] scale Scale factor for the argument. +/// @param[in] offset Offset factor for the argument. +/// @param[in] length Length of the array to index. +//------------------------------------------------------------------------------ + argument_node(shared_leaf x, + const T scale, + const T offset, + const size_t length) : + no_derivative> (x, argument_node::to_string(x, scale, + offset, + length)), + scale(scale), offset(offset), length(length) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the argument. +/// +/// Evaluate functions are only used by the minimization. So this node does not +/// evaluate the argument. Instead this only returns the data as if it were a +/// constant. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer result = this->arg->evaluate(); + backend::buffer o(1, offset); + backend::buffer s(1, scale); + result = result - o; + result = result/s; + result.real(); + backend::buffer upper(1, static_cast (length - 1)); + backend::buffer lower(1, static_cast (0)); + result = backend::min(result, upper); + result = backend::max(result, lower); + return result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduction method. +/// +/// If all the values in the data buffer are the same. Reduce to a single +/// constant. +/// +/// @returns A reduced representation of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + if (constant_cast(this->arg).get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// x' = (x - xmin)/dx (1) +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->arg->compile(stream, registers, + thread_mem, usage); + +#ifdef USE_INDEX_CACHE + registers[this] = jit::to_string('i', this); + stream << " const " + << jit::smallest_uint_type (length) << " " + << registers[this] << " = "; + compile_index (stream, registers[a.get()], length, + scale, offset); + this->endline(stream, usage); +#else + std::ostringstream source_buffer; + compile_index (source_buffer, registers[a.get()], + length, scale, offset); + registers[this] = source_buffer.str(); +#endif + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + auto temp = argument_cast(x); + return temp.get() && + this->arg->is_match(temp->get_arg()) && + (temp->get_size() == this->length) && + (temp->get_scale() == this->scale) && + (temp->get_offset() == this->offset); + } + +//------------------------------------------------------------------------------ +/// @brief Get argument scale. +/// +/// @returns The scale factor for x. +//------------------------------------------------------------------------------ + T get_scale() const { + return scale; + } + +//------------------------------------------------------------------------------ +/// @brief Get argument offset. +/// +/// @returns The offset factor for x. +//------------------------------------------------------------------------------ + T get_offset() const { + return offset; + } + +//------------------------------------------------------------------------------ +/// @brief Get the size of the array. +/// +/// @returns The size of the array. +//------------------------------------------------------------------------------ + size_t get_size() const { + return length; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"arg\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto a = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[a.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Define argument convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @param[in] length The array length. +/// @returns A reduced argument node. +//------------------------------------------------------------------------------ + template + shared_leaf argument(shared_leaf x, + const T scale, + const T offset, + const size_t length) { + auto temp = std::make_shared> (x, scale, + offset, + length)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif } - stream << "0),"; - if constexpr (jit::use_metal ()) { - stream << "(" << type << ")"; + +/// Convenience type alias for shared argument nodes. + template + using shared_argument = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a argument node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_argument argument_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); } - stream << length - 1 << ")"; -} //****************************************************************************** // 1D Piecewise node. @@ -104,11 +387,6 @@ void compile_index(std::ostringstream &stream, template class piecewise_1D_node final : public straight_node { private: -/// Scale factor for the argument. - const T scale; -/// Offset factor for the argument. - const T offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -127,20 +405,14 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] d Backend buffer. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] d Backend buffer. +/// @param[in] x Argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) { + shared_leaf x) { return piecewise_1D_node::to_string(d) + - jit::format_to_string(x->get_hash()) + - jit::format_to_string(scale) + - jit::format_to_string(offset); + jit::format_to_string(x->get_hash()); } //------------------------------------------------------------------------------ @@ -176,18 +448,11 @@ void compile_index(std::ostringstream &stream, /// /// @param[in] d Data to initialize the piecewise constant. /// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. //------------------------------------------------------------------------------ piecewise_1D_node(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) : - straight_node (x, piecewise_1D_node::to_string(d, x, - scale, - offset)), - data_hash(piecewise_1D_node::hash_data(d)), scale(scale), - offset(offset) {} + shared_leaf x) : + straight_node (x, piecewise_1D_node::to_string(d, x)), + data_hash(piecewise_1D_node::hash_data(d)) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of the piecewise constant. @@ -212,18 +477,8 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ virtual shared_leaf reduce() { if (constant_cast(this->arg).get()) { - const T arg = (this->arg->evaluate().at(0) + offset)/scale; - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(arg), - this->get_size() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i]); - } else { - const size_t i = std::max (std::min (std::real(arg), - this->get_size() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i]); - } + const size_t i = std::real(this->arg->evaluate().at(0)); + return constant (leaf_node::caches.backends[data_hash][i]); } if (evaluate().is_same()) { @@ -251,6 +506,7 @@ void compile_index(std::ostringstream &stream, /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -259,12 +515,14 @@ void compile_index(std::ostringstream &stream, jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->arg->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); + if (registers.find(leaf_node::caches.backends[data_hash].data()) == registers.end()) { registers[leaf_node::caches.backends[data_hash].data()] = jit::to_string('a', leaf_node::caches.backends[data_hash].data()); @@ -339,36 +597,20 @@ void compile_index(std::ostringstream &stream, /// c'_i = c_i - 3*d_i*i (4) /// d'_i = d_i (5) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { -#ifdef USE_INDEX_CACHE - if (indices.find(this->arg.get()) == indices.end()) { -#endif - const size_t length = leaf_node::caches.backends[data_hash].size(); - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); -#ifdef USE_INDEX_CACHE - indices[a.get()] = jit::to_string('i', a.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[a.get()] << " = "; - compile_index (stream, registers[a.get()], length, - scale, offset); - a->endline(stream, usage); - } -#endif + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -393,42 +635,22 @@ void compile_index(std::ostringstream &stream, #endif stream << registers[leaf_node::caches.backends[data_hash].data()]; if constexpr (jit::use_metal ()) { -#ifdef USE_INDEX_CACHE stream << ".read(" - << indices[this->arg.get()] + << registers[a.get()] << ").r"; -#else - stream << ".read("; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << ").r"; -#endif #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { -#ifdef USE_INDEX_CACHE stream << ", " - << indices[this->arg.get()]; -#else - stream << ", "; - compile_index (stream, registers[a.get()], length, - scale, offset); -#endif + << registers[a.get()] if constexpr (jit::complex_scalar || jit::double_base) { stream << ")"; } stream << ")"; #endif } else { -#ifdef USE_INDEX_CACHE stream << "[" - << indices[this->arg.get()] + << registers[a.get()] << "]"; -#else - stream << "["; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << "]"; -#endif } this->endline(stream, usage); } @@ -448,12 +670,25 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = piecewise_1D_cast(x); - if (x_cast.get()) { - return this->data_hash == x_cast->data_hash && - this->is_arg_match(x); - } + return x_cast.get() && + this->data_hash == x_cast->data_hash && + this->arg->is_match(x_cast->get_arg()); + } - return false; +//------------------------------------------------------------------------------ +/// @brief Query if the nodes arguments match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = piecewise_1D_cast(x); + + return x_cast.get() && + this->arg->is_match(x_cast->get_arg()); } //------------------------------------------------------------------------------ @@ -495,18 +730,13 @@ void compile_index(std::ostringstream &stream, return true; } -//------------------------------------------------------------------------------ -/// @brief Test the constant node has a zero. -/// -/// @returns True the node has a zero constant value. -//------------------------------------------------------------------------------ - virtual bool has_constant_zero() const { - return leaf_node::caches.backends[data_hash].has_zero(); - } - //------------------------------------------------------------------------------ /// @brief Test if node acts like a variable. /// +/// @note Even though @ref graph::leaf_node define a default of false. The +/// @ref graph::straight_node subclass overrides it so we need to +/// explicitly define these nodes as constants. +/// /// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ virtual bool is_all_variables() const { @@ -514,12 +744,21 @@ void compile_index(std::ostringstream &stream, } //------------------------------------------------------------------------------ -/// @brief Test if the node acts like a power of variable. +/// @brief Test the constant node has a zero. /// -/// @returns True. +/// @returns True the node has a zero constant value. //------------------------------------------------------------------------------ - virtual bool is_power_like() const { - return true; + virtual bool has_constant_zero() const { + return leaf_node::caches.backends[data_hash].has_zero(); + } + +//------------------------------------------------------------------------------ +/// @brief Test if the node acts like a power of variable. +/// +/// @returns True. +//------------------------------------------------------------------------------ + virtual bool is_power_like() const { + return true; } //------------------------------------------------------------------------------ @@ -539,48 +778,6 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf get_power_exponent() const { return one (); } - -//------------------------------------------------------------------------------ -/// @brief Check if the args match. -/// -/// @param[in] x Node to match. -/// @returns True if the arguments match. -//------------------------------------------------------------------------------ - bool is_arg_match(shared_leaf x) { - auto temp = piecewise_1D_cast(x); - return temp.get() && - this->arg->is_match(temp->get_arg()) && - (temp->get_size() == this->get_size()) && - (temp->get_scale() == this->scale) && - (temp->get_offset() == this->offset); - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_scale() const { - return scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_offset() const { - return offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. -/// -/// @returns The size of the buffer. -//------------------------------------------------------------------------------ - size_t get_size() const { - return leaf_node::caches.backends[data_hash].size(); - } }; //------------------------------------------------------------------------------ @@ -589,20 +786,14 @@ void compile_index(std::ostringstream &stream, /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] x Argument. -/// @param[in] scale Argument scale factor. -/// @param[in] offset Argument offset factor. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] x Argument. /// @returns A reduced piecewise_1D node. //------------------------------------------------------------------------------ template shared_leaf piecewise_1D(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) { - auto temp = std::make_shared> (d, x, - scale, - offset)->reduce(); + shared_leaf x) { + auto temp = std::make_shared> (d, x)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -620,6 +811,27 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define piecewise_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @returns A reduced piecewise_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf piecewise_1D(const backend::buffer &d, + shared_leaf x, + const T scale, + const T offset) { + return piecewise_1D (d, argument(x, scale, + offset, d.size())); + } + /// Convenience type alias for shared piecewise 1D nodes. template using shared_piecewise_1D = std::shared_ptr>; @@ -685,15 +897,6 @@ void compile_index(std::ostringstream &stream, template class piecewise_2D_node final : public branch_node { private: -/// Scale factor for the x argument. - const T x_scale; -/// Offset factor for the x argument. - const T x_offset; -/// Scale factor for the y argument. - const T y_scale; -/// Offset factor for the y argument. - const T y_offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -714,27 +917,15 @@ void compile_index(std::ostringstream &stream, /// /// @param[in] d Backend buffer. /// @param[in] x X argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. /// @param[in] y Y argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(const backend::buffer &d, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { + shared_leaf y) { return piecewise_2D_node::to_string(d) + jit::format_to_string(x->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + - jit::format_to_string(y->get_hash()) + - jit::format_to_string(y_scale) + - jit::format_to_string(y_offset); + jit::format_to_string(y->get_hash()); } //------------------------------------------------------------------------------ @@ -770,31 +961,19 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ /// @brief Construct 2D a piecewise constant node. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] n Number of columns. -/// @param[in] x X Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X Argument. +/// @param[in] y Y Argument. //------------------------------------------------------------------------------ piecewise_2D_node(const backend::buffer &d, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) : + shared_leaf y) : branch_node (x, y, - piecewise_2D_node::to_string(d, - x, x_scale, x_offset, - y, y_scale, y_offset)), - data_hash(piecewise_2D_node::hash_data(d)), - num_columns(n), x_scale(x_scale), x_offset(x_offset), y_scale(y_scale), - y_offset(y_offset) { - assert(d.size()%n == 0 && + piecewise_2D_node::to_string(d, x, y)), + data_hash(piecewise_2D_node::hash_data(d)), num_columns(n) { + assert(d.size()%get_num_columns() == 0 && "Expected the data buffer to be a multiple of the number of columns."); } @@ -813,44 +992,7 @@ void compile_index(std::ostringstream &stream, /// @returns The number of columns in the constant. //------------------------------------------------------------------------------ size_t get_num_rows() const { - return leaf_node::caches.backends[data_hash].size() / - num_columns; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_x_scale() const { - return x_scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_x_offset() const { - return x_offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get y argument scale. -/// -/// @returns The scale factor for y. -//------------------------------------------------------------------------------ - T get_y_scale() const { - return y_scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get y argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_y_offset() const { - return y_offset; + return leaf_node::caches.backends[data_hash].size()/num_columns; } //------------------------------------------------------------------------------ @@ -877,58 +1019,20 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf reduce() { if (constant_cast(this->left).get() && constant_cast(this->right).get()) { - const T l = (this->left->evaluate().at(0) + x_offset)/x_scale; - const T r = (this->right->evaluate().at(0) + y_offset)/y_scale; - - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); - } else { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); - } + const size_t i = std::real(this->left->evaluate().at(0)); + const size_t j = std::real(this->right->evaluate().at(0)); + + return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); } else if (constant_cast(this->left).get()) { - const T l = (this->left->evaluate().at(0) + x_offset)/x_scale; + const size_t i = std::real(this->left->evaluate().at(0)); - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), - this->right, y_scale, y_offset); - } else { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), - this->right, y_scale, y_offset); - } + return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), + this->right); } else if (constant_cast(this->right).get()) { - const T r = (this->right->evaluate().at(0) + y_offset)/y_scale; - - if constexpr (jit::float_base) { - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), - this->left, x_scale, x_offset); - } else { - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), - this->left, x_scale, x_offset); - } + const size_t j = std::real(this->right->evaluate().at(0)); + + return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), + this->left); } if (evaluate().is_same()) { @@ -957,6 +1061,7 @@ void compile_index(std::ostringstream &stream, /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -965,27 +1070,34 @@ void compile_index(std::ostringstream &stream, jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->left->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->right->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); + if (registers.find(leaf_node::caches.backends[data_hash].data()) == registers.end()) { registers[leaf_node::caches.backends[data_hash].data()] = jit::to_string('a', leaf_node::caches.backends[data_hash].data()); const size_t length = leaf_node::caches.backends[data_hash].size(); if constexpr (jit::use_metal ()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), this->get_num_columns() + })); #endif } else { if constexpr (jit::use_cuda()) { @@ -1017,11 +1129,17 @@ void compile_index(std::ostringstream &stream, const size_t length = leaf_node::caches.backends[data_hash].size(); if constexpr (jit::use_metal ()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #endif } } @@ -1062,68 +1180,46 @@ void compile_index(std::ostringstream &stream, /// c23'_ij = Σ_k,3Σ_l,3 Max(2*k-3,0)*Max(l-2,0)*(-i)^(k-2)*(-j)^(j-3) (17) /// c33'_ij = Σ_k,3Σ_l,3 Max(k-2,0)*Max(l-2,0)*(-i)^(k-3)*(-j)^(j-3) (18) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - const size_t length = leaf_node::caches.backends[data_hash].size(); - const size_t num_rows = length/num_columns; - - shared_leaf x = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf y = this->right->compile(stream, - registers, - indices, - usage); - + auto x = this->left->compile(stream, registers, + thread_mem, usage); + auto y = this->right->compile(stream, registers, + thread_mem, usage); + + auto temp = x*static_cast (this->get_num_columns()) + y; + if constexpr (!jit::use_metal ()) { + if (registers.find(temp.get()) == registers.end()) { +#ifndef USE_CUDA_TEXTURES #ifdef USE_INDEX_CACHE - if (indices.find(x.get()) == indices.end()) { - indices[x.get()] = jit::to_string('i', x.get()); - stream << " const " - << jit::smallest_uint_type (num_rows) << " " - << indices[x.get()] << " = "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - x->endline(stream, usage); - } - if (indices.find(y.get()) == indices.end()) { - indices[y.get()] = jit::to_string('i', y.get()); - stream << " const " - << jit::smallest_uint_type (num_columns) << " " - << indices[y.get()] << " = "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - y->endline(stream, usage); - } - - auto temp = this->left + this->right; - if constexpr (!jit::use_metal () -#ifdef USE_CUDA_TEXTURES - || !jit::use_cuda() -#endif - ) { - if (indices.find(temp.get()) == indices.end()) { - indices[temp.get()] = jit::to_string('i', temp.get()); + registers[temp.get()] = jit::to_string('i', temp.get()); stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[temp.get()] << " = " - << indices[x.get()] - << "*" << num_columns << " + " - << indices[y.get()] - << ";" << std::endl; + << jit::smallest_uint_type (this->get_num_columns()*this->get_num_rows()) + << " " << registers[temp.get()] << " = "; + compile_2D_index (stream, registers[x.get()], registers[y.get()], + this->get_num_columns()); + this->endline(stream, usage); +#else + std::ostringstream source_buffer; + temp->compile(source_buffer, + registers, + thread_mem, + usage); + registers[temp.get()] = source_buffer.str(); +#endif +#endif } } -#endif registers[this] = jit::to_string('r', this); stream << " const "; @@ -1147,60 +1243,31 @@ void compile_index(std::ostringstream &stream, } #endif stream << registers[leaf_node::caches.backends[data_hash].data()]; + if constexpr (jit::use_metal ()) { -#ifdef USE_INDEX_CACHE stream << ".read(" - << jit::smallest_uint_type (std::max(num_rows, - num_columns)) + << jit::smallest_uint_type (std::max(this->get_num_rows(), + this->get_num_columns())) << "2(" - << indices[y.get()] + << registers[y.get()] << "," - << indices[x.get()] + << registers[x.get()] << ")).r"; -#else - stream << ".read(uint2("; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << ","; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << ")).r"; -#endif #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { -#ifdef USE_INDEX_CACHE stream << ", " - << indices[y.get()] + << registers[y.get()] << ", " - << indices[x.get()]; -#else - stream << ", "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << ", "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); -#endif + << registers[x.get()]; if constexpr (jit::complex_scalar || jit::double_base) { stream << ")"; } stream << ")"; #endif } else { -#ifdef USE_INDEX_CACHE - stream << "[" - << indices[temp.get()] - << "]"; -#else - stream << "["; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << "*" << num_columns << " + "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << "]"; -#endif + stream << "[" << registers[temp.get()] << "]"; } + this->endline(stream, usage); } @@ -1218,12 +1285,28 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = piecewise_2D_cast(x); - if (x_cast.get()) { - return this->data_hash == x_cast->data_hash && - this->is_arg_match(x); - } + return x_cast.get() && + this->data_hash == x_cast->data_hash && + this->num_columns == x_cast->get_num_columns() && + this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right()); + } - return false; +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// Assumes both arguments are either set or not set. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = piecewise_2D_cast(x); + + return x_cast.get() && + this->num_columns == x_cast->get_num_columns() && + this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right()); } //------------------------------------------------------------------------------ @@ -1270,21 +1353,25 @@ void compile_index(std::ostringstream &stream, } //------------------------------------------------------------------------------ -/// @brief Test the constant node has a zero. +/// @brief Test if node acts like a variable. /// -/// @returns True the node has a zero constant value. +/// @note Even though @ref graph::leaf_node define a default of false. The +/// @ref graph::branch_node subclass overrides it so we need to +/// explicitly define these nodes as constants. +/// +/// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ - virtual bool has_constant_zero() const { - return leaf_node::caches.backends[data_hash].has_zero(); + virtual bool is_all_variables() const { + return false; } //------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. +/// @brief Test the constant node has a zero. /// -/// @returns True if the node acts like a variable. +/// @returns True the node has a zero constant value. //------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; + virtual bool has_constant_zero() const { + return leaf_node::caches.backends[data_hash].has_zero(); } //------------------------------------------------------------------------------ @@ -1313,25 +1400,6 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf get_power_exponent() const { return one (); } - -//------------------------------------------------------------------------------ -/// @brief Check if the args match. -/// -/// @param[in] x Node to match. -/// @returns True if the arguments match. -//------------------------------------------------------------------------------ - bool is_arg_match(shared_leaf x) { - auto temp = piecewise_2D_cast(x); - return temp.get() && - this->left->is_match(temp->get_left()) && - this->right->is_match(temp->get_right()) && - (temp->get_num_rows() == this->get_num_rows()) && - (temp->get_num_columns() == this->get_num_columns()) && - (temp->get_x_scale() == this->x_scale) && - (temp->get_x_offset() == this->x_offset) && - (temp->get_y_scale() == this->y_scale) && - (temp->get_y_offset() == this->y_offset); - } //------------------------------------------------------------------------------ /// @brief Do the rows match. @@ -1341,11 +1409,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_row_match(shared_leaf x) { auto temp = piecewise_1D_cast(x); - return temp.get() && - this->left->is_match(temp->get_arg()) && - (temp->get_size() == this->get_num_rows()) && - (temp->get_scale() == this->x_scale) && - (temp->get_offset() == this->x_offset); + return temp.get() && this->left->is_match(temp->get_arg()); } //------------------------------------------------------------------------------ @@ -1358,11 +1422,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_col_match(shared_leaf x) { auto temp = piecewise_1D_cast(x); - return temp.get() && - this->right->is_match(temp->get_arg()) && - (temp->get_size() == this->get_num_columns()) && - (temp->get_scale() == this->y_scale) && - (temp->get_offset() == this->y_offset); + return temp.get() && this->right->is_match(temp->get_arg()); } }; @@ -1372,28 +1432,18 @@ void compile_index(std::ostringstream &stream, /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] n Number of columns. -/// @param[in] x X argument. -/// @param[in] x_scale Scale for x argument. -/// @param[in] x_offset Offset for x argument. -/// @param[in] y Argument. -/// @param[in] y_scale Scale for y argument. -/// @param[in] y_offset Offset for y argument. -/// @returns A reduced sqrt node. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] y Y argument. +/// @returns A reduced piecewise_2D node. //------------------------------------------------------------------------------ - template + template shared_leaf piecewise_2D(const backend::buffer &d, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { - auto temp = std::make_shared> (d, n, - x, x_scale, x_offset, - y, y_scale, y_offset)->reduce(); + shared_leaf y) { + auto temp = std::make_shared> (d, n, x, y)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -1411,6 +1461,36 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define piecewise_2D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] x_scale Scale for x argument. +/// @param[in] x_offset Offset for x argument. +/// @param[in] y Y argument. +/// @param[in] y_scale Scale for y argument. +/// @param[in] y_offset Offset for y argument. +/// @returns A reduced sqrt node. +//------------------------------------------------------------------------------ + template + shared_leaf piecewise_2D(const backend::buffer &d, + const size_t n, + shared_leaf x, + const T x_scale, + const T x_offset, + shared_leaf y, + const T y_scale, + const T y_offset) { + return piecewise_2D (d, n, + argument(x, x_scale, x_offset, d.size()/n), + argument(y, y_scale, y_offset, n))->reduce(); + } + /// Convenience type alias for shared piecewise 2D nodes. template using shared_piecewise_2D = std::shared_ptr>; @@ -1447,59 +1527,42 @@ void compile_index(std::ostringstream &stream, template class index_1D_node final : public branch_node { private: -/// Scale factor for the argument. - const T scale; -/// Offset factor for the argument. - const T offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] v Value to index. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] v Value to index. +/// @param[in] x Argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(shared_leaf v, - shared_leaf x, - const T scale, - const T offset) { - return jit::format_to_string(v->get_hash()) + "[" + - jit::format_to_string(x->get_hash()) + - jit::format_to_string(scale) + - jit::format_to_string(offset) + "]"; + shared_leaf x) { + return jit::format_to_string(v->get_hash()) + + jit::format_to_string(x->get_hash()); } public: //------------------------------------------------------------------------------ /// @brief Construct a 1D index. /// -/// @param[in] var Node to index. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] var Node to index. +/// @param[in] x Argument. //------------------------------------------------------------------------------ index_1D_node(shared_leaf var, - shared_leaf x, - const T scale, - const T offset) : + shared_leaf x) : branch_node (var, x, - index_1D_node::to_string(var, x, - scale, offset)), - scale(scale), offset(offset) {} + index_1D_node::to_string(var, x)) {} //------------------------------------------------------------------------------ -/// @brief Evaluate the results of the piecewise constant. +/// @brief Evaluate the results of the 1D index. /// -/// Evaluate functions are only used by the minimization. So this node does not +/// Evaluate functions are only used by the reduction. So this node does not /// evaluate the argument. Instead this only returns the data as if it were a /// constant. /// /// @returns The evaluated value of the node. //------------------------------------------------------------------------------ virtual backend::buffer evaluate() { - return this->right->evaluate(); + return this->left->evaluate(); } //------------------------------------------------------------------------------ @@ -1513,61 +1576,41 @@ void compile_index(std::ostringstream &stream, } //------------------------------------------------------------------------------ -/// @brief the node. +/// @brief Compile the node. /// /// This node first evaluates the value of the argument then chooses the /// correct index of the variable. /// /// x' = (x - xmin)/dx (1) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { -#ifdef USE_INDEX_CACHE - if (indices.find(this->right.get()) == indices.end()) { -#endif - const size_t length = variable_cast(this->left)->size(); - shared_leaf a = this->right->compile(stream, - registers, - indices, - usage); -#ifdef USE_INDEX_CACHE - indices[a.get()] = jit::to_string('i', a.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[a.get()] << " = "; - compile_index (stream, registers[a.get()], length, - scale, offset); - a->endline(stream, usage); - } -#endif - + auto a = this->right->compile(stream, registers, + thread_mem, usage); + auto var = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - auto var = this->left->compile(stream, - registers, - indices, - usage); - stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()); -#ifdef USE_INDEX_CACHE - stream << "[" << indices[this->right.get()] << "]"; -#else - stream << "["; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << "]"; -#endif + stream << " " << registers[this] << " = "; + if (thread_mem.contains(var.get())) { + stream << thread_mem.at(var.get()); + } else { + stream << jit::to_string('v', var.get()); + } + stream << "[" << registers[a.get()] << "]"; + this->endline(stream, usage); } @@ -1585,19 +1628,16 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = index_1D_cast(x); - if (x_cast.get()) { - return this->left->is_match(x_cast->get_left()) && - this->is_arg_match(x); - } - - return false; + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->is_arg_match(x); } //------------------------------------------------------------------------------ /// @brief Convert the node to latex. //------------------------------------------------------------------------------ virtual void to_latex() const { - std::cout << "r\\_" << reinterpret_cast (this->left.get()) + std::cout << "v\\_" << reinterpret_cast (this->left.get()) << "\\left[i\\_" << reinterpret_cast (this->right.get()) << "\\right]"; @@ -1628,24 +1668,6 @@ void compile_index(std::ostringstream &stream, return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if node is a constant. -/// -/// @returns True if the node is a constant. -//------------------------------------------------------------------------------ - virtual bool is_constant() const { - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -1672,42 +1694,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_arg_match(shared_leaf x) { auto temp = index_1D_cast(x); - - if (temp.get()) { - return this->right->is_match(temp->get_right()) && - (temp->get_size() == this->get_size()) && - (temp->get_scale() == this->scale) && - (temp->get_offset() == this->offset); - } - - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_scale() const { - return scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_offset() const { - return offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. -/// -/// @returns The size of the buffer. -//------------------------------------------------------------------------------ - size_t get_size() const { - return variable_cast(this->left)->size(); + return temp.get() && this->right->is_match(temp->get_right()); } }; @@ -1717,22 +1704,16 @@ void compile_index(std::ostringstream &stream, /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] v Variable to index. -/// @param[in] x Argument. -/// @param[in] scale Argument scale factor. -/// @param[in] offset Argument offset factor. -/// @returns A reduced piecewise_1D node. +/// @param[in] v Variable to index. +/// @param[in] x Argument. +/// @returns A reduced index_1D node. //------------------------------------------------------------------------------ template shared_leaf index_1D(shared_leaf v, - shared_leaf x, - const T scale, - const T offset) { - assert(variable_cast(v).get() && - "index_1D requires a variable node for first arg."); - auto temp = std::make_shared> (v, x, - scale, - offset)->reduce(); + shared_leaf x) { + assert(argument_cast(x).get() && + "index_1D requires a argument node for second arg."); + auto temp = std::make_shared> (v, x)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -1750,6 +1731,28 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define index_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @returns A reduced index_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf index_1D(shared_leaf v, + shared_leaf x, + const T scale, + const T offset) { + assert(variable_cast(v).get() && + "index_1D requires a variable node for first arg."); + return index_1D (v, argument(x, scale, offset, variable_cast(v)->size())); + } + /// Convenience type alias for shared index 1D nodes. template using shared_index_1D = std::shared_ptr>; @@ -1787,74 +1790,58 @@ void compile_index(std::ostringstream &stream, template class index_2D_node final : public triple_node { private: -/// Scale factor for the x argument. - const T x_scale; -/// Offset factor for the x argument. - const T x_offset; -/// Scale factor for the y argument. - const T y_scale; -/// Offset factor for the y argument. - const T y_offset; -/// Number of columns. - const size_t num_columns; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] v Value to index. -/// @param[in] x Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] v Value to index. +/// @param[in] x X argument. +/// @param[in] y Y argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(shared_leaf v, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { - return jit::format_to_string(v->get_hash()) + "[" + + shared_leaf y) { + return jit::format_to_string(v->get_hash()) + jit::format_to_string(x->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + "," + - jit::format_to_string(y->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + "]"; + jit::format_to_string(y->get_hash()); } +/// Number of columns. + const size_t num_columns; + public: //------------------------------------------------------------------------------ /// @brief Construct a 2D index. /// -/// @param[in] var Node to index. -/// @param[in] n Number of columns. -/// @param[in] x X Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] var Node to index. +/// @param[in] n Number of columns. +/// @param[in] x X Argument. +/// @param[in] y Y Argument. //------------------------------------------------------------------------------ index_2D_node(shared_leaf var, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) : + shared_leaf y) : triple_node (var, x, y, - index_2D_node::to_string(var, - x, x_scale, x_offset, - y, y_scale, y_offset)), - num_columns(n), x_scale(x_scale), x_offset(x_offset), y_scale(y_scale), - y_offset(y_offset) { - assert(variable_cast(this->left)->size()%n == 0 && - "Expected the data buffer to be a multiple of the number of columns."); + index_2D_node::to_string(var, x, y)), + num_columns(n) {} + +//------------------------------------------------------------------------------ +/// @brief Get the number of columns. +/// +/// @returns The number of columns in the constant. +//------------------------------------------------------------------------------ + size_t get_num_columns() const { + return num_columns; + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of columns. +/// +/// @returns The number of columns in the constant. +//------------------------------------------------------------------------------ + size_t get_num_rows() const { + return variable_cast(this->left)->size()/num_columns; } //------------------------------------------------------------------------------ @@ -1889,88 +1876,59 @@ void compile_index(std::ostringstream &stream, /// x' = (x - xmin)/dx (1) /// y' = (y - ymin)/dy (2) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - const size_t length = variable_cast(this->left)->size(); - const size_t num_rows = length/num_columns; - - shared_leaf x = this->middle->compile(stream, - registers, - indices, - usage); - shared_leaf y = this->right->compile(stream, - registers, - indices, - usage); + auto x = this->middle->compile(stream, registers, + thread_mem, usage); + auto y = this->right->compile(stream, registers, + thread_mem, usage); + + auto temp = x*static_cast (this->get_num_columns()) + + y; + if (registers.find(temp.get()) == registers.end()) { #ifdef USE_INDEX_CACHE - if (indices.find(x.get()) == indices.end()) { - indices[x.get()] = jit::to_string('i', x.get()); + registers[temp.get()] = jit::to_string('i', temp.get()); stream << " const " - << jit::smallest_uint_type (num_rows) << " " - << indices[x.get()] << " = "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - x->endline(stream, usage); - } - if (indices.find(y.get()) == indices.end()) { - indices[y.get()] = jit::to_string('i', y.get()); - stream << " const " - << jit::smallest_uint_type (num_columns) << " " - << indices[y.get()] << " = "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - y->endline(stream, usage); + << jit::smallest_uint_type (this->get_num_columns()*this->get_num_rows()) + << " " << registers[temp.get()] << " = "; + compile_2D_index (stream, registers[x.get()], registers[y.get()], + this->get_num_columns()); + this->endline(stream, usage); + +#else + std::ostringstream source_buffer; + compile_2D_index (source_buffer, registers[x.get()], registers[y.get()], + this->get_num_columns()); + registers[temp.get()] = source_buffer.str(); +#endif } - auto temp = this->middle + this->right; - if constexpr (!jit::use_metal () || - !jit::use_cuda()) { - if (indices.find(temp.get()) == indices.end()) { - indices[temp.get()] = jit::to_string('i', temp.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[temp.get()] << " = " - << indices[x.get()] - << "*" << num_columns << " + " - << indices[y.get()] - << ";" << std::endl; - } - } -#endif + auto var = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - auto var = this->left->compile(stream, - registers, - indices, - usage); - stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()); -#ifdef USE_INDEX_CACHE - stream << "[" - << indices[temp.get()] - << "]"; -#else - stream << "["; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << "*" << num_columns << " + "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << "]"; -#endif + stream << " " << registers[this] << " = "; + if (thread_mem.contains(var.get())) { + stream << thread_mem.at(var.get()); + } else { + stream << jit::to_string('v', var.get()); + } + stream << "[" << registers[temp.get()] << "]"; + this->endline(stream, usage); } @@ -1988,12 +1946,11 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = index_2D_cast(x); - if (x_cast.get()) { - return this->left->is_match(x_cast->get_left()) && - this->is_arg_match(x); - } - - return false; + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->middle->is_match(x_cast->get_middle()) && + this->right->is_match(x_cast->get_right()) && + this->num_columns == x_cast->get_num_columns(); } //------------------------------------------------------------------------------ @@ -2035,24 +1992,6 @@ void compile_index(std::ostringstream &stream, return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if node is a constant. -/// -/// @returns True if the node is a constant. -//------------------------------------------------------------------------------ - virtual bool is_constant() const { - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -2079,95 +2018,353 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_arg_match(shared_leaf x) { auto temp = index_2D_cast(x); + return temp.get() && + this->middle->is_match(temp->get_middle()) && + this->right->is_match(temp->get_right()); + } + }; - if (temp.get()) { - return this->right->is_match(temp->get_right()) && - (temp->get_size() == this->get_size()) && - (temp->get_x_scale() == this->x_scale) && - (temp->get_x_offset() == this->x_offset) && - (temp->get_y_scale() == this->y_scale) && - (temp->get_y_offset() == this->y_offset); +//------------------------------------------------------------------------------ +/// @brief Define index_2D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] y Argument. +/// @returns A reduced sqrt node. +//------------------------------------------------------------------------------ + template + shared_leaf index_2D(shared_leaf v, + const size_t n, + shared_leaf x, + shared_leaf y) { + assert(argument_cast(x).get() && + "index_2D requires a argument node for second arg."); + assert(argument_cast(y).get() && + "index_2D requires a argument node for third arg."); + auto temp = std::make_shared> (v, n, x, y)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } - return false; +//------------------------------------------------------------------------------ +/// @brief Define index_2D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] x_scale Scale for x argument. +/// @param[in] x_offset Offset for x argument. +/// @param[in] y Argument. +/// @param[in] y_scale Scale for y argument. +/// @param[in] y_offset Offset for y argument. +/// @returns A reduced sqrt node. +//------------------------------------------------------------------------------ + template + shared_leaf index_2D(shared_leaf v, + const size_t n, + shared_leaf x, + const T x_scale, + const T x_offset, + shared_leaf y, + const T y_scale, + const T y_offset) { + assert(variable_cast(v).get() && + "index_2D requires a variable node for first arg."); + return index_2D (v, n, + argument(x, x_scale, x_offset, + variable_cast(v)->size()/n), + argument(y, y_scale, + y_offset, n))->reduce(); + } + +/// Convenience type alias for shared index 2D nodes. + template + using shared_index_2D = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a index 2D node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_index_2D index_2D_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// 1D Atomic Accumulate. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing a 1D accumulated array. +/// +/// This class is used to implement summation into an array. This uses atomic +/// add to avoid race conditions when multiple threads try to accumulate. +/// +/// Indicies are selected by +/// +/// x_norm' = (x - xmin)/dx (1) +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class atomic_accumulate_1D_node final : public triple_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string with the argument. +/// +/// @param[in] v Array to accumulate to. +/// @param[in] i Index of the array. +/// @param[in] x Argument to add to the existing value. +/// @returns A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(shared_leaf v, + shared_leaf i, + shared_leaf x) { + return jit::format_to_string(v->get_hash()) + + jit::format_to_string(i->get_hash()) + + jit::format_to_string(x->get_hash()); } + public: //------------------------------------------------------------------------------ -/// @brief Get x argument scale. +/// @brief Construct a 1D index. /// -/// @returns The scale factor for x. +/// @param[in] var Array node to accumulate to. +/// @param[in] index Index into the array. +/// @param[in] x Argument to add the existing array value. +//------------------------------------------------------------------------------ + atomic_accumulate_1D_node(shared_leaf var, + shared_leaf index, + shared_leaf x) : + triple_node (var, index, x, + atomic_accumulate_1D_node::to_string(var, index, x)) {} + //------------------------------------------------------------------------------ - T get_x_scale() const { - return x_scale; +/// @brief Evaluate the results of accumulate. +/// +/// Evaluate functions are only used by the reduction. So this node does not +/// evaluate the argument. Instead this only returns the data as if it were a +/// constant. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + return this->left->evaluate(); } //------------------------------------------------------------------------------ -/// @brief Get x argument offset. +/// @brief Transform node to derivative. /// -/// @returns The offset factor for x. +/// This node is effectively. +/// +/// y_i + x (1) +/// +/// So its effective derivative is +/// +/// ∂y_i/∂z + ∂x/dz +/// +/// @param[in] x The variable to take the derivative to. +/// @return The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + return constant (static_cast (this->left->is_match(x))) + + this->right->df(x); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// This node first evaluates the value of the argument then chooses the +/// correct index of the variable. +/// +/// x' = (x - xmin)/dx (1) +/// +/// @note Since this node accumulates, the right hand side is basically. +/// +/// y[i] = atomic_add(y[i], x) (2) +/// +/// @note The atomic add varies depending on the backend. +/// +/// - Metal atomic_fetch_add_explicit +/// - Cuda atomic_add +/// - CPU std::atomic_fetch_add_explicit +/// +/// @note These functions only take atomic data types. This changes the type +/// used the kernel argument. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->left->compile(stream, registers, + thread_mem, usage); + auto index = this->middle->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('v', a.get()) + + " + " + + registers[index.get()]; + stream << " atomic"; + if constexpr (jit::use_cuda()) { + stream << "Add("; + } else if constexpr (jit::use_metal ()){ + stream << "_fetch_add_explicit("; + } else { + stream << "_ref(*("; + } + stream << registers[this]; + if constexpr (jit::use_cuda() || + jit::use_metal ()) { + stream << ", "; + } else { + stream << ")).fetch_add("; + } + stream << registers[r.get()]; + if constexpr (jit::use_cuda()) { + stream << ")"; + } else { + stream << ", memory_order_relaxed)"; + } + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + //------------------------------------------------------------------------------ - T get_x_offset() const { - return x_offset; +/// @brief Query if the nodes match. +/// +/// Assumes both arguments are either set or not set. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + auto x_cast = atomic_accumulate_1D_cast(x); + + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->is_arg_match(x) && + this->right->is_match(x_cast->get_right()); } //------------------------------------------------------------------------------ -/// @brief Get y argument scale. +/// @brief Query if the nodes arguments match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. /// -/// @returns The scale factor for y. +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. //------------------------------------------------------------------------------ - T get_y_scale() const { - return y_scale; + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = atomic_accumulate_1D_cast(x); + + return x_cast.get() && + this->middle->is_match(x_cast->get_middle()); } //------------------------------------------------------------------------------ -/// @brief Get y argument offset. +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "v\\_" << reinterpret_cast (this->left.get()) + << "\\left[i\\_" + << reinterpret_cast (this->middle.get()) + << "\\right] + "; + this->right->to_latex(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. /// -/// @returns The offset factor for x. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. //------------------------------------------------------------------------------ - T get_y_offset() const { - return y_offset; + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"r_" << reinterpret_cast (this->left.get()) + << "\", shape = hexagon, style = filled, fillcolor = black, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto m = this->middle->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[m.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); } //------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. +/// @brief Get the exponent of a power. /// -/// @returns The size of the buffer. +/// @returns The exponent of a power like node. //------------------------------------------------------------------------------ - size_t get_size() const { - return variable_cast(this->left)->size(); + virtual shared_leaf get_power_exponent() const { + return one (); } }; //------------------------------------------------------------------------------ -/// @brief Define index_2D convenience function. +/// @brief Define atomic_accumulate_1D convenience function. /// /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] v Variable to index. -/// @param[in] n Number of columns. -/// @param[in] x X argument. -/// @param[in] x_scale Scale for x argument. -/// @param[in] x_offset Offset for x argument. -/// @param[in] y Argument. -/// @param[in] y_scale Scale for y argument. -/// @param[in] y_offset Offset for y argument. -/// @returns A reduced sqrt node. +/// @param[in] v Variable to index. +/// @param[in] i Index into the variable. +/// @param[in] x Argument. +/// @returns A reduced atomic_accumulate_1D node. //------------------------------------------------------------------------------ template - shared_leaf index_2D(shared_leaf v, - const size_t n, - shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { - assert(variable_cast(v).get() && - "index_2D requires a variable node for first arg."); - auto temp = std::make_shared> (v, n, - x, x_scale, x_offset, - y, y_scale, y_offset)->reduce(); + shared_leaf atomic_accumulate_1D(shared_leaf v, + shared_leaf i, + shared_leaf x) { + assert(argument_cast(i).get() && + "atomic_accumulate_1D requires a argument node for second arg."); + auto temp = std::make_shared> (v, i, x)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -2185,12 +2382,38 @@ void compile_index(std::ostringstream &stream, #endif } -/// Convenience type alias for shared index 2D nodes. +//------------------------------------------------------------------------------ +/// @brief Define atomic_accumulate_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] x Index Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @param[in] y Argument. +/// @returns A reduced atomic_accumulate_1D node. +//------------------------------------------------------------------------------ template - using shared_index_2D = std::shared_ptr>; + shared_leaf atomic_accumulate_1D(shared_leaf v, + shared_leaf x, + const T scale, + const T offset, + shared_leaf y) { + assert(variable_cast(v).get() && + "atomic_accumulate_1D requires a variable node for first arg."); + auto index = argument(x, scale, offset, variable_cast(v)->size()); + return atomic_accumulate_1D (v, index, y); + } + +/// Convenience type alias for shared atomic accumulate 1D nodes. + template + using shared_atomic_accumulate_1D = + std::shared_ptr>; //------------------------------------------------------------------------------ -/// @brief Cast to a index 2D node. +/// @brief Cast to a atomic accumulate 1D node. /// /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. @@ -2199,8 +2422,9 @@ void compile_index(std::ostringstream &stream, /// @returns An attempted dynamic cast. //------------------------------------------------------------------------------ template - shared_index_2D index_2D_cast(shared_leaf x) { - return std::dynamic_pointer_cast> (x); + shared_atomic_accumulate_1D + atomic_accumulate_1D_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); } } diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index c96f543..085d4fd 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -95,6 +95,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -103,8 +104,15 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!pre_funcs.contains("random_state")) { + pre_funcs.insert("random_state"); + + random_state_node::compile_random_state(stream); + } + + if (!visited.contains(this)) { visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -117,16 +125,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { return this->shared_from_this(); } @@ -157,15 +165,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if all the sub-nodes terminate in variables. -/// -/// @returns True if all the sub-nodes terminate in variables. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Get the exponent of a power. /// @@ -312,31 +311,25 @@ namespace graph { /// @param[in,out] stream String buffer stream. //------------------------------------------------------------------------------ static void compile_random(std::ostringstream &stream) { - jit::add_type (stream); - stream << " random("; + stream << "uint32_t random("; if constexpr (jit::use_metal ()) { stream << "device "; } - stream <<"mt_state &state) {" << std::endl - << " uint16_t k = state.index;" << std::endl - << " uint16_t j = (k + 1) % 624;" << std::endl - << " uint32_t x = (state.array[k] & 0x80000000U) |" << std::endl - << " (state.array[j] & 0x7fffffffU);" << std::endl - << " uint32_t xA = x >> 1;" << std::endl - << " if (x & 0x00000001U) {" << std::endl - << " xA ^= 0x9908b0dfU;" << std::endl - << " }" << std::endl - << " j = (k + 397) % 624;" << std::endl - << " x = state.array[j]^xA;" << std::endl - << " state.array[k] = x;" << std::endl - << " state.index = (k + 1) % 624;" << std::endl - << " uint32_t y = x^(x >> 11);" << std::endl - << " y = y^((y << 7) & 0x9d2c5680U);" << std::endl - << " y = y^((y << 15) & 0xefc60000U);" << std::endl - << " return static_cast<"; - jit::add_type (stream); - stream << "> (y^(y >> 18));" << std::endl - << "}" << std::endl; + stream << "mt_state &state) {" << std::endl + << " const uint16_t k = state.index;" << std::endl + << " state.index = (k + 1) % 624;" << std::endl + << " uint32_t x = (state.array[k] & 0x80000000U) |" << std::endl + << " (state.array[state.index] & 0x7fffffffU);" << std::endl + << " uint32_t xA = x >> 1;" << std::endl + << " xA = x & 0x1U ? xA^0x9908b0dfU : xA;" << std::endl + << " const uint16_t j = (k + 397) % 624;" << std::endl + << " x = state.array[j]^xA;" << std::endl + << " state.array[k] = x;" << std::endl + << " uint32_t y = x^(x >> 11);" << std::endl + << " y = y^((y << 7) & 0x9d2c5680U);" << std::endl + << " y = y^((y << 15) & 0xefc60000U);" << std::endl + << " return y^(y >> 18);" << std::endl + << "}" << std::endl; } //------------------------------------------------------------------------------ @@ -378,6 +371,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -386,12 +380,13 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->arg->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT @@ -400,29 +395,42 @@ namespace graph { ++usage[this]; #endif } + +// Need to do this after visited was checked so the random_state is created +// first. + if (!pre_funcs.contains("random")) { + pre_funcs.insert("random"); + + random_node::compile_random(stream); + } } //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); - - registers[this] = "random(" + registers[a.get()] + ")"; + auto a = this->arg->compile(stream, registers, + thread_mem, usage); + + if constexpr (jit::complex_scalar) { + registers[this] = "static_cast<" + + jit::get_type_string () + + "> (random(" + + registers[a.get()] + "))"; + } else { + registers[this] = "random(" + registers[a.get()] + ")"; + } } return this->shared_from_this(); @@ -474,15 +482,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if all the sub-nodes terminate in variables. -/// -/// @returns True if all the sub-nodes terminate in variables. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Get the exponent of a power. /// @@ -553,6 +552,25 @@ namespace graph { constexpr shared_leaf random_scale() { return constant (static_cast (std::numeric_limits::max())); } + +//------------------------------------------------------------------------------ +/// @brief Create a uniform random number. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] min Minimum value. +/// @param[in] max Maximum value. +/// @param[in] state Random state node. +/// @returns A uniform random constant. +//------------------------------------------------------------------------------ + template + constexpr shared_leaf uniform_random(const R min, + const R max, + shared_random_state state) { + auto random = graph::random (state); + return (max - min)/graph::random_scale ()*random + min; + } } #endif /* random_h */ diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index 96fdd2a..576d667 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -8,8 +8,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -17,6 +17,7 @@ #include #include #include +#include namespace jit { /// Complex scalar concept. @@ -110,19 +111,19 @@ namespace jit { if constexpr (jit::use_metal ()) { return "ushort"; } else { - return "unsigned char"; + return "uint8_t"; } } else if (max_size <= std::numeric_limits::max()) { if constexpr (jit::use_metal ()) { return "ushort"; } else { - return "unsigned short"; + return "uint16_t"; } } else if (max_size <= std::numeric_limits::max()) { if constexpr (jit::use_metal ()) { return "uint"; } else { - return "unsigned int"; + return "uint32_t"; } } else { if constexpr (jit::use_metal ()) { @@ -189,11 +190,11 @@ namespace jit { /// @returns The maximum number of digits needed. //------------------------------------------------------------------------------ template - constexpr int max_base() { + constexpr T max_base() { if constexpr (float_base) { - return std::numeric_limits::max(); + return static_cast (std::numeric_limits::max()); } else { - return std::numeric_limits::max(); + return static_cast (std::numeric_limits::max()); } } @@ -247,22 +248,27 @@ namespace jit { const NODE *pointer) { assert((prefix == 'r' || prefix == 'v' || prefix == 'o' || prefix == 'a' || - prefix == 'i' || prefix == 's') && - "Expected a variable (v), register (r), output (o), array (a), index (i), or state (s) prefix."); + prefix == 'i' || prefix == 's' || + prefix == 'l' || prefix == 't') && + "Expected a variable (v), register (r), output (o), array (a), index (i), state (s), logical (l), or (t) thread prefix."); return std::string(1, prefix) + format_to_string(reinterpret_cast (pointer)); } /// Type alias for mapping node pointers to register names. - typedef std::map register_map; + typedef std::unordered_map register_map; /// Type alias for counting register usage. - typedef std::map register_usage; + typedef std::unordered_map register_usage; /// Type alias for listing visited nodes. - typedef std::set visiter_map; + typedef std::unordered_set visiter_map; /// Type alias for indexing 1D textures. - typedef std::map texture1d_list; + typedef std::unordered_map texture1d_list; /// Type alias for indexing 2D textures. - typedef std::map> texture2d_list; + typedef std::unordered_map> texture2d_list; +/// Type alias for preamble defined functions. + typedef std::unordered_set preamble_map; +/// Type for tacking thread shared memory. + typedef std::unordered_set argument_set; //------------------------------------------------------------------------------ /// @brief Define a custom comparator class. diff --git a/graph_framework/solver.hpp b/graph_framework/solver.hpp index 244fabd..2196164 100644 --- a/graph_framework/solver.hpp +++ b/graph_framework/solver.hpp @@ -329,9 +329,7 @@ namespace solver { {this->t_next, graph::variable_cast(this->t)} }; - work.add_item(inputs, outputs, setters, - graph::shared_random_state (), + work.add_item(inputs, outputs, setters, {}, NULL, "solver_kernel", inputs.back()->size()); work.compile(); @@ -964,10 +962,10 @@ namespace solver { graph::variable_cast(lambda) }; - solver::newton(this->work, { + solver::newton (this->work, { var, graph::variable_cast(lambda) - }, inputs, loss, graph::shared_random_state ()); + }, inputs, {}, loss, NULL); inputs = { graph::variable_cast(this->t), @@ -997,9 +995,7 @@ namespace solver { {this->t_next, graph::variable_cast(this->t)} }; - this->work.add_item(inputs, outputs, setters, - graph::shared_random_state (), + this->work.add_item(inputs, outputs, setters, {}, NULL, "solver_kernel", inputs.back()->size()); this->work.compile(); } diff --git a/graph_framework/timing.hpp b/graph_framework/timing.hpp index 6439c09..763b3ac 100644 --- a/graph_framework/timing.hpp +++ b/graph_framework/timing.hpp @@ -20,9 +20,7 @@ namespace timing { /// Description of what is being timed. const std::string label; /// Starting time of the measure. - const std::chrono::high_resolution_clock::time_point start; -/// Ending time of the measure. - std::chrono::high_resolution_clock::time_point end; + std::chrono::high_resolution_clock::time_point start; public: //------------------------------------------------------------------------------ @@ -33,6 +31,21 @@ namespace timing { measure_diagnostic(const std::string message = "") : label(message), start(std::chrono::high_resolution_clock::now()) {} +//------------------------------------------------------------------------------ +/// @brief Construct a time diagnostic object. +/// +/// @param[in] md Object to copy. +//------------------------------------------------------------------------------ + measure_diagnostic(const measure_diagnostic &md) : + label(md.label), start(md.start) {} + +//------------------------------------------------------------------------------ +/// @brief Reset the time. +//------------------------------------------------------------------------------ + void reset() { + start = std::chrono::high_resolution_clock::now(); + } + //------------------------------------------------------------------------------ /// @brief Print the result. //------------------------------------------------------------------------------ @@ -86,6 +99,14 @@ namespace timing { measure_diagnostic_threaded(const std::string message = "") : label(message) {} +//------------------------------------------------------------------------------ +/// @brief Construct a time diagnostic object. +/// +/// @param[in] mdt Object to copy. +//------------------------------------------------------------------------------ + measure_diagnostic_threaded(const measure_diagnostic_threaded &mdt) : + label(mdt.label) {} + //------------------------------------------------------------------------------ /// @brief Start time for a given thread. /// diff --git a/graph_framework/trigonometry.hpp b/graph_framework/trigonometry.hpp index 558edfe..c3ee845 100644 --- a/graph_framework/trigonometry.hpp +++ b/graph_framework/trigonometry.hpp @@ -68,18 +68,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { - return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + return piecewise_1D(this->evaluate(), ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Sin(ArcTan(x, y)) -> y/Sqrt(x^2 + y^2) @@ -126,21 +123,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ - virtual shared_leaf compile(std::ostringstream &stream, - jit::register_map ®isters, - jit::register_map &indices, - const jit::register_usage &usage) { + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -326,17 +322,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Cos(ArcTan(x, y)) -> x/Sqrt(x^2 + y^2) @@ -383,22 +377,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -605,11 +597,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -618,13 +608,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -633,29 +623,29 @@ namespace graph { result.atan_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.atan_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.atan_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.atan_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } return this->shared_from_this(); @@ -686,26 +676,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf r = this->right->compile(stream, - registers, - indices, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -857,7 +843,7 @@ namespace graph { return atan(l, constant (static_cast (r))); } -/// Convenience type alias for shared add nodes. +/// Convenience type alias for shared atan nodes. template using shared_atan = std::shared_ptr>; diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index 50be5f2..16a7ff3 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -12,6 +12,158 @@ /// Name space for workflows. namespace workflow { +/// Items order + enum order { +/// Pre items + pre_run_item, +/// Items + run_item, +/// Post items + post_run_item + }; + +//------------------------------------------------------------------------------ +/// @brief Interface class representing items. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class item { + public: +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) = 0; + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() = 0; + }; + +//------------------------------------------------------------------------------ +/// @brief Callback item. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class callback_item : public item { + protected: +/// Callback function. + std::function callback; +/// Kernel function. + std::function kernel; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] callback Lambda function to run. +//------------------------------------------------------------------------------ + callback_item(std::function callback) : + callback(callback) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.run_function(callback); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Clear buffer item. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class zero_item : public item { + protected: +/// Kernel function. + std::function kernel; +/// Input nodes. + graph::input_nodes inputs; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] in Input variables. +//------------------------------------------------------------------------------ + zero_item(graph::input_nodes in) : + inputs(in) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_zero_call(inputs); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Copy one buffer item to another. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class copy_item : public item { + protected: +/// Kernel function. + std::function kernel; +/// Input nodes. + graph::copy_nodes maps; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] maps Input variables to copy. +//------------------------------------------------------------------------------ + copy_item(graph::copy_nodes maps) : + maps(maps) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_copy_call(maps); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + //------------------------------------------------------------------------------ /// @brief Class representing a work item. /// @@ -19,20 +171,22 @@ namespace workflow { /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. //------------------------------------------------------------------------------ template - class work_item { + class work_item : public item { protected: +/// Kernel function. + std::function kernel; /// Name of the GPU kernel. const std::string kernel_name; -/// Name of the GPU kernel. +/// Size of the GPU kernel. const size_t kernel_size; /// Input nodes. graph::input_nodes inputs; /// Output nodes. graph::output_nodes outputs; +/// Atomic nodes. + graph::input_nodes atomics; /// Random state node. graph::shared_random_state state; -/// Kernel function. - std::function kernel; public: //------------------------------------------------------------------------------ @@ -41,20 +195,22 @@ namespace workflow { /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. /// @param[in,out] context Jit context. //------------------------------------------------------------------------------ - work_item(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, + work_item(graph::input_nodes &in, + graph::output_nodes &out, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context) : - inputs(in), outputs(out), state(state), + inputs(in), outputs(out), atomics(atomics), state(state), kernel_name(name), kernel_size(size) { - context.add_kernel(name, in, out, maps, state, size); + context.add_kernel(name, in, out, maps, atomics, state, size); } //------------------------------------------------------------------------------ @@ -64,7 +220,7 @@ namespace workflow { //------------------------------------------------------------------------------ virtual void create_kernel_call(jit::context &context) { kernel = context.create_kernel_call(kernel_name, inputs, outputs, - state, kernel_size); + atomics, state, kernel_size); } //------------------------------------------------------------------------------ @@ -82,40 +238,66 @@ namespace workflow { /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. //------------------------------------------------------------------------------ template - class loop_item final : public work_item { -/// Iterations. - const size_t num_iterations; + class loop_item final : public item { + protected: +/// Kernel function. + std::function kernel; +/// Name of the GPU kernel. + const std::string kernel_name; +/// Size of the GPU kernel. + const size_t kernel_size; +/// Input nodes. + graph::input_nodes inputs; +/// Output nodes. + graph::output_nodes outputs; +/// Atomic nodes. + graph::input_nodes atomics; +/// Random state node. + graph::shared_random_state state; public: //------------------------------------------------------------------------------ /// @brief Construct a workflow item. /// -/// @param[in] inputs Input variables. -/// @param[in] outputs Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -/// @param[in,out] context Jit context. +/// @param[in] in Input variables. +/// @param[in] out Output nodes. +/// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. +/// @param[in] state Random state node. +/// @param[in] name Name of the work item. +/// @param[in] size Size of the work item. +/// @param[in,out] context Jit context. /// @param[in] iterations Number of iterations to run the loop. //------------------------------------------------------------------------------ - loop_item(graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes maps, - graph::shared_random_state state, + loop_item(graph::input_nodes &in, + graph::output_nodes &out, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context, const size_t iterations) : - work_item (inputs, outputs, maps, state, name, size, context), - num_iterations(iterations) {} + inputs(in), outputs(out), atomics(atomics),state(state), + kernel_name(name), kernel_size(size) { + context.add_kernel(name, in, out, maps, atomics, + state, size, iterations); + } + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_kernel_call(kernel_name, inputs, outputs, + atomics, state, kernel_size); + } //------------------------------------------------------------------------------ /// @brief Run the workitem. //------------------------------------------------------------------------------ virtual void run() { - for (size_t i = 0; i < num_iterations; i++) { - work_item::run(); - } + kernel(); } }; @@ -142,6 +324,7 @@ namespace workflow { /// @param[in] inputs Input variables. /// @param[in] outputs Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. @@ -149,15 +332,17 @@ namespace workflow { /// @param[in] tol Tolerance to solve the dispersion function to. /// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ - converge_item(graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes maps, - graph::shared_random_state state, + converge_item(graph::input_nodes &inputs, + graph::output_nodes &outputs, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context, const T tol=1.0E-30, const size_t max_iter=1000) : - work_item (inputs, outputs, maps, state, name, size, context), + work_item (inputs, outputs, maps, atomics, + state, name, size, context), tolerance(tol), max_iterations(max_iter) { context.add_max_reduction(size); } @@ -217,9 +402,11 @@ namespace workflow { /// JIT context. jit::context context; /// List of pre work items. - std::vector>> preitems; + std::vector>> preitems; /// List of work items. - std::vector>> items; + std::vector>> items; +/// List of pre work items. + std::vector>> postitems; /// Use reduction. bool add_reduction; @@ -238,96 +425,202 @@ namespace workflow { manager(const size_t index) : context(index), add_reduction(false) {} //------------------------------------------------------------------------------ -/// @brief Add a pre workflow item. -/// -/// @param[in] in Input variables. -/// @param[in] out Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -//------------------------------------------------------------------------------ - void add_preitem(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, - const std::string name, const size_t size) { - preitems.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context)); +/// @brief Add a pre callback function. +/// +/// @tparam O The @ref workflow::order +/// +/// @param[in] callback Lambda function to run. +//------------------------------------------------------------------------------ + template + void add_callback_item(std::function callback) { + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (callback)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (callback)); + } else { + postitems.push_back(std::make_unique> (callback)); + } } //------------------------------------------------------------------------------ /// @brief Add a workflow item. /// -/// @param[in] in Input variables. -/// @param[in] out Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -//------------------------------------------------------------------------------ +/// @tparam O The @ref workflow::order +/// +/// @param[in] in Input variables. +/// @param[in] out Output nodes. +/// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. +/// @param[in] state Random state node. +/// @param[in] name Name of the work item. +/// @param[in] size Size of the work item. +//------------------------------------------------------------------------------ + template void add_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size) { - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context)); + } } //------------------------------------------------------------------------------ -/// @brief Add a workflow item. +/// @brief Add a zero item. +/// +/// @tparam O The @ref workflow::order +/// +/// @param[in] in Input variables. +//------------------------------------------------------------------------------ + template + void add_zero_item(graph::input_nodes in) { + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in)); + } else { + postitems.push_back(std::make_unique> (in)); + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @tparam O The @ref workflow::order +/// +/// @param[in] maps Copy maps. +//------------------------------------------------------------------------------ + template + void add_copy_item(graph::copy_nodes maps) { + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (maps)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (maps)); + } else { + postitems.push_back(std::make_unique> (maps)); + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a loop item. +/// +/// @tparam O The @ref workflow::order /// /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. /// @param[in] iterations Number of iterations. //------------------------------------------------------------------------------ + template void add_loop_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size, const size_t iterations) { - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context, - iterations)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, + iterations)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, + iterations)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, + iterations)); + } } //------------------------------------------------------------------------------ /// @brief Add a converge item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. /// @param[in] tol Tolerance to converge the function to. /// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ + template void add_converge_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size, const T tol=1.0E-30, const size_t max_iter=1000) { add_reduction = true; - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context, tol, - max_iter)); + if constexpr (O == pre_run_item) { + items.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, tol, + max_iter)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, tol, + max_iter)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, + atomics, + state, + name, size, + context, tol, + max_iter)); + } } //------------------------------------------------------------------------------ @@ -342,23 +635,30 @@ namespace workflow { for (auto &item : items) { item->create_kernel_call(context); } - } - -//------------------------------------------------------------------------------ -/// @brief Run pre work items. -//------------------------------------------------------------------------------ - void pre_run() { - for (auto &item : preitems) { - item->run(); + for (auto &item : postitems) { + item->create_kernel_call(context); } } //------------------------------------------------------------------------------ /// @brief Run work items. +/// +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ + template void run() { - for (auto &item : items) { - item->run(); + if constexpr (O == pre_run_item) { + for (auto &item : preitems) { + item->run(); + } + } else if constexpr (O == run_item) { + for (auto &item : items) { + item->run(); + } + } else { + for (auto &item : postitems) { + item->run(); + } } } diff --git a/graph_korc/xkorc.cpp b/graph_korc/xkorc.cpp index d1b7cf1..2c66230 100644 --- a/graph_korc/xkorc.cpp +++ b/graph_korc/xkorc.cpp @@ -72,7 +72,7 @@ void run_korc() { pos->get_z())/b0; workflow::manager work(thread_number); - work.add_preitem({ + work.template add_item ({ graph::variable_cast(ux), graph::variable_cast(uy), graph::variable_cast(uz), @@ -82,7 +82,7 @@ void run_korc() { {u_init->get_y(), graph::variable_cast(uy)}, {u_init->get_z(), graph::variable_cast(uz)}, {gamma_init, graph::variable_cast(gamma)} - }, graph::shared_random_state (), "initialize_gamma", local_num_particles); + }, {}, NULL, "initialize_gamma", local_num_particles); auto u_prime = u_vec - dt*u_vec->cross(b_vec)/(2.0*gamma); @@ -118,7 +118,7 @@ void run_korc() { {u_next->get_y(), graph::variable_cast(uy)}, {u_next->get_z(), graph::variable_cast(uz)}, {gamma_next, graph::variable_cast(gamma)} - }, graph::shared_random_state (), "step", local_num_particles); + }, {}, NULL, "step", local_num_particles); work.compile(); @@ -142,7 +142,7 @@ void run_korc() { t_setup.print(); const timing::measure_diagnostic t_run("Run Time"); - work.pre_run(); + work.template run (); for (size_t i = 0; i < 1000000; i++) { /* sync.join(); work.wait(); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index b36e90f..a6abe04 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -4,37 +4,11 @@ //------------------------------------------------------------------------------ #include +#include +#include #include "../graph_framework/graph_framework.hpp" -//------------------------------------------------------------------------------ -/// @brief Build density. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] x The particle position. -//------------------------------------------------------------------------------ -template -graph::shared_leaf build_density(graph::shared_leaf x) { - return graph::exp(x*x/static_cast (-0.0001)); -} - -//------------------------------------------------------------------------------ -/// @brief Build parallel electric field. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] x The particle position. -//------------------------------------------------------------------------------ -template -graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { - const T te = 1; - const T q = 1;//1.602176634E-19; - auto n = build_density (x); - auto pe = n*te; - return static_cast (-1)/(q*n)*pe->df(x); -} - //------------------------------------------------------------------------------ /// @brief Pic code. /// @@ -42,137 +16,297 @@ graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { //------------------------------------------------------------------------------ template void run_pic() { - const size_t num_particles = 1000000; - auto x = graph::variable (num_particles, "x"); - auto vpara = graph::variable (num_particles, "v||"); - - std::normal_distribution norm(0, 0.25); - std::random_device rand_d; - std::mt19937_64 engine(rand_d()); - backend::buffer a(num_particles); - backend::buffer b(num_particles); - for (size_t i = 0; i < num_particles; i++) { - a[i] = norm(engine); - b[i] = norm(engine); - } - x->set(a); - vpara->set(b); + const timing::measure_diagnostic init("Init Time"); +// Sizes + const size_t num_particles = 3000000; + const size_t num_grid = 1000; + const size_t num_ions = 1; + const size_t num_steps = 100; + const size_t num_sub_steps = 2500; - const T m = 1;//9.1093837139E-31; - const T q = 1;//1.602176634E-19; - const T te = 1; - const T dt = 0.00001; + const std::vector ion_masses{2*pic::m_atomic}; + const std::vector ion_zs{1}; - const size_t num_grid = 1000; - auto epara = graph::variable (num_grid, "e||"); - auto n = graph::variable (num_grid, "n"); - auto grid_position = graph::variable (num_grid, "x_i"); - auto particle_index = graph::variable (num_grid, "i"); - - const T scale = 2.0/999.0; - const T offset = -1.0; - backend::buffer c(num_grid); - for (size_t i = 0; i < num_grid; i++) { - c[i] = scale*i + offset; + const pic::characteristics norms(ion_masses, ion_zs, + static_cast (2.5E19)); + + std::array density_fraction{1}; + + const T lmin = static_cast (-3.0); + const T lmax = static_cast (3.0); + const T ne0 = 0.4E18; + const T b0 = 0.050072; + const T r1 = 0.0; + const T r2 = 0.5; + const T a0 = std::numbers::pi_v*(r2*r2 - r1*r1); + const T ds = (lmax - lmin)/static_cast (num_grid - 1); + + std::vector> ions; + for (size_t i = 0; i < num_ions; i++) { + T num_real = 0; + for (size_t i = 0; i < num_grid; i++) { + const T x = ds*i + lmin; + const T b = b0*(x*x + static_cast (0.5)); + const T a = a0*b0/b; + num_real += ne0*density_fraction[0]*a*ds; + } + ions.emplace_back(ion_masses[i], ion_zs[i], num_particles, + num_real, norms); } - grid_position->set(c); - auto x1 = dt*vpara; - auto vpara1 = -q/m*graph::index_1D(epara, x, scale, offset); + const T b_cv = 1.2; + const T cyclotron_frequency = ion_zs[0]*pic::q*b_cv/ion_masses[0]; + const T gyro_period = 2*std::numbers::pi_v/cyclotron_frequency; + const T dtc = 0.25; + const pic::parameters params(b0, r1, r2, 3, 1.0E-4, + dtc*gyro_period, 2.5, 2.5, norms); - auto x2 = dt*(vpara + vpara1/2.0); - auto vpara2 = -q/m*graph::index_1D(epara, x + x1/2.0, scale, offset); + pic::mesh mesh(lmin, lmax, num_grid, norms); + + auto state = graph::random_state (jit::context::max_random_state_size(num_particles), 0); + + workflow::manager work(0); + + output::result_file f_file("fields.nc", num_grid); + output::data_set mesh_dataset(f_file); - auto x3 = dt*(vpara + vpara2/2.0); - auto vpara3 = -q/m*graph::index_1D(epara, x + x2/2.0, scale, offset); + output::result_file p_file("particles.nc", num_particles); + std::vector> p_datasets(num_ions, + output::data_set (p_file)); - auto x4 = dt*(vpara + vpara3); - auto vpara4 = -q/m*graph::index_1D(epara, x + x3, scale, offset); + std::vector ion_sync; + std::thread mesh_sync; - auto x_next = x + (x1 + static_cast (2)*(x2 + x3) + x4)/static_cast (6); - auto vpara_next = vpara + (vpara1 + static_cast (2)*(vpara2 + vpara3) + vpara4)/static_cast (6); + work.template add_zero_item ({ + graph::variable_cast(mesh.y[0]) + }); - auto next_index = particle_index; - auto next_epara = epara; - auto next_n = n; + std::vector, 3>> mesh_solves; + + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + + auto ion_inits = pic::build_initialization (ions[i], mesh, + norms, params, + graph::random_state_cast(state)); + + work.template add_item ({ + ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() + }, {}, { + {ion_inits[0], ions[i].get_x()}, + {ion_inits[1], ions[i].get_v_para()}, + {ion_inits[2], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "pre_initization_" + ion_tag, num_particles); + + work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { + ion_sync.push_back(std::thread([i, &p_file, &p_datasets]() { + p_datasets[i].write(p_file); + })); + }); - const size_t batch = 1000; -// Unroll the loop - for (size_t i = 0; i < batch; i++) { - auto indexed_particle = graph::index_1D(x, next_index, - static_cast (1), - static_cast (0)); - next_index = next_index + static_cast (1); - next_epara = next_epara - + build_parallel_electric_field (indexed_particle - grid_position); - next_n = next_n + build_density(indexed_particle - grid_position); + mesh_solves.emplace_back(mesh.build_mesh_solve(ions[i])); + work.template add_item ({ + ions[i].get_x() + }, { + mesh_solves[i][0], + mesh_solves[i][1], + mesh_solves[i][2] + }, {}, { + graph::variable_cast(mesh.y[0]) + }, NULL, "pre_sum_weights_" + ion_tag, num_particles); } - workflow::manager work(0); - work.add_item({ - graph::variable_cast(particle_index), - graph::variable_cast(epara), - graph::variable_cast(n) - }, {}, { - {graph::zero (), graph::variable_cast(particle_index)}, - {graph::zero (), graph::variable_cast(epara)}, - {graph::zero (), graph::variable_cast(n)} - }, NULL, "Index_reset", num_grid); - work.add_loop_item({ - graph::variable_cast(epara), - graph::variable_cast(n), - graph::variable_cast(grid_position), - graph::variable_cast(particle_index), - graph::variable_cast(x) - }, {}, { - {next_epara, graph::variable_cast(epara)}, - {next_index, graph::variable_cast(particle_index)}, - {next_n, graph::variable_cast(n)} - }, NULL, "Compute_efield", num_grid, num_particles/batch); - work.add_item({ - graph::variable_cast(x), - graph::variable_cast(vpara), - graph::variable_cast(epara) - }, {}, { - {x_next, graph::variable_cast(x)}, - {vpara_next, graph::variable_cast(vpara)} - }, NULL, "Particle_Push", num_particles); + work.template add_copy_item ({ + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[3])} + }); - work.compile(); - - output::result_file particles_file("pic_particles.nc", num_particles); - output::data_set p_dataset(particles_file); + work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_sync = std::thread([&f_file, &mesh_dataset]() { + mesh_dataset.write(f_file); + }); + }); - p_dataset.create_variable(particles_file, "x", x, work.get_context()); - p_dataset.create_variable(particles_file, "vpara", vpara, work.get_context()); + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + work.add_callback_item([i, &ion_sync]() { + if (ion_sync[i].joinable()) { + ion_sync[i].join(); + } + }); + + auto particle_step = pic::build_rk4_step(ions[i], mesh, norms, params); + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp(), + graph::variable_cast(mesh.y[0]), + graph::variable_cast(mesh.y[1]), + graph::variable_cast(mesh.y[2]), + graph::variable_cast(mesh.y[3]) + }, {}, { + {particle_step[0], ions[i].get_x()}, + {particle_step[1], ions[i].get_v_para()}, + {particle_step[2], ions[i].get_v_perp()} + }, {}, NULL, "particle_push_" + ion_tag, num_particles); + + auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, params, + graph::random_state_cast(state)); + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp() + }, {}, { + {particle_reinject[0], ions[i].get_x()}, + {particle_reinject[1], ions[i].get_v_para()}, + {particle_reinject[2], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "particle_reinjection_" + ion_tag, num_particles); - particles_file.end_define_mode(); - - output::result_file fields_file("pic_fields.nc", num_grid); - output::data_set f_dataset(fields_file); - - f_dataset.create_variable(fields_file, "epara", epara, work.get_context()); - f_dataset.create_variable(fields_file, "n", n, work.get_context()); - - fields_file.end_define_mode(); - std::thread sync_particles([]{}); - std::thread sync_fields([]{}); - - const size_t num_steps = 1000; - for (size_t i = 0; i < num_steps; i++) { - sync_particles.join(); - sync_fields.join(); - work.run(); - sync_particles = std::thread([&particles_file, &p_dataset] () -> void { - p_dataset.write(particles_file); + work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { + ion_sync[i] = std::thread([i, &p_file, &p_datasets]() { + p_datasets[i].write(p_file); + }); }); - sync_fields = std::thread([&fields_file, &f_dataset] () -> void { - f_dataset.write(fields_file); + } + + work.add_callback_item([&mesh_sync]() { + if (mesh_sync.joinable()) { + mesh_sync.join(); + } + }); + work.add_copy_item({ + {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, + {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])} + }); + work.add_zero_item({ + graph::variable_cast(mesh.y[0]) + }); + + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + + work.add_item({ + graph::variable_cast(ions[i].x) + }, { + mesh_solves[i][0], + mesh_solves[i][1], + mesh_solves[i][2] + }, {}, { + graph::variable_cast(mesh.y[0]) + }, NULL, "sum_weights_" + ion_tag, num_particles); + } + + work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_sync = std::thread([&f_file, &mesh_dataset]() { + mesh_dataset.write(f_file); }); + }); + + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + + work.add_callback_item([i, &ion_sync]() { + if (ion_sync[i].joinable()) { + ion_sync[i].join(); + } + }); + + graph::shared_leaf total_density = graph::zero (); + graph::shared_leaf total_flux = graph::zero (); + for (size_t j = 0; j < num_ions; j++) { + const std::string inner_ion_tag = jit::format_to_string(j); + + auto coll = pic::build_ion_ion_collision (ions[i], + ions[j], + mesh, + norms, + params, + total_density, + total_flux, + graph::random_state_cast(state)); + + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp(), + graph::variable_cast(mesh.y[0]) + }, {}, { + {coll[0], ions[i].get_v_para()}, + {coll[1], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "ion_ion_" + ion_tag + "_" + inner_ion_tag, num_particles); + } + + auto coll = pic::build_ion_electron_collision (ions[i], + mesh, + norms, + params, + total_density, + total_flux, + graph::random_state_cast(state)); + + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp() + }, {}, { + {coll[0], ions[i].get_v_para()}, + {coll[1], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "ion_elec_" + ion_tag, num_particles); + } + init.print(); + + const timing::measure_diagnostic compile("Compile Time"); + work.compile(); + compile.print(); + + mesh.define_variables(f_file, mesh_dataset, work); + f_file.end_define_mode(); + + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + ions[i].define_variables(p_file, p_datasets[i], work, ion_tag); + } + p_file.end_define_mode(); + + std::atomic_size_t counter = 0; +#ifndef PROFILE_KERNELS + std::thread progress = std::thread([&num_steps, &counter]() -> void { + using namespace std::chrono_literals; + do { + const size_t progress = (counter*100.0)/num_steps; + std::cout << "\33[2K\r" << std::setw(3) << progress << "% Complete" + << std::flush; + std::this_thread::sleep_for(1s); + } while (counter < num_steps); + }); +#endif + const timing::measure_diagnostic run("Run Time"); + work.template run (); + for (; counter < num_steps; counter++) { + for (size_t i = 0; i < num_sub_steps; i++) { + work.run(); + } + work.template run (); } + + counter = num_steps; work.wait(); - sync_particles.join(); - sync_fields.join(); +#ifndef PROFILE_KERNELS + progress.join(); +#endif + for (std::thread &ion : ion_sync) { + ion.join(); + } + mesh_sync.join(); + + std::cout << "\33[2K\r" << "100% Complete" << std::endl; + run.print(); } //------------------------------------------------------------------------------ @@ -186,7 +320,11 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; + jit::verbose = true; + + const timing::measure_diagnostic total("Total Time"); run_pic (); + total.print(); END_GPU } diff --git a/graph_tests/CMakeLists.txt b/graph_tests/CMakeLists.txt index 9a2fcc0..0cbbca6 100644 --- a/graph_tests/CMakeLists.txt +++ b/graph_tests/CMakeLists.txt @@ -13,6 +13,9 @@ add_test_target (erfi_test cpp) add_test_target (efit_test cpp) add_test_target (random_test cpp) add_test_target (workflow_test cpp) +add_test_target (pic_test cpp) +add_compile_test (no_derivative_test cpp) +add_test_target (logical_test cpp) target_compile_definitions (erfi_test PRIVATE diff --git a/graph_tests/arithmetic_test.cpp b/graph_tests/arithmetic_test.cpp index 81cf2a5..bf95f27 100644 --- a/graph_tests/arithmetic_test.cpp +++ b/graph_tests/arithmetic_test.cpp @@ -97,7 +97,7 @@ template void test_add() { "Expected to reduce to a constant one."); assert(done_plus_var->evaluate()[0] == static_cast (1.0) && "Expected value of one."); - + // Test common factors. auto var_a = graph::variable (1, ""); auto var_b = graph::variable (1, ""); @@ -188,7 +188,7 @@ template void test_add() { auto constant_factor = three*variable + (one + one)*var_b; assert(graph::multiply_cast(constant_factor).get() && "Expected multiply node."); - + // Test is_match auto match1 = graph::one () + variable; auto match2 = graph::one () + variable; @@ -231,7 +231,7 @@ template void test_add() { "Expected var_c in the second slot."); assert(graph::add_cast(add_fma_cast->get_right()) && "Expected add_node in the third slot."); - + // (a/(b*c) + d/(e*c)) -> (a/b + d/e)/c auto multiply_divide_factor = var_a/(var_b*var_c) + var_d/(var_e*var_c); auto multiply_divide_factor_cast = divide_cast(multiply_divide_factor); @@ -1026,6 +1026,17 @@ template void test_subtract() { "Expected 3 on the left."); assert(constant_combine6_cast->get_right()->is_match(var_a) && "Expected a on the right."); + +// (a + b) - a -> b + assert(((var_a + var_b) - var_a)->is_match(var_b) && "Expected b."); +// (a + b) - b -> a + assert(((var_a + var_b) - var_b)->is_match(var_a) && "Expected a."); +// a - (a + b) -> -b + assert((var_a - (var_a + var_b))->is_match(graph::none ()*var_b) && + "Expected b."); +// b - (a + b) -> -a + assert((var_b - (var_a + var_b))->is_match(graph::none ()*var_a) && + "Expected a."); } //------------------------------------------------------------------------------ @@ -2036,6 +2047,55 @@ template void test_multiply() { v1, 3.0)*v2) && "Expected fma(fma(fma(50,x,4),x,3),x,3)*y"); + +// Sqrt(a)*Sqrt(b) -> Sqrt(a*b) + auto sqsq = graph::sqrt(v1)*graph::sqrt(v2); + assert(sqsq->is_match(graph::sqrt(v1*v2)) && "Expected Sqrt(a*b)"); + + auto v3 = graph::variable (1, "v3"); + if constexpr (std::floating_point) { +// hypot(b,c)*Sqrt(a) -> Sqrt((b^2 + c^2)*a) + auto hypotsq = graph::hypot(v1, v2)*graph::sqrt(v3); + assert(hypotsq->is_match(graph::sqrt((v1*v1 + v2*v2)*v3)) && + "Expected Sqrt((b^2 + c^2)*a)"); +// Sqrt(a)*hypot(b,c) -> Sqrt(a*(b^2 + c^2)) + auto sqhypot = graph::sqrt(v3)*graph::hypot(v1, v2); + assert(sqhypot->is_match(graph::sqrt((v1*v1 + v2*v2)*v3)) && + "Expected Sqrt((b^2 + c^2)*a)"); + +// Sqrt(x^2)*copysign(1,x) -> x + auto sqcs = graph::sqrt(v1*v1)*graph::copysign(static_cast (1), v1); + assert(sqcs->is_match(v1) && "Expected x"); +// copysign(1,x)*Sqrt(x^2) -> x + auto cssq = graph::copysign(static_cast (1), v1)*graph::sqrt(v1*v1); + assert(cssq->is_match(v1) && "Expected x"); + } + +// (a + b/c)*c -> fma(a,c,b) + auto result = (v1 + v2/v3)*v3; + assert(result->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// c*(a + b/c) -> fma(a,c,b) + auto result2 = v3*(v1 + v2/v3); + assert(result2->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// (b/c + a)*c -> fma(a,c,b) + auto result3 = (v2/v3 + v1)*v3; + assert(result3->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// c*(b/c + a) -> fma(a,c,b) + auto result4 = v3*(v2/v3 + v1); + assert(result4->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); + +// (a - b/c)*c -> a*c - b + auto result5 = (v1 - v2/v3)*v3; + assert(result5->is_match(v1*v3 - v2) && "Expected a*c - b"); +// c*(a - b/c) -> a*c - b + auto result6 = v3*(v1 - v2/v3); + assert(result6->is_match(v1*v3 - v2) && "Expected a*c - b"); +// (b/c - a)*c -> b - a*c + auto result7 = (v2/v3 - v1)*v3; + assert(result7->is_match(v2 - v1*v3) && "Expected b - a*c"); +// c*(b/c - a) -> b - a*c + auto result8 = v3*(v2/v3 - v1); + assert(result8->is_match(v2 - v1*v3) && "Expected b - a*c"); } //------------------------------------------------------------------------------ @@ -3895,6 +3955,62 @@ template void test_fma() { -49.0))) && "Expected fma(fma(fma(fma(2,x,20),x,30),x,50),b,fma(fma(fma(2,x,-19),-29),-49)"); */ + + +// fma(sqrt(a),sqrt(b),c) -> sqrt(a*b) + c + auto sqsq = graph::fma(graph::sqrt(var_a),graph::sqrt(var_b),var_c); + assert(sqsq->is_match(graph::sqrt(var_a*var_b) + var_c) && + "Expected Sqrt(a*b) + c"); + + if constexpr (std::floating_point) { +// fma(hypot(b,c),Sqrt(a),d) -> Sqrt((b^2 + c^2)*a) + d + auto hypotsq = graph::fma(graph::hypot(var_a,var_b),graph::sqrt(var_c),var_d); + assert(hypotsq->is_match(graph::sqrt((var_a*var_a + + var_b*var_b)*var_c) + var_d) && + "Expected Sqrt((b^2 + c^2)*a) + d"); +// fma(Sqrt(a),hypot(b,c),d) -> Sqrt(a*(b^2 + c^2)) + d + auto sqhypot = fma(graph::sqrt(var_c), + graph::hypot(var_a, var_b), + var_d); + assert(sqhypot->is_match(graph::sqrt((var_a*var_a + + var_b*var_b)*var_c) + var_d) && + "Expected Sqrt((b^2 + c^2)*a) + d"); + +// fma(Sqrt(x^2),copysign(1,x),y) -> x + y + auto sqcs = fma(graph::sqrt(var_a*var_a), + graph::copysign(static_cast (1), var_a), + var_b); + assert(sqcs->is_match(var_a + var_b) && "Expected x"); +// fma(copysign(1,x),Sqrt(x^2),y) -> x + y + auto cssq = fma(graph::copysign(static_cast (1), var_a), + graph::sqrt(var_a*var_a), + var_b); + assert(cssq->is_match(var_a + var_b) && "Expected x"); + } +} + +//------------------------------------------------------------------------------ +/// @brief Tests for modulo nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_modulo() { + auto five = graph::constant(static_cast (5)); + auto four = graph::constant(static_cast (4)); + + auto result = five%four; + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant node."); + assert(result_cast->is(static_cast (1)) && "Expected 1"); + + auto x = graph::variable (1, ""); + auto result2 = five%x; + auto result2_cast = graph::modulo_cast(result2); + assert(result2_cast.get() && "Expected a variable node."); + + auto result3 = x%four; + auto result3_cast = graph::modulo_cast(result3); + assert(result3_cast.get() && "Expected a variable node."); } //------------------------------------------------------------------------------ @@ -3908,6 +4024,9 @@ template void run_tests() { test_multiply (); test_divide (); test_fma (); + if constexpr (std::floating_point) { + test_modulo (); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/backend_test.cpp b/graph_tests/backend_test.cpp index 50bbabd..1ac44ac 100644 --- a/graph_tests/backend_test.cpp +++ b/graph_tests/backend_test.cpp @@ -330,6 +330,86 @@ template void test_backend() { static_cast (1.0), static_cast (2.0) })); + avec.sin(); + assert(avec.size() == 2 && "Expected a size of 2"); + assert(avec.at(0) == std::sin(static_cast (1.0)) && + "Expected a value of sin(1)."); + assert(avec.at(1) == std::sin(static_cast (2.0)) && + "Expected a value of sin(2)."); + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + avec.cos(); + assert(avec.size() == 2 && "Expected a size of 2"); + assert(avec.at(0) == std::cos(static_cast (1.0)) && + "Expected a value of cos(1)."); + assert(avec.at(1) == std::cos(static_cast (2.0)) && + "Expected a value of cos(2)."); + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); + const backend::buffer arctanvec = backend::atan(avec, bvec); + assert(arctanvec.size() == 2 && "Expected a size of 2"); + if constexpr (jit::complex_scalar) { + const T temp = arctanvec.at(0) + - static_cast (std::atan(static_cast (3.0)/ + static_cast (1.0))); + assert(std::imag(temp) == 0 && + "Expected a value of atan(3/1)."); + if constexpr (jit::float_base) { + assert(std::real(temp) < 1.3E-7 && + "Expected a value of atan(3/1)."); + } else { + assert(std::real(temp) == 0 && + "Expected a value of atan(3/1)."); + } + assert(arctanvec.at(1) == static_cast (std::atan(static_cast (4.0)/ + static_cast (2.0))) && + "Expected a value of atan(4/2)."); + } else { + assert(arctanvec.at(0) == static_cast (std::atan2(static_cast (3.0), + static_cast (1.0))) && + "Expected a value of atan2(3,1)."); + assert(arctanvec.at(1) == static_cast (std::atan2(static_cast (4.0), + static_cast (2.0))) && + "Expected a value of atan2(4,2)."); + } + + if constexpr (std::floating_point) { + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); + const backend::buffer hypotvec = backend::hypot(avec, bvec); + assert(hypotvec.size() == 2 && "Expected a size of 2"); + assert(hypotvec.at(0) == std::hypot(static_cast (1.0), + static_cast (3.0)) && + "Expected a value of hypot(1,3)."); + assert(hypotvec.at(1) == std::hypot(static_cast (2.0), + static_cast (4.0)) && + "Expected a value of hypot(2,4)."); + } + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); const backend::buffer fma_vec_scale_scale = backend::fma(avec, bscalar, cscalar); assert(fma_vec_scale_scale.size() == 2 && "Expected a size of 2"); assert(fma_vec_scale_scale.at(0) == static_cast (-2.0) && @@ -500,7 +580,7 @@ template void test_backend() { })); exp_vec.set(std::vector ({ static_cast (-4.0), - static_cast (0.30) + static_cast (0.3) })); const backend::buffer vec_vec = backend::pow(base_vec, exp_vec); assert(vec_vec.size() == 2 && "Expected a size of 2"); @@ -510,8 +590,8 @@ template void test_backend() { std::abs(static_cast (8.6736173798840355e-19)) && "Expected 4^-4."); assert(vec_vec.at(1) == std::pow(static_cast (2.0), - static_cast (0.30)) && - "Expected 2^0.30."); + static_cast (0.3)) && + "Expected 2^0.3."); base_scalar.set(static_cast (4.0)); base_scalar.log(); @@ -564,6 +644,89 @@ template void test_backend() { static_cast (NAN) })); assert(!nan_vec.is_normal() && "Expected a NaN."); + + if constexpr (jit::complex_scalar) { + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + avec.erfi(); + assert(avec.at(0) == static_cast (special::erfi(static_cast (4.0))) && + "Expected a value of Erfi(4 + 0i)."); + assert(avec.at(1) == static_cast (special::erfi(static_cast (-2))) && + "Expected a value of Erfi(-2 + 0i)."); + } + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + bvec.set(std::vector ({ + static_cast (-3.0), + static_cast (0.3) + })); + assert((avec == avec).at(0) == static_cast (1) && "Expected true."); + assert((avec == avec).at(1) == static_cast (1) && "Expected true."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec == bvec).at(0) == static_cast (0) && "Expected false."); + assert((avec == bvec).at(1) == static_cast (0) && "Expected false."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec != avec).at(0) == static_cast (0) && "Expected false."); + assert((avec != avec).at(1) == static_cast (0) && "Expected false."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec != bvec).at(0) == static_cast (1) && "Expected true."); + assert((avec != bvec).at(1) == static_cast (1) && "Expected true."); + + if constexpr (std::floating_point) { + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + bvec.set(std::vector ({ + static_cast (-3.0), + static_cast (0.3) + })); + const backend::buffer copysignvec = backend::copysign(avec, bvec); + assert(copysignvec.size() == 2 && "Expected a size of 2"); + assert(copysignvec.at(0) == static_cast (-4.0) && + "Expected a value of -4."); + assert(copysignvec.at(1) == static_cast (2.0) && + "Expected a value of 2."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + avec.erf(); + assert(avec.at(0) == static_cast (std::erf(static_cast (4.0))) && + "Expected a value of Erf(4)."); + assert(avec.at(1) == static_cast (std::erf(static_cast (-2.0))) && + "Expected a value of Erf(-2)."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + const backend::buffer modvec = avec % bvec; + assert((modvec.at(0) == static_cast (std::fmod(static_cast (4.0), + static_cast (-3.0)))) && + "Expected a value of 4 % -3."); + assert((modvec.at(1) == static_cast (std::fmod(static_cast (-2.0), + static_cast (0.3)))) && + "Expected a value of -2 % 0.3."); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index 72ff6cc..a3dff5d 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -98,7 +98,7 @@ void run_tests(const enum graph_type type, } } - graph_node state = graph_random_state(c_context, 0); + graph_node state = graph_random_state(c_context, 1, 0); graph_node rand = graph_random(c_context, state); const size_t max_device = graph_get_max_concurrency(c_context) - 1; @@ -195,25 +195,29 @@ void run_tests(const enum graph_type type, graph_node *map_inputs2 = NULL; graph_node *map_outputs2 = NULL; - graph_add_pre_item(c_context, - NULL, 0, - &rand, 1, - NULL, NULL, 0, - state, - "c_binding_pre_kernel", 1); + graph_add_item(c_context, + NULL, 0, + &rand, 1, + NULL, NULL, 0, + NULL, 0, + state, + "c_binding_pre_kernel", 1); graph_add_item(c_context, inputs, 1, outputs, 5, map_inputs, map_outputs, 0, + NULL, 0, NULL, "c_binding", 1); graph_add_item(c_context, inputs2, 4, outputs2, 4, map_inputs2, map_outputs2, 0, + NULL, 0, NULL, "c_binding_piecewise", 1); graph_add_converge_item(c_context, &z, 1, &root2, 1, &z, &dz, 1, + NULL, 0, NULL, "c_binding_converge", 1, 1.0E-30, 1000); graph_compile(c_context); @@ -268,11 +272,7 @@ void run_tests(const enum graph_type type, assert(value[2] == 2.0f && "Value of dydm does not match."); assert(value[3] == 1.0f && "Value of dydb does not match."); assert(value[4] == 1.0f && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(value[5] == 2546248192.0f && "Value of rand does not match."); - } else { - assert(value[5] == 2357136128.0f && "Value of rand does not match."); - } + assert(value[5] == (float)2357136044 && "Value of rand does not match."); assert(value[6] == 1.0f && "Value of root does not match."); assert(value[7] == 4.0f && "Value of p1 does not match."); assert(value[8] == 8.0f && "Value of p2 does not match."); @@ -299,11 +299,7 @@ void run_tests(const enum graph_type type, assert(value[2] == 2.0 && "Value of dydm does not match."); assert(value[3] == 1.0 && "Value of dydb does not match."); assert(value[4] == 1.0 && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(value[5] == 2546248239.0 && "Value of rand does not match."); - } else { - assert(value[5] == 2357136044.0 && "Value of rand does not match."); - } + assert(value[5] == (double)2357136044 && "Value of rand does not match."); assert(value[6] == 1.0 && "Value of root does not match."); assert(value[7] == 4.0 && "Value of p1 does not match."); assert(value[8] == 8.0 && "Value of p2 does not match."); @@ -330,11 +326,7 @@ void run_tests(const enum graph_type type, assert(crealf(value[2]) == 2.0f && "Value of dydm does not match."); assert(crealf(value[3]) == 1.0f && "Value of dydb does not match."); assert(crealf(value[4]) == 1.0f && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(crealf(value[5]) == 2546248192.0f && "Value of rand does not match."); - } else { - assert(crealf(value[5]) == 2357136128.0f && "Value of rand does not match."); - } + assert(crealf(value[5]) == crealf(2357136044) && "Value of rand does not match."); assert(crealf(value[6]) == 1.0f && "Value of root does not match."); assert(crealf(value[7]) == 4.0f && "Value of p1 does not match."); assert(crealf(value[8]) == 8.0f && "Value of p2 does not match."); @@ -361,11 +353,7 @@ void run_tests(const enum graph_type type, assert(creal(value[2]) == 2.0 && "Value of dydm does not match."); assert(creal(value[3]) == 1.0 && "Value of dydb does not match."); assert(creal(value[4]) == 1.0 && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(creal(value[5]) == 2546248239.0 && "Value of rand does not match."); - } else { - assert(creal(value[5]) == 2357136044.0 && "Value of rand does not match."); - } + assert(creal(value[5]) == creal(2357136044) && "Value of rand does not match."); assert(creal(value[6]) == 1.0 && "Value of root does not match."); assert(creal(value[7]) == 4.0 && "Value of p1 does not match."); assert(creal(value[8]) == 8.0 && "Value of p2 does not match."); diff --git a/graph_tests/efit_test.cpp b/graph_tests/efit_test.cpp index 9a15675..3b734dc 100644 --- a/graph_tests/efit_test.cpp +++ b/graph_tests/efit_test.cpp @@ -166,7 +166,7 @@ void run_test() { graph::variable_cast(z) }, { bvec->get_x(), bvec->get_y(), bvec->get_z(), ne, te, div - }, {}, graph::shared_random_state (), "test_kernel", xy_x_grid.size()); + }, {}, {}, NULL, "test_kernel", xy_x_grid.size()); work.compile(); work.run(); diff --git a/graph_tests/f_binding_test.f90 b/graph_tests/f_binding_test.f90 index 899e0b6..6562ac1 100644 --- a/graph_tests/f_binding_test.f90 +++ b/graph_tests/f_binding_test.f90 @@ -143,7 +143,7 @@ SUBROUTINE run_test_float(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -180,7 +180,8 @@ SUBROUTINE run_test_float(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -189,17 +190,17 @@ SUBROUTINE run_test_float(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -222,13 +223,8 @@ SUBROUTINE run_test_float(use_safe_math) CALL graph%copy_to_host(dydy, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(value(1) .eq. 2546248192.0_C_FLOAT, & - 'Value of rand does not match.') - ELSE - CALL assert(value(1) .eq. 2357136128.0_C_FLOAT, & - 'Value of rand does not match.') - END IF + CALL assert(value(1) .eq. 2357136044.0_C_FLOAT, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of root does not match.') CALL graph%copy_to_host(p1, value) @@ -336,7 +332,7 @@ SUBROUTINE run_test_double(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -373,7 +369,8 @@ SUBROUTINE run_test_double(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -382,17 +379,17 @@ SUBROUTINE run_test_double(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -415,13 +412,8 @@ SUBROUTINE run_test_double(use_safe_math) CALL graph%copy_to_host(dydy, value) CALL assert(value(1) .eq. 1.0_C_DOUBLE, 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(value(1) .eq. 2546248239.0_C_DOUBLE, & - 'Value of rand does not match.') - ELSE - CALL assert(value(1) .eq. 2357136044.0_C_DOUBLE, & - 'Value of rand does not match.') - END IF + CALL assert(value(1) .eq. 2357136044_C_DOUBLE, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(value(1) .eq. 1.0_C_DOUBLE, 'Value of root does not match.') CALL graph%copy_to_host(p1, value) @@ -531,7 +523,7 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -570,7 +562,8 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'c_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -579,17 +572,17 @@ SUBROUTINE run_test_complex_float(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -616,13 +609,8 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL assert(REAL(value(1)) .eq. 1.0_C_FLOAT, & 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(REAL(value(1)) .eq. 2546248192.0_C_FLOAT, & - 'Value of rand does not match.') - ELSE - CALL assert(REAL(value(1)) .eq. 2357136128.0_C_FLOAT, & - 'Value of rand does not match.') - END IF + CALL assert(REAL(value(1)) .eq. 2357136044.0_C_FLOAT, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(REAL(value(1)) .eq. 1.0_C_FLOAT, & 'Value of root does not match.') @@ -737,7 +725,7 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -788,7 +776,8 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -797,17 +786,17 @@ SUBROUTINE run_test_complex_double(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -835,13 +824,8 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL assert(DBLE(value(1)) .eq. 1.0_C_DOUBLE, & 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(DBLE(value(1)) .eq. 2546248239.0_C_DOUBLE, & - 'Value of rand does not match.') - ELSE - CALL assert(DBLE(value(1)) .eq. 2357136044.0_C_DOUBLE, & - 'Value of rand does not match.') - END IF + CALL assert(DBLE(value(1)) .eq. 2357136044_C_DOUBLE, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(DBLE(value(1)) .eq. 1.0_C_DOUBLE, & 'Value of root does not match.') diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index 7b811b9..114323d 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -10,7 +10,7 @@ #include -#include "../graph_framework/dispersion.hpp" +#include "../graph_framework/graph_framework.hpp" //------------------------------------------------------------------------------ /// @brief Assert when difference is greater than the tolerance. @@ -52,14 +52,15 @@ void compile(graph::input_nodes inputs, const T expected, const T tolerance) { jit::context source(0); + graph::input_nodes atomics; + graph::shared_random_state state; source.add_kernel("test_kernel", inputs, outputs, setters, - graph::shared_random_state (), - inputs.back()->size()); + atomics, state, inputs.back()->size()); source.compile(); auto run = source.create_kernel_call("test_kernel", inputs, outputs, - graph::shared_random_state (), 1); + atomics, state, 1); run(); T result; @@ -76,6 +77,13 @@ void compile(graph::input_nodes inputs, //------------------------------------------------------------------------------ template void run_math_tests() { auto v1 = graph::variable (1, "v1"); + + compile ({ + graph::variable_cast(v1) + }, { + graph::index () + }, {}, static_cast (0), 0.0); + auto v2 = graph::variable (1, "v2"); auto v3 = graph::variable (1, "v3"); @@ -344,6 +352,40 @@ template void run_math_tests() { graph::variable_cast(v1), graph::variable_cast(v2) }, {atan_node}, {}, atan_node->evaluate().at(0), result); + + if constexpr (std::floating_point) { + auto module_node = v1%v2; + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {module_node}, {}, module_node->evaluate().at(0), 0.0); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + auto if_node = graph::if_(v1 > v2, true_v, false_v); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {if_node}, {}, false_v->evaluate().at(0), 0.0); + + if_node = graph::if_(v1 < v2, true_v, false_v); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {if_node}, {}, true_v->evaluate().at(0), 0.0); + + auto hypot_node = graph::hypot(v1, v2); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {hypot_node}, {}, hypot_node->evaluate().at(0), 5.0E-16); + + auto min_node = graph::min(v1, v2); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {min_node}, {}, min_node->evaluate().at(0), 0.0); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/logical_test.cpp b/graph_tests/logical_test.cpp new file mode 100644 index 0000000..fc21be3 --- /dev/null +++ b/graph_tests/logical_test.cpp @@ -0,0 +1,375 @@ +//------------------------------------------------------------------------------ +/// @file logical.cpp +/// @brief Tests for logic nodes. +//------------------------------------------------------------------------------ + +// Turn on asserts even in release builds. +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include +#include + +#include "../graph_framework/graph_framework.hpp" + +//------------------------------------------------------------------------------ +/// @brief Tests for isinf nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_isinf() { + auto zero = graph::zero (); + auto one = graph::one (); + auto nan = graph::constant (NAN); + auto inf = graph::constant (INFINITY); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + assert(graph::isinf(zero)->is_match(false_v) && "Expected false."); + assert(graph::isinf(one)->is_match(false_v) && "Expected false."); + assert(graph::isinf(nan)->is_match(false_v) && "Expected false."); + assert(graph::isinf(inf)->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for isnan nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_isnan() { + auto zero = graph::zero (); + auto one = graph::one (); + auto nan = graph::constant (NAN); + auto inf = graph::constant (INFINITY); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + assert(graph::isnan(zero)->is_match(false_v) && "Expected false."); + assert(graph::isnan(one)->is_match(false_v) && "Expected false."); + assert(graph::isnan(nan)->is_match(true_v) && "Expected true."); + assert(graph::isnan(inf)->is_match(false_v) && "Expected false."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for not nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_not() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = !true_v; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = !false_v; + assert(result2->is_match(true_v) && "Expected true."); + +// !(a == b) -> a != b + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto result3 = !(v1 == v2); + auto result3_cast = graph::not_equal_cast(result3); + assert(result3_cast.get() && "Expected a not equal node."); + +// !(a != b) -> a == b + auto result4 = !(v1 != v2); + auto result4_cast = graph::equal_cast(result4); + assert(result4_cast.get() && "Expected an equal node."); + + if constexpr (!jit::complex_scalar) { +// !(a < b) -> a >= b + auto result5 = !(v1 < v2); + auto result5_cast = graph::greater_than_equal_cast(result5); + assert(result5_cast.get() && "Expected a greater than equal node."); + +// !(a <= b) -> a > b + auto result6 = !(v1 <= v2); + auto result6_cast = graph::greater_than_cast(result6); + assert(result6_cast.get() && "Expected a greater than node."); + +// !(a > b) -> a <= b + auto result7 = !(v1 > v2); + auto result7_cast = graph::less_than_equal_cast(result7); + assert(result7_cast.get() && "Expected a less than equal node."); + +// !(a >= b) -> a < b + auto result8 = !(v1 >= v2); + auto result8_cast = graph::less_than_cast(result8); + assert(result8_cast.get() && "Expected a less than node."); + } + +// !!a -> a + auto result9 = !!v1; + auto result9_cast = graph::variable_cast(result9); + assert(result9_cast.get() && "Expected v1"); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v == true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = false_v == false_v; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = true_v == false_v; + assert(result3->is_match(false_v) && "Expected false."); + auto result4 = false_v == true_v; + assert(result4->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 == v2)->is_match(v2 == v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for not equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_not_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v != true_v; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = false_v != false_v; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = true_v != false_v; + assert(result3->is_match(true_v) && "Expected true."); + auto result4 = false_v != true_v; + assert(result4->is_match(true_v) && "Expected true."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 != v2)->is_match(v2 != v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for greater than nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_greater_than() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one > none; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = none > one; + assert(result2->is_match(false_v) && "Expected false."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for less than nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_less_than() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one < none; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = none < one; + assert(result2->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for greater than equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_greater_than_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one >= none; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = none >= one; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = one >= one; + assert(result3->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for less than equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_less_than_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one <= none; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = none <= one; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = one <= one; + assert(result3->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for and nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_and() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v && true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = true_v && false_v; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = false_v && false_v; + assert(result3->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 && v2)->is_match(v2 && v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for or nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_or() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v || true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = true_v || true_v; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = false_v || false_v; + assert(result3->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 || v2)->is_match(v2 || v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for if nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_if() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = graph::if_(true_v, true_v, false_v); + assert(result1->is_match(true_v) && "Exected the true condition."); + auto result2 = graph::if_(false_v, true_v, false_v); + assert(result2->is_match(false_v) && "Exected the false condition."); + +// If(c, a, a) -> a + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto result = graph::if_(v1, v2, v2); + assert(result->is_match(v2)); + auto result_df = result->df(v1); + assert(result_df->is_match(false_v) && "Expected 0"); + +// If(!a, b, c) -> If(a, c, b) + auto test_not = graph::if_(graph::not_(v1), v1, v2); + auto test_not_cast = if_cast(test_not); + assert(test_not_cast.get() && "Expected if node."); + assert(test_not_cast->get_left()->is_match(v1) && "Expected v1"); + assert(test_not_cast->get_middle()->is_match(v2) && "Expected v2"); + assert(test_not_cast->get_right()->is_match(v1) && "Expected v1"); +} + +//------------------------------------------------------------------------------ +/// @brief Test for min nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_min() { + auto one = graph::one (); + auto none = graph::none (); + + assert(graph::min(one, none)->is_match(none) && + "Expected -1."); + assert(graph::min(none, one)->is_match(none) && + "Expected -1."); + assert(graph::min(none, none)->is_match(none) && + "Expected -1."); + assert(graph::min(one, one)->is_match(one) && + "Expected 1."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto zero = graph::zero (); + + auto min_result = graph::min(v1, v2); + assert(min_result->df(v1)->is_match(zero) && + "Expected 1."); + assert(min_result->df(v2)->is_match(zero) && + "Expected 1."); + + auto v3 = graph::variable (1, ""); + assert(min_result->df(v3)->is_match(zero) && + "Expected 0."); + + assert(min_result->df(min_result)->is_match(one) && + "Expected 1"); +} + +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified backend. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_tests() { + test_equal (); + test_not_equal (); + test_not (); + if constexpr (std::floating_point) { + test_isinf (); + test_isnan (); + test_greater_than (); + test_less_than (); + test_and (); + test_or (); + test_if (); + test_min (); + } +} + +//------------------------------------------------------------------------------ +/// @brief Main program of the test. +/// +/// @param[in] argc Number of commandline arguments. +/// @param[in] argv Array of commandline arguments. +//------------------------------------------------------------------------------ +int main(int argc, const char * argv[]) { + (void)argc; + (void)argv; + run_tests (); + run_tests (); + run_tests> (); + run_tests> (); +} diff --git a/graph_tests/math_test.cpp b/graph_tests/math_test.cpp index 3a0f65a..48428e9 100644 --- a/graph_tests/math_test.cpp +++ b/graph_tests/math_test.cpp @@ -504,6 +504,14 @@ void test_pow() { graph::pow(expr_a, 2.0) * graph::pow(expr_c, 2.0)) && "Expected b*c^2*d^2."); + +// hypot(a,b)^2 -> a^2 + b^2 + if constexpr (std::floating_point) { + assert((graph::pow(graph::hypot(var_a, var_b), + static_cast(2))->is_match(var_a*var_a + + var_b*var_b)) && + "Expected a^2 + b^2"); + } } //------------------------------------------------------------------------------ @@ -530,7 +538,34 @@ void test_log() { } //------------------------------------------------------------------------------ -/// @brief Tests for log nodes. +/// @brief Tests for erfi nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_erf() { + auto a = graph::variable (1, ""); + auto erf = graph::erf(a); + + assert(graph::erf_cast(erf) && + "Expected an erf node."); + + auto derfda = erf->df(a); + assert(graph::multiply_cast(derfda) && + "Expected a multiply node."); + + auto erfc = graph::erf(graph::one ()); + assert(graph::constant_cast(erfc) && + "Expected a constant node."); + +// Test node properties. + assert(!erf->is_constant() && "Did not expect a constant."); + assert(erf->is_all_variables() && "Expected a variable."); + assert(!erf->is_power_like() && "Did not expect a power like."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for erfi nodes. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ @@ -556,6 +591,101 @@ void test_erfi() { assert(!erfi->is_power_like() && "Did not expect a power like."); } +//------------------------------------------------------------------------------ +/// @brief Tests for hypot nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_hypot() { + auto a = graph::constant (0.5); + auto b = graph::constant (1.2); + + auto result = graph::hypot (a, b); + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant."); + assert(result_cast->is(std::hypot(static_cast (0.5), + static_cast (1.2))) && + "Expected hypot(0.5, 1.2)"); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + + assert(graph::hypot (v1, v2)->is_match(graph::hypot (v2, v1)) && + "Expected match."); + +// hypot(sqrt(a), sqrt(b)) -> sqrt(a + b) + auto result2 = graph::hypot(sqrt(v1), sqrt(v2)); + auto result2_cast = graph::sqrt_cast(result2); + assert(result2_cast.get() && "Expected a sqrt node."); + assert(result2->is_match(graph::sqrt(v1 + v2)) && + "Expected sqrt(a + b)."); + +// hypot(a, sqrt(b)) -> sqrt(a^2 + b) + auto result3 = graph::hypot(v1, sqrt(v2)); + auto result3_cast = graph::sqrt_cast(result3); + assert(result3_cast.get() && "Expected a sqrt node."); + assert(result3->is_match(graph::sqrt(v1*v1 + v2)) && + "Expected sqrt(a^2 + b)."); + +// hypot(sqrt(a), b) -> sqrt(a + b^2) + auto result4 = graph::hypot(sqrt(v1), v2); + auto result4_cast = graph::sqrt_cast(result4); + assert(result4_cast.get() && "Expected a sqrt node."); + assert(result4->is_match(graph::sqrt(v1 + v2*v2)) && + "Expected sqrt(a + b^2)."); + +// hypoy(a,a) -> sqrt(2)sqrt(a^2) + auto result5 = graph::hypot(v1, v1); + auto result5_cast = graph::multiply_cast(result5); + assert(result5_cast.get() && "Expected a multiply node."); + assert(result5->is_match(std::numbers::sqrt2_v*graph::sqrt(v1*v1)) && + "Expected sqrt(2)sqrt(a^2)."); + +// d hypoy(a,b)/dx -> 0 + auto result6 = graph::hypot(v1, v2)->df(a); + auto result6_cast = graph::constant_cast(result6); + assert(result6_cast.get() && "Expected a constant."); + assert(result6_cast->is(0) && "Expected zero"); + +// d hypoy(a,b)/da -> 0 + auto result7 = graph::hypot(v1, v2)->df(v1); + auto result7_cast = graph::divide_cast(result7); + assert(result7_cast.get() && "Expected a divide node."); + assert(result7_cast->is_match(v1/graph::hypot(v1, v2)) && + "v1/hypot(v1, v2)"); + +// d hypoy(a,b)/db -> 0 + auto result8 = graph::hypot(v1, v2)->df(v2); + auto result8_cast = graph::divide_cast(result8); + assert(result8_cast.get() && "Expected a divide node."); + assert(result8_cast->is_match(v2/graph::hypot(v1, v2)) && + "v2/hypot(v1, v2)"); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for copysign nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_copysign() { + auto a = graph::constant (0.5); + auto b = graph::constant (-1.2); + + auto result = graph::copysign (a, b); + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant."); + assert(result_cast->is(-0.5) && "Expected -0.5"); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + + auto result2 = graph::copysign(v1, v2); + auto result2_cast = graph::copysign_cast(result2); + assert(result2_cast.get() && "Expected a copysign node."); +} + //------------------------------------------------------------------------------ /// @brief Tests function for variable like expressions. /// @@ -593,6 +723,10 @@ template void run_tests() { if constexpr (jit::complex_scalar) { test_erfi (); } + if constexpr (std::floating_point) { + test_erf (); + test_hypot (); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp new file mode 100644 index 0000000..91e8479 --- /dev/null +++ b/graph_tests/no_derivative_test.cpp @@ -0,0 +1,72 @@ +//------------------------------------------------------------------------------ +/// @file no_derivative_test.cpp +/// @brief Test for nodes with no derivatives. +//------------------------------------------------------------------------------ + +#include "../graph_framework/node.hpp" + +//------------------------------------------------------------------------------ +/// @brief Dummy node. +//------------------------------------------------------------------------------ +class dummy : public graph::no_derivative> { +public: +//------------------------------------------------------------------------------ +/// @brief A dummy constructor. +//------------------------------------------------------------------------------ + dummy() : + graph::no_derivative> ("") {} + +//------------------------------------------------------------------------------ +/// @brief Dummy evaluate method. +/// +/// @returns An empty buffer. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + return backend::buffer (); + }; + +//------------------------------------------------------------------------------ +/// @brief Dummy reduce method. +/// +/// @returns Returns the dummy node. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Dummy to vizgraph method. +/// +/// @returns A reference to this. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Dummy get power exponent method. +/// +/// @returns One. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf get_power_exponent() const { + return graph::one (); + } +}; + +//------------------------------------------------------------------------------ +/// @brief Test function. +/// +/// This test checks for a failure to compiler if a df method is called on a +/// node without a derivative. +//------------------------------------------------------------------------------ +void test() { + dummy a; +#ifndef CHECK_TEST + a.df(a); +#endif +} diff --git a/graph_tests/node_test.cpp b/graph_tests/node_test.cpp index b6fefcd..1e003e3 100644 --- a/graph_tests/node_test.cpp +++ b/graph_tests/node_test.cpp @@ -15,6 +15,26 @@ #include "../graph_framework/trigonometry.hpp" #include "../graph_framework/arithmetic.hpp" +//------------------------------------------------------------------------------ +/// @brief Tests for constant nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_index() { + auto index = graph::index (); + auto index_cast = graph::index_cast(index); + assert(index_cast.get() && "Expected a index type."); + auto dindex = index->df(index); + auto dindex_cast = graph::constant_cast(dindex); + assert(dindex_cast.get() && "Expected a constant type for derivative."); + assert(dindex_cast->is(1.0) && "Constant value expected one."); + auto dindex2 = index->df(graph::zero ()); + auto dindex2_cast = graph::constant_cast(dindex2); + assert(dindex2_cast.get() && "Expected a constant type for derivative."); + assert(dindex2_cast->is(0.0) && "Constant value expected one."); +} + //------------------------------------------------------------------------------ /// @brief Tests for constant nodes. /// @@ -169,6 +189,7 @@ void test_pseudo_variable() { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { + test_index (); test_constant (); test_variable (); test_pseudo_variable (); diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp new file mode 100644 index 0000000..6ccbf08 --- /dev/null +++ b/graph_tests/pic_test.cpp @@ -0,0 +1,344 @@ +//------------------------------------------------------------------------------ +/// @file pic_test.cpp +/// @brief Tests for the particle in cell functions interface. +//------------------------------------------------------------------------------ + +// Turn on asserts even in release builds. +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include +#include + +#include "../graph_framework/graph_framework.hpp" + +//------------------------------------------------------------------------------ +/// @brief Run interpolation test. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_interpolation_test() { + const size_t num_mesh = 100; + const size_t num_particles = 10000; + +// Characteristic factors + const std::vector ion_masses{pic::m_hydrogen}; + const std::vector ion_zs{1}; + + const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); + std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], + num_particles, 0, norms)}; + pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); + + std::function func([&norms](const T x) -> T { + return std::sin(std::exp(x)); + }); + + for (size_t i = 0; i < num_mesh; i++) { + mesh.data()[i] = func(mesh.dx*i + mesh.xmin); + } + + const T dxp = (mesh.xmax - mesh.xmin)/(num_particles - 1); + for (size_t i = 0; i < num_particles; i++) { + ions[0].x_data()[i] = dxp*i + mesh.xmin; + } + + auto weights = mesh.build_weights(ions[0].x); + auto field = mesh.build_interpolation(ions[0].x); + auto weight = weights[0] + weights[1] + weights[2]; + + workflow::manager work(0); + work.add_item({ + graph::variable_cast(mesh.y[0]), + graph::variable_cast(ions[0].x) + }, { + weight, + field + }, {}, {}, NULL, "Mesh_Interpolation", num_particles); + work.compile(); + work.run(); + work.wait(); + +// The weights should sum to 1. + for (size_t i = 0; i < num_particles; i++) { + const T received = work.check_value(i, weight); + const T diff = static_cast (1) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.7E-12) && + "Weight not equal to 1±4.7E-12"); + } else { + assert(diff*diff < static_cast (7.1E-30) && + "Weight not equal to 1±7.1E-30"); + } + } + + for (size_t i = 0, ie = num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.0E-7) && + "Profile not equal ±4.0E-7"); + } else { + assert(diff*diff < static_cast (4.0E-7) && + "Profile not equal ±4.0E-7"); + } + } + for (size_t i = num_particles/10, ie = 2*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (jit::use_cuda()) { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.22E-4) && + "Profile not equal ±1.22E-4"); + } else { + assert(diff*diff < static_cast (1.04E-6) && + "Profile not equal ±1.04E-6"); + } + } else { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.4E-6) && + "Profile not equal ±1.4E-6"); + } else { + assert(diff*diff < static_cast (1.1E-6) && + "Profile not equal ±1.1E-7"); + } + } + } + std::cout << std::endl; + for (size_t i = 2*num_particles/10, ie = 3*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (jit::use_cuda()) { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.1E-4) && + "Profile not equal ±3.1E-4"); + } else { + assert(diff*diff < static_cast (1.42E-8) && + "Profile not equal ±1.42E-8"); + } + } else { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.8E-6) && + "Profile not equal ±3.8E-6"); + } else { + assert(diff*diff < static_cast (1.5E-8) && + "Profile not equal ±1.5E-8"); + } + } + } + for (size_t i = 3*num_particles/10, ie = 4*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (5.9E-4) && + "Profile not equal ±5.9E-4"); + } else { + assert(diff*diff < static_cast (7.9E-6) && + "Profile not equal ±7.9E-6"); + } + } + for (size_t i = 4*num_particles/10, ie = 5*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.9E-5) && + "Profile not equal ±1.9E-5"); + } else { + assert(diff*diff < static_cast (2.1E-8) && + "Profile not equal ±2.1E-8"); + } + } + for (size_t i = 5*num_particles/10, ie = 6*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (jit::use_metal ()) { + assert(diff*diff < static_cast (1.6E-5) && + "Profile not equal ±1.6E-5"); + } else { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.7E-5) && + "Profile not equal ±1.7E-5"); + } else { + assert(diff*diff < static_cast (2.9E-6) && + "Profile not equal ±2.9E-6"); + } + } + } + for (size_t i = 6*num_particles/10, ie = 7*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.0E-2) && + "Profile not equal ±4.0E-2"); + } else { + assert(diff*diff < static_cast (7.0E-6) && + "Profile not equal ±7.0E-6"); + } + } + for (size_t i = 7*num_particles/10, ie = 8*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.5E-3) && + "Profile not equal ±1.5E-3"); + } else { + assert(diff*diff < static_cast (1.5E-3) && + "Profile not equal ±1.5E-3"); + } + } + for (size_t i = 8*num_particles/10, ie = 9*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (5.0E-3) && + "Profile not equal ±5.0E-3"); + } else { + assert(diff*diff < static_cast (3.0E-3) && + "Profile not equal ±3.0E-3"); + } + } + for (size_t i = 9*num_particles/10, ie = 10*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); + const T received = work.check_value(i, field); + const T diff = func(x) - received; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.8E-2) && + "Profile not equal ±1.8E-2"); + } else { + assert(diff*diff < static_cast (1.8E-2) && + "Profile not equal ±1.8E-2"); + } + } +} + +//------------------------------------------------------------------------------ +/// @brief Field solve test. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_field_solve_test() { + const size_t num_mesh = 100; + const size_t num_particles = 1000000; + +// Characteristic factors + const std::vector ion_masses{pic::m_hydrogen}; + const std::vector ion_zs{1}; + + const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); + std::vector> ions{ + pic::ion (ion_masses[0], ion_zs[0], num_particles, 0, norms) + }; + pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); + +// Initialize particle positions. + backend::buffer buffer(num_particles); + + std::mt19937 gen(0); + std::normal_distribution dist(0.0, 1.0); + + for (size_t i = 0; i < num_particles; i++) { + do { + buffer[i] = dist(gen); + } while(buffer[i] < -3.0 || buffer[i] > 3.0); + } + ions[0].x->set(buffer); + +// Count particles in mesh bins. This builds a histogram of particle counts. + std::vector counts(num_mesh, 0); + for (size_t i = 0; i < num_mesh; i++) { + const T bin_low = i*mesh.dx + mesh.xmin - mesh.dx/2; + const T bin_high = i*mesh.dx + mesh.xmin + mesh.dx/2; + for (size_t j = 0; j < num_particles; j++) { + if (buffer[j] >= bin_low && buffer[j] < bin_high) { + counts[i]++; + } + } + } + + auto mesh_solve = mesh.build_mesh_solve(ions[0]); + + workflow::manager work(0); + work.add_zero_item({ + graph::variable_cast(mesh.y[0]) + }); + work.add_item({ + ions[0].get_x() + }, { + mesh_solve[0], + mesh_solve[1], + mesh_solve[2] + }, {}, { + graph::variable_cast(mesh.y[0]) + }, NULL, "sum_weights", num_particles); + + work.compile(); + + const timing::measure_diagnostic t_run("Run Time"); + work.run(); + work.wait(); + t_run.print(); + + for (size_t i = 0; i < num_mesh; i++) { + const T recieved = work.check_value(i, mesh.y[0]); + const T error = std::abs((counts[i] - recieved)/counts[i]); + if constexpr (std::is_same_v) { + assert(error < 0.155 && "Error outside tolarance range."); + } else { + assert(error < 0.192 && "Error outside tolarance range."); + } + } +} + +//------------------------------------------------------------------------------ +/// @brief Coordinate tests. +//------------------------------------------------------------------------------ +template void run_coord_tests() { + auto x = graph::variable (1, "x"); + auto y = graph::variable (1, "y"); + + std::array, 3> sphere = pic::cartesian_to_sphereical(x, y); + std::array, 2> cart = pic::sphereical_to_cartesian(sphere[0], + sphere[1], + sphere[2]); + + assert(cart[0]->is_match(x) && "x not converted correctly."); + assert(cart[1]->is_match(y) && "y not converted correctly."); +} + +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified precision. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_tests() { + run_interpolation_test (); + run_field_solve_test (); + run_coord_tests (); +} + +//------------------------------------------------------------------------------ +/// @brief Main program of the test. +/// +/// @param[in] argc Number of commandline arguments. +/// @param[in] argv Array of commandline arguments. +//------------------------------------------------------------------------------ +int main(int argc, const char * argv[]) { + START_GPU + + (void)argc; + (void)argv; + run_tests (); + run_tests (); + + END_GPU +} diff --git a/graph_tests/piecewise_test.cpp b/graph_tests/piecewise_test.cpp index eae3207..8be948c 100644 --- a/graph_tests/piecewise_test.cpp +++ b/graph_tests/piecewise_test.cpp @@ -56,13 +56,15 @@ template void compile(graph::input_nodes inputs, const T expected, const T tolerance) { jit::context source(0); - source.add_kernel("test_kernel", inputs, outputs, setters, - graph::shared_random_state (), inputs.back()->size()); + graph::input_nodes atomics; + graph::shared_random_state state; + source.add_kernel("test_kernel", inputs, outputs, setters, atomics, state, + inputs.back()->size()); source.compile(); auto run = source.create_kernel_call("test_kernel", inputs, outputs, - graph::shared_random_state (), 1); + atomics, state, 1); run(); T result; diff --git a/graph_tests/random_test.cpp b/graph_tests/random_test.cpp index e0c0f93..ee7398f 100644 --- a/graph_tests/random_test.cpp +++ b/graph_tests/random_test.cpp @@ -42,15 +42,16 @@ T autocorrelation(const std::vector &sequence, /// @tparam N Number of random numbers to use. //------------------------------------------------------------------------------ template void test_dist() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(N), 0); auto random = graph::random (graph::random_state_cast(state)); const T max = 1.0; const T min = -1.0; auto random_real = (max - min)/graph::random_scale ()*random + min; workflow::manager work(0); - work.add_item({}, {random_real}, {}, graph::random_state_cast(state), - "step", N); + work.add_item({}, { + random_real + }, {}, {}, graph::random_state_cast(state), "step", N); work.compile(); work.run(); @@ -68,7 +69,7 @@ template void test_dist() { /// @brief Test graph properties of random numbers. //------------------------------------------------------------------------------ template void test_graph() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(1), 0); auto random = graph::random (graph::random_state_cast(state)); // r + r -> r + r @@ -140,14 +141,14 @@ template void test_graph() { /// @brief Test multiple randoms in a single kernel. //------------------------------------------------------------------------------ template void test_multi() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(1), 0); auto random1 = graph::random (graph::random_state_cast(state)); auto random2 = graph::random (graph::random_state_cast(state)); workflow::manager work(0); work.add_item({}, { random1, random2 - }, {}, graph::random_state_cast(state), "multi_random", 1); + }, {}, {}, graph::random_state_cast(state), "multi_random", 1); work.compile(); } diff --git a/graph_tests/workflow_test.cpp b/graph_tests/workflow_test.cpp index e61b619..55477c8 100644 --- a/graph_tests/workflow_test.cpp +++ b/graph_tests/workflow_test.cpp @@ -16,8 +16,87 @@ /// @brief Test setting multiple variables with the same map. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_maps() { +template void test_zeros() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer(1, static_cast (1)); + a->set(buffer); + b->set(buffer); + + workflow::manager work(0); + work.template add_zero_item ({ + graph::variable_cast(a), + graph::variable_cast(b) + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); + work.template run(); + assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); + assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); +} + +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order +//------------------------------------------------------------------------------ +template void test_copy() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer1(1, static_cast (1)); + backend::buffer buffer2(1, static_cast (2)); + a->set(buffer1); + b->set(buffer2); + + workflow::manager work(0); + work.template add_copy_item ({ + {graph::variable_cast(a), graph::variable_cast(b)} + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (2) && "Expected two."); + work.template run (); + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); +} + +//------------------------------------------------------------------------------ +/// @brief Test callback functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order +//------------------------------------------------------------------------------ +template void test_callbacks() { + int i = 1; + + workflow::manager work(0); + work.template add_callback_item ([&i]() { + i = 2; + }); + + work.compile(); + + assert(i == 1 && "Expected 1"); + work.template run (); + work.wait(); + assert(i == 2 && "Expected 2"); +} + +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order +//------------------------------------------------------------------------------ +template void test_maps() { auto a = graph::variable (1, ""); auto b = graph::variable (1, ""); backend::buffer buffer(1, static_cast (1)); @@ -27,19 +106,19 @@ template void test_maps() { auto zero = graph::zero (); workflow::manager work(0); - work.add_item({ + work.template add_item ({ graph::variable_cast(a), graph::variable_cast(b) }, {}, { {zero, graph::variable_cast(a)}, {zero, graph::variable_cast(b)} - }, NULL, "test_maps", 1); + }, {}, NULL, "test_maps", 1); work.compile(); assert(work.check_value(0, a) == static_cast (1) && "Expected one."); assert(work.check_value(0, b) == static_cast (1) && "Expected one."); - work.run(); + work.template run (); assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); } @@ -48,8 +127,9 @@ template void test_maps() { /// @brief Test loop items. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_loops() { +template void test_loops() { auto a = graph::variable (1, ""); backend::buffer buffer(1, static_cast (0)); a->set(buffer); @@ -57,27 +137,42 @@ template void test_loops() { auto a_next = a + static_cast (1); workflow::manager work(0); - work.add_loop_item({ + work.template add_loop_item ({ graph::variable_cast(a) }, {}, { {a_next, graph::variable_cast(a)} - }, NULL, "test_maps", 1, 10); + }, {}, NULL, "test_maps", 1, 10); work.compile(); assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); - work.run(); + work.template run (); assert(work.check_value(0, a) == static_cast (10) && "Expected ten."); } +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified backend. +/// +/// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order +//------------------------------------------------------------------------------ +template void run_tests_order() { + test_zeros (); + test_copy (); + test_callbacks (); + test_maps (); + test_loops (); +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified backend. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { - test_maps (); - test_loops (); + run_tests_order (); + run_tests_order (); + run_tests_order (); } //------------------------------------------------------------------------------