From 98121765d919b69e129fb8a708532efe9ce82e12 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 1 Jun 2026 11:00:28 -0400 Subject: [PATCH 01/51] Save progress. This builds up the initalization needed to interpolate fields. --- graph_pic/xpic.cpp | 109 ++++++++++++++++++++++++------- graph_playground/xplayground.cpp | 63 ++++++++++++++++++ 2 files changed, 148 insertions(+), 24 deletions(-) diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index b36e90f..bce50ca 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -4,6 +4,7 @@ //------------------------------------------------------------------------------ #include +#include #include "../graph_framework/graph_framework.hpp" @@ -42,28 +43,88 @@ graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { //------------------------------------------------------------------------------ template void run_pic() { +// Constants const size_t num_particles = 1000000; + const size_t num_grid = 200; + const T c = 299792458.0; + const T epsilon0 = 8.854E-12; + const T q = 1.602E-19; + const T m_hydrogen = 1.6738E-27; + const T m_electron = 9.1093837139E-31; + const T kb = 1.380650E-23; + const uint8_t Z = 1; + +// Characteristic factors + const T mchar = (m_hydrogen + m_electron)/2; + const T qchar = (q + q)/2; + const T nechar = 2.5E19; + const T wpechar = std::sqrt(nechar*qchar*qchar/(mchar*epsilon0)); + const T tchar = 1/wpechar; + const T echar = mchar*c/(qchar*tchar); + const T bchar = echar/c; + +// Particle initalization. auto x = graph::variable (num_particles, "x"); - auto vpara = graph::variable (num_particles, "v||"); - - std::normal_distribution norm(0, 0.25); - std::random_device rand_d; - std::mt19937_64 engine(rand_d()); - backend::buffer a(num_particles); - backend::buffer b(num_particles); - for (size_t i = 0; i < num_particles; i++) { - a[i] = norm(engine); - b[i] = norm(engine); + auto vpara = graph::variable (num_particles, "v_{||}"); + auto vperp = graph::variable (num_particles, "v_{\\perp}"); + + { + std::uniform_real_distribution position_dist(-0.25, 0.25); + std::uniform_real_distribution phi_dist(0.0, 2.0*std::numbers::pi_v); + std::uniform_real_distribution r_dist(0.0, 1.0); + + std::random_device rand_d; + std::mt19937_64 engine(rand_d()); + + backend::buffer pos_buffer(num_particles); + backend::buffer vpara_buffer(num_particles); + backend::buffer vperp_buffer(num_particles); + + for (size_t i = 0; i < num_particles; i++) { + pos_buffer[i] = position_dist(engine); + T phi = phi_dist(engine); + T r = r_dist(engine); + vpara_buffer[i] = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::sin(phi)); + phi = phi_dist(engine); + r = r_dist(engine); + const T vperp1 = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::cos(phi)); + const T vperp2 = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::sin(phi)); + vperp_buffer[i] = std::sqrt(vperp1*vperp1 + vperp2*vperp2); + } + x->set(pos_buffer); + vpara->set(vpara_buffer); + vperp->set(vperp_buffer); } - x->set(a); - vpara->set(b); - const T m = 1;//9.1093837139E-31; - const T q = 1;//1.602176634E-19; - const T te = 1; - const T dt = 0.00001; +// Electron initialization. + auto te = graph::variable (num_grid, static_cast (1.0), "t_{e}"); + +// Magnetic field. + auto bfield = (x*x + static_cast (1))/bchar; + +// Electric field. + auto efield = graph::variable (num_grid, "E_{||}"); + +// Time step + const T gyro_frequency = q*1*1.2/2; + const T gyro_period = 2*std::numbers::pi_v/gyro_frequency; + T dt = 0.25*gyro_period; + + const size_t timeIterations = std::ceil(10/0.25); + const size_t outputCadence = std::ceil(600/0.25); + +// Normalize + dt /= tchar; + vpara = vpara/c; + vperp = vperp/c; + x = x*wpechar/c; + + bfield = bfield/bchar; + + + +// Build the field solver; - const size_t num_grid = 1000; auto epara = graph::variable (num_grid, "e||"); auto n = graph::variable (num_grid, "n"); auto grid_position = graph::variable (num_grid, "x_i"); @@ -71,23 +132,23 @@ void run_pic() { const T scale = 2.0/999.0; const T offset = -1.0; - backend::buffer c(num_grid); + backend::buffer coe(num_grid); for (size_t i = 0; i < num_grid; i++) { - c[i] = scale*i + offset; + coe[i] = scale*i + offset; } - grid_position->set(c); + grid_position->set(coe); auto x1 = dt*vpara; - auto vpara1 = -q/m*graph::index_1D(epara, x, scale, offset); + auto vpara1 = -q/m_electron*graph::index_1D(epara, x, scale, offset); auto x2 = dt*(vpara + vpara1/2.0); - auto vpara2 = -q/m*graph::index_1D(epara, x + x1/2.0, scale, offset); + auto vpara2 = -q/m_electron*graph::index_1D(epara, x + x1/2.0, scale, offset); auto x3 = dt*(vpara + vpara2/2.0); - auto vpara3 = -q/m*graph::index_1D(epara, x + x2/2.0, scale, offset); + auto vpara3 = -q/m_electron*graph::index_1D(epara, x + x2/2.0, scale, offset); auto x4 = dt*(vpara + vpara3); - auto vpara4 = -q/m*graph::index_1D(epara, x + x3, scale, offset); + auto vpara4 = -q/m_electron*graph::index_1D(epara, x + x3, scale, offset); auto x_next = x + (x1 + static_cast (2)*(x2 + x3) + x4)/static_cast (6); auto vpara_next = vpara + (vpara1 + static_cast (2)*(vpara2 + vpara3) + vpara4)/static_cast (6); diff --git a/graph_playground/xplayground.cpp b/graph_playground/xplayground.cpp index 7781207..654e2c2 100644 --- a/graph_playground/xplayground.cpp +++ b/graph_playground/xplayground.cpp @@ -17,6 +17,69 @@ int main(int argc, const char * argv[]) { // Insert code here. No code should be committed to this file beyond this // template. + const size_t num_mesh = 10; + const size_t num_particles = 100; + + auto xmesh = graph::variable (num_mesh, "x_mesh"); + auto ymesh = graph::variable (num_mesh, "y_mesh"); + + const double xmin = -3; + const double xmax = 3; + const double dx = (xmax - xmin)/(num_mesh - 1); + + for (size_t i = 0; i < num_mesh; i++) { + graph::variable_cast(xmesh)->data()[i] = dx*i + xmin; + graph::variable_cast(ymesh)->data()[i] = std::sin(std::exp(graph::variable_cast(xmesh)->data()[i])); + } + + auto xp = graph::variable (num_particles, "xp"); + const double dxp = (xmax - xmin)/(num_particles - 1); + for (size_t i = 0; i < num_particles; i++) { + graph::variable_cast(xp)->data()[i] = dxp*i + xmin; + } + + auto x = graph::index_1D(xmesh, xp, dx, xmin) - xp; + auto xnorm1 = 1.5 + (x - dx)/dx; + auto xnorm2 = x/dx; + auto xnorm3 = 1.5 - (x + dx)/dx; + + auto w0 = 0.5*xnorm1*xnorm1; + auto w1 = 0.75 - xnorm2*xnorm2; + auto w2 = 0.5*xnorm3*xnorm3; + + (1.5 - ((x + dx)/dx))->to_latex(); + std::cout << std::endl; + + auto weigth = w0 + w1 + w2; + + workflow::manager work(0); + work.add_item({ + graph::variable_cast(xmesh), + graph::variable_cast(xp) + }, { + x, + weigth, + w0, + w1, + w2 + }, {}, NULL, "Mesh_Interpolation", num_particles); + work.compile(); + + output::result_file file("/Users/m4c/Projects/graph_framework/build/mesh.nc", num_particles); + output::data_set dataset(file); + dataset.create_variable(file, "x", x, work.get_context()); + dataset.create_variable(file, "xp", xp, work.get_context()); + dataset.create_variable(file, "weigth", weigth, work.get_context()); + dataset.create_variable(file, "w0", w0, work.get_context()); + dataset.create_variable(file, "w1", w1, work.get_context()); + dataset.create_variable(file, "w2", w2, work.get_context()); + + file.end_define_mode(); + + work.run(); + work.wait(); + + dataset.write(file); END_GPU } From 29baca7383e7b54090667e2417a955a72fc2742a Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 2 Jun 2026 16:06:56 -0400 Subject: [PATCH 02/51] Add routine and unit test for PIC field interpolation. --- graph_framework.xcodeproj/project.pbxproj | 6 + graph_framework/CMakeLists.txt | 1 + graph_framework/graph_framework.hpp | 1 + graph_framework/particle_in_cell.hpp | 81 +++++++++ graph_playground/xplayground.cpp | 63 ------- graph_tests/CMakeLists.txt | 1 + graph_tests/pic_test.cpp | 208 ++++++++++++++++++++++ 7 files changed, 298 insertions(+), 63 deletions(-) create mode 100644 graph_framework/particle_in_cell.hpp create mode 100644 graph_tests/pic_test.cpp diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index e6213ad..ae90183 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -47,6 +47,7 @@ C78F3DA82DC41BCA002E3D94 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C79141B622DAAD0C00E0BA0D /* xrays.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C79141B522DAAD0C00E0BA0D /* xrays.cpp */; }; C7B676082AA9023F005AB34C /* xrays_bench.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7B676072AA9023F005AB34C /* xrays_bench.cpp */; }; + C7C16FCF2FCF3EF2008D4ABB /* particle_in_cell.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */; }; C7D12D9A2DBAB31F00925420 /* random.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C7D12D992DBAB31F00925420 /* random.hpp */; }; C7D371132A0595A40074676E /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C7DC9EEC2E39790100524F6F /* graph_c_binding.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7DC9EE22E39768300524F6F /* graph_c_binding.cpp */; }; @@ -420,6 +421,8 @@ C7B676092AA90243005AB34C /* CMakeLists.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = CMakeLists.txt; sourceTree = ""; }; C7B677D829E45C9500D3ADC6 /* backend.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = backend.hpp; sourceTree = ""; }; C7B677DA29E464AE00D3ADC6 /* cpu_context.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = cpu_context.hpp; sourceTree = ""; }; + C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = particle_in_cell.hpp; sourceTree = ""; }; + C7C16FD02FCF488F008D4ABB /* pic_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = pic_test.cpp; sourceTree = ""; }; C7CEA0042948D02A00F61D09 /* timing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = timing.hpp; sourceTree = ""; }; C7CEA0052948EB0F00F61D09 /* cuda_context.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = cuda_context.hpp; sourceTree = ""; }; C7D12D992DBAB31F00925420 /* random.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = random.hpp; sourceTree = ""; }; @@ -775,6 +778,7 @@ C7CEA0042948D02A00F61D09 /* timing.hpp */, C73BBE9629F8669F0027BB7F /* newton.hpp */, C7E134492A3CB3EC0083F6A7 /* output.hpp */, + C7C16FCE2FCF3EF2008D4ABB /* particle_in_cell.hpp */, ); path = graph_framework; sourceTree = ""; @@ -810,6 +814,7 @@ C7DC9EF12E3A688F00524F6F /* c_binding_test.c */, C7AE06662E3C2AEE00586BCD /* f_binding_test.f90 */, C74F2ADE2F6DC10E00B48216 /* workflow_test.cpp */, + C7C16FD02FCF488F008D4ABB /* pic_test.cpp */, ); path = graph_tests; sourceTree = ""; @@ -850,6 +855,7 @@ C736903F2A38C958001733B0 /* dispersion.hpp in Headers */, C73690402A38C958001733B0 /* solver.hpp in Headers */, C7D12D9A2DBAB31F00925420 /* random.hpp in Headers */, + C7C16FCF2FCF3EF2008D4ABB /* particle_in_cell.hpp in Headers */, C73690412A38C958001733B0 /* backend.hpp in Headers */, C73690422A38C958001733B0 /* equilibrium.hpp in Headers */, C73690432A38C958001733B0 /* jit.hpp in Headers */, diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 87ddff0..93169d0 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -66,4 +66,5 @@ target_precompile_headers (graph_framework $<$:$> $<$:$<$:$>> $<$:$<$:$>> + $<$:$> ) diff --git a/graph_framework/graph_framework.hpp b/graph_framework/graph_framework.hpp index 6139869..93b894c 100644 --- a/graph_framework/graph_framework.hpp +++ b/graph_framework/graph_framework.hpp @@ -27,6 +27,7 @@ #include "trigonometry.hpp" #include "vector.hpp" #include "workflow.hpp" +#include "particle_in_cell.hpp" #ifdef USE_CUDA #include "cuda_context.hpp" diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp new file mode 100644 index 0000000..41fe198 --- /dev/null +++ b/graph_framework/particle_in_cell.hpp @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +/// @file particle_in_cell.hpp +/// @brief Utilities needed for a particle in cell code. +/// +/// Defines graphs for use in Particle In Cell (PIC) codes. +//------------------------------------------------------------------------------ + +#ifndef particle_in_cell_h +#define particle_in_cell_h + +#include "piecewise.hpp" +#include "workflow.hpp" + +namespace pic { +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] xmesh X position of mesh points. +/// @param[in] ymesh Y position of the mesh. +/// @param[in] xp X position of the particles. +/// @param[in] xmin Minimum X position of the mesh. +/// @param[in] dx Size of the mesh cells. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_interpolation(graph::shared_leaf xmesh, + graph::shared_leaf ymesh, + graph::shared_leaf xp, + const T xmin, + const T dx) { + auto x = graph::index_1D(xmesh, xp, dx, xmin) - xp; + auto xnorm1 = 1.5 + (x - dx)/dx; + auto xnorm2 = x/dx; + auto xnorm3 = 1.5 - (x + dx)/dx; + + auto w0 = 0.5*xnorm1*xnorm1; + auto w1 = 0.75 - xnorm2*xnorm2; + auto w2 = 0.5*xnorm3*xnorm3; + + auto ymesh0 = graph::index_1D(ymesh, xp - dx, dx, xmin); + auto ymesh1 = graph::index_1D(ymesh, xp, dx, xmin); + auto ymesh2 = graph::index_1D(ymesh, xp + dx, dx, xmin); + +// Run only for unit tests. + if constexpr (UNIT_TEST) { + auto xp_cast = graph::variable_cast(xp); + assert(xp_cast.get() && "Expected a variable."); + + auto weight = w0 + w1 + w2; + + workflow::manager work(0); + work.add_item({ + graph::variable_cast(xmesh), + graph::variable_cast(xp) + }, { + weight + }, {}, NULL, "Mesh_Interpolation", xp_cast->size()); + work.compile(); + work.run(); + work.wait(); + +// The weights should sum to 1. + for (size_t i = 0, ie = xp_cast->size(); i < ie; i++) { + const T recieved = work.check_value(i, weight); + const T diff = static_cast (1) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.2E-14) && + "Weight not equal to 1±3.2E-14"); + } else { + assert(diff*diff < static_cast (5.0E-32) && + "Weight not equal to 1±5.0E-32"); + } + } + } + + return w0*ymesh0 + w1*ymesh1 + w2*ymesh2; + } +} + +#endif /* particle_in_cell_h */ diff --git a/graph_playground/xplayground.cpp b/graph_playground/xplayground.cpp index 654e2c2..7781207 100644 --- a/graph_playground/xplayground.cpp +++ b/graph_playground/xplayground.cpp @@ -17,69 +17,6 @@ int main(int argc, const char * argv[]) { // Insert code here. No code should be committed to this file beyond this // template. - const size_t num_mesh = 10; - const size_t num_particles = 100; - - auto xmesh = graph::variable (num_mesh, "x_mesh"); - auto ymesh = graph::variable (num_mesh, "y_mesh"); - - const double xmin = -3; - const double xmax = 3; - const double dx = (xmax - xmin)/(num_mesh - 1); - - for (size_t i = 0; i < num_mesh; i++) { - graph::variable_cast(xmesh)->data()[i] = dx*i + xmin; - graph::variable_cast(ymesh)->data()[i] = std::sin(std::exp(graph::variable_cast(xmesh)->data()[i])); - } - - auto xp = graph::variable (num_particles, "xp"); - const double dxp = (xmax - xmin)/(num_particles - 1); - for (size_t i = 0; i < num_particles; i++) { - graph::variable_cast(xp)->data()[i] = dxp*i + xmin; - } - - auto x = graph::index_1D(xmesh, xp, dx, xmin) - xp; - auto xnorm1 = 1.5 + (x - dx)/dx; - auto xnorm2 = x/dx; - auto xnorm3 = 1.5 - (x + dx)/dx; - - auto w0 = 0.5*xnorm1*xnorm1; - auto w1 = 0.75 - xnorm2*xnorm2; - auto w2 = 0.5*xnorm3*xnorm3; - - (1.5 - ((x + dx)/dx))->to_latex(); - std::cout << std::endl; - - auto weigth = w0 + w1 + w2; - - workflow::manager work(0); - work.add_item({ - graph::variable_cast(xmesh), - graph::variable_cast(xp) - }, { - x, - weigth, - w0, - w1, - w2 - }, {}, NULL, "Mesh_Interpolation", num_particles); - work.compile(); - - output::result_file file("/Users/m4c/Projects/graph_framework/build/mesh.nc", num_particles); - output::data_set dataset(file); - dataset.create_variable(file, "x", x, work.get_context()); - dataset.create_variable(file, "xp", xp, work.get_context()); - dataset.create_variable(file, "weigth", weigth, work.get_context()); - dataset.create_variable(file, "w0", w0, work.get_context()); - dataset.create_variable(file, "w1", w1, work.get_context()); - dataset.create_variable(file, "w2", w2, work.get_context()); - - file.end_define_mode(); - - work.run(); - work.wait(); - - dataset.write(file); END_GPU } diff --git a/graph_tests/CMakeLists.txt b/graph_tests/CMakeLists.txt index 9a2fcc0..d3b85bc 100644 --- a/graph_tests/CMakeLists.txt +++ b/graph_tests/CMakeLists.txt @@ -13,6 +13,7 @@ add_test_target (erfi_test cpp) add_test_target (efit_test cpp) add_test_target (random_test cpp) add_test_target (workflow_test cpp) +add_test_target (pic_test cpp) target_compile_definitions (erfi_test PRIVATE diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp new file mode 100644 index 0000000..eaa849b --- /dev/null +++ b/graph_tests/pic_test.cpp @@ -0,0 +1,208 @@ +//------------------------------------------------------------------------------ +/// @file pic_test.cpp +/// @brief Tests for the particle in cell functions interface. +//------------------------------------------------------------------------------ + +// Turn on asserts even in release builds. +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include + +#include "../graph_framework/graph_framework.hpp" + +//------------------------------------------------------------------------------ +/// @brief Run interpolation test. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_interpolation_test() { + const size_t num_mesh = 100; + const size_t num_particles = 10000; + + auto xmesh = graph::variable (num_mesh, "x_mesh"); + auto ymesh = graph::variable (num_mesh, "y_mesh"); + + const T xmin = static_cast (-3); + const T xmax = static_cast (3); + const T dx = (xmax - xmin)/(num_mesh - 1); + + std::function func([](const T x) -> T { + return std::sin(std::exp(x)); + }); + + for (size_t i = 0; i < num_mesh; i++) { + graph::variable_cast(xmesh)->data()[i] = dx*i + xmin; + graph::variable_cast(ymesh)->data()[i] = func(graph::variable_cast(xmesh)->data()[i]); + } + + auto xp = graph::variable (num_particles, "xp"); + const T dxp = (xmax - xmin)/(num_particles - 1); + for (size_t i = 0; i < num_particles; i++) { + graph::variable_cast(xp)->data()[i] = dxp*i + xmin; + } + + auto field = pic::build_interpolation (xmesh, ymesh, xp, xmin, dx); + + workflow::manager work(0); + work.add_item({ + graph::variable_cast(xmesh), + graph::variable_cast(ymesh), + graph::variable_cast(xp) + }, { + field + }, {}, NULL, "Mesh_Interpolation", num_particles); + work.compile(); + + work.run(); + work.wait(); + + auto xp_cast = graph::variable_cast(xp); + for (size_t i = 0, ie = xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.0E-7) && + "Profile not equal to 1±4.0E-7"); + } else { + assert(diff*diff < static_cast (4.0E-7) && + "Profile not equal to 1±4.0E-7"); + } + } + for (size_t i = xp_cast->size()/10, ie = 2*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.4E-6) && + "Profile not equal to 1±1.4E-6"); + } else { + assert(diff*diff < static_cast (1.1E-6) && + "Profile not equal to 1±1.1E-7"); + } + } + for (size_t i = 2*xp_cast->size()/10, ie = 3*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.8E-6) && + "Profile not equal to 1±3.8E-6"); + } else { + assert(diff*diff < static_cast (1.5E-8) && + "Profile not equal to 1±1.5E-8"); + } + } + for (size_t i = 3*xp_cast->size()/10, ie = 4*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (5.9E-4) && + "Profile not equal to 1±5.9E-4"); + } else { + assert(diff*diff < static_cast (7.9E-6) && + "Profile not equal to 1±7.9E-6"); + } + } + for (size_t i = 4*xp_cast->size()/10, ie = 5*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.9E-5) && + "Profile not equal to 1±1.9E-5"); + } else { + assert(diff*diff < static_cast (2.1E-8) && + "Profile not equal to 1±2.1E-8"); + } + } + for (size_t i = 5*xp_cast->size()/10, ie = 6*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.6E-5) && + "Profile not equal to 1±1.6E-5"); + } else { + assert(diff*diff < static_cast (2.9E-6) && + "Profile not equal to 1±2.9E-6"); + } + } + for (size_t i = 6*xp_cast->size()/10, ie = 7*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.0E-2) && + "Profile not equal to 1±4.0E-2"); + } else { + assert(diff*diff < static_cast (7.0E-6) && + "Profile not equal to 1±7.0E-6"); + } + } + for (size_t i = 7*xp_cast->size()/10, ie = 8*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.5E-3) && + "Profile not equal to 1±1.5E-3"); + } else { + assert(diff*diff < static_cast (1.5E-3) && + "Profile not equal to 1±1.5E-3"); + } + } + for (size_t i = 8*xp_cast->size()/10, ie = 9*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (5.0E-3) && + "Profile not equal to 1±5.0E-3"); + } else { + assert(diff*diff < static_cast (3.0E-3) && + "Profile not equal to 1±3.0E-3"); + } + } + for (size_t i = 9*xp_cast->size()/10, ie = 10*xp_cast->size()/10; i < ie; i++) { + const T x = work.check_value(i, xp); + const T recieved = work.check_value(i, field); + const T diff = func(x) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.8E-2) && + "Profile not equal to 1±1.8E-2"); + } else { + assert(diff*diff < static_cast (1.8E-2) && + "Profile not equal to 1±1.8E-2"); + } + } +} + +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified backend. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_tests() { + run_interpolation_test (); +} + +//------------------------------------------------------------------------------ +/// @brief Main program of the test. +/// +/// @param[in] argc Number of commandline arguments. +/// @param[in] argv Array of commandline arguments. +//------------------------------------------------------------------------------ +int main(int argc, const char * argv[]) { + START_GPU + + (void)argc; + (void)argv; + run_tests (); + run_tests (); + + END_GPU +} From 48f84f9b0895fac68620f83ccd295b2e5b37d56e Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 3 Jun 2026 16:05:41 -0400 Subject: [PATCH 03/51] Minor formatting changes. --- graph_framework.xcodeproj/project.pbxproj | 93 +++++++++++++++++++++++ graph_framework/particle_in_cell.hpp | 12 +-- graph_tests/pic_test.cpp | 85 ++++++++++----------- 3 files changed, 141 insertions(+), 49 deletions(-) diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index ae90183..bcdc0fa 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -9,6 +9,8 @@ /* Begin PBXBuildFile section */ C70D93152A30FF4E006A4227 /* special_functions.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C70D93132A30FF4E006A4227 /* special_functions.hpp */; }; C713426A2947F39400672AD4 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C715C79B2FD09E7D003EEFF4 /* pic_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7C16FD02FCF488F008D4ABB /* pic_test.cpp */; }; + C715C79C2FD09FF3003EEFF4 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C7170CC02C66A228003274E2 /* efit_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7D3C5B02C654AD3008AD8C6 /* efit_test.cpp */; }; C7170CC12C66A238003274E2 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C73690382A38C958001733B0 /* node.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C79141AE22DA9C3000E0BA0D /* node.hpp */; }; @@ -165,6 +167,15 @@ /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ + C715C7922FD09E29003EEFF4 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C7170CB72C66A10D003274E2 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -356,6 +367,7 @@ C713425C2942665300672AD4 /* register.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = register.hpp; sourceTree = ""; }; C71342652947D57900672AD4 /* metal_context.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = metal_context.hpp; sourceTree = ""; }; C71342682947F36100672AD4 /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = System/Library/Frameworks/Metal.framework; sourceTree = SDKROOT; }; + C715C7942FD09E29003EEFF4 /* pic_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = pic_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7167B222AC5CE8500E03131 /* fix_NaN.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = fix_NaN.py; sourceTree = ""; }; C7170CB92C66A10D003274E2 /* efit_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = efit_test; sourceTree = BUILT_PRODUCTS_DIR; }; C717CB8D2A02E361008FBDD8 /* FindNetCDF.cmake */ = {isa = PBXFileReference; lastKnownFileType = text; path = FindNetCDF.cmake; sourceTree = ""; }; @@ -451,6 +463,14 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + C715C7912FD09E29003EEFF4 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C715C79C2FD09FF3003EEFF4 /* Metal.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7170CB62C66A10D003274E2 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -746,6 +766,7 @@ C7DC9EE82E39789900524F6F /* libgraph_c.a */, C74F2AD22F6D9A6E00B48216 /* graph_pic */, C74F2AE32F6DE8C500B48216 /* workflow_test */, + C715C7942FD09E29003EEFF4 /* pic_test */, ); name = Products; sourceTree = ""; @@ -879,6 +900,25 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXNativeTarget section */ + C715C7932FD09E29003EEFF4 /* pic_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C715C79A2FD09E29003EEFF4 /* Build configuration list for PBXNativeTarget "pic_test" */; + buildPhases = ( + C715C7902FD09E29003EEFF4 /* Sources */, + C715C7912FD09E29003EEFF4 /* Frameworks */, + C715C7922FD09E29003EEFF4 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = pic_test; + packageProductDependencies = ( + ); + productName = pic_test; + productReference = C715C7942FD09E29003EEFF4 /* pic_test */; + productType = "com.apple.product-type.tool"; + }; C7170CB82C66A10D003274E2 /* efit_test */ = { isa = PBXNativeTarget; buildConfigurationList = C7170CBF2C66A10D003274E2 /* Build configuration list for PBXNativeTarget "efit_test" */; @@ -1289,6 +1329,9 @@ LastUpgradeCheck = 2610; ORGANIZATIONNAME = "Cianciosa, Mark R."; TargetAttributes = { + C715C7932FD09E29003EEFF4 = { + CreatedOnToolsVersion = 26.4; + }; C7170CB82C66A10D003274E2 = { CreatedOnToolsVersion = 15.4; }; @@ -1392,11 +1435,20 @@ C7DC9EE72E39789900524F6F /* graph_c */, C74F2AD12F6D9A6E00B48216 /* graph_pic */, C74F2AE22F6DE8C500B48216 /* workflow_test */, + C715C7932FD09E29003EEFF4 /* pic_test */, ); }; /* End PBXProject section */ /* Begin PBXSourcesBuildPhase section */ + C715C7902FD09E29003EEFF4 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C715C79B2FD09E7D003EEFF4 /* pic_test.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7170CB52C66A10D003274E2 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1648,6 +1700,38 @@ /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + C715C7982FD09E29003EEFF4 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C715C7992FD09E29003EEFF4 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C7170CBD2C66A10D003274E2 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -2697,6 +2781,15 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + C715C79A2FD09E29003EEFF4 /* Build configuration list for PBXNativeTarget "pic_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C715C7982FD09E29003EEFF4 /* Debug */, + C715C7992FD09E29003EEFF4 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C7170CBF2C66A10D003274E2 /* Build configuration list for PBXNativeTarget "efit_test" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 41fe198..82f7689 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -30,13 +30,13 @@ namespace pic { const T xmin, const T dx) { auto x = graph::index_1D(xmesh, xp, dx, xmin) - xp; - auto xnorm1 = 1.5 + (x - dx)/dx; + auto xnorm1 = static_cast (1.5) + (x - dx)/dx; auto xnorm2 = x/dx; - auto xnorm3 = 1.5 - (x + dx)/dx; + auto xnorm3 = static_cast (1.5) - (x + dx)/dx; - auto w0 = 0.5*xnorm1*xnorm1; - auto w1 = 0.75 - xnorm2*xnorm2; - auto w2 = 0.5*xnorm3*xnorm3; + auto w0 = static_cast (0.5)*xnorm1*xnorm1; + auto w1 = static_cast (0.75) - xnorm2*xnorm2; + auto w2 = static_cast (0.5)*xnorm3*xnorm3; auto ymesh0 = graph::index_1D(ymesh, xp - dx, dx, xmin); auto ymesh1 = graph::index_1D(ymesh, xp, dx, xmin); @@ -55,7 +55,7 @@ namespace pic { graph::variable_cast(xp) }, { weight - }, {}, NULL, "Mesh_Interpolation", xp_cast->size()); + }, {}, NULL, "build_interpolation_unit_test", xp_cast->size()); work.compile(); work.run(); work.wait(); diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index eaa849b..8e4a520 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -54,135 +54,134 @@ template void run_interpolation_test() { field }, {}, NULL, "Mesh_Interpolation", num_particles); work.compile(); - work.run(); work.wait(); auto xp_cast = graph::variable_cast(xp); for (size_t i = 0, ie = xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (4.0E-7) && - "Profile not equal to 1±4.0E-7"); + "Profile not equal ±4.0E-7"); } else { assert(diff*diff < static_cast (4.0E-7) && - "Profile not equal to 1±4.0E-7"); + "Profile not equal ±4.0E-7"); } } for (size_t i = xp_cast->size()/10, ie = 2*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (1.4E-6) && - "Profile not equal to 1±1.4E-6"); + "Profile not equal ±1.4E-6"); } else { assert(diff*diff < static_cast (1.1E-6) && - "Profile not equal to 1±1.1E-7"); + "Profile not equal ±1.1E-7"); } } for (size_t i = 2*xp_cast->size()/10, ie = 3*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (3.8E-6) && - "Profile not equal to 1±3.8E-6"); + "Profile not equal ±3.8E-6"); } else { assert(diff*diff < static_cast (1.5E-8) && - "Profile not equal to 1±1.5E-8"); + "Profile not equal ±1.5E-8"); } } for (size_t i = 3*xp_cast->size()/10, ie = 4*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (5.9E-4) && - "Profile not equal to 1±5.9E-4"); + "Profile not equal ±5.9E-4"); } else { assert(diff*diff < static_cast (7.9E-6) && - "Profile not equal to 1±7.9E-6"); + "Profile not equal ±7.9E-6"); } } for (size_t i = 4*xp_cast->size()/10, ie = 5*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (1.9E-5) && - "Profile not equal to 1±1.9E-5"); + "Profile not equal ±1.9E-5"); } else { assert(diff*diff < static_cast (2.1E-8) && - "Profile not equal to 1±2.1E-8"); + "Profile not equal ±2.1E-8"); } } for (size_t i = 5*xp_cast->size()/10, ie = 6*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (1.6E-5) && - "Profile not equal to 1±1.6E-5"); + "Profile not equal ±1.6E-5"); } else { assert(diff*diff < static_cast (2.9E-6) && - "Profile not equal to 1±2.9E-6"); + "Profile not equal ±2.9E-6"); } } for (size_t i = 6*xp_cast->size()/10, ie = 7*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (4.0E-2) && - "Profile not equal to 1±4.0E-2"); + "Profile not equal ±4.0E-2"); } else { assert(diff*diff < static_cast (7.0E-6) && - "Profile not equal to 1±7.0E-6"); + "Profile not equal ±7.0E-6"); } } for (size_t i = 7*xp_cast->size()/10, ie = 8*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (1.5E-3) && - "Profile not equal to 1±1.5E-3"); + "Profile not equal ±1.5E-3"); } else { assert(diff*diff < static_cast (1.5E-3) && - "Profile not equal to 1±1.5E-3"); + "Profile not equal ±1.5E-3"); } } for (size_t i = 8*xp_cast->size()/10, ie = 9*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (5.0E-3) && - "Profile not equal to 1±5.0E-3"); + "Profile not equal ±5.0E-3"); } else { assert(diff*diff < static_cast (3.0E-3) && - "Profile not equal to 1±3.0E-3"); + "Profile not equal ±3.0E-3"); } } for (size_t i = 9*xp_cast->size()/10, ie = 10*xp_cast->size()/10; i < ie; i++) { const T x = work.check_value(i, xp); - const T recieved = work.check_value(i, field); - const T diff = func(x) - recieved; + const T received = work.check_value(i, field); + const T diff = func(x) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (1.8E-2) && - "Profile not equal to 1±1.8E-2"); + "Profile not equal ±1.8E-2"); } else { assert(diff*diff < static_cast (1.8E-2) && - "Profile not equal to 1±1.8E-2"); + "Profile not equal ±1.8E-2"); } } } //------------------------------------------------------------------------------ -/// @brief Run tests with a specified backend. +/// @brief Run tests with a specified precision. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ @@ -201,7 +200,7 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; - run_tests (); + //run_tests (); run_tests (); END_GPU From ec3b744b4c06712eb48bd4aac6569958dd9af5b5 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 16 Jun 2026 11:34:31 -0400 Subject: [PATCH 04/51] Add framework enhancements necessary to impliment particle reinjection. First is add a modulo operation. This requires disabling derivatives. The second enhancement is to add logic and conditional nodes. These have not been added to the c or fortran bindings yet. --- CMakeLists.txt | 14 + graph_framework.xcodeproj/project.pbxproj | 99 + graph_framework/CMakeLists.txt | 1 + graph_framework/arithmetic.hpp | 262 ++ graph_framework/backend.hpp | 360 +++ graph_framework/graph_framework.hpp | 1 + graph_framework/logical.hpp | 2623 +++++++++++++++++++++ graph_framework/node.hpp | 72 +- graph_framework/particle_in_cell.hpp | 236 +- graph_framework/random.hpp | 19 + graph_framework/register.hpp | 5 +- graph_pic/xpic.cpp | 18 +- graph_tests/CMakeLists.txt | 2 + graph_tests/arithmetic_test.cpp | 27 + graph_tests/jit_test.cpp | 24 +- graph_tests/logical_test.cpp | 265 +++ graph_tests/no_derivative_test.cpp | 52 + graph_tests/pic_test.cpp | 8 +- 18 files changed, 4058 insertions(+), 30 deletions(-) create mode 100644 graph_framework/logical.hpp create mode 100644 graph_tests/logical_test.cpp create mode 100644 graph_tests/no_derivative_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9faf51a..0c42789 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -404,4 +404,18 @@ macro (add_test_target target lang) endif () endmacro () +macro (add_compile_test target lang) + cmake_path (GET CMAKE_CXX_COMPILER FILENAME compiler_command) + add_test (NAME ${target}_off + COMMAND ${compiler_command} -std=c++23 -DUSE_VERBOSE=false -DCHECK_TEST -c ${CMAKE_CURRENT_SOURCE_DIR}/${target}.${lang} + ) + add_test (NAME ${target}_on + COMMAND ${compiler_command} -std=c++23 -DUSE_VERBOSE=false -c ${CMAKE_CURRENT_SOURCE_DIR}/${target}.${lang} + ) + set_tests_properties (${target}_on + PROPERTIES + WILL_FAIL true + ) +endmacro () + add_subdirectory (graph_tests) diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index bcdc0fa..dd2f561 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -43,6 +43,8 @@ C74F2ADD2F6D9B0D00B48216 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C74F2AEA2F6DE8E400B48216 /* workflow_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C74F2ADE2F6DC10E00B48216 /* workflow_test.cpp */; }; C74F2AEB2F6DE8EC00B48216 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C76263472FE0887300F283DF /* logical_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C762633B2FE0754F00F283DF /* logical_test.cpp */; }; + C76263482FE0891300F283DF /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C7DC9EEF2E397BE600524F6F /* Cocoa.framework */; }; C78F3D972DC41AF2002E3D94 /* random_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C78F3D8A2DC122C7002E3D94 /* random_test.cpp */; }; C78F3D982DC41B05002E3D94 /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C78F3DA72DC41BB8002E3D94 /* xkorc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C78F3D882DC122B1002E3D94 /* xkorc.cpp */; }; @@ -248,6 +250,15 @@ ); runOnlyForDeploymentPostprocessing = 1; }; + C762633E2FE0882200F283DF /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C78F3D8D2DC41ACA002E3D94 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -403,8 +414,11 @@ C75C42932E5CA60B00B0950B /* main.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = main.dox; sourceTree = ""; }; C75C42952E5CC80B00B0950B /* tutorial.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = tutorial.dox; sourceTree = ""; }; C760B1AB2BC6D760001737A3 /* get_includes.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = get_includes.py; sourceTree = ""; }; + C762633B2FE0754F00F283DF /* logical_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = logical_test.cpp; sourceTree = ""; }; + C76263402FE0882200F283DF /* logical_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = logical_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7678FBD2B45C2850025F37E /* bin.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = bin.py; sourceTree = ""; }; C77707F62F5F288B00BA4E87 /* kernel_optimization.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = kernel_optimization.dox; sourceTree = ""; }; + C77CA28F2FDB7CBA00D71BF6 /* no_derivative_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = no_derivative_test.cpp; sourceTree = ""; }; C77E6DF522DD64E700469621 /* trigonometry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = trigonometry.hpp; sourceTree = ""; }; C78F3D872DC122B1002E3D94 /* CMakeLists.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = CMakeLists.txt; sourceTree = ""; }; C78F3D882DC122B1002E3D94 /* xkorc.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = xkorc.cpp; sourceTree = ""; }; @@ -458,6 +472,7 @@ C7E5648628A2A324000F31A2 /* vector_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = vector_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E5649228A2A34A000F31A2 /* physics_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = physics_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E7D02F283565A200E09896 /* vector_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; path = vector_test.cpp; sourceTree = ""; }; + C7EBEA5A2FE0648C005C0463 /* logical.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = logical.hpp; sourceTree = ""; }; C7FA0DFD29590B7400A31E4D /* jit_test.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.objcpp; fileEncoding = 4; path = jit_test.cpp; sourceTree = ""; }; C7FA0E0329590EF300A31E4D /* jit_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = jit_test; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ @@ -533,6 +548,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C762633D2FE0882200F283DF /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C76263482FE0891300F283DF /* Cocoa.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C78F3D8C2DC41ACA002E3D94 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -767,6 +790,7 @@ C74F2AD22F6D9A6E00B48216 /* graph_pic */, C74F2AE32F6DE8C500B48216 /* workflow_test */, C715C7942FD09E29003EEFF4 /* pic_test */, + C76263402FE0882200F283DF /* logical_test */, ); name = Products; sourceTree = ""; @@ -777,6 +801,7 @@ C7931E7028073BE70033B488 /* CMakeLists.txt */, C7D453872EBFD05D00A828DB /* graph_framework.hpp */, C79141AE22DA9C3000E0BA0D /* node.hpp */, + C7EBEA5A2FE0648C005C0463 /* logical.hpp */, C72358F52C4027A10084A489 /* commandline_parser.hpp */, C70B705629F4F86A00098AA0 /* piecewise.hpp */, C7922EEB29E0ABDF000BB9C7 /* workflow.hpp */, @@ -836,6 +861,8 @@ C7AE06662E3C2AEE00586BCD /* f_binding_test.f90 */, C74F2ADE2F6DC10E00B48216 /* workflow_test.cpp */, C7C16FD02FCF488F008D4ABB /* pic_test.cpp */, + C77CA28F2FDB7CBA00D71BF6 /* no_derivative_test.cpp */, + C762633B2FE0754F00F283DF /* logical_test.cpp */, ); path = graph_tests; sourceTree = ""; @@ -1064,6 +1091,25 @@ productReference = C74F2AE32F6DE8C500B48216 /* workflow_test */; productType = "com.apple.product-type.tool"; }; + C762633F2FE0882200F283DF /* logical_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C76263442FE0882200F283DF /* Build configuration list for PBXNativeTarget "logical_test" */; + buildPhases = ( + C762633C2FE0882200F283DF /* Sources */, + C762633D2FE0882200F283DF /* Frameworks */, + C762633E2FE0882200F283DF /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = logical_test; + packageProductDependencies = ( + ); + productName = logical_test; + productReference = C76263402FE0882200F283DF /* logical_test */; + productType = "com.apple.product-type.tool"; + }; C78F3D8E2DC41ACA002E3D94 /* random_test */ = { isa = PBXNativeTarget; buildConfigurationList = C78F3D932DC41ACA002E3D94 /* Build configuration list for PBXNativeTarget "random_test" */; @@ -1356,6 +1402,9 @@ C74F2AE22F6DE8C500B48216 = { CreatedOnToolsVersion = 26.1; }; + C762633F2FE0882200F283DF = { + CreatedOnToolsVersion = 26.4; + }; C78F3D8E2DC41ACA002E3D94 = { CreatedOnToolsVersion = 16.3; }; @@ -1436,6 +1485,7 @@ C74F2AD12F6D9A6E00B48216 /* graph_pic */, C74F2AE22F6DE8C500B48216 /* workflow_test */, C715C7932FD09E29003EEFF4 /* pic_test */, + C762633F2FE0882200F283DF /* logical_test */, ); }; /* End PBXProject section */ @@ -1513,6 +1563,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C762633C2FE0882200F283DF /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C76263472FE0887300F283DF /* logical_test.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C78F3D8B2DC41ACA002E3D94 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1998,6 +2056,38 @@ }; name = Release; }; + C76263452FE0882200F283DF /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C76263462FE0882200F283DF /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.4; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C78F3D942DC41ACA002E3D94 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -2862,6 +2952,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + C76263442FE0882200F283DF /* Build configuration list for PBXNativeTarget "logical_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C76263452FE0882200F283DF /* Debug */, + C76263462FE0882200F283DF /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C78F3D932DC41ACA002E3D94 /* Build configuration list for PBXNativeTarget "random_test" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 93169d0..535bc9c 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -67,4 +67,5 @@ target_precompile_headers (graph_framework $<$:$<$:$>> $<$:$<$:$>> $<$:$> + $<$:$> ) diff --git a/graph_framework/arithmetic.hpp b/graph_framework/arithmetic.hpp index 19734d1..8a9aab0 100644 --- a/graph_framework/arithmetic.hpp +++ b/graph_framework/arithmetic.hpp @@ -5402,6 +5402,268 @@ namespace graph { shared_fma fma_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// Modulo node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An Modulo node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class modulo_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "%" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an modulo node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + modulo_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + modulo_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of modulo. +/// +/// result = l % r +/// +/// @returns The value of l % r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result % r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an modulo node. +/// +/// @returns A reduced modulo node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] indices List of defined indices. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = fmod(" + << registers[l.get()] << "," + << registers[r.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\%"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"%\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf modulo(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(shared_leaf l, + shared_leaf r) { + return modulo (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(const L l, + shared_leaf r) { + return modulo (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build modulo node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator%(shared_leaf l, + const R r) { + return modulo (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared modulo nodes. + template + using shared_modulo = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a modulo node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_modulo modulo_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* arithmetic_h */ diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 39170d0..b1f0108 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -772,6 +772,60 @@ namespace backend { } } +//------------------------------------------------------------------------------ +/// @brief Not operation. +/// +/// @returns The negation of the buffer. +//------------------------------------------------------------------------------ + buffer operator!() requires(std::floating_point) { + for (size_t i = 0, ie = memory.size(); i < ie; i++) { + memory[i] = !memory[i]; + } + return memory; + } + +//------------------------------------------------------------------------------ +/// @brief Apply condition. +/// +/// @params[in] t True condition. +/// @params[in] f False condition. +//------------------------------------------------------------------------------ + buffer if_(const buffer &t, + const buffer &f) { + if (size() == 1) { + return memory[0] ? t : f; + } else { + if (t.size() == 1) { + if (f.size() == 1) { + for (size_t i = 0, ie = size(); i < ie; i++) { + memory[i] = memory[i] ? t.at(0) : f.at(0); + } + return memory; + } else { + assert(size() == f.size() && "Incompatable buffersize."); + for (size_t i = 0, ie = size(); i < ie; i++) { + memory[i] = memory[i] ? t.at(0) : f[i]; + } + return memory; + } + } else { + assert(size() == t.size() && "Incompatable buffersize."); + if (f.size() == 1) { + for (size_t i = 0, ie = size(); i < ie; i++) { + memory[i] = memory[i] ? t[i] : f.at(0); + } + return memory; + } else { + assert(size() == f.size() && "Incompatable buffersize."); + for (size_t i = 0, ie = size(); i < ie; i++) { + memory[i] = memory[i] ? t[i] : f[i]; + } + return memory; + } + } + } + } + /// Type def to retrieve the backend T type. typedef T base; }; @@ -1044,6 +1098,312 @@ namespace backend { return a; } +//------------------------------------------------------------------------------ +/// @brief Modulo operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a % b. +//------------------------------------------------------------------------------ + template + inline buffer operator%(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = std::fmod(a[i], right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = std::fmod(left, b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = std::fmod(a[i], b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a == b. +//------------------------------------------------------------------------------ + template + inline buffer operator==(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] == right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left == b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] == b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Not equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a == b. +//------------------------------------------------------------------------------ + template + inline buffer operator!=(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] != right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left != b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] != b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Greater than operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a > b. +//------------------------------------------------------------------------------ + template + inline buffer operator>(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] > right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left > b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] > b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Less than operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a < b. +//------------------------------------------------------------------------------ + template + inline buffer operator<(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] < right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left < b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] < b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Greater than equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a >= b. +//------------------------------------------------------------------------------ + template + inline buffer operator>=(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] >= right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left >= b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] >= b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Less than equal operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a <= b. +//------------------------------------------------------------------------------ + template + inline buffer operator<=(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] <= right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left <= b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] <= b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief And operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a && b. +//------------------------------------------------------------------------------ + template + inline buffer operator&&(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] && right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left && b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] && b.at(i)); + } + return a; + } + +//------------------------------------------------------------------------------ +/// @brief Or operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns a || b. +//------------------------------------------------------------------------------ + template + inline buffer operator||(buffer &a, + buffer &b) { + if (b.size() == 1) { + const T right = b.at(0); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] || right); + } + return a; + } else if (a.size() == 1) { + const T left = a.at(0); + for (size_t i = 0, ie = b.size(); i < ie; i++) { + b[i] = static_cast (left || b.at(i)); + } + return b; + } + + assert(a.size() == b.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = a.size(); i < ie; i++) { + a[i] = static_cast (a[i] || b.at(i)); + } + return a; + } + //------------------------------------------------------------------------------ /// @brief Take the power. /// diff --git a/graph_framework/graph_framework.hpp b/graph_framework/graph_framework.hpp index 93b894c..251b89e 100644 --- a/graph_framework/graph_framework.hpp +++ b/graph_framework/graph_framework.hpp @@ -28,6 +28,7 @@ #include "vector.hpp" #include "workflow.hpp" #include "particle_in_cell.hpp" +#include "logical.hpp" #ifdef USE_CUDA #include "cuda_context.hpp" diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp new file mode 100644 index 0000000..d54daff --- /dev/null +++ b/graph_framework/logical.hpp @@ -0,0 +1,2623 @@ +//------------------------------------------------------------------------------ +/// @file logical.hpp +/// @brief Nodes for boolean logic. +/// +/// Defines a tree of operations that allows automatic differentiation. +//------------------------------------------------------------------------------ +#ifndef logical_h +#define logical_h + +#include "node.hpp" + +/// Name space for graph nodes. +namespace graph { +/// Convenience type for true constant. + template + constexpr shared_leaf true_constant() { + return one (); + } + +/// Convenience type for false constant. + template + constexpr shared_leaf false_constant() { + return zero (); + } + +//****************************************************************************** +// Not node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Not node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class not_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "!" + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an not node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + not_node(shared_leaf arg) : + no_derivative> (arg, + not_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of equal. +/// +/// result = !a +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + return !arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an equal node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + + auto equalc = equal_cast(this->arg); + if (equalc.get()) { + return equalc->get_left() != equalc->get_right(); + } + + auto nequalc = not_equal_cast(this->arg); + if (nequalc.get()) { + return nequalc->get_left() == nequalc->get_right(); + } + + auto ltc = less_than_cast(this->arg); + if (ltc.get()) { + return ltc->get_left() >= ltc->get_right(); + } + + auto gtc = greater_than_cast(this->arg); + if (gtc.get()) { + return gtc->get_left() <= gtc->get_right(); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf arg = this->arg->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = !" + << registers[arg.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool arg_brackets = add_cast(this->arg).get() || + subtract_cast(this->arg).get(); + std::cout << "\neg"; + if (arg_brackets) { + std::cout << "\\left("; + } + this->arg->to_latex(); + if (arg_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build not node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf not_(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf operator!(shared_leaf arg) { + return not_ (arg); + } + +/// Convenience type alias for shared equal nodes. + template + using shared_not = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_not not_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "==" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of equal. +/// +/// result = l == r +/// +/// @returns The value of l == r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result == r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an equal node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "==" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = equal_cast(x); + if (x_cast.get()) { +// equal is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "="; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"==\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(shared_leaf l, + shared_leaf r) { + return equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(const L l, + shared_leaf r) { + return equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator==(shared_leaf l, + const R r) { + return equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared equal nodes. + template + using shared_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_equal equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Not equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A not equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class not_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "!=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an not equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + not_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + not_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of not equal. +/// +/// result = l != r +/// +/// @returns The value of l != r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result != r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an not equal node. +/// +/// @returns A reduced not equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "!=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = not_equal_cast(x); + if (x_cast.get()) { +// equal is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\ne"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build note equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf not_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(shared_leaf l, + shared_leaf r) { + return not_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(const L l, + shared_leaf r) { + return not_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build not equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator!=(shared_leaf l, + const R r) { + return not_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared not equal nodes. + template + using shared_not_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_not_equal not_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Greater than node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A greater than node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class greater_than_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + ">" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a greater than node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + greater_than_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + greater_than_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of greater than. +/// +/// result = l > r +/// +/// @returns The value of l > r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result > r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce greater than node. +/// +/// @returns A reduced not equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << ">" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << ">"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \">\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf greater_than(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(shared_leaf l, + shared_leaf r) { + return greater_than (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(const L l, + shared_leaf r) { + return greater_than (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>(shared_leaf l, + const R r) { + return greater_than (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared greater than nodes. + template + using shared_greater_than = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a greater than node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_greater_than greater_than_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Less than node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A less than node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class less_than_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "<" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a less than node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + less_than_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + less_than_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than. +/// +/// result = l < r +/// +/// @returns The value of l < r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result < r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce less than node. +/// +/// @returns A reduced less than node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "<" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "<"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"<\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf less_than(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(shared_leaf l, + shared_leaf r) { + return less_than (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(const L l, + shared_leaf r) { + return less_than (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<(shared_leaf l, + const R r) { + return less_than (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared less than nodes. + template + using shared_less_than = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a less than node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_less_than less_than_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Greater than equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A greater than equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class greater_than_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + ">=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a greater than equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + greater_than_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + greater_than_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of greater than equal. +/// +/// result = l >= r +/// +/// @returns The value of l >= r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result >= r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce greater than equal node. +/// +/// @returns A reduced greater than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << ">=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\ge"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \">=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf greater_than_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(shared_leaf l, + shared_leaf r) { + return greater_than_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(const L l, + shared_leaf r) { + return greater_than_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build greater than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator>=(shared_leaf l, + const R r) { + return greater_than_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared greater than equal nodes. + template + using shared_greater_than_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a greater than equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_greater_than_equal greater_than_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Less than equal node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A less than equal node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class less_than_equal_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "<=" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a less than equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + less_than_equal_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + less_than_equal_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l <= r +/// +/// @returns The value of l <= r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result <= r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce less than equal node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "<=" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\le"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"<=\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf less_than_equal(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(shared_leaf l, + shared_leaf r) { + return less_than_equal (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(const L l, + shared_leaf r) { + return less_than_equal (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build less than equal node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Float type for the constant. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator<=(shared_leaf l, + const R r) { + return less_than_equal (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared less than equal nodes. + template + using shared_less_than_equal = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a less than equal node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_less_than_equal less_than_equal_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// And node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A and node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class and_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "&&" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an and node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + and_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + and_node::to_string(l.get(), r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l && r +/// +/// @returns The value of l && r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result && r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce and node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "&&" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = and_cast(x); + if (x_cast.get()) { +// and is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\land"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"&&\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf and_(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(shared_leaf l, + shared_leaf r) { + return and_ (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(const bool l, + shared_leaf r) { + return and_ (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build and node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator&&(shared_leaf l, + const bool r) { + return and_ (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared and nodes. + template + using shared_and = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an and node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_and and_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// Or node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A or node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class or_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left node pointer. +/// @param[in] r Right node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return jit::format_to_string(reinterpret_cast (l)) + "||" + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a or node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + or_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + or_node::to_string(l.get(), r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of less than equal. +/// +/// result = l || r +/// +/// @returns The value of l || r. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return l_result || r_result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce or node. +/// +/// @returns A reduced less than equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto l = constant_cast(this->left); + auto r = constant_cast(this->right); + + if (l.get() && r.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf l = this->left->compile(stream, + registers, + indices, + usage); + shared_leaf r = this->right->compile(stream, + registers, + indices, + usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = " + << registers[l.get()] << "||" + << registers[r.get()]; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = and_cast(x); + if (x_cast.get()) { +// or is commutative. + if ((this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right()))) { + return true; + } + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + bool l_brackets = add_cast(this->left).get() || + subtract_cast(this->left).get(); + bool r_brackets = add_cast(this->right).get() || + subtract_cast(this->right).get(); + if (l_brackets) { + std::cout << "\\left("; + } + this->left->to_latex(); + if (l_brackets) { + std::cout << "\\right)"; + } + std::cout << "\\lor"; + if (r_brackets) { + std::cout << "\\left("; + } + this->right->to_latex(); + if (r_brackets) { + std::cout << "\\right)"; + } + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"||\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf or_(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(shared_leaf l, + shared_leaf r) { + return or_ (l, r); + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(const bool l, + shared_leaf r) { + return or_ (constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build or node from two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + template + shared_leaf operator||(shared_leaf l, + const bool r) { + return or_ (l, constant (static_cast (r))); + } + +/// Convenience type alias for shared or nodes. + template + using shared_or = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a or node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_or or_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// If node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An If conditional node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class if_node final : public triple_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] c Condition node. +/// @param[in] t True condition. +/// @param[in] f False condition. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *c, + leaf_node *t, + leaf_node *f) { + return "if(" + + jit::format_to_string(reinterpret_cast (c)) + "," + + jit::format_to_string(reinterpret_cast (t)) + "," + + jit::format_to_string(reinterpret_cast (f)) + ")"; + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] c Condition node. +/// @param[in] t True condition branch. +/// @param[in] f False condition branch. +//------------------------------------------------------------------------------ + if_node(shared_leaf c, + shared_leaf t, + shared_leaf f) : + triple_node (c, t, f, + if_node::to_string(c.get(), + t.get(), + f.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of if. +/// +/// result = if(c, t, f) +/// +/// @returns The value of if(c, t, f). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer c_result = this->left->evaluate(); + backend::buffer t_result = this->middle->evaluate(); + backend::buffer f_result = this->right->evaluate(); + return c_result.if_(t_result, f_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an if node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto c = constant_cast(this->left); + auto t = constant_cast(this->middle); + auto f = constant_cast(this->right); + + if (c.get() && t.get() && f.get()) { + return constant (this->evaluate()); + } + +// If(c, a, a) -> a + if (this->middle->is_match(this->right)) { + return this->middle; + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d if(c,t,f)/dx = if(c,dt/dx,df/dx) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = if_ (this->left, + this->middle->df(x), + this->right->df(x)); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + shared_leaf c = this->left->compile(stream, + registers, + indices, + usage); + registers[this] = jit::to_string('r', this); + stream << " "; + jit::add_type (stream); + stream << " " << registers[this] << ";" << std::endl + << " if(" << registers[c.get()] << ") {" << std::endl; + + shared_leaf t = this->middle->compile(stream, + registers, + indices, + usage); + stream << " " << registers[this] << " = " << registers[t.get()] << ";" + << " } else {" << std::endl; + + shared_leaf f = this->right->compile(stream, + registers, + indices, + usage); + + stream << " " << registers[this] << " = " << registers[f.get()] << ";" + << " }" << std::endl; + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "if\\left("; + this->left->to_latex(); + std::cout << ","; + this->middle->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return if_ (this->left, + this->middle->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"if\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto c = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[c.get()] << ";" << std::endl; + auto t = this->middle->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[t.get()] << ";" << std::endl; + auto f = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[f.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build an if node from a condition and two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] c Condition branch. +/// @param[in] t True branch. +/// @param[in] f False branch. +//------------------------------------------------------------------------------ + template + shared_leaf if_(shared_leaf c, + shared_leaf t, + shared_leaf f) { + auto temp = std::make_shared> (c, t, f)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared add nodes. + template + using shared_if = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an if node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_if if_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } +} + +#endif /* logical_h */ diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 7add76f..a8f65e4 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -346,6 +346,7 @@ #include #include #include +#include #include "backend.hpp" @@ -416,7 +417,9 @@ namespace graph { /// @returns The derivative of the node. //------------------------------------------------------------------------------ virtual std::shared_ptr> - df(std::shared_ptr> x) = 0; + df(std::shared_ptr> x) { + return std::shared_ptr> (); + }; //------------------------------------------------------------------------------ /// @brief Compile preamble. @@ -516,7 +519,7 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Convert the node to latex. //------------------------------------------------------------------------------ - virtual void to_latex() const = 0; + virtual void to_latex() const {}; //------------------------------------------------------------------------------ /// @brief Convert the node to vizgraph. @@ -1373,6 +1376,71 @@ namespace graph { } }; +//------------------------------------------------------------------------------ +/// @brief Type trait for not having a valid derivative. +/// +/// @tparam T Type (Only used to compile time errors) +//------------------------------------------------------------------------------ + template + struct has_no_derivative : std::false_type {}; + +//------------------------------------------------------------------------------ +/// @brief Nodes without derivatives. +/// +/// Some functions have no derivative. This can be used as a base class to +/// case a compile error if a derivative node is attempted. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// @tparam BASE_NODE Base code to subclass from. +//------------------------------------------------------------------------------ + template> + class no_derivative : public BASE_NODE { + public: + template + std::shared_ptr> + df(std::shared_ptr> x) requires(has_no_derivative::value); + +//------------------------------------------------------------------------------ +/// @brief Constructor for base leaf nodes base classes. +/// +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + leaf_node (s, 0, false) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for straight node base classes. +/// +/// @param[in] arg Node argument. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf arg, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + straight_node (arg, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base branch nodes base classes. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf l, + shared_leaf r, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + branch_node (l, r, s) {} + }; + //****************************************************************************** // Variable node. //****************************************************************************** diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 82f7689..6582232 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -8,39 +8,120 @@ #ifndef particle_in_cell_h #define particle_in_cell_h +#include + #include "piecewise.hpp" #include "workflow.hpp" +#include "random.hpp" namespace pic { //------------------------------------------------------------------------------ -/// @brief Build interpolation expression. +/// @brief ion class. /// /// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class ion { + public: +/// Charge + const T charge; +/// Particle mass + const T mass; +/// Position + graph::shared_leaf x; +/// Parallel velocity. + graph::shared_leaf v_para; +/// Perpendicular velocity. + graph::shared_leaf v_perp; + +//------------------------------------------------------------------------------ +/// @brief Construct an ion object. +/// +/// @param[in] charge Ion charge. +/// @param[in] mass Ion mass. +/// @param[in] x Ion position. +/// @param[in] v_para Parallel velocity. +/// @param[in] v_perp Perpendicular velocity. +//------------------------------------------------------------------------------ + ion(const T charge, + const T mass, + graph::shared_leaf x, + graph::shared_leaf v_para, + graph::shared_leaf v_perp) : + charge(charge), mass(mass), x(x), v_para(v_para), v_perp(v_perp) {} + }; + +//------------------------------------------------------------------------------ +/// @brief Mesh class. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class mesh { + public: +/// Mesh x positions. + graph::shared_leaf x; +/// Mesh y values. + graph::shared_leaf y; +/// Min x + const T xmin; +/// Max x + const T xmax; +/// Min mesh spacing. + const T dx; + +//------------------------------------------------------------------------------ +/// @brief Construct a mesh object. /// /// @param[in] xmesh X position of mesh points. /// @param[in] ymesh Y position of the mesh. -/// @param[in] xp X position of the particles. -/// @param[in] xmin Minimum X position of the mesh. -/// @param[in] dx Size of the mesh cells. +//------------------------------------------------------------------------------ + mesh(graph::shared_leaf xmesh, + graph::shared_leaf ymesh) : + x(xmesh), y(ymesh), + xmin(graph::variable_cast(xmesh)->data()[0]), + xmax(graph::variable_cast(xmesh)->data()[graph::variable_cast(xmesh)->size() - 1]), + dx(graph::variable_cast(xmesh)->data()[1] - + graph::variable_cast(xmesh)->data()[0]) {} + }; + +//------------------------------------------------------------------------------ +/// @brief Build Magnetic field. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] xp X position of the particles. +/// @returns The magnetic field expression. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_magnetic_field(graph::shared_leaf xp, + const T bchar) { + return (xp*xp + static_cast (1))/bchar; + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] mesh Mesh object. +/// @param[in] xp X position of the particles. //------------------------------------------------------------------------------ template - graph::shared_leaf build_interpolation(graph::shared_leaf xmesh, - graph::shared_leaf ymesh, - graph::shared_leaf xp, - const T xmin, - const T dx) { - auto x = graph::index_1D(xmesh, xp, dx, xmin) - xp; - auto xnorm1 = static_cast (1.5) + (x - dx)/dx; - auto xnorm2 = x/dx; - auto xnorm3 = static_cast (1.5) - (x + dx)/dx; + graph::shared_leaf build_interpolation(pic::mesh &mesh, + graph::shared_leaf xp) { + auto x = graph::index_1D(mesh.x, xp, mesh.dx, mesh.xmin) - xp; + auto xnorm1 = static_cast (1.5) + (x - mesh.dx)/mesh.dx; + auto xnorm2 = x/mesh.dx; + auto xnorm3 = static_cast (1.5) - (x + mesh.dx)/mesh.dx; auto w0 = static_cast (0.5)*xnorm1*xnorm1; auto w1 = static_cast (0.75) - xnorm2*xnorm2; auto w2 = static_cast (0.5)*xnorm3*xnorm3; - auto ymesh0 = graph::index_1D(ymesh, xp - dx, dx, xmin); - auto ymesh1 = graph::index_1D(ymesh, xp, dx, xmin); - auto ymesh2 = graph::index_1D(ymesh, xp + dx, dx, xmin); + auto ymesh0 = graph::index_1D(mesh.y, xp - mesh.dx, mesh.dx, mesh.xmin); + auto ymesh1 = graph::index_1D(mesh.y, xp, mesh.dx, mesh.xmin); + auto ymesh2 = graph::index_1D(mesh.y, xp + mesh.dx, mesh.dx, mesh.xmin); // Run only for unit tests. if constexpr (UNIT_TEST) { @@ -51,7 +132,7 @@ namespace pic { workflow::manager work(0); work.add_item({ - graph::variable_cast(xmesh), + graph::variable_cast(mesh.x), graph::variable_cast(xp) }, { weight @@ -76,6 +157,127 @@ namespace pic { return w0*ymesh0 + w1*ymesh1 + w2*ymesh2; } + +//------------------------------------------------------------------------------ +/// @brief Build F expressions. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] bchar Characteristic magnetic field. +/// @param[in] z Runga Kutta substep. +/// @param[in] dt Time step. +//------------------------------------------------------------------------------ + template + std::array, 3> build_F_expressions(ion &ion, + mesh &mesh, + const T bchar, + const std::array, 3> z, + const T dt) { + auto bfield = build_magnetic_field (z[0], bchar); + auto efield = build_interpolation (mesh, ion.x); + auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; + return { + z[1]*dt, + temp*dt, + (ion.charge/ion.mass*efield - temp)*dt + }; + } + +//------------------------------------------------------------------------------ +/// @brief Build Runga Kutta step update. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] bchar Characteristic magnetic field. +/// @returns Step update expressions for x, v_para, and x_perp. +//------------------------------------------------------------------------------ + template + std::array, 3> build_rk4_step(ion &ion, + mesh &mesh, + const T bchar, + const T dt) { +// Step 1 + std::array, 3> Z1{ion.x, ion.v_para, ion.v_perp}; + std::array, 3> dZ1(build_F_expressions (ion, mesh, bchar, Z1, dt)); + +// Step 2 + std::array, 3> Z2{ + Z1[0] + dZ1[0]/static_cast (2), + Z1[1] + dZ1[1]/static_cast (2), + Z1[2] + dZ1[2]/static_cast (2) + }; + std::array, 3> dZ2(build_F_expressions (ion, mesh, bchar, Z2, dt)); + +// Step 3 + std::array, 3> Z3{ + Z1[0] + dZ2[0]/static_cast (2), + Z1[1] + dZ2[1]/static_cast (2), + Z1[2] + dZ2[2]/static_cast (2) + }; + std::array, 3> dZ3(build_F_expressions (ion, mesh, bchar, Z3, dt)); + +// Step 4 + std::array, 3> Z4{ + Z1[0] + dZ3[0], + Z1[1] + dZ3[1], + Z1[2] + dZ3[2] + }; + std::array, 3> dZ4(build_F_expressions (ion, mesh, bchar, Z4, dt)); + +// Rk4 Solution + return { + Z1[0] + (dZ1[0] + static_cast (2)*(dZ2[0] + dZ3[0]) + dZ4[0])/static_cast (6), + Z1[1] + (dZ1[1] + static_cast (2)*(dZ2[1] + dZ3[1]) + dZ4[1])/static_cast (6), + Z1[2] + (dZ1[2] + static_cast (2)*(dZ2[2] + dZ3[2]) + dZ4[2])/static_cast (6) + }; + } + +//------------------------------------------------------------------------------ +/// @brief Build magnetic moment. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_magnetic_moment(ion &ion, mesh &mesh) { + auto efield = build_interpolation (mesh, ion.x); + return 0.5*ion.mass*ion.v_perp*ion.v_perp/efield; + } + +//------------------------------------------------------------------------------ +/// @brief Build +//------------------------------------------------------------------------------ + template + std::array,3> build_initialization(mesh &mesh, + graph::shared_random_state state) { + auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, + state); + auto phi_dist = graph::uniform_random (static_cast (0.0), + static_cast (2.0)*std::numbers::pi_v, + state); + auto r_dist = graph::uniform_random (static_cast (0.0), + static_cast (1.0), + state); +// FIXME: This should be in a separate file of physics constants. + const T kb = static_cast (1.380650E-23); + auto vpara = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::sin(phi_dist); + + phi_dist = graph::uniform_random (static_cast (0.0), + static_cast (2.0)*std::numbers::pi_v, + state); + r_dist = graph::uniform_random (static_cast (0.0), + static_cast (1.0), + state); + auto vperp1 = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::cos(phi_dist); + auto vperp2 = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::sin(phi_dist); + auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); + + return {position_dist, vpara, vperp}; + } } #endif /* particle_in_cell_h */ diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index c96f543..df1fe10 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -553,6 +553,25 @@ namespace graph { constexpr shared_leaf random_scale() { return constant (static_cast (std::numeric_limits::max())); } + +//------------------------------------------------------------------------------ +/// @brief Create a uniform random number. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] min Minimum value. +/// @param[in] max Maximum value. +/// @param[in] state Random state node. +/// @returns A uniform random constant. +//------------------------------------------------------------------------------ + template + constexpr shared_leaf uniform_random(const R min, + const R max, + shared_random_state state) { + auto random = graph::random (state); + return (max - min)/graph::random_scale ()*random + min; + } } #endif /* random_h */ diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index 96fdd2a..dedd7b9 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -247,8 +247,9 @@ namespace jit { const NODE *pointer) { assert((prefix == 'r' || prefix == 'v' || prefix == 'o' || prefix == 'a' || - prefix == 'i' || prefix == 's') && - "Expected a variable (v), register (r), output (o), array (a), index (i), or state (s) prefix."); + prefix == 'i' || prefix == 's' || + prefix == 'l') && + "Expected a variable (v), register (r), output (o), array (a), index (i), state (s), or logical (l) prefix."); return std::string(1, prefix) + format_to_string(reinterpret_cast (pointer)); } diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index bce50ca..0e0a64b 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -99,11 +99,17 @@ void run_pic() { // Electron initialization. auto te = graph::variable (num_grid, static_cast (1.0), "t_{e}"); -// Magnetic field. - auto bfield = (x*x + static_cast (1))/bchar; - // Electric field. auto efield = graph::variable (num_grid, "E_{||}"); + auto meshx = graph::variable (num_grid, "x_{m}"); + { + backend::buffer pos_buffer(num_grid); + const T dx = (0.25 - -0.25)/(num_particles - 1); + for (size_t i = 0; i < num_particles; i++) { + pos_buffer[i] = dx*i - 0.25; + } + meshx->set(pos_buffer); + } // Time step const T gyro_frequency = q*1*1.2/2; @@ -119,9 +125,11 @@ void run_pic() { vperp = vperp/c; x = x*wpechar/c; - bfield = bfield/bchar; - + pic::ion ion(q/qchar, m_hydrogen/mchar, x, vpara, vperp); + pic::mesh mesh(meshx, efield); + std::array, 3> rk4_step(pic::build_rk4_step(ion, mesh, bchar, dt)); + auto mu = pic::build_magnetic_moment (ion, mesh); // Build the field solver; diff --git a/graph_tests/CMakeLists.txt b/graph_tests/CMakeLists.txt index d3b85bc..0cbbca6 100644 --- a/graph_tests/CMakeLists.txt +++ b/graph_tests/CMakeLists.txt @@ -14,6 +14,8 @@ add_test_target (efit_test cpp) add_test_target (random_test cpp) add_test_target (workflow_test cpp) add_test_target (pic_test cpp) +add_compile_test (no_derivative_test cpp) +add_test_target (logical_test cpp) target_compile_definitions (erfi_test PRIVATE diff --git a/graph_tests/arithmetic_test.cpp b/graph_tests/arithmetic_test.cpp index 81cf2a5..f519894 100644 --- a/graph_tests/arithmetic_test.cpp +++ b/graph_tests/arithmetic_test.cpp @@ -3897,6 +3897,30 @@ template void test_fma() { */ } +//------------------------------------------------------------------------------ +/// @brief Tests for modulo nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_modulo() { + auto five = graph::constant(static_cast (5)); + auto four = graph::constant(static_cast (4)); + + auto result = five%four; + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant node."); + assert(result_cast->is(static_cast (1)) && "Expected 1"); + + auto x = graph::variable (1, ""); + auto result2 = five%x; + auto result2_cast = graph::modulo_cast(result2); + assert(result2_cast.get() && "Expected a variable node."); + + auto result3 = x%four; + auto result3_cast = graph::modulo_cast(result3); + assert(result3_cast.get() && "Expected a variable node."); +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified backend. /// @@ -3908,6 +3932,9 @@ template void run_tests() { test_multiply (); test_divide (); test_fma (); + if constexpr (std::floating_point) { + test_modulo (); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index 7b811b9..9a121a3 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -10,7 +10,7 @@ #include -#include "../graph_framework/dispersion.hpp" +#include "../graph_framework/graph_framework.hpp" //------------------------------------------------------------------------------ /// @brief Assert when difference is greater than the tolerance. @@ -344,6 +344,28 @@ template void run_math_tests() { graph::variable_cast(v1), graph::variable_cast(v2) }, {atan_node}, {}, atan_node->evaluate().at(0), result); + + if constexpr (std::floating_point) { + auto module_node = v1%v2; + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {module_node}, {}, module_node->evaluate().at(0), 0.0); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + auto if_node = graph::if_(v1 > v2, true_v, false_v); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {if_node}, {}, false_v->evaluate().at(0), 0.0); + + if_node = graph::if_(v1 < v2, true_v, false_v); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {if_node}, {}, true_v->evaluate().at(0), 0.0); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/logical_test.cpp b/graph_tests/logical_test.cpp new file mode 100644 index 0000000..4b4e465 --- /dev/null +++ b/graph_tests/logical_test.cpp @@ -0,0 +1,265 @@ +//------------------------------------------------------------------------------ +/// @file logical.cpp +/// @brief Tests for logic nodes. +//------------------------------------------------------------------------------ + +// Turn on asserts even in release builds. +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include + +#include "../graph_framework/graph_framework.hpp" + +//------------------------------------------------------------------------------ +/// @brief Tests for equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_not() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = !true_v; + assert(result1->is_match(false_v) && "Expected flase."); + auto result2 = !false_v; + assert(result2->is_match(true_v) && "Expected true."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto result3 = !(v1 == v2); + auto result3_cast = graph::not_equal_cast(result3); + assert(result3_cast.get() && "Expected a not equal node."); + + auto result4 = !(v1 != v2); + auto result4_cast = graph::equal_cast(result4); + assert(result4_cast.get() && "Expected an equal node."); + + auto result5 = !(v1 < v2); + auto result5_cast = graph::greater_than_equal_cast(result5); + assert(result5_cast.get() && "Expected a greater than equal node."); + + auto result6 = !(v1 > v2); + auto result6_cast = graph::less_than_equal_cast(result6); + assert(result6_cast.get() && "Expected a less than equal node."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v == true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = false_v == false_v; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = true_v == false_v; + assert(result3->is_match(false_v) && "Expected false."); + auto result4 = false_v == true_v; + assert(result4->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 == v2)->is_match(v2 == v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for not equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_not_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v != true_v; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = false_v != false_v; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = true_v != false_v; + assert(result3->is_match(true_v) && "Expected true."); + auto result4 = false_v != true_v; + assert(result4->is_match(true_v) && "Expected true."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 != v2)->is_match(v2 != v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for greater than nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_greater_than() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one > none; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = none > one; + assert(result2->is_match(false_v) && "Expected false."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for less than nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_less_than() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one < none; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = none < one; + assert(result2->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for greater than equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_greater_than_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one >= none; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = none >= one; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = one >= one; + assert(result3->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for less than equal nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_less_than_equal() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto one = graph::one (); + auto none = graph::none (); + + auto result1 = one <= none; + assert(result1->is_match(false_v) && "Expected false."); + auto result2 = none <= one; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = one <= one; + assert(result3->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for and nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_and() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v && true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = true_v && false_v; + assert(result2->is_match(false_v) && "Expected false."); + auto result3 = false_v && false_v; + assert(result3->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 && v2)->is_match(v2 && v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for or nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_or() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = true_v || true_v; + assert(result1->is_match(true_v) && "Expected true."); + auto result2 = true_v || true_v; + assert(result2->is_match(true_v) && "Expected true."); + auto result3 = false_v || false_v; + assert(result3->is_match(false_v) && "Expected false."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + assert((v1 || v2)->is_match(v2 || v1) && "Expected match."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for if nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_if() { + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + auto result1 = graph::if_(true_v, true_v, false_v); + assert(result1->is_match(true_v) && "Exected the true condition."); + auto result2 = graph::if_(false_v, true_v, false_v); + assert(result2->is_match(false_v) && "Exected the false condition."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto result = graph::if_(v1, v2, v2); + assert(result->is_match(v2)); + auto result_df = result->df(v1); + assert(result_df->is_match(false_v) && "Expected 0"); +} + +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified backend. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_tests() { + test_equal (); + test_not_equal (); + if constexpr (std::floating_point) { + test_not (); + test_greater_than (); + test_less_than (); + test_and (); + test_or (); + test_if (); + } +} + +//------------------------------------------------------------------------------ +/// @brief Main program of the test. +/// +/// @param[in] argc Number of commandline arguments. +/// @param[in] argv Array of commandline arguments. +//------------------------------------------------------------------------------ +int main(int argc, const char * argv[]) { + (void)argc; + (void)argv; + run_tests (); + run_tests (); + run_tests> (); + run_tests> (); +} diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp new file mode 100644 index 0000000..9902883 --- /dev/null +++ b/graph_tests/no_derivative_test.cpp @@ -0,0 +1,52 @@ +//------------------------------------------------------------------------------ +/// @file no_derivative_test.cpp +/// @brief Test for nodes with no derivatives. +//------------------------------------------------------------------------------ + +#include "../graph_framework/node.hpp" + +//------------------------------------------------------------------------------ +/// @brief Dummy node. +//------------------------------------------------------------------------------ +class dummy : public graph::no_derivative> { +public: + dummy() : graph::no_derivative> ("") {} + + virtual backend::buffer evaluate() { + return backend::buffer (); + }; + + virtual graph::shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + return this->shared_from_this(); + } + + virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + return this->shared_from_this(); + } + + virtual bool is_all_variables() const { + return false; + } + + virtual graph::shared_leaf get_power_exponent() const { + return graph::one (); + } +}; + +//------------------------------------------------------------------------------ +/// @brief Test function. +/// +/// This test checks for a failure to compiler if a df method is called on a +/// node without a derivative. +//------------------------------------------------------------------------------ +void test() { + dummy a; +#ifndef CHECK_TEST + a.df(a); +#endif +} diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 8e4a520..2c80ae8 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -37,18 +37,20 @@ template void run_interpolation_test() { graph::variable_cast(ymesh)->data()[i] = func(graph::variable_cast(xmesh)->data()[i]); } + pic::mesh efield_mesh(xmesh, ymesh); + auto xp = graph::variable (num_particles, "xp"); const T dxp = (xmax - xmin)/(num_particles - 1); for (size_t i = 0; i < num_particles; i++) { graph::variable_cast(xp)->data()[i] = dxp*i + xmin; } - auto field = pic::build_interpolation (xmesh, ymesh, xp, xmin, dx); + auto field = pic::build_interpolation (efield_mesh, xp); workflow::manager work(0); work.add_item({ - graph::variable_cast(xmesh), - graph::variable_cast(ymesh), + graph::variable_cast(efield_mesh.x), + graph::variable_cast(efield_mesh.y), graph::variable_cast(xp) }, { field From 03416e169871b50b4edb1a3fd56b2724ee45451c Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 25 Jun 2026 14:42:15 -0400 Subject: [PATCH 05/51] Refactor simplify field solve. Among the changes include. 1. Add an index node constant. This node allows access to the current kernel index. 2. Correct if node code generation and use ? : to allow the register to be const. 3. Refactor workflows and gpu contexts to add a memzero work item and fold the loop interation into the kernel itself. 4. Refactor backend buffers to inherit from std::vector directly. This eliminates a low of code. 5. Refactor copy and passed backend buffer code to use macro functions. 6. Add function to generate and organize graphs for PIC codes. 7. Add unit test for PIC field solver. This checkes the particle counts aganist a standard historgram. 8. Disable PIC driver to avoid compilation error since the API is in flux. --- CMakeLists.txt | 1 + graph_docs/use_cases.dox | 2 +- graph_framework.xcodeproj/project.pbxproj | 2 +- .../xcschemes/arithmetic_test.xcscheme | 2 +- .../xcschemes/graph_driver.xcscheme | 2 +- .../xcshareddata/xcschemes/graph_pic.xcscheme | 2 +- .../xcshareddata/xcschemes/jit_test.xcscheme | 2 +- .../xcshareddata/xcschemes/math_test.xcscheme | 2 +- .../xcschemes/physics_test.xcscheme | 2 +- graph_framework/backend.hpp | 886 ++++++------------ graph_framework/cpu_context.hpp | 50 +- graph_framework/cuda_context.hpp | 57 +- graph_framework/jit.hpp | 111 ++- graph_framework/logical.hpp | 18 +- graph_framework/metal_context.hpp | 47 +- graph_framework/node.hpp | 170 +++- graph_framework/particle_in_cell.hpp | 386 ++++++-- graph_framework/workflow.hpp | 147 ++- graph_pic/xpic.cpp | 24 +- graph_tests/jit_test.cpp | 7 + graph_tests/node_test.cpp | 21 + graph_tests/pic_test.cpp | 184 +++- graph_tests/workflow_test.cpp | 28 + 23 files changed, 1347 insertions(+), 806 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c42789..8baf9e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,6 +394,7 @@ macro (add_test_target target lang) add_test (NAME ${target} COMMAND ${target} ) + if (${USE_PCH}) if (${BUILD_C_BINDING}) diff --git a/graph_docs/use_cases.dox b/graph_docs/use_cases.dox index f47fa27..1025793 100644 --- a/graph_docs/use_cases.dox +++ b/graph_docs/use_cases.dox @@ -125,7 +125,7 @@ * branch is absorbed in the upper hybrid resonance, @f$\omega_{h}@f$, while the * O-Mode branch can pass through it. * - * @subsubsection use_cases_rf_correctness Comparison to GENRAY + * @subsubsection use_cases_rf_correctness_genray Comparison to GENRAY * Genray is an RF-Ray tracing * code written in Fortran which operates in a cylindrical geometry. Toroidal * equilibria can be imported using the diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index dd2f561..fbfad82 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -1372,7 +1372,7 @@ isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = YES; - LastUpgradeCheck = 2610; + LastUpgradeCheck = 2640; ORGANIZATIONNAME = "Cianciosa, Mark R."; TargetAttributes = { C715C7932FD09E29003EEFF4 = { diff --git a/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme b/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme index 1688675..b5495d8 100644 --- a/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme +++ b/graph_framework.xcodeproj/xcshareddata/xcschemes/arithmetic_test.xcscheme @@ -1,6 +1,6 @@ - class buffer { - private: -/// The data buffer to hold the data. - std::vector memory; - + class buffer : public std::vector { public: + using std::vector::size; + using std::vector::data; + using std::vector::assign; + //------------------------------------------------------------------------------ /// @brief Construct an empty buffer backend. //------------------------------------------------------------------------------ buffer() : - memory() {} + std::vector () {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend with a size. @@ -44,7 +44,7 @@ namespace backend { /// @param[in] s Size of he data buffer. //------------------------------------------------------------------------------ buffer(const size_t s) : - memory(s) {} + std::vector (s) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend with a size. @@ -53,7 +53,7 @@ namespace backend { /// @param[in] d Scalar data to initialize. //------------------------------------------------------------------------------ buffer(const size_t s, const T d) : - memory(s, d) {} + std::vector (s, d) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend from a vector. @@ -61,7 +61,7 @@ namespace backend { /// @param[in] d Array buffer. //------------------------------------------------------------------------------ buffer(const std::vector &d) : - memory(d) {} + std::vector (d) {} //------------------------------------------------------------------------------ /// @brief Construct a buffer backend from a buffer backend. @@ -69,27 +69,19 @@ namespace backend { /// @param[in] d Backend buffer. //------------------------------------------------------------------------------ buffer(const buffer &d) : - memory(d.memory) {} - -//------------------------------------------------------------------------------ -/// @brief Index operator. -//------------------------------------------------------------------------------ - T &operator[] (const size_t index) { - return memory[index]; - } - -//------------------------------------------------------------------------------ -/// @brief Const index operator. -//------------------------------------------------------------------------------ - const T &operator[] (const size_t index) const { - return memory[index]; - } + std::vector (d) {} //------------------------------------------------------------------------------ -/// @brief Get value at. +/// @brief Construct a buffer backend linearly. +/// +/// @param[in] min Minimum value.. +/// @param[in] dx Step size. +/// @param[in] num Number of mesh points. //------------------------------------------------------------------------------ - const T at(const size_t index) const { - return memory.at(index); + buffer(const T min, const T dx, const size_t num) : std::vector (num) { + for (size_t i = 0; i < num; i++) { + (*this)[i] = dx*i + min; + } } //------------------------------------------------------------------------------ @@ -98,7 +90,7 @@ namespace backend { /// @param[in] d Scalar data to set. //------------------------------------------------------------------------------ void set(const T d) { - memory.assign(memory.size(), d); + assign(size(), d); } //------------------------------------------------------------------------------ @@ -107,14 +99,7 @@ namespace backend { /// @param[in] d Vector data to set. //------------------------------------------------------------------------------ void set(const std::vector &d) { - memory.assign(d.cbegin(), d.cend()); - } - -//------------------------------------------------------------------------------ -/// @brief Get size of the buffer. -//------------------------------------------------------------------------------ - size_t size() const { - return memory.size(); + assign(d.cbegin(), d.cend()); } //------------------------------------------------------------------------------ @@ -123,9 +108,9 @@ namespace backend { /// @returns Returns true if every element is the same. //------------------------------------------------------------------------------ bool is_same() const { - const T same = memory.at(0); - for (size_t i = 1, ie = memory.size(); i < ie; i++) { - if (memory.at(i) != same) { + const T same = (*this)[0]; + for (size_t i = 1, ie = size(); i < ie; i++) { + if ((*this)[i] != same) { return false; } } @@ -139,7 +124,7 @@ namespace backend { /// @returns Returns true if every element is zero. //------------------------------------------------------------------------------ bool is_zero() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d != static_cast (0.0)) { return false; } @@ -154,7 +139,7 @@ namespace backend { /// @returns Returns true if any element is zero. //------------------------------------------------------------------------------ bool has_zero() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d == static_cast (0.0)) { return true; } @@ -169,7 +154,7 @@ namespace backend { /// @returns Returns true if every element is negative. //------------------------------------------------------------------------------ bool is_negative() const { - for (const T &d : memory) { + for (const T &d : *this) { if (std::real(d) > std::real(static_cast (0.0))) { return false; } @@ -184,7 +169,7 @@ namespace backend { /// @returns Returns true if every element is negative. //------------------------------------------------------------------------------ bool is_even() const { - for (const T &d : memory) { + for (const T &d : *this) { if (std::fmod(std::real(d), std::real(static_cast (2.0)))) { return false; } @@ -199,7 +184,7 @@ namespace backend { /// @returns Returns true if every element is negative one. //------------------------------------------------------------------------------ bool is_none() const { - for (const T &d : memory) { + for (const T &d : *this) { if (d != static_cast (-1.0)) { return false; } @@ -208,67 +193,56 @@ namespace backend { return true; } +//------------------------------------------------------------------------------ +/// @brief Applies an operation over all elements in the buffer. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define apply_op(op) \ +for (T &d : *this) { \ + d = op(d); \ +} + //------------------------------------------------------------------------------ /// @brief Take sqrt. //------------------------------------------------------------------------------ void sqrt() { - for (T &d : memory) { - d = std::sqrt(d); - } + apply_op(std::sqrt) } //------------------------------------------------------------------------------ /// @brief Take exp. //------------------------------------------------------------------------------ void exp() { - for (T &d : memory) { - d = std::exp(d); - } + apply_op(std::exp) } //------------------------------------------------------------------------------ /// @brief Take log. //------------------------------------------------------------------------------ void log() { - for (T &d : memory) { - d = std::log(d); - } + apply_op(std::log) } //------------------------------------------------------------------------------ /// @brief Take sin. //------------------------------------------------------------------------------ void sin() { - for (T &d : memory) { - d = std::sin(d); - } + apply_op(std::sin) } //------------------------------------------------------------------------------ /// @brief Take cos. //------------------------------------------------------------------------------ void cos() { - for (T &d : memory) { - d = std::cos(d); - } + apply_op(std::cos) } //------------------------------------------------------------------------------ /// @brief Take erfi. //------------------------------------------------------------------------------ void erfi() requires(jit::complex_scalar) { - for (T &d : memory) { - d = special::erfi(d); - } - } - -//------------------------------------------------------------------------------ -/// @brief Get a pointer to the basic memory buffer. -/// -/// @returns The pointer to the buffer memory. -//------------------------------------------------------------------------------ - T *data() { - return memory.data(); + apply_op(special::erfi) } //------------------------------------------------------------------------------ @@ -277,7 +251,7 @@ namespace backend { /// @returns False if any NaN or Inf is found. //------------------------------------------------------------------------------ bool is_normal() const { - for (const T &x : memory) { + for (const T &x : *this) { if constexpr (jit::complex_scalar) { if (std::isnan(std::real(x)) || std::isinf(std::real(x)) || std::isnan(std::imag(x)) || std::isinf(std::imag(x))) { @@ -303,7 +277,7 @@ namespace backend { buffer b(num_columns); const size_t num_rows = size()/num_columns; for (size_t j = 0; j < num_columns; j++) { - b[j] = memory[index*num_rows + j]; + b[j] = (*this)[index*num_rows + j]; } return b; } @@ -319,11 +293,44 @@ namespace backend { const size_t num_rows = size()/num_columns; buffer b(num_rows); for (size_t i = 0; i < num_rows; i++) { - b[i] = memory[i*num_rows + index]; + b[i] = (*this)[i*num_rows + index]; } return b; } +//------------------------------------------------------------------------------ +/// @brief Applies an operatator along a row. +/// +/// @param opp The operation to apply. +/// @param oppeq The assignment operator to apply. +//------------------------------------------------------------------------------ +#define row_op(opp, oppeq) \ +if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] oppeq x[i]; \ + } \ + } \ +} else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = (*this)[i] opp x[i*num_columns + j]; \ + } \ + } \ + *this = m; \ +} + //------------------------------------------------------------------------------ /// @brief Add row operation. /// @@ -333,32 +340,41 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void add_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_rows + j] += x[i]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i] + x[i*num_columns + j]; - } - } - memory = m; - } - } + row_op(+, +=) + } + +//------------------------------------------------------------------------------ +/// @brief Applies an operatator along a column. +/// +/// @param opp The operation to apply. +/// @param oppeq The assignment operator to apply. +//------------------------------------------------------------------------------ +#define col_op(opp, oppeq) \ +if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] oppeq x[j]; \ + } \ + } \ +} else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = (*this)[j] opp x[i*num_columns + j]; \ + } \ + } \ + *this = m; \ +} //------------------------------------------------------------------------------ /// @brief Add col operation. @@ -369,31 +385,7 @@ namespace backend { /// @param[in] x The other operand. //------------------------------------------------------------------------------ void add_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] += x[j]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j] + x[i*num_columns + j]; - } - } - memory = m; - } + col_op(+, +=) } //------------------------------------------------------------------------------ @@ -405,31 +397,7 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void subtract_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] -= x[i]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_columns; i++) { - for (size_t j = 0; j < num_rows; j++) { - m[i*num_columns + j] = memory[i] - x[i*num_columns + j]; - } - } - memory = m; - } + row_op(-, -=) } //------------------------------------------------------------------------------ @@ -441,31 +409,7 @@ namespace backend { /// @param[in] x The other operand. //------------------------------------------------------------------------------ void subtract_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] -= x[j]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j] - x[i*num_columns + j]; - } - } - memory = m; - } + col_op(-, -=) } //------------------------------------------------------------------------------ @@ -477,31 +421,7 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void multiply_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] *= x[i]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i]*x[i*num_columns + j]; - } - } - memory = m; - } + row_op(*, *=) } //------------------------------------------------------------------------------ @@ -513,31 +433,7 @@ namespace backend { /// @param[in] x The other operand. //------------------------------------------------------------------------------ void multiply_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] *= x[j]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j]*x[i*num_columns + j]; - } - } - memory = m; - } + col_op(*, *=) } //------------------------------------------------------------------------------ @@ -549,31 +445,7 @@ namespace backend { /// @param[in] x The right operand. //------------------------------------------------------------------------------ void divide_row(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] /= x[i]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[i]/x[i*num_columns + j]; - } - } - memory = m; - } + row_op(/, /=) } //------------------------------------------------------------------------------ @@ -585,31 +457,7 @@ namespace backend { /// @param[in] x The other operand. //------------------------------------------------------------------------------ void divide_col(const buffer &x) { - if (size() > x.size()) { - assert(size()%x.size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - const size_t num_columns = size()/x.size(); - const size_t num_rows = x.size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] /= x[j]; - } - } - } else { - assert(x.size()%size() == 0 && - "Vector operand size is not a multiple of matrix operand size"); - - std::vector m(x.size()); - const size_t num_columns = x.size()/size(); - const size_t num_rows = size(); - for (size_t i = 0; i < num_rows; i++) { - for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = memory[j]/x[i*num_columns + j]; - } - } - memory = m; - } + col_op(/, /=) } //------------------------------------------------------------------------------ @@ -630,9 +478,9 @@ namespace backend { for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { if constexpr (jit::complex_scalar) { - memory[i*num_columns + j] = std::atan(x[i]/memory[i*num_columns + j]); + (*this)[i*num_columns + j] = std::atan(x[i]/(*this)[i*num_columns + j]); } else { - memory[i*num_columns + j] = std::atan2(x[i], memory[i*num_columns + j]); + (*this)[i*num_columns + j] = std::atan2(x[i], (*this)[i*num_columns + j]); } } } @@ -646,13 +494,13 @@ namespace backend { for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { if constexpr (jit::complex_scalar) { - m[i*num_columns + j] = std::atan(x[i*num_columns + j]/memory[i]); + m[i*num_columns + j] = std::atan(x[i*num_columns + j]/(*this)[i]); } else { - m[i*num_columns + j] = std::atan2(x[i*num_columns + j], memory[i]); + m[i*num_columns + j] = std::atan2(x[i*num_columns + j], (*this)[i]); } } } - memory = m; + *this = m; } } @@ -674,9 +522,9 @@ namespace backend { for (size_t i = 0; i < num_columns; i++) { for (size_t j = 0; j < num_rows; j++) { if constexpr (jit::complex_scalar) { - memory[i*num_columns + j] = std::atan(x[j]/memory[i*num_columns + j]); + (*this)[i*num_columns + j] = std::atan(x[j]/(*this)[i*num_columns + j]); } else { - memory[i*num_columns + j] = std::atan2(x[j], memory[i*num_columns + j]); + (*this)[i*num_columns + j] = std::atan2(x[j], (*this)[i*num_columns + j]); } } } @@ -690,13 +538,13 @@ namespace backend { for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { if constexpr (jit::complex_scalar) { - m[i*num_columns + j] = std::atan(x[i*num_columns + j]/memory[j]); + m[i*num_columns + j] = std::atan(x[i*num_columns + j]/(*this)[j]); } else { - m[i*num_columns + j] = std::atan2(x[i*num_columns + j], memory[j]); + m[i*num_columns + j] = std::atan2(x[i*num_columns + j], (*this)[j]); } } } - memory = m; + *this = m; } } @@ -717,7 +565,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] = std::pow(memory[i*num_columns + j], x[i]); + (*this)[i*num_columns + j] = std::pow((*this)[i*num_columns + j], x[i]); } } } else { @@ -729,10 +577,10 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_columns; i++) { for (size_t j = 0; j < num_rows; j++) { - m[i*num_columns + j] = std::pow(memory[i], x[i*num_columns + j]); + m[i*num_columns + j] = std::pow((*this)[i], x[i*num_columns + j]); } } - memory = m; + *this = m; } } @@ -753,7 +601,7 @@ namespace backend { const size_t num_rows = x.size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - memory[i*num_columns + j] = std::pow(memory[i*num_columns + j], x[j]); + (*this)[i*num_columns + j] = std::pow((*this)[i*num_columns + j], x[j]); } } } else { @@ -765,10 +613,10 @@ namespace backend { const size_t num_rows = size(); for (size_t i = 0; i < num_rows; i++) { for (size_t j = 0; j < num_columns; j++) { - m[i*num_columns + j] = std::pow(memory[j], x[i*num_columns + j]); + m[i*num_columns + j] = std::pow((*this)[j], x[i*num_columns + j]); } } - memory = m; + *this = m; } } @@ -778,10 +626,10 @@ namespace backend { /// @returns The negation of the buffer. //------------------------------------------------------------------------------ buffer operator!() requires(std::floating_point) { - for (size_t i = 0, ie = memory.size(); i < ie; i++) { - memory[i] = !memory[i]; + for (T &d : *this) { + d = !d; } - return memory; + return *this; } //------------------------------------------------------------------------------ @@ -793,34 +641,34 @@ namespace backend { buffer if_(const buffer &t, const buffer &f) { if (size() == 1) { - return memory[0] ? t : f; + return (*this)[0] ? t : f; } else { if (t.size() == 1) { if (f.size() == 1) { - for (size_t i = 0, ie = size(); i < ie; i++) { - memory[i] = memory[i] ? t.at(0) : f.at(0); + for (T &d : *this) { + d = d ? t[0] : f[0]; } - return memory; + return *this; } else { assert(size() == f.size() && "Incompatable buffersize."); for (size_t i = 0, ie = size(); i < ie; i++) { - memory[i] = memory[i] ? t.at(0) : f[i]; + (*this)[i] = (*this)[i] ? t[0] : f[i]; } - return memory; + return *this; } } else { assert(size() == t.size() && "Incompatable buffersize."); if (f.size() == 1) { for (size_t i = 0, ie = size(); i < ie; i++) { - memory[i] = memory[i] ? t[i] : f.at(0); + (*this)[i] = (*this)[i] ? t[i] : f[0]; } - return memory; + return *this; } else { assert(size() == f.size() && "Incompatable buffersize."); for (size_t i = 0, ie = size(); i < ie; i++) { - memory[i] = memory[i] ? t[i] : f[i]; + (*this)[i] = (*this)[i] ? t[i] : f[i]; } - return memory; + return *this; } } } @@ -831,63 +679,99 @@ namespace backend { }; //------------------------------------------------------------------------------ -/// @brief Add operation. +/// @brief Equal operation. /// /// @tparam T Base type of the calculation. /// /// @param[in] a Left operand. /// @param[in] b Right operand. -/// @returns a + b. +/// @returns a == b. //------------------------------------------------------------------------------ template - inline buffer operator+(buffer &a, - buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] += right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] += left; - } - return b; + inline bool operator==(const buffer &a, + const buffer &b) { + if (a.size() != b.size()) { + return false; } - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] += b.at(i); + if (a[i] != b[i]) { + return false; + } } - return a; + return true; } //------------------------------------------------------------------------------ -/// @brief Equal operation. +/// @brief Applies an associative operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define build_assoc_op(op) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l op right; \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r op left; \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] op b[i]; \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Add operation. /// /// @tparam T Base type of the calculation. /// /// @param[in] a Left operand. /// @param[in] b Right operand. -/// @returns a == b. +/// @returns a + b. //------------------------------------------------------------------------------ template - inline bool operator==(const buffer &a, - const buffer &b) { - if (a.size() != b.size()) { - return false; - } - - for (size_t i = 0, ie = a.size(); i < ie; i++) { - if (a.at(i) != b.at(i)) { - return false; - } - } - return true; + inline buffer operator+(buffer &a, + buffer &b) { + build_assoc_op(+=) } +//------------------------------------------------------------------------------ +/// @brief Applies a non-associative operator. +/// +/// @param op The operation to apply. +/// @param opeq The assign operation to apply. +//------------------------------------------------------------------------------ +#define build_non_assoc_op(op, opeq) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l opeq right; \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = left op r; \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] opeq b[i]; \ +} \ +return a; + //------------------------------------------------------------------------------ /// @brief Subtract operation. /// @@ -900,26 +784,7 @@ namespace backend { template inline buffer operator-(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] -= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = left - b.at(i); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] -= b.at(i); - } - return a; + build_non_assoc_op(-, -=) } //------------------------------------------------------------------------------ @@ -934,26 +799,7 @@ namespace backend { template inline buffer operator*(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] *= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] *= left; - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] *= b.at(i); - } - return a; + build_assoc_op(*=) } //------------------------------------------------------------------------------ @@ -968,26 +814,7 @@ namespace backend { template inline buffer operator/(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] /= right; - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = left/b.at(i); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] /= b.at(i); - } - return a; + build_non_assoc_op(/, /=) } //------------------------------------------------------------------------------ @@ -1012,25 +839,25 @@ namespace backend { #endif if (a.size() == 1) { - const T left = a.at(0); + const T left = a[0]; if (b.size() == 1) { - const T middle = b.at(0); + const T middle = b[0]; for (size_t i = 0, ie = c.size(); i < ie; i++) { if constexpr (use_fma) { - c[i] = std::fma(left, middle, c.at(i)); + c[i] = std::fma(left, middle, c[i]); } else { - c[i] = left*middle + c.at(i); + c[i] = left*middle + c[i]; } } return c; } else if (c.size() == 1) { - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = b.size(); i < ie; i++) { if constexpr (use_fma) { - b[i] = std::fma(left, b.at(i), right); + b[i] = std::fma(left, b[i], right); } else { - b[i] = left*b.at(i) + right; + b[i] = left*b[i] + right; } } return b; @@ -1040,21 +867,21 @@ namespace backend { "Size mismatch between middle and right."); for (size_t i = 0, ie = b.size(); i < ie; i++) { if constexpr (use_fma) { - b[i] = std::fma(left, b.at(i), c.at(i)); + b[i] = std::fma(left, b[i], c[i]); } else { - b[i] = left*b.at(i) + c.at(i); + b[i] = left*b[i] + c[i]; } } return b; } else if (b.size() == 1) { - const T middle = b.at(0); + const T middle = b[0]; if (c.size() == 1) { - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), middle, right); + a[i] = std::fma(a[i], middle, right); } else { - a[i] = a.at(i)*middle + right; + a[i] = a[i]*middle + right; } } return a; @@ -1064,21 +891,21 @@ namespace backend { "Size mismatch between left and right."); for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), middle, c.at(i)); + a[i] = std::fma(a[i], middle, c[i]); } else { - a[i] = a.at(i)*middle + c.at(i); + a[i] = a[i]*middle + c[i]; } } return a; } else if (c.size() == 1) { assert(a.size() == b.size() && "Size mismatch between left and middle."); - const T right = c.at(0); + const T right = c[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), b.at(i), right); + a[i] = std::fma(a[i], b[i], right); } else { - a[i] = a.at(i)*b.at(i) + right; + a[i] = a[i]*b[i] + right; } } return a; @@ -1090,9 +917,9 @@ namespace backend { "Left, middle and right sizes are incompatible."); for (size_t i = 0, ie = a.size(); i < ie; i++) { if constexpr (use_fma) { - a[i] = std::fma(a.at(i), b.at(i), c.at(i)); + a[i] = std::fma(a[i], b[i], c[i]); } else { - a[i] = a.at(i)*b.at(i) + c.at(i); + a[i] = a[i]*b[i] + c[i]; } } return a; @@ -1111,15 +938,15 @@ namespace backend { inline buffer operator%(buffer &a, buffer &b) { if (b.size() == 1) { - const T right = b.at(0); + const T right = b[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { a[i] = std::fmod(a[i], right); } return a; } else if (a.size() == 1) { - const T left = a.at(0); + const T left = a[0]; for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = std::fmod(left, b.at(i)); + b[i] = std::fmod(left, b[i]); } return b; } @@ -1127,11 +954,38 @@ namespace backend { assert(a.size() == b.size() && "Left and right sizes are incompatible."); for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = std::fmod(a[i], b.at(i)); + a[i] = std::fmod(a[i], b[i]); } return a; } +//------------------------------------------------------------------------------ +/// @brief Applies a logical operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define logic_op(op) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = static_cast (a[i] op right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (size_t i = 0, ie = b.size(); i < ie; i++) { \ + b[i] = static_cast (left op b[i]); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = static_cast (a[i] op b[i]); \ +} \ +return a; + //------------------------------------------------------------------------------ /// @brief Equal operation. /// @@ -1144,26 +998,7 @@ namespace backend { template inline buffer operator==(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] == right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left == b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] == b.at(i)); - } - return a; + logic_op(==) } //------------------------------------------------------------------------------ @@ -1178,26 +1013,7 @@ namespace backend { template inline buffer operator!=(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] != right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left != b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] != b.at(i)); - } - return a; + logic_op(!=) } //------------------------------------------------------------------------------ @@ -1212,26 +1028,7 @@ namespace backend { template inline buffer operator>(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] > right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left > b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] > b.at(i)); - } - return a; + logic_op(>) } //------------------------------------------------------------------------------ @@ -1246,26 +1043,7 @@ namespace backend { template inline buffer operator<(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] < right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left < b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] < b.at(i)); - } - return a; + logic_op(<) } //------------------------------------------------------------------------------ @@ -1280,26 +1058,7 @@ namespace backend { template inline buffer operator>=(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] >= right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left >= b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] >= b.at(i)); - } - return a; + logic_op(>=) } //------------------------------------------------------------------------------ @@ -1314,26 +1073,7 @@ namespace backend { template inline buffer operator<=(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] <= right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left <= b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] <= b.at(i)); - } - return a; + logic_op(<=) } //------------------------------------------------------------------------------ @@ -1348,26 +1088,7 @@ namespace backend { template inline buffer operator&&(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] && right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left && b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] && b.at(i)); - } - return a; + logic_op(&&) } //------------------------------------------------------------------------------ @@ -1382,26 +1103,7 @@ namespace backend { template inline buffer operator||(buffer &a, buffer &b) { - if (b.size() == 1) { - const T right = b.at(0); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] || right); - } - return a; - } else if (a.size() == 1) { - const T left = a.at(0); - for (size_t i = 0, ie = b.size(); i < ie; i++) { - b[i] = static_cast (left || b.at(i)); - } - return b; - } - - assert(a.size() == b.size() && - "Left and right sizes are incompatible."); - for (size_t i = 0, ie = a.size(); i < ie; i++) { - a[i] = static_cast (a[i] || b.at(i)); - } - return a; + logic_op(||) } //------------------------------------------------------------------------------ @@ -1417,7 +1119,7 @@ namespace backend { inline buffer pow(buffer &base, buffer &exponent) { if (exponent.size() == 1) { - const T right = exponent.at(0); + const T right = exponent[0]; if (std::imag(right) == 0) { const int64_t right_int = static_cast (std::real(right)); if (std::real(right) - right_int) { @@ -1427,14 +1129,14 @@ namespace backend { } for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), right); + base[i] = std::pow(base[i], right); } return base; } if (right_int > 0) { for (size_t i = 0, ie = base.size(); i < ie; i++) { - const T left = base.at(i); + const T left = base[i]; for (size_t j = 0, je = right_int - 1; j < je; j++) { base[i] *= left; } @@ -1447,7 +1149,7 @@ namespace backend { return base; } else { for (size_t i = 0, ie = base.size(); i < ie; i++) { - const T left = static_cast (1.0)/base.at(i); + const T left = static_cast (1.0)/base[i]; base[i] = left; for (size_t j = 0, je = std::abs(right_int) - 1; j < je; j++) { base[i] *= left; @@ -1457,14 +1159,14 @@ namespace backend { } } else { for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), right); + base[i] = std::pow(base[i], right); } return base; } } else if (base.size() == 1) { - const T left = base.at(0); + const T left = base[0]; for (size_t i = 0, ie = exponent.size(); i < ie; i++) { - exponent[i] = std::pow(left, exponent.at(i)); + exponent[i] = std::pow(left, exponent[i]); } return exponent; } @@ -1472,7 +1174,7 @@ namespace backend { assert(base.size() == exponent.size() && "Left and right sizes are incompatible."); for (size_t i = 0, ie = base.size(); i < ie; i++) { - base[i] = std::pow(base.at(i), exponent.at(i)); + base[i] = std::pow(base[i], exponent[i]); } return base; } @@ -1490,7 +1192,7 @@ namespace backend { inline buffer atan(buffer &x, buffer &y) { if (y.size() == 1) { - const T right = y.at(0); + const T right = y[0]; for (size_t i = 0, ie = x.size(); i < ie; i++) { if constexpr (jit::complex_scalar) { x[i] = std::atan(right/x[i]); @@ -1500,7 +1202,7 @@ namespace backend { } return x; } else if (x.size() == 1) { - const T left = x.at(0); + const T left = x[0]; for (size_t i = 0, ie = y.size(); i < ie; i++) { if constexpr (jit::complex_scalar) { y[i] = std::atan(y[i]/left); diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index d0b48db..774a073 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -243,9 +243,8 @@ namespace gpu { for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { - backend::buffer buffer = input->evaluate(); - std::vector arg(buffer.size()); - memcpy(arg.data(), buffer.data(), buffer.size()*sizeof(T)); + std::vector arg(input->size()); + memcpy(arg.data(), input->data(), input->size()*sizeof(T)); kernel_arguments[input.get()] = arg; } buffers[reinterpret_cast (input.get())] = kernel_arguments[input.get()].data(); @@ -302,6 +301,7 @@ namespace gpu { /// /// @param[in] argument Node to reduce. /// @param[in] run Function to run before reduction. +/// @returns A lambda function to run the kernel. //------------------------------------------------------------------------------ std::function create_max_call(graph::shared_leaf &argument, std::function run) { @@ -321,6 +321,33 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create buffer that will be memset to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector buffers; + std::vector sizes; + + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + std::vector arg(input->size()); + memcpy(arg.data(), input->data(), input->size()*sizeof(T)); + kernel_arguments[input.get()] = arg; + } + buffers.push_back(kernel_arguments[input.get()].data()); + sizes.push_back(input->size()*sizeof(T)); + } + + return [buffers, sizes] () mutable { + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + std::memset(buffers[i], 0, sizes[i]); + } + }; + } + //------------------------------------------------------------------------------ /// @brief Hold the current thread until the command buffer has completed. /// @@ -424,6 +451,7 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, @@ -435,7 +463,8 @@ namespace gpu { jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" void " << name << "(" << std::endl; @@ -485,6 +514,9 @@ namespace gpu { << std::endl; } source_buffer << " for (size_t i = 0; i < " << size << "; i++) {" << std::endl; + if (iterations > 1) { + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } for (auto &input : inputs) { registers[input.get()] = jit::to_string('r', input.get()); @@ -510,6 +542,7 @@ namespace gpu { /// @param[in,out] registers Map of used registers. /// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, @@ -517,7 +550,8 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, jit::register_map &indices, - const jit::register_usage &usage) { + const jit::register_usage &usage, + const size_t iterations=1) { std::unordered_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { @@ -579,8 +613,10 @@ namespace gpu { } } - source_buffer << " }" << std::endl; - source_buffer << "}" << std::endl; + if (iterations > 1) { + source_buffer << " }" << std::endl; + } + source_buffer << " }" << std::endl << "}" << std::endl; } //------------------------------------------------------------------------------ diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index d03b9f4..ef38317 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -330,14 +330,13 @@ namespace gpu { for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { kernel_arguments.try_emplace(input.get()); - const backend::buffer backend = input->evaluate(); check_error(cuMemAllocManaged(&kernel_arguments[input.get()], - backend.size()*sizeof(T), + input->size()*sizeof(T), CU_MEM_ATTACH_GLOBAL), "cuMemAllocManaged"); check_error(cuMemcpyHtoD(kernel_arguments[input.get()], - &backend[0], - backend.size()*sizeof(T)), + input->data(), + input->size()*sizeof(T)), "cuMemcpyHtoD"); buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); needed_buffers.insert(input.get()); @@ -575,6 +574,41 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create buffer that will be memset to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector buffers; + std::vector sizes; + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + kernel_arguments.try_emplace(input.get()); + check_error(cuMemAllocManaged(&kernel_arguments[input.get()], + input->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[input.get()], + input->data(), + input->size()*sizeof(T)), + "cuMemcpyHtoD"); + buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); + } + buffers.push_back(kernel_arguments[input.get()]); + sizes.push_back(inputs->size()*sizeof(T)); + } + + return [this, buffers, sizes] () mutable { + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + check_error_async(cuMemsetD8Async(buffers[i], 0, + size[i], stream), + "cuMemsetD8Async"); + } + }; + } + //------------------------------------------------------------------------------ /// @brief Hold the current thread until the stream has completed. //------------------------------------------------------------------------------ @@ -709,6 +743,7 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, @@ -720,7 +755,8 @@ namespace gpu { jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" __global__ void " << name << "(" << std::endl; @@ -822,7 +858,9 @@ namespace gpu { source_buffer << "offset[0] + "; } source_buffer << "index < " << size << ") {" << std::endl; - + if (iterations > 1) { + source_buffer = " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } for (auto &input : inputs) { #ifdef USE_INPUT_CACHE @@ -858,6 +896,7 @@ namespace gpu { /// @param[in,out] registers Map of used registers. /// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, @@ -865,7 +904,8 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, jit::register_map &indices, - const jit::register_usage &usage) { + const jit::register_usage &usage, + const size_t iterations=1) { std::unordered_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { @@ -942,6 +982,9 @@ namespace gpu { } } + if (iterations > 1) { + source_buffer << " }" << std::endl; + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index b73afaa..2f10fc6 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -108,19 +108,21 @@ namespace jit { /// /// Build the source code for a kernel graph. /// -/// @param[in] name Name to call the kernel. -/// @param[in] inputs Input variables of the kernel. -/// @param[in] outputs Output nodes of the graph to compute. -/// @param[in] setters Map outputs back to input values. -/// @param[in] state Random state node. -/// @param[in] size Size of the kernel. +/// @param[in] name Name to call the kernel. +/// @param[in] inputs Input variables of the kernel. +/// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] setters Map outputs back to input values. +/// @param[in] state Random state node. +/// @param[in] size Size of the kernel. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void add_kernel(const std::string name, graph::input_nodes inputs, graph::output_nodes outputs, graph::map_nodes setters, graph::shared_random_state state, - const size_t size) { + const size_t size, + const size_t iterations=1) { kernel_names.push_back(name); if (state.get() && !used_random) { @@ -166,7 +168,90 @@ namespace jit { size, is_constant, registers, usage, kernel_1dtextures[name], - kernel_2dtextures[name]); + kernel_2dtextures[name], + iterations); + + register_map indices; + for (auto &[out, in] : setters) { + out->compile(source_buffer, registers, indices, usage); + } + for (auto &out : outputs) { + out->compile(source_buffer, registers, indices, usage); + } + + gpu_context.create_kernel_postfix(source_buffer, outputs, + setters, state, + registers, indices, usage, + iterations); + +// Delete the registers so that they can be used again in other kernels. + std::vector removed_elements; + for (auto &[key, value] : registers) { + if (value[0] == 'r') { + removed_elements.push_back(key); + } + } + + for (auto &key : removed_elements) { + registers.erase(key); + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a loop kernel. +/// +/// Build the source code for a kernel graph. +/// +/// @param[in] name Name to call the kernel. +/// @param[in] inputs Input variables of the kernel. +/// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] setters Map outputs back to input values. +/// @param[in] state Random state node. +/// @param[in] size Size of the kernel. +/// @param[in] iterations Number of iterations of the loop. +//------------------------------------------------------------------------------ + void add_loop_kernel(const std::string name, + graph::input_nodes inputs, + graph::output_nodes outputs, + graph::map_nodes setters, + graph::shared_random_state state, + const size_t size, + const size_t iterations) { + kernel_names.push_back(name); + + kernel_names.push_back(name); + + std::vector is_constant(inputs.size(), true); + visiter_map visited; + register_usage usage; + kernel_1dtextures[name] = texture1d_list(); + kernel_2dtextures[name] = texture2d_list(); + for (auto &[out, in] : setters) { + auto found = std::distance(inputs.begin(), + std::find(inputs.begin(), + inputs.end(), in)); + if (found < is_constant.size()) { + is_constant[found] = false; + } + out->compile_preamble(source_buffer, registers, + visited, usage, + kernel_1dtextures[name], + kernel_2dtextures[name], + gpu_context.remaining_const_memory); + } + for (auto &out : outputs) { + out->compile_preamble(source_buffer, registers, + visited, usage, + kernel_1dtextures[name], + kernel_2dtextures[name], + gpu_context.remaining_const_memory); + } + + for (auto &in : inputs) { + if (usage.find(in.get()) == usage.end()) { + usage[in.get()] = 0; + } + } register_map indices; for (auto &[out, in] : setters) { @@ -202,6 +287,16 @@ namespace jit { gpu_context.create_reduction(source_buffer, size); } +//------------------------------------------------------------------------------ +/// @brief Add zero. +/// +/// @param[in] inputs Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes inputs) { + return gpu_context.create_zero_call(inputs); + } + //------------------------------------------------------------------------------ /// @brief Print the kernel source. //------------------------------------------------------------------------------ diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index d54daff..19cdd72 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -2487,25 +2487,21 @@ namespace graph { indices, usage); registers[this] = jit::to_string('r', this); - stream << " "; - jit::add_type (stream); - stream << " " << registers[this] << ";" << std::endl - << " if(" << registers[c.get()] << ") {" << std::endl; - shared_leaf t = this->middle->compile(stream, registers, indices, usage); - stream << " " << registers[this] << " = " << registers[t.get()] << ";" - << " } else {" << std::endl; - shared_leaf f = this->right->compile(stream, registers, indices, usage); - - stream << " " << registers[this] << " = " << registers[f.get()] << ";" - << " }" << std::endl; + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = " + << registers[c.get()] << " ? " + << registers[t.get()] << " : " + << registers[f.get()]; + this->endline(stream, usage); } return this->shared_from_this(); diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index b88c7ce..8d8c105 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -348,6 +348,39 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create buffer that will be memset to zero. +/// +/// @param[in] inputs Input nodes of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector> buffers; + for (auto &input : inputs) { + if (!kernel_arguments.contains(input.get())) { + kernel_arguments[input.get()] = [device newBufferWithBytes:input->data() + length:input->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + buffers.push_back(kernel_arguments[input.get()]); + } + buffers.push_back(kernel_arguments[input.get()]); + } + + return [this, buffers] () mutable { + command_buffer = [queue commandBuffer]; + id encoder = [command_buffer blitCommandEncoder]; + + for (id buffer : buffers) { + [encoder fillBuffer:buffer + range:NSMakeRange(0, buffer.length) + value:0]; + } + [encoder endEncoding]; + + [command_buffer commit]; + }; + } + //------------------------------------------------------------------------------ /// @brief Get the compile options. //------------------------------------------------------------------------------ @@ -452,6 +485,7 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, @@ -463,7 +497,8 @@ namespace gpu { jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, - jit::texture2d_list &textures2d) { + jit::texture2d_list &textures2d, + const size_t iterations=1) { source_buffer << std::endl; source_buffer << "kernel void " << name << "(" << std::endl; @@ -531,6 +566,9 @@ namespace gpu { source_buffer << "offset + "; } source_buffer << "index < " << size << ") {" << std::endl; + if (iterations > 1) { + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + } for (auto &input : inputs) { #ifdef USE_INPUT_CACHE @@ -576,6 +614,7 @@ namespace gpu { /// @param[in,out] registers Map of used registers. /// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. +/// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes &outputs, @@ -583,7 +622,8 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, jit::register_map &indices, - const jit::register_usage &usage) { + const jit::register_usage &usage, + const size_t iterations=1) { std::unordered_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { @@ -621,6 +661,9 @@ namespace gpu { } } + if (iterations > 1) { + source_buffer << " }" << std::endl; + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index a8f65e4..8d256a4 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -719,6 +719,174 @@ namespace graph { std::cout << stream.str() << std::endl; } +//****************************************************************************** +/// @brief Index node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing kernel thread index. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class index_node final : public leaf_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string() { + return "i"; + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a constant node from a vector. +//------------------------------------------------------------------------------ + index_node() : + leaf_node (index_node::to_string(), 1, false) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate method. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + return backend::buffer (); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + return this->is_match(x) ? one () : zero (); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] indices List of defined indices. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual std::shared_ptr> + compile(std::ostringstream &stream, + jit::register_map ®isters, + jit::register_map &indices, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + registers[this] = jit::to_string('i', this); + stream << " const "; + if constexpr (jit::use_cuda()) { + stream << "int " << registers[this] << " = index"; + } else if constexpr (jit::use_metal ()) { + stream << "int " << registers[this] << " = index"; + } else { + stream << "size_t " << registers[this] << " = i"; + } + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "i"; + }; + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('i', this); + registers[this] = name; + stream << " " << name + << " [label = \"i\", shape = box, style = \"rounded,filled\", fillcolor = black, fontcolor = white];" << std::endl; + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the exponent of a power. +/// +/// @returns The exponent of a power like node. +//------------------------------------------------------------------------------ + virtual shared_leaf get_power_exponent() const { + return one (); + } + +//------------------------------------------------------------------------------ +/// @brief Test if all the sub-nodes terminate in variables. +/// +/// @returns True if all the sub-nodes terminate in variables. +//------------------------------------------------------------------------------ + virtual bool is_all_variables() const { + return false; + } + }; + +//------------------------------------------------------------------------------ +/// @brief Construct an index. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @returns A reduced constant node. +//------------------------------------------------------------------------------ + template + shared_leaf index() { + auto temp = std::make_shared> (); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared index nodes. + template + using shared_index = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a index node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_index index_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //****************************************************************************** // Constant node. //****************************************************************************** @@ -730,6 +898,7 @@ namespace graph { //------------------------------------------------------------------------------ template class constant_node final : public leaf_node { + private: //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -740,7 +909,6 @@ namespace graph { return jit::format_to_string (d); } - private: /// Storage buffer for the data. const backend::buffer data; diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 6582232..d4bcbec 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -13,11 +13,106 @@ #include "piecewise.hpp" #include "workflow.hpp" #include "random.hpp" +#include "logical.hpp" namespace pic { +// FIXME: This should be in a separate file of physics constants. +/// Speed of light m/s. + template + constexpr T c = static_cast (299792458.0); +/// Vacuum permitivity F/m. + template + constexpr T epsilon0 = static_cast (8.8541878188E-12); +/// Fundamental charge coulombs. + template + constexpr T q = static_cast (1.602176634E-19); +/// Hydrogen mass kg. + template + constexpr T m_hydrogen = static_cast (1.67362192595E-27); +/// Electron mass kg. + template + constexpr T m_electron = static_cast (9.1093837139E-31); +/// Boltzman constant. + template + constexpr T kb = static_cast (1.380649E-23); + +//------------------------------------------------------------------------------ +/// @brief Characteristic factors. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class characteristics { + private: +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_masses Ion masses. +/// @returns (∑(m_i) + me)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_m(const std::vector &ion_masses) { + T total_m = m_electron; + for (const T &mass : ion_masses) { + total_m += mass; + } + return total_m/(ion_masses.size() + 1); + } + +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_zs Ion Z. +/// @returns (∑(Z_i)*q + q)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_q(const std::vector &ion_zs) { + T total_q = pic::q; + for (const uint8_t &z : ion_zs) { + total_q += z*pic::q; + } + return total_q/(ion_zs.size() + 1); + } + + public: +/// Mass + const T m; +/// Charge + const T q; +/// Electron density. + const T ne; +/// Plasma Frequency. + const T wpe; +/// Time. + const T t; +/// Length + const T l; +/// Velocity + const T v; +/// Electric field; + const T efield; +/// Magnetic field; + const T bfield; + +//------------------------------------------------------------------------------ +/// @brief Construct the characteristics. +/// +/// @param[in] ion_masses Ion masses for all species. +/// @param[in] ion_zs Ion Z effective all species. +/// @param[in] ne Characteristic density. +//------------------------------------------------------------------------------ + characteristics(const std::vector &ion_masses, + const std::vector &ion_zs, + const T ne) : + m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), + wpe(std::sqrt(ne*q*q/(m*epsilon0))), + t(1/wpe), l(wpe/c), v(c), efield(m*c/(q*t)), + bfield(efield/c) {} + }; + //------------------------------------------------------------------------------ /// @brief ion class. /// +/// These values need to be initalized using normalized quantities. +/// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template @@ -27,28 +122,38 @@ namespace pic { const T charge; /// Particle mass const T mass; -/// Position +/// Normalized Position graph::shared_leaf x; -/// Parallel velocity. +/// Normalized Parallel velocity. graph::shared_leaf v_para; -/// Perpendicular velocity. +/// Normalized Perpendicular velocity. graph::shared_leaf v_perp; +/// Mesh Weights + std::array, 3> weights; +/// Mesh index + graph::shared_leaf indices; //------------------------------------------------------------------------------ /// @brief Construct an ion object. /// -/// @param[in] charge Ion charge. -/// @param[in] mass Ion mass. -/// @param[in] x Ion position. -/// @param[in] v_para Parallel velocity. -/// @param[in] v_perp Perpendicular velocity. -//------------------------------------------------------------------------------ - ion(const T charge, - const T mass, - graph::shared_leaf x, - graph::shared_leaf v_para, - graph::shared_leaf v_perp) : - charge(charge), mass(mass), x(x), v_para(v_para), v_perp(v_perp) {} +/// @param[in] mass Ion mass. +/// @param[in] z Ion Z. +/// @param[in] num_ions Number of ions. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + ion(const T mass, + const uint8_t z, + const size_t num_ions, + const characteristics &norms) : + charge(z*pic::q/norms.q), mass(mass/norms.m), + x(graph::variable (num_ions, "x")), + v_para(graph::variable (num_ions, "v_{||}")), + v_perp(graph::variable (num_ions, "v_{\\perp}")), + weights({ + graph::variable (num_ions, "w_{0}"), + graph::variable (num_ions, "w_{1}"), + graph::variable (num_ions, "w_{2}") + }), indices(graph::variable (num_ions, "m_{i}")) {} }; //------------------------------------------------------------------------------ @@ -59,30 +164,91 @@ namespace pic { template class mesh { public: -/// Mesh x positions. - graph::shared_leaf x; -/// Mesh y values. - graph::shared_leaf y; /// Min x const T xmin; /// Max x const T xmax; -/// Min mesh spacing. +/// Dx const T dx; +/// Particle index. + graph::shared_leaf index; +/// Mesh y values. + graph::shared_leaf y; //------------------------------------------------------------------------------ /// @brief Construct a mesh object. /// -/// @param[in] xmesh X position of mesh points. -/// @param[in] ymesh Y position of the mesh. -//------------------------------------------------------------------------------ - mesh(graph::shared_leaf xmesh, - graph::shared_leaf ymesh) : - x(xmesh), y(ymesh), - xmin(graph::variable_cast(xmesh)->data()[0]), - xmax(graph::variable_cast(xmesh)->data()[graph::variable_cast(xmesh)->size() - 1]), - dx(graph::variable_cast(xmesh)->data()[1] - - graph::variable_cast(xmesh)->data()[0]) {} +/// @param[in] x_min Minimum X postion of mesh. +/// @param[in] x_max Maximum X position of mesh. +/// @param[in] num Number of mesh points. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + mesh(const T x_min, + const T x_max, + const size_t num, + const characteristics &norms) : + y(graph::variable (num, "y_{m}")), + index(graph::variable (num, "pi_{m}")), + xmin(x_min/norms.l), xmax(x_max/norms.l), + dx((xmax - xmin)/(num - 1)) {} + +//------------------------------------------------------------------------------ +/// @brief Build x index. +/// +/// @param[in] ion A @ref pic::ion object. +/// @returns The indexed mesh X position. +//------------------------------------------------------------------------------ + graph::shared_leaf build_x_index(ion &ion) const { + const size_t s = graph::variable_cast(y)->size(); + const backend::buffer buffer(xmin, dx, s); + return piecewise_1D(buffer, ion.x, dx, xmin); + } + +//------------------------------------------------------------------------------ +/// @brief Build i index. +/// +/// @param[in] ion A @ref pic::ion object. +/// @returns The indexed mesh X position. +//------------------------------------------------------------------------------ + graph::shared_leaf build_i_index(ion &ion) const { + const size_t s = graph::variable_cast(y)->size(); + const backend::buffer buffer(static_cast (0), + static_cast (1), s); + return piecewise_1D(buffer, ion.x, dx, xmin); + } + +//------------------------------------------------------------------------------ +/// @brief Build mesh accumulation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @returns Expressions for mesh accumulation. +//------------------------------------------------------------------------------ + std::array, 2> build_mesh_solve(ion &ion) const { + auto next_index = index; + auto next_weight = y; + auto kernel_index = graph::index (); + + auto index_i = graph::index_1D(ion.indices, next_index, + static_cast (1), + static_cast (0)); + auto index_w0 = graph::index_1D(ion.weights[0], next_index, + static_cast (1), + static_cast (0)); + auto index_w1 = graph::index_1D(ion.weights[1], next_index, + static_cast (1), + static_cast (0)); + auto index_w2 = graph::index_1D(ion.weights[2], next_index, + static_cast (1), + static_cast (0)); + next_index = next_index + static_cast (1); + next_weight = graph::if_(index_i - static_cast (1) == kernel_index, + next_weight + index_w0, next_weight); + next_weight = graph::if_(index_i == kernel_index, + next_weight + index_w1, next_weight); + next_weight = graph::if_(index_i + static_cast (1) == kernel_index, + next_weight + index_w2, next_weight); + return {next_index, next_weight}; + } }; //------------------------------------------------------------------------------ @@ -90,27 +256,29 @@ namespace pic { /// /// @tparam T Base type of the calculation. /// -/// @param[in] xp X position of the particles. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] norms A @ref pic::characteristics object. /// @returns The magnetic field expression. //------------------------------------------------------------------------------ template - graph::shared_leaf build_magnetic_field(graph::shared_leaf xp, - const T bchar) { - return (xp*xp + static_cast (1))/bchar; + graph::shared_leaf build_magnetic_field(ion ion, + const characteristics &norms) { + return (ion.x*ion.x + static_cast (1))/norms.bfield; } //------------------------------------------------------------------------------ -/// @brief Build interpolation expression. +/// @brief Build interpolation weights. /// /// @tparam T Base type of the calculation. /// /// @param[in] mesh Mesh object. -/// @param[in] xp X position of the particles. +/// @param[in] ion A @ref pic::ion object. +/// @returns The interpolated mesh weights. //------------------------------------------------------------------------------ - template - graph::shared_leaf build_interpolation(pic::mesh &mesh, - graph::shared_leaf xp) { - auto x = graph::index_1D(mesh.x, xp, mesh.dx, mesh.xmin) - xp; + template + std::array, 3> build_weights(mesh &mesh, + ion &ion) { + auto x = mesh.build_x_index(ion) - ion.x; auto xnorm1 = static_cast (1.5) + (x - mesh.dx)/mesh.dx; auto xnorm2 = x/mesh.dx; auto xnorm3 = static_cast (1.5) - (x + mesh.dx)/mesh.dx; @@ -118,44 +286,29 @@ namespace pic { auto w0 = static_cast (0.5)*xnorm1*xnorm1; auto w1 = static_cast (0.75) - xnorm2*xnorm2; auto w2 = static_cast (0.5)*xnorm3*xnorm3; + + return {w0, w1, w2}; + } - auto ymesh0 = graph::index_1D(mesh.y, xp - mesh.dx, mesh.dx, mesh.xmin); - auto ymesh1 = graph::index_1D(mesh.y, xp, mesh.dx, mesh.xmin); - auto ymesh2 = graph::index_1D(mesh.y, xp + mesh.dx, mesh.dx, mesh.xmin); - -// Run only for unit tests. - if constexpr (UNIT_TEST) { - auto xp_cast = graph::variable_cast(xp); - assert(xp_cast.get() && "Expected a variable."); - - auto weight = w0 + w1 + w2; - - workflow::manager work(0); - work.add_item({ - graph::variable_cast(mesh.x), - graph::variable_cast(xp) - }, { - weight - }, {}, NULL, "build_interpolation_unit_test", xp_cast->size()); - work.compile(); - work.run(); - work.wait(); - -// The weights should sum to 1. - for (size_t i = 0, ie = xp_cast->size(); i < ie; i++) { - const T recieved = work.check_value(i, weight); - const T diff = static_cast (1) - recieved; - if constexpr (std::same_as) { - assert(diff*diff < static_cast (3.2E-14) && - "Weight not equal to 1±3.2E-14"); - } else { - assert(diff*diff < static_cast (5.0E-32) && - "Weight not equal to 1±5.0E-32"); - } - } - } +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] mesh Mesh object. +/// @param[in] ion A @ref pic::ion object. +/// @returns The interpolated mesh quantity. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_interpolation(mesh &mesh, + ion &ion) { + auto weights = build_weights (mesh, ion); + + auto ymesh0 = graph::index_1D(mesh.y, ion.x - mesh.dx, mesh.dx, mesh.xmin); + auto ymesh1 = graph::index_1D(mesh.y, ion.x, mesh.dx, mesh.xmin); + auto ymesh2 = graph::index_1D(mesh.y, ion.x + mesh.dx, mesh.dx, mesh.xmin); - return w0*ymesh0 + w1*ymesh1 + w2*ymesh2; + return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; } //------------------------------------------------------------------------------ @@ -165,17 +318,18 @@ namespace pic { /// /// @param[in] ion A @ref pic::ion object. /// @param[in] mesh A @ref pic::mesh object. -/// @param[in] bchar Characteristic magnetic field. /// @param[in] z Runga Kutta substep. -/// @param[in] dt Time step. +/// @param[in] dt Normalized time step. +/// @param[in] norms A @ref pic::characteristics object. +/// @returns the Forces on the particles. //------------------------------------------------------------------------------ template std::array, 3> build_F_expressions(ion &ion, mesh &mesh, - const T bchar, const std::array, 3> z, - const T dt) { - auto bfield = build_magnetic_field (z[0], bchar); + const T dt, + const characteristics &norms) { + auto bfield = build_magnetic_field (z[0], norms); auto efield = build_interpolation (mesh, ion.x); auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; return { @@ -192,17 +346,20 @@ namespace pic { /// /// @param[in] ion A @ref pic::ion object. /// @param[in] mesh A @ref pic::mesh object. -/// @param[in] bchar Characteristic magnetic field. +/// @param[in] dt Normalized time step. +/// @param[in] norms A @ref pic::characteristics object. /// @returns Step update expressions for x, v_para, and x_perp. //------------------------------------------------------------------------------ template std::array, 3> build_rk4_step(ion &ion, mesh &mesh, - const T bchar, - const T dt) { + const T dt, + const characteristics &norms) { + std::array, 3> ion_norm = ion.normalize(norms); + // Step 1 std::array, 3> Z1{ion.x, ion.v_para, ion.v_perp}; - std::array, 3> dZ1(build_F_expressions (ion, mesh, bchar, Z1, dt)); + std::array, 3> dZ1(build_F_expressions (ion, mesh, Z1, dt, norms)); // Step 2 std::array, 3> Z2{ @@ -210,7 +367,7 @@ namespace pic { Z1[1] + dZ1[1]/static_cast (2), Z1[2] + dZ1[2]/static_cast (2) }; - std::array, 3> dZ2(build_F_expressions (ion, mesh, bchar, Z2, dt)); + std::array, 3> dZ2(build_F_expressions (ion, mesh, Z2, dt, norms)); // Step 3 std::array, 3> Z3{ @@ -218,7 +375,7 @@ namespace pic { Z1[1] + dZ2[1]/static_cast (2), Z1[2] + dZ2[2]/static_cast (2) }; - std::array, 3> dZ3(build_F_expressions (ion, mesh, bchar, Z3, dt)); + std::array, 3> dZ3(build_F_expressions (ion, mesh, Z3, dt, norms)); // Step 4 std::array, 3> Z4{ @@ -226,7 +383,7 @@ namespace pic { Z1[1] + dZ3[1], Z1[2] + dZ3[2] }; - std::array, 3> dZ4(build_F_expressions (ion, mesh, bchar, Z4, dt)); + std::array, 3> dZ4(build_F_expressions (ion, mesh, Z4, dt, norms)); // Rk4 Solution return { @@ -239,8 +396,11 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Build magnetic moment. /// +/// @tparam T Base type of the calculation. +/// /// @param[in] ion A @ref pic::ion object. /// @param[in] mesh A @ref pic::mesh object. +/// @returns The magnetic moment. //------------------------------------------------------------------------------ template graph::shared_leaf build_magnetic_moment(ion &ion, mesh &mesh) { @@ -249,11 +409,20 @@ namespace pic { } //------------------------------------------------------------------------------ -/// @brief Build +/// @brief Build initializtion. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] state Random state node. +/// @returns Initialized normalized values for x, v||, and v⟂ //------------------------------------------------------------------------------ template std::array,3> build_initialization(mesh &mesh, + const characteristics &norms, graph::shared_random_state state) { +// The mesh is already normalized so position_dist will be a normalized quantity. auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, state); auto phi_dist = graph::uniform_random (static_cast (0.0), @@ -262,9 +431,8 @@ namespace pic { auto r_dist = graph::uniform_random (static_cast (0.0), static_cast (1.0), state); -// FIXME: This should be in a separate file of physics constants. - const T kb = static_cast (1.380650E-23); - auto vpara = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::sin(phi_dist); + + auto vpara = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); phi_dist = graph::uniform_random (static_cast (0.0), static_cast (2.0)*std::numbers::pi_v, @@ -272,11 +440,39 @@ namespace pic { r_dist = graph::uniform_random (static_cast (0.0), static_cast (1.0), state); - auto vperp1 = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::cos(phi_dist); - auto vperp2 = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))%graph::sin(phi_dist); + auto vperp1 = graph::sqrt(-pic::kb*graph::log(static_cast (1) - r_dist))*graph::cos(phi_dist); + auto vperp2 = graph::sqrt(-pic::kb*graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); - return {position_dist, vpara, vperp}; + return {position_dist, vpara/norms.v, vperp/norms.v}; + } + +//------------------------------------------------------------------------------ +/// @brief Build reinjected expressions. +/// +/// If the particles leave the mesh, reinitalize them using the same +/// initialization. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] state Random state node. +/// @returns Reinjected values for x, v||, and v⟂ +//------------------------------------------------------------------------------ + template + std::array, 3> build_reinjection(ion &ion, + mesh &mesh, + const characteristics &norms, + graph::shared_random_state state) { + auto resampled = build_initialization(mesh, state); + auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; + + auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); + auto reinject_vpara = graph::if_(is_outside, resampled[1], ion.v_para); + auto reinject_vperp = graph::if_(is_outside, resampled[2], ion.v_perp); + return {reinject_x, reinject_vpara, reinject_vperp}; } } diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index 50be5f2..fa3f40a 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -12,6 +12,68 @@ /// Name space for workflows. namespace workflow { +//------------------------------------------------------------------------------ +/// @brief Interface class representing items. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class item { + public: +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) = 0; + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() = 0; + }; + +//------------------------------------------------------------------------------ +/// @brief Clear buffer item. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class zero_item : public item { + protected: +/// Kernel function. + std::function kernel; +/// Input nodes. + graph::input_nodes inputs; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] in Input variables. +//------------------------------------------------------------------------------ + zero_item(graph::input_nodes in) : + inputs(in) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_zero_call(inputs); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + //------------------------------------------------------------------------------ /// @brief Class representing a work item. /// @@ -19,11 +81,13 @@ namespace workflow { /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. //------------------------------------------------------------------------------ template - class work_item { + class work_item : public item { protected: +/// Kernel function. + std::function kernel; /// Name of the GPU kernel. const std::string kernel_name; -/// Name of the GPU kernel. +/// Size of the GPU kernel. const size_t kernel_size; /// Input nodes. graph::input_nodes inputs; @@ -31,8 +95,6 @@ namespace workflow { graph::output_nodes outputs; /// Random state node. graph::shared_random_state state; -/// Kernel function. - std::function kernel; public: //------------------------------------------------------------------------------ @@ -82,40 +144,61 @@ namespace workflow { /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. //------------------------------------------------------------------------------ template - class loop_item final : public work_item { -/// Iterations. - const size_t num_iterations; + class loop_item final : public item { + protected: +/// Kernel function. + std::function kernel; +/// Name of the GPU kernel. + const std::string kernel_name; +/// Size of the GPU kernel. + const size_t kernel_size; +/// Input nodes. + graph::input_nodes inputs; +/// Output nodes. + graph::output_nodes outputs; +/// Random state node. + graph::shared_random_state state; public: //------------------------------------------------------------------------------ /// @brief Construct a workflow item. /// -/// @param[in] inputs Input variables. -/// @param[in] outputs Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -/// @param[in,out] context Jit context. +/// @param[in] in Input variables. +/// @param[in] out Output nodes. +/// @param[in] maps Setter maps. +/// @param[in] state Random state node. +/// @param[in] name Name of the work item. +/// @param[in] size Size of the work item. +/// @param[in,out] context Jit context. /// @param[in] iterations Number of iterations to run the loop. //------------------------------------------------------------------------------ - loop_item(graph::input_nodes inputs, - graph::output_nodes outputs, + loop_item(graph::input_nodes in, + graph::output_nodes out, graph::map_nodes maps, graph::shared_random_state state, const std::string name, const size_t size, jit::context &context, const size_t iterations) : - work_item (inputs, outputs, maps, state, name, size, context), - num_iterations(iterations) {} + inputs(in), outputs(out), state(state), + kernel_name(name), kernel_size(size) { + context.add_kernel(name, in, out, maps, state, size, iterations); + } + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_kernel_call(kernel_name, inputs, outputs, + state, kernel_size); + } //------------------------------------------------------------------------------ /// @brief Run the workitem. //------------------------------------------------------------------------------ virtual void run() { - for (size_t i = 0; i < num_iterations; i++) { - work_item::run(); - } + kernel(); } }; @@ -217,9 +300,9 @@ namespace workflow { /// JIT context. jit::context context; /// List of pre work items. - std::vector>> preitems; + std::vector>> preitems; /// List of work items. - std::vector>> items; + std::vector>> items; /// Use reduction. bool add_reduction; @@ -258,6 +341,15 @@ namespace workflow { context)); } +//------------------------------------------------------------------------------ +/// @brief Add a pre zero item. +/// +/// @param[in] in Input variables. +//------------------------------------------------------------------------------ + void add_prezero_item(graph::input_nodes in) { + preitems.push_back(std::make_unique> (in)); + } + //------------------------------------------------------------------------------ /// @brief Add a workflow item. /// @@ -279,6 +371,15 @@ namespace workflow { context)); } +//------------------------------------------------------------------------------ +/// @brief Add a zero item. +/// +/// @param[in] in Input variables. +//------------------------------------------------------------------------------ + void add_zero_item(graph::input_nodes in) { + items.push_back(std::make_unique> (in)); + } + //------------------------------------------------------------------------------ /// @brief Add a workflow item. /// diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 0e0a64b..06976d5 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -43,25 +43,22 @@ graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { //------------------------------------------------------------------------------ template void run_pic() { +#if 0 // Constants const size_t num_particles = 1000000; const size_t num_grid = 200; - const T c = 299792458.0; - const T epsilon0 = 8.854E-12; - const T q = 1.602E-19; - const T m_hydrogen = 1.6738E-27; - const T m_electron = 9.1093837139E-31; - const T kb = 1.380650E-23; const uint8_t Z = 1; // Characteristic factors - const T mchar = (m_hydrogen + m_electron)/2; - const T qchar = (q + q)/2; - const T nechar = 2.5E19; - const T wpechar = std::sqrt(nechar*qchar*qchar/(mchar*epsilon0)); - const T tchar = 1/wpechar; - const T echar = mchar*c/(qchar*tchar); - const T bchar = echar/c; + const std::vector ion_masses{pic::m_hydrogen}; + const std::vector ion_zs{1}; + + const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); + std::vector> ions; + for(size_t i = 0, ie = ion_masses.size(); i < ie; i++) { + ions.emplace_back(ion_masses[i], ion_zs[i], num_particles); + } + pic::mesh mesh(-0.25, 0.25, num_grid); // Particle initalization. auto x = graph::variable (num_particles, "x"); @@ -242,6 +239,7 @@ void run_pic() { work.wait(); sync_particles.join(); sync_fields.join(); +#endif } //------------------------------------------------------------------------------ diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index 9a121a3..d027da6 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -76,6 +76,13 @@ void compile(graph::input_nodes inputs, //------------------------------------------------------------------------------ template void run_math_tests() { auto v1 = graph::variable (1, "v1"); + + compile ({ + graph::variable_cast(v1) + }, { + graph::index () + }, {}, static_cast (0), 0.0); + auto v2 = graph::variable (1, "v2"); auto v3 = graph::variable (1, "v3"); diff --git a/graph_tests/node_test.cpp b/graph_tests/node_test.cpp index b6fefcd..1e003e3 100644 --- a/graph_tests/node_test.cpp +++ b/graph_tests/node_test.cpp @@ -15,6 +15,26 @@ #include "../graph_framework/trigonometry.hpp" #include "../graph_framework/arithmetic.hpp" +//------------------------------------------------------------------------------ +/// @brief Tests for constant nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_index() { + auto index = graph::index (); + auto index_cast = graph::index_cast(index); + assert(index_cast.get() && "Expected a index type."); + auto dindex = index->df(index); + auto dindex_cast = graph::constant_cast(dindex); + assert(dindex_cast.get() && "Expected a constant type for derivative."); + assert(dindex_cast->is(1.0) && "Constant value expected one."); + auto dindex2 = index->df(graph::zero ()); + auto dindex2_cast = graph::constant_cast(dindex2); + assert(dindex2_cast.get() && "Expected a constant type for derivative."); + assert(dindex2_cast->is(0.0) && "Constant value expected one."); +} + //------------------------------------------------------------------------------ /// @brief Tests for constant nodes. /// @@ -169,6 +189,7 @@ void test_pseudo_variable() { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { + test_index (); test_constant (); test_variable (); test_pseudo_variable (); diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 2c80ae8..a70a111 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -9,6 +9,7 @@ #endif #include +#include #include "../graph_framework/graph_framework.hpp" @@ -21,47 +22,58 @@ template void run_interpolation_test() { const size_t num_mesh = 100; const size_t num_particles = 10000; - auto xmesh = graph::variable (num_mesh, "x_mesh"); - auto ymesh = graph::variable (num_mesh, "y_mesh"); +// Characteristic factors + const std::vector ion_masses{pic::m_hydrogen}; + const std::vector ion_zs{1}; - const T xmin = static_cast (-3); - const T xmax = static_cast (3); - const T dx = (xmax - xmin)/(num_mesh - 1); + const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); + std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], num_particles, norms)}; + pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); - std::function func([](const T x) -> T { + std::function func([&norms](const T x) -> T { return std::sin(std::exp(x)); }); for (size_t i = 0; i < num_mesh; i++) { - graph::variable_cast(xmesh)->data()[i] = dx*i + xmin; - graph::variable_cast(ymesh)->data()[i] = func(graph::variable_cast(xmesh)->data()[i]); + graph::variable_cast(mesh.y)->data()[i] = func(mesh.dx*i + mesh.xmin); } - pic::mesh efield_mesh(xmesh, ymesh); - - auto xp = graph::variable (num_particles, "xp"); - const T dxp = (xmax - xmin)/(num_particles - 1); + const T dxp = (mesh.xmax - mesh.xmin)/(num_particles - 1); for (size_t i = 0; i < num_particles; i++) { - graph::variable_cast(xp)->data()[i] = dxp*i + xmin; + graph::variable_cast(ions[0].x)->data()[i] = dxp*i + mesh.xmin; } - auto field = pic::build_interpolation (efield_mesh, xp); + auto weights = pic::build_weights (mesh, ions[0]); + auto field = pic::build_interpolation (mesh, ions[0]); + auto weight = weights[0] + weights[1] + weights[2]; workflow::manager work(0); work.add_item({ - graph::variable_cast(efield_mesh.x), - graph::variable_cast(efield_mesh.y), - graph::variable_cast(xp) + graph::variable_cast(mesh.y), + graph::variable_cast(ions[0].x) }, { + weight, field }, {}, NULL, "Mesh_Interpolation", num_particles); work.compile(); work.run(); work.wait(); - auto xp_cast = graph::variable_cast(xp); - for (size_t i = 0, ie = xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); +// The weights should sum to 1. + for (size_t i = 0; i < num_particles; i++) { + const T recieved = work.check_value(i, weight); + const T diff = static_cast (1) - recieved; + if constexpr (std::same_as) { + assert(diff*diff < static_cast (4.7E-12) && + "Weight not equal to 1±4.7E-12"); + } else { + assert(diff*diff < static_cast (7.1E-30) && + "Weight not equal to 1±7.1E-30"); + } + } + + for (size_t i = 0, ie = num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -72,8 +84,8 @@ template void run_interpolation_test() { "Profile not equal ±4.0E-7"); } } - for (size_t i = xp_cast->size()/10, ie = 2*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = num_particles/10, ie = 2*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -84,8 +96,8 @@ template void run_interpolation_test() { "Profile not equal ±1.1E-7"); } } - for (size_t i = 2*xp_cast->size()/10, ie = 3*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 2*num_particles/10, ie = 3*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -96,8 +108,8 @@ template void run_interpolation_test() { "Profile not equal ±1.5E-8"); } } - for (size_t i = 3*xp_cast->size()/10, ie = 4*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 3*num_particles/10, ie = 4*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -108,8 +120,8 @@ template void run_interpolation_test() { "Profile not equal ±7.9E-6"); } } - for (size_t i = 4*xp_cast->size()/10, ie = 5*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 4*num_particles/10, ie = 5*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -120,8 +132,8 @@ template void run_interpolation_test() { "Profile not equal ±2.1E-8"); } } - for (size_t i = 5*xp_cast->size()/10, ie = 6*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 5*num_particles/10, ie = 6*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -132,8 +144,8 @@ template void run_interpolation_test() { "Profile not equal ±2.9E-6"); } } - for (size_t i = 6*xp_cast->size()/10, ie = 7*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 6*num_particles/10, ie = 7*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -144,8 +156,8 @@ template void run_interpolation_test() { "Profile not equal ±7.0E-6"); } } - for (size_t i = 7*xp_cast->size()/10, ie = 8*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 7*num_particles/10, ie = 8*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -156,8 +168,8 @@ template void run_interpolation_test() { "Profile not equal ±1.5E-3"); } } - for (size_t i = 8*xp_cast->size()/10, ie = 9*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 8*num_particles/10, ie = 9*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -168,8 +180,8 @@ template void run_interpolation_test() { "Profile not equal ±3.0E-3"); } } - for (size_t i = 9*xp_cast->size()/10, ie = 10*xp_cast->size()/10; i < ie; i++) { - const T x = work.check_value(i, xp); + for (size_t i = 9*num_particles/10, ie = 10*num_particles/10; i < ie; i++) { + const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; if constexpr (std::same_as) { @@ -182,6 +194,99 @@ template void run_interpolation_test() { } } +//------------------------------------------------------------------------------ +/// @brief Field solve test. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void run_field_solve_test() { + const size_t num_mesh = 100; + const size_t num_particles = 1000000; + +// Characteristic factors + const std::vector ion_masses{pic::m_hydrogen}; + const std::vector ion_zs{1}; + + const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); + std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], num_particles, norms)}; + pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); + +// Initialize particle positions. + backend::buffer buffer(num_particles); + + std::mt19937 gen(0); + std::normal_distribution dist(0.0, 1.0); + + for (size_t i = 0; i < num_particles; i++) { + do { + buffer[i] = dist(gen); + } while(buffer[i] < -3.0 || buffer[i] > 3.0); + } + ions[0].x->set(buffer); + +// Count particles in mesh bins. This builds a histogram of particle counts. + std::vector counts(num_mesh, 0); + for (size_t i = 0; i < num_mesh; i++) { + const T bin_low = i*mesh.dx + mesh.xmin - mesh.dx/2; + const T bin_high = i*mesh.dx + mesh.xmin + mesh.dx/2; + for (size_t j = 0; j < num_particles; j++) { + if (buffer[j] >= bin_low && buffer[j] < bin_high) { + counts[i]++; + } + } + } + + auto weights = pic::build_weights (mesh, ions[0]); + auto mesh_i = mesh.build_i_index(ions[0]); + auto mesh_solve = mesh.build_mesh_solve(ions[0]); + + workflow::manager work(0); + work.add_zero_item({ + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y) + }); + work.add_item({ + graph::variable_cast(ions[0].x), + graph::variable_cast(ions[0].weights[0]), + graph::variable_cast(ions[0].weights[1]), + graph::variable_cast(ions[0].weights[2]), + graph::variable_cast(ions[0].indices) + }, {}, { + {weights[0], graph::variable_cast(ions[0].weights[0])}, + {weights[1], graph::variable_cast(ions[0].weights[1])}, + {weights[2], graph::variable_cast(ions[0].weights[2])}, + {mesh_i, graph::variable_cast(ions[0].indices)} + }, NULL, "compute_weights", num_particles); + work.add_loop_item({ + graph::variable_cast(ions[0].indices), + graph::variable_cast(ions[0].weights[0]), + graph::variable_cast(ions[0].weights[1]), + graph::variable_cast(ions[0].weights[2]), + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y) + }, {}, { + {mesh_solve[0], graph::variable_cast(mesh.index)}, + {mesh_solve[1], graph::variable_cast(mesh.y)} + }, NULL, "sum_weights", num_mesh, num_particles); + + work.compile(); + + const timing::measure_diagnostic t_run("Run Time"); + work.run(); + work.wait(); + t_run.print(); + + for (size_t i = 0; i < num_mesh; i++) { + const T recieved = work.check_value(i, mesh.y); + const T error = std::abs((counts[i] - recieved)/counts[i]); + if constexpr (std::is_same_v) { + assert(error < 0.155 && "Error outside tolarance range."); + } else { + assert(error < 0.192 && "Error outside tolarance range."); + } + } +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified precision. /// @@ -189,6 +294,7 @@ template void run_interpolation_test() { //------------------------------------------------------------------------------ template void run_tests() { run_interpolation_test (); + run_field_solve_test (); } //------------------------------------------------------------------------------ @@ -202,7 +308,7 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; - //run_tests (); + run_tests (); run_tests (); END_GPU diff --git a/graph_tests/workflow_test.cpp b/graph_tests/workflow_test.cpp index e61b619..be12d47 100644 --- a/graph_tests/workflow_test.cpp +++ b/graph_tests/workflow_test.cpp @@ -12,6 +12,33 @@ #include "../graph_framework/graph_framework.hpp" +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_zeros() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer(1, static_cast (1)); + a->set(buffer); + b->set(buffer); + + workflow::manager work(0); + work.add_zero_item({ + graph::variable_cast(a), + graph::variable_cast(b) + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); + work.run(); + assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); + assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); +} + //------------------------------------------------------------------------------ /// @brief Test setting multiple variables with the same map. /// @@ -76,6 +103,7 @@ template void test_loops() { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { + test_zeros (); test_maps (); test_loops (); } From 4abd2281cce9e275847b9622f22bd5596da03a40 Mon Sep 17 00:00:00 2001 From: m4c Date: Thu, 25 Jun 2026 16:31:09 -0400 Subject: [PATCH 06/51] Fix CUDA build errors. --- CMakeLists.txt | 1 - graph_framework/cuda_context.hpp | 17 +++++++++-------- graph_framework/piecewise.hpp | 12 ++++++++---- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8baf9e4..0c42789 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,7 +394,6 @@ macro (add_test_target target lang) add_test (NAME ${target} COMMAND ${target} ) - if (${USE_PCH}) if (${BUILD_C_BINDING}) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index ef38317..7448992 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -580,9 +580,8 @@ namespace gpu { /// @param[in] inputs Input nodes of the kernel. /// @returns A lambda function to run the kernel. //------------------------------------------------------------------------------ - std::function create_zero_call(graph::input_nodes &inputs) { - std::vector buffers; - std::vector sizes; + std::function create_zero_call(graph::input_nodes &inputs) { + std::vector buffers; for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { kernel_arguments.try_emplace(input.get()); @@ -594,16 +593,18 @@ namespace gpu { input->data(), input->size()*sizeof(T)), "cuMemcpyHtoD"); - buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); + buffers.push_back(kernel_arguments[input.get()]); } buffers.push_back(kernel_arguments[input.get()]); - sizes.push_back(inputs->size()*sizeof(T)); } - return [this, buffers, sizes] () mutable { + return [this, buffers] () mutable { for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffers[i]), + "cuMemGetAddressRange"); check_error_async(cuMemsetD8Async(buffers[i], 0, - size[i], stream), + size, stream), "cuMemsetD8Async"); } }; @@ -859,7 +860,7 @@ namespace gpu { } source_buffer << "index < " << size << ") {" << std::endl; if (iterations > 1) { - source_buffer = " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; + source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; } for (auto &input : inputs) { diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index c9fca38..6fab9fc 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -30,11 +30,13 @@ void compile_index(std::ostringstream &stream, const T offset) { const std::string type = jit::type_to_string (); stream << "(" << jit::smallest_uint_type (length) << ")min"; - if constexpr (!jit::use_metal ()) { + if constexpr (!jit::use_metal () && + !jit::use_cuda()) { stream << "<" << type << ">"; } stream << "(max"; - if constexpr (!jit::use_metal ()) { + if constexpr (!jit::use_metal () && + !jit::use_cuda ()) { stream << "<" << type << ">"; } stream << "("; @@ -54,11 +56,13 @@ void compile_index(std::ostringstream &stream, stream << ")"; } stream << ","; - if constexpr (jit::use_metal ()) { + if constexpr (jit::use_metal () || + jit::use_cuda()) { stream << "(" << type << ")"; } stream << "0),"; - if constexpr (jit::use_metal ()) { + if constexpr (jit::use_metal () || + jit::use_cuda()) { stream << "(" << type << ")"; } stream << length - 1 << ")"; From 2eade325a5fef7cdd5ba0cba745edef2ed7f8ccf Mon Sep 17 00:00:00 2001 From: m4c Date: Thu, 25 Jun 2026 17:30:26 -0400 Subject: [PATCH 07/51] Update test tolarance for cuda builds --- graph_tests/pic_test.cpp | 41 ++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index a70a111..c04d23e 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -88,24 +88,45 @@ template void run_interpolation_test() { const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; - if constexpr (std::same_as) { - assert(diff*diff < static_cast (1.4E-6) && - "Profile not equal ±1.4E-6"); + if constexpr (jit::use_cuda) { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.22E-4) && + "Profile not equal ±1.22E-4"); + } else { + assert(diff*diff < static_cast (1.04E-6) && + "Profile not equal ±1.04E-6"); + } } else { - assert(diff*diff < static_cast (1.1E-6) && - "Profile not equal ±1.1E-7"); + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.4E-6) && + "Profile not equal ±1.4E-6"); + } else { + assert(diff*diff < static_cast (1.1E-6) && + "Profile not equal ±1.1E-7"); + } } } + std::cout << std::endl; for (size_t i = 2*num_particles/10, ie = 3*num_particles/10; i < ie; i++) { const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; - if constexpr (std::same_as) { - assert(diff*diff < static_cast (3.8E-6) && - "Profile not equal ±3.8E-6"); + if constexpr (jit::use_cuda) { + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.1E-4) && + "Profile not equal ±3.1E-4"); + } else { + assert(diff*diff < static_cast (1.42E-8) && + "Profile not equal ±1.42E-8"); + } } else { - assert(diff*diff < static_cast (1.5E-8) && - "Profile not equal ±1.5E-8"); + if constexpr (std::same_as) { + assert(diff*diff < static_cast (3.8E-6) && + "Profile not equal ±3.8E-6"); + } else { + assert(diff*diff < static_cast (1.5E-8) && + "Profile not equal ±1.5E-8"); + } } } for (size_t i = 3*num_particles/10, ie = 4*num_particles/10; i < ie; i++) { From aef0b36fcfcd12e347be2c41cba6a38b2ed54eb6 Mon Sep 17 00:00:00 2001 From: m4c Date: Thu, 25 Jun 2026 17:53:32 -0400 Subject: [PATCH 08/51] Adjust float test tolarance on CPU. --- graph_tests/pic_test.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index c04d23e..482e321 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -157,12 +157,17 @@ template void run_interpolation_test() { const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; - if constexpr (std::same_as) { + if constexpr (jit::use_metal ()) { assert(diff*diff < static_cast (1.6E-5) && "Profile not equal ±1.6E-5"); } else { - assert(diff*diff < static_cast (2.9E-6) && - "Profile not equal ±2.9E-6"); + if constexpr (std::same_as) { + assert(diff*diff < static_cast (1.7E-5) && + "Profile not equal ±1.7E-5"); + } else { + assert(diff*diff < static_cast (2.9E-6) && + "Profile not equal ±2.9E-6"); + } } } for (size_t i = 6*num_particles/10, ie = 7*num_particles/10; i < ie; i++) { From 774071ce427441328a660dd81f07597c366a1efe Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 25 Jun 2026 18:13:55 -0400 Subject: [PATCH 09/51] jit::use_cuda is a function. --- graph_tests/pic_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 482e321..2e3e1dd 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -88,7 +88,7 @@ template void run_interpolation_test() { const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; - if constexpr (jit::use_cuda) { + if constexpr (jit::use_cuda()) { if constexpr (std::same_as) { assert(diff*diff < static_cast (1.22E-4) && "Profile not equal ±1.22E-4"); @@ -111,7 +111,7 @@ template void run_interpolation_test() { const T x = work.check_value(i, ions[0].x); const T received = work.check_value(i, field); const T diff = func(x) - received; - if constexpr (jit::use_cuda) { + if constexpr (jit::use_cuda()) { if constexpr (std::same_as) { assert(diff*diff < static_cast (3.1E-4) && "Profile not equal ±3.1E-4"); From 5449ea7c6224f22feb17f39c0551bb1dcb7ad2a4 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 1 Jul 2026 22:10:11 -0400 Subject: [PATCH 10/51] Add the ability to interpolate the filtered electric field. Start building main PIC code. --- graph_framework/particle_in_cell.hpp | 663 ++++++++++++++++++++------- graph_pic/xpic.cpp | 276 +++-------- graph_tests/pic_test.cpp | 33 +- 3 files changed, 593 insertions(+), 379 deletions(-) diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index d4bcbec..751a437 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -87,6 +87,8 @@ namespace pic { const T l; /// Velocity const T v; +/// Electron temperature; + const T te; /// Electric field; const T efield; /// Magnetic field; @@ -104,20 +106,60 @@ namespace pic { const T ne) : m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), wpe(std::sqrt(ne*q*q/(m*epsilon0))), - t(1/wpe), l(wpe/c), v(c), efield(m*c/(q*t)), + t(1/wpe), l(wpe/c), v(c), te(m*v*v/kb), efield(m*c/(q*t)), bfield(efield/c) {} }; +//------------------------------------------------------------------------------ +/// @brief Parameter Class +//------------------------------------------------------------------------------ + template + class parameters { + public: +/// Initial magnetic field + const T b0; +/// Geometry + const T a0; +/// Filter Iterations. + const size_t filter_iterations; +/// Smoothing parameters. + const T smoothing; +/// Time step. + const T dt; + +//------------------------------------------------------------------------------ +/// @brief Construct a parameters object. +/// +/// @param[in] b0 Initial magnetic field. +/// @param[in] r1 +/// @param[in] r2 +/// @param[in] filter_iterations Number of times to apply smoothing filter. +/// @param[in] smoothing Smoothing parameter. +/// @param[in] dt Time step. +/// @param[in] norms A @ref pic::characteristics object +//------------------------------------------------------------------------------ + parameters(const T b0, const T r1, const T r2, + const size_t filter_iterations, + const T smoothing, const T dt, + const characteristics &norms) : + b0(b0/norms.bfield), + a0(std::numbers::pi_v*(r2*r2 - r1*r1)/(norms.l*norms.l)), + filter_iterations(filter_iterations), smoothing(smoothing), + dt(dt/norms.t) {} + }; + //------------------------------------------------------------------------------ /// @brief ion class. /// -/// These values need to be initalized using normalized quantities. +/// These values need to be initialized using normalized quantities. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template class ion { public: +/// Atomic number. + const T z; /// Charge const T charge; /// Particle mass @@ -132,6 +174,9 @@ namespace pic { std::array, 3> weights; /// Mesh index graph::shared_leaf indices; +/// Number of real particles + const T num_real; +/// //------------------------------------------------------------------------------ /// @brief Construct an ion object. @@ -139,13 +184,16 @@ namespace pic { /// @param[in] mass Ion mass. /// @param[in] z Ion Z. /// @param[in] num_ions Number of ions. +/// @param[in] num_real Number of real particles. /// @param[in] norms A @ref pic::characteristics object. //------------------------------------------------------------------------------ ion(const T mass, const uint8_t z, const size_t num_ions, + const T num_real, const characteristics &norms) : - charge(z*pic::q/norms.q), mass(mass/norms.m), + z(z), charge(z*pic::q/norms.q), + mass(mass/norms.m), num_real(num_real), x(graph::variable (num_ions, "x")), v_para(graph::variable (num_ions, "v_{||}")), v_perp(graph::variable (num_ions, "v_{\\perp}")), @@ -154,6 +202,78 @@ namespace pic { graph::variable (num_ions, "w_{1}"), graph::variable (num_ions, "w_{2}") }), indices(graph::variable (num_ions, "m_{i}")) {} + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_x() const { + return graph::variable_cast(x); + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + size_t size() const { + return get_x()->size(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for x. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *x_data() const { + return get_x()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_para() const { + return graph::variable_cast(v_para); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the parallel velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_para_data() const { + return get_v_para()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_perp() const { + return graph::variable_cast(v_perp); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the perpendicular velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_perp_data() const { + return graph::variable_cast(v_perp)->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Conversion factor from super particles to real particles. +/// +/// @returns The super to real conversion factor. +//------------------------------------------------------------------------------ + T super_to_real() const { + return num_real/size(); + } }; //------------------------------------------------------------------------------ @@ -163,6 +283,34 @@ namespace pic { //------------------------------------------------------------------------------ template class mesh { + private: +//------------------------------------------------------------------------------ +/// @brief Build y index. +/// +/// @tparam I Mesh index. +/// +/// @param[in] x The x position. +/// @param[in] scale Scale factor. +/// @param[in] iterations Iterations. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_y_index(graph::shared_leaf x, + const T scale, + const size_t iterations=0) const { + auto low = iterations ? build_y_index (x - dx, iterations - 1) : + graph::index_1D(y[I], x, dx, xmin + dx); + auto center = graph::index_1D(y[I], x, dx, xmin); + auto high = iterations ? build_y_index (x + dx, iterations - 1) : + graph::index_1D(y[I], x, dx, xmin + dx); + + const T center_w = static_cast (0.5); + const T side_w = static_cast (0.25); + + auto b = center_w*center + side_w*low + side_w*high; + return (1 - scale)*center + scale*b; + } + public: /// Min x const T xmin; @@ -173,7 +321,16 @@ namespace pic { /// Particle index. graph::shared_leaf index; /// Mesh y values. - graph::shared_leaf y; + std::array, 4> y; +/// Mesh point. + enum offset { +/// Lower index. + low, +/// Center index. + center, +/// Higher index. + high + }; //------------------------------------------------------------------------------ /// @brief Construct a mesh object. @@ -187,7 +344,12 @@ namespace pic { const T x_max, const size_t num, const characteristics &norms) : - y(graph::variable (num, "y_{m}")), + y({ + graph::variable (num, "y^{0}_{m}"), + graph::variable (num, "y^{1}_{m}"), + graph::variable (num, "y^{2}_{m}"), + graph::variable (num, "y^{3}_{m}") + }), index(graph::variable (num, "pi_{m}")), xmin(x_min/norms.l), xmax(x_max/norms.l), dx((xmax - xmin)/(num - 1)) {} @@ -195,26 +357,81 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Build x index. /// -/// @param[in] ion A @ref pic::ion object. +/// @param[in] x The x position. /// @returns The indexed mesh X position. //------------------------------------------------------------------------------ - graph::shared_leaf build_x_index(ion &ion) const { - const size_t s = graph::variable_cast(y)->size(); - const backend::buffer buffer(xmin, dx, s); - return piecewise_1D(buffer, ion.x, dx, xmin); + graph::shared_leaf build_x_index(graph::shared_leaf x) const { + const backend::buffer buffer(xmin, dx, size()); + return graph::piecewise_1D(buffer, x, dx, xmin); } //------------------------------------------------------------------------------ /// @brief Build i index. /// -/// @param[in] ion A @ref pic::ion object. +/// @param[in] x The x position. /// @returns The indexed mesh X position. //------------------------------------------------------------------------------ - graph::shared_leaf build_i_index(ion &ion) const { - const size_t s = graph::variable_cast(y)->size(); + graph::shared_leaf build_i_index(graph::shared_leaf x) const { const backend::buffer buffer(static_cast (0), - static_cast (1), s); - return piecewise_1D(buffer, ion.x, dx, xmin); + static_cast (1), size()); + return graph::piecewise_1D(buffer, x, dx, xmin); + } + +//------------------------------------------------------------------------------ +/// @brief Build y index. +/// +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] params A @ref pic::parameters object. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_y_index(graph::shared_leaf x, + const parameters ¶ms) const { + if constexpr (O == low) { + return build_y_index (x - dx, params.scale, + params.filter_iterations); + } else if constexpr (O == center) { + return build_y_index (x, params.scale, + params.filter_iterations); + } else { + return build_y_index (x + dx, params.scale, + params.filter_iterations); + } + } + +//------------------------------------------------------------------------------ +/// @brief Build dy/dx index. +/// +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] params A @ref pic::parameters object. +/// @returns The indexed mesh Y position. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_dydx_index(graph::shared_leaf x, + const parameters ¶ms) const { + const T two = 2; + if constexpr (O == low) { + auto low = build_y_index (x - two*dx, params.scale, + params.filter_iterations); + auto high = build_y_index (x, params.scale, + params.filter_iterations); + } else if constexpr (O == center) { + auto low = build_y_index (x - dx, params.scale, + params.filter_iterations); + auto high = build_y_index (x + dx, params.scale, + params.filter_iterations); + } else { + auto low = build_y_index (x, params.scale, + params.filter_iterations); + auto high = build_y_index (x + two*dx, params.scale, + params.filter_iterations); + } } //------------------------------------------------------------------------------ @@ -223,9 +440,9 @@ namespace pic { /// @param[in] ion A @ref pic::ion object. /// @returns Expressions for mesh accumulation. //------------------------------------------------------------------------------ - std::array, 2> build_mesh_solve(ion &ion) const { + std::array, 2> build_mesh_solve(const ion &ion) const { auto next_index = index; - auto next_weight = y; + auto next_weight = y[0]; auto kernel_index = graph::index (); auto index_i = graph::index_1D(ion.indices, next_index, @@ -249,39 +466,44 @@ namespace pic { next_weight + index_w2, next_weight); return {next_index, next_weight}; } - }; //------------------------------------------------------------------------------ -/// @brief Build Magnetic field. +/// @brief Get the number of computational ions. /// -/// @tparam T Base type of the calculation. +/// @return The number of particles. +//------------------------------------------------------------------------------ + size_t size() const { + return graph::variable_cast(y[0])->size(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. /// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] norms A @ref pic::characteristics object. -/// @returns The magnetic field expression. +/// @tparam I Mesh index. +/// +/// @return The number of particles. //------------------------------------------------------------------------------ - template - graph::shared_leaf build_magnetic_field(ion ion, - const characteristics &norms) { - return (ion.x*ion.x + static_cast (1))/norms.bfield; - } + template + T *data() const { + return graph::variable_cast(y[I])->data(); + } + }; //------------------------------------------------------------------------------ /// @brief Build interpolation weights. /// /// @tparam T Base type of the calculation. -/// +/// @param[in] x The x position. /// @param[in] mesh Mesh object. -/// @param[in] ion A @ref pic::ion object. /// @returns The interpolated mesh weights. //------------------------------------------------------------------------------ template - std::array, 3> build_weights(mesh &mesh, - ion &ion) { - auto x = mesh.build_x_index(ion) - ion.x; - auto xnorm1 = static_cast (1.5) + (x - mesh.dx)/mesh.dx; - auto xnorm2 = x/mesh.dx; - auto xnorm3 = static_cast (1.5) - (x + mesh.dx)/mesh.dx; + std::array, 3> build_weights(graph::shared_leaf x, + const mesh &mesh) { + auto x_off = mesh.build_x_index(x) - x; + auto xnorm1 = static_cast (1.5) + (x_off - mesh.dx)/mesh.dx; + auto xnorm2 = x_off/mesh.dx; + auto xnorm3 = static_cast (1.5) - (x_off + mesh.dx)/mesh.dx; auto w0 = static_cast (0.5)*xnorm1*xnorm1; auto w1 = static_cast (0.75) - xnorm2*xnorm2; @@ -290,23 +512,242 @@ namespace pic { return {w0, w1, w2}; } +//------------------------------------------------------------------------------ +/// @brief Build initialization. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] state Random state node. +/// @returns Initialized normalized values for x, v||, and v⟂ +//------------------------------------------------------------------------------ + template + std::array,3> build_initialization(const mesh &mesh, + const characteristics &norms, + const graph::shared_random_state state) { +// The mesh is already normalized so position_dist will be a normalized quantity. + auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, + state); + auto phi_dist = graph::uniform_random (static_cast (0.0), + static_cast (2.0)*std::numbers::pi_v, + state); + auto r_dist = graph::uniform_random (static_cast (0.0), + static_cast (1.0), + state); + + auto vpara = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); + + phi_dist = graph::uniform_random (static_cast (0.0), + static_cast (2.0)*std::numbers::pi_v, + state); + r_dist = graph::uniform_random (static_cast (0.0), + static_cast (1.0), + state); + auto vperp1 = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::cos(phi_dist); + auto vperp2 = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); + auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); + + return {position_dist, vpara/norms.v, vperp/norms.v}; + } + +//------------------------------------------------------------------------------ +/// @brief Build reinjected expressions. +/// +/// If the particles leave the mesh, reinitalize them using the same +/// initialization. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] state Random state node. +/// @returns Reinjected values for x, v||, and v⟂ +//------------------------------------------------------------------------------ + template + std::array, 3> build_reinjection(const ion &ion, + const mesh &mesh, + const characteristics &norms, + const graph::shared_random_state state) { + auto resampled = build_initialization(mesh, state); + auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; + + auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); + auto reinject_vpara = graph::if_(is_outside, resampled[1], ion.v_para); + auto reinject_vperp = graph::if_(is_outside, resampled[2], ion.v_perp); + return {reinject_x, reinject_vpara, reinject_vperp}; + } + +//------------------------------------------------------------------------------ +/// @brief Build a magnetic field. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] norms A @ref pic::characteristics object. +/// @returns The expression for the magnetic field. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_magnetic_field(graph::shared_leaf x, + const characteristics &norms) { + return (static_cast (0.1)*x*x + static_cast (0.5))/norms.bfield; + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the density. +/// +/// @tparam T Base type of the calculation. +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The expression for the density. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_density(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto y = mesh.template build_y_index (x, params); + +// Compression factor. + auto cf = build_magnetic_field(x, norms)/params.b0; +// Scale factor. + const T sf = ion.super_to_real()/(params.a0*mesh.dx); + + return ion.z*y*cf*sf; + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the density gradient. +/// +/// @tparam T Base type of the calculation. +/// @tparam I Mesh index. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The expression for the density. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_density_gradient(graph::shared_leaf x, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto y = mesh.template build_dydx_index (x, params); + +// Compression factor. + auto cf = build_magnetic_field(x, norms)/params.b0; +// Scale factor. + const T sf = ion.super_to_real()/(params.a0*mesh.dx); + + return ion.z*y*cf*sf; + } + +//------------------------------------------------------------------------------ +/// @brief Build Expressions for Electron temperature. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] norms A @ref pic::characteristics object. +/// @returns The expressions for the electron temperature. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_electron_temperature(graph::shared_leaf x, + const characteristics &norms) { + return graph::one ()/norms.te; + } + +//------------------------------------------------------------------------------ +/// @brief Build expressions for the electric field. +/// +/// @tparam T Base type of the calculation. +/// @tparam O Mesh offset. +/// +/// @param[in] x The x position. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +//------------------------------------------------------------------------------ + template::offset O=mesh::center> + graph::shared_leaf build_electric_efield(graph::shared_leaf x, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto n0 = build_density (x, mesh, norms, params); + auto n1 = build_density (x, mesh, norms, params); + auto n2 = build_density (x, mesh, norms, params); + auto n3 = build_density (x, mesh, norms, params); + + auto dn0dx = build_density_gradient (x, mesh, norms, params); + auto dn1dx = build_density_gradient (x, mesh, norms, params); + auto dn2dx = build_density_gradient (x, mesh, norms, params); + auto dn3dx = build_density_gradient (x, mesh, norms, params); + + auto n = (n0 + n1 + n2 + n3)/static_cast (4); + auto dndx = (dn0dx + dn1dx + dn2dx + dn3dx)/static_cast (4); + + auto te = build_magnetic_field(x, norms); + auto pressure = te*n/q; + + return graph::none ()/n*(dndx*te/q + pressure->df(x)); + } + //------------------------------------------------------------------------------ /// @brief Build interpolation expression. /// /// @tparam T Base type of the calculation. /// +/// @param[in] x The x position. /// @param[in] mesh Mesh object. -/// @param[in] ion A @ref pic::ion object. /// @returns The interpolated mesh quantity. //------------------------------------------------------------------------------ template - graph::shared_leaf build_interpolation(mesh &mesh, - ion &ion) { - auto weights = build_weights (mesh, ion); + graph::shared_leaf build_interpolation(graph::shared_leaf x, + mesh &mesh) { + auto weights = build_weights (x, mesh); - auto ymesh0 = graph::index_1D(mesh.y, ion.x - mesh.dx, mesh.dx, mesh.xmin); - auto ymesh1 = graph::index_1D(mesh.y, ion.x, mesh.dx, mesh.xmin); - auto ymesh2 = graph::index_1D(mesh.y, ion.x + mesh.dx, mesh.dx, mesh.xmin); + auto ymesh0 = graph::index_1D(mesh.y[0], x - mesh.dx, mesh.dx, mesh.xmin); + auto ymesh1 = graph::index_1D(mesh.y[0], x, mesh.dx, mesh.xmin); + auto ymesh2 = graph::index_1D(mesh.y[0], x + mesh.dx, mesh.dx, mesh.xmin); + + return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x The x position. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @returns The interpolated mesh quantity. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_interpolate_efield(graph::shared_leaf x, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { + auto weights = build_weights (x, mesh); + + auto ymesh0 = build_electric_efield::low> (x, mesh, norms, params); + auto ymesh1 = build_electric_efield::center> (x, mesh, norms, params); + auto ymesh2 = build_electric_efield::high> (x, mesh, norms, params); return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; } @@ -316,26 +757,26 @@ namespace pic { /// /// @tparam T Base type of the calculation. /// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] z Runga Kutta substep. -/// @param[in] dt Normalized time step. -/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] z Runga Kutta substep. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. /// @returns the Forces on the particles. //------------------------------------------------------------------------------ template - std::array, 3> build_F_expressions(ion &ion, - mesh &mesh, - const std::array, 3> z, - const T dt, - const characteristics &norms) { + std::array, 3> build_F_expressions(const std::array, 3> z, + const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { auto bfield = build_magnetic_field (z[0], norms); - auto efield = build_interpolation (mesh, ion.x); + auto efield = build_interpolate_efield (z[0], mesh, norms, params); auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; return { - z[1]*dt, - temp*dt, - (ion.charge/ion.mass*efield - temp)*dt + z[1]*params.dt, + temp*params.dt, + (ion.charge/ion.mass*efield - temp)*params.dt }; } @@ -344,22 +785,20 @@ namespace pic { /// /// @tparam T Base type of the calculation. /// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] dt Normalized time step. -/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. /// @returns Step update expressions for x, v_para, and x_perp. //------------------------------------------------------------------------------ template - std::array, 3> build_rk4_step(ion &ion, - mesh &mesh, - const T dt, - const characteristics &norms) { - std::array, 3> ion_norm = ion.normalize(norms); - + std::array, 3> build_rk4_step(const ion &ion, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms) { // Step 1 std::array, 3> Z1{ion.x, ion.v_para, ion.v_perp}; - std::array, 3> dZ1(build_F_expressions (ion, mesh, Z1, dt, norms)); + std::array, 3> dZ1(build_F_expressions (Z1, ion, mesh, norms, params)); // Step 2 std::array, 3> Z2{ @@ -367,7 +806,7 @@ namespace pic { Z1[1] + dZ1[1]/static_cast (2), Z1[2] + dZ1[2]/static_cast (2) }; - std::array, 3> dZ2(build_F_expressions (ion, mesh, Z2, dt, norms)); + std::array, 3> dZ2(build_F_expressions (Z2, ion, mesh, norms, params)); // Step 3 std::array, 3> Z3{ @@ -375,7 +814,7 @@ namespace pic { Z1[1] + dZ2[1]/static_cast (2), Z1[2] + dZ2[2]/static_cast (2) }; - std::array, 3> dZ3(build_F_expressions (ion, mesh, Z3, dt, norms)); + std::array, 3> dZ3(build_F_expressions (Z3, ion, mesh, norms, params)); // Step 4 std::array, 3> Z4{ @@ -383,7 +822,7 @@ namespace pic { Z1[1] + dZ3[1], Z1[2] + dZ3[2] }; - std::array, 3> dZ4(build_F_expressions (ion, mesh, Z4, dt, norms)); + std::array, 3> dZ4(build_F_expressions (Z4, ion, mesh, norms, params)); // Rk4 Solution return { @@ -392,88 +831,6 @@ namespace pic { Z1[2] + (dZ1[2] + static_cast (2)*(dZ2[2] + dZ3[2]) + dZ4[2])/static_cast (6) }; } - -//------------------------------------------------------------------------------ -/// @brief Build magnetic moment. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] mesh A @ref pic::mesh object. -/// @returns The magnetic moment. -//------------------------------------------------------------------------------ - template - graph::shared_leaf build_magnetic_moment(ion &ion, mesh &mesh) { - auto efield = build_interpolation (mesh, ion.x); - return 0.5*ion.mass*ion.v_perp*ion.v_perp/efield; - } - -//------------------------------------------------------------------------------ -/// @brief Build initializtion. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] norms A @ref pic::characteristics object. -/// @param[in] state Random state node. -/// @returns Initialized normalized values for x, v||, and v⟂ -//------------------------------------------------------------------------------ - template - std::array,3> build_initialization(mesh &mesh, - const characteristics &norms, - graph::shared_random_state state) { -// The mesh is already normalized so position_dist will be a normalized quantity. - auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, - state); - auto phi_dist = graph::uniform_random (static_cast (0.0), - static_cast (2.0)*std::numbers::pi_v, - state); - auto r_dist = graph::uniform_random (static_cast (0.0), - static_cast (1.0), - state); - - auto vpara = graph::sqrt(-kb*graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); - - phi_dist = graph::uniform_random (static_cast (0.0), - static_cast (2.0)*std::numbers::pi_v, - state); - r_dist = graph::uniform_random (static_cast (0.0), - static_cast (1.0), - state); - auto vperp1 = graph::sqrt(-pic::kb*graph::log(static_cast (1) - r_dist))*graph::cos(phi_dist); - auto vperp2 = graph::sqrt(-pic::kb*graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); - auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); - - return {position_dist, vpara/norms.v, vperp/norms.v}; - } - -//------------------------------------------------------------------------------ -/// @brief Build reinjected expressions. -/// -/// If the particles leave the mesh, reinitalize them using the same -/// initialization. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] norms A @ref pic::characteristics object. -/// @param[in] state Random state node. -/// @returns Reinjected values for x, v||, and v⟂ -//------------------------------------------------------------------------------ - template - std::array, 3> build_reinjection(ion &ion, - mesh &mesh, - const characteristics &norms, - graph::shared_random_state state) { - auto resampled = build_initialization(mesh, state); - auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; - - auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); - auto reinject_vpara = graph::if_(is_outside, resampled[1], ion.v_para); - auto reinject_vperp = graph::if_(is_outside, resampled[2], ion.v_perp); - return {reinject_x, reinject_vpara, reinject_vperp}; - } } #endif /* particle_in_cell_h */ diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 06976d5..966126c 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -8,34 +8,6 @@ #include "../graph_framework/graph_framework.hpp" -//------------------------------------------------------------------------------ -/// @brief Build density. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] x The particle position. -//------------------------------------------------------------------------------ -template -graph::shared_leaf build_density(graph::shared_leaf x) { - return graph::exp(x*x/static_cast (-0.0001)); -} - -//------------------------------------------------------------------------------ -/// @brief Build parallel electric field. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] x The particle position. -//------------------------------------------------------------------------------ -template -graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { - const T te = 1; - const T q = 1;//1.602176634E-19; - auto n = build_density (x); - auto pe = n*te; - return static_cast (-1)/(q*n)*pe->df(x); -} - //------------------------------------------------------------------------------ /// @brief Pic code. /// @@ -43,203 +15,85 @@ graph::shared_leaf build_parallel_electric_field(graph::shared_leaf x) { //------------------------------------------------------------------------------ template void run_pic() { -#if 0 -// Constants +// Sizes const size_t num_particles = 1000000; - const size_t num_grid = 200; - const uint8_t Z = 1; + const size_t num_grid = 100; + const size_t num_ions = 1; + + const pic::characteristics norms({ + pic::m_hydrogen + }, {1}, static_cast (2.5E19)); + + std::array ion_masses{pic::m_hydrogen}; + std::array ion_zs{1}; + std::array density_fraction{1}; + + const T lmin = static_cast (-3.0); + const T lmax = static_cast (3.0); + const T ne0 = 0.4E18; + const T b0 = 0.050072; + const T r1 = 0.0; + const T r2 = 0.5; + const T a0 = std::numbers::pi_v*(r2*r2 - r1*r1); + const T ds = (lmax - lmin)/static_cast (num_grid - 1); -// Characteristic factors - const std::vector ion_masses{pic::m_hydrogen}; - const std::vector ion_zs{1}; - - const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); std::vector> ions; - for(size_t i = 0, ie = ion_masses.size(); i < ie; i++) { - ions.emplace_back(ion_masses[i], ion_zs[i], num_particles); - } - pic::mesh mesh(-0.25, 0.25, num_grid); - -// Particle initalization. - auto x = graph::variable (num_particles, "x"); - auto vpara = graph::variable (num_particles, "v_{||}"); - auto vperp = graph::variable (num_particles, "v_{\\perp}"); - - { - std::uniform_real_distribution position_dist(-0.25, 0.25); - std::uniform_real_distribution phi_dist(0.0, 2.0*std::numbers::pi_v); - std::uniform_real_distribution r_dist(0.0, 1.0); - - std::random_device rand_d; - std::mt19937_64 engine(rand_d()); - - backend::buffer pos_buffer(num_particles); - backend::buffer vpara_buffer(num_particles); - backend::buffer vperp_buffer(num_particles); - - for (size_t i = 0; i < num_particles; i++) { - pos_buffer[i] = position_dist(engine); - T phi = phi_dist(engine); - T r = r_dist(engine); - vpara_buffer[i] = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::sin(phi)); - phi = phi_dist(engine); - r = r_dist(engine); - const T vperp1 = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::cos(phi)); - const T vperp2 = std::fmod(std::sqrt(-kb*std::log(1 - r)), std::sin(phi)); - vperp_buffer[i] = std::sqrt(vperp1*vperp1 + vperp2*vperp2); + for(size_t i = 0; i < num_ions; i++) { + T num_real = 0; + for (size_t i = 0; i < num_grid; i++) { + const T x = ds*i + lmin; + const T b = static_cast (0.1)*x*x + static_cast (0.5); + const T a = a0*b0/b; + num_real += ne0*density_fraction[0]*a*ds; } - x->set(pos_buffer); - vpara->set(vpara_buffer); - vperp->set(vperp_buffer); + ions.emplace_back(ion_masses[i], ion_zs[i], num_particles, + num_real, norms); } -// Electron initialization. - auto te = graph::variable (num_grid, static_cast (1.0), "t_{e}"); + const T gyro_period = ion_zs[0]*pic::q*b0/ion_masses[0]; + const T dtc = 0.25; + const pic::parameters params(b0, r1, r2, 3, 1.0E-4, + dtc*gyro_period, norms); -// Electric field. - auto efield = graph::variable (num_grid, "E_{||}"); - auto meshx = graph::variable (num_grid, "x_{m}"); - { - backend::buffer pos_buffer(num_grid); - const T dx = (0.25 - -0.25)/(num_particles - 1); - for (size_t i = 0; i < num_particles; i++) { - pos_buffer[i] = dx*i - 0.25; - } - meshx->set(pos_buffer); - } - -// Time step - const T gyro_frequency = q*1*1.2/2; - const T gyro_period = 2*std::numbers::pi_v/gyro_frequency; - T dt = 0.25*gyro_period; - - const size_t timeIterations = std::ceil(10/0.25); - const size_t outputCadence = std::ceil(600/0.25); - -// Normalize - dt /= tchar; - vpara = vpara/c; - vperp = vperp/c; - x = x*wpechar/c; - - pic::ion ion(q/qchar, m_hydrogen/mchar, x, vpara, vperp); - pic::mesh mesh(meshx, efield); + pic::mesh mesh(lmin, lmax, num_grid, norms); - std::array, 3> rk4_step(pic::build_rk4_step(ion, mesh, bchar, dt)); - auto mu = pic::build_magnetic_moment (ion, mesh); - -// Build the field solver; - - auto epara = graph::variable (num_grid, "e||"); - auto n = graph::variable (num_grid, "n"); - auto grid_position = graph::variable (num_grid, "x_i"); - auto particle_index = graph::variable (num_grid, "i"); - - const T scale = 2.0/999.0; - const T offset = -1.0; - backend::buffer coe(num_grid); - for (size_t i = 0; i < num_grid; i++) { - coe[i] = scale*i + offset; - } - grid_position->set(coe); - - auto x1 = dt*vpara; - auto vpara1 = -q/m_electron*graph::index_1D(epara, x, scale, offset); - - auto x2 = dt*(vpara + vpara1/2.0); - auto vpara2 = -q/m_electron*graph::index_1D(epara, x + x1/2.0, scale, offset); - - auto x3 = dt*(vpara + vpara2/2.0); - auto vpara3 = -q/m_electron*graph::index_1D(epara, x + x2/2.0, scale, offset); - - auto x4 = dt*(vpara + vpara3); - auto vpara4 = -q/m_electron*graph::index_1D(epara, x + x3, scale, offset); - - auto x_next = x + (x1 + static_cast (2)*(x2 + x3) + x4)/static_cast (6); - auto vpara_next = vpara + (vpara1 + static_cast (2)*(vpara2 + vpara3) + vpara4)/static_cast (6); - - auto next_index = particle_index; - auto next_epara = epara; - auto next_n = n; - - const size_t batch = 1000; -// Unroll the loop - for (size_t i = 0; i < batch; i++) { - auto indexed_particle = graph::index_1D(x, next_index, - static_cast (1), - static_cast (0)); - next_index = next_index + static_cast (1); - next_epara = next_epara - + build_parallel_electric_field (indexed_particle - grid_position); - next_n = next_n + build_density(indexed_particle - grid_position); - } + auto state = graph::random_state (jit::context::random_state_size, 0); workflow::manager work(0); - work.add_item({ - graph::variable_cast(particle_index), - graph::variable_cast(epara), - graph::variable_cast(n) - }, {}, { - {graph::zero (), graph::variable_cast(particle_index)}, - {graph::zero (), graph::variable_cast(epara)}, - {graph::zero (), graph::variable_cast(n)} - }, NULL, "Index_reset", num_grid); - work.add_loop_item({ - graph::variable_cast(epara), - graph::variable_cast(n), - graph::variable_cast(grid_position), - graph::variable_cast(particle_index), - graph::variable_cast(x) - }, {}, { - {next_epara, graph::variable_cast(epara)}, - {next_index, graph::variable_cast(particle_index)}, - {next_n, graph::variable_cast(n)} - }, NULL, "Compute_efield", num_grid, num_particles/batch); - work.add_item({ - graph::variable_cast(x), - graph::variable_cast(vpara), - graph::variable_cast(epara) - }, {}, { - {x_next, graph::variable_cast(x)}, - {vpara_next, graph::variable_cast(vpara)} - }, NULL, "Particle_Push", num_particles); + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + + auto ion_inits = pic::build_initialization (mesh, norms, + graph::random_state_cast(state)); + work.add_preitem({ + ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() + }, {}, { + {ion_inits[0], ions[i].get_x()}, + {ion_inits[1], ions[i].get_v_para()}, + {ion_inits[2], ions[i].get_v_perp()} + }, graph::random_state_cast(state), + "pre_initization_" + ion_tag, num_particles); + + auto mesh_i = mesh.build_i_index(ions[i].x); + auto weights = pic::build_weights (ions[i].x, mesh); + work.add_preitem({ + ions[i].get_x(), + graph::variable_cast(ions[i].weights[0]), + graph::variable_cast(ions[i].weights[1]), + graph::variable_cast(ions[i].weights[2]), + graph::variable_cast(ions[i].indices) + }, {}, { + {weights[0], graph::variable_cast(ions[i].weights[0])}, + {weights[1], graph::variable_cast(ions[i].weights[1])}, + {weights[2], graph::variable_cast(ions[i].weights[2])}, + {mesh_i, graph::variable_cast(ions[i].indices)} + }, NULL, "pre_compute_weights_" + ion_tag, num_particles); + } work.compile(); - - output::result_file particles_file("pic_particles.nc", num_particles); - output::data_set p_dataset(particles_file); - - p_dataset.create_variable(particles_file, "x", x, work.get_context()); - p_dataset.create_variable(particles_file, "vpara", vpara, work.get_context()); + work.pre_run(); - particles_file.end_define_mode(); - - output::result_file fields_file("pic_fields.nc", num_grid); - output::data_set f_dataset(fields_file); - - f_dataset.create_variable(fields_file, "epara", epara, work.get_context()); - f_dataset.create_variable(fields_file, "n", n, work.get_context()); - - fields_file.end_define_mode(); - std::thread sync_particles([]{}); - std::thread sync_fields([]{}); - - const size_t num_steps = 1000; - for (size_t i = 0; i < num_steps; i++) { - sync_particles.join(); - sync_fields.join(); - work.run(); - sync_particles = std::thread([&particles_file, &p_dataset] () -> void { - p_dataset.write(particles_file); - }); - sync_fields = std::thread([&fields_file, &f_dataset] () -> void { - f_dataset.write(fields_file); - }); - } work.wait(); - sync_particles.join(); - sync_fields.join(); -#endif } //------------------------------------------------------------------------------ diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 2e3e1dd..f2515e0 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -27,7 +27,8 @@ template void run_interpolation_test() { const std::vector ion_zs{1}; const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); - std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], num_particles, norms)}; + std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], + num_particles, 0, norms)}; pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); std::function func([&norms](const T x) -> T { @@ -35,21 +36,21 @@ template void run_interpolation_test() { }); for (size_t i = 0; i < num_mesh; i++) { - graph::variable_cast(mesh.y)->data()[i] = func(mesh.dx*i + mesh.xmin); + mesh.data()[i] = func(mesh.dx*i + mesh.xmin); } const T dxp = (mesh.xmax - mesh.xmin)/(num_particles - 1); for (size_t i = 0; i < num_particles; i++) { - graph::variable_cast(ions[0].x)->data()[i] = dxp*i + mesh.xmin; + ions[0].x_data()[i] = dxp*i + mesh.xmin; } - auto weights = pic::build_weights (mesh, ions[0]); - auto field = pic::build_interpolation (mesh, ions[0]); + auto weights = pic::build_weights (ions[0].x, mesh); + auto field = pic::build_interpolation (ions[0].x, mesh); auto weight = weights[0] + weights[1] + weights[2]; workflow::manager work(0); work.add_item({ - graph::variable_cast(mesh.y), + graph::variable_cast(mesh.y[0]), graph::variable_cast(ions[0].x) }, { weight, @@ -61,8 +62,8 @@ template void run_interpolation_test() { // The weights should sum to 1. for (size_t i = 0; i < num_particles; i++) { - const T recieved = work.check_value(i, weight); - const T diff = static_cast (1) - recieved; + const T received = work.check_value(i, weight); + const T diff = static_cast (1) - received; if constexpr (std::same_as) { assert(diff*diff < static_cast (4.7E-12) && "Weight not equal to 1±4.7E-12"); @@ -234,7 +235,9 @@ template void run_field_solve_test() { const std::vector ion_zs{1}; const pic::characteristics norms(ion_masses, ion_zs, static_cast (2.5E19)); - std::vector> ions{pic::ion (ion_masses[0], ion_zs[0], num_particles, norms)}; + std::vector> ions{ + pic::ion (ion_masses[0], ion_zs[0], num_particles, 0, norms) + }; pic::mesh mesh(-3.0*norms.l, 3.0*norms.l, num_mesh, norms); // Initialize particle positions. @@ -262,14 +265,14 @@ template void run_field_solve_test() { } } - auto weights = pic::build_weights (mesh, ions[0]); - auto mesh_i = mesh.build_i_index(ions[0]); + auto weights = pic::build_weights (ions[0].x, mesh); + auto mesh_i = mesh.build_i_index(ions[0].x); auto mesh_solve = mesh.build_mesh_solve(ions[0]); workflow::manager work(0); work.add_zero_item({ graph::variable_cast(mesh.index), - graph::variable_cast(mesh.y) + graph::variable_cast(mesh.y[0]) }); work.add_item({ graph::variable_cast(ions[0].x), @@ -289,10 +292,10 @@ template void run_field_solve_test() { graph::variable_cast(ions[0].weights[1]), graph::variable_cast(ions[0].weights[2]), graph::variable_cast(mesh.index), - graph::variable_cast(mesh.y) + graph::variable_cast(mesh.y[0]) }, {}, { {mesh_solve[0], graph::variable_cast(mesh.index)}, - {mesh_solve[1], graph::variable_cast(mesh.y)} + {mesh_solve[1], graph::variable_cast(mesh.y[0])} }, NULL, "sum_weights", num_mesh, num_particles); work.compile(); @@ -303,7 +306,7 @@ template void run_field_solve_test() { t_run.print(); for (size_t i = 0; i < num_mesh; i++) { - const T recieved = work.check_value(i, mesh.y); + const T recieved = work.check_value(i, mesh.y[0]); const T error = std::abs((counts[i] - recieved)/counts[i]); if constexpr (std::is_same_v) { assert(error < 0.155 && "Error outside tolarance range."); From f6a4e2b320aa954794447f165723381491d8e9d7 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 6 Jul 2026 14:10:34 -0400 Subject: [PATCH 11/51] Assemble pic code. Currently does not store any results but has all the steps for initalization, particle push, field solve, and particle reinjection (uniform x dist only). --- .../xcshareddata/xcschemes/pic_test.xcscheme | 80 +++++++++++++ graph_framework/cpu_context.hpp | 37 +++++- graph_framework/cuda_context.hpp | 57 ++++++++- graph_framework/jit.hpp | 18 ++- graph_framework/metal_context.hpp | 46 ++++++- graph_framework/node.hpp | 4 + graph_framework/particle_in_cell.hpp | 51 ++++---- graph_framework/workflow.hpp | 84 ++++++++++++- graph_pic/xpic.cpp | 112 ++++++++++++++++++ 9 files changed, 457 insertions(+), 32 deletions(-) create mode 100644 graph_framework.xcodeproj/xcshareddata/xcschemes/pic_test.xcscheme diff --git a/graph_framework.xcodeproj/xcshareddata/xcschemes/pic_test.xcscheme b/graph_framework.xcodeproj/xcshareddata/xcschemes/pic_test.xcscheme new file mode 100644 index 0000000..c4918ca --- /dev/null +++ b/graph_framework.xcodeproj/xcshareddata/xcschemes/pic_test.xcscheme @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 774a073..8ef6960 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -322,7 +322,7 @@ namespace gpu { } //------------------------------------------------------------------------------ -/// @brief Create buffer that will be memset to zero. +/// @brief Create kernel call that will be memset a buffer to zero. /// /// @param[in] inputs Input nodes of the kernel. /// @returns A lambda function to run the kernel. @@ -348,6 +348,41 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector sources; + std::vector destinations; + std::vector sizes; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + std::vector arg(in->size()); + memcpy(arg.data(), in->data(), in->size()*sizeof(T)); + kernel_arguments[in.get()] = arg; + } + destinations.push_back(kernel_arguments[in.get()].data()); + sizes.push_back(in->size()*sizeof(T)); + + if (!kernel_arguments.contains(out.get())) { + std::vector arg(out->size()); + memcpy(arg.data(), out->data(), out->size()*sizeof(T)); + kernel_arguments[out.get()] = arg; + } + sources.push_back(kernel_arguments[out.get()].data()); + } + + return [sources, destinations, sizes] () mutable { + for (size_t i = 0, ie = sources.size(); i < ie; i++) { + std::memcpy(destinations[i], sources[i], sizes[i]); + } + }; + } + //------------------------------------------------------------------------------ /// @brief Hold the current thread until the command buffer has completed. /// diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 7448992..b1aed55 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -575,13 +575,13 @@ namespace gpu { } //------------------------------------------------------------------------------ -/// @brief Create buffer that will be memset to zero. +/// @brief Create kernel call that will be memset a buffer to zero. /// /// @param[in] inputs Input nodes of the kernel. /// @returns A lambda function to run the kernel. //------------------------------------------------------------------------------ std::function create_zero_call(graph::input_nodes &inputs) { - std::vector buffers; + std::vector buffers; for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { kernel_arguments.try_emplace(input.get()); @@ -593,13 +593,62 @@ namespace gpu { input->data(), input->size()*sizeof(T)), "cuMemcpyHtoD"); - buffers.push_back(kernel_arguments[input.get()]); } buffers.push_back(kernel_arguments[input.get()]); } return [this, buffers] () mutable { - for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + for (CUdeviceptr &buffer : buffers) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); + check_error_async(cuMemsetD8Async(buffer, 0, size, stream), + "cuMemsetD8Async"); + } + }; + } + +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector sources; + std::vector destinations; + std::vector sizes; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + kernel_arguments.try_emplace(in.get()); + check_error(cuMemAllocManaged(&kernel_arguments[in.get()], + in->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[in.get()], + in->data(), + in->size()*sizeof(T)), + "cuMemcpyHtoD"); + } + destinations.push_back(kernel_arguments[in.get()]); + + if (!kernel_arguments.contains(out.get())) { + kernel_arguments.try_emplace(out.get()); + check_error(cuMemAllocManaged(&kernel_arguments[out.get()], + out->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[out.get()], + out->data(), + out->size()*sizeof(T)), + "cuMemcpyHtoD"); + } + sources.push_back(kernel_arguments[out.get()]); + } + + return [this, sources, destinations] () mutable { + for (size_t i = 0, ie = sources.size(); i < ie; i++) { size_t size; check_error(cuMemGetAddressRange(NULL, &size, buffers[i]), "cuMemGetAddressRange"); diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index 2f10fc6..c6d3672 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -187,7 +187,9 @@ namespace jit { // Delete the registers so that they can be used again in other kernels. std::vector removed_elements; for (auto &[key, value] : registers) { - if (value[0] == 'r') { + if (value[0] == 'r' || + value[0] == 'l' || + value[0] == 'i') { removed_elements.push_back(key); } } @@ -268,7 +270,9 @@ namespace jit { // Delete the registers so that they can be used again in other kernels. std::vector removed_elements; for (auto &[key, value] : registers) { - if (value[0] == 'r') { + if (value[0] == 'r' || + value[0] == 'l' || + value[0] == 'i') { removed_elements.push_back(key); } } @@ -297,6 +301,16 @@ namespace jit { return gpu_context.create_zero_call(inputs); } +//------------------------------------------------------------------------------ +/// @brief Add copy. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes setters) { + return gpu_context.create_copy_call(setters); + } + //------------------------------------------------------------------------------ /// @brief Print the kernel source. //------------------------------------------------------------------------------ diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 8d8c105..08c7f60 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -349,7 +349,7 @@ namespace gpu { } //------------------------------------------------------------------------------ -/// @brief Create buffer that will be memset to zero. +/// @brief Create kernel call that will be memset a buffer to zero. /// /// @param[in] inputs Input nodes of the kernel. /// @returns A lambda function to run the kernel. @@ -361,7 +361,6 @@ namespace gpu { kernel_arguments[input.get()] = [device newBufferWithBytes:input->data() length:input->size()*sizeof(float) options:MTLResourceStorageModeShared]; - buffers.push_back(kernel_arguments[input.get()]); } buffers.push_back(kernel_arguments[input.get()]); } @@ -381,6 +380,49 @@ namespace gpu { }; } +//------------------------------------------------------------------------------ +/// @brief Create kernel call that will to copy one buffer to another. +/// +/// @param[in] setters Input variables of the kernel. +/// @returns A lambda function to run the kernel. +//------------------------------------------------------------------------------ + std::function create_copy_call(graph::copy_nodes &setters) { + std::vector> sources; + std::vector> destinations; + + for (auto &[out, in] : setters) { + if (!kernel_arguments.contains(in.get())) { + kernel_arguments[in.get()] = [device newBufferWithBytes:in->data() + length:in->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + } + destinations.push_back(kernel_arguments[in.get()]); + + if (!kernel_arguments.contains(out.get())) { + kernel_arguments[out.get()] = [device newBufferWithBytes:out->data() + length:out->size()*sizeof(float) + options:MTLResourceStorageModeShared]; + } + sources.push_back(kernel_arguments[out.get()]); + } + + return [this, sources, destinations] () mutable { + command_buffer = [queue commandBuffer]; + id encoder = [command_buffer blitCommandEncoder]; + + for (size_t i = 0, ie = sources.size(); i < ie; i++) { + [encoder copyFromBuffer:sources[i] + sourceOffset:0 + toBuffer:destinations[i] + destinationOffset:0 + size:sources[i].length]; + } + [encoder endEncoding]; + + [command_buffer commit]; + }; + } + //------------------------------------------------------------------------------ /// @brief Get the compile options. //------------------------------------------------------------------------------ diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 8d256a4..7222a6b 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -1949,6 +1949,10 @@ namespace graph { template using map_nodes = std::vector, shared_variable>>; +/// Convenience type alias for copying buffers. + template + using copy_nodes = std::vector, + shared_variable>>; //------------------------------------------------------------------------------ /// @brief Cast to a variable node. diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 751a437..6724ef9 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -391,13 +391,13 @@ namespace pic { graph::shared_leaf build_y_index(graph::shared_leaf x, const parameters ¶ms) const { if constexpr (O == low) { - return build_y_index (x - dx, params.scale, + return build_y_index (x - dx, params.smoothing, params.filter_iterations); } else if constexpr (O == center) { - return build_y_index (x, params.scale, + return build_y_index (x, params.smoothing, params.filter_iterations); } else { - return build_y_index (x + dx, params.scale, + return build_y_index (x + dx, params.smoothing, params.filter_iterations); } } @@ -417,20 +417,23 @@ namespace pic { const parameters ¶ms) const { const T two = 2; if constexpr (O == low) { - auto low = build_y_index (x - two*dx, params.scale, + auto low = build_y_index (x - two*dx, params.smoothing, params.filter_iterations); - auto high = build_y_index (x, params.scale, + auto high = build_y_index (x, params.smoothing, params.filter_iterations); + return (high - low)/two; } else if constexpr (O == center) { - auto low = build_y_index (x - dx, params.scale, + auto low = build_y_index (x - dx, params.smoothing, params.filter_iterations); - auto high = build_y_index (x + dx, params.scale, + auto high = build_y_index (x + dx, params.smoothing, params.filter_iterations); + return (high - low)/two; } else { - auto low = build_y_index (x, params.scale, + auto low = build_y_index (x, params.smoothing, params.filter_iterations); - auto high = build_y_index (x + two*dx, params.scale, + auto high = build_y_index (x + two*dx, params.smoothing, params.filter_iterations); + return (high - low)/two; } } @@ -570,7 +573,7 @@ namespace pic { const mesh &mesh, const characteristics &norms, const graph::shared_random_state state) { - auto resampled = build_initialization(mesh, state); + auto resampled = build_initialization(mesh, norms, state); auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); @@ -678,24 +681,26 @@ namespace pic { /// @tparam O Mesh offset. /// /// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. /// @param[in] mesh A @ref pic::mesh object. /// @param[in] norms A @ref pic::characteristics object. /// @param[in] params A @ref pic::parameters object. //------------------------------------------------------------------------------ template::offset O=mesh::center> graph::shared_leaf build_electric_efield(graph::shared_leaf x, + const ion &ion, const mesh &mesh, const characteristics &norms, const parameters ¶ms) { - auto n0 = build_density (x, mesh, norms, params); - auto n1 = build_density (x, mesh, norms, params); - auto n2 = build_density (x, mesh, norms, params); - auto n3 = build_density (x, mesh, norms, params); + auto n0 = build_density (x, ion, mesh, norms, params); + auto n1 = build_density (x, ion, mesh, norms, params); + auto n2 = build_density (x, ion, mesh, norms, params); + auto n3 = build_density (x, ion, mesh, norms, params); - auto dn0dx = build_density_gradient (x, mesh, norms, params); - auto dn1dx = build_density_gradient (x, mesh, norms, params); - auto dn2dx = build_density_gradient (x, mesh, norms, params); - auto dn3dx = build_density_gradient (x, mesh, norms, params); + auto dn0dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn1dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn2dx = build_density_gradient (x, ion, mesh, norms, params); + auto dn3dx = build_density_gradient (x, ion, mesh, norms, params); auto n = (n0 + n1 + n2 + n3)/static_cast (4); auto dndx = (dn0dx + dn1dx + dn2dx + dn3dx)/static_cast (4); @@ -733,6 +738,7 @@ namespace pic { /// @tparam T Base type of the calculation. /// /// @param[in] x The x position. +/// @param[in] ion A @ref pic::ion object. /// @param[in] mesh A @ref pic::mesh object. /// @param[in] norms A @ref pic::characteristics object. /// @param[in] params A @ref pic::parameters object. @@ -740,14 +746,15 @@ namespace pic { //------------------------------------------------------------------------------ template graph::shared_leaf build_interpolate_efield(graph::shared_leaf x, + const ion &ion, const mesh &mesh, const characteristics &norms, const parameters ¶ms) { auto weights = build_weights (x, mesh); - auto ymesh0 = build_electric_efield::low> (x, mesh, norms, params); - auto ymesh1 = build_electric_efield::center> (x, mesh, norms, params); - auto ymesh2 = build_electric_efield::high> (x, mesh, norms, params); + auto ymesh0 = build_electric_efield::low> (x, ion, mesh, norms, params); + auto ymesh1 = build_electric_efield::center> (x, ion, mesh, norms, params); + auto ymesh2 = build_electric_efield::high> (x, ion, mesh, norms, params); return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; } @@ -771,7 +778,7 @@ namespace pic { const characteristics &norms, const parameters ¶ms) { auto bfield = build_magnetic_field (z[0], norms); - auto efield = build_interpolate_efield (z[0], mesh, norms, params); + auto efield = build_interpolate_efield (z[0], ion, mesh, norms, params); auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; return { z[1]*params.dt, diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index fa3f40a..d4f794d 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -74,6 +74,46 @@ namespace workflow { } }; +//------------------------------------------------------------------------------ +/// @brief Copy one buffer item to another. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class copy_item : public item { + protected: +/// Kernel function. + std::function kernel; +/// Input nodes. + graph::copy_nodes maps; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] maps Input variables to copy. +//------------------------------------------------------------------------------ + copy_item(graph::copy_nodes maps) : + maps(maps) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.create_copy_call(maps); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + //------------------------------------------------------------------------------ /// @brief Class representing a work item. /// @@ -350,6 +390,39 @@ namespace workflow { preitems.push_back(std::make_unique> (in)); } +//------------------------------------------------------------------------------ +/// @brief Add a pre copy item. +/// +/// @param[in] maps Copy maps. +//------------------------------------------------------------------------------ + void add_precopy_item(graph::copy_nodes maps) { + preitems.push_back(std::make_unique> (maps)); + } + +//------------------------------------------------------------------------------ +/// @brief Add a pre loop item. +/// +/// @param[in] in Input variables. +/// @param[in] out Output nodes. +/// @param[in] maps Setter maps. +/// @param[in] state Random state node. +/// @param[in] name Name of the work item. +/// @param[in] size Size of the work item. +/// @param[in] iterations Number of iterations. +//------------------------------------------------------------------------------ + void add_preloop_item(graph::input_nodes in, + graph::output_nodes out, + graph::map_nodes maps, + graph::shared_random_state state, + const std::string name, const size_t size, + const size_t iterations) { + preitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, + iterations)); + } + //------------------------------------------------------------------------------ /// @brief Add a workflow item. /// @@ -381,7 +454,16 @@ namespace workflow { } //------------------------------------------------------------------------------ -/// @brief Add a workflow item. +/// @brief Add a copy item. +/// +/// @param[in] maps Copy maps. +//------------------------------------------------------------------------------ + void add_copy_item(graph::copy_nodes maps) { + items.push_back(std::make_unique> (maps)); + } + +//------------------------------------------------------------------------------ +/// @brief Add a loop item. /// /// @param[in] in Input variables. /// @param[in] out Output nodes. diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 966126c..ed94f89 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -88,12 +88,124 @@ void run_pic() { {weights[2], graph::variable_cast(ions[i].weights[2])}, {mesh_i, graph::variable_cast(ions[i].indices)} }, NULL, "pre_compute_weights_" + ion_tag, num_particles); + + if (i == 0) { + work.add_prezero_item({ + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y[0]) + }); + } else { + work.add_prezero_item({ + graph::variable_cast(mesh.index) + }); + } + + auto mesh_solve = mesh.build_mesh_solve(ions[i]); + work.add_preloop_item({ + graph::variable_cast(ions[i].indices), + graph::variable_cast(ions[i].weights[0]), + graph::variable_cast(ions[i].weights[1]), + graph::variable_cast(ions[i].weights[2]), + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y[0]) + }, {}, { + {mesh_solve[0], graph::variable_cast(mesh.index)}, + {mesh_solve[1], graph::variable_cast(mesh.y[0])} + }, NULL, "pre_sum_weights_" + ion_tag, num_grid, num_particles); + + if (i == ions.size() - 1) { + work.add_precopy_item({ + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[3])} + }); + } + + auto particle_step = pic::build_rk4_step(ions[i], mesh, norms, params); + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp(), + graph::variable_cast(mesh.y[0]), + graph::variable_cast(mesh.y[1]), + graph::variable_cast(mesh.y[2]), + graph::variable_cast(mesh.y[3]) + }, {}, { + {particle_step[0], ions[i].get_x()}, + {particle_step[1], ions[i].get_v_para()}, + {particle_step[2], ions[i].get_v_perp()} + }, NULL, "particle_push_" + ion_tag, num_particles); + + auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, + graph::random_state_cast(state)); + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp() + }, {}, { + {particle_reinject[0], ions[i].get_x()}, + {particle_reinject[1], ions[i].get_v_para()}, + {particle_reinject[2], ions[i].get_v_perp()} + }, graph::random_state_cast(state), + "particle_reinjection_" + ion_tag, num_particles); + + work.add_item({ + ions[i].get_x(), + graph::variable_cast(ions[i].weights[0]), + graph::variable_cast(ions[i].weights[1]), + graph::variable_cast(ions[i].weights[2]), + graph::variable_cast(ions[i].indices) + }, {}, { + {weights[0], graph::variable_cast(ions[i].weights[0])}, + {weights[1], graph::variable_cast(ions[i].weights[1])}, + {weights[2], graph::variable_cast(ions[i].weights[2])}, + {mesh_i, graph::variable_cast(ions[i].indices)} + }, NULL, "compute_weights_" + ion_tag, num_particles); + + if (i == 0) { + work.add_zero_item({ + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y[0]) + }); + } else { + work.add_zero_item({ + graph::variable_cast(mesh.index) + }); + } + if (i == 0) { + work.add_copy_item({ + {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, + {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])} + }); + } + + work.add_loop_item({ + graph::variable_cast(ions[i].indices), + graph::variable_cast(ions[i].weights[0]), + graph::variable_cast(ions[i].weights[1]), + graph::variable_cast(ions[i].weights[2]), + graph::variable_cast(mesh.index), + graph::variable_cast(mesh.y[0]) + }, {}, { + {mesh_solve[0], graph::variable_cast(mesh.index)}, + {mesh_solve[1], graph::variable_cast(mesh.y[0])} + }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles); } work.compile(); + + const timing::measure_diagnostic prerun("Pre Run Time"); work.pre_run(); + work.wait(); + prerun.print(); + const timing::measure_diagnostic run("Run Time"); + for (size_t i = 0; i < 100; i++) { + work.run(); + } work.wait(); + run.print(); } //------------------------------------------------------------------------------ From ca13c37d79d126eab5758284ebb5e5016809232c Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 6 Jul 2026 15:21:46 -0400 Subject: [PATCH 12/51] Enable batch loop unrolling for the field solve and test optimal unroll iterations. --- graph_framework/particle_in_cell.hpp | 46 +++++++++++++++------------- graph_pic/xpic.cpp | 13 ++++++-- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 6724ef9..cb318f4 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -440,33 +440,37 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Build mesh accumulation. /// -/// @param[in] ion A @ref pic::ion object. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] batch The batch size. /// @returns Expressions for mesh accumulation. //------------------------------------------------------------------------------ - std::array, 2> build_mesh_solve(const ion &ion) const { + std::array, 2> build_mesh_solve(const ion &ion, + const size_t batch=1) const { auto next_index = index; auto next_weight = y[0]; auto kernel_index = graph::index (); - auto index_i = graph::index_1D(ion.indices, next_index, - static_cast (1), - static_cast (0)); - auto index_w0 = graph::index_1D(ion.weights[0], next_index, - static_cast (1), - static_cast (0)); - auto index_w1 = graph::index_1D(ion.weights[1], next_index, - static_cast (1), - static_cast (0)); - auto index_w2 = graph::index_1D(ion.weights[2], next_index, - static_cast (1), - static_cast (0)); - next_index = next_index + static_cast (1); - next_weight = graph::if_(index_i - static_cast (1) == kernel_index, - next_weight + index_w0, next_weight); - next_weight = graph::if_(index_i == kernel_index, - next_weight + index_w1, next_weight); - next_weight = graph::if_(index_i + static_cast (1) == kernel_index, - next_weight + index_w2, next_weight); + for (size_t i = 0; i < batch; i++) { + auto index_i = graph::index_1D(ion.indices, next_index, + static_cast (1), + static_cast (0)); + auto index_w0 = graph::index_1D(ion.weights[0], next_index, + static_cast (1), + static_cast (0)); + auto index_w1 = graph::index_1D(ion.weights[1], next_index, + static_cast (1), + static_cast (0)); + auto index_w2 = graph::index_1D(ion.weights[2], next_index, + static_cast (1), + static_cast (0)); + next_index = next_index + static_cast (1); + next_weight = graph::if_(index_i - static_cast (1) == kernel_index, + next_weight + index_w0, next_weight); + next_weight = graph::if_(index_i == kernel_index, + next_weight + index_w1, next_weight); + next_weight = graph::if_(index_i + static_cast (1) == kernel_index, + next_weight + index_w2, next_weight); + } return {next_index, next_weight}; } diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index ed94f89..e1ddb32 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -15,9 +15,11 @@ //------------------------------------------------------------------------------ template void run_pic() { + const timing::measure_diagnostic init("Init Time"); // Sizes const size_t num_particles = 1000000; const size_t num_grid = 100; + const size_t num_batch = 10; const size_t num_ions = 1; const pic::characteristics norms({ @@ -100,7 +102,7 @@ void run_pic() { }); } - auto mesh_solve = mesh.build_mesh_solve(ions[i]); + auto mesh_solve = mesh.build_mesh_solve(ions[i], num_batch); work.add_preloop_item({ graph::variable_cast(ions[i].indices), graph::variable_cast(ions[i].weights[0]), @@ -111,7 +113,7 @@ void run_pic() { }, {}, { {mesh_solve[0], graph::variable_cast(mesh.index)}, {mesh_solve[1], graph::variable_cast(mesh.y[0])} - }, NULL, "pre_sum_weights_" + ion_tag, num_grid, num_particles); + }, NULL, "pre_sum_weights_" + ion_tag, num_grid, num_particles/num_batch); if (i == ions.size() - 1) { work.add_precopy_item({ @@ -190,10 +192,13 @@ void run_pic() { }, {}, { {mesh_solve[0], graph::variable_cast(mesh.index)}, {mesh_solve[1], graph::variable_cast(mesh.y[0])} - }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles); + }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles/num_batch); } + init.print(); + const timing::measure_diagnostic compile("Compile Time"); work.compile(); + compile.print(); const timing::measure_diagnostic prerun("Pre Run Time"); work.pre_run(); @@ -219,7 +224,9 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; + const timing::measure_diagnostic total("Run Time"); run_pic (); + total.print(); END_GPU } From 2123c999911617fc0696be21bb4315bcb6fa28c2 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 6 Jul 2026 15:43:53 -0400 Subject: [PATCH 13/51] Use the correct cuda call for device to device copy. --- graph_framework/cuda_context.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index b1aed55..7eb23d4 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -650,11 +650,12 @@ namespace gpu { return [this, sources, destinations] () mutable { for (size_t i = 0, ie = sources.size(); i < ie; i++) { size_t size; - check_error(cuMemGetAddressRange(NULL, &size, buffers[i]), + check_error(cuMemGetAddressRange(NULL, &size, sources[i]), "cuMemGetAddressRange"); - check_error_async(cuMemsetD8Async(buffers[i], 0, - size, stream), - "cuMemsetD8Async"); + check_error_async(cuMemcpyDtoDAsync(destinations[i], + sources[i], + size, stream), + "cuMemcpyDtoDAsync"); } }; } From 8afaa28808aecc96442aabee27630f02888fe24e Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 6 Jul 2026 18:20:27 -0400 Subject: [PATCH 14/51] Add test coverage for preitems and copy items. --- graph_tests/workflow_test.cpp | 144 ++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/graph_tests/workflow_test.cpp b/graph_tests/workflow_test.cpp index be12d47..71674b2 100644 --- a/graph_tests/workflow_test.cpp +++ b/graph_tests/workflow_test.cpp @@ -12,6 +12,118 @@ #include "../graph_framework/graph_framework.hpp" +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_prezeros() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer(1, static_cast (1)); + a->set(buffer); + b->set(buffer); + + workflow::manager work(0); + work.add_prezero_item({ + graph::variable_cast(a), + graph::variable_cast(b) + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); + work.pre_run(); + assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); + assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); +} + +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_precopy() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer1(1, static_cast (1)); + backend::buffer buffer2(1, static_cast (2)); + a->set(buffer1); + b->set(buffer2); + + workflow::manager work(0); + work.add_precopy_item({ + {graph::variable_cast(a), graph::variable_cast(b)} + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (2) && "Expected two."); + work.pre_run(); + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); +} + +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_premaps() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer(1, static_cast (1)); + a->set(buffer); + b->set(buffer); + + auto zero = graph::zero (); + + workflow::manager work(0); + work.add_preitem({ + graph::variable_cast(a), + graph::variable_cast(b) + }, {}, { + {zero, graph::variable_cast(a)}, + {zero, graph::variable_cast(b)} + }, NULL, "test_maps", 1); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); + work.pre_run(); + assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); + assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); +} + +//------------------------------------------------------------------------------ +/// @brief Test loop items. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_preloops() { + auto a = graph::variable (1, ""); + backend::buffer buffer(1, static_cast (0)); + a->set(buffer); + + auto a_next = a + static_cast (1); + + workflow::manager work(0); + work.add_preloop_item({ + graph::variable_cast(a) + }, {}, { + {a_next, graph::variable_cast(a)} + }, NULL, "test_maps", 1, 10); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); + work.pre_run(); + assert(work.check_value(0, a) == static_cast (10) && "Expected ten."); +} + //------------------------------------------------------------------------------ /// @brief Test setting multiple variables with the same map. /// @@ -39,6 +151,33 @@ template void test_zeros() { assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); } +//------------------------------------------------------------------------------ +/// @brief Test setting multiple variables with the same map. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_copy() { + auto a = graph::variable (1, ""); + auto b = graph::variable (1, ""); + backend::buffer buffer1(1, static_cast (1)); + backend::buffer buffer2(1, static_cast (2)); + a->set(buffer1); + b->set(buffer2); + + workflow::manager work(0); + work.add_copy_item({ + {graph::variable_cast(a), graph::variable_cast(b)} + }); + + work.compile(); + + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (2) && "Expected two."); + work.run(); + assert(work.check_value(0, a) == static_cast (1) && "Expected one."); + assert(work.check_value(0, b) == static_cast (1) && "Expected one."); +} + //------------------------------------------------------------------------------ /// @brief Test setting multiple variables with the same map. /// @@ -103,7 +242,12 @@ template void test_loops() { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { + test_prezeros (); + test_precopy (); + test_premaps (); + test_preloops (); test_zeros (); + test_copy (); test_maps (); test_loops (); } From 977d342eab8587c0525584c5ef5c134dc7b0a92c Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 13 Jul 2026 22:13:09 -0400 Subject: [PATCH 15/51] Refactor workflow manager to define pre run and post items. Add an ability to define allback functions in the workflows. Add fileio functions for the pic code. --- graph_c_binding/graph_c_binding.cpp | 48 ++--- graph_docs/kernel_optimization.dox | 2 +- graph_framework.xcodeproj/project.pbxproj | 15 +- graph_framework/cpu_context.hpp | 10 + graph_framework/cuda_context.hpp | 13 ++ graph_framework/jit.hpp | 9 + graph_framework/metal_context.hpp | 22 +- graph_framework/node.hpp | 1 + graph_framework/output.hpp | 84 +++----- graph_framework/particle_in_cell.hpp | 89 ++++++-- graph_framework/workflow.hpp | 247 ++++++++++++++-------- graph_korc/xkorc.cpp | 4 +- graph_pic/xpic.cpp | 97 +++++++-- graph_tests/c_binding_test.c | 12 +- graph_tests/workflow_test.cpp | 163 ++++---------- 15 files changed, 477 insertions(+), 339 deletions(-) diff --git a/graph_c_binding/graph_c_binding.cpp b/graph_c_binding/graph_c_binding.cpp index 1076d09..d2a49ae 100644 --- a/graph_c_binding/graph_c_binding.cpp +++ b/graph_c_binding/graph_c_binding.cpp @@ -2035,13 +2035,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } else { auto d = reinterpret_cast *> (c); @@ -2072,13 +2072,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } break; @@ -2113,13 +2113,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } else { auto d = reinterpret_cast *> (c); @@ -2150,13 +2150,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } break; @@ -2191,13 +2191,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } else { auto d = reinterpret_cast> *> (c); @@ -2228,13 +2228,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } break; @@ -2269,13 +2269,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } else { auto d = reinterpret_cast> *> (c); @@ -2306,13 +2306,13 @@ extern "C" { if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_preitem(in, out, map, rand, name, size); + d->work.add_item (in, out, map, rand, name, size); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_preitem(in, out, map, NULL, name, size); + d->work.add_item (in, out, map, NULL, name, size); } } break; @@ -3074,40 +3074,40 @@ extern "C" { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } break; case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast *> (c); - d->work.pre_run(); + d->work.run (); } break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast> *> (c); - d->work.pre_run(); + d->work.run (); } break; case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work.pre_run(); + d->work.run (); } else { auto d = reinterpret_cast> *> (c); - d->work.pre_run(); + d->work.run (); } break; } diff --git a/graph_docs/kernel_optimization.dox b/graph_docs/kernel_optimization.dox index e4302fc..6158df3 100644 --- a/graph_docs/kernel_optimization.dox +++ b/graph_docs/kernel_optimization.dox @@ -68,7 +68,7 @@ void field_solve_example() { timing::measure_diagnostic compile("compile"); workflow::manager work(0); - work.add_preitem({ + work.add_item ({ graph::variable_cast(particle_positions) }, {}, { {random_real, variable_cast(particle_positions)} diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index fbfad82..f862cd1 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -2525,7 +2525,15 @@ GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; MACOSX_DEPLOYMENT_TARGET = 15.0; OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; - OTHER_LDFLAGS = ""; + "OTHER_CPLUSPLUSFLAGS[arch=*]" = ( + "$(OTHER_CFLAGS)", + "-fsanitize=undefined", + "-fsanitize=float-divide-by-zero", + ); + OTHER_LDFLAGS = ( + "-fsanitize=float-divide-by-zero", + "-fsenitize=undefined", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SKIP_INSTALL = YES; @@ -2542,7 +2550,10 @@ GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; MACOSX_DEPLOYMENT_TARGET = 15.0; OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; - OTHER_LDFLAGS = ""; + OTHER_LDFLAGS = ( + "-fsanitize=float-divide-by-zero", + "-fsenitize=undefined", + ); PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SKIP_INSTALL = YES; diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 8ef6960..9eb5666 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -397,6 +397,16 @@ namespace gpu { } } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return callback; + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 7eb23d4..d017a54 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -668,6 +668,19 @@ namespace gpu { check_error(cuCtxSynchronize(), "cuCtxSynchronize"); } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return [this]() { + check_error_async(cuLaunchHostFunc(stream, callback.target()), + "cuLaunchHostFunc"); + } + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index c6d3672..143b2bf 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -415,6 +415,15 @@ namespace jit { gpu_context.wait(); } +//------------------------------------------------------------------------------ +/// @brief Run a function. +/// +/// @returns A lambda function to run run the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return gpu_context.run_function(callback); + } + //------------------------------------------------------------------------------ /// @brief Copy contexts of buffer to device. /// diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 08c7f60..b9c61b5 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -147,8 +147,6 @@ namespace gpu { kernel_arguments[input.get()] = [device newBufferWithBytes:buffer.data() length:buffer.size()*buffer_element_size options:MTLResourceStorageModeShared]; - buffers.push_back(kernel_arguments[input.get()]); - needed_buffers.insert(input.get()); } if (!needed_buffers.contains(input.get())) { buffers.push_back(kernel_arguments[input.get()]); @@ -159,8 +157,6 @@ namespace gpu { if (!kernel_arguments.contains(output.get())) { kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float) options:MTLResourceStorageModeShared]; - buffers.push_back(kernel_arguments[output.get()]); - needed_buffers.insert(output.get()); } if (!needed_buffers.contains(output.get())) { buffers.push_back(kernel_arguments[output.get()]); @@ -443,6 +439,24 @@ namespace gpu { [command_buffer waitUntilCompleted]; } +//------------------------------------------------------------------------------ +/// @brief Run a callback function in the queue. +/// +/// @param[in] callback The callback function to run. +/// @returns Lambda to call the function. +//------------------------------------------------------------------------------ + std::function run_function(std::function callback) { + return [this, callback]() { + command_buffer = [queue commandBuffer]; + + [command_buffer addCompletedHandler:[callback](id commandBuffer) { + callback(); + }]; + + [command_buffer commit]; + }; + } + //------------------------------------------------------------------------------ /// @brief Print out the results. /// diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 7222a6b..786c364 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -920,6 +920,7 @@ namespace graph { //------------------------------------------------------------------------------ constant_node(const backend::buffer &d) : leaf_node (constant_node::to_string(d.at(0)), 1, false), data(d) { + assert(d.is_normal() && "Denormal encountered"); assert(d.size() == 1 && "Constants need to be scalar functions."); } diff --git a/graph_framework/output.hpp b/graph_framework/output.hpp index 02b2f75..04c2fed 100644 --- a/graph_framework/output.hpp +++ b/graph_framework/output.hpp @@ -369,33 +369,17 @@ namespace output { for (variable &var : variables) { sync.lock(); if constexpr (jit::float_base) { - if constexpr (jit::complex_scalar) { - check_error(nc_put_vara_float(result.get_ncid(), - var.id, - start.data(), - count.data(), - reinterpret_cast (var.buffer))); - } else { - check_error(nc_put_vara_float(result.get_ncid(), - var.id, - start.data(), - count.data(), - var.buffer)); - } + check_error(nc_put_vara_float(result.get_ncid(), + var.id, + start.data(), + count.data(), + reinterpret_cast (var.buffer))); } else { - if constexpr (jit::complex_scalar) { - check_error(nc_put_vara_double(result.get_ncid(), - var.id, - start.data(), - count.data(), - reinterpret_cast (var.buffer))); - } else { - check_error(nc_put_vara_double(result.get_ncid(), - var.id, - start.data(), - count.data(), - var.buffer)); - } + check_error(nc_put_vara_double(result.get_ncid(), + var.id, + start.data(), + count.data(), + reinterpret_cast (var.buffer))); } sync.unlock(); } @@ -430,41 +414,21 @@ namespace output { sync.lock(); if constexpr (jit::float_base) { - if constexpr (jit::complex_scalar) { - check_error(nc_get_varm_float(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - reinterpret_cast (ref.buffer))); - } else { - check_error(nc_get_varm_float(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - ref.buffer)); - } + check_error(nc_get_varm_float(result.get_ncid(), + ref.id, + ref_start.data(), + ref_count.data(), + stride.data(), + map.data(), + reinterpret_cast (ref.buffer))); } else { - if constexpr (jit::complex_scalar) { - check_error(nc_get_varm_double(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - reinterpret_cast (ref.buffer))); - } else { - check_error(nc_get_varm_double(result.get_ncid(), - ref.id, - ref_start.data(), - ref_count.data(), - stride.data(), - map.data(), - ref.buffer)); - } + check_error(nc_get_varm_double(result.get_ncid(), + ref.id, + ref_start.data(), + ref_count.data(), + stride.data(), + map.data(), + reinterpret_cast (ref.buffer))); } sync.unlock(); diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index cb318f4..fa99c3d 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -29,6 +29,9 @@ namespace pic { /// Hydrogen mass kg. template constexpr T m_hydrogen = static_cast (1.67362192595E-27); +/// Atomic mass + template + constexpr T m_atomic = static_cast (1.66053906892E-27); /// Electron mass kg. template constexpr T m_electron = static_cast (9.1093837139E-31); @@ -126,6 +129,10 @@ namespace pic { const T smoothing; /// Time step. const T dt; +/// Parallel temperature. + const T t_para; +/// Perpendicular temperature. + const T t_perp; //------------------------------------------------------------------------------ /// @brief Construct a parameters object. @@ -141,11 +148,12 @@ namespace pic { parameters(const T b0, const T r1, const T r2, const size_t filter_iterations, const T smoothing, const T dt, + const T t_para, const T t_perp, const characteristics &norms) : b0(b0/norms.bfield), a0(std::numbers::pi_v*(r2*r2 - r1*r1)/(norms.l*norms.l)), filter_iterations(filter_iterations), smoothing(smoothing), - dt(dt/norms.t) {} + dt(dt/norms.t), t_para(t_para), t_perp(t_perp) {} }; //------------------------------------------------------------------------------ @@ -193,7 +201,7 @@ namespace pic { const T num_real, const characteristics &norms) : z(z), charge(z*pic::q/norms.q), - mass(mass/norms.m), num_real(num_real), + mass(mass), num_real(num_real), x(graph::variable (num_ions, "x")), v_para(graph::variable (num_ions, "v_{||}")), v_perp(graph::variable (num_ions, "v_{\\perp}")), @@ -274,6 +282,26 @@ namespace pic { T super_to_real() const { return num_real/size(); } + +//------------------------------------------------------------------------------ +/// @brief Define variables. +/// +/// @param[in] file A @ref output::result_file object to define variables. +/// @param[in,out] data A @ref output::data_set object to create variable. +/// @param[in,out] work A @ref workflow::manager object where data was +/// computed. +/// @param[in] tag Unique identity for give the ion species. +//------------------------------------------------------------------------------ + void define_variables(const output::result_file &file, + output::data_set &data, + workflow::manager &work, + const std::string tag) { + data.create_variable(file, "x_" + tag, x, work.get_context()); + data.create_variable(file, "vpara_" + tag, v_para, + work.get_context()); + data.create_variable(file, "vperp_" + tag, v_perp, + work.get_context()); + } }; //------------------------------------------------------------------------------ @@ -494,6 +522,23 @@ namespace pic { T *data() const { return graph::variable_cast(y[I])->data(); } + +//------------------------------------------------------------------------------ +/// @brief Define variables. +/// +/// @param[in] file A @ref output::result_file object to define variables. +/// @param[in,out] data A @ref output::data_set object to create variable. +/// @param[in] work A @ref workflow::manager object where data was +/// computed. +//------------------------------------------------------------------------------ + void define_variables(const output::result_file &file, + output::data_set &data, + workflow::manager &work) { + data.create_variable(file, "y_0", y[0], work.get_context()); + data.create_variable(file, "y_1", y[1], work.get_context()); + data.create_variable(file, "y_2", y[2], work.get_context()); + data.create_variable(file, "y_3", y[3], work.get_context()); + } }; //------------------------------------------------------------------------------ @@ -524,14 +569,18 @@ namespace pic { /// /// @tparam T Base type of the calculation. /// -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. /// @param[in] state Random state node. /// @returns Initialized normalized values for x, v||, and v⟂ //------------------------------------------------------------------------------ template - std::array,3> build_initialization(const mesh &mesh, + std::array,3> build_initialization(const pic::ion &ion, + const mesh &mesh, const characteristics &norms, + const parameters ¶ms, const graph::shared_random_state state) { // The mesh is already normalized so position_dist will be a normalized quantity. auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, @@ -543,7 +592,9 @@ namespace pic { static_cast (1.0), state); - auto vpara = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); + const T vtpara = std::sqrt(2*params.t_para*q/ion.mass); + auto vpara = vtpara*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + * graph::sin(phi_dist); phi_dist = graph::uniform_random (static_cast (0.0), static_cast (2.0)*std::numbers::pi_v, @@ -551,8 +602,12 @@ namespace pic { r_dist = graph::uniform_random (static_cast (0.0), static_cast (1.0), state); - auto vperp1 = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::cos(phi_dist); - auto vperp2 = graph::sqrt(-graph::log(static_cast (1) - r_dist))*graph::sin(phi_dist); + + const T vtperp = std::sqrt(2*params.t_perp*q/ion.mass); + auto vperp1 = vtperp*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + * graph::cos(phi_dist); + auto vperp2 = vtperp*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + * graph::sin(phi_dist); auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); return {position_dist, vpara/norms.v, vperp/norms.v}; @@ -566,9 +621,10 @@ namespace pic { /// /// @tparam T Base type of the calculation. /// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] mesh A @ref pic::mesh object. -/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] ion A @ref pic::ion object. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. /// @param[in] state Random state node. /// @returns Reinjected values for x, v||, and v⟂ //------------------------------------------------------------------------------ @@ -576,8 +632,9 @@ namespace pic { std::array, 3> build_reinjection(const ion &ion, const mesh &mesh, const characteristics &norms, + const parameters ¶ms, const graph::shared_random_state state) { - auto resampled = build_initialization(mesh, norms, state); + auto resampled = build_initialization(ion, mesh, norms, params, state); auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); @@ -598,7 +655,7 @@ namespace pic { template graph::shared_leaf build_magnetic_field(graph::shared_leaf x, const characteristics &norms) { - return (static_cast (0.1)*x*x + static_cast (0.5))/norms.bfield; + return (x*x*norms.l*norms.l + static_cast (0.5))/norms.bfield; } //------------------------------------------------------------------------------ @@ -675,7 +732,7 @@ namespace pic { template graph::shared_leaf build_electron_temperature(graph::shared_leaf x, const characteristics &norms) { - return graph::one ()/norms.te; + return graph::constant (static_cast (2.5)*q/(norms.te*kb)); } //------------------------------------------------------------------------------ @@ -709,7 +766,7 @@ namespace pic { auto n = (n0 + n1 + n2 + n3)/static_cast (4); auto dndx = (dn0dx + dn1dx + dn2dx + dn3dx)/static_cast (4); - auto te = build_magnetic_field(x, norms); + auto te = build_electron_temperature(x, norms); auto pressure = te*n/q; return graph::none ()/n*(dndx*te/q + pressure->df(x)); @@ -787,7 +844,7 @@ namespace pic { return { z[1]*params.dt, temp*params.dt, - (ion.charge/ion.mass*efield - temp)*params.dt + (ion.charge/ion.mass*norms.m*efield - temp)*params.dt }; } diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index d4f794d..46bf7e1 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -12,6 +12,16 @@ /// Name space for workflows. namespace workflow { +/// Items order + enum order { +/// Pre items + pre_run_item, +/// Items + run_item, +/// Post items + post_run_item + }; + //------------------------------------------------------------------------------ /// @brief Interface class representing items. /// @@ -34,6 +44,46 @@ namespace workflow { virtual void run() = 0; }; +//------------------------------------------------------------------------------ +/// @brief Callback item. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class callback_item : public item { + protected: +/// Callback function. + std::function callback; +/// Kernel function. + std::function kernel; + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a workflow item. +/// +/// @param[in] callback Lambda function to run. +//------------------------------------------------------------------------------ + callback_item(std::function callback) : + callback(callback) {} + +//------------------------------------------------------------------------------ +/// @brief Set the kernel function. +/// +/// @param[in,out] context Jit context. +//------------------------------------------------------------------------------ + virtual void create_kernel_call(jit::context &context) { + kernel = context.run_function(callback); + } + +//------------------------------------------------------------------------------ +/// @brief Run the work item. +//------------------------------------------------------------------------------ + virtual void run() { + kernel(); + } + }; + //------------------------------------------------------------------------------ /// @brief Clear buffer item. /// @@ -343,6 +393,8 @@ namespace workflow { std::vector>> preitems; /// List of work items. std::vector>> items; +/// List of pre work items. + std::vector>> postitems; /// Use reduction. bool add_reduction; @@ -361,71 +413,28 @@ namespace workflow { manager(const size_t index) : context(index), add_reduction(false) {} //------------------------------------------------------------------------------ -/// @brief Add a pre workflow item. -/// -/// @param[in] in Input variables. -/// @param[in] out Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -//------------------------------------------------------------------------------ - void add_preitem(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, - const std::string name, const size_t size) { - preitems.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context)); - } - -//------------------------------------------------------------------------------ -/// @brief Add a pre zero item. +/// @brief Add a pre callback function. /// -/// @param[in] in Input variables. -//------------------------------------------------------------------------------ - void add_prezero_item(graph::input_nodes in) { - preitems.push_back(std::make_unique> (in)); - } - -//------------------------------------------------------------------------------ -/// @brief Add a pre copy item. +/// @tparam O The @ref workflow::order /// -/// @param[in] maps Copy maps. +/// @param[in] callback Lambda function to run. //------------------------------------------------------------------------------ - void add_precopy_item(graph::copy_nodes maps) { - preitems.push_back(std::make_unique> (maps)); - } - -//------------------------------------------------------------------------------ -/// @brief Add a pre loop item. -/// -/// @param[in] in Input variables. -/// @param[in] out Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. -/// @param[in] iterations Number of iterations. -//------------------------------------------------------------------------------ - void add_preloop_item(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, - const std::string name, const size_t size, - const size_t iterations) { - preitems.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context, - iterations)); + template + void add_callback_item(std::function callback) { + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (callback)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (callback)); + } else { + postitems.push_back(std::make_unique> (callback)); + } } //------------------------------------------------------------------------------ /// @brief Add a workflow item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. @@ -433,38 +442,71 @@ namespace workflow { /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. //------------------------------------------------------------------------------ + template void add_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, graph::shared_random_state state, const std::string name, const size_t size) { - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context)); + } } //------------------------------------------------------------------------------ /// @brief Add a zero item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] in Input variables. //------------------------------------------------------------------------------ + template void add_zero_item(graph::input_nodes in) { - items.push_back(std::make_unique> (in)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in)); + } else { + postitems.push_back(std::make_unique> (in)); + } } //------------------------------------------------------------------------------ /// @brief Add a copy item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] maps Copy maps. //------------------------------------------------------------------------------ + template void add_copy_item(graph::copy_nodes maps) { - items.push_back(std::make_unique> (maps)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (maps)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (maps)); + } else { + postitems.push_back(std::make_unique> (maps)); + } } //------------------------------------------------------------------------------ /// @brief Add a loop item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. @@ -473,22 +515,39 @@ namespace workflow { /// @param[in] size Size of the work item. /// @param[in] iterations Number of iterations. //------------------------------------------------------------------------------ + template void add_loop_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, graph::shared_random_state state, const std::string name, const size_t size, const size_t iterations) { - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context, - iterations)); + if constexpr (O == pre_run_item) { + preitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, + iterations)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, + iterations)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, + iterations)); + } } //------------------------------------------------------------------------------ /// @brief Add a converge item. /// +/// @tparam O The @ref workflow::order +/// /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. @@ -498,6 +557,7 @@ namespace workflow { /// @param[in] tol Tolerance to converge the function to. /// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ + template void add_converge_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, @@ -506,11 +566,25 @@ namespace workflow { const T tol=1.0E-30, const size_t max_iter=1000) { add_reduction = true; - items.push_back(std::make_unique> (in, out, - maps, state, - name, size, - context, tol, - max_iter)); + if constexpr (O == pre_run_item) { + items.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, tol, + max_iter)); + } else if constexpr (O == run_item) { + items.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, tol, + max_iter)); + } else { + postitems.push_back(std::make_unique> (in, out, + maps, state, + name, size, + context, tol, + max_iter)); + } } //------------------------------------------------------------------------------ @@ -525,23 +599,30 @@ namespace workflow { for (auto &item : items) { item->create_kernel_call(context); } - } - -//------------------------------------------------------------------------------ -/// @brief Run pre work items. -//------------------------------------------------------------------------------ - void pre_run() { - for (auto &item : preitems) { - item->run(); + for (auto &item : postitems) { + item->create_kernel_call(context); } } //------------------------------------------------------------------------------ /// @brief Run work items. +/// +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ + template void run() { - for (auto &item : items) { - item->run(); + if constexpr (O == pre_run_item) { + for (auto &item : preitems) { + item->run(); + } + } else if constexpr (O == run_item) { + for (auto &item : items) { + item->run(); + } + } else { + for (auto &item : postitems) { + item->run(); + } } } diff --git a/graph_korc/xkorc.cpp b/graph_korc/xkorc.cpp index d1b7cf1..f6ee85e 100644 --- a/graph_korc/xkorc.cpp +++ b/graph_korc/xkorc.cpp @@ -72,7 +72,7 @@ void run_korc() { pos->get_z())/b0; workflow::manager work(thread_number); - work.add_preitem({ + work.template add_item ({ graph::variable_cast(ux), graph::variable_cast(uy), graph::variable_cast(uz), @@ -142,7 +142,7 @@ void run_korc() { t_setup.print(); const timing::measure_diagnostic t_run("Run Time"); - work.pre_run(); + work.template run (); for (size_t i = 0; i < 1000000; i++) { /* sync.join(); work.wait(); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index e1ddb32..dff79b4 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -17,13 +17,15 @@ template void run_pic() { const timing::measure_diagnostic init("Init Time"); // Sizes - const size_t num_particles = 1000000; + const size_t num_particles = 10000;//00; const size_t num_grid = 100; const size_t num_batch = 10; const size_t num_ions = 1; + const size_t num_steps = 1; + const size_t num_sub_steps = 1; const pic::characteristics norms({ - pic::m_hydrogen + 2*pic::m_atomic }, {1}, static_cast (2.5E19)); std::array ion_masses{pic::m_hydrogen}; @@ -55,19 +57,29 @@ void run_pic() { const T gyro_period = ion_zs[0]*pic::q*b0/ion_masses[0]; const T dtc = 0.25; const pic::parameters params(b0, r1, r2, 3, 1.0E-4, - dtc*gyro_period, norms); + dtc*gyro_period, 2.5, 2.5, norms); pic::mesh mesh(lmin, lmax, num_grid, norms); auto state = graph::random_state (jit::context::random_state_size, 0); workflow::manager work(0); + + output::result_file f_file("fields.nc", num_grid); + output::data_set mesh_dataset(f_file); + + output::result_file p_file("particles.nc", num_particles); + std::vector> p_datasets(num_ions, + output::data_set (p_file)); + for (size_t i = 0; i < num_ions; i++) { const std::string ion_tag = jit::format_to_string(i); - auto ion_inits = pic::build_initialization (mesh, norms, + auto ion_inits = pic::build_initialization (ions[i], mesh, + norms, params, graph::random_state_cast(state)); - work.add_preitem({ + auto efield = build_interpolate_efield (ions[i].x, ions[i], mesh, norms, params); + work.template add_item ({ ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() }, {}, { {ion_inits[0], ions[i].get_x()}, @@ -78,7 +90,7 @@ void run_pic() { auto mesh_i = mesh.build_i_index(ions[i].x); auto weights = pic::build_weights (ions[i].x, mesh); - work.add_preitem({ + work.template add_item ({ ions[i].get_x(), graph::variable_cast(ions[i].weights[0]), graph::variable_cast(ions[i].weights[1]), @@ -92,18 +104,18 @@ void run_pic() { }, NULL, "pre_compute_weights_" + ion_tag, num_particles); if (i == 0) { - work.add_prezero_item({ + work.template add_zero_item ({ graph::variable_cast(mesh.index), graph::variable_cast(mesh.y[0]) }); } else { - work.add_prezero_item({ + work.template add_zero_item ({ graph::variable_cast(mesh.index) }); } auto mesh_solve = mesh.build_mesh_solve(ions[i], num_batch); - work.add_preloop_item({ + work.template add_loop_item ({ graph::variable_cast(ions[i].indices), graph::variable_cast(ions[i].weights[0]), graph::variable_cast(ions[i].weights[1]), @@ -116,13 +128,22 @@ void run_pic() { }, NULL, "pre_sum_weights_" + ion_tag, num_grid, num_particles/num_batch); if (i == ions.size() - 1) { - work.add_precopy_item({ + work.template add_copy_item ({ {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])}, {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[2])}, {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[3])} }); } + if (i == 0) { + work.template add_callback_item ([&f_file, &mesh_dataset]() { + mesh_dataset.write(f_file); + }); + } + work.template add_callback_item ([i, &p_file, &p_datasets]() { + p_datasets[i].write(p_file); + }); + auto particle_step = pic::build_rk4_step(ions[i], mesh, norms, params); work.add_item({ ions[i].get_x(), @@ -132,13 +153,13 @@ void run_pic() { graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3]) - }, {}, { + }, {efield}, { {particle_step[0], ions[i].get_x()}, {particle_step[1], ions[i].get_v_para()}, {particle_step[2], ions[i].get_v_perp()} }, NULL, "particle_push_" + ion_tag, num_particles); - - auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, +#if 0 + auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, params, graph::random_state_cast(state)); work.add_item({ ions[i].get_x(), @@ -193,6 +214,7 @@ void run_pic() { {mesh_solve[0], graph::variable_cast(mesh.index)}, {mesh_solve[1], graph::variable_cast(mesh.y[0])} }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles/num_batch); +#endif } init.print(); @@ -200,16 +222,49 @@ void run_pic() { work.compile(); compile.print(); - const timing::measure_diagnostic prerun("Pre Run Time"); - work.pre_run(); - work.wait(); - prerun.print(); + mesh.define_variables(f_file, mesh_dataset, work); + f_file.end_define_mode(); - const timing::measure_diagnostic run("Run Time"); - for (size_t i = 0; i < 100; i++) { - work.run(); + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + auto efield = build_interpolate_efield (ions[i].x, ions[i], mesh, norms, params); + ions[i].define_variables(p_file, p_datasets[i], work, ion_tag); + p_datasets[i].create_variable(p_file, "efield" + ion_tag, efield, work.get_context()); } + p_file.end_define_mode(); + + std::atomic_size_t counter = 0; + std::thread progress = std::thread([&num_steps, &counter]() -> void { + using namespace std::chrono_literals; + do { + const size_t progress = (counter*100.0)/num_steps; + std::cout << "\33[2K\r" << std::setw(3) << progress << "% Complete" + << std::flush; + std::this_thread::sleep_for(1s); + } while (counter < num_steps); + }); + + const timing::measure_diagnostic run("Run Time"); + work.template run (); + work.template run (); work.wait(); + + //auto bfield = build_magnetic_field (ions[0].x, norms); + //for (size_t i = 0; i < num_particles; i++) { + // work.print(i, {ions[0].x, efield}); + //} + + for (; counter < num_steps; counter++) { + for (size_t i = 0; i < num_sub_steps; i++) { + work.run(); + } + work.template run (); + work.wait(); + } + + counter = num_steps; + progress.join(); + std::cout << "\33[2K\r" << "100% Complete" << std::endl; run.print(); } @@ -224,7 +279,7 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; - const timing::measure_diagnostic total("Run Time"); + const timing::measure_diagnostic total("Total Time"); run_pic (); total.print(); diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index 72ff6cc..953c968 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -195,12 +195,12 @@ void run_tests(const enum graph_type type, graph_node *map_inputs2 = NULL; graph_node *map_outputs2 = NULL; - graph_add_pre_item(c_context, - NULL, 0, - &rand, 1, - NULL, NULL, 0, - state, - "c_binding_pre_kernel", 1); + graph_add_item(c_context, + NULL, 0, + &rand, 1, + NULL, NULL, 0, + state, + "c_binding_pre_kernel", 1); graph_add_item(c_context, inputs, 1, outputs, 5, diff --git a/graph_tests/workflow_test.cpp b/graph_tests/workflow_test.cpp index 71674b2..cc2ca7d 100644 --- a/graph_tests/workflow_test.cpp +++ b/graph_tests/workflow_test.cpp @@ -16,8 +16,9 @@ /// @brief Test setting multiple variables with the same map. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_prezeros() { +template void test_zeros() { auto a = graph::variable (1, ""); auto b = graph::variable (1, ""); backend::buffer buffer(1, static_cast (1)); @@ -25,7 +26,7 @@ template void test_prezeros() { b->set(buffer); workflow::manager work(0); - work.add_prezero_item({ + work.template add_zero_item ({ graph::variable_cast(a), graph::variable_cast(b) }); @@ -34,7 +35,7 @@ template void test_prezeros() { assert(work.check_value(0, a) == static_cast (1) && "Expected one."); assert(work.check_value(0, b) == static_cast (1) && "Expected one."); - work.pre_run(); + work.template run(); assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); } @@ -43,8 +44,9 @@ template void test_prezeros() { /// @brief Test setting multiple variables with the same map. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_precopy() { +template void test_copy() { auto a = graph::variable (1, ""); auto b = graph::variable (1, ""); backend::buffer buffer1(1, static_cast (1)); @@ -53,7 +55,7 @@ template void test_precopy() { b->set(buffer2); workflow::manager work(0); - work.add_precopy_item({ + work.template add_copy_item ({ {graph::variable_cast(a), graph::variable_cast(b)} }); @@ -61,129 +63,40 @@ template void test_precopy() { assert(work.check_value(0, a) == static_cast (1) && "Expected one."); assert(work.check_value(0, b) == static_cast (2) && "Expected two."); - work.pre_run(); + work.template run (); assert(work.check_value(0, a) == static_cast (1) && "Expected one."); assert(work.check_value(0, b) == static_cast (1) && "Expected one."); } //------------------------------------------------------------------------------ -/// @brief Test setting multiple variables with the same map. -/// -/// @tparam T Base type of the calculation. -//------------------------------------------------------------------------------ -template void test_premaps() { - auto a = graph::variable (1, ""); - auto b = graph::variable (1, ""); - backend::buffer buffer(1, static_cast (1)); - a->set(buffer); - b->set(buffer); - - auto zero = graph::zero (); - - workflow::manager work(0); - work.add_preitem({ - graph::variable_cast(a), - graph::variable_cast(b) - }, {}, { - {zero, graph::variable_cast(a)}, - {zero, graph::variable_cast(b)} - }, NULL, "test_maps", 1); - - work.compile(); - - assert(work.check_value(0, a) == static_cast (1) && "Expected one."); - assert(work.check_value(0, b) == static_cast (1) && "Expected one."); - work.pre_run(); - assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); - assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); -} - -//------------------------------------------------------------------------------ -/// @brief Test loop items. -/// -/// @tparam T Base type of the calculation. -//------------------------------------------------------------------------------ -template void test_preloops() { - auto a = graph::variable (1, ""); - backend::buffer buffer(1, static_cast (0)); - a->set(buffer); - - auto a_next = a + static_cast (1); - - workflow::manager work(0); - work.add_preloop_item({ - graph::variable_cast(a) - }, {}, { - {a_next, graph::variable_cast(a)} - }, NULL, "test_maps", 1, 10); - - work.compile(); - - assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); - work.pre_run(); - assert(work.check_value(0, a) == static_cast (10) && "Expected ten."); -} - -//------------------------------------------------------------------------------ -/// @brief Test setting multiple variables with the same map. +/// @brief Test callback functions. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_zeros() { - auto a = graph::variable (1, ""); - auto b = graph::variable (1, ""); - backend::buffer buffer(1, static_cast (1)); - a->set(buffer); - b->set(buffer); +template void test_callbacks() { + int i = 1; workflow::manager work(0); - work.add_zero_item({ - graph::variable_cast(a), - graph::variable_cast(b) + work.template add_callback_item ([&i]() { + i = 2; }); work.compile(); - assert(work.check_value(0, a) == static_cast (1) && "Expected one."); - assert(work.check_value(0, b) == static_cast (1) && "Expected one."); - work.run(); - assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); - assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); + assert(i == 1 && "Expected 1"); + work.template run (); + work.wait(); + assert(i == 2 && "Expected 2"); } //------------------------------------------------------------------------------ /// @brief Test setting multiple variables with the same map. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_copy() { - auto a = graph::variable (1, ""); - auto b = graph::variable (1, ""); - backend::buffer buffer1(1, static_cast (1)); - backend::buffer buffer2(1, static_cast (2)); - a->set(buffer1); - b->set(buffer2); - - workflow::manager work(0); - work.add_copy_item({ - {graph::variable_cast(a), graph::variable_cast(b)} - }); - - work.compile(); - - assert(work.check_value(0, a) == static_cast (1) && "Expected one."); - assert(work.check_value(0, b) == static_cast (2) && "Expected two."); - work.run(); - assert(work.check_value(0, a) == static_cast (1) && "Expected one."); - assert(work.check_value(0, b) == static_cast (1) && "Expected one."); -} - -//------------------------------------------------------------------------------ -/// @brief Test setting multiple variables with the same map. -/// -/// @tparam T Base type of the calculation. -//------------------------------------------------------------------------------ -template void test_maps() { +template void test_maps() { auto a = graph::variable (1, ""); auto b = graph::variable (1, ""); backend::buffer buffer(1, static_cast (1)); @@ -193,7 +106,7 @@ template void test_maps() { auto zero = graph::zero (); workflow::manager work(0); - work.add_item({ + work.template add_item ({ graph::variable_cast(a), graph::variable_cast(b) }, {}, { @@ -205,7 +118,7 @@ template void test_maps() { assert(work.check_value(0, a) == static_cast (1) && "Expected one."); assert(work.check_value(0, b) == static_cast (1) && "Expected one."); - work.run(); + work.template run (); assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); assert(work.check_value(0, b) == static_cast (0) && "Expected zero."); } @@ -214,8 +127,9 @@ template void test_maps() { /// @brief Test loop items. /// /// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order //------------------------------------------------------------------------------ -template void test_loops() { +template void test_loops() { auto a = graph::variable (1, ""); backend::buffer buffer(1, static_cast (0)); a->set(buffer); @@ -223,7 +137,7 @@ template void test_loops() { auto a_next = a + static_cast (1); workflow::manager work(0); - work.add_loop_item({ + work.template add_loop_item ({ graph::variable_cast(a) }, {}, { {a_next, graph::variable_cast(a)} @@ -232,24 +146,33 @@ template void test_loops() { work.compile(); assert(work.check_value(0, a) == static_cast (0) && "Expected zero."); - work.run(); + work.template run (); assert(work.check_value(0, a) == static_cast (10) && "Expected ten."); } +//------------------------------------------------------------------------------ +/// @brief Run tests with a specified backend. +/// +/// @tparam T Base type of the calculation. +/// @tparam O The @ref workflow::order +//------------------------------------------------------------------------------ +template void run_tests_order() { + test_zeros (); + test_copy (); + test_callbacks (); + test_maps (); + test_loops (); +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified backend. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template void run_tests() { - test_prezeros (); - test_precopy (); - test_premaps (); - test_preloops (); - test_zeros (); - test_copy (); - test_maps (); - test_loops (); + run_tests_order (); + run_tests_order (); + run_tests_order (); } //------------------------------------------------------------------------------ From 85afa52e83305fb070649974758906ea184e162d Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 15 Jul 2026 16:48:37 -0400 Subject: [PATCH 16/51] Inital working pic code. Fix bug where or was accidently using add nodes. --- graph_framework/logical.hpp | 2 +- graph_framework/metal_context.hpp | 106 ++++++++++++++++++++++++--- graph_framework/particle_in_cell.hpp | 49 +++++++------ graph_framework/timing.hpp | 27 ++++++- graph_pic/xpic.cpp | 46 ++++++------ 5 files changed, 169 insertions(+), 61 deletions(-) diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index 19cdd72..aed937e 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -2275,7 +2275,7 @@ namespace graph { template shared_leaf or_(shared_leaf l, shared_leaf r) { - auto temp = std::make_shared> (l, r)->reduce(); + auto temp = std::make_shared> (l, r)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index b9c61b5..637f186 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -13,6 +13,9 @@ #import #include "random.hpp" +#include "timing.hpp" + +//#define PROFILE /// Name space for GPU backends. namespace gpu { @@ -39,6 +42,13 @@ namespace gpu { /// Buffer mutability descriptor. std::map> bufferMutability; +#ifdef PROFILE +/// Timer map. + std::vector times; +/// Current index. + size_t index; +#endif + public: /// Size of random state needed. constexpr static size_t random_state_size = 1024; @@ -69,7 +79,11 @@ namespace gpu { //------------------------------------------------------------------------------ metal_context(const size_t index) : device([MTLCopyAllDevices() objectAtIndex:index]), - queue([device newCommandQueue]) {} + queue([device newCommandQueue]) +#ifdef PROFILE + , index(0) +#endif + {} //------------------------------------------------------------------------------ /// @brief Compile the kernels. @@ -238,8 +252,21 @@ namespace gpu { } if (state.get()) { - return [this, num_rays, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures] () mutable { +#ifdef PROFILE + size_t k = index++; + times.emplace_back(kernel_name); +#endif + return [this, num_rays, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures +#ifdef PROFILE + ,k +#endif + ] () mutable { command_buffer = [queue commandBuffer]; +#ifdef PROFILE + [command_buffer addScheduledHandler:[this, k](id commandBuffer) { + times[k].reset(); + }]; +#endif for (uint32_t i = 0; i < num_rays; i += threads_per_group) { id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; @@ -261,12 +288,29 @@ namespace gpu { threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; } - +#ifdef PROFILE + [command_buffer addCompletedHandler:[this, k](id commandBuffer) { + times[k].print(); + }]; +#endif [command_buffer commit]; }; } else { - return [this, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures] () mutable { +#ifdef PROFILE + size_t j = index++; + times.emplace_back(kernel_name); +#endif + return [this, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures +#ifdef PROFILE + , j +#endif + ] () mutable { command_buffer = [queue commandBuffer]; +#ifdef PROFILE + [command_buffer addScheduledHandler:[this, j](id commandBuffer) { + times[j].reset(); + }]; +#endif id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; [encoder setComputePipelineState:pipline]; @@ -279,7 +323,11 @@ namespace gpu { [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; - +#ifdef PROFILE + [command_buffer addCompletedHandler:[this, j](id commandBuffer) { + times[j].print(); + }]; +#endif [command_buffer commit]; }; } @@ -361,8 +409,21 @@ namespace gpu { buffers.push_back(kernel_arguments[input.get()]); } - return [this, buffers] () mutable { +#ifdef PROFILE + size_t j = index++; + times.emplace_back("zero_call"); +#endif + return [this, buffers +#ifdef PROFILE + , j +#endif + ] () mutable { command_buffer = [queue commandBuffer]; +#ifdef PROFILE + [command_buffer addScheduledHandler:[this, j](id commandBuffer) { + times[j].reset(); + }]; +#endif id encoder = [command_buffer blitCommandEncoder]; for (id buffer : buffers) { @@ -371,7 +432,11 @@ namespace gpu { value:0]; } [encoder endEncoding]; - +#ifdef PROFILE + [command_buffer addCompletedHandler:[this, j](id commandBuffer) { + times[j].print(); + }]; +#endif [command_buffer commit]; }; } @@ -402,8 +467,21 @@ namespace gpu { sources.push_back(kernel_arguments[out.get()]); } - return [this, sources, destinations] () mutable { +#ifdef PROFILE + size_t j = index++; + times.emplace_back("copy_call"); +#endif + return [this, sources, destinations +#ifdef PROFILE + , j +#endif + ] () mutable { command_buffer = [queue commandBuffer]; +#ifdef PROFILE + [command_buffer addScheduledHandler:[this, j](id commandBuffer) { + times[j].reset(); + }]; +#endif id encoder = [command_buffer blitCommandEncoder]; for (size_t i = 0, ie = sources.size(); i < ie; i++) { @@ -414,7 +492,11 @@ namespace gpu { size:sources[i].length]; } [encoder endEncoding]; - +#ifdef PROFILE + [command_buffer addCompletedHandler:[this, j](id commandBuffer) { + times[j].print(); + }]; +#endif [command_buffer commit]; }; } @@ -450,7 +532,13 @@ namespace gpu { command_buffer = [queue commandBuffer]; [command_buffer addCompletedHandler:[callback](id commandBuffer) { +#ifdef PROFILE + timing::measure_diagnostic timer("callback"); +#endif callback(); +#ifdef PROFILE + timer.print(); +#endif }]; [command_buffer commit]; diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index fa99c3d..d24fbe3 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -54,11 +54,11 @@ namespace pic { /// @returns (∑(m_i) + me)/(n_i + 1); //------------------------------------------------------------------------------ T make_m(const std::vector &ion_masses) { - T total_m = m_electron; + T total_m = static_cast (0); for (const T &mass : ion_masses) { total_m += mass; } - return total_m/(ion_masses.size() + 1); + return total_m/ion_masses.size(); } //------------------------------------------------------------------------------ @@ -68,11 +68,11 @@ namespace pic { /// @returns (∑(Z_i)*q + q)/(n_i + 1); //------------------------------------------------------------------------------ T make_q(const std::vector &ion_zs) { - T total_q = pic::q; + T total_q = static_cast (0); for (const uint8_t &z : ion_zs) { total_q += z*pic::q; } - return total_q/(ion_zs.size() + 1); + return total_q/ion_zs.size(); } public: @@ -109,7 +109,7 @@ namespace pic { const T ne) : m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), wpe(std::sqrt(ne*q*q/(m*epsilon0))), - t(1/wpe), l(wpe/c), v(c), te(m*v*v/kb), efield(m*c/(q*t)), + t(1/wpe), l(c/wpe), v(c), te(m*v*v/kb), efield(m*c/(q*t)), bfield(efield/c) {} }; @@ -585,28 +585,28 @@ namespace pic { // The mesh is already normalized so position_dist will be a normalized quantity. auto position_dist = graph::uniform_random (mesh.xmin, mesh.xmax, state); - auto phi_dist = graph::uniform_random (static_cast (0.0), - static_cast (2.0)*std::numbers::pi_v, + auto phi_dist = graph::uniform_random (static_cast (0), + static_cast (2)*std::numbers::pi_v, state); - auto r_dist = graph::uniform_random (static_cast (0.0), - static_cast (1.0), + auto r_dist = graph::uniform_random (std::numeric_limits::min(), + static_cast (1), state); const T vtpara = std::sqrt(2*params.t_para*q/ion.mass); - auto vpara = vtpara*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + auto vpara = vtpara*graph::sqrt(-graph::log(r_dist)) * graph::sin(phi_dist); - phi_dist = graph::uniform_random (static_cast (0.0), - static_cast (2.0)*std::numbers::pi_v, + phi_dist = graph::uniform_random (static_cast (0), + static_cast (2)*std::numbers::pi_v, state); - r_dist = graph::uniform_random (static_cast (0.0), - static_cast (1.0), + r_dist = graph::uniform_random (std::numeric_limits::min(), + static_cast (1), state); const T vtperp = std::sqrt(2*params.t_perp*q/ion.mass); - auto vperp1 = vtperp*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + auto vperp1 = vtperp*graph::sqrt(-graph::log(r_dist)) * graph::cos(phi_dist); - auto vperp2 = vtperp*graph::sqrt(-graph::log(static_cast (1) - r_dist)) + auto vperp2 = vtperp*graph::sqrt(-graph::log(r_dist)) * graph::sin(phi_dist); auto vperp = graph::sqrt(vperp1*vperp1 + vperp2*vperp2); @@ -650,12 +650,14 @@ namespace pic { /// /// @param[in] x The x position. /// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. /// @returns The expression for the magnetic field. //------------------------------------------------------------------------------ template graph::shared_leaf build_magnetic_field(graph::shared_leaf x, - const characteristics &norms) { - return (x*x*norms.l*norms.l + static_cast (0.5))/norms.bfield; + const characteristics &norms, + const parameters ¶ms) { + return (x*x*(norms.l*norms.l) + static_cast (0.5))*params.b0; } //------------------------------------------------------------------------------ @@ -682,7 +684,7 @@ namespace pic { auto y = mesh.template build_y_index (x, params); // Compression factor. - auto cf = build_magnetic_field(x, norms)/params.b0; + auto cf = build_magnetic_field(x, norms, params)/params.b0; // Scale factor. const T sf = ion.super_to_real()/(params.a0*mesh.dx); @@ -713,7 +715,7 @@ namespace pic { auto y = mesh.template build_dydx_index (x, params); // Compression factor. - auto cf = build_magnetic_field(x, norms)/params.b0; + auto cf = build_magnetic_field(x, norms, params)/params.b0; // Scale factor. const T sf = ion.super_to_real()/(params.a0*mesh.dx); @@ -767,9 +769,10 @@ namespace pic { auto dndx = (dn0dx + dn1dx + dn2dx + dn3dx)/static_cast (4); auto te = build_electron_temperature(x, norms); - auto pressure = te*n/q; + const T scale = norms.q/q; + auto pressure = te*n*scale; - return graph::none ()/n*(dndx*te/q + pressure->df(x)); + return graph::none ()/n*(dndx*te*scale + pressure->df(x)); } //------------------------------------------------------------------------------ @@ -838,7 +841,7 @@ namespace pic { const mesh &mesh, const characteristics &norms, const parameters ¶ms) { - auto bfield = build_magnetic_field (z[0], norms); + auto bfield = build_magnetic_field (z[0], norms, params); auto efield = build_interpolate_efield (z[0], ion, mesh, norms, params); auto temp = 0.5*z[2]*z[1]*bfield->df(z[0])/bfield; return { diff --git a/graph_framework/timing.hpp b/graph_framework/timing.hpp index 6439c09..763b3ac 100644 --- a/graph_framework/timing.hpp +++ b/graph_framework/timing.hpp @@ -20,9 +20,7 @@ namespace timing { /// Description of what is being timed. const std::string label; /// Starting time of the measure. - const std::chrono::high_resolution_clock::time_point start; -/// Ending time of the measure. - std::chrono::high_resolution_clock::time_point end; + std::chrono::high_resolution_clock::time_point start; public: //------------------------------------------------------------------------------ @@ -33,6 +31,21 @@ namespace timing { measure_diagnostic(const std::string message = "") : label(message), start(std::chrono::high_resolution_clock::now()) {} +//------------------------------------------------------------------------------ +/// @brief Construct a time diagnostic object. +/// +/// @param[in] md Object to copy. +//------------------------------------------------------------------------------ + measure_diagnostic(const measure_diagnostic &md) : + label(md.label), start(md.start) {} + +//------------------------------------------------------------------------------ +/// @brief Reset the time. +//------------------------------------------------------------------------------ + void reset() { + start = std::chrono::high_resolution_clock::now(); + } + //------------------------------------------------------------------------------ /// @brief Print the result. //------------------------------------------------------------------------------ @@ -86,6 +99,14 @@ namespace timing { measure_diagnostic_threaded(const std::string message = "") : label(message) {} +//------------------------------------------------------------------------------ +/// @brief Construct a time diagnostic object. +/// +/// @param[in] mdt Object to copy. +//------------------------------------------------------------------------------ + measure_diagnostic_threaded(const measure_diagnostic_threaded &mdt) : + label(mdt.label) {} + //------------------------------------------------------------------------------ /// @brief Start time for a given thread. /// diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index dff79b4..b1ae47e 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -17,19 +17,19 @@ template void run_pic() { const timing::measure_diagnostic init("Init Time"); // Sizes - const size_t num_particles = 10000;//00; - const size_t num_grid = 100; + const size_t num_particles = 3000000; + const size_t num_grid = 1000; const size_t num_batch = 10; const size_t num_ions = 1; - const size_t num_steps = 1; - const size_t num_sub_steps = 1; + const size_t num_steps = 100; + const size_t num_sub_steps = 2400; - const pic::characteristics norms({ - 2*pic::m_atomic - }, {1}, static_cast (2.5E19)); + const std::vector ion_masses{2*pic::m_atomic}; + const std::vector ion_zs{1}; + + const pic::characteristics norms(ion_masses, ion_zs, + static_cast (2.5E19)); - std::array ion_masses{pic::m_hydrogen}; - std::array ion_zs{1}; std::array density_fraction{1}; const T lmin = static_cast (-3.0); @@ -42,11 +42,11 @@ void run_pic() { const T ds = (lmax - lmin)/static_cast (num_grid - 1); std::vector> ions; - for(size_t i = 0; i < num_ions; i++) { + for (size_t i = 0; i < num_ions; i++) { T num_real = 0; for (size_t i = 0; i < num_grid; i++) { const T x = ds*i + lmin; - const T b = static_cast (0.1)*x*x + static_cast (0.5); + const T b = b0*(x*x + static_cast (0.5)); const T a = a0*b0/b; num_real += ne0*density_fraction[0]*a*ds; } @@ -54,9 +54,11 @@ void run_pic() { num_real, norms); } - const T gyro_period = ion_zs[0]*pic::q*b0/ion_masses[0]; + const T b_cv = 1.2; + const T cyclotron_frequency = ion_zs[0]*pic::q*b_cv/ion_masses[0]; + const T gyro_period = 2*std::numbers::pi_v/cyclotron_frequency; const T dtc = 0.25; - const pic::parameters params(b0, r1, r2, 3, 1.0E-4, + const pic::parameters params(b0, r1, r2, 100, 1.0E-4, dtc*gyro_period, 2.5, 2.5, norms); pic::mesh mesh(lmin, lmax, num_grid, norms); @@ -78,7 +80,6 @@ void run_pic() { auto ion_inits = pic::build_initialization (ions[i], mesh, norms, params, graph::random_state_cast(state)); - auto efield = build_interpolate_efield (ions[i].x, ions[i], mesh, norms, params); work.template add_item ({ ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() }, {}, { @@ -153,12 +154,12 @@ void run_pic() { graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3]) - }, {efield}, { + }, {}, { {particle_step[0], ions[i].get_x()}, {particle_step[1], ions[i].get_v_para()}, {particle_step[2], ions[i].get_v_perp()} }, NULL, "particle_push_" + ion_tag, num_particles); -#if 0 + auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, params, graph::random_state_cast(state)); work.add_item({ @@ -214,7 +215,6 @@ void run_pic() { {mesh_solve[0], graph::variable_cast(mesh.index)}, {mesh_solve[1], graph::variable_cast(mesh.y[0])} }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles/num_batch); -#endif } init.print(); @@ -227,13 +227,12 @@ void run_pic() { for (size_t i = 0; i < num_ions; i++) { const std::string ion_tag = jit::format_to_string(i); - auto efield = build_interpolate_efield (ions[i].x, ions[i], mesh, norms, params); ions[i].define_variables(p_file, p_datasets[i], work, ion_tag); - p_datasets[i].create_variable(p_file, "efield" + ion_tag, efield, work.get_context()); } p_file.end_define_mode(); std::atomic_size_t counter = 0; +#ifndef PROFILE std::thread progress = std::thread([&num_steps, &counter]() -> void { using namespace std::chrono_literals; do { @@ -243,17 +242,12 @@ void run_pic() { std::this_thread::sleep_for(1s); } while (counter < num_steps); }); - +#endif const timing::measure_diagnostic run("Run Time"); work.template run (); work.template run (); work.wait(); - //auto bfield = build_magnetic_field (ions[0].x, norms); - //for (size_t i = 0; i < num_particles; i++) { - // work.print(i, {ions[0].x, efield}); - //} - for (; counter < num_steps; counter++) { for (size_t i = 0; i < num_sub_steps; i++) { work.run(); @@ -263,7 +257,9 @@ void run_pic() { } counter = num_steps; +#ifndef PROFILE progress.join(); +#endif std::cout << "\33[2K\r" << "100% Complete" << std::endl; run.print(); } From d6d8e88f0d34333372e176a975851bb47fcf901c Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 17 Jul 2026 16:37:22 -0400 Subject: [PATCH 17/51] Fix error where the recursive pick filtering indexing was running the correct number of iterations. Remove unneeded character from index string reps. Add kernel profiling capability. --- CMakeLists.txt | 1 + graph_framework/CMakeLists.txt | 1 + graph_framework/cpu_context.hpp | 44 +++++++++- graph_framework/cuda_context.hpp | 110 ++++++++++++++++++++---- graph_framework/logical.hpp | 2 +- graph_framework/metal_context.hpp | 124 ++++++++------------------- graph_framework/particle_in_cell.hpp | 12 +-- graph_framework/piecewise.hpp | 16 ++-- graph_pic/xpic.cpp | 52 +++++++++-- 9 files changed, 230 insertions(+), 132 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c42789..f56d106 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ option (USE_CONSTANT_CACHE "Cache the value of constants in kernel registers." O option (SHOW_USE_COUNT "Add a comment showing the use count in kernel sources." OFF) option (USE_INDEX_CACHE "Cache index values instead of computing them every time." OFF) option (USE_VERBOSE "Verbose jit option." OFF) +option (PROFILE_KERNELS "Display kernel timing information" OFF) option (BUILD_C_BINDING "Build C interface." OFF) option (BUILD_Fortran_BINDING "Build Fortran interface." OFF) diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 535bc9c..5fc4de0 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -26,6 +26,7 @@ target_compile_definitions (graph_framework $<$:SHOW_USE_COUNT> $<$:USE_INDEX_CACHE> $,USE_VERBOSE=true,USE_VERBOSE=false> + $<$:PROFILE_KERNELS> ) target_include_directories (graph_framework diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 9eb5666..f9f6efc 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -272,8 +272,18 @@ namespace gpu { << std::endl; } - return [kernel, buffers, state] () mutable { + return [kernel, buffers, state +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("callback"); +#endif kernel(buffers, state->data()); +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } else { auto kernel = entry.toPtr &)> (); @@ -290,8 +300,18 @@ namespace gpu { << std::endl; } - return [kernel, buffers] () mutable { + return [kernel, buffers +#ifdef PROFILE_KERNELS + , kernel_name +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("callback"); +#endif kernel(buffers); +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } } @@ -342,9 +362,15 @@ namespace gpu { } return [buffers, sizes] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("zero buffer"); +#endif for (size_t i = 0, ie = buffers.size(); i < ie; i++) { std::memset(buffers[i], 0, sizes[i]); } +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } @@ -377,9 +403,15 @@ namespace gpu { } return [sources, destinations, sizes] () mutable { +#ifdef PROFILE_KERNELS + timing::measure_diagnostic timer("copy buffer"); +#endif for (size_t i = 0, ie = sources.size(); i < ie; i++) { std::memcpy(destinations[i], sources[i], sizes[i]); } +#ifdef PROFILE_KERNELS + timer.print(); +#endif }; } @@ -404,7 +436,15 @@ namespace gpu { /// @returns Lambda to call the function. //------------------------------------------------------------------------------ std::function run_function(std::function callback) { +#ifdef PROFILE_KERNELS + return [callback]() { + timing::measure_diagnostic timer("callback"); + callback(); + timer.print(); + }; +#else return callback; +#endif } //------------------------------------------------------------------------------ diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index d017a54..d9f6a1e 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -91,6 +91,10 @@ namespace gpu { /// Cuda stream. CUstream stream; +#ifdef PROFILE_KERNELS + std::vector timers; +#endif + //------------------------------------------------------------------------------ /// @brief Check results of async cuda functions. /// @@ -107,8 +111,10 @@ namespace gpu { } public: +/// Random state size multiplyer. + constexpr static size_t random_state_scale = 1000; /// Size of random state needed. - constexpr static size_t random_state_size = 1024; + constexpr static size_t random_state_size = 1024*random_state_scale; /// Remaining constant memory in bytes. int remaining_const_memory; @@ -490,41 +496,73 @@ namespace gpu { check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); unsigned int threads_per_group = value; - unsigned int thread_groups = num_rays/threads_per_group + (num_rays%threads_per_group ? 1 : 0); + unsigned int total_parallel = state.get() ? random_state_size : num_rays; + unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); int min_grid; check_error(cuOccupancyMaxPotentialBlockSize(&min_grid, &value, function, 0, 0, 0), "cuOccupancyMaxPotentialBlockSize"); if (jit::verbose) { - std::cout << " Kernel name : " << kernel_name << std::endl; + std::cout << " Kernel name : " << kernel_name << std::endl; std::cout << " Threads per group : " << threads_per_group << std::endl; std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; + std::cout << " Total parallel : " << total_parallel; std::cout << " Min grid size : " << min_grid << std::endl; std::cout << " Suggested Block size : " << value << std::endl; } - +#ifdef PROFILE_KERNELS + timers.emplace_back(kernel_name); +#endif if (state.get()) { - return [this, num_rays, function, threads_per_group, buffers] () mutable { - for (uint32_t i = 0; i < num_rays; i += threads_per_group) { + return [this, num_rays, function, thread_groups, threads_per_group, buffers +#ifdef PROFILE_KERNELS + , timer = &timers.back() +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.reset(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif + for (uint32_t i = 0, ie = threads_per_group*thread_groups; i < num_rays; i += ie) { check_error_async(cuStreamWriteValue32(stream, offset_buffer, i, CU_STREAM_WRITE_VALUE_DEFAULT), "cuStreamWriteValue32"); check_error_async(cuLaunchKernel(function, - 1, 1, 1, + thread_groups, 1, 1, threads_per_group, 1, 1, 0, stream, buffers.data(), NULL), "cuLaunchKernel"); +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.print(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif } }; } else { - return [this, function, thread_groups, threads_per_group, buffers] () mutable { + return [this, function, thread_groups, threads_per_group, buffers +#ifdef PROFILE_KERNELS + , timer = &timers.back() +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.reset(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif check_error_async(cuLaunchKernel(function, thread_groups, 1, 1, threads_per_group, 1, 1, 0, stream, buffers.data(), NULL), "cuLaunchKernel"); +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.print(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif }; } } @@ -597,14 +635,31 @@ namespace gpu { buffers.push_back(kernel_arguments[input.get()]); } - return [this, buffers] () mutable { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); +#ifdef PROFILE_KERNELS + timers.emplace_back("zero buffer"); +#endif + return [this, buffers, size +#ifdef PROFILE_KERNELS + , timer = &timers.back() +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.reset(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif for (CUdeviceptr &buffer : buffers) { - size_t size; - check_error(cuMemGetAddressRange(NULL, &size, buffer), - "cuMemGetAddressRange"); check_error_async(cuMemsetD8Async(buffer, 0, size, stream), "cuMemsetD8Async"); } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.print(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif }; } @@ -647,11 +702,11 @@ namespace gpu { sources.push_back(kernel_arguments[out.get()]); } - return [this, sources, destinations] () mutable { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, sources[i]), + "cuMemGetAddressRange"); + return [this, sources, destinations, size] () mutable { for (size_t i = 0, ie = sources.size(); i < ie; i++) { - size_t size; - check_error(cuMemGetAddressRange(NULL, &size, sources[i]), - "cuMemGetAddressRange"); check_error_async(cuMemcpyDtoDAsync(destinations[i], sources[i], size, stream), @@ -675,10 +730,27 @@ namespace gpu { /// @returns Lambda to call the function. //------------------------------------------------------------------------------ std::function run_function(std::function callback) { - return [this]() { - check_error_async(cuLaunchHostFunc(stream, callback.target()), +#ifdef PROFILE_KERNELS + timers.emplace_back("callback"); +#endif + return [this, callback +#ifdef PROFILE_KERNELS + , timer = &timers.back() +#endif + ]() mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.reset(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif + check_error_async(cuLaunchHostFunc(stream, callback.target (), NULL), "cuLaunchHostFunc"); - } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.print(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif + }; } //------------------------------------------------------------------------------ diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index aed937e..faf12b5 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -2199,7 +2199,7 @@ namespace graph { return true; } - auto x_cast = and_cast(x); + auto x_cast = or_cast(x); if (x_cast.get()) { // or is commutative. if ((this->left->is_match(x_cast->get_left()) && diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 637f186..04980c1 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -15,8 +15,6 @@ #include "random.hpp" #include "timing.hpp" -//#define PROFILE - /// Name space for GPU backends. namespace gpu { //------------------------------------------------------------------------------ @@ -42,16 +40,11 @@ namespace gpu { /// Buffer mutability descriptor. std::map> bufferMutability; -#ifdef PROFILE -/// Timer map. - std::vector times; -/// Current index. - size_t index; -#endif - public: +/// Random state size multiplyer. + constexpr static size_t random_state_scale = 1000; /// Size of random state needed. - constexpr static size_t random_state_size = 1024; + constexpr static size_t random_state_size = 1024*random_state_scale; /// Remaining constant memory in bytes. NOT USED. int remaining_const_memory; @@ -79,11 +72,7 @@ namespace gpu { //------------------------------------------------------------------------------ metal_context(const size_t index) : device([MTLCopyAllDevices() objectAtIndex:index]), - queue([device newCommandQueue]) -#ifdef PROFILE - , index(0) -#endif - {} + queue([device newCommandQueue]) {} //------------------------------------------------------------------------------ /// @brief Compile the kernels. @@ -197,7 +186,7 @@ namespace gpu { descriptor.textureType = MTLTextureType1D; descriptor.pixelFormat = MTLPixelFormatR32Float; descriptor.width = size; - descriptor.storageMode = MTLStorageModeManaged; + descriptor.storageMode = MTLStorageModeShared; descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined; descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked; descriptor.usage = MTLTextureUsageShaderRead; @@ -218,7 +207,7 @@ namespace gpu { descriptor.pixelFormat = MTLPixelFormatR32Float; descriptor.width = size[1]; descriptor.height = size[0]; - descriptor.storageMode = MTLStorageModeManaged; + descriptor.storageMode = MTLStorageModeShared; descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined; descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked; descriptor.usage = MTLTextureUsageShaderRead; @@ -239,9 +228,10 @@ namespace gpu { NSRange range = NSMakeRange(0, buffers.size()); NSRange tex_range = NSMakeRange(0, textures.size()); - NSUInteger threads_per_group = pipline.maxTotalThreadsPerThreadgroup; + NSUInteger total_parallel = state.get() ? random_state_size : num_rays; NSUInteger thread_width = pipline.threadExecutionWidth; - NSUInteger thread_groups = num_rays/threads_per_group + (num_rays%threads_per_group ? 1 : 0); + NSUInteger threads_per_group = total_parallel < pipline.maxTotalThreadsPerThreadgroup ? thread_width : pipline.maxTotalThreadsPerThreadgroup; + NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); if (jit::verbose) { std::cout << " Kernel name : " << kernel_name << std::endl; @@ -249,25 +239,18 @@ namespace gpu { std::cout << " Threads per group : " << threads_per_group << std::endl; std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; + std::cout << " Total parallel size : " << total_parallel << std::endl; } if (state.get()) { -#ifdef PROFILE - size_t k = index++; - times.emplace_back(kernel_name); -#endif + return [this, num_rays, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures -#ifdef PROFILE - ,k +#ifdef PROFILE_KERNELS + , kernel_name #endif ] () mutable { command_buffer = [queue commandBuffer]; -#ifdef PROFILE - [command_buffer addScheduledHandler:[this, k](id commandBuffer) { - times[k].reset(); - }]; -#endif - for (uint32_t i = 0; i < num_rays; i += threads_per_group) { + for (NSUInteger i = 0, ie = thread_groups*threads_per_group; i < num_rays; i += ie) { id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; for (size_t j = 0, je = buffers.size() - 1; j < je; j++) { @@ -284,33 +267,24 @@ namespace gpu { [encoder setTextures:textures.data() withRange:tex_range]; - [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) + [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; } -#ifdef PROFILE - [command_buffer addCompletedHandler:[this, k](id commandBuffer) { - times[k].print(); +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[kernel_name](id commandBuffer) { + std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; }]; #endif [command_buffer commit]; }; } else { -#ifdef PROFILE - size_t j = index++; - times.emplace_back(kernel_name); -#endif return [this, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures -#ifdef PROFILE - , j +#ifdef PROFILE_KERNELS + , kernel_name #endif ] () mutable { command_buffer = [queue commandBuffer]; -#ifdef PROFILE - [command_buffer addScheduledHandler:[this, j](id commandBuffer) { - times[j].reset(); - }]; -#endif id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; [encoder setComputePipelineState:pipline]; @@ -323,9 +297,9 @@ namespace gpu { [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)]; [encoder endEncoding]; -#ifdef PROFILE - [command_buffer addCompletedHandler:[this, j](id commandBuffer) { - times[j].print(); +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[kernel_name](id commandBuffer) { + std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; }]; #endif [command_buffer commit]; @@ -409,32 +383,21 @@ namespace gpu { buffers.push_back(kernel_arguments[input.get()]); } -#ifdef PROFILE - size_t j = index++; - times.emplace_back("zero_call"); -#endif - return [this, buffers -#ifdef PROFILE - , j -#endif - ] () mutable { + const NSRange range = NSMakeRange(0, buffers.front().length); + + return [this, buffers, range] () mutable { command_buffer = [queue commandBuffer]; -#ifdef PROFILE - [command_buffer addScheduledHandler:[this, j](id commandBuffer) { - times[j].reset(); - }]; -#endif id encoder = [command_buffer blitCommandEncoder]; for (id buffer : buffers) { [encoder fillBuffer:buffer - range:NSMakeRange(0, buffer.length) + range:range value:0]; } [encoder endEncoding]; -#ifdef PROFILE - [command_buffer addCompletedHandler:[this, j](id commandBuffer) { - times[j].print(); +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[](id commandBuffer) { + std::cout << std::endl << " zero buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; }]; #endif [command_buffer commit]; @@ -467,21 +430,8 @@ namespace gpu { sources.push_back(kernel_arguments[out.get()]); } -#ifdef PROFILE - size_t j = index++; - times.emplace_back("copy_call"); -#endif - return [this, sources, destinations -#ifdef PROFILE - , j -#endif - ] () mutable { + return [this, sources, destinations] () mutable { command_buffer = [queue commandBuffer]; -#ifdef PROFILE - [command_buffer addScheduledHandler:[this, j](id commandBuffer) { - times[j].reset(); - }]; -#endif id encoder = [command_buffer blitCommandEncoder]; for (size_t i = 0, ie = sources.size(); i < ie; i++) { @@ -492,9 +442,9 @@ namespace gpu { size:sources[i].length]; } [encoder endEncoding]; -#ifdef PROFILE - [command_buffer addCompletedHandler:[this, j](id commandBuffer) { - times[j].print(); +#ifdef PROFILE_KERNELS + [command_buffer addCompletedHandler:[](id commandBuffer) { + std::cout << std::endl << " copy buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl; }]; #endif [command_buffer commit]; @@ -532,11 +482,11 @@ namespace gpu { command_buffer = [queue commandBuffer]; [command_buffer addCompletedHandler:[callback](id commandBuffer) { -#ifdef PROFILE +#ifdef PROFILE_KERNELS timing::measure_diagnostic timer("callback"); #endif callback(); -#ifdef PROFILE +#ifdef PROFILE_KERNELS timer.print(); #endif }]; @@ -739,7 +689,7 @@ namespace gpu { << " = " << jit::to_string('s', state.get()) << "[thread_index];" #ifdef SHOW_USE_COUNT - << " // used " << usage.at(input.get()) + << " // used " << usage.at(state.get()) #endif << std::endl; #else diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index d24fbe3..fcfcb1f 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -326,11 +326,11 @@ namespace pic { graph::shared_leaf build_y_index(graph::shared_leaf x, const T scale, const size_t iterations=0) const { - auto low = iterations ? build_y_index (x - dx, iterations - 1) : + auto low = iterations ? build_y_index (x - dx, scale, iterations - 1) : graph::index_1D(y[I], x, dx, xmin + dx); auto center = graph::index_1D(y[I], x, dx, xmin); - auto high = iterations ? build_y_index (x + dx, iterations - 1) : - graph::index_1D(y[I], x, dx, xmin + dx); + auto high = iterations ? build_y_index (x + dx, scale, iterations - 1) : + graph::index_1D(y[I], x, dx, xmin - dx); const T center_w = static_cast (0.5); const T side_w = static_cast (0.25); @@ -420,13 +420,13 @@ namespace pic { const parameters ¶ms) const { if constexpr (O == low) { return build_y_index (x - dx, params.smoothing, - params.filter_iterations); + params.filter_iterations - 1); } else if constexpr (O == center) { return build_y_index (x, params.smoothing, - params.filter_iterations); + params.filter_iterations - 1); } else { return build_y_index (x + dx, params.smoothing, - params.filter_iterations); + params.filter_iterations - 1); } } diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index 6fab9fc..097ac75 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -1469,10 +1469,10 @@ void compile_index(std::ostringstream &stream, shared_leaf x, const T scale, const T offset) { - return jit::format_to_string(v->get_hash()) + "[" + + return jit::format_to_string(v->get_hash()) + jit::format_to_string(x->get_hash()) + jit::format_to_string(scale) + - jit::format_to_string(offset) + "]"; + jit::format_to_string(offset); } public: @@ -1559,9 +1559,9 @@ void compile_index(std::ostringstream &stream, stream << " const "; jit::add_type (stream); auto var = this->left->compile(stream, - registers, - indices, - usage); + registers, + indices, + usage); stream << " " << registers[this] << " = " << jit::to_string('v', var.get()); #ifdef USE_INDEX_CACHE @@ -1821,13 +1821,13 @@ void compile_index(std::ostringstream &stream, shared_leaf y, const T y_scale, const T y_offset) { - return jit::format_to_string(v->get_hash()) + "[" + + return jit::format_to_string(v->get_hash()) + jit::format_to_string(x->get_hash()) + jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + "," + + jit::format_to_string(x_offset) + jit::format_to_string(y->get_hash()) + jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + "]"; + jit::format_to_string(x_offset); } public: diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index b1ae47e..4218f4d 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -5,6 +5,7 @@ #include #include +#include #include "../graph_framework/graph_framework.hpp" @@ -21,8 +22,8 @@ void run_pic() { const size_t num_grid = 1000; const size_t num_batch = 10; const size_t num_ions = 1; - const size_t num_steps = 100; - const size_t num_sub_steps = 2400; + const size_t num_steps = 1; + const size_t num_sub_steps = 100; const std::vector ion_masses{2*pic::m_atomic}; const std::vector ion_zs{1}; @@ -58,7 +59,7 @@ void run_pic() { const T cyclotron_frequency = ion_zs[0]*pic::q*b_cv/ion_masses[0]; const T gyro_period = 2*std::numbers::pi_v/cyclotron_frequency; const T dtc = 0.25; - const pic::parameters params(b0, r1, r2, 100, 1.0E-4, + const pic::parameters params(b0, r1, r2, 3, 1.0E-4, dtc*gyro_period, 2.5, 2.5, norms); pic::mesh mesh(lmin, lmax, num_grid, norms); @@ -74,6 +75,9 @@ void run_pic() { std::vector> p_datasets(num_ions, output::data_set (p_file)); + std::vector ion_sync(num_ions); + std::mutex mesh_sync; + for (size_t i = 0; i < num_ions; i++) { const std::string ion_tag = jit::format_to_string(i); @@ -136,13 +140,29 @@ void run_pic() { }); } + + work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { + ion_sync[i].lock(); + std::thread async([i, &p_file, &p_datasets, &ion_sync]() { + p_datasets[i].write(p_file); + ion_sync[i].unlock(); + }); + async.detach(); + }); if (i == 0) { - work.template add_callback_item ([&f_file, &mesh_dataset]() { - mesh_dataset.write(f_file); + work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_sync.lock(); + std::thread async([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_dataset.write(f_file); + mesh_sync.unlock(); + }); + async.detach(); }); } - work.template add_callback_item ([i, &p_file, &p_datasets]() { - p_datasets[i].write(p_file); + + work.add_callback_item([i, &ion_sync]() { + ion_sync[i].lock(); + ion_sync[i].unlock(); }); auto particle_step = pic::build_rk4_step(ions[i], mesh, norms, params); @@ -187,6 +207,10 @@ void run_pic() { }, NULL, "compute_weights_" + ion_tag, num_particles); if (i == 0) { + work.add_callback_item([&mesh_sync]() { + mesh_sync.lock(); + mesh_sync.unlock(); + }); work.add_zero_item({ graph::variable_cast(mesh.index), graph::variable_cast(mesh.y[0]) @@ -245,21 +269,29 @@ void run_pic() { #endif const timing::measure_diagnostic run("Run Time"); work.template run (); - work.template run (); work.wait(); + work.template run (); for (; counter < num_steps; counter++) { for (size_t i = 0; i < num_sub_steps; i++) { work.run(); } - work.template run (); work.wait(); + work.template run (); } counter = num_steps; + work.wait(); #ifndef PROFILE progress.join(); #endif + for (std::mutex &ion : ion_sync) { + ion.lock(); + ion.unlock(); + } + mesh_sync.lock(); + mesh_sync.unlock(); + std::cout << "\33[2K\r" << "100% Complete" << std::endl; run.print(); } @@ -275,6 +307,8 @@ int main(int argc, const char * argv[]) { (void)argc; (void)argv; + jit::verbose = true; + const timing::measure_diagnostic total("Total Time"); run_pic (); total.print(); From e0e627e623a8aa6aaf15ca8e9c2ac0f7754f4583 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 17 Jul 2026 17:05:06 -0400 Subject: [PATCH 18/51] Fix compile error and allow copy and zero buffers to have different sizes. --- graph_framework/cuda_context.hpp | 50 +++++++++++++++++++++++-------- graph_framework/metal_context.hpp | 13 ++++---- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index d9f6a1e..bb13fbe 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -635,15 +635,19 @@ namespace gpu { buffers.push_back(kernel_arguments[input.get()]); } - size_t size; - check_error(cuMemGetAddressRange(NULL, &size, buffer), - "cuMemGetAddressRange"); + std::vector sizes; + for (CUdeviceptr &buffer : buffers) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); + size.push_back(size); + } #ifdef PROFILE_KERNELS timers.emplace_back("zero buffer"); #endif - return [this, buffers, size + return [this, buffers, sizes #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer = &timers.back() #endif ] () mutable { #ifdef PROFILE_KERNELS @@ -651,8 +655,9 @@ namespace gpu { timer.reset(); }.target (), NULL), "cuLaunchHostFunc"); #endif - for (CUdeviceptr &buffer : buffers) { - check_error_async(cuMemsetD8Async(buffer, 0, size, stream), + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + check_error_async(cuMemsetD8Async(buffers[i], 0, sizes[i], + stream), "cuMemsetD8Async"); } #ifdef PROFILE_KERNELS @@ -702,16 +707,37 @@ namespace gpu { sources.push_back(kernel_arguments[out.get()]); } - size_t size; - check_error(cuMemGetAddressRange(NULL, &size, sources[i]), - "cuMemGetAddressRange"); - return [this, sources, destinations, size] () mutable { + std::vector sizes; + for (CUdeviceptr &buffer : buffers) { + size_t size; + check_error(cuMemGetAddressRange(NULL, &size, buffer), + "cuMemGetAddressRange"); + size.push_back(size); + } +#ifdef PROFILE_KERNELS + timers.emplace_back("copy buffer"); +#endif + return [this, sources, destinations, sizes +#ifdef PROFILE_KERNELS + , timer = &timers.back() +#endif + ] () mutable { +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.reset(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif for (size_t i = 0, ie = sources.size(); i < ie; i++) { check_error_async(cuMemcpyDtoDAsync(destinations[i], sources[i], - size, stream), + sizes[i], stream), "cuMemcpyDtoDAsync"); } +#ifdef PROFILE_KERNELS + check_error_async(cuLaunchHostFunc(stream, [&timer]() { + timer.print(); + }.target (), NULL), "cuLaunchHostFunc"); +#endif }; } diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 04980c1..8cd14ee 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -383,15 +383,18 @@ namespace gpu { buffers.push_back(kernel_arguments[input.get()]); } - const NSRange range = NSMakeRange(0, buffers.front().length); + std::vector ranges; + for (id buffer : buffers) { + ranges.push_back(NSMakeRange(0, buffer.length)); + } - return [this, buffers, range] () mutable { + return [this, buffers, ranges] () mutable { command_buffer = [queue commandBuffer]; id encoder = [command_buffer blitCommandEncoder]; - for (id buffer : buffers) { - [encoder fillBuffer:buffer - range:range + for (size_t i = 0, ie = buffers.size(); i < ie; i++) { + [encoder fillBuffer:buffers[i] + range:ranges[i] value:0]; } [encoder endEncoding]; From 86a7df485e79c37d37db7a71f9eda359c9110c14 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 17 Jul 2026 17:08:57 -0400 Subject: [PATCH 19/51] Fix compile error on cuda backends. --- graph_framework/cuda_context.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index bb13fbe..848d5aa 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -640,7 +640,7 @@ namespace gpu { size_t size; check_error(cuMemGetAddressRange(NULL, &size, buffer), "cuMemGetAddressRange"); - size.push_back(size); + sizes.push_back(size); } #ifdef PROFILE_KERNELS timers.emplace_back("zero buffer"); @@ -708,11 +708,11 @@ namespace gpu { } std::vector sizes; - for (CUdeviceptr &buffer : buffers) { + for (CUdeviceptr &buffer : sources) { size_t size; check_error(cuMemGetAddressRange(NULL, &size, buffer), "cuMemGetAddressRange"); - size.push_back(size); + sizes.push_back(size); } #ifdef PROFILE_KERNELS timers.emplace_back("copy buffer"); From fddd1c4407931d25844688e73516f7eb7b51c441 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 17 Jul 2026 17:11:57 -0400 Subject: [PATCH 20/51] Delete duplicate variable delcaration. --- graph_framework/cuda_context.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 848d5aa..e9b8065 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -677,7 +677,6 @@ namespace gpu { std::function create_copy_call(graph::copy_nodes &setters) { std::vector sources; std::vector destinations; - std::vector sizes; for (auto &[out, in] : setters) { if (!kernel_arguments.contains(in.get())) { From 60b96eacdf4ae1ba2f886b20d1eea1e7631251c5 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Sun, 19 Jul 2026 23:30:05 -0400 Subject: [PATCH 21/51] Check the correct variable for Kernel Profiling. --- graph_framework/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 5fc4de0..3d8767c 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -26,7 +26,7 @@ target_compile_definitions (graph_framework $<$:SHOW_USE_COUNT> $<$:USE_INDEX_CACHE> $,USE_VERBOSE=true,USE_VERBOSE=false> - $<$:PROFILE_KERNELS> + $<$:PROFILE_KERNELS> ) target_include_directories (graph_framework From edda8afc10e71523c968ecc2e4dc0800c0c1bcdf Mon Sep 17 00:00:00 2001 From: m4c Date: Mon, 20 Jul 2026 16:48:08 -0400 Subject: [PATCH 22/51] Fix function signatures in cuda backends so host functions run correctly. Avoid underlow by keeping max_base teh same floating type as the template T --- graph_framework/cuda_context.hpp | 94 ++++++++++++++++---------------- graph_framework/register.hpp | 6 +- graph_pic/xpic.cpp | 5 +- 3 files changed, 51 insertions(+), 54 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index e9b8065..ca56dc6 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -16,6 +16,7 @@ #include #include "random.hpp" +#include "timing.hpp" /// Maximum number of registers to use. #define MAX_REG 128 @@ -91,10 +92,6 @@ namespace gpu { /// Cuda stream. CUstream stream; -#ifdef PROFILE_KERNELS - std::vector timers; -#endif - //------------------------------------------------------------------------------ /// @brief Check results of async cuda functions. /// @@ -495,8 +492,8 @@ namespace gpu { int value; check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); - unsigned int threads_per_group = value; unsigned int total_parallel = state.get() ? random_state_size : num_rays; + unsigned int threads_per_group = total_parallel < 1024 ? 32 : value; unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); int min_grid; @@ -508,23 +505,23 @@ namespace gpu { std::cout << " Threads per group : " << threads_per_group << std::endl; std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; - std::cout << " Total parallel : " << total_parallel; + std::cout << " Total parallel : " << total_parallel << std::endl; std::cout << " Min grid size : " << min_grid << std::endl; std::cout << " Suggested Block size : " << value << std::endl; } #ifdef PROFILE_KERNELS - timers.emplace_back(kernel_name); + timing::measure_diagnostic timer(kernel_name); #endif if (state.get()) { return [this, num_rays, function, thread_groups, threads_per_group, buffers #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer #endif ] () mutable { #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.reset(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); #endif for (uint32_t i = 0, ie = threads_per_group*thread_groups; i < num_rays; i += ie) { check_error_async(cuStreamWriteValue32(stream, offset_buffer, i, @@ -536,32 +533,32 @@ namespace gpu { 0, stream, buffers.data(), NULL), "cuLaunchKernel"); + } #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.print(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); #endif - } }; } else { return [this, function, thread_groups, threads_per_group, buffers #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer #endif ] () mutable { #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.reset(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); #endif check_error_async(cuLaunchKernel(function, thread_groups, 1, 1, threads_per_group, 1, 1, 0, stream, buffers.data(), NULL), "cuLaunchKernel"); #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.print(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); #endif }; } @@ -643,17 +640,17 @@ namespace gpu { sizes.push_back(size); } #ifdef PROFILE_KERNELS - timers.emplace_back("zero buffer"); + timing::measure_diagnostic timer("zero buffer"); #endif return [this, buffers, sizes #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer #endif ] () mutable { #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.reset(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); #endif for (size_t i = 0, ie = buffers.size(); i < ie; i++) { check_error_async(cuMemsetD8Async(buffers[i], 0, sizes[i], @@ -661,9 +658,9 @@ namespace gpu { "cuMemsetD8Async"); } #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.print(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); #endif }; } @@ -714,17 +711,17 @@ namespace gpu { sizes.push_back(size); } #ifdef PROFILE_KERNELS - timers.emplace_back("copy buffer"); + timing::measure_diagnostic timer("copy buffer"); #endif return [this, sources, destinations, sizes #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer #endif ] () mutable { #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.reset(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); #endif for (size_t i = 0, ie = sources.size(); i < ie; i++) { check_error_async(cuMemcpyDtoDAsync(destinations[i], @@ -733,9 +730,9 @@ namespace gpu { "cuMemcpyDtoDAsync"); } #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.print(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); #endif }; } @@ -756,24 +753,25 @@ namespace gpu { //------------------------------------------------------------------------------ std::function run_function(std::function callback) { #ifdef PROFILE_KERNELS - timers.emplace_back("callback"); + timing::measure_diagnostic timer("callback"); #endif return [this, callback #ifdef PROFILE_KERNELS - , timer = &timers.back() + , timer #endif ]() mutable { #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.reset(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->reset(); + }, &timer), "cuLaunchHostFunc"); #endif - check_error_async(cuLaunchHostFunc(stream, callback.target (), NULL), - "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast *> (arg)->operator()(); + }, &callback), "cuLaunchHostFunc"); #ifdef PROFILE_KERNELS - check_error_async(cuLaunchHostFunc(stream, [&timer]() { - timer.print(); - }.target (), NULL), "cuLaunchHostFunc"); + check_error_async(cuLaunchHostFunc(stream, [](void *arg) { + reinterpret_cast (arg)->print(); + }, &timer), "cuLaunchHostFunc"); #endif }; } diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index dedd7b9..da3ce1d 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -189,11 +189,11 @@ namespace jit { /// @returns The maximum number of digits needed. //------------------------------------------------------------------------------ template - constexpr int max_base() { + constexpr T max_base() { if constexpr (float_base) { - return std::numeric_limits::max(); + return static_cast (std::numeric_limits::max()); } else { - return std::numeric_limits::max(); + return static_cast (std::numeric_limits::max()); } } diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 4218f4d..070868d 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -140,7 +140,6 @@ void run_pic() { }); } - work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { ion_sync[i].lock(); std::thread async([i, &p_file, &p_datasets, &ion_sync]() { @@ -256,7 +255,7 @@ void run_pic() { p_file.end_define_mode(); std::atomic_size_t counter = 0; -#ifndef PROFILE +#ifndef PROFILE_KERNELS std::thread progress = std::thread([&num_steps, &counter]() -> void { using namespace std::chrono_literals; do { @@ -282,7 +281,7 @@ void run_pic() { counter = num_steps; work.wait(); -#ifndef PROFILE +#ifndef PROFILE_KERNELS progress.join(); #endif for (std::mutex &ion : ion_sync) { From 6a7d13eba8ac637665baa41df0a29585fba301ed Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 29 Jul 2026 13:34:31 -0400 Subject: [PATCH 23/51] Remove indices cache since indices are now stored in registers. Clean up show useage count to endline methods. --- graph_framework.xcodeproj/project.pbxproj | 2 + graph_framework/arithmetic.hpp | 151 +-- graph_framework/backend.hpp | 67 + graph_framework/cpu_context.hpp | 18 +- graph_framework/cuda_context.hpp | 37 +- graph_framework/jit.hpp | 14 +- graph_framework/logical.hpp | 78 +- graph_framework/math.hpp | 79 +- graph_framework/metal_context.hpp | 83 +- graph_framework/node.hpp | 45 +- graph_framework/piecewise.hpp | 1390 ++++++++++----------- graph_framework/random.hpp | 5 - graph_framework/trigonometry.hpp | 57 +- graph_pic/xpic.cpp | 2 +- graph_tests/no_derivative_test.cpp | 1 - 15 files changed, 959 insertions(+), 1070 deletions(-) diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index f862cd1..ecdb0aa 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -2211,6 +2211,7 @@ "\"CXX_ARGS=\\\"-I/Users/m4c/Projects/graph_framework/graph_framework -I/usr/local/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/v1 -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks -fgnuc-version=4.2.1 -std=gnu++2a\\\"\"", STATIC, "MACOS_LIB_RT=\\\"/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/lib/darwin/libclang_rt.osx.a\\\"", + USE_INPUT_CACHE, USE_INDEX_CACHE, "USE_VERBOSE=false", "$(inherited)", @@ -2387,6 +2388,7 @@ USE_METAL, "MACOS_LIB_RT=\\\"/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/lib/darwin/libclang_rt.osx.a\\\"", "USE_VERBOSE=false", + USE_INPUT_CACHE, USE_INDEX_CACHE, "\"CXX_ARGS=\\\"-I/Users/m4c/Projects/graph_framework/graph_framework -I/usr/local/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/v1 -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/21.0.0/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include -I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks -fgnuc-version=4.2.1 -std=gnu++2a\\\"\"", "$(inherited)", diff --git a/graph_framework/arithmetic.hpp b/graph_framework/arithmetic.hpp index 8a9aab0..5710aed 100644 --- a/graph_framework/arithmetic.hpp +++ b/graph_framework/arithmetic.hpp @@ -193,11 +193,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -206,13 +204,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -221,29 +219,29 @@ namespace graph { result.add_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.add_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.add_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.add_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // Identity reductions. @@ -637,23 +635,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -949,11 +943,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -962,13 +954,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -976,30 +968,30 @@ namespace graph { backend::buffer result = pl1->evaluate(); result.subtract_row(pr2->evaluate()); return piecewise_2D(result, - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.subtract_col(pr2->evaluate()); return piecewise_2D(result, - pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pl2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.subtract_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.subtract_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // (c1 + a) - c2 -> c3 + a // c1 - (c2 + a) -> c3 - a @@ -1467,23 +1459,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -1901,11 +1889,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -1914,13 +1900,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -1929,29 +1915,29 @@ namespace graph { result.multiply_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.multiply_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.multiply_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.multiply_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } // Move constants to the left. @@ -2508,23 +2494,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -2835,11 +2817,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -2848,13 +2828,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -2863,29 +2843,29 @@ namespace graph { result.divide_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.divide_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.divide_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.divide_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } if (this->left->is_match(this->right)) { @@ -3500,23 +3480,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -5071,27 +5047,22 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf m = this->middle->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -5479,23 +5450,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 65792e3..98b504a 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -238,6 +238,13 @@ for (T &d : *this) { \ apply_op(std::cos) } +//------------------------------------------------------------------------------ +/// @brief Take cos. +//------------------------------------------------------------------------------ + void real() { + apply_op(std::real) + } + //------------------------------------------------------------------------------ /// @brief Take erfi. //------------------------------------------------------------------------------ @@ -702,6 +709,66 @@ if (size() > x.size()) { \ return true; } +//------------------------------------------------------------------------------ +/// @brief Applies an associative function. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define build_assoc_func(func) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l = func(std::real(l), \ + std::real(right)); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = func(std::real(r), \ + std::real(left)); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = func(std::real(a[i]), \ + std::real(b[i])); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Max operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns max(a, b). +//------------------------------------------------------------------------------ + template + inline buffer max(buffer &a, + buffer &b) { + build_assoc_func(std::max); + } + +//------------------------------------------------------------------------------ +/// @brief Min operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns min(a, b). +//------------------------------------------------------------------------------ + template + inline buffer min(buffer &a, + buffer &b) { + build_assoc_func(std::min); + } + //------------------------------------------------------------------------------ /// @brief Applies an associative operator. /// diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index f9f6efc..961c209 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -592,11 +592,8 @@ namespace gpu { registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " mt_state &" << registers[state.get()] << " = " - << jit::to_string('s', state.get()) << "[0];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(state.get()) -#endif - << std::endl; + << jit::to_string('s', state.get()) << "[0]"; + state->endline(source_buffer, usage); } source_buffer << " for (size_t i = 0; i < " << size << "; i++) {" << std::endl; if (iterations > 1) { @@ -609,11 +606,8 @@ namespace gpu { jit::add_type (source_buffer); source_buffer << " " << registers[input.get()] << " = " << jit::to_string('v', input.get()) - << "[i]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) -#endif - << std::endl; + << "[i]"; + input->endline(source_buffer, usage); } } @@ -625,7 +619,6 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ @@ -634,7 +627,6 @@ namespace gpu { graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage, const size_t iterations=1) { std::unordered_set out_registers; @@ -642,7 +634,6 @@ namespace gpu { if (!out->is_match(in)) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('v', in.get()); source_buffer << "[i] = "; @@ -672,7 +663,6 @@ namespace gpu { !out_registers.contains(out.get())) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('o', out.get()); source_buffer << "[i] = "; diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index ca56dc6..f8f873c 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -933,13 +933,7 @@ namespace gpu { } for (size_t i = 1, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { - source_buffer << ", // " << inputs[i - 1]->get_symbol() -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - << " used " << usage.at(inputs[i - 1].get()) -#endif -#endif - << std::endl; + inputs[i]->endline(source_buffer, usage, ','); source_buffer << " "; if (is_constant[i]) { source_buffer << "const "; @@ -954,15 +948,8 @@ namespace gpu { if (!used_args.contains(outputs[i].get())) { if (i == 0) { if (inputs.size()) { - source_buffer << ", // " - << inputs[inputs.size() - 1]->get_symbol(); -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - source_buffer << " used " - << usage.at(inputs[inputs.size() - 1].get()); -#endif -#endif - source_buffer << std::endl; + inputs[inputs.size() - 1]->endline(source_buffer, + usage, ','); } } else { source_buffer << "," << std::endl; @@ -1003,11 +990,8 @@ namespace gpu { registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " mt_state &" << registers[state.get()] << " = " << jit::to_string('s', state.get()) - << "[threadIdx.x];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(state.get()) -#endif - << std::endl; + << "[index]"; + state->endline(source_buffer, usage); #else registers[state.get()] = jit::to_string('s', state.get()) + "[threadIdx.x]"; #endif @@ -1033,11 +1017,8 @@ namespace gpu { if (state.get()) { source_buffer << "offset[0] + "; } - source_buffer << "index]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) -#endif - << std::endl; + source_buffer << "index]"; + input->endline(source_buffer, usage); } #else registers[input.get()] = jit::to_string('v', input.get()) + "[index]"; @@ -1053,7 +1034,6 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ @@ -1062,7 +1042,6 @@ namespace gpu { graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage, const size_t iterations=1) { std::unordered_set out_registers; @@ -1070,7 +1049,6 @@ namespace gpu { if (!out->is_match(in)) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('v', in.get()) @@ -1108,7 +1086,6 @@ namespace gpu { !out_registers.contains(out.get())) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('o', out.get()) diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index 143b2bf..e057403 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -171,17 +171,16 @@ namespace jit { kernel_2dtextures[name], iterations); - register_map indices; for (auto &[out, in] : setters) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, usage); } for (auto &out : outputs) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, usage); } gpu_context.create_kernel_postfix(source_buffer, outputs, setters, state, - registers, indices, usage, + registers, usage, iterations); // Delete the registers so that they can be used again in other kernels. @@ -255,17 +254,16 @@ namespace jit { } } - register_map indices; for (auto &[out, in] : setters) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, usage); } for (auto &out : outputs) { - out->compile(source_buffer, registers, indices, usage); + out->compile(source_buffer, registers, usage); } gpu_context.create_kernel_postfix(source_buffer, outputs, setters, state, - registers, indices, usage); + registers, usage); // Delete the registers so that they can be used again in other kernels. std::vector removed_elements; diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index faf12b5..9bd463c 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -108,16 +108,18 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf arg = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -305,20 +307,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -585,20 +588,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -865,20 +869,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -1120,20 +1125,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -1375,20 +1381,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -1630,20 +1637,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('l', this); @@ -1884,21 +1892,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, - usage); + usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -2126,8 +2135,8 @@ namespace graph { or_node(shared_leaf l, shared_leaf r) : no_derivative> (l, r, - or_node::to_string(l.get(), r.get())) {} + branch_node> (l, r, + or_node::to_string(l.get(), r.get())) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of less than equal. @@ -2161,21 +2170,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, - usage); + usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -2475,25 +2485,25 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf c = this->left->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); shared_leaf t = this->middle->compile(stream, registers, - indices, usage); shared_leaf f = this->right->compile(stream, registers, - indices, usage); stream << " const "; jit::add_type (stream); diff --git a/graph_framework/math.hpp b/graph_framework/math.hpp index f22f3f6..3fa328e 100644 --- a/graph_framework/math.hpp +++ b/graph_framework/math.hpp @@ -75,17 +75,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Handle cases like sqrt(c*x) where c is constant or cases like sqrt((x^a)*y). @@ -158,19 +156,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -381,17 +376,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Reduce exp(log(x)) -> x @@ -428,19 +421,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -646,17 +636,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Reduce log(exp(x)) -> x @@ -693,19 +681,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -910,11 +895,9 @@ namespace graph { auto pl1 = piecewise_1D_cast(this->left); auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -922,13 +905,13 @@ namespace graph { if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -937,29 +920,29 @@ namespace graph { result.pow_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.pow_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.pow_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.pow_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } auto lp = pow_cast(this->left); @@ -1191,24 +1174,21 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r; auto temp = constant_cast(this->right); if (!temp.get() || !temp->is_integer()) { - r = this->right->compile(stream, registers, indices, usage); + r = this->right->compile(stream, registers, usage); } registers[this] = jit::to_string('r', this); @@ -1484,17 +1464,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } return this->shared_from_this(); @@ -1526,19 +1504,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 8cd14ee..8519a8d 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -232,14 +232,16 @@ namespace gpu { NSUInteger thread_width = pipline.threadExecutionWidth; NSUInteger threads_per_group = total_parallel < pipline.maxTotalThreadsPerThreadgroup ? thread_width : pipline.maxTotalThreadsPerThreadgroup; NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); + NSUInteger thread_group_memory = device.maxThreadgroupMemoryLength; if (jit::verbose) { std::cout << " Kernel name : " << kernel_name << std::endl; - std::cout << " Thread execution width : " << thread_width << std::endl; - std::cout << " Threads per group : " << threads_per_group << std::endl; - std::cout << " Number of groups : " << thread_groups << std::endl; - std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; - std::cout << " Total parallel size : " << total_parallel << std::endl; + std::cout << " Thread execution width : " << thread_width << std::endl; + std::cout << " Threads per group : " << threads_per_group << std::endl; + std::cout << " Number of groups : " << thread_groups << std::endl; + std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; + std::cout << " Total parallel size : " << total_parallel << std::endl; + std::cout << " Max thread group memory : " << thread_group_memory << std::endl; } if (state.get()) { @@ -609,14 +611,8 @@ namespace gpu { source_buffer << " " << (is_constant[i] ? "constant" : "device") << " float *" << jit::to_string('v', inputs[i].get()) - << " [[buffer(" << buffer_count++ << ")]], // " - << inputs[i]->get_symbol() -#ifndef USE_INPUT_CACHE -#ifdef SHOW_USE_COUNT - << " used " << usage.at(inputs[i].get()) -#endif -#endif - << std::endl; + << " [[buffer(" << buffer_count++ << ")]]"; + inputs[i]->endline(source_buffer, usage, ','); used_args.insert(inputs[i].get()); } } @@ -653,48 +649,59 @@ namespace gpu { << " [[texture(" << index++ << ")]]," << std::endl; } - if (state.get()) { - source_buffer << " uint thread_index [[thread_index_in_threadgroup]]," - << std::endl; - } source_buffer << " uint index [[thread_position_in_grid]]) {" << std::endl << " if ("; if (state.get()) { source_buffer << "offset + "; } source_buffer << "index < " << size << ") {" << std::endl; + + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (is_constant[i]) { +#ifdef USE_INPUT_CACHE + if (usage.at(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } +#else + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; +#endif + } + } + if (iterations > 1) { source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; } - for (auto &input : inputs) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (!is_constant[i]) { #ifdef USE_INPUT_CACHE - if (usage.at(input.get())) { - registers[input.get()] = jit::to_string('r', input.get()); - source_buffer << " const "; - jit::add_type (source_buffer); - source_buffer << " " << registers[input.get()] << " = " - << jit::to_string('v', input.get()) - << "[index]; // " << input->get_symbol() -#ifdef SHOW_USE_COUNT - << " used " << usage.at(input.get()) -#endif - << std::endl; - } + if (usage.at(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } #else - registers[input.get()] = jit::to_string('v', input.get()) + "[index]"; + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; #endif + } } if (state.get()) { #ifdef USE_INPUT_CACHE registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " device mt_state &" << registers[state.get()] << " = " << jit::to_string('s', state.get()) - << "[thread_index];" -#ifdef SHOW_USE_COUNT - << " // used " << usage.at(state.get()) -#endif - << std::endl; + << "[index]"; + state->endline(source_buffer, usage); #else registers[state.get()] = jit::to_string('s', state.get()) + "[thread_index]"; #endif @@ -709,7 +716,6 @@ namespace gpu { /// @param[in] setters Map outputs back to input values. /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. -/// @param[in,out] indices Map of used indices. /// @param[in] usage List of register usage count. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ @@ -718,7 +724,6 @@ namespace gpu { graph::map_nodes &setters, graph::shared_random_state state, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage, const size_t iterations=1) { std::unordered_set out_registers; @@ -726,7 +731,6 @@ namespace gpu { if (!out->is_match(in)) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('v', in.get()) @@ -745,7 +749,6 @@ namespace gpu { !out_registers.contains(out.get())) { graph::shared_leaf a = out->compile(source_buffer, registers, - indices, usage); source_buffer << " " << jit::to_string('o', out.get()) << "[index] = "; diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 786c364..3dce5fd 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -233,12 +233,10 @@ /// virtual shared_leaf /// compile(std::ostringstream &stream, /// jit::register_map ®isters, -/// jit::register_map &indices, /// const jit::register_usage &usage) { /// if (registers.find(this) == registers.end()) { /// shared_leaf a = this->arg->compile(stream, /// registers, -/// indices, /// usage); /// /// registers[this] = jit::to_string('r', this); @@ -456,14 +454,12 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual std::shared_ptr> compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) = 0; //------------------------------------------------------------------------------ @@ -638,14 +634,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in] usage List of register usage count. +/// @param[in] end The end character. //------------------------------------------------------------------------------ virtual void endline(std::ostringstream &stream, - const jit::register_usage &usage) + const jit::register_usage &usage, + const char end=';') #ifndef SHOW_USE_COUNT const #endif - final { - stream << ";" + { + stream << end #ifdef SHOW_USE_COUNT << " // used " << usage.at(this) #endif @@ -771,14 +769,12 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual std::shared_ptr> compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { registers[this] = jit::to_string('i', this); @@ -786,7 +782,7 @@ namespace graph { if constexpr (jit::use_cuda()) { stream << "int " << registers[this] << " = index"; } else if constexpr (jit::use_metal ()) { - stream << "int " << registers[this] << " = index"; + stream << "uint " << registers[this] << " = index"; } else { stream << "size_t " << registers[this] << " = i"; } @@ -948,14 +944,12 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { #ifdef USE_CONSTANT_CACHE @@ -1283,16 +1277,14 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { - return this->arg->compile(stream, registers, indices, usage); + return this->arg->compile(stream, registers, usage); } //------------------------------------------------------------------------------ @@ -1743,14 +1735,12 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { return this->shared_from_this(); } @@ -1877,6 +1867,27 @@ namespace graph { virtual shared_leaf get_power_exponent() const { return one (); } + +//------------------------------------------------------------------------------ +/// @brief End a line in the kernel source. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in] usage List of register usage count. +/// @param[in] end The end character. +//------------------------------------------------------------------------------ + virtual void endline(std::ostringstream &stream, + const jit::register_usage &usage, + const char end=';') + #ifndef SHOW_USE_COUNT + const + #endif + { + stream << end << " // " << symbol + #ifdef SHOW_USE_COUNT + << " used " << usage.at(this) + #endif + << std::endl; + } }; //------------------------------------------------------------------------------ diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index 097ac75..f4f3bf1 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -22,51 +22,330 @@ namespace graph { /// @param[in] scale Argument scale factor. /// @param[in] offset Argument offset factor. //------------------------------------------------------------------------------ -template -void compile_index(std::ostringstream &stream, - const std::string ®ister_name, - const size_t length, - const T scale, - const T offset) { - const std::string type = jit::type_to_string (); - stream << "(" << jit::smallest_uint_type (length) << ")min"; - if constexpr (!jit::use_metal () && - !jit::use_cuda()) { - stream << "<" << type << ">"; + template + void compile_index(std::ostringstream &stream, + const std::string ®ister_name, + const size_t length, + const T scale, + const T offset) { + const std::string type = jit::type_to_string (); + stream << "(" << jit::smallest_uint_type (length) << ")min"; + if constexpr (!jit::use_metal () && + !jit::use_cuda()) { + stream << "<" << type << ">"; + } + stream << "(max"; + if constexpr (!jit::use_metal () && + !jit::use_cuda ()) { + stream << "<" << type << ">"; + } + stream << "("; + if constexpr (jit::complex_scalar) { + stream << "real("; + } + stream << "(" << register_name << " - "; + if constexpr (jit::complex_scalar) { + stream << jit::get_type_string (); + } + stream << offset << ")/"; + if constexpr (jit::complex_scalar) { + stream << jit::get_type_string (); + } + stream << scale; + if constexpr (jit::complex_scalar) { + stream << ")"; + } + stream << ","; + if constexpr (jit::use_metal () || + jit::use_cuda()) { + stream << "(" << type << ")"; + } + stream << "0),"; + if constexpr (jit::use_metal () || + jit::use_cuda()) { + stream << "(" << type << ")"; + } + stream << length - 1 << ")"; } - stream << "(max"; - if constexpr (!jit::use_metal () && - !jit::use_cuda ()) { - stream << "<" << type << ">"; - } - stream << "("; - if constexpr (jit::complex_scalar) { - stream << "real("; - } - stream << "(" << register_name << " - "; - if constexpr (jit::complex_scalar) { - stream << jit::get_type_string (); - } - stream << offset << ")/"; - if constexpr (jit::complex_scalar) { - stream << jit::get_type_string (); - } - stream << scale; - if constexpr (jit::complex_scalar) { - stream << ")"; + +//------------------------------------------------------------------------------ +/// @brief Compile an 2D index. +/// +/// 2D indicies are flattened to a single index. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in] x_register_name Register for the x argument. +/// @param[in] y_register_name Register for the x argument. +/// @param[in] num_columns The y index. +/// @param +//------------------------------------------------------------------------------ + template + void compile_2D_index(std::ostringstream &stream, + const std::string &x_register_name, + const std::string &y_register_name, + const size_t num_columns) { + stream << x_register_name << "*" << num_columns << " + " + << y_register_name; } - stream << ","; - if constexpr (jit::use_metal () || - jit::use_cuda()) { - stream << "(" << type << ")"; + +//****************************************************************************** +// Argument node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Node class to contain the index argument. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class argument_node final : public no_derivative> { + private: +/// Scale factor. + const T scale; +/// Offset factor. + const T offset; +/// Length + const size_t length; + +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string with the argument. +/// +/// @param[in] x Argument. +/// @param[in] scale Scale factor for the argument. +/// @param[in] offset Offset factor for the argument. +/// @param[in] length Length of the array to index. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(shared_leaf x, + const T scale, + const T offset, + const size_t length) { + return jit::format_to_string(x->get_hash()) + + jit::format_to_string(scale) + + jit::format_to_string(offset) + + jit::format_to_string(length); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an Argument node. +/// +/// @param[in] x Argument. +/// @param[in] scale Scale factor for the argument. +/// @param[in] offset Offset factor for the argument. +/// @param[in] length Length of the array to index. +//------------------------------------------------------------------------------ + argument_node(shared_leaf x, + const T scale, + const T offset, + const size_t length) : + no_derivative> (x, argument_node::to_string(x, scale, + offset, + length)), + scale(scale), offset(offset), length(length) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the argument. +/// +/// Evaluate functions are only used by the minimization. So this node does not +/// evaluate the argument. Instead this only returns the data as if it were a +/// constant. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer result = this->arg->evaluate(); + backend::buffer o(1, offset); + backend::buffer s(1, scale); + result = result - o; + result = result/s; + result.real(); + backend::buffer upper(1, static_cast (length - 1)); + backend::buffer lower(1, static_cast (0)); + result = backend::min(result, upper); + result = backend::max(result, lower); + return result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduction method. +/// +/// If all the values in the data buffer are the same. Reduce to a single +/// constant. +/// +/// @returns A reduced representation of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + if (constant_cast(this->arg).get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// x' = (x - xmin)/dx (1) +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->arg->compile(stream, + registers, + usage); + +#ifdef USE_INDEX_CACHE + registers[this] = jit::to_string('i', this); + stream << " const " + << jit::smallest_uint_type (length) << " " + << registers[this] << " = "; + compile_index (stream, registers[a.get()], length, + scale, offset); + this->endline(stream, usage); +#else + std::ostringstream source_buffer; + compile_index (source_buffer, registers[a.get()], + length, scale, offset); + registers[this] = source_buffer.str(); +#endif + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + auto temp = argument_cast(x); + return temp.get() && + this->arg->is_match(temp->get_arg()) && + (temp->get_size() == this->length) && + (temp->get_scale() == this->scale) && + (temp->get_offset() == this->offset); + } + +//------------------------------------------------------------------------------ +/// @brief Get argument scale. +/// +/// @returns The scale factor for x. +//------------------------------------------------------------------------------ + T get_scale() const { + return scale; + } + +//------------------------------------------------------------------------------ +/// @brief Get argument offset. +/// +/// @returns The offset factor for x. +//------------------------------------------------------------------------------ + T get_offset() const { + return offset; + } + +//------------------------------------------------------------------------------ +/// @brief Get the size of the array. +/// +/// @returns The size of the array. +//------------------------------------------------------------------------------ + size_t get_size() const { + return length; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"arg\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto a = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[a.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Define argument convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @param[in] length The array length. +/// @returns A reduced argument node. +//------------------------------------------------------------------------------ + template + shared_leaf argument(shared_leaf x, + const T scale, + const T offset, + const size_t length) { + auto temp = std::make_shared> (x, scale, + offset, + length)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif } - stream << "0),"; - if constexpr (jit::use_metal () || - jit::use_cuda()) { - stream << "(" << type << ")"; + +/// Convenience type alias for shared argument nodes. + template + using shared_argument = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a argument node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_argument argument_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); } - stream << length - 1 << ")"; -} //****************************************************************************** // 1D Piecewise node. @@ -108,11 +387,6 @@ void compile_index(std::ostringstream &stream, template class piecewise_1D_node final : public straight_node { private: -/// Scale factor for the argument. - const T scale; -/// Offset factor for the argument. - const T offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -131,20 +405,14 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] d Backend buffer. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] d Backend buffer. +/// @param[in] x Argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) { + shared_leaf x) { return piecewise_1D_node::to_string(d) + - jit::format_to_string(x->get_hash()) + - jit::format_to_string(scale) + - jit::format_to_string(offset); + jit::format_to_string(x->get_hash()); } //------------------------------------------------------------------------------ @@ -180,18 +448,11 @@ void compile_index(std::ostringstream &stream, /// /// @param[in] d Data to initialize the piecewise constant. /// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. //------------------------------------------------------------------------------ piecewise_1D_node(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) : - straight_node (x, piecewise_1D_node::to_string(d, x, - scale, - offset)), - data_hash(piecewise_1D_node::hash_data(d)), scale(scale), - offset(offset) {} + shared_leaf x) : + straight_node (x, piecewise_1D_node::to_string(d, x)), + data_hash(piecewise_1D_node::hash_data(d)) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of the piecewise constant. @@ -216,18 +477,8 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ virtual shared_leaf reduce() { if (constant_cast(this->arg).get()) { - const T arg = (this->arg->evaluate().at(0) + offset)/scale; - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(arg), - this->get_size() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i]); - } else { - const size_t i = std::max (std::min (std::real(arg), - this->get_size() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i]); - } + const size_t i = std::real(this->arg->evaluate().at(0)); + return constant (leaf_node::caches.backends[data_hash][i]); } if (evaluate().is_same()) { @@ -345,34 +596,17 @@ void compile_index(std::ostringstream &stream, /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { -#ifdef USE_INDEX_CACHE - if (indices.find(this->arg.get()) == indices.end()) { -#endif - const size_t length = leaf_node::caches.backends[data_hash].size(); - shared_leaf a = this->arg->compile(stream, - registers, - indices, - usage); -#ifdef USE_INDEX_CACHE - indices[a.get()] = jit::to_string('i', a.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[a.get()] << " = "; - compile_index (stream, registers[a.get()], length, - scale, offset); - a->endline(stream, usage); - } -#endif + shared_leaf a = this->arg->compile(stream, + registers, + usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -397,42 +631,22 @@ void compile_index(std::ostringstream &stream, #endif stream << registers[leaf_node::caches.backends[data_hash].data()]; if constexpr (jit::use_metal ()) { -#ifdef USE_INDEX_CACHE stream << ".read(" - << indices[this->arg.get()] + << registers[a.get()] << ").r"; -#else - stream << ".read("; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << ").r"; -#endif #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { -#ifdef USE_INDEX_CACHE stream << ", " - << indices[this->arg.get()]; -#else - stream << ", "; - compile_index (stream, registers[a.get()], length, - scale, offset); -#endif + << registers[a.get()] if constexpr (jit::complex_scalar || jit::double_base) { stream << ")"; } stream << ")"; #endif } else { -#ifdef USE_INDEX_CACHE stream << "[" - << indices[this->arg.get()] + << registers[a.get()] << "]"; -#else - stream << "["; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << "]"; -#endif } this->endline(stream, usage); } @@ -452,12 +666,25 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = piecewise_1D_cast(x); - if (x_cast.get()) { - return this->data_hash == x_cast->data_hash && - this->is_arg_match(x); - } + return x_cast.get() && + this->data_hash == x_cast->data_hash && + this->arg->is_match(x_cast->get_arg()); + } - return false; +//------------------------------------------------------------------------------ +/// @brief Query if the nodes arguments match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = piecewise_1D_cast(x); + + return x_cast.get() && + this->arg->is_match(x_cast->get_arg()); } //------------------------------------------------------------------------------ @@ -543,48 +770,6 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf get_power_exponent() const { return one (); } - -//------------------------------------------------------------------------------ -/// @brief Check if the args match. -/// -/// @param[in] x Node to match. -/// @returns True if the arguments match. -//------------------------------------------------------------------------------ - bool is_arg_match(shared_leaf x) { - auto temp = piecewise_1D_cast(x); - return temp.get() && - this->arg->is_match(temp->get_arg()) && - (temp->get_size() == this->get_size()) && - (temp->get_scale() == this->scale) && - (temp->get_offset() == this->offset); - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_scale() const { - return scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_offset() const { - return offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. -/// -/// @returns The size of the buffer. -//------------------------------------------------------------------------------ - size_t get_size() const { - return leaf_node::caches.backends[data_hash].size(); - } }; //------------------------------------------------------------------------------ @@ -593,20 +778,14 @@ void compile_index(std::ostringstream &stream, /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] x Argument. -/// @param[in] scale Argument scale factor. -/// @param[in] offset Argument offset factor. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] x Argument. /// @returns A reduced piecewise_1D node. //------------------------------------------------------------------------------ template shared_leaf piecewise_1D(const backend::buffer &d, - shared_leaf x, - const T scale, - const T offset) { - auto temp = std::make_shared> (d, x, - scale, - offset)->reduce(); + shared_leaf x) { + auto temp = std::make_shared> (d, x)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -624,6 +803,27 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define piecewise_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @returns A reduced piecewise_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf piecewise_1D(const backend::buffer &d, + shared_leaf x, + const T scale, + const T offset) { + return piecewise_1D (d, argument(x, scale, + offset, d.size())); + } + /// Convenience type alias for shared piecewise 1D nodes. template using shared_piecewise_1D = std::shared_ptr>; @@ -689,15 +889,6 @@ void compile_index(std::ostringstream &stream, template class piecewise_2D_node final : public branch_node { private: -/// Scale factor for the x argument. - const T x_scale; -/// Offset factor for the x argument. - const T x_offset; -/// Scale factor for the y argument. - const T y_scale; -/// Offset factor for the y argument. - const T y_offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. /// @@ -718,27 +909,15 @@ void compile_index(std::ostringstream &stream, /// /// @param[in] d Backend buffer. /// @param[in] x X argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. /// @param[in] y Y argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(const backend::buffer &d, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { + shared_leaf y) { return piecewise_2D_node::to_string(d) + jit::format_to_string(x->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + - jit::format_to_string(y->get_hash()) + - jit::format_to_string(y_scale) + - jit::format_to_string(y_offset); + jit::format_to_string(y->get_hash()); } //------------------------------------------------------------------------------ @@ -774,31 +953,19 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ /// @brief Construct 2D a piecewise constant node. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] n Number of columns. -/// @param[in] x X Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X Argument. +/// @param[in] y Y Argument. //------------------------------------------------------------------------------ piecewise_2D_node(const backend::buffer &d, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) : + shared_leaf y) : branch_node (x, y, - piecewise_2D_node::to_string(d, - x, x_scale, x_offset, - y, y_scale, y_offset)), - data_hash(piecewise_2D_node::hash_data(d)), - num_columns(n), x_scale(x_scale), x_offset(x_offset), y_scale(y_scale), - y_offset(y_offset) { - assert(d.size()%n == 0 && + piecewise_2D_node::to_string(d, x, y)), + data_hash(piecewise_2D_node::hash_data(d)), num_columns(n) { + assert(d.size()%get_num_columns() == 0 && "Expected the data buffer to be a multiple of the number of columns."); } @@ -817,44 +984,7 @@ void compile_index(std::ostringstream &stream, /// @returns The number of columns in the constant. //------------------------------------------------------------------------------ size_t get_num_rows() const { - return leaf_node::caches.backends[data_hash].size() / - num_columns; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_x_scale() const { - return x_scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_x_offset() const { - return x_offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get y argument scale. -/// -/// @returns The scale factor for y. -//------------------------------------------------------------------------------ - T get_y_scale() const { - return y_scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get y argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_y_offset() const { - return y_offset; + return leaf_node::caches.backends[data_hash].size()/num_columns; } //------------------------------------------------------------------------------ @@ -881,58 +1011,20 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf reduce() { if (constant_cast(this->left).get() && constant_cast(this->right).get()) { - const T l = (this->left->evaluate().at(0) + x_offset)/x_scale; - const T r = (this->right->evaluate().at(0) + y_offset)/y_scale; - - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); - } else { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); - } + const size_t i = std::real(this->left->evaluate().at(0)); + const size_t j = std::real(this->right->evaluate().at(0)); + + return constant (leaf_node::caches.backends[data_hash][i*this->get_num_columns() + j]); } else if (constant_cast(this->left).get()) { - const T l = (this->left->evaluate().at(0) + x_offset)/x_scale; + const size_t i = std::real(this->left->evaluate().at(0)); - if constexpr (jit::float_base) { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), - this->right, y_scale, y_offset); - } else { - const size_t i = std::max (std::min (std::real(l), - this->get_num_rows() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), - this->right, y_scale, y_offset); - } + return piecewise_1D(leaf_node::caches.backends[data_hash].index_row(i, this->get_num_columns()), + this->right); } else if (constant_cast(this->right).get()) { - const T r = (this->right->evaluate().at(0) + y_offset)/y_scale; - - if constexpr (jit::float_base) { - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), - this->left, x_scale, x_offset); - } else { - const size_t j = std::max (std::min (std::real(r), - this->get_num_columns() - 1), - 0); - return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), - this->left, x_scale, x_offset); - } + const size_t j = std::real(this->right->evaluate().at(0)); + + return piecewise_1D(leaf_node::caches.backends[data_hash].index_column(j, this->get_num_columns()), + this->left); } if (evaluate().is_same()) { @@ -985,11 +1077,16 @@ void compile_index(std::ostringstream &stream, const size_t length = leaf_node::caches.backends[data_hash].size(); if constexpr (jit::use_metal ()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), this->get_num_columns() + })); #endif } else { if constexpr (jit::use_cuda()) { @@ -1021,11 +1118,17 @@ void compile_index(std::ostringstream &stream, const size_t length = leaf_node::caches.backends[data_hash].size(); if constexpr (jit::use_metal ()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { textures2d.try_emplace(leaf_node::caches.backends[data_hash].data(), - std::array ({length/num_columns, num_columns})); + std::array ({ + length/this->get_num_columns(), + this->get_num_columns() + })); #endif } } @@ -1068,66 +1171,39 @@ void compile_index(std::ostringstream &stream, /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - const size_t length = leaf_node::caches.backends[data_hash].size(); - const size_t num_rows = length/num_columns; - - shared_leaf x = this->left->compile(stream, - registers, - indices, - usage); - shared_leaf y = this->right->compile(stream, - registers, - indices, - usage); + auto x = this->left->compile(stream, registers, usage); + auto y = this->right->compile(stream, registers, usage); + auto temp = x*static_cast (this->get_num_columns()) + y; + if constexpr (!jit::use_metal ()) { + if (registers.find(temp.get()) == registers.end()) { +#ifndef USE_CUDA_TEXTURES #ifdef USE_INDEX_CACHE - if (indices.find(x.get()) == indices.end()) { - indices[x.get()] = jit::to_string('i', x.get()); - stream << " const " - << jit::smallest_uint_type (num_rows) << " " - << indices[x.get()] << " = "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - x->endline(stream, usage); - } - if (indices.find(y.get()) == indices.end()) { - indices[y.get()] = jit::to_string('i', y.get()); - stream << " const " - << jit::smallest_uint_type (num_columns) << " " - << indices[y.get()] << " = "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - y->endline(stream, usage); - } - - auto temp = this->left + this->right; - if constexpr (!jit::use_metal () -#ifdef USE_CUDA_TEXTURES - || !jit::use_cuda() -#endif - ) { - if (indices.find(temp.get()) == indices.end()) { - indices[temp.get()] = jit::to_string('i', temp.get()); + registers[temp.get()] = jit::to_string('i', temp.get()); stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[temp.get()] << " = " - << indices[x.get()] - << "*" << num_columns << " + " - << indices[y.get()] - << ";" << std::endl; + << jit::smallest_uint_type (this->get_num_columns()*this->get_num_rows()) + << " " << registers[temp.get()] << " = "; + compile_2D_index (stream, registers[x.get()], registers[y.get()], + this->get_num_columns()); + this->endline(stream, usage); +#else + std::ostringstream source_buffer; + temp->compile(source_buffer, + registers, + usage); + registers[temp.get()] = source_buffer.str(); +#endif +#endif } } -#endif registers[this] = jit::to_string('r', this); stream << " const "; @@ -1151,60 +1227,31 @@ void compile_index(std::ostringstream &stream, } #endif stream << registers[leaf_node::caches.backends[data_hash].data()]; + if constexpr (jit::use_metal ()) { -#ifdef USE_INDEX_CACHE stream << ".read(" - << jit::smallest_uint_type (std::max(num_rows, - num_columns)) + << jit::smallest_uint_type (std::max(this->get_num_rows(), + this->get_num_columns())) << "2(" - << indices[y.get()] + << registers[y.get()] << "," - << indices[x.get()] + << registers[x.get()] << ")).r"; -#else - stream << ".read(uint2("; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << ","; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << ")).r"; -#endif #ifdef USE_CUDA_TEXTURES } else if constexpr (jit::use_cuda()) { -#ifdef USE_INDEX_CACHE stream << ", " - << indices[y.get()] + << registers[y.get()] << ", " - << indices[x.get()]; -#else - stream << ", "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << ", "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); -#endif + << registers[x.get()]; if constexpr (jit::complex_scalar || jit::double_base) { stream << ")"; } stream << ")"; #endif } else { -#ifdef USE_INDEX_CACHE - stream << "[" - << indices[temp.get()] - << "]"; -#else - stream << "["; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << "*" << num_columns << " + "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << "]"; -#endif + stream << "[" << registers[temp.get()] << "]"; } + this->endline(stream, usage); } @@ -1222,12 +1269,28 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = piecewise_2D_cast(x); - if (x_cast.get()) { - return this->data_hash == x_cast->data_hash && - this->is_arg_match(x); - } + return x_cast.get() && + this->data_hash == x_cast->data_hash && + this->num_columns == x_cast->get_num_columns() && + this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right()); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// Assumes both arguments are either set or not set. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = piecewise_2D_cast(x); - return false; + return x_cast.get() && + this->num_columns == x_cast->get_num_columns() && + this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right()); } //------------------------------------------------------------------------------ @@ -1317,25 +1380,6 @@ void compile_index(std::ostringstream &stream, virtual shared_leaf get_power_exponent() const { return one (); } - -//------------------------------------------------------------------------------ -/// @brief Check if the args match. -/// -/// @param[in] x Node to match. -/// @returns True if the arguments match. -//------------------------------------------------------------------------------ - bool is_arg_match(shared_leaf x) { - auto temp = piecewise_2D_cast(x); - return temp.get() && - this->left->is_match(temp->get_left()) && - this->right->is_match(temp->get_right()) && - (temp->get_num_rows() == this->get_num_rows()) && - (temp->get_num_columns() == this->get_num_columns()) && - (temp->get_x_scale() == this->x_scale) && - (temp->get_x_offset() == this->x_offset) && - (temp->get_y_scale() == this->y_scale) && - (temp->get_y_offset() == this->y_offset); - } //------------------------------------------------------------------------------ /// @brief Do the rows match. @@ -1345,11 +1389,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_row_match(shared_leaf x) { auto temp = piecewise_1D_cast(x); - return temp.get() && - this->left->is_match(temp->get_arg()) && - (temp->get_size() == this->get_num_rows()) && - (temp->get_scale() == this->x_scale) && - (temp->get_offset() == this->x_offset); + return temp.get() && this->left->is_match(temp->get_arg()); } //------------------------------------------------------------------------------ @@ -1362,11 +1402,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_col_match(shared_leaf x) { auto temp = piecewise_1D_cast(x); - return temp.get() && - this->right->is_match(temp->get_arg()) && - (temp->get_size() == this->get_num_columns()) && - (temp->get_scale() == this->y_scale) && - (temp->get_offset() == this->y_offset); + return temp.get() && this->right->is_match(temp->get_arg()); } }; @@ -1376,28 +1412,18 @@ void compile_index(std::ostringstream &stream, /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] d Data to initialize the piecewise constant. -/// @param[in] n Number of columns. -/// @param[in] x X argument. -/// @param[in] x_scale Scale for x argument. -/// @param[in] x_offset Offset for x argument. -/// @param[in] y Argument. -/// @param[in] y_scale Scale for y argument. -/// @param[in] y_offset Offset for y argument. -/// @returns A reduced sqrt node. +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] y Y argument. +/// @returns A reduced piecewise_2D node. //------------------------------------------------------------------------------ - template + template shared_leaf piecewise_2D(const backend::buffer &d, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { - auto temp = std::make_shared> (d, n, - x, x_scale, x_offset, - y, y_scale, y_offset)->reduce(); + shared_leaf y) { + auto temp = std::make_shared> (d, n, x, y)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -1415,6 +1441,36 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define piecewise_2D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] d Data to initialize the piecewise constant. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] x_scale Scale for x argument. +/// @param[in] x_offset Offset for x argument. +/// @param[in] y Y argument. +/// @param[in] y_scale Scale for y argument. +/// @param[in] y_offset Offset for y argument. +/// @returns A reduced sqrt node. +//------------------------------------------------------------------------------ + template + shared_leaf piecewise_2D(const backend::buffer &d, + const size_t n, + shared_leaf x, + const T x_scale, + const T x_offset, + shared_leaf y, + const T y_scale, + const T y_offset) { + return piecewise_2D (d, n, + argument(x, x_scale, x_offset, d.size()/n), + argument(y, y_scale, y_offset, n))->reduce(); + } + /// Convenience type alias for shared piecewise 2D nodes. template using shared_piecewise_2D = std::shared_ptr>; @@ -1451,47 +1507,30 @@ void compile_index(std::ostringstream &stream, template class index_1D_node final : public branch_node { private: -/// Scale factor for the argument. - const T scale; -/// Offset factor for the argument. - const T offset; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] v Value to index. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] v Value to index. +/// @param[in] x Argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(shared_leaf v, - shared_leaf x, - const T scale, - const T offset) { + shared_leaf x) { return jit::format_to_string(v->get_hash()) + - jit::format_to_string(x->get_hash()) + - jit::format_to_string(scale) + - jit::format_to_string(offset); + jit::format_to_string(x->get_hash()); } public: //------------------------------------------------------------------------------ /// @brief Construct a 1D index. /// -/// @param[in] var Node to index. -/// @param[in] x Argument. -/// @param[in] scale Scale factor for the argument. -/// @param[in] offset Offset factor for the argument. +/// @param[in] var Node to index. +/// @param[in] x Argument. //------------------------------------------------------------------------------ index_1D_node(shared_leaf var, - shared_leaf x, - const T scale, - const T offset) : + shared_leaf x) : branch_node (var, x, - index_1D_node::to_string(var, x, - scale, offset)), - scale(scale), offset(offset) {} + index_1D_node::to_string(var, x)) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of the piecewise constant. @@ -1503,7 +1542,7 @@ void compile_index(std::ostringstream &stream, /// @returns The evaluated value of the node. //------------------------------------------------------------------------------ virtual backend::buffer evaluate() { - return this->right->evaluate(); + return this->left->evaluate(); } //------------------------------------------------------------------------------ @@ -1526,52 +1565,24 @@ void compile_index(std::ostringstream &stream, /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { -#ifdef USE_INDEX_CACHE - if (indices.find(this->right.get()) == indices.end()) { -#endif - const size_t length = variable_cast(this->left)->size(); - shared_leaf a = this->right->compile(stream, - registers, - indices, - usage); -#ifdef USE_INDEX_CACHE - indices[a.get()] = jit::to_string('i', a.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[a.get()] << " = "; - compile_index (stream, registers[a.get()], length, - scale, offset); - a->endline(stream, usage); - } -#endif + auto a = this->right->compile(stream, registers, usage); + auto var = this->left->compile(stream, registers, usage); registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - auto var = this->left->compile(stream, - registers, - indices, - usage); stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()); -#ifdef USE_INDEX_CACHE - stream << "[" << indices[this->right.get()] << "]"; -#else - stream << "["; - compile_index (stream, registers[a.get()], length, - scale, offset); - stream << "]"; -#endif + << jit::to_string('v', var.get()) + << "[" << registers[a.get()] << "]"; + this->endline(stream, usage); } @@ -1676,42 +1687,7 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_arg_match(shared_leaf x) { auto temp = index_1D_cast(x); - - if (temp.get()) { - return this->right->is_match(temp->get_right()) && - (temp->get_size() == this->get_size()) && - (temp->get_scale() == this->scale) && - (temp->get_offset() == this->offset); - } - - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_scale() const { - return scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_offset() const { - return offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. -/// -/// @returns The size of the buffer. -//------------------------------------------------------------------------------ - size_t get_size() const { - return variable_cast(this->left)->size(); + return temp.get() && this->right->is_match(temp->get_right()); } }; @@ -1723,20 +1699,14 @@ void compile_index(std::ostringstream &stream, /// /// @param[in] v Variable to index. /// @param[in] x Argument. -/// @param[in] scale Argument scale factor. -/// @param[in] offset Argument offset factor. /// @returns A reduced piecewise_1D node. //------------------------------------------------------------------------------ template shared_leaf index_1D(shared_leaf v, - shared_leaf x, - const T scale, - const T offset) { - assert(variable_cast(v).get() && - "index_1D requires a variable node for first arg."); - auto temp = std::make_shared> (v, x, - scale, - offset)->reduce(); + shared_leaf x) { + assert(argument_cast(x).get() && + "index_1D requires a argument node for second arg."); + auto temp = std::make_shared> (v, x)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { if (leaf_node::caches.nodes.find(i) == @@ -1754,6 +1724,28 @@ void compile_index(std::ostringstream &stream, #endif } +//------------------------------------------------------------------------------ +/// @brief Define index_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] x Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @returns A reduced piecewise_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf index_1D(shared_leaf v, + shared_leaf x, + const T scale, + const T offset) { + assert(variable_cast(v).get() && + "index_1D requires a variable node for first arg."); + return index_1D (v, argument(x, scale, offset, variable_cast(v)->size())); + } + /// Convenience type alias for shared index 1D nodes. template using shared_index_1D = std::shared_ptr>; @@ -1791,74 +1783,58 @@ void compile_index(std::ostringstream &stream, template class index_2D_node final : public triple_node { private: -/// Scale factor for the x argument. - const T x_scale; -/// Offset factor for the x argument. - const T x_offset; -/// Scale factor for the y argument. - const T y_scale; -/// Offset factor for the y argument. - const T y_offset; -/// Number of columns. - const size_t num_columns; - //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string with the argument. /// -/// @param[in] v Value to index. -/// @param[in] x Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] v Value to index. +/// @param[in] x X argument. +/// @param[in] y Y argument. /// @return A string rep of the node. //------------------------------------------------------------------------------ static std::string to_string(shared_leaf v, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) { + shared_leaf y) { return jit::format_to_string(v->get_hash()) + jit::format_to_string(x->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset) + - jit::format_to_string(y->get_hash()) + - jit::format_to_string(x_scale) + - jit::format_to_string(x_offset); + jit::format_to_string(y->get_hash()); } +/// Number of columns. + const size_t num_columns; + public: //------------------------------------------------------------------------------ /// @brief Construct a 2D index. /// -/// @param[in] var Node to index. -/// @param[in] n Number of columns. -/// @param[in] x X Argument. -/// @param[in] x_scale Scale factor for the argument. -/// @param[in] x_offset Offset factor for the x argument. -/// @param[in] y Y Argument. -/// @param[in] y_scale Scale factor for the y argument. -/// @param[in] y_offset Offset factor for the y argument. +/// @param[in] var Node to index. +/// @param[in] n Number of columns. +/// @param[in] x X Argument. +/// @param[in] y Y Argument. //------------------------------------------------------------------------------ index_2D_node(shared_leaf var, const size_t n, shared_leaf x, - const T x_scale, - const T x_offset, - shared_leaf y, - const T y_scale, - const T y_offset) : + shared_leaf y) : triple_node (var, x, y, - index_2D_node::to_string(var, - x, x_scale, x_offset, - y, y_scale, y_offset)), - num_columns(n), x_scale(x_scale), x_offset(x_offset), y_scale(y_scale), - y_offset(y_offset) { - assert(variable_cast(this->left)->size()%n == 0 && - "Expected the data buffer to be a multiple of the number of columns."); + index_2D_node::to_string(var, x, y)), + num_columns(n) {} + +//------------------------------------------------------------------------------ +/// @brief Get the number of columns. +/// +/// @returns The number of columns in the constant. +//------------------------------------------------------------------------------ + size_t get_num_columns() const { + return num_columns; + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of columns. +/// +/// @returns The number of columns in the constant. +//------------------------------------------------------------------------------ + size_t get_num_rows() const { + return variable_cast(this->left)->size()/num_columns; } //------------------------------------------------------------------------------ @@ -1895,86 +1871,50 @@ void compile_index(std::ostringstream &stream, /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - const size_t length = variable_cast(this->left)->size(); - const size_t num_rows = length/num_columns; - - shared_leaf x = this->middle->compile(stream, - registers, - indices, - usage); - shared_leaf y = this->right->compile(stream, - registers, - indices, - usage); + auto x = this->middle->compile(stream, registers, usage); + auto y = this->right->compile(stream, registers, usage); + auto temp = x*static_cast (this->get_num_columns()) + + y; + + if (registers.find(temp.get()) == registers.end()) { #ifdef USE_INDEX_CACHE - if (indices.find(x.get()) == indices.end()) { - indices[x.get()] = jit::to_string('i', x.get()); - stream << " const " - << jit::smallest_uint_type (num_rows) << " " - << indices[x.get()] << " = "; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - x->endline(stream, usage); - } - if (indices.find(y.get()) == indices.end()) { - indices[y.get()] = jit::to_string('i', y.get()); + registers[temp.get()] = jit::to_string('i', temp.get()); stream << " const " - << jit::smallest_uint_type (num_columns) << " " - << indices[y.get()] << " = "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - y->endline(stream, usage); + << jit::smallest_uint_type (this->get_num_columns()*this->get_num_rows()) + << " " << registers[temp.get()] << " = "; + compile_2D_index (stream, registers[x.get()], registers[y.get()], + this->get_num_columns()); + this->endline(stream, usage); + +#else + std::ostringstream source_buffer; + compile_2D_index (source_buffer, registers[x.get()], registers[y.get()], + this->get_num_columns()); + registers[temp.get()] = source_buffer.str(); +#endif } - auto temp = this->middle + this->right; - if constexpr (!jit::use_metal () || - !jit::use_cuda()) { - if (indices.find(temp.get()) == indices.end()) { - indices[temp.get()] = jit::to_string('i', temp.get()); - stream << " const " - << jit::smallest_uint_type (length) << " " - << indices[temp.get()] << " = " - << indices[x.get()] - << "*" << num_columns << " + " - << indices[y.get()] - << ";" << std::endl; - } - } -#endif + auto var = this->left->compile(stream, + registers, + usage); + registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - auto var = this->left->compile(stream, - registers, - indices, - usage); stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()); -#ifdef USE_INDEX_CACHE - stream << "[" - << indices[temp.get()] - << "]"; -#else - stream << "["; - compile_index (stream, registers[x.get()], num_rows, - x_scale, x_offset); - stream << "*" << num_columns << " + "; - compile_index (stream, registers[y.get()], num_columns, - y_scale, y_offset); - stream << "]"; -#endif + << jit::to_string('v', var.get()) + << "[" << registers[temp.get()] << "]"; + this->endline(stream, usage); } @@ -1992,12 +1932,11 @@ void compile_index(std::ostringstream &stream, virtual bool is_match(shared_leaf x) { auto x_cast = index_2D_cast(x); - if (x_cast.get()) { - return this->left->is_match(x_cast->get_left()) && - this->is_arg_match(x); - } - - return false; + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->middle->is_match(x_cast->get_middle()) && + this->right->is_match(x_cast->get_right()) && + this->num_columns == x_cast->get_num_columns(); } //------------------------------------------------------------------------------ @@ -2083,64 +2022,50 @@ void compile_index(std::ostringstream &stream, //------------------------------------------------------------------------------ bool is_arg_match(shared_leaf x) { auto temp = index_2D_cast(x); - - if (temp.get()) { - return this->right->is_match(temp->get_right()) && - (temp->get_size() == this->get_size()) && - (temp->get_x_scale() == this->x_scale) && - (temp->get_x_offset() == this->x_offset) && - (temp->get_y_scale() == this->y_scale) && - (temp->get_y_offset() == this->y_offset); - } - - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument scale. -/// -/// @returns The scale factor for x. -//------------------------------------------------------------------------------ - T get_x_scale() const { - return x_scale; - } - -//------------------------------------------------------------------------------ -/// @brief Get x argument offset. -/// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_x_offset() const { - return x_offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get y argument scale. -/// -/// @returns The scale factor for y. -//------------------------------------------------------------------------------ - T get_y_scale() const { - return y_scale; + return temp.get() && + this->middle->is_match(temp->get_middle()) && + this->right->is_match(temp->get_right()); } + }; //------------------------------------------------------------------------------ -/// @brief Get y argument offset. +/// @brief Define index_2D convenience function. /// -/// @returns The offset factor for x. -//------------------------------------------------------------------------------ - T get_y_offset() const { - return y_offset; - } - -//------------------------------------------------------------------------------ -/// @brief Get the size of the buffer. +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @returns The size of the buffer. +/// @param[in] v Variable to index. +/// @param[in] n Number of columns. +/// @param[in] x X argument. +/// @param[in] y Argument. +/// @returns A reduced sqrt node. //------------------------------------------------------------------------------ - size_t get_size() const { - return variable_cast(this->left)->size(); + template + shared_leaf index_2D(shared_leaf v, + const size_t n, + shared_leaf x, + shared_leaf y) { + assert(argument_cast(x).get() && + "index_2D requires a argument node for second arg."); + assert(argument_cast(y).get() && + "index_2D requires a argument node for third arg."); + auto temp = std::make_shared> (v, n, x, y)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } } - }; +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } //------------------------------------------------------------------------------ /// @brief Define index_2D convenience function. @@ -2169,24 +2094,11 @@ void compile_index(std::ostringstream &stream, const T y_offset) { assert(variable_cast(v).get() && "index_2D requires a variable node for first arg."); - auto temp = std::make_shared> (v, n, - x, x_scale, x_offset, - y, y_scale, y_offset)->reduce(); -// Test for hash collisions. - for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { - if (leaf_node::caches.nodes.find(i) == - leaf_node::caches.nodes.end()) { - leaf_node::caches.nodes[i] = temp; - return temp; - } else if (temp->is_match(leaf_node::caches.nodes[i])) { - return leaf_node::caches.nodes[i]; - } - } -#if defined(__clang__) || defined(__GNUC__) - __builtin_unreachable(); -#else - assert(false && "Should never reach."); -#endif + return index_2D (v, n, + argument(x, x_scale, x_offset, + variable_cast(v)->size()/n), + argument(y, y_scale, + y_offset, n))->reduce(); } /// Convenience type alias for shared index 2D nodes. diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index df1fe10..22161fe 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -119,14 +119,12 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { return this->shared_from_this(); } @@ -407,19 +405,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = "random(" + registers[a.get()] + ")"; diff --git a/graph_framework/trigonometry.hpp b/graph_framework/trigonometry.hpp index 558edfe..25b376b 100644 --- a/graph_framework/trigonometry.hpp +++ b/graph_framework/trigonometry.hpp @@ -68,18 +68,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { - return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + return piecewise_1D(this->evaluate(), ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Sin(ArcTan(x, y)) -> y/Sqrt(x^2 + y^2) @@ -128,18 +125,15 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -326,17 +320,15 @@ namespace graph { auto ap1 = piecewise_1D_cast(this->arg); if (ap1.get()) { return piecewise_1D(this->evaluate(), - ap1->get_arg(), - ap1->get_scale(), - ap1->get_offset()); + ap1->get_arg()); } auto ap2 = piecewise_2D_cast(this->arg); if (ap2.get()) { return piecewise_2D(this->evaluate(), ap2->get_num_columns(), - ap2->get_left(), ap2->get_x_scale(), ap2->get_x_offset(), - ap2->get_right(), ap2->get_y_scale(), ap2->get_y_offset()); + ap2->get_left(), + ap2->get_right()); } // Cos(ArcTan(x, y)) -> x/Sqrt(x^2 + y^2) @@ -385,19 +377,16 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf a = this->arg->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); @@ -605,11 +594,9 @@ namespace graph { auto pr1 = piecewise_1D_cast(this->right); if (pl1.get() && (r.get() || pl1->is_arg_match(this->right))) { - return piecewise_1D(this->evaluate(), pl1->get_arg(), - pl1->get_scale(), pl1->get_offset()); + return piecewise_1D(this->evaluate(), pl1->get_arg()); } else if (pr1.get() && (l.get() || pr1->is_arg_match(this->left))) { - return piecewise_1D(this->evaluate(), pr1->get_arg(), - pr1->get_scale(), pr1->get_offset()); + return piecewise_1D(this->evaluate(), pr1->get_arg()); } auto pl2 = piecewise_2D_cast(this->left); @@ -618,13 +605,13 @@ namespace graph { if (pl2.get() && (r.get() || pl2->is_arg_match(this->right))) { return piecewise_2D(this->evaluate(), pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pr2.get() && (l.get() || pr2->is_arg_match(this->left))) { return piecewise_2D(this->evaluate(), pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } // Combine 2D and 1D piecewise constants if a row or column matches. @@ -633,29 +620,29 @@ namespace graph { result.atan_row(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pr2.get() && pr2->is_col_match(this->left)) { backend::buffer result = pl1->evaluate(); result.atan_col(pr2->evaluate()); return piecewise_2D(result, pr2->get_num_columns(), - pr2->get_left(), pr2->get_x_scale(), pr2->get_x_offset(), - pr2->get_right(), pr2->get_y_scale(), pr2->get_y_offset()); + pr2->get_left(), + pr2->get_right()); } else if (pl2.get() && pl2->is_row_match(this->right)) { backend::buffer result = pl2->evaluate(); result.atan_row(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } else if (pl2.get() && pl2->is_col_match(this->right)) { backend::buffer result = pl2->evaluate(); result.atan_col(pr1->evaluate()); return piecewise_2D(result, pl2->get_num_columns(), - pl2->get_left(), pl2->get_x_scale(), pl2->get_x_offset(), - pl2->get_right(), pl2->get_y_scale(), pl2->get_y_offset()); + pl2->get_left(), + pl2->get_right()); } return this->shared_from_this(); @@ -688,23 +675,19 @@ namespace graph { /// /// @param[in,out] stream String buffer stream. /// @param[in,out] registers List of defined registers. -/// @param[in,out] indices List of defined indices. /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { shared_leaf l = this->left->compile(stream, registers, - indices, usage); shared_leaf r = this->right->compile(stream, registers, - indices, usage); registers[this] = jit::to_string('r', this); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 070868d..daf87d1 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -20,7 +20,7 @@ void run_pic() { // Sizes const size_t num_particles = 3000000; const size_t num_grid = 1000; - const size_t num_batch = 10; + const size_t num_batch = 1; const size_t num_ions = 1; const size_t num_steps = 1; const size_t num_sub_steps = 100; diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp index 9902883..c4e4873 100644 --- a/graph_tests/no_derivative_test.cpp +++ b/graph_tests/no_derivative_test.cpp @@ -19,7 +19,6 @@ class dummy : public graph::no_derivative compile(std::ostringstream &stream, jit::register_map ®isters, - jit::register_map &indices, const jit::register_usage &usage) { return this->shared_from_this(); } From a727c2cb59e44c0e06de193131231e2d08eba4a1 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 29 Jul 2026 18:19:29 -0400 Subject: [PATCH 24/51] Enable threadgroup memory usage in metal kernels. This reduces the particle_push kernel from 0.25s to 0.03s. --- graph_framework/arithmetic.hpp | 113 +++++++++--------- graph_framework/cpu_context.hpp | 27 +++-- graph_framework/jit.hpp | 93 ++------------- graph_framework/logical.hpp | 180 ++++++++++++++--------------- graph_framework/math.hpp | 68 ++++++----- graph_framework/metal_context.hpp | 146 ++++++++++++++++++++--- graph_framework/node.hpp | 44 ++++--- graph_framework/piecewise.hpp | 96 +++++++++------ graph_framework/random.hpp | 21 ++-- graph_framework/register.hpp | 8 +- graph_framework/trigonometry.hpp | 51 ++++---- graph_tests/no_derivative_test.cpp | 1 + 12 files changed, 468 insertions(+), 380 deletions(-) diff --git a/graph_framework/arithmetic.hpp b/graph_framework/arithmetic.hpp index 5710aed..9be6ad1 100644 --- a/graph_framework/arithmetic.hpp +++ b/graph_framework/arithmetic.hpp @@ -633,22 +633,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -1457,22 +1457,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -2492,22 +2492,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -3478,22 +3478,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -5045,25 +5045,24 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf m = this->middle->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto m = this->middle->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -5448,22 +5447,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 961c209..5e8780e 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -12,7 +12,6 @@ #include #include #include -#include // Clang headers will define IBAction and IBOutlet these so undefined them // here. @@ -278,7 +277,7 @@ namespace gpu { #endif ] () mutable { #ifdef PROFILE_KERNELS - timing::measure_diagnostic timer("callback"); + timing::measure_diagnostic timer(kernel_name); #endif kernel(buffers, state->data()); #ifdef PROFILE_KERNELS @@ -306,7 +305,7 @@ namespace gpu { #endif ] () mutable { #ifdef PROFILE_KERNELS - timing::measure_diagnostic timer("callback"); + timing::measure_diagnostic timer(kernel_name); #endif kernel(buffers); #ifdef PROFILE_KERNELS @@ -536,6 +535,8 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, @@ -549,6 +550,8 @@ namespace gpu { const jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" void " << name << "(" << std::endl; @@ -562,7 +565,7 @@ namespace gpu { } source_buffer << ") {" << std::endl; - std::unordered_set used_args; + jit::argument_set used_args; for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { source_buffer << " "; @@ -620,6 +623,8 @@ namespace gpu { /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, @@ -628,13 +633,14 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { - std::unordered_set out_registers; + jit::argument_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('v', in.get()); source_buffer << "[i] = "; if constexpr (SAFE_MATH) { @@ -661,9 +667,8 @@ namespace gpu { for (auto &out : outputs) { if (!graph::variable_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()); source_buffer << "[i] = "; if constexpr (SAFE_MATH) { diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index e057403..1e48a1c 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -163,24 +163,31 @@ namespace jit { } } + argument_set thread_shared; + jit::register_map thread_mem; + gpu_context.create_kernel_prefix(source_buffer, name, inputs, outputs, state, size, is_constant, registers, usage, kernel_1dtextures[name], kernel_2dtextures[name], + thread_shared, + thread_mem, iterations); for (auto &[out, in] : setters) { - out->compile(source_buffer, registers, usage); + out->compile(source_buffer, registers, thread_mem, usage); } for (auto &out : outputs) { - out->compile(source_buffer, registers, usage); + out->compile(source_buffer, registers, thread_mem, usage); } gpu_context.create_kernel_postfix(source_buffer, outputs, setters, state, registers, usage, + thread_shared, + thread_mem, iterations); // Delete the registers so that they can be used again in other kernels. @@ -198,88 +205,6 @@ namespace jit { } } -//------------------------------------------------------------------------------ -/// @brief Add a loop kernel. -/// -/// Build the source code for a kernel graph. -/// -/// @param[in] name Name to call the kernel. -/// @param[in] inputs Input variables of the kernel. -/// @param[in] outputs Output nodes of the graph to compute. -/// @param[in] setters Map outputs back to input values. -/// @param[in] state Random state node. -/// @param[in] size Size of the kernel. -/// @param[in] iterations Number of iterations of the loop. -//------------------------------------------------------------------------------ - void add_loop_kernel(const std::string name, - graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes setters, - graph::shared_random_state state, - const size_t size, - const size_t iterations) { - kernel_names.push_back(name); - - kernel_names.push_back(name); - - std::vector is_constant(inputs.size(), true); - visiter_map visited; - register_usage usage; - kernel_1dtextures[name] = texture1d_list(); - kernel_2dtextures[name] = texture2d_list(); - for (auto &[out, in] : setters) { - auto found = std::distance(inputs.begin(), - std::find(inputs.begin(), - inputs.end(), in)); - if (found < is_constant.size()) { - is_constant[found] = false; - } - out->compile_preamble(source_buffer, registers, - visited, usage, - kernel_1dtextures[name], - kernel_2dtextures[name], - gpu_context.remaining_const_memory); - } - for (auto &out : outputs) { - out->compile_preamble(source_buffer, registers, - visited, usage, - kernel_1dtextures[name], - kernel_2dtextures[name], - gpu_context.remaining_const_memory); - } - - for (auto &in : inputs) { - if (usage.find(in.get()) == usage.end()) { - usage[in.get()] = 0; - } - } - - for (auto &[out, in] : setters) { - out->compile(source_buffer, registers, usage); - } - for (auto &out : outputs) { - out->compile(source_buffer, registers, usage); - } - - gpu_context.create_kernel_postfix(source_buffer, outputs, - setters, state, - registers, usage); - -// Delete the registers so that they can be used again in other kernels. - std::vector removed_elements; - for (auto &[key, value] : registers) { - if (value[0] == 'r' || - value[0] == 'l' || - value[0] == 'i') { - removed_elements.push_back(key); - } - } - - for (auto &key : removed_elements) { - registers.erase(key); - } - } - //------------------------------------------------------------------------------ /// @brief Add max reduction kernel. /// diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index 9bd463c..41fd531 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -109,18 +109,19 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf arg = this->arg->compile(stream, - registers, - usage); + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -308,21 +309,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -589,21 +590,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -870,21 +871,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -1126,21 +1127,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -1382,21 +1383,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -1638,21 +1639,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -1893,21 +1894,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -2171,21 +2172,21 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('l', this); stream << " const bool "; @@ -2486,25 +2487,24 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf c = this->left->compile(stream, - registers, - usage); + auto c = this->left->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); - shared_leaf t = this->middle->compile(stream, - registers, - usage); - shared_leaf f = this->right->compile(stream, - registers, - usage); + auto t = this->middle->compile(stream, registers, + thread_mem, usage); + auto f = this->right->compile(stream, registers, + thread_mem, usage); stream << " const "; jit::add_type (stream); stream << " " << registers[this] << " = " diff --git a/graph_framework/math.hpp b/graph_framework/math.hpp index 3fa328e..abc51f0 100644 --- a/graph_framework/math.hpp +++ b/graph_framework/math.hpp @@ -154,19 +154,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -419,19 +420,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -679,19 +681,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -1172,23 +1175,25 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); shared_leaf r; auto temp = constant_cast(this->right); if (!temp.get() || !temp->is_integer()) { - r = this->right->compile(stream, registers, usage); + r = this->right->compile(stream, registers, + thread_mem, usage); } registers[this] = jit::to_string('r', this); @@ -1502,19 +1507,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 8519a8d..d3d639f 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -8,8 +8,6 @@ #ifndef metal_context_h #define metal_context_h -#include - #import #include "random.hpp" @@ -584,6 +582,8 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, @@ -597,16 +597,36 @@ namespace gpu { const jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { source_buffer << std::endl; source_buffer << "kernel void " << name << "(" << std::endl; bufferMutability[name] = std::vector (); + size_t used_thread_mem = 0; + size_t buffer_count = 0; - std::unordered_set used_args; + jit::argument_set used_args; for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { + if (!is_constant[i] && iterations > 1) { + const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*4 : 32*4; + if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + } + } else if (is_constant[i] && + inputs[i]->size() < size && + inputs[i]->size() < 1024) { + const size_t needed_mem = inputs[i]->size()*4; + if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); + } + } bufferMutability[name].push_back(is_constant[i] ? MTLMutabilityMutable : MTLMutabilityImmutable); source_buffer << " " << (is_constant[i] ? "constant" : "device") << " float *" @@ -649,7 +669,12 @@ namespace gpu { << " [[texture(" << index++ << ")]]," << std::endl; } - source_buffer << " uint index [[thread_position_in_grid]]) {" << std::endl + if (thread_shared.size()) { + source_buffer << " ushort t_index [[thread_position_in_threadgroup]]," << std::endl; + } + source_buffer << " " + << jit::smallest_uint_type (size) + << " index [[thread_position_in_grid]]) {" << std::endl << " if ("; if (state.get()) { source_buffer << "offset + "; @@ -659,7 +684,7 @@ namespace gpu { for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (is_constant[i]) { #ifdef USE_INPUT_CACHE - if (usage.at(inputs[i].get())) { + if (usage.at(inputs[i].get()) && inputs[i]->size() == size) { registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); source_buffer << " const "; jit::add_type (source_buffer); @@ -674,6 +699,64 @@ namespace gpu { } } + if (thread_shared.size()) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " threadgroup float " + << jit::to_string('t', inputs[i].get()) + << "[" << inputs[i]->size() << "]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " if (t_index < " + << inputs[i]->size() + << ") {" << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "[t_index]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " }" << std::endl + << " threadgroup_barrier(mem_flags::mem_threadgroup);" + << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + thread_shared.erase(inputs[i].get()); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) { + source_buffer << " threadgroup float " + << jit::to_string('t', inputs[i].get()) + << "[" + << (inputs[i]->size() > 1024 ? 1024 : 32) + << "]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } + } + } + if (iterations > 1) { source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; } @@ -685,13 +768,22 @@ namespace gpu { registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); source_buffer << " const "; jit::add_type (source_buffer); - source_buffer << " " << registers[inputs[i].get()] << " = " - << jit::to_string('v', inputs[i].get()) - << "[index]"; + source_buffer << " " << registers[inputs[i].get()] << " = "; + if (thread_shared.contains(inputs[i].get())) { + source_buffer << jit::to_string('t', inputs[i].get()) + << "[t_index]"; + } else { + source_buffer << jit::to_string('v', inputs[i].get()) + << "[index]"; + } inputs[i]->endline(source_buffer, usage); } #else - registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + if (thread_shared.contains(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; + } else { + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + } #endif } } @@ -717,6 +809,8 @@ namespace gpu { /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, @@ -725,16 +819,22 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { - std::unordered_set out_registers; + jit::argument_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); - source_buffer << " " - << jit::to_string('v', in.get()) - << "[index] = "; + auto a = out->compile(source_buffer, registers, + thread_mem, usage); + source_buffer << " "; + if (thread_shared.contains(in.get())) { + source_buffer << jit::to_string('t', in.get()) + << "[t_index] = "; + } else { + source_buffer << jit::to_string('v', in.get()) + << "[index] = "; + } if constexpr (SAFE_MATH) { source_buffer << "isnan(" << registers[a.get()] << ") ? 0.0 : "; @@ -747,9 +847,8 @@ namespace gpu { for (auto &out : outputs) { if (!graph::variable_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) << "[index] = "; if constexpr (SAFE_MATH) { @@ -764,6 +863,15 @@ namespace gpu { if (iterations > 1) { source_buffer << " }" << std::endl; } + for (auto &[out, in] : setters) { + if (thread_shared.contains(in.get())) { + source_buffer << " " + << jit::to_string('v', in.get()) + << "[index] = " + << jit::to_string('t', in.get()) + << "[t_index];" << std::endl; + } + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 3dce5fd..d531726 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -233,10 +233,12 @@ /// virtual shared_leaf /// compile(std::ostringstream &stream, /// jit::register_map ®isters, +/// const jit::register_map &thread_mem, /// const jit::register_usage &usage) { /// if (registers.find(this) == registers.end()) { /// shared_leaf a = this->arg->compile(stream, /// registers, +/// thread_mem, /// usage); /// /// registers[this] = jit::to_string('r', this); @@ -452,14 +454,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual std::shared_ptr> compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) = 0; //------------------------------------------------------------------------------ @@ -767,14 +771,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual std::shared_ptr> compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { registers[this] = jit::to_string('i', this); @@ -942,14 +948,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { #ifdef USE_CONSTANT_CACHE @@ -1275,16 +1283,18 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { - return this->arg->compile(stream, registers, usage); + return this->arg->compile(stream, registers, thread_mem, usage); } //------------------------------------------------------------------------------ @@ -1733,14 +1743,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { return this->shared_from_this(); } diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index f4f3bf1..3c1b1ce 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -191,19 +191,20 @@ namespace graph { /// /// x' = (x - xmin)/dx (1) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - auto a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); #ifdef USE_INDEX_CACHE registers[this] = jit::to_string('i', this); @@ -594,19 +595,20 @@ namespace graph { /// c'_i = c_i - 3*d_i*i (4) /// d'_i = d_i (5) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -1169,18 +1171,22 @@ namespace graph { /// c23'_ij = Σ_k,3Σ_l,3 Max(2*k-3,0)*Max(l-2,0)*(-i)^(k-2)*(-j)^(j-3) (17) /// c33'_ij = Σ_k,3Σ_l,3 Max(k-2,0)*Max(l-2,0)*(-i)^(k-3)*(-j)^(j-3) (18) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - auto x = this->left->compile(stream, registers, usage); - auto y = this->right->compile(stream, registers, usage); + auto x = this->left->compile(stream, registers, + thread_mem, usage); + auto y = this->right->compile(stream, registers, + thread_mem, usage); auto temp = x*static_cast (this->get_num_columns()) + y; if constexpr (!jit::use_metal ()) { @@ -1198,6 +1204,7 @@ namespace graph { std::ostringstream source_buffer; temp->compile(source_buffer, registers, + thread_mem, usage); registers[temp.get()] = source_buffer.str(); #endif @@ -1563,25 +1570,33 @@ namespace graph { /// /// x' = (x - xmin)/dx (1) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - auto a = this->right->compile(stream, registers, usage); - auto var = this->left->compile(stream, registers, usage); - + auto a = this->right->compile(stream, registers, + thread_mem, usage); + auto var = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()) - << "[" << registers[a.get()] << "]"; + stream << " " << registers[this] << " = "; + if (thread_mem.contains(var.get())) { + stream << thread_mem.at(var.get()); + } else { + stream << jit::to_string('v', var.get()); + } + stream << "[" << registers[a.get()] << "]"; this->endline(stream, usage); } @@ -1869,18 +1884,22 @@ namespace graph { /// x' = (x - xmin)/dx (1) /// y' = (y - ymin)/dy (2) /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - auto x = this->middle->compile(stream, registers, usage); - auto y = this->right->compile(stream, registers, usage); + auto x = this->middle->compile(stream, registers, + thread_mem, usage); + auto y = this->right->compile(stream, registers, + thread_mem, usage); auto temp = x*static_cast (this->get_num_columns()) + y; @@ -1903,17 +1922,20 @@ namespace graph { #endif } - auto var = this->left->compile(stream, - registers, - usage); + auto var = this->left->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; jit::add_type (stream); - stream << " " << registers[this] << " = " - << jit::to_string('v', var.get()) - << "[" << registers[temp.get()] << "]"; + stream << " " << registers[this] << " = "; + if (thread_mem.contains(var.get())) { + stream << thread_mem.at(var.get()); + } else { + stream << jit::to_string('v', var.get()); + } + stream << "[" << registers[temp.get()] << "]"; this->endline(stream, usage); } diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index 22161fe..5c65ee7 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -117,14 +117,16 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { return this->shared_from_this(); } @@ -403,19 +405,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = "random(" + registers[a.get()] + ")"; } diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index da3ce1d..dbd7cad 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace jit { /// Complex scalar concept. @@ -45,6 +46,9 @@ namespace jit { /// Verbose output. static bool verbose = USE_VERBOSE; +/// Type for tacking thread shared memory. + typedef std::unordered_set argument_set; + //------------------------------------------------------------------------------ /// @brief Convert a base type to a string. /// @@ -248,8 +252,8 @@ namespace jit { assert((prefix == 'r' || prefix == 'v' || prefix == 'o' || prefix == 'a' || prefix == 'i' || prefix == 's' || - prefix == 'l') && - "Expected a variable (v), register (r), output (o), array (a), index (i), state (s), or logical (l) prefix."); + prefix == 'l' || prefix == 't') && + "Expected a variable (v), register (r), output (o), array (a), index (i), state (s), logical (l), or (t) thread prefix."); return std::string(1, prefix) + format_to_string(reinterpret_cast (pointer)); } diff --git a/graph_framework/trigonometry.hpp b/graph_framework/trigonometry.hpp index 25b376b..ead690a 100644 --- a/graph_framework/trigonometry.hpp +++ b/graph_framework/trigonometry.hpp @@ -123,18 +123,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ - virtual shared_leaf compile(std::ostringstream &stream, - jit::register_map ®isters, - const jit::register_usage &usage) { + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -375,19 +377,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf a = this->arg->compile(stream, - registers, - usage); + auto a = this->arg->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; @@ -673,22 +676,22 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Compile the node. /// -/// @param[in,out] stream String buffer stream. -/// @param[in,out] registers List of defined registers. -/// @param[in] usage List of register usage count. +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ virtual shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { - shared_leaf l = this->left->compile(stream, - registers, - usage); - shared_leaf r = this->right->compile(stream, - registers, - usage); + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); registers[this] = jit::to_string('r', this); stream << " const "; diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp index c4e4873..02c39c9 100644 --- a/graph_tests/no_derivative_test.cpp +++ b/graph_tests/no_derivative_test.cpp @@ -19,6 +19,7 @@ class dummy : public graph::no_derivative compile(std::ostringstream &stream, jit::register_map ®isters, + const jit::register_map &thread_mem, const jit::register_usage &usage) { return this->shared_from_this(); } From f953bc11549ac16c4b33a2a008733791ed2263b8 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 11:26:36 -0400 Subject: [PATCH 25/51] Enable shared memory caching in the Cuda backend. --- graph_framework/cuda_context.hpp | 171 ++++++++++++++++++++++++++---- graph_framework/metal_context.hpp | 3 +- graph_framework/register.hpp | 5 +- 3 files changed, 156 insertions(+), 23 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index f8f873c..95cc008 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -8,7 +8,6 @@ #ifndef cuda_context_h #define cuda_context_h -#include #include #include @@ -492,6 +491,7 @@ namespace gpu { int value; check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); + unsigned int total_parallel = state.get() ? random_state_size : num_rays; unsigned int threads_per_group = total_parallel < 1024 ? 32 : value; unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); @@ -508,6 +508,12 @@ namespace gpu { std::cout << " Total parallel : " << total_parallel << std::endl; std::cout << " Min grid size : " << min_grid << std::endl; std::cout << " Suggested Block size : " << value << std::endl; + + check_error(cuDeviceGetAttribute(&value, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, + device), "cuDeviceGetAttribute"); + + std::cout << " Max shared memory : " << value << std::endl; } #ifdef PROFILE_KERNELS timing::measure_diagnostic timer(kernel_name); @@ -902,6 +908,8 @@ namespace gpu { /// @param[in] usage List of register usage count. /// @param[in] textures1d List of 1D kernel textures. /// @param[in] textures2d List of 2D kernel textures. +/// @param[out] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of loop iterations. //------------------------------------------------------------------------------ void create_kernel_prefix(std::ostringstream &source_buffer, @@ -915,13 +923,37 @@ namespace gpu { const jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { source_buffer << std::endl; source_buffer << "extern \"C\" __global__ void " << name << "(" << std::endl; + + int used_thread_mem = 0; + int max_shared_mem; + check_error(cuDeviceGetAttribute(&max_shared_mem, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, + device), "cuDeviceGetAttribute"); - std::unordered_set used_args; + jit::argument_set used_args; if (inputs.size()) { + if (!is_constant[0] && iterations > 1) { + const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[0].get()); + } + } else if (is_constant[i] && + inputs[i]->size() < size && + inputs[i]->size() < 1024) { + const size_t needed_mem = inputs[i]->size()*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[0].get()); + thread_mem[inputs[0].get()] = jit::to_string('t', inputs[0].get()); + } + } source_buffer << " "; if (is_constant[0]) { source_buffer << "const "; @@ -982,8 +1014,12 @@ namespace gpu { << jit::to_string('a', key); } #endif - source_buffer << ") {" << std::endl - << " const int index = blockIdx.x*blockDim.x + threadIdx.x;" + source_buffer << ") {" << std::endl; + if (thread_shared.size()) { + source_buffer << " const int t_index = threadIdx.x;" + << std::endl; + } + source_buffer << " const int index = blockIdx.x*blockDim.x + threadIdx.x;" << std::endl; if (state.get()) { #ifdef USE_INPUT_CACHE @@ -1001,28 +1037,114 @@ namespace gpu { source_buffer << "offset[0] + "; } source_buffer << "index < " << size << ") {" << std::endl; + + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (is_constant[i]) { +#ifdef USE_INPUT_CACHE + if (usage.at(inputs[i].get()) && inputs[i]->size() == size) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } +#else + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; +#endif + } + } + + if (thread_shared.size()) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " __shared__ "; + jit::add_type (source_buffer); + source_buffer << jit::to_string('t', inputs[i].get()) + << "[" << inputs[i]->size() << "]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " if (t_index < " + << inputs[i]->size() + << ") {" << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "[t_index]"; + inputs[i]->endline(source_buffer, usage); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + source_buffer << " }" << std::endl + << " __syncthreads();" + << std::endl; + break; + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { + thread_shared.erase(inputs[i].get()); + } + } + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) { + source_buffer << " __shared__ "; + jit::add_type (source_buffer); + source_buffer << jit::to_string('t', inputs[i].get()) + << "[" + << (inputs[i]->size() > 1024 ? 1024 : 32) + << "]"; + inputs[i]->endline(source_buffer, usage); + source_buffer << " " + << jit::to_string('t', inputs[i].get()) + << "[t_index] = " + << jit::to_string('v', inputs[i].get()) + << "[index]"; + inputs[i]->endline(source_buffer, usage); + } + } + } + if (iterations > 1) { source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl; } - for (auto &input : inputs) { + for (size_t i = 0, ie = inputs.size(); i < ie; i++) { + if (!is_constant[i]) { #ifdef USE_INPUT_CACHE - if (usage.at(input.get())) { - registers[input.get()] = jit::to_string('r', input.get()); - source_buffer << " const "; - jit::add_type (source_buffer); - source_buffer << " " << registers[input.get()] << " = " - << jit::to_string('v', input.get()) - << "["; - if (state.get()) { - source_buffer << "offset[0] + "; + if (usage.at(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('r', inputs[i].get()); + source_buffer << " const "; + jit::add_type (source_buffer); + source_buffer << " " << registers[inputs[i].get()] << " = "; + if (thread_shared.contains(inputs[i].get())) { + source_buffer << jit::to_string('t', inputs[i].get()) + << "[t_index]"; + } else { + source_buffer << jit::to_string('v', inputs[i].get()) + << "[index]"; + } + inputs[i]->endline(source_buffer, usage); } - source_buffer << "index]"; - input->endline(source_buffer, usage); - } #else - registers[input.get()] = jit::to_string('v', input.get()) + "[index]"; + if (thread_shared.contains(inputs[i].get())) { + registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; + } else { + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + } #endif + } } } @@ -1035,6 +1157,8 @@ namespace gpu { /// @param[in] state Random states. /// @param[in,out] registers Map of used registers. /// @param[in] usage List of register usage count. +/// @param[in] thread_shared Set of inputs that use thread shared memory. +/// @param[out] thread_mem Registers of thread shared memory. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void create_kernel_postfix(std::ostringstream &source_buffer, @@ -1043,6 +1167,8 @@ namespace gpu { graph::shared_random_state state, jit::register_map ®isters, const jit::register_usage &usage, + const jit::argument_set &thread_shared, + jit::register_map &thread_mem, const size_t iterations=1) { std::unordered_set out_registers; for (auto &[out, in] : setters) { @@ -1121,6 +1247,15 @@ namespace gpu { if (iterations > 1) { source_buffer << " }" << std::endl; } + for (auto &[out, in] : setters) { + if (thread_shared.contains(in.get())) { + source_buffer << " " + << jit::to_string('v', in.get()) + << "[index] = " + << jit::to_string('t', in.get()) + << "[t_index];" << std::endl; + } + } source_buffer << " }" << std::endl << "}" << std::endl; } diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index d3d639f..ec81180 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -230,7 +230,6 @@ namespace gpu { NSUInteger thread_width = pipline.threadExecutionWidth; NSUInteger threads_per_group = total_parallel < pipline.maxTotalThreadsPerThreadgroup ? thread_width : pipline.maxTotalThreadsPerThreadgroup; NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); - NSUInteger thread_group_memory = device.maxThreadgroupMemoryLength; if (jit::verbose) { std::cout << " Kernel name : " << kernel_name << std::endl; @@ -239,7 +238,7 @@ namespace gpu { std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; std::cout << " Total parallel size : " << total_parallel << std::endl; - std::cout << " Max thread group memory : " << thread_group_memory << std::endl; + std::cout << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl; } if (state.get()) { diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index dbd7cad..90195aa 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -46,9 +46,6 @@ namespace jit { /// Verbose output. static bool verbose = USE_VERBOSE; -/// Type for tacking thread shared memory. - typedef std::unordered_set argument_set; - //------------------------------------------------------------------------------ /// @brief Convert a base type to a string. /// @@ -268,6 +265,8 @@ namespace jit { typedef std::map texture1d_list; /// Type alias for indexing 2D textures. typedef std::map> texture2d_list; +/// Type for tacking thread shared memory. + typedef std::unordered_set argument_set; //------------------------------------------------------------------------------ /// @brief Define a custom comparator class. From eb4ed9c33192b99ab1aa62c9837d4e104b9bc69c Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 11:57:51 -0400 Subject: [PATCH 26/51] Use the correct index for the start of the loop. --- graph_framework/cuda_context.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 95cc008..b649b48 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -945,8 +945,8 @@ namespace gpu { thread_shared.insert(inputs[0].get()); } } else if (is_constant[i] && - inputs[i]->size() < size && - inputs[i]->size() < 1024) { + inputs[0]->size() < size && + inputs[0]->size() < 1024) { const size_t needed_mem = inputs[i]->size()*sizeof(T); if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; From c29698e0e6833937f224a310ed79bc163f6d16ed Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 11:59:32 -0400 Subject: [PATCH 27/51] Fix more index errors. --- graph_framework/cuda_context.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index b649b48..f4d7324 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -939,15 +939,15 @@ namespace gpu { jit::argument_set used_args; if (inputs.size()) { if (!is_constant[0] && iterations > 1) { - const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); + const size_t needed_mem = inputs[0]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); } - } else if (is_constant[i] && + } else if (is_constant[0] && inputs[0]->size() < size && inputs[0]->size() < 1024) { - const size_t needed_mem = inputs[i]->size()*sizeof(T); + const size_t needed_mem = inputs[0]->size()*sizeof(T); if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); From 90deb8749078995e81413da3637a78a4f9e3431b Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 12:02:42 -0400 Subject: [PATCH 28/51] Add shared memory registers to the compile method calls for the output. --- graph_framework/cuda_context.hpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index f4d7324..27aae05 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -1173,9 +1173,8 @@ namespace gpu { std::unordered_set out_registers; for (auto &[out, in] : setters) { if (!out->is_match(in)) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); + auto a = out->compile(source_buffer, registers, + thread_mem, usage); source_buffer << " " << jit::to_string('v', in.get()) << "["; @@ -1210,9 +1209,8 @@ namespace gpu { for (auto &out : outputs) { if (!graph::variable_cast(out).get() && !out_registers.contains(out.get())) { - graph::shared_leaf a = out->compile(source_buffer, - registers, - usage); + auto a = out->compile(source_buffer, egisters, + thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) << "["; From 9417c3331a4b3b1c205af037fea3538a71b9b59a Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 12:03:50 -0400 Subject: [PATCH 29/51] Fix typo --- graph_framework/cuda_context.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 27aae05..0886874 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -1209,7 +1209,7 @@ namespace gpu { for (auto &out : outputs) { if (!graph::variable_cast(out).get() && !out_registers.contains(out.get())) { - auto a = out->compile(source_buffer, egisters, + auto a = out->compile(source_buffer, registers, thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) From 0a74da5056dc6368395fd84b276f13e4fea7bb16 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 30 Jul 2026 12:44:43 -0400 Subject: [PATCH 30/51] Shared memory is currently assuming a given thread size. So over ride the number of threads used for cuda kernels. Long term we should use something more robust. --- graph_framework/cuda_context.hpp | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 0886874..cb62031 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -90,6 +90,8 @@ namespace gpu { CUdeviceptr offset_buffer; /// Cuda stream. CUstream stream; +/// Assumed thread sizes. + std::map assumed_thread_size; //------------------------------------------------------------------------------ /// @brief Check results of async cuda functions. @@ -491,7 +493,10 @@ namespace gpu { int value; check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); - + if (assumed_thread_size[kernel_name] != -1) { + value = assumed_thread_size[kernel_name]; + } + unsigned int total_parallel = state.get() ? random_state_size : num_rays; unsigned int threads_per_group = total_parallel < 1024 ? 32 : value; unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); @@ -943,6 +948,9 @@ namespace gpu { if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); + if (!assumed_thread_size.contains(name)) { + assumed_thread_size[name] = inputs[i]->size() > 1024 ? 1024 : 32; + } } } else if (is_constant[0] && inputs[0]->size() < size && @@ -952,6 +960,9 @@ namespace gpu { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); thread_mem[inputs[0].get()] = jit::to_string('t', inputs[0].get()); + if (!assumed_thread_size.contains(name)) { + assumed_thread_size[name] = 1024; + } } } source_buffer << " "; @@ -964,6 +975,28 @@ namespace gpu { used_args.insert(inputs[0].get()); } for (size_t i = 1, ie = inputs.size(); i < ie; i++) { + if (!is_constant[i] && iterations > 1) { + const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + if (!assumed_thread_size.contains(name)) { + assumed_thread_size[name] = inputs[i]->size() > 1024 ? 1024 : 32; + } + } + } else if (is_constant[i] && + inputs[i]->size() < size && + inputs[i]->size() < 1024) { + const size_t needed_mem = inputs[i]->size()*sizeof(T); + if (used_thread_mem + needed_mem < max_shared_mem) { + used_thread_mem += needed_mem; + thread_shared.insert(inputs[i].get()); + thread_mem[inputs[i].get()] = jit::to_string('t', inputs[0].get()); + if (!assumed_thread_size.contains(name)) { + assumed_thread_size[name] = 1024; + } + } + } if (!used_args.contains(inputs[i].get())) { inputs[i]->endline(source_buffer, usage, ','); source_buffer << " "; From 95503a67aaa79f5fc0a3438d7485aac8fc045f5a Mon Sep 17 00:00:00 2001 From: m4c Date: Fri, 31 Jul 2026 13:55:57 -0400 Subject: [PATCH 31/51] Decouple shared memory from thread size. --- graph_framework/cuda_context.hpp | 109 ++++++++++++++++--------------- 1 file changed, 58 insertions(+), 51 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index cb62031..f36e9aa 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -18,7 +18,7 @@ #include "timing.hpp" /// Maximum number of registers to use. -#define MAX_REG 128 +#define MAX_REG 256 namespace gpu { //------------------------------------------------------------------------------ @@ -90,8 +90,6 @@ namespace gpu { CUdeviceptr offset_buffer; /// Cuda stream. CUstream stream; -/// Assumed thread sizes. - std::map assumed_thread_size; //------------------------------------------------------------------------------ /// @brief Check results of async cuda functions. @@ -229,6 +227,19 @@ namespace gpu { if (jit::verbose) { std::cout << "CUDA GPU info." << std::endl; std::cout << " Major compute capability : " << compute_version << std::endl; + + int value; + check_error(cuDeviceGetAttribute(&value, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, + device), "cuDeviceGetAttribute"); + + std::cout << " Max shared memory : " << value << std::endl; + + check_error(cuDeviceGetAttribute(&value, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); + + std::cout << " Warp size : " << value << std::endl; } check_error(cuDeviceGetAttribute(&compute_version, @@ -493,12 +504,13 @@ namespace gpu { int value; check_error(cuFuncGetAttribute(&value, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function), "cuFuncGetAttribute"); - if (assumed_thread_size[kernel_name] != -1) { - value = assumed_thread_size[kernel_name]; - } + int warp_size; + check_error(cuDeviceGetAttribute(&warp_size, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); unsigned int total_parallel = state.get() ? random_state_size : num_rays; - unsigned int threads_per_group = total_parallel < 1024 ? 32 : value; + unsigned int threads_per_group = total_parallel < 1024 ? warp_size : value; unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); int min_grid; @@ -513,12 +525,6 @@ namespace gpu { std::cout << " Total parallel : " << total_parallel << std::endl; std::cout << " Min grid size : " << min_grid << std::endl; std::cout << " Suggested Block size : " << value << std::endl; - - check_error(cuDeviceGetAttribute(&value, - CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, - device), "cuDeviceGetAttribute"); - - std::cout << " Max shared memory : " << value << std::endl; } #ifdef PROFILE_KERNELS timing::measure_diagnostic timer(kernel_name); @@ -934,23 +940,27 @@ namespace gpu { source_buffer << std::endl; source_buffer << "extern \"C\" __global__ void " << name << "(" << std::endl; - + int used_thread_mem = 0; int max_shared_mem; check_error(cuDeviceGetAttribute(&max_shared_mem, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, device), "cuDeviceGetAttribute"); + int warp_size; + check_error(cuDeviceGetAttribute(&warp_size, + CU_DEVICE_ATTRIBUTE_WARP_SIZE, + device), "cuDeviceGetAttribute"); + jit::argument_set used_args; if (inputs.size()) { if (!is_constant[0] && iterations > 1) { - const size_t needed_mem = inputs[0]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); + const size_t needed_mem = inputs[0]->size() > 1024 ? + 1024*sizeof(T) : + warp_size*sizeof(T); if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); - if (!assumed_thread_size.contains(name)) { - assumed_thread_size[name] = inputs[i]->size() > 1024 ? 1024 : 32; - } } } else if (is_constant[0] && inputs[0]->size() < size && @@ -960,9 +970,6 @@ namespace gpu { used_thread_mem += needed_mem; thread_shared.insert(inputs[0].get()); thread_mem[inputs[0].get()] = jit::to_string('t', inputs[0].get()); - if (!assumed_thread_size.contains(name)) { - assumed_thread_size[name] = 1024; - } } } source_buffer << " "; @@ -976,13 +983,12 @@ namespace gpu { } for (size_t i = 1, ie = inputs.size(); i < ie; i++) { if (!is_constant[i] && iterations > 1) { - const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*sizeof(T) : 32*sizeof(T); + const size_t needed_mem = inputs[i]->size() > 1024 ? + 1024*sizeof(T) : + warp_size*sizeof(T); if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[i].get()); - if (!assumed_thread_size.contains(name)) { - assumed_thread_size[name] = inputs[i]->size() > 1024 ? 1024 : 32; - } } } else if (is_constant[i] && inputs[i]->size() < size && @@ -991,10 +997,7 @@ namespace gpu { if (used_thread_mem + needed_mem < max_shared_mem) { used_thread_mem += needed_mem; thread_shared.insert(inputs[i].get()); - thread_mem[inputs[i].get()] = jit::to_string('t', inputs[0].get()); - if (!assumed_thread_size.contains(name)) { - assumed_thread_size[name] = 1024; - } + thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); } } if (!used_args.contains(inputs[i].get())) { @@ -1094,33 +1097,28 @@ namespace gpu { if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { source_buffer << " __shared__ "; jit::add_type (source_buffer); - source_buffer << jit::to_string('t', inputs[i].get()) + source_buffer << " " << jit::to_string('t', inputs[i].get()) << "[" << inputs[i]->size() << "]"; inputs[i]->endline(source_buffer, usage); } } for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " if (t_index < " + source_buffer << " for(int j = t_index; j < " << inputs[i]->size() - << ") {" << std::endl; - break; - } - } - for (size_t i = 0, ie = inputs.size(); i < ie; i++) { - if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " " + << "; j += blockDim.x) {" << std::endl + << " " << jit::to_string('t', inputs[i].get()) - << "[t_index] = " + << "[j] = " << jit::to_string('v', inputs[i].get()) - << "[t_index]"; + << "[j]"; inputs[i]->endline(source_buffer, usage); + source_buffer << " }" << std::endl; } } for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " }" << std::endl - << " __syncthreads();" + source_buffer << " __syncthreads();" << std::endl; break; } @@ -1134,9 +1132,9 @@ namespace gpu { if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) { source_buffer << " __shared__ "; jit::add_type (source_buffer); - source_buffer << jit::to_string('t', inputs[i].get()) + source_buffer << " " << jit::to_string('t', inputs[i].get()) << "[" - << (inputs[i]->size() > 1024 ? 1024 : 32) + << (inputs[i]->size() > 1024 ? 1024 : warp_size) << "]"; inputs[i]->endline(source_buffer, usage); source_buffer << " " @@ -1208,13 +1206,18 @@ namespace gpu { if (!out->is_match(in)) { auto a = out->compile(source_buffer, registers, thread_mem, usage); - source_buffer << " " - << jit::to_string('v', in.get()) - << "["; - if (state.get()) { - source_buffer << "offset[0] + "; + source_buffer << " "; + if (thread_shared.contains(in.get())) { + source_buffer << jit::to_string('t', in.get()) + << "[t_index] = "; + } else { + source_buffer << jit::to_string('v', in.get()) + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index] = "; } - source_buffer << "index] = "; if constexpr (SAFE_MATH) { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); @@ -1282,7 +1285,11 @@ namespace gpu { if (thread_shared.contains(in.get())) { source_buffer << " " << jit::to_string('v', in.get()) - << "[index] = " + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index] = " << jit::to_string('t', in.get()) << "[t_index];" << std::endl; } From 72ab0b3ce73a67931035f2e454ea54d719ebdc6a Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 31 Jul 2026 14:49:20 -0400 Subject: [PATCH 32/51] Decouple threadgroup memory from thread width. --- graph_framework/metal_context.hpp | 53 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index ec81180..b95c16e 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -94,6 +94,10 @@ namespace gpu { if (jit::verbose) { std::cout << "Metal GPU info." << std::endl; + std::cout << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl; + std::cout << " Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl; + std::cout << " Device name : " << device.name << std::endl; + std::cout << " Architecture : " << device.architecture << std::endl; } } @@ -129,10 +133,10 @@ namespace gpu { compute.buffers[i].mutability = bufferMutability[kernel_name][i]; } - id pipline = [device newComputePipelineStateWithDescriptor:compute - options:MTLPipelineOptionNone - reflection:NULL - error:&error]; + id pipeline = [device newComputePipelineStateWithDescriptor:compute + options:MTLPipelineOptionNone + reflection:NULL + error:&error]; if (error) { NSLog(@"%@", error); @@ -227,8 +231,10 @@ namespace gpu { NSRange tex_range = NSMakeRange(0, textures.size()); NSUInteger total_parallel = state.get() ? random_state_size : num_rays; - NSUInteger thread_width = pipline.threadExecutionWidth; - NSUInteger threads_per_group = total_parallel < pipline.maxTotalThreadsPerThreadgroup ? thread_width : pipline.maxTotalThreadsPerThreadgroup; + NSUInteger thread_width = pipeline.threadExecutionWidth; + NSUInteger threads_per_group = total_parallel < pipeline.maxTotalThreadsPerThreadgroup ? + thread_width : + pipeline.maxTotalThreadsPerThreadgroup; NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); if (jit::verbose) { @@ -238,12 +244,13 @@ namespace gpu { std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; std::cout << " Total parallel size : " << total_parallel << std::endl; - std::cout << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl; + std::cout << " Allocated thread mem : " << pipeline.staticThreadgroupMemoryLength << std::endl; + std::cout << " Required threads : " << pipeline.requiredThreadsPerThreadgroup.width << std::endl; } if (state.get()) { - return [this, num_rays, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures + return [this, num_rays, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures #ifdef PROFILE_KERNELS , kernel_name #endif @@ -256,7 +263,7 @@ namespace gpu { offsets[j] = i*sizeof(float); } - [encoder setComputePipelineState:pipline]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffers:buffers.data() offsets:offsets.data() withRange:range]; @@ -278,7 +285,7 @@ namespace gpu { [command_buffer commit]; }; } else { - return [this, pipline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures + return [this, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures #ifdef PROFILE_KERNELS , kernel_name #endif @@ -286,7 +293,7 @@ namespace gpu { command_buffer = [queue commandBuffer]; id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; - [encoder setComputePipelineState:pipline]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffers:buffers.data() offsets:offsets.data() withRange:range]; @@ -611,7 +618,9 @@ namespace gpu { for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { if (!is_constant[i] && iterations > 1) { - const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*4 : 32*4; + const size_t needed_mem = inputs[i]->size() > 1024 ? + 1024*4 : + 32*4; if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) { used_thread_mem += needed_mem; thread_shared.insert(inputs[i].get()); @@ -670,6 +679,7 @@ namespace gpu { } if (thread_shared.size()) { source_buffer << " ushort t_index [[thread_position_in_threadgroup]]," << std::endl; + source_buffer << " ushort t_total [[threads_per_threadgroup]]," << std::endl; } source_buffer << " " << jit::smallest_uint_type (size) @@ -709,26 +719,21 @@ namespace gpu { } for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " if (t_index < " + source_buffer << " for(int j = t_index; j < " << inputs[i]->size() - << ") {" << std::endl; - break; - } - } - for (size_t i = 0, ie = inputs.size(); i < ie; i++) { - if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " " + << "; j += t_total) {" << std::endl + << " " << jit::to_string('t', inputs[i].get()) - << "[t_index] = " + << "[j] = " << jit::to_string('v', inputs[i].get()) - << "[t_index]"; + << "[j]"; inputs[i]->endline(source_buffer, usage); + source_buffer << " }" << std::endl; } } for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (thread_shared.contains(inputs[i].get()) && is_constant[i]) { - source_buffer << " }" << std::endl - << " threadgroup_barrier(mem_flags::mem_threadgroup);" + source_buffer << " threadgroup_barrier(mem_flags::mem_threadgroup);" << std::endl; break; } From f927fc6f21448ec446b5d1ddb90b24ce79988198 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 31 Jul 2026 15:57:38 -0400 Subject: [PATCH 33/51] Fix issues when input cache was disabled. --- CMakeLists.txt | 14 +++++---- graph_framework/cuda_context.hpp | 42 +++++++++++++++++--------- graph_framework/metal_context.hpp | 50 +++++++++++++++++++++++++------ 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f56d106..4f924a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,10 +7,10 @@ project (graph_framework CXX) #------------------------------------------------------------------------------- option (USE_PCH "Enable the use of precompiled headers" ON) option (SAVE_KERNEL_SOURCE "Writes the kernel source code to a file." OFF) -option (USE_INPUT_CACHE "Cache the values kernel input values." OFF) -option (USE_CONSTANT_CACHE "Cache the value of constants in kernel registers." OFF) +option (USE_INPUT_CACHE "Cache the values kernel input values." ON) +option (USE_CONSTANT_CACHE "Cache the value of constants in kernel registers." ON) option (SHOW_USE_COUNT "Add a comment showing the use count in kernel sources." OFF) -option (USE_INDEX_CACHE "Cache index values instead of computing them every time." OFF) +option (USE_INDEX_CACHE "Cache index values instead of computing them every time." ON) option (USE_VERBOSE "Verbose jit option." OFF) option (PROFILE_KERNELS "Display kernel timing information" OFF) option (BUILD_C_BINDING "Build C interface." OFF) @@ -375,8 +375,12 @@ macro (add_tool_target target lang) graph_framework ) - if (${USE_PCH} AND ${BUILD_C_BINDING}) - target_precompile_headers (${target} REUSE_FROM graph_c) + if (${USE_PCH}) + if (${BUILD_C_BINDING}) + target_precompile_headers (${target} REUSE_FROM graph_c) + elseif (NOT ${target} MATCHES xrays) + target_precompile_headers (${target} REUSE_FROM xrays) + endif () endif () endmacro () diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index f36e9aa..87e739c 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -954,6 +954,7 @@ namespace gpu { jit::argument_set used_args; if (inputs.size()) { +#ifdef USE_INPUT_CACHE if (!is_constant[0] && iterations > 1) { const size_t needed_mem = inputs[0]->size() > 1024 ? 1024*sizeof(T) : @@ -972,6 +973,7 @@ namespace gpu { thread_mem[inputs[0].get()] = jit::to_string('t', inputs[0].get()); } } +#endif source_buffer << " "; if (is_constant[0]) { source_buffer << "const "; @@ -982,6 +984,7 @@ namespace gpu { used_args.insert(inputs[0].get()); } for (size_t i = 1, ie = inputs.size(); i < ie; i++) { +#ifdef USE_INPUT_CACHE if (!is_constant[i] && iterations > 1) { const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*sizeof(T) : @@ -1000,6 +1003,7 @@ namespace gpu { thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); } } +#endif if (!used_args.contains(inputs[i].get())) { inputs[i]->endline(source_buffer, usage, ','); source_buffer << " "; @@ -1057,17 +1061,7 @@ namespace gpu { } source_buffer << " const int index = blockIdx.x*blockDim.x + threadIdx.x;" << std::endl; - if (state.get()) { -#ifdef USE_INPUT_CACHE - registers[state.get()] = jit::to_string('r', state.get()); - source_buffer << " mt_state &" << registers[state.get()] << " = " - << jit::to_string('s', state.get()) - << "[index]"; - state->endline(source_buffer, usage); -#else - registers[state.get()] = jit::to_string('s', state.get()) + "[threadIdx.x]"; -#endif - } + source_buffer << " if ("; if (state.get()) { source_buffer << "offset[0] + "; @@ -1087,7 +1081,10 @@ namespace gpu { inputs[i]->endline(source_buffer, usage); } #else - registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + + "[" + + (state.get() ? "offset[0] + " : "") + + "index]"; #endif } } @@ -1164,7 +1161,11 @@ namespace gpu { << "[t_index]"; } else { source_buffer << jit::to_string('v', inputs[i].get()) - << "[index]"; + << "["; + if (state.get()) { + source_buffer << "offset[0] + "; + } + source_buffer << "index]"; } inputs[i]->endline(source_buffer, usage); } @@ -1172,11 +1173,24 @@ namespace gpu { if (thread_shared.contains(inputs[i].get())) { registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; } else { - registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[" + + (state.get() ? "offset[0] + " : "") + + "index]"; } #endif } } + if (state.get()) { +#ifdef USE_INPUT_CACHE + registers[state.get()] = jit::to_string('r', state.get()); + source_buffer << " mt_state &" << registers[state.get()] << " = " + << jit::to_string('s', state.get()) + << "[index]"; + state->endline(source_buffer, usage); +#else + registers[state.get()] = jit::to_string('s', state.get()) + "[threadIdx.x]"; +#endif + } } //------------------------------------------------------------------------------ diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index b95c16e..abac551 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -617,6 +617,7 @@ namespace gpu { jit::argument_set used_args; for (size_t i = 0, ie = inputs.size(); i < ie; i++) { if (!used_args.contains(inputs[i].get())) { +#ifdef USE_INPUT_CACHE if (!is_constant[i] && iterations > 1) { const size_t needed_mem = inputs[i]->size() > 1024 ? 1024*4 : @@ -635,6 +636,7 @@ namespace gpu { thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get()); } } +#endif bufferMutability[name].push_back(is_constant[i] ? MTLMutabilityMutable : MTLMutabilityImmutable); source_buffer << " " << (is_constant[i] ? "constant" : "device") << " float *" @@ -699,11 +701,18 @@ namespace gpu { jit::add_type (source_buffer); source_buffer << " " << registers[inputs[i].get()] << " = " << jit::to_string('v', inputs[i].get()) - << "[index]"; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; inputs[i]->endline(source_buffer, usage); } #else - registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + + "[" + + (state.get() ? "offset + " : "") + + "index]"; #endif } } @@ -755,7 +764,11 @@ namespace gpu { << jit::to_string('t', inputs[i].get()) << "[t_index] = " << jit::to_string('v', inputs[i].get()) - << "[index]"; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; inputs[i]->endline(source_buffer, usage); } } @@ -778,7 +791,11 @@ namespace gpu { << "[t_index]"; } else { source_buffer << jit::to_string('v', inputs[i].get()) - << "[index]"; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index]"; } inputs[i]->endline(source_buffer, usage); } @@ -786,7 +803,9 @@ namespace gpu { if (thread_shared.contains(inputs[i].get())) { registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]"; } else { - registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[index]"; + registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "[" + + (state.get() ? "offset + " : "") + + "index]"; } #endif } @@ -799,7 +818,8 @@ namespace gpu { << "[index]"; state->endline(source_buffer, usage); #else - registers[state.get()] = jit::to_string('s', state.get()) + "[thread_index]"; + registers[state.get()] = jit::to_string('s', state.get()) + + "[index]"; #endif } } @@ -837,7 +857,11 @@ namespace gpu { << "[t_index] = "; } else { source_buffer << jit::to_string('v', in.get()) - << "[index] = "; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = "; } if constexpr (SAFE_MATH) { source_buffer << "isnan(" << registers[a.get()] @@ -854,7 +878,11 @@ namespace gpu { auto a = out->compile(source_buffer, registers, thread_mem, usage); source_buffer << " " << jit::to_string('o', out.get()) - << "[index] = "; + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = "; if constexpr (SAFE_MATH) { source_buffer << "isnan(" << registers[a.get()] << ") ? 0.0 : "; @@ -871,7 +899,11 @@ namespace gpu { if (thread_shared.contains(in.get())) { source_buffer << " " << jit::to_string('v', in.get()) - << "[index] = " + << "["; + if (state.get()) { + source_buffer << "offset + "; + } + source_buffer << "index] = " << jit::to_string('t', in.get()) << "[t_index];" << std::endl; } From 16c65d7a1d65f80916c2c3db1a505e26ed0c83a5 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 4 Aug 2026 14:39:25 -0400 Subject: [PATCH 34/51] Rewrite the sum kernel to use atomic accumulation. The kernel is 10x faster than the previous kernel however it is 100x slower when run with orther kernels. So far this only affects the metal kernel. A standalone test case doesn't show this problem so It's not clear where it's comming from. Commit this work in progress to see of the Cuda kernel has the same problem. --- graph_driver/xrays.cpp | 2 +- graph_framework.xcodeproj/project.pbxproj | 8 +- graph_framework/absorption.hpp | 10 +- graph_framework/cpu_context.hpp | 42 ++- graph_framework/dispersion.hpp | 9 +- graph_framework/equilibrium.hpp | 8 +- graph_framework/jit.hpp | 23 +- graph_framework/metal_context.hpp | 70 +++- graph_framework/newton.hpp | 4 +- graph_framework/node.hpp | 22 +- graph_framework/particle_in_cell.hpp | 111 +++--- graph_framework/piecewise.hpp | 393 ++++++++++++++++++---- graph_framework/random.hpp | 18 - graph_framework/solver.hpp | 14 +- graph_framework/workflow.hpp | 107 ++++-- graph_korc/xkorc.cpp | 4 +- graph_pic/xpic.cpp | 96 ++---- graph_tests/efit_test.cpp | 2 +- graph_tests/jit_test.cpp | 7 +- graph_tests/no_derivative_test.cpp | 4 - graph_tests/pic_test.cpp | 35 +- graph_tests/piecewise_test.cpp | 8 +- graph_tests/random_test.cpp | 7 +- graph_tests/workflow_test.cpp | 4 +- 24 files changed, 627 insertions(+), 381 deletions(-) diff --git a/graph_driver/xrays.cpp b/graph_driver/xrays.cpp index cb8ec2d..e09c8b4 100644 --- a/graph_driver/xrays.cpp +++ b/graph_driver/xrays.cpp @@ -740,7 +740,7 @@ void bin_power(const commandline::parser &cl, {z, graph::variable_cast(z_last)}, {p_next, graph::variable_cast(power)}, {k_next, graph::variable_cast(k_sum)} - }, graph::shared_random_state (), "power", local_num_rays); + }, {}, NULL, "power", local_num_rays); work.compile(); output::result_file file(stream.str()); diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index ecdb0aa..cb6fe6a 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -2005,7 +2005,7 @@ "$(inherited)", ); LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.1; + MACOSX_DEPLOYMENT_TARGET = 26.0; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Debug; @@ -2019,7 +2019,7 @@ CODE_SIGN_STYLE = Automatic; GCC_C_LANGUAGE_STANDARD = gnu17; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.1; + MACOSX_DEPLOYMENT_TARGET = 26.0; PRODUCT_NAME = "$(TARGET_NAME)"; }; name = Release; @@ -2230,7 +2230,7 @@ "build/_deps/llvm-build/lib", /usr/local/lib, ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -2407,7 +2407,7 @@ "build/_deps/llvm-build/lib", /usr/local/lib, ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; diff --git a/graph_framework/absorption.hpp b/graph_framework/absorption.hpp index 2816851..1b48885 100644 --- a/graph_framework/absorption.hpp +++ b/graph_framework/absorption.hpp @@ -233,7 +233,7 @@ namespace absorption { {graph::zero (), graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, NULL, + work.add_item(inputs, {}, setters, {}, NULL, "root_find_init_kernel", inputs.back()->size()); inputs.push_back(graph::variable_cast(this->t)); @@ -245,8 +245,7 @@ namespace absorption { kvec + kamp_vec, x, y, z, t, eq); - solver::newton(work, {kamp}, inputs, {D}, - graph::shared_random_state ()); + solver::newton (work, {kamp}, inputs, {}, {D}, NULL); inputs = { graph::variable_cast(this->kamp), @@ -260,7 +259,7 @@ namespace absorption { setters = { {klen + kamp, graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, NULL, + work.add_item(inputs, {}, setters, {}, NULL, "final_kamp", inputs.back()->size()); } @@ -426,8 +425,7 @@ namespace absorption { {kamp1, graph::variable_cast(this->kamp)} }; - work.add_item(inputs, {}, setters, - graph::shared_random_state (), + work.add_item(inputs, {}, setters, {}, NULL, "weak_damping_kimg_kernel", inputs.back()->size()); } diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 5e8780e..53a33cc 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -37,6 +37,7 @@ #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "random.hpp" +#include "piecewise.hpp" #ifndef NDEBUG //------------------------------------------------------------------------------ @@ -223,6 +224,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @param[in] tex1d_list List of 1D textures. @@ -232,6 +234,7 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, @@ -249,11 +252,21 @@ namespace gpu { buffers[reinterpret_cast (input.get())] = kernel_arguments[input.get()].data(); } for (auto &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - std::vector arg(num_rays); - kernel_arguments[output.get()] = arg; + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + std::vector arg(num_rays); + kernel_arguments[output.get()] = arg; + } + buffers[reinterpret_cast (output.get())] = kernel_arguments[output.get()].data(); + } + } + for (auto &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + std::vector arg(atomic->size()); + memcpy(arg.data(), atomic->data(), atomic->size()*sizeof(T)); + kernel_arguments[atomic.get()] = arg; } - buffers[reinterpret_cast (output.get())] = kernel_arguments[output.get()].data(); + buffers[reinterpret_cast (atomic.get())] = kernel_arguments[atomic.get()].data(); } if (state.get()) { @@ -528,6 +541,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -543,6 +557,7 @@ namespace gpu { const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t size, const std::vector &is_constant, @@ -581,7 +596,8 @@ namespace gpu { } } for (auto &output : outputs) { - if (!used_args.contains(output.get())) { + if (!used_args.contains(output.get()) && + !graph::atomic_accumulate_1D_cast(output).get()) { source_buffer << " "; jit::add_type (source_buffer); source_buffer << " *" << jit::to_string('o', output.get()) @@ -591,6 +607,19 @@ namespace gpu { used_args.insert(output.get()); } } + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + source_buffer << " "; + jit::add_type (source_buffer); + source_buffer << " *" + << jit::to_string('v', atomics[i].get()) + << " = args[" + << reinterpret_cast (atomics[i].get()) + << "];" + << std::endl; + used_args.insert(atomics[i].get()); + } + } if (state.get()) { registers[state.get()] = jit::to_string('r', state.get()); source_buffer << " mt_state &" @@ -665,7 +694,8 @@ namespace gpu { } } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { auto a = out->compile(source_buffer, registers, thread_mem, usage); diff --git a/graph_framework/dispersion.hpp b/graph_framework/dispersion.hpp index f9d2526..5a73345 100644 --- a/graph_framework/dispersion.hpp +++ b/graph_framework/dispersion.hpp @@ -1461,10 +1461,11 @@ namespace dispersion { workflow::manager work(index); - solver::newton(work, {x}, inputs, this->D, - graph::shared_random_state (), - tolerance, max_iterations); + solver::newton (work, {x}, inputs, + {}, this->D, NULL, + tolerance, + max_iterations); work.compile(); work.run(); diff --git a/graph_framework/equilibrium.hpp b/graph_framework/equilibrium.hpp index 9d03531..d996989 100644 --- a/graph_framework/equilibrium.hpp +++ b/graph_framework/equilibrium.hpp @@ -1599,11 +1599,11 @@ namespace equilibrium { }; workflow::manager work(device_number); - solver::newton(work, { + solver::newton (work, { x_axis, z_axis - }, inputs, (psi_cache - psimin)/dpsi, graph::shared_random_state (), static_cast (1.0E-30), 1000, static_cast (0.1)); - work.add_item(inputs, {b_mod}, {}, - graph::shared_random_state (), + }, inputs, {}, (psi_cache - psimin)/dpsi, + NULL, static_cast (1.0E-30), 1000, static_cast (0.1)); + work.add_item(inputs, {b_mod}, {}, {}, NULL, "bmod_at_axis", inputs.back()->size()); work.compile(); work.run(); diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index 1e48a1c..f6ee4a1 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -112,15 +112,17 @@ namespace jit { /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. /// @param[in] setters Map outputs back to input values. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] size Size of the kernel. /// @param[in] iterations Number of iterations of the loop. //------------------------------------------------------------------------------ void add_kernel(const std::string name, - graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes setters, - graph::shared_random_state state, + graph::input_nodes &inputs, + graph::output_nodes &outputs, + graph::map_nodes &setters, + graph::input_nodes &atomics, + graph::shared_random_state &state, const size_t size, const size_t iterations=1) { kernel_names.push_back(name); @@ -167,7 +169,8 @@ namespace jit { jit::register_map thread_mem; gpu_context.create_kernel_prefix(source_buffer, - name, inputs, outputs, state, + name, inputs, outputs, + atomics, state, size, is_constant, registers, usage, kernel_1dtextures[name], @@ -199,6 +202,11 @@ namespace jit { removed_elements.push_back(key); } } + for (auto &out : outputs) { + if (graph::atomic_accumulate_1D_cast(out).get()) { + removed_elements.push_back(out.get()); + } + } for (auto &key : removed_elements) { registers.erase(key); @@ -282,6 +290,7 @@ namespace jit { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @returns A lambda function to run the kernel. @@ -289,9 +298,11 @@ namespace jit { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays) { - return gpu_context.create_kernel_call(kernel_name, inputs, outputs, state, num_rays, + return gpu_context.create_kernel_call(kernel_name, inputs, outputs, + atomics, state, num_rays, kernel_1dtextures[kernel_name], kernel_2dtextures[kernel_name]); } diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index abac551..7d8b560 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -12,6 +12,7 @@ #include "random.hpp" #include "timing.hpp" +#include "piecewise.hpp" /// Name space for GPU backends. namespace gpu { @@ -107,6 +108,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace. /// @param[in] tex1d_list List of 1D textures. @@ -116,14 +118,20 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, const jit::texture2d_list &tex2d_list) { NSError *error; - id function = [library newFunctionWithName:[NSString stringWithCString:kernel_name.c_str() - encoding:NSUTF8StringEncoding]]; + MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new]; + funcDesc.options = MTLFunctionOptionNone; + funcDesc.name = [NSString stringWithCString:kernel_name.c_str() + encoding:NSUTF8StringEncoding]; + + id function = [library newFunctionWithDescriptor:funcDesc + error:&error]; MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new]; compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES; @@ -159,13 +167,25 @@ namespace gpu { } } for (graph::shared_leaf &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float) + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float) + options:MTLResourceStorageModeShared]; + } + if (!needed_buffers.contains(output.get())) { + buffers.push_back(kernel_arguments[output.get()]); + needed_buffers.insert(output.get()); + } + } + } + for (graph::shared_variable &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + kernel_arguments[atomic.get()] = [device newBufferWithLength:atomic->size()*buffer_element_size options:MTLResourceStorageModeShared]; } - if (!needed_buffers.contains(output.get())) { - buffers.push_back(kernel_arguments[output.get()]); - needed_buffers.insert(output.get()); + if (!needed_buffers.contains(atomic.get())) { + buffers.push_back(kernel_arguments[atomic.get()]); + needed_buffers.insert(atomic.get()); } } if (state.get()) { @@ -244,8 +264,6 @@ namespace gpu { std::cout << " Number of groups : " << thread_groups << std::endl; std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; std::cout << " Total parallel size : " << total_parallel << std::endl; - std::cout << " Allocated thread mem : " << pipeline.staticThreadgroupMemoryLength << std::endl; - std::cout << " Required threads : " << pipeline.requiredThreadsPerThreadgroup.width << std::endl; } if (state.get()) { @@ -322,13 +340,21 @@ namespace gpu { //------------------------------------------------------------------------------ std::function create_max_call(graph::shared_leaf &argument, std::function run) { + NSError *error; + + MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new]; + funcDesc.options = MTLFunctionOptionNone; + funcDesc.name = @"max_reduction"; + + id function = [library newFunctionWithDescriptor:funcDesc + error:&error]; + MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new]; compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES; - compute.computeFunction = [library newFunctionWithName:@"max_reduction"]; + compute.computeFunction = function; compute.maxTotalThreadsPerThreadgroup = 1024; compute.buffers[0].mutability = MTLMutabilityImmutable; - NSError *error; id max_state = [device newComputePipelineStateWithDescriptor:compute options:MTLPipelineOptionNone reflection:NULL @@ -467,6 +493,8 @@ namespace gpu { MTLCompileOptions *options = [MTLCompileOptions new]; options.mathMode = MTLMathModeFast; options.mathFloatingPointFunctions = MTLMathFloatingPointFunctionsFast; + options.optimizationLevel = MTLLibraryOptimizationLevelDefault; + options.languageVersion = MTLLanguageVersion3_2; return options; } @@ -581,6 +609,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -596,7 +625,8 @@ namespace gpu { const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, - graph::shared_random_state state, + graph::input_nodes atomics, + graph::shared_random_state &state, const size_t size, const std::vector &is_constant, jit::register_map ®isters, @@ -647,7 +677,8 @@ namespace gpu { } } for (size_t i = 0, ie = outputs.size(); i < ie; i++) { - if (!used_args.contains(outputs[i].get())) { + if (!used_args.contains(outputs[i].get()) && + !graph::atomic_accumulate_1D_cast(outputs[i]).get()) { bufferMutability[name].push_back(MTLMutabilityMutable); source_buffer << " device float *" << jit::to_string('o', outputs[i].get()) @@ -656,6 +687,16 @@ namespace gpu { used_args.insert(outputs[i].get()); } } + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + bufferMutability[name].push_back(MTLMutabilityMutable); + source_buffer << " device atomic_float *" + << jit::to_string('v', atomics[i].get()) + << " [[buffer(" << buffer_count++ << ")]]," + << std::endl; + used_args.insert(atomics[i].get()); + } + } if (state.get()) { bufferMutability[name].push_back(MTLMutabilityMutable); source_buffer << " device mt_state *" @@ -873,7 +914,8 @@ namespace gpu { } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { auto a = out->compile(source_buffer, registers, thread_mem, usage); diff --git a/graph_framework/newton.hpp b/graph_framework/newton.hpp index 62d5308..ca386eb 100644 --- a/graph_framework/newton.hpp +++ b/graph_framework/newton.hpp @@ -22,6 +22,7 @@ namespace solver { /// @param[in,out] work Workflow manager. /// @param[in] vars The unknowns to solver for. /// @param[in] inputs Inputs for jit compile. +/// @param[in] atomics Atomic inputs for jit compile. /// @param[in] func Function to find the root of. /// @param[in] state Random state node. /// @param[in] tolerance Tolerance to solve the dispersion function @@ -34,6 +35,7 @@ namespace solver { void newton(workflow::manager &work, graph::output_nodes vars, graph::input_nodes inputs, + graph::input_nodes atomics, graph::shared_leaf func, graph::shared_random_state state, const T tolerance = 1.0E-30, @@ -45,7 +47,7 @@ namespace solver { graph::variable_cast(x)}); } - work.add_converge_item(inputs, {func*func}, setters, state, + work.add_converge_item(inputs, {func*func}, setters, atomics, state, "loss_kernel", inputs.back()->size(), tolerance, max_iterations); } diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index d531726..fe6a570 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -563,7 +563,9 @@ namespace graph { /// /// @returns True if all the sub-nodes terminate in variables. //------------------------------------------------------------------------------ - virtual bool is_all_variables() const = 0; + virtual bool is_all_variables() const { + return false; + } //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. @@ -832,15 +834,6 @@ namespace graph { virtual shared_leaf get_power_exponent() const { return one (); } - -//------------------------------------------------------------------------------ -/// @brief Test if all the sub-nodes terminate in variables. -/// -/// @returns True if all the sub-nodes terminate in variables. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } }; //------------------------------------------------------------------------------ @@ -1067,15 +1060,6 @@ namespace graph { return data.has_zero(); } -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index fcfcb1f..2cddc09 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -468,38 +468,18 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Build mesh accumulation. /// -/// @param[in] ion A @ref pic::ion object. -/// @param[in] batch The batch size. +/// @param[in] ion A @ref pic::ion object. /// @returns Expressions for mesh accumulation. //------------------------------------------------------------------------------ - std::array, 2> build_mesh_solve(const ion &ion, - const size_t batch=1) const { - auto next_index = index; - auto next_weight = y[0]; - auto kernel_index = graph::index (); - - for (size_t i = 0; i < batch; i++) { - auto index_i = graph::index_1D(ion.indices, next_index, - static_cast (1), - static_cast (0)); - auto index_w0 = graph::index_1D(ion.weights[0], next_index, - static_cast (1), - static_cast (0)); - auto index_w1 = graph::index_1D(ion.weights[1], next_index, - static_cast (1), - static_cast (0)); - auto index_w2 = graph::index_1D(ion.weights[2], next_index, - static_cast (1), - static_cast (0)); - next_index = next_index + static_cast (1); - next_weight = graph::if_(index_i - static_cast (1) == kernel_index, - next_weight + index_w0, next_weight); - next_weight = graph::if_(index_i == kernel_index, - next_weight + index_w1, next_weight); - next_weight = graph::if_(index_i + static_cast (1) == kernel_index, - next_weight + index_w2, next_weight); - } - return {next_index, next_weight}; + std::array, 3> build_mesh_solve(const ion &ion) const { + auto weights = build_weights(ion.x); + auto sum_low = graph::atomic_accumulate_1D(y[0], ion.x - dx, + dx, xmin, weights[0]); + auto sum = graph::atomic_accumulate_1D(y[0], ion.x, + dx, xmin, weights[1]); + auto sum_high = graph::atomic_accumulate_1D(y[0], ion.x + dx, + dx, xmin, weights[2]); + return {sum_low, sum, sum_high}; } //------------------------------------------------------------------------------ @@ -539,30 +519,42 @@ namespace pic { data.create_variable(file, "y_2", y[2], work.get_context()); data.create_variable(file, "y_3", y[3], work.get_context()); } - }; //------------------------------------------------------------------------------ /// @brief Build interpolation weights. /// -/// @tparam T Base type of the calculation. -/// @param[in] x The x position. -/// @param[in] mesh Mesh object. +/// @param[in] x The x position. /// @returns The interpolated mesh weights. //------------------------------------------------------------------------------ - template - std::array, 3> build_weights(graph::shared_leaf x, - const mesh &mesh) { - auto x_off = mesh.build_x_index(x) - x; - auto xnorm1 = static_cast (1.5) + (x_off - mesh.dx)/mesh.dx; - auto xnorm2 = x_off/mesh.dx; - auto xnorm3 = static_cast (1.5) - (x_off + mesh.dx)/mesh.dx; - - auto w0 = static_cast (0.5)*xnorm1*xnorm1; - auto w1 = static_cast (0.75) - xnorm2*xnorm2; - auto w2 = static_cast (0.5)*xnorm3*xnorm3; - - return {w0, w1, w2}; - } + std::array, 3> build_weights(graph::shared_leaf x) const { + auto x_off = build_x_index(x) - x; + auto xnorm1 = static_cast (1.5) + (x_off - dx)/dx; + auto xnorm2 = x_off/dx; + auto xnorm3 = static_cast (1.5) - (x_off + dx)/dx; + + auto w0 = static_cast (0.5)*xnorm1*xnorm1; + auto w1 = static_cast (0.75) - xnorm2*xnorm2; + auto w2 = static_cast (0.5)*xnorm3*xnorm3; + + return {w0, w1, w2}; + } + +//------------------------------------------------------------------------------ +/// @brief Build interpolation expression. +/// +/// @param[in] x The x position. +/// @returns The interpolated mesh quantity. +//------------------------------------------------------------------------------ + graph::shared_leaf build_interpolation(graph::shared_leaf x) const { + auto weights = build_weights(x); + + auto ymesh0 = graph::index_1D(y[0], x - dx, dx, xmin); + auto ymesh1 = graph::index_1D(y[0], x, dx, xmin); + auto ymesh2 = graph::index_1D(y[0], x + dx, dx, xmin); + + return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; + } + }; //------------------------------------------------------------------------------ /// @brief Build initialization. @@ -775,27 +767,6 @@ namespace pic { return graph::none ()/n*(dndx*te*scale + pressure->df(x)); } -//------------------------------------------------------------------------------ -/// @brief Build interpolation expression. -/// -/// @tparam T Base type of the calculation. -/// -/// @param[in] x The x position. -/// @param[in] mesh Mesh object. -/// @returns The interpolated mesh quantity. -//------------------------------------------------------------------------------ - template - graph::shared_leaf build_interpolation(graph::shared_leaf x, - mesh &mesh) { - auto weights = build_weights (x, mesh); - - auto ymesh0 = graph::index_1D(mesh.y[0], x - mesh.dx, mesh.dx, mesh.xmin); - auto ymesh1 = graph::index_1D(mesh.y[0], x, mesh.dx, mesh.xmin); - auto ymesh2 = graph::index_1D(mesh.y[0], x + mesh.dx, mesh.dx, mesh.xmin); - - return weights[0]*ymesh0 + weights[1]*ymesh1 + weights[2]*ymesh2; - } - //------------------------------------------------------------------------------ /// @brief Build interpolation expression. /// @@ -814,7 +785,7 @@ namespace pic { const mesh &mesh, const characteristics &norms, const parameters ¶ms) { - auto weights = build_weights (x, mesh); + auto weights = mesh.build_weights(x); auto ymesh0 = build_electric_efield::low> (x, ion, mesh, norms, params); auto ymesh1 = build_electric_efield::center> (x, ion, mesh, norms, params); diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index 3c1b1ce..85d4c05 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -79,7 +79,6 @@ namespace graph { /// @param[in] x_register_name Register for the x argument. /// @param[in] y_register_name Register for the x argument. /// @param[in] num_columns The y index. -/// @param //------------------------------------------------------------------------------ template void compile_2D_index(std::ostringstream &stream, @@ -737,15 +736,6 @@ namespace graph { return leaf_node::caches.backends[data_hash].has_zero(); } -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -1352,15 +1342,6 @@ namespace graph { return leaf_node::caches.backends[data_hash].has_zero(); } -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -1540,9 +1521,9 @@ namespace graph { index_1D_node::to_string(var, x)) {} //------------------------------------------------------------------------------ -/// @brief Evaluate the results of the piecewise constant. +/// @brief Evaluate the results of the 1D index. /// -/// Evaluate functions are only used by the minimization. So this node does not +/// Evaluate functions are only used by the reduction. So this node does not /// evaluate the argument. Instead this only returns the data as if it were a /// constant. /// @@ -1563,7 +1544,7 @@ namespace graph { } //------------------------------------------------------------------------------ -/// @brief the node. +/// @brief Compile the node. /// /// This node first evaluates the value of the argument then chooses the /// correct index of the variable. @@ -1615,19 +1596,16 @@ namespace graph { virtual bool is_match(shared_leaf x) { auto x_cast = index_1D_cast(x); - if (x_cast.get()) { - return this->left->is_match(x_cast->get_left()) && - this->is_arg_match(x); - } - - return false; + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->is_arg_match(x); } //------------------------------------------------------------------------------ /// @brief Convert the node to latex. //------------------------------------------------------------------------------ virtual void to_latex() const { - std::cout << "r\\_" << reinterpret_cast (this->left.get()) + std::cout << "v\\_" << reinterpret_cast (this->left.get()) << "\\left[i\\_" << reinterpret_cast (this->right.get()) << "\\right]"; @@ -1658,24 +1636,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if node is a constant. -/// -/// @returns True if the node is a constant. -//------------------------------------------------------------------------------ - virtual bool is_constant() const { - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -1712,9 +1672,9 @@ namespace graph { /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// -/// @param[in] v Variable to index. -/// @param[in] x Argument. -/// @returns A reduced piecewise_1D node. +/// @param[in] v Variable to index. +/// @param[in] x Argument. +/// @returns A reduced index_1D node. //------------------------------------------------------------------------------ template shared_leaf index_1D(shared_leaf v, @@ -1749,7 +1709,7 @@ namespace graph { /// @param[in] x Argument. /// @param[in] scale Argument scale factor. /// @param[in] offset Argument offset factor. -/// @returns A reduced piecewise_1D node. +/// @returns A reduced index_1D node. //------------------------------------------------------------------------------ template shared_leaf index_1D(shared_leaf v, @@ -2000,24 +1960,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if node is a constant. -/// -/// @returns True if the node is a constant. -//------------------------------------------------------------------------------ - virtual bool is_constant() const { - return false; - } - -//------------------------------------------------------------------------------ -/// @brief Test if node acts like a variable. -/// -/// @returns True if the node acts like a variable. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Test if the node acts like a power of variable. /// @@ -2140,6 +2082,319 @@ namespace graph { shared_index_2D index_2D_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// 1D Atomic Accumulate. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing a 1D accumulated array. +/// +/// This class is used to implement summation into an array. This uses atomic +/// add to avoid race conditions when multiple threads try to accumulate. +/// +/// Indicies are selected by +/// +/// x_norm' = (x - xmin)/dx (1) +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class atomic_accumulate_1D_node final : public triple_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string with the argument. +/// +/// @param[in] v Array to accumulate to. +/// @param[in] i Index of the array. +/// @param[in] x Argument to add to the existing value. +/// @returns A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(shared_leaf v, + shared_leaf i, + shared_leaf x) { + return jit::format_to_string(v->get_hash()) + + jit::format_to_string(i->get_hash()) + + jit::format_to_string(x->get_hash()); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a 1D index. +/// +/// @param[in] var Array node to accumulate to. +/// @param[in] index Index into the array. +/// @param[in] x Argument to add the existing array value. +//------------------------------------------------------------------------------ + atomic_accumulate_1D_node(shared_leaf var, + shared_leaf index, + shared_leaf x) : + triple_node (var, index, x, + atomic_accumulate_1D_node::to_string(var, index, x)) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of accumulate. +/// +/// Evaluate functions are only used by the reduction. So this node does not +/// evaluate the argument. Instead this only returns the data as if it were a +/// constant. +/// +/// @returns The evaluated value of the node. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + return this->left->evaluate(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// This node is effectively. +/// +/// y_i + x (1) +/// +/// So its effective derivative is +/// +/// ∂y_i/∂z + ∂x/dz +/// +/// @param[in] x The variable to take the derivative to. +/// @return The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + return constant (static_cast (this->left->is_match(x))) + + this->right->df(x); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// This node first evaluates the value of the argument then chooses the +/// correct index of the variable. +/// +/// x' = (x - xmin)/dx (1) +/// +/// @note Since this node accumulates, the right hand side is basically. +/// +/// y[i] = atomic_add(y[i], x) (2) +/// +/// @note The atomic add varies depending on the backend. +/// +/// - Metal atomic_fetch_add_explicit +/// - Cuda atomic_add +/// - CPU std::atomic_fetch_add_explicit +/// +/// @note These functions only take atomic data types. This changes the type +/// used the kernel argument. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->left->compile(stream, registers, + thread_mem, usage); + auto index = this->middle->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('v', a.get()) + + "[" + + registers[index.get()] + + "]"; + stream << " atomic"; + if constexpr (jit::use_cuda()) { + stream << "Add(&"; + } else if constexpr (jit::use_metal ()){ + stream << "_fetch_add_explicit(&"; + } else { + stream << "_ref("; + } + stream << registers[this]; + if constexpr (jit::use_cuda() || + jit::use_metal ()) { + stream << ", "; + } else { + stream << ").fetch_add("; + } + stream << registers[r.get()]; + if constexpr (jit::use_cuda()) { + stream << ")"; + } else { + stream << ", memory_order_relaxed)"; + } + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// Assumes both arguments are either set or not set. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + auto x_cast = atomic_accumulate_1D_cast(x); + + return x_cast.get() && + this->left->is_match(x_cast->get_left()) && + this->is_arg_match(x) && + this->right->is_match(x_cast->get_right()); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes arguments match. +/// +/// The argument of this node can be deferred so we need to check if the +/// arguments are null. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_arg_match(shared_leaf x) { + auto x_cast = atomic_accumulate_1D_cast(x); + + return x_cast.get() && + this->middle->is_match(x_cast->get_middle()); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "v\\_" << reinterpret_cast (this->left.get()) + << "\\left[i\\_" + << reinterpret_cast (this->middle.get()) + << "\\right] + "; + this->right->to_latex(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"r_" << reinterpret_cast (this->left.get()) + << "\", shape = hexagon, style = filled, fillcolor = black, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto m = this->middle->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[m.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the exponent of a power. +/// +/// @returns The exponent of a power like node. +//------------------------------------------------------------------------------ + virtual shared_leaf get_power_exponent() const { + return one (); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Define atomic_accumulate_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] i Index into the variable. +/// @param[in] x Argument. +/// @returns A reduced atomic_accumulate_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf atomic_accumulate_1D(shared_leaf v, + shared_leaf i, + shared_leaf x) { + assert(argument_cast(i).get() && + "atomic_accumulate_1D requires a argument node for second arg."); + auto temp = std::make_shared> (v, i, x)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Define atomic_accumulate_1D convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] v Variable to index. +/// @param[in] x Index Argument. +/// @param[in] scale Argument scale factor. +/// @param[in] offset Argument offset factor. +/// @param[in] y Argument. +/// @returns A reduced atomic_accumulate_1D node. +//------------------------------------------------------------------------------ + template + shared_leaf atomic_accumulate_1D(shared_leaf v, + shared_leaf x, + const T scale, + const T offset, + shared_leaf y) { + assert(variable_cast(v).get() && + "atomic_accumulate_1D requires a variable node for first arg."); + auto index = argument(x, scale, offset, variable_cast(v)->size()); + return atomic_accumulate_1D (v, index, y); + } + +/// Convenience type alias for shared atomic accumulate 1D nodes. + template + using shared_atomic_accumulate_1D = + std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a atomic accumulate 1D node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_atomic_accumulate_1D + atomic_accumulate_1D_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* piecewise_h */ diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index 5c65ee7..083551b 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -157,15 +157,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if all the sub-nodes terminate in variables. -/// -/// @returns True if all the sub-nodes terminate in variables. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Get the exponent of a power. /// @@ -472,15 +463,6 @@ namespace graph { return this->shared_from_this(); } -//------------------------------------------------------------------------------ -/// @brief Test if all the sub-nodes terminate in variables. -/// -/// @returns True if all the sub-nodes terminate in variables. -//------------------------------------------------------------------------------ - virtual bool is_all_variables() const { - return false; - } - //------------------------------------------------------------------------------ /// @brief Get the exponent of a power. /// diff --git a/graph_framework/solver.hpp b/graph_framework/solver.hpp index 244fabd..2196164 100644 --- a/graph_framework/solver.hpp +++ b/graph_framework/solver.hpp @@ -329,9 +329,7 @@ namespace solver { {this->t_next, graph::variable_cast(this->t)} }; - work.add_item(inputs, outputs, setters, - graph::shared_random_state (), + work.add_item(inputs, outputs, setters, {}, NULL, "solver_kernel", inputs.back()->size()); work.compile(); @@ -964,10 +962,10 @@ namespace solver { graph::variable_cast(lambda) }; - solver::newton(this->work, { + solver::newton (this->work, { var, graph::variable_cast(lambda) - }, inputs, loss, graph::shared_random_state ()); + }, inputs, {}, loss, NULL); inputs = { graph::variable_cast(this->t), @@ -997,9 +995,7 @@ namespace solver { {this->t_next, graph::variable_cast(this->t)} }; - this->work.add_item(inputs, outputs, setters, - graph::shared_random_state (), + this->work.add_item(inputs, outputs, setters, {}, NULL, "solver_kernel", inputs.back()->size()); this->work.compile(); } diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index 46bf7e1..edf6510 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -183,6 +183,8 @@ namespace workflow { graph::input_nodes inputs; /// Output nodes. graph::output_nodes outputs; +/// Atomic nodes. + graph::input_nodes atomics; /// Random state node. graph::shared_random_state state; @@ -193,20 +195,22 @@ namespace workflow { /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. /// @param[in,out] context Jit context. //------------------------------------------------------------------------------ - work_item(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, + work_item(graph::input_nodes &in, + graph::output_nodes &out, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context) : - inputs(in), outputs(out), state(state), + inputs(in), outputs(out), atomics(atomics), state(state), kernel_name(name), kernel_size(size) { - context.add_kernel(name, in, out, maps, state, size); + context.add_kernel(name, in, out, maps, atomics, state, size); } //------------------------------------------------------------------------------ @@ -216,7 +220,7 @@ namespace workflow { //------------------------------------------------------------------------------ virtual void create_kernel_call(jit::context &context) { kernel = context.create_kernel_call(kernel_name, inputs, outputs, - state, kernel_size); + atomics, state, kernel_size); } //------------------------------------------------------------------------------ @@ -246,6 +250,8 @@ namespace workflow { graph::input_nodes inputs; /// Output nodes. graph::output_nodes outputs; +/// Atomic nodes. + graph::input_nodes atomics; /// Random state node. graph::shared_random_state state; @@ -256,22 +262,25 @@ namespace workflow { /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. /// @param[in,out] context Jit context. /// @param[in] iterations Number of iterations to run the loop. //------------------------------------------------------------------------------ - loop_item(graph::input_nodes in, - graph::output_nodes out, - graph::map_nodes maps, - graph::shared_random_state state, + loop_item(graph::input_nodes &in, + graph::output_nodes &out, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context, const size_t iterations) : - inputs(in), outputs(out), state(state), + inputs(in), outputs(out), atomics(atomics),state(state), kernel_name(name), kernel_size(size) { - context.add_kernel(name, in, out, maps, state, size, iterations); + context.add_kernel(name, in, out, maps, atomics, + state, size, iterations); } //------------------------------------------------------------------------------ @@ -281,7 +290,7 @@ namespace workflow { //------------------------------------------------------------------------------ virtual void create_kernel_call(jit::context &context) { kernel = context.create_kernel_call(kernel_name, inputs, outputs, - state, kernel_size); + atomics, state, kernel_size); } //------------------------------------------------------------------------------ @@ -315,6 +324,7 @@ namespace workflow { /// @param[in] inputs Input variables. /// @param[in] outputs Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. @@ -322,15 +332,17 @@ namespace workflow { /// @param[in] tol Tolerance to solve the dispersion function to. /// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ - converge_item(graph::input_nodes inputs, - graph::output_nodes outputs, - graph::map_nodes maps, - graph::shared_random_state state, + converge_item(graph::input_nodes &inputs, + graph::output_nodes &outputs, + graph::map_nodes &maps, + graph::input_nodes &atomics, + graph::shared_random_state &state, const std::string name, const size_t size, jit::context &context, const T tol=1.0E-30, const size_t max_iter=1000) : - work_item (inputs, outputs, maps, state, name, size, context), + work_item (inputs, outputs, maps, atomics, + state, name, size, context), tolerance(tol), max_iterations(max_iter) { context.add_max_reduction(size); } @@ -435,32 +447,40 @@ namespace workflow { /// /// @tparam O The @ref workflow::order /// -/// @param[in] in Input variables. -/// @param[in] out Output nodes. -/// @param[in] maps Setter maps. -/// @param[in] state Random state node. -/// @param[in] name Name of the work item. -/// @param[in] size Size of the work item. +/// @param[in] in Input variables. +/// @param[in] out Output nodes. +/// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. +/// @param[in] state Random state node. +/// @param[in] name Name of the work item. +/// @param[in] size Size of the work item. //------------------------------------------------------------------------------ template void add_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size) { if constexpr (O == pre_run_item) { preitems.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context)); } else if constexpr (O == run_item) { items.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context)); } else { postitems.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context)); } @@ -471,7 +491,7 @@ namespace workflow { /// /// @tparam O The @ref workflow::order /// -/// @param[in] in Input variables. +/// @param[in] in Input variables. //------------------------------------------------------------------------------ template void add_zero_item(graph::input_nodes in) { @@ -510,6 +530,7 @@ namespace workflow { /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. @@ -519,24 +540,30 @@ namespace workflow { void add_loop_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size, const size_t iterations) { if constexpr (O == pre_run_item) { - preitems.push_back(std::make_unique> (in, out, - maps, state, + preitems.push_back(std::make_unique> (in, out, maps, + atomics, + state, name, size, context, iterations)); } else if constexpr (O == run_item) { items.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context, iterations)); } else { postitems.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context, iterations)); @@ -551,6 +578,7 @@ namespace workflow { /// @param[in] in Input variables. /// @param[in] out Output nodes. /// @param[in] maps Setter maps. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random state node. /// @param[in] name Name of the work item. /// @param[in] size Size of the work item. @@ -561,6 +589,7 @@ namespace workflow { void add_converge_item(graph::input_nodes in, graph::output_nodes out, graph::map_nodes maps, + graph::input_nodes atomics, graph::shared_random_state state, const std::string name, const size_t size, const T tol=1.0E-30, @@ -568,19 +597,25 @@ namespace workflow { add_reduction = true; if constexpr (O == pre_run_item) { items.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context, tol, max_iter)); } else if constexpr (O == run_item) { items.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context, tol, max_iter)); } else { postitems.push_back(std::make_unique> (in, out, - maps, state, + maps, + atomics, + state, name, size, context, tol, max_iter)); diff --git a/graph_korc/xkorc.cpp b/graph_korc/xkorc.cpp index f6ee85e..2c66230 100644 --- a/graph_korc/xkorc.cpp +++ b/graph_korc/xkorc.cpp @@ -82,7 +82,7 @@ void run_korc() { {u_init->get_y(), graph::variable_cast(uy)}, {u_init->get_z(), graph::variable_cast(uz)}, {gamma_init, graph::variable_cast(gamma)} - }, graph::shared_random_state (), "initialize_gamma", local_num_particles); + }, {}, NULL, "initialize_gamma", local_num_particles); auto u_prime = u_vec - dt*u_vec->cross(b_vec)/(2.0*gamma); @@ -118,7 +118,7 @@ void run_korc() { {u_next->get_y(), graph::variable_cast(uy)}, {u_next->get_z(), graph::variable_cast(uz)}, {gamma_next, graph::variable_cast(gamma)} - }, graph::shared_random_state (), "step", local_num_particles); + }, {}, NULL, "step", local_num_particles); work.compile(); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index daf87d1..7fd2398 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -20,10 +20,9 @@ void run_pic() { // Sizes const size_t num_particles = 3000000; const size_t num_grid = 1000; - const size_t num_batch = 1; const size_t num_ions = 1; const size_t num_steps = 1; - const size_t num_sub_steps = 100; + const size_t num_sub_steps = 1; const std::vector ion_masses{2*pic::m_atomic}; const std::vector ion_zs{1}; @@ -84,53 +83,32 @@ void run_pic() { auto ion_inits = pic::build_initialization (ions[i], mesh, norms, params, graph::random_state_cast(state)); + + if (i == 0) { + work.template add_zero_item ({ + graph::variable_cast(mesh.y[0]) + }); + } + work.template add_item ({ ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() }, {}, { {ion_inits[0], ions[i].get_x()}, {ion_inits[1], ions[i].get_v_para()}, {ion_inits[2], ions[i].get_v_perp()} - }, graph::random_state_cast(state), + }, {}, graph::random_state_cast(state), "pre_initization_" + ion_tag, num_particles); - auto mesh_i = mesh.build_i_index(ions[i].x); - auto weights = pic::build_weights (ions[i].x, mesh); + auto mesh_solve = mesh.build_mesh_solve(ions[i]); work.template add_item ({ - ions[i].get_x(), - graph::variable_cast(ions[i].weights[0]), - graph::variable_cast(ions[i].weights[1]), - graph::variable_cast(ions[i].weights[2]), - graph::variable_cast(ions[i].indices) + ions[i].get_x() + }, { + mesh_solve[0], + mesh_solve[1], + mesh_solve[2] }, {}, { - {weights[0], graph::variable_cast(ions[i].weights[0])}, - {weights[1], graph::variable_cast(ions[i].weights[1])}, - {weights[2], graph::variable_cast(ions[i].weights[2])}, - {mesh_i, graph::variable_cast(ions[i].indices)} - }, NULL, "pre_compute_weights_" + ion_tag, num_particles); - - if (i == 0) { - work.template add_zero_item ({ - graph::variable_cast(mesh.index), - graph::variable_cast(mesh.y[0]) - }); - } else { - work.template add_zero_item ({ - graph::variable_cast(mesh.index) - }); - } - - auto mesh_solve = mesh.build_mesh_solve(ions[i], num_batch); - work.template add_loop_item ({ - graph::variable_cast(ions[i].indices), - graph::variable_cast(ions[i].weights[0]), - graph::variable_cast(ions[i].weights[1]), - graph::variable_cast(ions[i].weights[2]), - graph::variable_cast(mesh.index), graph::variable_cast(mesh.y[0]) - }, {}, { - {mesh_solve[0], graph::variable_cast(mesh.index)}, - {mesh_solve[1], graph::variable_cast(mesh.y[0])} - }, NULL, "pre_sum_weights_" + ion_tag, num_grid, num_particles/num_batch); + }, NULL, "pre_sum_weights_" + ion_tag, num_particles); if (i == ions.size() - 1) { work.template add_copy_item ({ @@ -177,7 +155,7 @@ void run_pic() { {particle_step[0], ions[i].get_x()}, {particle_step[1], ions[i].get_v_para()}, {particle_step[2], ions[i].get_v_perp()} - }, NULL, "particle_push_" + ion_tag, num_particles); + }, {}, NULL, "particle_push_" + ion_tag, num_particles); auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, params, graph::random_state_cast(state)); @@ -189,37 +167,17 @@ void run_pic() { {particle_reinject[0], ions[i].get_x()}, {particle_reinject[1], ions[i].get_v_para()}, {particle_reinject[2], ions[i].get_v_perp()} - }, graph::random_state_cast(state), + }, {}, graph::random_state_cast(state), "particle_reinjection_" + ion_tag, num_particles); - work.add_item({ - ions[i].get_x(), - graph::variable_cast(ions[i].weights[0]), - graph::variable_cast(ions[i].weights[1]), - graph::variable_cast(ions[i].weights[2]), - graph::variable_cast(ions[i].indices) - }, {}, { - {weights[0], graph::variable_cast(ions[i].weights[0])}, - {weights[1], graph::variable_cast(ions[i].weights[1])}, - {weights[2], graph::variable_cast(ions[i].weights[2])}, - {mesh_i, graph::variable_cast(ions[i].indices)} - }, NULL, "compute_weights_" + ion_tag, num_particles); - if (i == 0) { work.add_callback_item([&mesh_sync]() { mesh_sync.lock(); mesh_sync.unlock(); }); - work.add_zero_item({ - graph::variable_cast(mesh.index), - graph::variable_cast(mesh.y[0]) - }); - } else { work.add_zero_item({ graph::variable_cast(mesh.index) }); - } - if (i == 0) { work.add_copy_item({ {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, @@ -227,17 +185,15 @@ void run_pic() { }); } - work.add_loop_item({ - graph::variable_cast(ions[i].indices), - graph::variable_cast(ions[i].weights[0]), - graph::variable_cast(ions[i].weights[1]), - graph::variable_cast(ions[i].weights[2]), - graph::variable_cast(mesh.index), - graph::variable_cast(mesh.y[0]) + work.add_item({ + graph::variable_cast(ions[i].x) + }, { + mesh_solve[0], + mesh_solve[1], + mesh_solve[2] }, {}, { - {mesh_solve[0], graph::variable_cast(mesh.index)}, - {mesh_solve[1], graph::variable_cast(mesh.y[0])} - }, NULL, "sum_weights_" + ion_tag, num_grid, num_particles/num_batch); + graph::variable_cast(mesh.y[0]) + }, NULL, "sum_weights_" + ion_tag, num_particles); } init.print(); diff --git a/graph_tests/efit_test.cpp b/graph_tests/efit_test.cpp index 9a15675..3b734dc 100644 --- a/graph_tests/efit_test.cpp +++ b/graph_tests/efit_test.cpp @@ -166,7 +166,7 @@ void run_test() { graph::variable_cast(z) }, { bvec->get_x(), bvec->get_y(), bvec->get_z(), ne, te, div - }, {}, graph::shared_random_state (), "test_kernel", xy_x_grid.size()); + }, {}, {}, NULL, "test_kernel", xy_x_grid.size()); work.compile(); work.run(); diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index d027da6..e40f60a 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -52,14 +52,15 @@ void compile(graph::input_nodes inputs, const T expected, const T tolerance) { jit::context source(0); + graph::input_nodes atomics; + graph::shared_random_state state; source.add_kernel("test_kernel", inputs, outputs, setters, - graph::shared_random_state (), - inputs.back()->size()); + atomics, state, inputs.back()->size()); source.compile(); auto run = source.create_kernel_call("test_kernel", inputs, outputs, - graph::shared_random_state (), 1); + atomics, state, 1); run(); T result; diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp index 02c39c9..f20362a 100644 --- a/graph_tests/no_derivative_test.cpp +++ b/graph_tests/no_derivative_test.cpp @@ -29,10 +29,6 @@ class dummy : public graph::no_derivativeshared_from_this(); } - virtual bool is_all_variables() const { - return false; - } - virtual graph::shared_leaf get_power_exponent() const { return graph::one (); } diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index f2515e0..871ac0a 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -44,8 +44,8 @@ template void run_interpolation_test() { ions[0].x_data()[i] = dxp*i + mesh.xmin; } - auto weights = pic::build_weights (ions[0].x, mesh); - auto field = pic::build_interpolation (ions[0].x, mesh); + auto weights = mesh.build_weights(ions[0].x); + auto field = mesh.build_interpolation(ions[0].x); auto weight = weights[0] + weights[1] + weights[2]; workflow::manager work(0); @@ -55,7 +55,7 @@ template void run_interpolation_test() { }, { weight, field - }, {}, NULL, "Mesh_Interpolation", num_particles); + }, {}, {}, NULL, "Mesh_Interpolation", num_particles); work.compile(); work.run(); work.wait(); @@ -265,38 +265,21 @@ template void run_field_solve_test() { } } - auto weights = pic::build_weights (ions[0].x, mesh); - auto mesh_i = mesh.build_i_index(ions[0].x); auto mesh_solve = mesh.build_mesh_solve(ions[0]); workflow::manager work(0); work.add_zero_item({ - graph::variable_cast(mesh.index), graph::variable_cast(mesh.y[0]) }); work.add_item({ - graph::variable_cast(ions[0].x), - graph::variable_cast(ions[0].weights[0]), - graph::variable_cast(ions[0].weights[1]), - graph::variable_cast(ions[0].weights[2]), - graph::variable_cast(ions[0].indices) + graph::variable_cast(ions[0].x) + }, { + mesh_solve[0], + mesh_solve[1], + mesh_solve[2] }, {}, { - {weights[0], graph::variable_cast(ions[0].weights[0])}, - {weights[1], graph::variable_cast(ions[0].weights[1])}, - {weights[2], graph::variable_cast(ions[0].weights[2])}, - {mesh_i, graph::variable_cast(ions[0].indices)} - }, NULL, "compute_weights", num_particles); - work.add_loop_item({ - graph::variable_cast(ions[0].indices), - graph::variable_cast(ions[0].weights[0]), - graph::variable_cast(ions[0].weights[1]), - graph::variable_cast(ions[0].weights[2]), - graph::variable_cast(mesh.index), graph::variable_cast(mesh.y[0]) - }, {}, { - {mesh_solve[0], graph::variable_cast(mesh.index)}, - {mesh_solve[1], graph::variable_cast(mesh.y[0])} - }, NULL, "sum_weights", num_mesh, num_particles); + }, NULL, "sum_weights", num_particles); work.compile(); diff --git a/graph_tests/piecewise_test.cpp b/graph_tests/piecewise_test.cpp index eae3207..8be948c 100644 --- a/graph_tests/piecewise_test.cpp +++ b/graph_tests/piecewise_test.cpp @@ -56,13 +56,15 @@ template void compile(graph::input_nodes inputs, const T expected, const T tolerance) { jit::context source(0); - source.add_kernel("test_kernel", inputs, outputs, setters, - graph::shared_random_state (), inputs.back()->size()); + graph::input_nodes atomics; + graph::shared_random_state state; + source.add_kernel("test_kernel", inputs, outputs, setters, atomics, state, + inputs.back()->size()); source.compile(); auto run = source.create_kernel_call("test_kernel", inputs, outputs, - graph::shared_random_state (), 1); + atomics, state, 1); run(); T result; diff --git a/graph_tests/random_test.cpp b/graph_tests/random_test.cpp index e0c0f93..fa5228c 100644 --- a/graph_tests/random_test.cpp +++ b/graph_tests/random_test.cpp @@ -49,8 +49,9 @@ template void test_dist() { auto random_real = (max - min)/graph::random_scale ()*random + min; workflow::manager work(0); - work.add_item({}, {random_real}, {}, graph::random_state_cast(state), - "step", N); + work.add_item({}, { + random_real + }, {}, {}, graph::random_state_cast(state), "step", N); work.compile(); work.run(); @@ -147,7 +148,7 @@ template void test_multi() { workflow::manager work(0); work.add_item({}, { random1, random2 - }, {}, graph::random_state_cast(state), "multi_random", 1); + }, {}, {}, graph::random_state_cast(state), "multi_random", 1); work.compile(); } diff --git a/graph_tests/workflow_test.cpp b/graph_tests/workflow_test.cpp index cc2ca7d..55477c8 100644 --- a/graph_tests/workflow_test.cpp +++ b/graph_tests/workflow_test.cpp @@ -112,7 +112,7 @@ template void test_maps() { }, {}, { {zero, graph::variable_cast(a)}, {zero, graph::variable_cast(b)} - }, NULL, "test_maps", 1); + }, {}, NULL, "test_maps", 1); work.compile(); @@ -141,7 +141,7 @@ template void test_loops() { graph::variable_cast(a) }, {}, { {a_next, graph::variable_cast(a)} - }, NULL, "test_maps", 1, 10); + }, {}, NULL, "test_maps", 1, 10); work.compile(); From a17d2ad78f2fb659d9b8b17f8cb6ebc15c8c1fd4 Mon Sep 17 00:00:00 2001 From: m4c Date: Tue, 4 Aug 2026 20:28:46 -0400 Subject: [PATCH 35/51] Enable atomics for the cuda backend. --- graph_framework/cuda_context.hpp | 51 ++++++++++++++++++++++++++++++-- graph_pic/xpic.cpp | 4 +-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 87e739c..1d8f9e3 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -16,6 +16,7 @@ #include "random.hpp" #include "timing.hpp" +#include "piecewise.hpp" /// Maximum number of registers to use. #define MAX_REG 256 @@ -322,6 +323,7 @@ namespace gpu { /// @param[in] kernel_name Name of the kernel for later reference. /// @param[in] inputs Input nodes of the kernel. /// @param[in] outputs Output nodes of the kernel. +/// @param[in] atomics Atomic nodes of the kernel. /// @param[in] state Random states. /// @param[in] num_rays Number of rays to trace.' /// @param[in] tex1d_list List of 1D textures. @@ -331,6 +333,7 @@ namespace gpu { std::function create_kernel_call(const std::string kernel_name, graph::input_nodes inputs, graph::output_nodes outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t num_rays, const jit::texture1d_list &tex1d_list, @@ -376,6 +379,25 @@ namespace gpu { needed_buffers.insert(output.get()); } } + for (auto &atomic : atomics) { + if (!kernel_arguments.contains(atomic.get())) { + kernel_arguments.try_emplace(atomic.get()); + check_error(cuMemAllocManaged(&kernel_arguments[atomic.get()], + atomic->size()*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + check_error(cuMemcpyHtoD(kernel_arguments[atomic.get()], + atomic->data(), + atomic->size()*sizeof(T)), + "cuMemcpyHtoD"); + buffers.push_back(reinterpret_cast (&kernel_arguments[atomic.get()])); + needed_buffers.insert(atomic.get()); + } + if (!needed_buffers.contains(atomic.get())) { + buffers.push_back(reinterpret_cast (&kernel_arguments[atomic.get()])); + needed_buffers.insert(atomic.get()); + } + } const size_t num_buffers = buffers.size(); if (state.get()) { @@ -912,6 +934,7 @@ namespace gpu { /// @param[in] name Name to call the kernel. /// @param[in] inputs Input variables of the kernel. /// @param[in] outputs Output nodes of the graph to compute. +/// @param[in] atomics Input variables for atomic operations. /// @param[in] state Random states. /// @param[in] size Size of the input buffer. /// @param[in] is_constant Flags if the input is read only. @@ -927,6 +950,7 @@ namespace gpu { const std::string name, graph::input_nodes &inputs, graph::output_nodes &outputs, + graph::input_nodes atomics, graph::shared_random_state state, const size_t size, const std::vector &is_constant, @@ -1017,7 +1041,8 @@ namespace gpu { } } for (size_t i = 0, ie = outputs.size(); i < ie; i++) { - if (!used_args.contains(outputs[i].get())) { + if (!used_args.contains(outputs[i].get()) && + !graph::atomic_accumulate_1D_cast(outputs[i]).get()) { if (i == 0) { if (inputs.size()) { inputs[inputs.size() - 1]->endline(source_buffer, @@ -1034,6 +1059,27 @@ namespace gpu { used_args.insert(outputs[i].get()); } } + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { + if (!used_args.contains(atomics[i].get())) { + if (i == 0) { + if (outputs.size()) { + outputs[outputs.size() - 1]->endline(source_buffer, + usage, ','); + } else if (inputs.size()) { + inputs[inputs.size() - 1]->endline(source_buffer, + usage, ','); + } + } else { + source_buffer << "," << std::endl; + } + + source_buffer << " "; + jit::add_type (source_buffer); + source_buffer << " * __restrict__ " + << jit::to_string('v', atomics[i].get()); + used_args.insert(atomics[i].get()); + } + } if (state.get()) { source_buffer << "," << std::endl << " mt_state * __restrict__ " @@ -1257,7 +1303,8 @@ namespace gpu { } for (auto &out : outputs) { - if (!graph::variable_cast(out).get() && + if (!graph::variable_cast(out).get() && + !graph::atomic_accumulate_1D_cast(out).get() && !out_registers.contains(out.get())) { auto a = out->compile(source_buffer, registers, thread_mem, usage); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 7fd2398..6a0b36c 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -21,8 +21,8 @@ void run_pic() { const size_t num_particles = 3000000; const size_t num_grid = 1000; const size_t num_ions = 1; - const size_t num_steps = 1; - const size_t num_sub_steps = 1; + const size_t num_steps = 100; + const size_t num_sub_steps = 2500; const std::vector ion_masses{2*pic::m_atomic}; const std::vector ion_zs{1}; From 884f4471780eedd425596137beedc86145850deb Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 4 Aug 2026 20:57:33 -0400 Subject: [PATCH 36/51] Argument nodes allow us to compute cell indicies and position without reading from constant memory. --- graph_framework/particle_in_cell.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 2cddc09..3afae5e 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -389,8 +389,7 @@ namespace pic { /// @returns The indexed mesh X position. //------------------------------------------------------------------------------ graph::shared_leaf build_x_index(graph::shared_leaf x) const { - const backend::buffer buffer(xmin, dx, size()); - return graph::piecewise_1D(buffer, x, dx, xmin); + return dx*graph::argument(x, dx, xmin, size()) + xmin; } //------------------------------------------------------------------------------ @@ -400,9 +399,7 @@ namespace pic { /// @returns The indexed mesh X position. //------------------------------------------------------------------------------ graph::shared_leaf build_i_index(graph::shared_leaf x) const { - const backend::buffer buffer(static_cast (0), - static_cast (1), size()); - return graph::piecewise_1D(buffer, x, dx, xmin); + return graph::argument(x, dx, xmin, size()); } //------------------------------------------------------------------------------ From 925c3eebbe42ee1f8e76cfcee93efe114d257f19 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 10 Aug 2026 19:59:42 -0400 Subject: [PATCH 37/51] Update C and Fortran bindings for new API changes. --- graph_c_binding/graph_c_binding.cpp | 5540 +++++++++++++++-- graph_c_binding/graph_c_binding.h | 402 ++ .../graph_fortran_binding.f90 | 35 +- graph_framework/backend.hpp | 30 +- graph_tests/c_binding_test.c | 4 + graph_tests/f_binding_test.f90 | 36 +- 6 files changed, 5471 insertions(+), 576 deletions(-) diff --git a/graph_c_binding/graph_c_binding.cpp b/graph_c_binding/graph_c_binding.cpp index d2a49ae..ffcab88 100644 --- a/graph_c_binding/graph_c_binding.cpp +++ b/graph_c_binding/graph_c_binding.cpp @@ -1886,103 +1886,4213 @@ extern "C" { } } -//****************************************************************************** -// JIT -//****************************************************************************** //------------------------------------------------------------------------------ -/// @brief Create 2D piecewise node with complex arguments. +/// @brief Create an atomic accumulate 1D index. +/// +/// @param[in] c The graph C context. +/// @param[in] variable The variable to index. +/// @param[in] index The function argument. +/// @param[in] scale Scale factor argument. +/// @param[in] offset Offset factor argument. +/// @param[in] arg Argument. +/// @returns An atomic accumulate 1D node. +//---------------------------------------------- +//------------------------------------------------------------------------------ + graph_node graph_atomic_accumulate_1D(STRUCT_TAG graph_c_context *c, + graph_node variable, + graph_node index, + const double scale, + const double offset, + graph_node arg) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast (scale), + static_cast (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::atomic_accumulate_1D(d->nodes[variable], + d->nodes[index], + static_cast> (scale), + static_cast> (offset), + d->nodes[arg]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an index code. /// /// @param[in] c The graph C context. -/// @returns The number of concurrent devices. +/// @returns An index node. //------------------------------------------------------------------------------ - size_t graph_get_max_concurrency(graph_c_context *c) { + graph_node graph_index(STRUCT_TAG graph_c_context *c) { switch (c->type) { case FLOAT: if (c->safe_math) { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } case DOUBLE: if (c->safe_math) { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context::max_concurrency(); + auto d = reinterpret_cast *> (c); + auto temp = graph::index (); + d->nodes[temp.get()] = temp; + return temp.get(); } case COMPLEX_FLOAT: if (c->safe_math) { - return jit::context, true>::max_concurrency(); + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::index, true> (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context>::max_concurrency(); + auto d = reinterpret_cast> *> (c); + auto temp = graph::index> (); + d->nodes[temp.get()] = temp; + return temp.get(); } case COMPLEX_DOUBLE: if (c->safe_math) { - return jit::context, true>::max_concurrency(); + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::index, true> (); + d->nodes[temp.get()] = temp; + return temp.get(); } else { - return jit::context>::max_concurrency(); + auto d = reinterpret_cast> *> (c); + auto temp = graph::index> (); + d->nodes[temp.get()] = temp; + return temp.get(); } } } -//****************************************************************************** -// Workflows -//****************************************************************************** //------------------------------------------------------------------------------ -/// @brief Choose the device number. +/// @brief Create not node. /// /// @param[in] c The graph C context. -/// @param[in] num The device number. +/// @param[in] arg The function argument. +/// @returns !arg //------------------------------------------------------------------------------ - void graph_set_device_number(STRUCT_TAG graph_c_context *c, - const size_t num) { + graph_node graph_not(STRUCT_TAG graph_c_context *c, + graph_node arg) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast *> (c); - d->work = workflow::manager (num); + auto temp = !d->nodes[arg]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left == right +//------------------------------------------------------------------------------ + graph_node graph_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work = workflow::manager, true> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast> *> (c); - d->work = workflow::manager> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } - break; case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - d->work = workflow::manager, true> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); } else { auto d = reinterpret_cast> *> (c); - d->work = workflow::manager> (num); + auto temp = d->nodes[left] == d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a not equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left != right +//------------------------------------------------------------------------------ + graph_node graph_not_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = d->nodes[left] != d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a greater than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left > right +//------------------------------------------------------------------------------ + graph_node graph_greater_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] > d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left < right +//------------------------------------------------------------------------------ + graph_node graph_less_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] < d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a greater than equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left >= right +//------------------------------------------------------------------------------ + graph_node graph_greater_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] >= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left <= right +//------------------------------------------------------------------------------ + graph_node graph_less_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] <= d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an and node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left && right +//------------------------------------------------------------------------------ + graph_node graph_and(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] && d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create an or node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left || right +//------------------------------------------------------------------------------ + graph_node graph_or(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = d->nodes[left] || d->nodes[right]; + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + case COMPLEX_DOUBLE: + std::cerr << "Operation not supported for complex types." << std::endl; + exit(1); + } + } + +//------------------------------------------------------------------------------ +/// @brief Create a if node. +/// +/// @param[in] c The graph C context. +/// @param[in] condition The logical condition. +/// @param[in] t The true case. +/// @param[in] f The false case. +/// @returns condiiton ? t : f +//------------------------------------------------------------------------------ + graph_node graph_if(STRUCT_TAG graph_c_context *c, + graph_node condition, + graph_node t, + graph_node f) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } else { + auto d = reinterpret_cast> *> (c); + auto temp = graph::if_(d->nodes[condition], + d->nodes[t], + d->nodes[f]); + d->nodes[temp.get()] = temp; + return temp.get(); + } + } + } + +//****************************************************************************** +// JIT +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Create 2D piecewise node with complex arguments. +/// +/// @param[in] c The graph C context. +/// @returns The number of concurrent devices. +//------------------------------------------------------------------------------ + size_t graph_get_max_concurrency(graph_c_context *c) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + return jit::context::max_concurrency(); + } else { + return jit::context::max_concurrency(); + } + + case DOUBLE: + if (c->safe_math) { + return jit::context::max_concurrency(); + } else { + return jit::context::max_concurrency(); + } + + case COMPLEX_FLOAT: + if (c->safe_math) { + return jit::context, true>::max_concurrency(); + } else { + return jit::context>::max_concurrency(); + } + + case COMPLEX_DOUBLE: + if (c->safe_math) { + return jit::context, true>::max_concurrency(); + } else { + return jit::context>::max_concurrency(); + } + } + } + +//****************************************************************************** +// Workflows +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Choose the device number. +/// +/// @param[in] c The graph C context. +/// @param[in] num The device number. +//------------------------------------------------------------------------------ + void graph_set_device_number(STRUCT_TAG graph_c_context *c, + const size_t num) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } else { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } else { + auto d = reinterpret_cast *> (c); + d->work = workflow::manager (num); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work = workflow::manager, true> (num); + } else { + auto d = reinterpret_cast> *> (c); + d->work = workflow::manager> (num); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work = workflow::manager, true> (num); + } else { + auto d = reinterpret_cast> *> (c); + d->work = workflow::manager> (num); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add pre workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_pre_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item(in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item(in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add pre workflow loop item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_pre_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add workflow loopitem. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item(in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item(in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add post workflow loopitem. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_post_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_item (in, out, map, atom, rand, name, size); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_item (in, out, map, atom, NULL, name, size); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_loop_item (in, out, map, atom, rand, name, size, iterations); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_loop_item (in, out, map, atom, NULL, name, size, iterations); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a pre converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_pre_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); + } } break; } } //------------------------------------------------------------------------------ -/// @brief Add pre workflow item. +/// @brief Add a post converge item. /// /// @param[in] c The graph C context. /// @param[in] inputs Array of input nodes. @@ -1992,18 +6102,25 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. //------------------------------------------------------------------------------ - void graph_add_pre_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, - graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size) { + void graph_add_post_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter) { switch (c->type) { case FLOAT: if (c->safe_math) { @@ -2014,7 +6131,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2028,20 +6145,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_item (in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } else { auto d = reinterpret_cast *> (c); @@ -2051,7 +6180,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2065,20 +6194,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_item (in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } break; @@ -2092,7 +6233,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2106,20 +6247,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_item (in, out, map, NULL, name, size); + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); } } else { auto d = reinterpret_cast *> (c); @@ -2129,7 +6282,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2143,20 +6296,32 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + d->work.add_converge_item(in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } } else { - d->work.add_item (in, out, map, NULL, name, size); + d->work.add_converge_item(in, out, map, atom, NULL, name, + size, tol, max_iter); } } break; @@ -2170,7 +6335,7 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } @@ -2184,24 +6349,249 @@ extern "C" { if (temp.get()) { map.push_back({d->nodes[map_outputs[i]], temp}); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes, true> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes, true> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; + exit(1); + } + } + if (random_state) { + auto rand = graph::random_state_cast(d->nodes[random_state]); + if (rand.get()) { + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); + } else { + std::cerr << "Invalid random state." << std::endl; + exit(1); + } + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Work input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::output_nodes> out; + for (size_t i = 0; i < num_outputs; i++) { + out.push_back(d->nodes[outputs[i]]); + } + graph::map_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + if (temp.get()) { + map.push_back({d->nodes[map_outputs[i]], temp}); + } else { + std::cerr << "Work map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + graph::input_nodes> atom; + for (size_t i = 0; i < num_atomics; i++) { + auto temp = graph::variable_cast(d->nodes[atomics[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); } } if (random_state) { auto rand = graph::random_state_cast(d->nodes[random_state]); if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + d->work.add_converge_item (in, out, map, atom, rand, name, + size, tol, max_iter); } else { std::cerr << "Invalid random state." << std::endl; exit(1); } - } else { - d->work.add_item (in, out, map, NULL, name, size); + } else { + d->work.add_converge_item (in, out, map, atom, NULL, name, + size, tol, max_iter); + } + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a pre zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_pre_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_zero_item (in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); + } else { + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; + exit(1); + } } + + d->work.add_zero_item (in); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2211,31 +6601,40 @@ extern "C" { exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item (in); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item (in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2252,31 +6651,8 @@ extern "C" { exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_item (in, out, map, NULL, name, size); - } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2289,59 +6665,22 @@ extern "C" { exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item (in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_item (in, out, map, NULL, name, size); - } + + d->work.add_zero_item (in); } break; } } //------------------------------------------------------------------------------ -/// @brief Add workflow item. +/// @brief Add a copy item. /// -/// @param[in] c The graph C context. -/// @param[in] inputs Array of input nodes. -/// @param[in] num_inputs Number of inputs. -/// @param[in] outputs Array of output nodes. -/// @param[in] num_outputs Number of outputs. -/// @param[in] map_inputs Array of map input nodes. -/// @param[in] map_outputs Array of map output nodes. -/// @param[in] num_maps Number of maps. -/// @param[in] random_state Optional random state, can be NULL if not used. -/// @param[in] name Name for the kernel. -/// @param[in] size Number of elements to operate on. +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. //------------------------------------------------------------------------------ - void graph_add_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, - graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size) { + void graph_add_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { switch (c->type) { case FLOAT: if (c->safe_math) { @@ -2356,34 +6695,43 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item(in); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2393,38 +6741,47 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item(in); } break; - case DOUBLE: + case COMPLEX_DOUBLE: if (c->safe_math) { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast, true> *> (c); + graph::input_nodes, true> in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { @@ -2434,68 +6791,98 @@ extern "C" { exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item(in); + } else { + auto d = reinterpret_cast> *> (c); + graph::input_nodes> in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Work input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item(in); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a post zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_post_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; + auto d = reinterpret_cast *> (c); + graph::input_nodes in; for (size_t i = 0; i < num_inputs; i++) { auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); + + d->work.add_zero_item (in); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + in.push_back(temp); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); + + d->work.add_zero_item (in); + } else { + auto d = reinterpret_cast *> (c); + graph::input_nodes in; + for (size_t i = 0; i < num_inputs; i++) { + auto temp = graph::variable_cast(d->nodes[inputs[i]]); + if (temp.get()) { + in.push_back(temp); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2508,35 +6895,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2545,35 +6909,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; @@ -2586,35 +6927,12 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } else { auto d = reinterpret_cast> *> (c); graph::input_nodes> in; @@ -2623,393 +6941,462 @@ extern "C" { if (temp.get()) { in.push_back(temp); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; - for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); - } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; - exit(1); - } - } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_item(in, out, map, rand, name, size); - } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_item(in, out, map, NULL, name, size); } + + d->work.add_zero_item (in); } break; } } //------------------------------------------------------------------------------ -/// @brief Add a converge item. +/// @brief Add a pre copy item. /// -/// @param[in] c The graph C context. -/// @param[in] inputs Array of input nodes. -/// @param[in] num_inputs Number of inputs. -/// @param[in] outputs Array of output nodes. -/// @param[in] num_outputs Number of outputs. -/// @param[in] map_inputs Array of map input nodes. -/// @param[in] map_outputs Array of map output nodes. -/// @param[in] num_maps Number of maps. -/// @param[in] random_state Optional random state, can be NULL if not used. -/// @param[in] name Name for the kernel. -/// @param[in] size Number of elements to operate on. -/// @param[in] tol Tolerance to converge the function to. -/// @param[in] max_iter Maximum number of iterations before giving up. +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. //------------------------------------------------------------------------------ - void graph_add_converge_item(STRUCT_TAG graph_c_context *c, - graph_node *inputs, size_t num_inputs, - graph_node *outputs, size_t num_outputs, + void graph_add_pre_copy_item(STRUCT_TAG graph_c_context *c, graph_node *map_inputs, - graph_node *map_outputs, size_t num_maps, - graph_node random_state, - const char *name, - const size_t size, - const double tol, - const size_t max_iter) { + graph_node *map_outputs, size_t num_maps) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } else { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } } - graph::map_nodes map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; + exit(1); + } + } + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); + } else { + std::cerr << "Preitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; + } + } - case DOUBLE: +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps) { + switch (c->type) { + case FLOAT: if (c->safe_math) { - auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + + d->work.add_copy_item(map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item(map); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item(map); } else { auto d = reinterpret_cast *> (c); - graph::input_nodes in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); - } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; - exit(1); - } - } - graph::output_nodes out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes map; + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); - } else { - std::cerr << "Invalid random state." << std::endl; - exit(1); - } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); - } + + d->work.add_copy_item(map); } break; case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - graph::input_nodes, true> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; + + d->work.add_copy_item(map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item(map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item(map); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Work map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; + + d->work.add_copy_item(map); + } + break; + } + } + +//------------------------------------------------------------------------------ +/// @brief Add a post copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_post_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; - case COMPLEX_DOUBLE: + case DOUBLE: if (c->safe_math) { - auto d = reinterpret_cast, true> *> (c); - graph::input_nodes, true> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes, true> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes, true> map; + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast *> (c); + graph::copy_nodes map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } else { - auto d = reinterpret_cast> *> (c); - graph::input_nodes> in; - for (size_t i = 0; i < num_inputs; i++) { - auto temp = graph::variable_cast(d->nodes[inputs[i]]); - if (temp.get()) { - in.push_back(temp); + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - graph::output_nodes> out; - for (size_t i = 0; i < num_outputs; i++) { - out.push_back(d->nodes[outputs[i]]); - } - graph::map_nodes> map; + + d->work.add_copy_item (map); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + graph::copy_nodes, true> map; for (size_t i = 0; i < num_maps; i++) { - auto temp = graph::variable_cast(d->nodes[map_inputs[i]]); - if (temp.get()) { - map.push_back({d->nodes[map_outputs[i]], temp}); + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Work map input " << i << " is not a variable." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } } - if (random_state) { - auto rand = graph::random_state_cast(d->nodes[random_state]); - if (rand.get()) { - d->work.add_converge_item(in, out, map, rand, name, - size, tol, max_iter); + + d->work.add_copy_item (map); + } else { + auto d = reinterpret_cast> *> (c); + graph::copy_nodes> map; + for (size_t i = 0; i < num_maps; i++) { + auto temp_in = graph::variable_cast(d->nodes[map_inputs[i]]); + auto temp_out = graph::variable_cast(d->nodes[map_outputs[i]]); + if (temp_in.get() && temp_out.get()) { + map.push_back({temp_out, temp_in}); } else { - std::cerr << "Invalid random state." << std::endl; + std::cerr << "Postitem map input " << i << " is not a variable." << std::endl; exit(1); } - } else { - d->work.add_converge_item(in, out, map, NULL, name, - size, tol, max_iter); } + + d->work.add_copy_item (map); } break; } @@ -3162,6 +7549,55 @@ extern "C" { } } +//------------------------------------------------------------------------------ +/// @brief Run post work items. +/// +/// @param[in] c The graph C context. +//------------------------------------------------------------------------------ + void graph_post_run(graph_c_context *c) { + switch (c->type) { + case FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast *> (c); + d->work.run (); + } + break; + + case DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast *> (c); + d->work.run (); + } + break; + + case COMPLEX_FLOAT: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast> *> (c); + d->work.run (); + } + break; + + case COMPLEX_DOUBLE: + if (c->safe_math) { + auto d = reinterpret_cast, true> *> (c); + d->work.run (); + } else { + auto d = reinterpret_cast> *> (c); + d->work.run (); + } + break; + } + } + //------------------------------------------------------------------------------ /// @brief Wait for work items to complete. /// diff --git a/graph_c_binding/graph_c_binding.h b/graph_c_binding/graph_c_binding.h index 49a27b5..3c56586 100644 --- a/graph_c_binding/graph_c_binding.h +++ b/graph_c_binding/graph_c_binding.h @@ -485,6 +485,152 @@ extern "C" { const double y_scale, const double y_offset); +//------------------------------------------------------------------------------ +/// @brief Create an atomic accumulate 1D index. +/// +/// @param[in] c The graph C context. +/// @param[in] variable The variable to index. +/// @param[in] index The function argument. +/// @param[in] scale Scale factor argument. +/// @param[in] offset Offset factor argument. +/// @param[in] arg Argument. +/// @returns An atomic accumulate 1D node. +//------------------------------------------------------------------------------ + graph_node graph_atomic_accumulate_1D(STRUCT_TAG graph_c_context *c, + graph_node variable, + graph_node index, + const double scale, + const double offset, + graph_node arg); + +//------------------------------------------------------------------------------ +/// @brief Create an index code. +/// +/// @param[in] c The graph C context. +/// @returns An index node. +//------------------------------------------------------------------------------ + graph_node graph_index(STRUCT_TAG graph_c_context *c); + +//------------------------------------------------------------------------------ +/// @brief Create not node. +/// +/// @param[in] c The graph C context. +/// @param[in] arg The function argument. +/// @returns !arg +//------------------------------------------------------------------------------ + graph_node graph_not(STRUCT_TAG graph_c_context *c, + graph_node arg); + +//------------------------------------------------------------------------------ +/// @brief Create an equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left == right +//------------------------------------------------------------------------------ + graph_node graph_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a not equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left != right +//------------------------------------------------------------------------------ + graph_node graph_not_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a greater than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left > right +//------------------------------------------------------------------------------ + graph_node graph_greater_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left < right +//------------------------------------------------------------------------------ + graph_node graph_less_than(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a greater than equal node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left >= right +//------------------------------------------------------------------------------ + graph_node graph_greater_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a less than node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left <= right +//------------------------------------------------------------------------------ + graph_node graph_less_than_equal(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create an and node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left && right +//------------------------------------------------------------------------------ + graph_node graph_and(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create an or node. +/// +/// @param[in] c The graph C context. +/// @param[in] left The left operand. +/// @param[in] right The right operand. +/// @returns left || right +//------------------------------------------------------------------------------ + graph_node graph_or(STRUCT_TAG graph_c_context *c, + graph_node left, + graph_node right); + +//------------------------------------------------------------------------------ +/// @brief Create a if node. +/// +/// @param[in] c The graph C context. +/// @param[in] condition The logical condition. +/// @param[in] t The true case. +/// @param[in] f The false case. +/// @returns condiiton ? t : f +//------------------------------------------------------------------------------ + graph_node graph_if(STRUCT_TAG graph_c_context *c, + graph_node condition, + graph_node t, + graph_node f); + //------------------------------------------------------------------------------ /// @brief Create 2D piecewise node with complex arguments. /// @@ -513,6 +659,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -522,6 +670,7 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size); @@ -537,6 +686,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -546,10 +697,154 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size); +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size); + +//------------------------------------------------------------------------------ +/// @brief Add pre loop workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_pre_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations); + +//------------------------------------------------------------------------------ +/// @brief Add workflow loop item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] iterations Number of loop iterations. +//------------------------------------------------------------------------------ + void graph_add_loop_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const size_t iterations); + +//------------------------------------------------------------------------------ +/// @brief Add post workflow item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +//------------------------------------------------------------------------------ + void graph_add_post_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size); + +//------------------------------------------------------------------------------ +/// @brief Add a pre converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_pre_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter); + //------------------------------------------------------------------------------ /// @brief Add a converge item. /// @@ -561,6 +856,8 @@ extern "C" { /// @param[in] map_inputs Array of map input nodes. /// @param[in] map_outputs Array of map output nodes. /// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. /// @param[in] random_state Optional random state, can be NULL if not used. /// @param[in] name Name for the kernel. /// @param[in] size Number of elements to operate on. @@ -572,12 +869,110 @@ extern "C" { graph_node *outputs, size_t num_outputs, graph_node *map_inputs, graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, graph_node random_state, const char *name, const size_t size, const double tol, const size_t max_iter); +//------------------------------------------------------------------------------ +/// @brief Add a post converge item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +/// @param[in] outputs Array of output nodes. +/// @param[in] num_outputs Number of outputs. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +/// @param[in] atomics Array of atomic nodes. +/// @param[in] num_atomics Number of atomics. +/// @param[in] random_state Optional random state, can be NULL if not used. +/// @param[in] name Name for the kernel. +/// @param[in] size Number of elements to operate on. +/// @param[in] tol Tolerance to converge the function to. +/// @param[in] max_iter Maximum number of iterations before giving up. +//------------------------------------------------------------------------------ + void graph_add_post_converge_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs, + graph_node *outputs, size_t num_outputs, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps, + graph_node *atomics, size_t num_atomics, + graph_node random_state, + const char *name, + const size_t size, + const double tol, + const size_t max_iter); + +//------------------------------------------------------------------------------ +/// @brief Add a pre zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_pre_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a post zero item. +/// +/// @param[in] c The graph C context. +/// @param[in] inputs Array of input nodes. +/// @param[in] num_inputs Number of inputs. +//------------------------------------------------------------------------------ + void graph_add_post_zero_item(STRUCT_TAG graph_c_context *c, + graph_node *inputs, size_t num_inputs); + +//------------------------------------------------------------------------------ +/// @brief Add a pre copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_pre_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + +//------------------------------------------------------------------------------ +/// @brief Add a copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + +//------------------------------------------------------------------------------ +/// @brief Add a post copy item. +/// +/// @param[in] c The graph C context. +/// @param[in] map_inputs Array of map input nodes. +/// @param[in] map_outputs Array of map output nodes. +/// @param[in] num_maps Number of maps. +//------------------------------------------------------------------------------ + void graph_add_post_copy_item(STRUCT_TAG graph_c_context *c, + graph_node *map_inputs, + graph_node *map_outputs, size_t num_maps); + //------------------------------------------------------------------------------ /// @brief Compile the work items. /// @@ -599,6 +994,13 @@ extern "C" { //------------------------------------------------------------------------------ void graph_run(STRUCT_TAG graph_c_context *c); +//------------------------------------------------------------------------------ +/// @brief Run post work items. +/// +/// @param[in] c The graph C context. +//------------------------------------------------------------------------------ + void graph_post_run(STRUCT_TAG graph_c_context *c); + //------------------------------------------------------------------------------ /// @brief Wait for work items to complete. /// diff --git a/graph_fortran_binding/graph_fortran_binding.f90 b/graph_fortran_binding/graph_fortran_binding.f90 index bc5d7c6..e677071 100644 --- a/graph_fortran_binding/graph_fortran_binding.f90 +++ b/graph_fortran_binding/graph_fortran_binding.f90 @@ -742,6 +742,8 @@ SUBROUTINE graph_set_device_number(c, num) & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -749,6 +751,7 @@ SUBROUTINE graph_set_device_number(c, num) & SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles) & BIND(C, NAME='graph_add_pre_item') USE, INTRINSIC :: ISO_C_BINDING @@ -761,6 +764,8 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -777,6 +782,8 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -784,6 +791,7 @@ SUBROUTINE graph_add_pre_item(c, inputs, num_inputs, & SUBROUTINE graph_add_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles) & BIND(C, NAME='graph_add_item') USE, INTRINSIC :: ISO_C_BINDING @@ -796,6 +804,8 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -812,6 +822,8 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. !> @param[in] num_maps Number of maps. +!> @param[in] atomics Array of atomics nodes. +!> @param[in] num_atomics Number of atomics. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -821,6 +833,7 @@ SUBROUTINE graph_add_item(c, inputs, num_inputs, & SUBROUTINE graph_add_converge_item(c, inputs, num_inputs, & outputs, num_outputs, & map_inputs, map_outputs, num_maps, & + atomics, num_atomics, & random_state, name, num_particles, & tol, max_iter) & BIND(C, NAME='graph_add_converge_item') @@ -834,6 +847,8 @@ SUBROUTINE graph_add_converge_item(c, inputs, num_inputs, & INTEGER(C_INTPTR_T), VALUE :: map_inputs INTEGER(C_INTPTR_T), VALUE :: map_outputs INTEGER(C_LONG), VALUE :: num_maps + INTEGER(C_INTPTR_T), VALUE :: atomics + INTEGER(C_LONG), VALUE :: num_atomics TYPE(C_PTR), VALUE :: random_state CHARACTER(kind=C_CHAR), DIMENSION(*) :: name INTEGER(C_LONG), VALUE :: num_particles @@ -2030,13 +2045,15 @@ SUBROUTINE graph_context_set_device_number(this, num) !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, num_particles) + atomics, random_state, name, & + num_particles) IMPLICIT NONE @@ -2046,6 +2063,7 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2056,6 +2074,8 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & LOC(outputs), INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles) END SUBROUTINE @@ -2068,13 +2088,15 @@ SUBROUTINE graph_context_add_pre_item(this, inputs, outputs, & !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, num_particles) + atomics, random_state, name, & + num_particles) IMPLICIT NONE @@ -2084,6 +2106,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2094,6 +2117,8 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & LOC(outputs), INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles) END SUBROUTINE @@ -2106,6 +2131,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & !> @param[in] outputs Array of output nodes. !> @param[in] map_inputs Array of map input nodes. !> @param[in] map_outputs Array of map output nodes. +!> @param[in] atomics Array of atomic nodes. !> @param[in] random_state Optional random state, can be NULL if not used. !> @param[in] name Name for the kernel. !> @param[in] num_particles Number of elements to operate on. @@ -2114,7 +2140,7 @@ SUBROUTINE graph_context_add_item(this, inputs, outputs, & !------------------------------------------------------------------------------- SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & map_inputs, map_outputs, & - random_state, name, & + atomics, random_state, name, & num_particles, tol, max_iter) IMPLICIT NONE @@ -2125,6 +2151,7 @@ SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: outputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_inputs INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: map_outputs + INTEGER(C_INTPTR_T), DIMENSION(:), INTENT(IN) :: atomics TYPE(C_PTR), INTENT(IN) :: random_state CHARACTER(kind=C_CHAR,len=*), INTENT(IN) :: name INTEGER(C_LONG), INTENT(IN) :: num_particles @@ -2138,6 +2165,8 @@ SUBROUTINE graph_context_add_converge_item(this, inputs, outputs, & INT(SIZE(outputs), KIND=C_LONG), & LOC(map_inputs), LOC(map_outputs), & INT(SIZE(map_inputs), KIND=C_LONG), & + LOC(atomics), & + INT(SIZE(atomics), KIND=C_LONG), & random_state, name, num_particles, & tol, max_iter) diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 98b504a..49fad41 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -648,18 +648,30 @@ if (size() > x.size()) { \ buffer if_(const buffer &t, const buffer &f) { if (size() == 1) { - return (*this)[0] ? t : f; + if constexpr (std::floating_point) { + return (*this)[0] ? t : f; + } else { + return (*this)[0] != static_cast (0) ? t : f; + } } else { if (t.size() == 1) { if (f.size() == 1) { for (T &d : *this) { - d = d ? t[0] : f[0]; + if constexpr (std::floating_point) { + d = d ? t[0] : f[0]; + } else { + d = d != static_cast (0) ? t[0] : f[0]; + } } return *this; } else { assert(size() == f.size() && "Incompatable buffersize."); for (size_t i = 0, ie = size(); i < ie; i++) { - (*this)[i] = (*this)[i] ? t[0] : f[i]; + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[0] : f[i]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[0] : f[i]; + } } return *this; } @@ -667,13 +679,21 @@ if (size() > x.size()) { \ assert(size() == t.size() && "Incompatable buffersize."); if (f.size() == 1) { for (size_t i = 0, ie = size(); i < ie; i++) { - (*this)[i] = (*this)[i] ? t[i] : f[0]; + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[i] : f[0]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[i] : f[0]; + } } return *this; } else { assert(size() == f.size() && "Incompatable buffersize."); for (size_t i = 0, ie = size(); i < ie; i++) { - (*this)[i] = (*this)[i] ? t[i] : f[i]; + if constexpr (std::floating_point) { + (*this)[i] = (*this)[i] ? t[i] : f[i]; + } else { + (*this)[i] = (*this)[i] != static_cast (0) ? t[i] : f[i]; + } } return *this; } diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index 953c968..7b20415 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -199,21 +199,25 @@ void run_tests(const enum graph_type type, NULL, 0, &rand, 1, NULL, NULL, 0, + NULL, 0, state, "c_binding_pre_kernel", 1); graph_add_item(c_context, inputs, 1, outputs, 5, map_inputs, map_outputs, 0, + NULL, 0, NULL, "c_binding", 1); graph_add_item(c_context, inputs2, 4, outputs2, 4, map_inputs2, map_outputs2, 0, + NULL, 0, NULL, "c_binding_piecewise", 1); graph_add_converge_item(c_context, &z, 1, &root2, 1, &z, &dz, 1, + NULL, 0, NULL, "c_binding_converge", 1, 1.0E-30, 1000); graph_compile(c_context); diff --git a/graph_tests/f_binding_test.f90 b/graph_tests/f_binding_test.f90 index 899e0b6..2aa0f0b 100644 --- a/graph_tests/f_binding_test.f90 +++ b/graph_tests/f_binding_test.f90 @@ -180,7 +180,8 @@ SUBROUTINE run_test_float(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -189,17 +190,17 @@ SUBROUTINE run_test_float(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -373,7 +374,8 @@ SUBROUTINE run_test_double(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -382,17 +384,17 @@ SUBROUTINE run_test_double(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -570,7 +572,8 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'c_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -579,17 +582,17 @@ SUBROUTINE run_test_complex_float(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile @@ -788,7 +791,8 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL graph%set_device_number(graph%get_max_concurrency() - 1) CALL graph%add_pre_item(graph_null_array, (/ graph_ptr(rand) /), & - graph_null_array, graph_null_array, state, & + graph_null_array, graph_null_array, & + graph_null_array, state, & 'f_binding_pre_kernel' // C_NULL_CHAR, & 1_C_LONG) CALL graph%add_item((/ graph_ptr(x) /), (/ & @@ -797,17 +801,17 @@ SUBROUTINE run_test_complex_double(use_safe_math) graph_ptr(dydm), & graph_ptr(dydb), & graph_ptr(dydy) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_item((/ & graph_ptr(i), graph_ptr(j), graph_ptr(variable), graph_ptr(variable2) & /), (/ & graph_ptr(p1), graph_ptr(p2), graph_ptr(i1), graph_ptr(i2) & - /), graph_null_array, graph_null_array, C_NULL_PTR, & + /), graph_null_array, graph_null_array, graph_null_array, C_NULL_PTR, & 'f_binding_piecewise' // C_NULL_CHAR, 1_C_LONG) CALL graph%add_converge_item((/ graph_ptr(z) /), (/ graph_ptr(root2) /), & (/ graph_ptr(z) /), (/ graph_ptr(dz) /), & - C_NULL_PTR, & + graph_null_array, C_NULL_PTR, & 'f_binding_converge' // C_NULL_CHAR, & 1_C_LONG, 1.0E-30_C_DOUBLE, 1000_C_LONG) CALL graph%compile From bfeb5a235126ca397dfca7b5554e38cb0444f392 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 11 Aug 2026 15:01:23 -0400 Subject: [PATCH 38/51] Fix some issues where the random state array could be larger than the total problem size. --- graph_c_binding/graph_c_binding.cpp | 162 +++++++++--------- graph_c_binding/graph_c_binding.h | 2 + graph_docs/code_performance.dox | 2 +- graph_docs/kernel_optimization.dox | 2 +- .../graph_fortran_binding.f90 | 11 +- graph_framework/cuda_context.hpp | 2 +- graph_framework/jit.hpp | 10 ++ graph_framework/metal_context.hpp | 4 +- graph_framework/piecewise.hpp | 26 +++ graph_pic/xpic.cpp | 2 +- graph_tests/c_binding_test.c | 2 +- graph_tests/f_binding_test.f90 | 8 +- graph_tests/random_test.cpp | 6 +- 13 files changed, 142 insertions(+), 97 deletions(-) diff --git a/graph_c_binding/graph_c_binding.cpp b/graph_c_binding/graph_c_binding.cpp index ffcab88..826ded2 100644 --- a/graph_c_binding/graph_c_binding.cpp +++ b/graph_c_binding/graph_c_binding.cpp @@ -1259,22 +1259,24 @@ extern "C" { /// @brief Construct a random state node. /// /// @param[in] c The graph C context. +/// @param[in] size The number of randoms needed. /// @param[in] seed Intial random seed. /// @returns A random state node. //------------------------------------------------------------------------------ graph_node graph_random_state(STRUCT_TAG graph_c_context *c, + const size_t size, const uint32_t seed) { switch (c->type) { case FLOAT: if (c->safe_math) { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); @@ -1283,13 +1285,13 @@ extern "C" { case DOUBLE: if (c->safe_math) { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast *> (c); - auto temp = graph::random_state (jit::context::random_state_size, + auto temp = graph::random_state (jit::context::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); @@ -1298,13 +1300,13 @@ extern "C" { case COMPLEX_FLOAT: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - auto temp = graph::random_state, true> (jit::context, true>::random_state_size, + auto temp = graph::random_state, true> (jit::context, true>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast> *> (c); - auto temp = graph::random_state> (jit::context>::random_state_size, seed); + auto temp = graph::random_state> (jit::context>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } @@ -1312,13 +1314,13 @@ extern "C" { case COMPLEX_DOUBLE: if (c->safe_math) { auto d = reinterpret_cast, true> *> (c); - auto temp = graph::random_state, true> (jit::context, true>::random_state_size, + auto temp = graph::random_state, true> (jit::context, true>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } else { auto d = reinterpret_cast> *> (c); - auto temp = graph::random_state> (jit::context>::random_state_size, seed); + auto temp = graph::random_state> (jit::context>::max_random_state_size(size), seed); d->nodes[temp.get()] = temp; return temp.get(); } @@ -2735,7 +2737,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -2782,7 +2784,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -2833,7 +2835,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -2880,7 +2882,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -2931,7 +2933,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -2978,7 +2980,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3029,7 +3031,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3076,7 +3078,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3156,7 +3158,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3203,7 +3205,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3254,7 +3256,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3301,7 +3303,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3352,7 +3354,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3399,7 +3401,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3450,7 +3452,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3497,7 +3499,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3577,7 +3579,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3624,7 +3626,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3675,7 +3677,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3722,7 +3724,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3773,7 +3775,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3820,7 +3822,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3871,7 +3873,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -3918,7 +3920,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4000,7 +4002,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4047,7 +4049,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4098,7 +4100,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4145,7 +4147,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4196,7 +4198,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4243,7 +4245,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4294,7 +4296,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4341,7 +4343,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4423,7 +4425,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4470,7 +4472,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4521,7 +4523,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4568,7 +4570,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4619,7 +4621,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4666,7 +4668,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4717,7 +4719,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4764,7 +4766,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4846,7 +4848,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4893,7 +4895,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4944,7 +4946,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -4991,7 +4993,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5042,7 +5044,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5089,7 +5091,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5140,7 +5142,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5187,7 +5189,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5271,7 +5273,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5320,7 +5322,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5373,7 +5375,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5422,7 +5424,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5475,7 +5477,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5524,7 +5526,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5577,7 +5579,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5626,7 +5628,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Preitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5712,7 +5714,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5761,7 +5763,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5814,7 +5816,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5863,7 +5865,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5916,7 +5918,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -5965,7 +5967,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6018,7 +6020,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6067,7 +6069,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Work atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6153,7 +6155,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6202,7 +6204,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6255,7 +6257,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6304,7 +6306,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6357,7 +6359,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6406,7 +6408,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6459,7 +6461,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); @@ -6508,7 +6510,7 @@ extern "C" { for (size_t i = 0; i < num_atomics; i++) { auto temp = graph::variable_cast(d->nodes[atomics[i]]); if (temp.get()) { - in.push_back(temp); + atom.push_back(temp); } else { std::cerr << "Postitem atomic " << i << " is not a variable." << std::endl; exit(1); diff --git a/graph_c_binding/graph_c_binding.h b/graph_c_binding/graph_c_binding.h index 3c56586..3024076 100644 --- a/graph_c_binding/graph_c_binding.h +++ b/graph_c_binding/graph_c_binding.h @@ -385,10 +385,12 @@ extern "C" { /// @brief Construct a random state node. /// /// @param[in] c The graph C context. +/// @param[in] size The number of randoms needed. /// @param[in] seed Initial random seed. /// @returns A random state node. //------------------------------------------------------------------------------ graph_node graph_random_state(STRUCT_TAG graph_c_context *c, + const size_t size, const uint32_t seed); //------------------------------------------------------------------------------ diff --git a/graph_docs/code_performance.dox b/graph_docs/code_performance.dox index 0fc897e..d70d500 100644 --- a/graph_docs/code_performance.dox +++ b/graph_docs/code_performance.dox @@ -112,7 +112,7 @@ for (size_t i = 0, ie = threads.size(); i < ie; i++) { {v_next->get_x(), graph::variable_cast(vx)}, {v_next->get_y(), graph::variable_cast(vy)}, {v_next->get_z(), graph::variable_cast(vz)} - }, NULL, "Lorentz_kernel", local_size); + }, {}, NULL, "Lorentz_kernel", local_size); work.compile(); time_steps.start_time(thread_number); diff --git a/graph_docs/kernel_optimization.dox b/graph_docs/kernel_optimization.dox index 6158df3..ac9641e 100644 --- a/graph_docs/kernel_optimization.dox +++ b/graph_docs/kernel_optimization.dox @@ -57,7 +57,7 @@ void field_solve_example() { + graph::exp(static_cast (-1)*arg*arg/static_cast (10)); } - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(num_particles), 0); auto random = graph::random (graph::random_state_cast(state)); const T max = 1.0; const T min = -1.0; diff --git a/graph_fortran_binding/graph_fortran_binding.f90 b/graph_fortran_binding/graph_fortran_binding.f90 index e677071..184ee63 100644 --- a/graph_fortran_binding/graph_fortran_binding.f90 +++ b/graph_fortran_binding/graph_fortran_binding.f90 @@ -568,14 +568,16 @@ TYPE(C_PTR) FUNCTION graph_atan(c, left, right) & !> @brief Construct a random state node. !> !> @param[in] c The graph C context. +!> @param[in] size Number of randoms needed. !> @param[in] seed Initial random seed. !> @returns A random state node. !------------------------------------------------------------------------------- - TYPE(C_PTR) FUNCTION graph_random_state(c, seed) & + TYPE(C_PTR) FUNCTION graph_random_state(c, size, seed) & BIND(C, NAME='graph_random_state') USE, INTRINSIC :: ISO_C_BINDING IMPLICIT NONE TYPE(C_PTR), VALUE :: c + INTEGER(C_LONG), value :: size INTEGER(C_INT32_T), VALUE :: seed END FUNCTION @@ -1603,20 +1605,23 @@ FUNCTION graph_context_atan(this, left, right) !> @brief Get random size. !> !> @param[in,out] this @ref graph_context instance. +!> @param[in] size Number of random numbers needed. !> @param[in] seed Initial random seed. !> @returns The random size. !------------------------------------------------------------------------------- - FUNCTION graph_context_random_state(this, seed) + FUNCTION graph_context_random_state(this, size, seed) IMPLICIT NONE ! Declare Arguments TYPE(C_PTR) :: graph_context_random_state CLASS(graph_context), INTENT(INOUT) :: this + INTEGER(C_LONG), INTENT(IN) :: size INTEGER(C_INT32_T), INTENT(IN) :: seed ! Start of executable. - graph_context_random_state = graph_random_state(this%c_context, seed) + graph_context_random_state = graph_random_state(this%c_context, & + size, seed) END FUNCTION diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 1d8f9e3..cb7b19e 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -531,7 +531,7 @@ namespace gpu { CU_DEVICE_ATTRIBUTE_WARP_SIZE, device), "cuDeviceGetAttribute"); - unsigned int total_parallel = state.get() ? random_state_size : num_rays; + unsigned int total_parallel = state.get() ? state->size() : num_rays; unsigned int threads_per_group = total_parallel < 1024 ? warp_size : value; unsigned int thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index f6ee4a1..cbdde38 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -79,6 +79,16 @@ namespace jit { /// Size of random state needed. constexpr static size_t random_state_size = gpu_context_type::random_state_size; +//------------------------------------------------------------------------------ +/// @brief Get the number of random states needed. +/// +/// @param[in] size Number of random numbers needed. +/// @returns The maximum number of random states needed. +//------------------------------------------------------------------------------ + static size_t max_random_state_size(const size_t size) { + return std::min(size, random_state_size); + } + //------------------------------------------------------------------------------ /// @brief Get the maximum number of concurrent instances. /// diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 7d8b560..fed467c 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -40,7 +40,7 @@ namespace gpu { std::map> bufferMutability; public: -/// Random state size multiplyer. +/// Random state size multiplier. constexpr static size_t random_state_scale = 1000; /// Size of random state needed. constexpr static size_t random_state_size = 1024*random_state_scale; @@ -250,7 +250,7 @@ namespace gpu { NSRange range = NSMakeRange(0, buffers.size()); NSRange tex_range = NSMakeRange(0, textures.size()); - NSUInteger total_parallel = state.get() ? random_state_size : num_rays; + NSUInteger total_parallel = state.get() ? state->size() : num_rays; NSUInteger thread_width = pipeline.threadExecutionWidth; NSUInteger threads_per_group = total_parallel < pipeline.maxTotalThreadsPerThreadgroup ? thread_width : diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index 85d4c05..02e324e 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -727,6 +727,19 @@ namespace graph { return true; } +//------------------------------------------------------------------------------ +/// @brief Test if node acts like a variable. +/// +/// @note Even though @ref graph::leaf_node define a default of false. The +/// @ref graph::straight_node subclass overrides it so we need to +/// explicitly define these nodes as constants. +/// +/// @returns True if the node acts like a variable. +//------------------------------------------------------------------------------ + virtual bool is_all_variables() const { + return false; + } + //------------------------------------------------------------------------------ /// @brief Test the constant node has a zero. /// @@ -1333,6 +1346,19 @@ namespace graph { return true; } +//------------------------------------------------------------------------------ +/// @brief Test if node acts like a variable. +/// +/// @note Even though @ref graph::leaf_node define a default of false. The +/// @ref graph::branch_node subclass overrides it so we need to +/// explicitly define these nodes as constants. +/// +/// @returns True if the node acts like a variable. +//------------------------------------------------------------------------------ + virtual bool is_all_variables() const { + return false; + } + //------------------------------------------------------------------------------ /// @brief Test the constant node has a zero. /// diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 6a0b36c..c2b44f3 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -63,7 +63,7 @@ void run_pic() { pic::mesh mesh(lmin, lmax, num_grid, norms); - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(num_particles), 0); workflow::manager work(0); diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index 7b20415..433ae52 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -98,7 +98,7 @@ void run_tests(const enum graph_type type, } } - graph_node state = graph_random_state(c_context, 0); + graph_node state = graph_random_state(c_context, 1, 0); graph_node rand = graph_random(c_context, state); const size_t max_device = graph_get_max_concurrency(c_context) - 1; diff --git a/graph_tests/f_binding_test.f90 b/graph_tests/f_binding_test.f90 index 2aa0f0b..3ceb8a5 100644 --- a/graph_tests/f_binding_test.f90 +++ b/graph_tests/f_binding_test.f90 @@ -143,7 +143,7 @@ SUBROUTINE run_test_float(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -337,7 +337,7 @@ SUBROUTINE run_test_double(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -533,7 +533,7 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) @@ -740,7 +740,7 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL assert(graph_ptr(graph%atan(one, zero)) .eq. graph_ptr(zero), & 'Expected atan(one, zero) = zero.') - state = graph%random_state(0) + state = graph%random_state(1_C_LONG, 0) rand = graph%random(state) i = graph%variable(1_C_LONG, 'i' // C_NULL_CHAR) diff --git a/graph_tests/random_test.cpp b/graph_tests/random_test.cpp index fa5228c..ee7398f 100644 --- a/graph_tests/random_test.cpp +++ b/graph_tests/random_test.cpp @@ -42,7 +42,7 @@ T autocorrelation(const std::vector &sequence, /// @tparam N Number of random numbers to use. //------------------------------------------------------------------------------ template void test_dist() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(N), 0); auto random = graph::random (graph::random_state_cast(state)); const T max = 1.0; const T min = -1.0; @@ -69,7 +69,7 @@ template void test_dist() { /// @brief Test graph properties of random numbers. //------------------------------------------------------------------------------ template void test_graph() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(1), 0); auto random = graph::random (graph::random_state_cast(state)); // r + r -> r + r @@ -141,7 +141,7 @@ template void test_graph() { /// @brief Test multiple randoms in a single kernel. //------------------------------------------------------------------------------ template void test_multi() { - auto state = graph::random_state (jit::context::random_state_size, 0); + auto state = graph::random_state (jit::context::max_random_state_size(1), 0); auto random1 = graph::random (graph::random_state_cast(state)); auto random2 = graph::random (graph::random_state_cast(state)); From fc10826af3c4cef0452dc98a827810252c97243d Mon Sep 17 00:00:00 2001 From: m4c Date: Wed, 12 Aug 2026 16:05:51 -0400 Subject: [PATCH 39/51] Fix JIT compile issues on Linux systems. Needed to include the atomic header. --- graph_framework/CMakeLists.txt | 2 +- graph_framework/cpu_context.hpp | 8 +++++--- graph_tests/arithmetic_test.cpp | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/graph_framework/CMakeLists.txt b/graph_framework/CMakeLists.txt index 3d8767c..abd15ff 100644 --- a/graph_framework/CMakeLists.txt +++ b/graph_framework/CMakeLists.txt @@ -17,7 +17,7 @@ execute_process (COMMAND ${Python_EXECUTABLE} get_includes.py --compiler=${CMAKE target_compile_definitions (graph_framework INTERFACE $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -fgnuc-version=4.2.1 -std=gnu++2a"> - $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -std=gnu++2a -fno-use-cxa-atexit"> + $<$:CXX_ARGS="-I${CMAKE_CURRENT_SOURCE_DIR}${jit_include_paths} -std=gnu++2a -fno-use-cxa-atexit -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -D__GCC_ATOMIC_POINTER_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR_LOCK_FREE=2 -D__GCC_ATOMIC_SHORT_LOCK_FREE=2 -D__GCC_ATOMIC_INT_LOCK_FREE=2 -D__GCC_ATOMIC_LONG_LOCK_FREE=2 -D__GCC_ATOMIC_LLONG_LOCK_FREE -D__GCC_ATOMIC_CHAR8_T_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR16_T_LOCK_FREE=2 -D__GCC_ATOMIC_CHAR32_T_LOCK_FREE=2 -D__GCC_ATOMIC_BOOL_LOCK_FREE=2 -D__GCC_ATOMIC_WCHAR_T_LOCK_FREE=2"> EFIT_FILE="${CMAKE_CURRENT_SOURCE_DIR}/../graph_tests/efit.nc" VMEC_FILE="${CMAKE_CURRENT_SOURCE_DIR}/../graph_tests/vmec.nc" $<$:HEADER_DIR="$"> diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 53a33cc..94bef50 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -38,6 +38,7 @@ #include "random.hpp" #include "piecewise.hpp" +#include "timing.hpp" #ifndef NDEBUG //------------------------------------------------------------------------------ @@ -362,7 +363,7 @@ namespace gpu { std::function create_zero_call(graph::input_nodes &inputs) { std::vector buffers; std::vector sizes; - + for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { std::vector arg(input->size()); @@ -396,7 +397,7 @@ namespace gpu { std::vector sources; std::vector destinations; std::vector sizes; - + for (auto &[out, in] : setters) { if (!kernel_arguments.contains(in.get())) { std::vector arg(in->size()); @@ -531,7 +532,8 @@ namespace gpu { } else { source_buffer << "#include " << std::endl; } - source_buffer << "using namespace std;" << std::endl; + source_buffer << "#include " << std::endl + << "using namespace std;" << std::endl; } //------------------------------------------------------------------------------ diff --git a/graph_tests/arithmetic_test.cpp b/graph_tests/arithmetic_test.cpp index f519894..98e9c6e 100644 --- a/graph_tests/arithmetic_test.cpp +++ b/graph_tests/arithmetic_test.cpp @@ -97,7 +97,7 @@ template void test_add() { "Expected to reduce to a constant one."); assert(done_plus_var->evaluate()[0] == static_cast (1.0) && "Expected value of one."); - + // Test common factors. auto var_a = graph::variable (1, ""); auto var_b = graph::variable (1, ""); @@ -188,7 +188,7 @@ template void test_add() { auto constant_factor = three*variable + (one + one)*var_b; assert(graph::multiply_cast(constant_factor).get() && "Expected multiply node."); - + // Test is_match auto match1 = graph::one () + variable; auto match2 = graph::one () + variable; @@ -231,7 +231,7 @@ template void test_add() { "Expected var_c in the second slot."); assert(graph::add_cast(add_fma_cast->get_right()) && "Expected add_node in the third slot."); - + // (a/(b*c) + d/(e*c)) -> (a/b + d/e)/c auto multiply_divide_factor = var_a/(var_b*var_c) + var_d/(var_e*var_c); auto multiply_divide_factor_cast = divide_cast(multiply_divide_factor); From fa37129d56d381918bb22c431cfb7432b4eb0df0 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 26 Aug 2026 18:01:01 -0400 Subject: [PATCH 40/51] Fix issue with Metal random kernels where the offset was being iterated wrong. This fixes the performance regression experianced when using atomic types since there was a 1024000 population all getting assigned to the same bin. Fix issues with NaNs and Infs by checking for them in the particle reinjection kernel. --- CMakeLists.txt | 1 + graph_docs/code_performance.dox | 2 +- graph_framework/backend.hpp | 33 ++- graph_framework/logical.hpp | 412 +++++++++++++++++++++++++-- graph_framework/metal_context.hpp | 4 - graph_framework/node.hpp | 4 +- graph_framework/particle_in_cell.hpp | 23 +- graph_framework/piecewise.hpp | 13 +- graph_pic/xpic.cpp | 6 +- graph_tests/logical_test.cpp | 95 +++++- 10 files changed, 533 insertions(+), 60 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f924a4..3bcbe28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,6 +208,7 @@ FetchContent_GetProperties ( ) # Do not build llvm until pull command is finished. +add_dependencies (CASPluginTest_exports pull_llvm) add_dependencies (gpu-resource-headers pull_llvm) add_dependencies (llvm-offload-resource-headers pull_llvm) add_dependencies (LLVMDemangle pull_llvm) diff --git a/graph_docs/code_performance.dox b/graph_docs/code_performance.dox index d70d500..391c45e 100644 --- a/graph_docs/code_performance.dox +++ b/graph_docs/code_performance.dox @@ -25,7 +25,7 @@ * * The figure above shows the advantage even a single GPU has over CPU * execution. In single precision, the M2's GPU is almost @f$100\times@f$ faster - * a single CPU core while the a single A100 has a nearly $800\times$ advantage. + * a single CPU core while the a single A100 has a nearly @f$800\times@f$ advantage. * An interesting thing to note is the M2 Max CPU show no advantage between * single and double precision execution. * diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 49fad41..e926874 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -632,9 +632,14 @@ if (size() > x.size()) { \ /// /// @returns The negation of the buffer. //------------------------------------------------------------------------------ - buffer operator!() requires(std::floating_point) { + buffer operator!() { for (T &d : *this) { - d = !d; + if constexpr (jit::complex_scalar) { + assert(d.imag() == 0.0 && "Imaginary part not zero."); + d = static_cast (!d.real()); + } else { + d = !d; + } } return *this; } @@ -701,6 +706,30 @@ if (size() > x.size()) { \ } } +//------------------------------------------------------------------------------ +/// @brief Applies a logical is operator. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define logic_is(op) \ +for (T &d : *this) { \ + d = std::op(d); \ +} + +//------------------------------------------------------------------------------ +/// @brief isinf query. +//------------------------------------------------------------------------------ + void isinf() { + logic_is(isinf) + } + +//------------------------------------------------------------------------------ +/// @brief isnan query. +//------------------------------------------------------------------------------ + void isnan() { + logic_is(isnan) + } + /// Type def to retrieve the backend T type. typedef T base; }; diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index 41fd531..90b717c 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -23,6 +23,344 @@ namespace graph { return zero (); } +//****************************************************************************** +// IsInf node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief isinf node. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class isinf_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "isinf" + + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an isinf node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + isinf_node(shared_leaf arg) : + no_derivative> (arg, + isinf_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of isinf. +/// +/// result = isinf(a) +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + arg.isinf(); + return arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an isinf node. +/// +/// @returns A reduced isinf node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = isinf(" + << registers[arg.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "isinf\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build not node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf isinf(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared isinf nodes. + template + using shared_isinf = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a isinf node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_isinf isinf_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// IsNaN node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief isnan node. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class isnan_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] arg Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *arg) { + return "isnan" + + jit::format_to_string(reinterpret_cast (arg)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an isnan node. +/// +/// @param[in] arg Node argument. +//------------------------------------------------------------------------------ + isnan_node(shared_leaf arg) : + no_derivative> (arg, + isnan_node::to_string(arg.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of isnan. +/// +/// result = isnan(a) +/// +/// @returns The value of !a. +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer arg = this->arg->evaluate(); + arg.isnan(); + return arg; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce an isnan node. +/// +/// @returns A reduced isnan node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto arg = constant_cast(this->arg); + + if (arg.get()) { + return constant (this->evaluate()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto arg = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('l', this); + stream << " const bool "; + stream << registers[this] << " = isnan(" + << registers[arg.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "isnan\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"!\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto arg = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[arg.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build isnan node from the argument leaves. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] arg Arguement +//------------------------------------------------------------------------------ + template + shared_leaf isnan(shared_leaf arg) { + auto temp = std::make_shared> (arg)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared isnan nodes. + template + using shared_isnan = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an isnan node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_isnan isnan_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //****************************************************************************** // Not node. //****************************************************************************** @@ -34,7 +372,7 @@ namespace graph { /// @tparam T Base type of the operands. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. //------------------------------------------------------------------------------ - template + template class not_node final : public no_derivative> { private: //------------------------------------------------------------------------------ @@ -59,7 +397,7 @@ namespace graph { not_node::to_string(arg.get())) {} //------------------------------------------------------------------------------ -/// @brief Evaluate the results of equal. +/// @brief Evaluate the results of not. /// /// result = !a /// @@ -71,7 +409,7 @@ namespace graph { } //------------------------------------------------------------------------------ -/// @brief Reduce an equal node. +/// @brief Reduce a not node. /// /// @returns A reduced equal node. //------------------------------------------------------------------------------ @@ -83,24 +421,48 @@ namespace graph { return constant (this->evaluate()); } +// !(a == b) -> a != b auto equalc = equal_cast(this->arg); if (equalc.get()) { return equalc->get_left() != equalc->get_right(); } +// !(a != b) -> a == b auto nequalc = not_equal_cast(this->arg); if (nequalc.get()) { return nequalc->get_left() == nequalc->get_right(); } - auto ltc = less_than_cast(this->arg); - if (ltc.get()) { - return ltc->get_left() >= ltc->get_right(); + if constexpr (!jit::complex_scalar) { +// !(a < b) -> a >= b + auto ltc = less_than_cast(this->arg); + if (ltc.get()) { + return ltc->get_left() >= ltc->get_right(); + } + +// !(a <= b) -> a > b + auto lec = less_than_equal_cast(this->arg); + if (lec.get()) { + return lec->get_left() > lec->get_right(); + } + +// !(a > b) -> a <= b + auto gtc = greater_than_cast(this->arg); + if (gtc.get()) { + return gtc->get_left() <= gtc->get_right(); + } + +// !(a >= b) -> a < b + auto gec = greater_than_equal_cast(this->arg); + if (gec.get()) { + return gec->get_left() < gec->get_right(); + } } - auto gtc = greater_than_cast(this->arg); - if (gtc.get()) { - return gtc->get_left() <= gtc->get_right(); +// !!a -> a + auto n = not_cast(this->arg); + if (n.get()) { + return n->get_arg(); } return this->shared_from_this(); @@ -124,9 +486,15 @@ namespace graph { thread_mem, usage); registers[this] = jit::to_string('l', this); - stream << " const bool "; - stream << registers[this] << " = !" - << registers[arg.get()]; + stream << " const bool " + << registers[this] << " = !"; + if constexpr (jit::complex_scalar) { + stream << "real("; + } + stream << registers[arg.get()]; + if constexpr (jit::complex_scalar) { + stream << ")"; + } this->endline(stream, usage); } @@ -180,7 +548,7 @@ namespace graph { /// /// @param[in] arg Arguement //------------------------------------------------------------------------------ - template + template shared_leaf not_(shared_leaf arg) { auto temp = std::make_shared> (arg)->reduce(); // Test for hash collisions. @@ -202,7 +570,7 @@ namespace graph { } //------------------------------------------------------------------------------ -/// @brief Build equal node from two leaves. +/// @brief Build not node from two leaves. /// /// Note use templates here to defer this so it can be used in the above /// classes. @@ -212,17 +580,17 @@ namespace graph { /// /// @param[in] arg Arguement //------------------------------------------------------------------------------ - template + template shared_leaf operator!(shared_leaf arg) { return not_ (arg); } -/// Convenience type alias for shared equal nodes. - template +/// Convenience type alias for shared not nodes. + template using shared_not = std::shared_ptr>; //------------------------------------------------------------------------------ -/// @brief Cast to a equal node. +/// @brief Cast to a not node. /// /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. @@ -230,7 +598,7 @@ namespace graph { /// @param[in] x Leaf node to attempt cast. /// @returns An attempted dynamic cast. //------------------------------------------------------------------------------ - template + template shared_not not_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } @@ -2458,6 +2826,12 @@ namespace graph { return this->middle; } +// If(!a, b, c) -> If(a, c, b) + auto n = not_cast(this->left); + if (n.get()) { + return if_(n->get_arg(), this->right, this->middle); + } + return this->shared_from_this(); } diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index fed467c..2571b02 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -277,10 +277,6 @@ namespace gpu { for (NSUInteger i = 0, ie = thread_groups*threads_per_group; i < num_rays; i += ie) { id encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial]; - for (size_t j = 0, je = buffers.size() - 1; j < je; j++) { - offsets[j] = i*sizeof(float); - } - [encoder setComputePipelineState:pipeline]; [encoder setBuffers:buffers.data() offsets:offsets.data() diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index fe6a570..2d7ea1a 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -915,7 +915,9 @@ namespace graph { //------------------------------------------------------------------------------ constant_node(const backend::buffer &d) : leaf_node (constant_node::to_string(d.at(0)), 1, false), data(d) { - assert(d.is_normal() && "Denormal encountered"); + if constexpr (SAFE_MATH) { + assert(d.is_normal() && "Denormal encountered"); + } assert(d.size() == 1 && "Constants need to be scalar functions."); } diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 3afae5e..ae5887f 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -180,11 +180,8 @@ namespace pic { graph::shared_leaf v_perp; /// Mesh Weights std::array, 3> weights; -/// Mesh index - graph::shared_leaf indices; /// Number of real particles const T num_real; -/// //------------------------------------------------------------------------------ /// @brief Construct an ion object. @@ -209,7 +206,7 @@ namespace pic { graph::variable (num_ions, "w_{0}"), graph::variable (num_ions, "w_{1}"), graph::variable (num_ions, "w_{2}") - }), indices(graph::variable (num_ions, "m_{i}")) {} + }) {} //------------------------------------------------------------------------------ /// @brief Get x case as variable. @@ -346,8 +343,6 @@ namespace pic { const T xmax; /// Dx const T dx; -/// Particle index. - graph::shared_leaf index; /// Mesh y values. std::array, 4> y; /// Mesh point. @@ -377,9 +372,7 @@ namespace pic { graph::variable (num, "y^{1}_{m}"), graph::variable (num, "y^{2}_{m}"), graph::variable (num, "y^{3}_{m}") - }), - index(graph::variable (num, "pi_{m}")), - xmin(x_min/norms.l), xmax(x_max/norms.l), + }), xmin(x_min/norms.l), xmax(x_max/norms.l), dx((xmax - xmin)/(num - 1)) {} //------------------------------------------------------------------------------ @@ -626,9 +619,15 @@ namespace pic { auto resampled = build_initialization(ion, mesh, norms, params, state); auto is_outside = ion.x <= mesh.xmin || ion.x >= mesh.xmax; - auto reinject_x = graph::if_(is_outside, resampled[0], ion.x); - auto reinject_vpara = graph::if_(is_outside, resampled[1], ion.v_para); - auto reinject_vperp = graph::if_(is_outside, resampled[2], ion.v_perp); + auto reinject_x = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[0], ion.x); + auto reinject_vpara = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[1], ion.v_para); + auto reinject_vperp = graph::if_(is_outside || + graph::isnan(ion.x) || + graph::isinf(ion.x), resampled[2], ion.v_perp); return {reinject_x, reinject_vpara, reinject_vperp}; } diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index 02e324e..ea99b6b 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -2231,23 +2231,22 @@ namespace graph { thread_mem, usage); registers[this] = jit::to_string('v', a.get()) - + "[" - + registers[index.get()] - + "]"; + + " + " + + registers[index.get()]; stream << " atomic"; if constexpr (jit::use_cuda()) { - stream << "Add(&"; + stream << "Add("; } else if constexpr (jit::use_metal ()){ - stream << "_fetch_add_explicit(&"; + stream << "_fetch_add_explicit("; } else { - stream << "_ref("; + stream << "_ref(*("; } stream << registers[this]; if constexpr (jit::use_cuda() || jit::use_metal ()) { stream << ", "; } else { - stream << ").fetch_add("; + stream << ")).fetch_add("; } stream << registers[r.get()]; if constexpr (jit::use_cuda()) { diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index c2b44f3..ffded51 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -175,14 +175,14 @@ void run_pic() { mesh_sync.lock(); mesh_sync.unlock(); }); - work.add_zero_item({ - graph::variable_cast(mesh.index) - }); work.add_copy_item({ {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])} }); + work.add_zero_item({ + graph::variable_cast(mesh.y[0]) + }); } work.add_item({ diff --git a/graph_tests/logical_test.cpp b/graph_tests/logical_test.cpp index 4b4e465..977490a 100644 --- a/graph_tests/logical_test.cpp +++ b/graph_tests/logical_test.cpp @@ -9,11 +9,52 @@ #endif #include +#include #include "../graph_framework/graph_framework.hpp" //------------------------------------------------------------------------------ -/// @brief Tests for equal nodes. +/// @brief Tests for isinf nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_isinf() { + auto zero = graph::zero (); + auto one = graph::one (); + auto nan = graph::constant (NAN); + auto inf = graph::constant (INFINITY); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + assert(graph::isinf(zero)->is_match(false_v) && "Expected false."); + assert(graph::isinf(one)->is_match(false_v) && "Expected false."); + assert(graph::isinf(nan)->is_match(false_v) && "Expected false."); + assert(graph::isinf(inf)->is_match(true_v) && "Expected true."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for isnan nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_isnan() { + auto zero = graph::zero (); + auto one = graph::one (); + auto nan = graph::constant (NAN); + auto inf = graph::constant (INFINITY); + + auto true_v = graph::true_constant (); + auto false_v = graph::false_constant (); + + assert(graph::isnan(zero)->is_match(false_v) && "Expected false."); + assert(graph::isnan(one)->is_match(false_v) && "Expected false."); + assert(graph::isnan(nan)->is_match(true_v) && "Expected true."); + assert(graph::isnan(inf)->is_match(false_v) && "Expected false."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for not nodes. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ @@ -22,27 +63,48 @@ template void test_not() { auto false_v = graph::false_constant (); auto result1 = !true_v; - assert(result1->is_match(false_v) && "Expected flase."); + assert(result1->is_match(false_v) && "Expected false."); auto result2 = !false_v; assert(result2->is_match(true_v) && "Expected true."); +// !(a == b) -> a != b auto v1 = graph::variable (1, ""); auto v2 = graph::variable (1, ""); auto result3 = !(v1 == v2); auto result3_cast = graph::not_equal_cast(result3); assert(result3_cast.get() && "Expected a not equal node."); +// !(a != b) -> a == b auto result4 = !(v1 != v2); auto result4_cast = graph::equal_cast(result4); assert(result4_cast.get() && "Expected an equal node."); - - auto result5 = !(v1 < v2); - auto result5_cast = graph::greater_than_equal_cast(result5); - assert(result5_cast.get() && "Expected a greater than equal node."); - - auto result6 = !(v1 > v2); - auto result6_cast = graph::less_than_equal_cast(result6); - assert(result6_cast.get() && "Expected a less than equal node."); + + if constexpr (!jit::complex_scalar) { +// !(a < b) -> a >= b + auto result5 = !(v1 < v2); + auto result5_cast = graph::greater_than_equal_cast(result5); + assert(result5_cast.get() && "Expected a greater than equal node."); + +// !(a <= b) -> a > b + auto result6 = !(v1 <= v2); + auto result6_cast = graph::greater_than_cast(result6); + assert(result6_cast.get() && "Expected a greater than node."); + +// !(a > b) -> a <= b + auto result7 = !(v1 > v2); + auto result7_cast = graph::less_than_equal_cast(result7); + assert(result7_cast.get() && "Expected a less than equal node."); + +// !(a >= b) -> a < b + auto result8 = !(v1 >= v2); + auto result8_cast = graph::less_than_cast(result8); + assert(result8_cast.get() && "Expected a less than node."); + } + +// !!a -> a + auto result9 = !!v1; + auto result9_cast = graph::variable_cast(result9); + assert(result9_cast.get() && "Expected v1"); } //------------------------------------------------------------------------------ @@ -223,12 +285,21 @@ template void test_if() { auto result2 = graph::if_(false_v, true_v, false_v); assert(result2->is_match(false_v) && "Exected the false condition."); +// If(c, a, a) -> a auto v1 = graph::variable (1, ""); auto v2 = graph::variable (1, ""); auto result = graph::if_(v1, v2, v2); assert(result->is_match(v2)); auto result_df = result->df(v1); assert(result_df->is_match(false_v) && "Expected 0"); + +// If(!a, b, c) -> If(a, c, b) + auto test_not = graph::if_(graph::not_(v1), v1, v2); + auto test_not_cast = if_cast(test_not); + assert(test_not_cast.get() && "Expected if node."); + assert(test_not_cast->get_left()->is_match(v1) && "Expected v1"); + assert(test_not_cast->get_middle()->is_match(v2) && "Expected v2"); + assert(test_not_cast->get_right()->is_match(v1) && "Expected v1"); } //------------------------------------------------------------------------------ @@ -239,8 +310,10 @@ template void test_if() { template void run_tests() { test_equal (); test_not_equal (); + test_not (); if constexpr (std::floating_point) { - test_not (); + test_isinf (); + test_isnan (); test_greater_than (); test_less_than (); test_and (); From e30ee20cbe301a7e9e5834aa8fea3807afa40458 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 31 Aug 2026 16:11:12 -0400 Subject: [PATCH 41/51] Create a custom node for the apply_u collision operator. --- graph_framework/node.hpp | 222 ++++++++++++++--- graph_framework/particle_in_cell.hpp | 347 +++++++++++++++++++++++++++ graph_framework/random.hpp | 2 +- 3 files changed, 541 insertions(+), 30 deletions(-) diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 2d7ea1a..63b6d27 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -1231,7 +1231,7 @@ namespace graph { /// @returns The evaluated value of the node. //------------------------------------------------------------------------------ virtual backend::buffer evaluate() { - return this->arg->evaluate(); + return arg->evaluate(); } //------------------------------------------------------------------------------ @@ -1253,10 +1253,10 @@ namespace graph { jit::texture2d_list &textures2d, int &avail_const_mem) { if (visited.find(this) == visited.end()) { - this->arg->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); + arg->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1280,14 +1280,16 @@ namespace graph { jit::register_map ®isters, const jit::register_map &thread_mem, const jit::register_usage &usage) { - return this->arg->compile(stream, registers, thread_mem, usage); + return arg->compile(stream, registers, thread_mem, usage); } //------------------------------------------------------------------------------ /// @brief Get the argument. +/// +/// @returns The argument. //------------------------------------------------------------------------------ - shared_leaf get_arg() { - return this->arg; + shared_leaf get_arg() const { + return arg; } //------------------------------------------------------------------------------ @@ -1296,7 +1298,7 @@ namespace graph { /// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ virtual bool is_all_variables() const { - return this->arg->is_all_variables(); + return arg->is_all_variables(); } //------------------------------------------------------------------------------ @@ -1381,14 +1383,14 @@ namespace graph { jit::texture2d_list &textures2d, int &avail_const_mem) { if (visited.find(this) == visited.end()) { - this->left->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); - this->right->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); + left->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); + right->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1400,16 +1402,20 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Get the left branch. +/// +/// @returns The left argument. //------------------------------------------------------------------------------ - shared_leaf get_left() { - return this->left; + shared_leaf get_left() const { + return left; } //------------------------------------------------------------------------------ /// @brief Get the right branch. +/// +/// @returns The right argument. //------------------------------------------------------------------------------ - shared_leaf get_right() { - return this->right; + shared_leaf get_right() const { + return right; } //------------------------------------------------------------------------------ @@ -1418,8 +1424,8 @@ namespace graph { /// @returns True if the node acts like a variable. //------------------------------------------------------------------------------ virtual bool is_all_variables() const { - return this->left->is_all_variables() && - this->right->is_all_variables(); + return left->is_all_variables() && + right->is_all_variables(); } //------------------------------------------------------------------------------ @@ -1452,7 +1458,6 @@ namespace graph { shared_leaf middle; public: - //------------------------------------------------------------------------------ /// @brief Reduces and assigns the left and right branches. /// @@ -1515,10 +1520,12 @@ namespace graph { } //------------------------------------------------------------------------------ -/// @brief Get the right branch. +/// @brief Get the middle branch. +/// +/// @returns The middle branch. //------------------------------------------------------------------------------ - shared_leaf get_middle() { - return this->middle; + shared_leaf get_middle() const { + return middle; } //------------------------------------------------------------------------------ @@ -1528,11 +1535,134 @@ namespace graph { //------------------------------------------------------------------------------ virtual bool is_all_variables() const { return this->left->is_all_variables() && - this->middle->is_all_variables() && + middle->is_all_variables() && this->right->is_all_variables(); } }; +//****************************************************************************** +// Base N arg node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief Class representing a N branch node. +/// +/// @tparam N Number of branches of the node. +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// This ensures that the base leaf type has the common type between the two +/// template arguments. +//------------------------------------------------------------------------------ + template + class n_branch_node : public leaf_node { + protected: +/// Branches of the tree. + std::array, N> branches; + +//------------------------------------------------------------------------------ +/// @brief Check if any sub-node has a pseudo variable. +/// +/// @param[in] branches Array of branches. +/// @returns True if any branch contains pseudo. +//------------------------------------------------------------------------------ + bool any_has_pseudo(std::array, N> &branches) { + for (auto &b : branches) { + const bool test = b->has_pseudo(); + if (test) { + return test; + } + } + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Check if any sub-node has a pseudo variable. +/// +/// @param[in] branches Array of branches. +/// @returns True if any branch contains pseudo. +//------------------------------------------------------------------------------ + size_t total_complexity(std::array, N> &branches) { + size_t complexity = 1; + for (auto &b : branches) { + complexity += b->get_complexity(); + } + return complexity; + } + + public: +//------------------------------------------------------------------------------ +/// @brief Reduces and assigns the branches. +/// +/// @param[in] branches Array of branches. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + n_branch_node(std::array, N> branches, + const std::string s) : + leaf_node (s, n_branch_node::total_complexity(branches), + n_branch_node::any_has_pseudo(branches)), + branches(branches) {} + +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + int &avail_const_mem) { + if (visited.find(this) == visited.end()) { + for (auto &b : branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); + } + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Get the Nth arg. +/// +/// @param[in] index The argument index. +/// @returns The argument at the index. +//------------------------------------------------------------------------------ + shared_leaf get_arg(const size_t index) const { + return branches[index]; + } + +//------------------------------------------------------------------------------ +/// @brief Test if node acts like a variable. +/// +/// @returns True if the node acts like a variable. +//------------------------------------------------------------------------------ + virtual bool is_all_variables() const { + for (auto b : branches) { + const bool test = b->is_all_variables(); + if (!test) { + return test; + } + } + return true; + } + }; + //------------------------------------------------------------------------------ /// @brief Type trait for not having a valid derivative. /// @@ -1550,8 +1680,10 @@ namespace graph { /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. /// @tparam BASE_NODE Base code to subclass from. +/// @tparam N Number of sub-nodes. //------------------------------------------------------------------------------ - template> + template, size_t N=1> class no_derivative : public BASE_NODE { public: template @@ -1596,6 +1728,38 @@ namespace graph { no_derivative>>) : branch_node (l, r, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base triple nodes base classes. +/// +/// @param[in] l Left branch. +/// @param[in] m Middle branch. +/// @param[in] r Right branch. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(shared_leaf l, + shared_leaf m, + shared_leaf r, + const std::string s) + requires(std::is_base_of_v, + no_derivative>>) : + triple_node (l, m, r, s) {} + +//------------------------------------------------------------------------------ +/// @brief Constructor for base n branch nodes base classes. +/// +/// @param[in] b Array of branches. +/// @param[in] s Node string to hash. +//------------------------------------------------------------------------------ + no_derivative(std::array, N> &b, + const std::string s) + requires(std::is_base_of_v, + no_derivative, + N>>) : + n_branch_node (b, s) {} }; //****************************************************************************** diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index ae5887f..943ba13 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -301,6 +301,353 @@ namespace pic { } }; +//------------------------------------------------------------------------------ +/// @brief U Collision node. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class apply_u_node final : public graph::no_derivative, + 7> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] branches Array of branches. +/// @return A string rep of a the node. +//------------------------------------------------------------------------------ + static std::string to_string(std::array, 7> &branches) { + std::string s = "apply_u"; + for (auto &b : branches) { + s += jit::format_to_string(reinterpret_cast (b.get())); + } + return s; + } + +//------------------------------------------------------------------------------ +/// @brief Define a CPU evaluator. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +//------------------------------------------------------------------------------ + static void func(backend::buffer &x, + const backend::buffer &i, + const backend::buffer &rand, + const backend::buffer &mof, + const backend::buffer &tbnu_e_dt, + const backend::buffer &A, + const backend::buffer &B) { + assert(x.size() == i.size() == rand.size() == mof.size() == tbnu_e_dt.size() && + "Expected all arguments to have the same length."); + + for (size_t j = 0, ej = x.size(); j < ej; j++) { + const T mof_j = mof[j]; + T temp_x = x[j]; + const uint32_t rand_j = reintrepet_cast (rand[j]); + const T tbnu_e_dt_j = tbnu_e_dt[j]; + for (uint8_t k = 0, ke = i[j]; k < ke; k++) { + const T E0 = mof_j*temp_x; + const int8_t rm = 4*((rand_j >> k) & 1) - 2; + const T C = rm*std::sqrt(tbnu_e_dt_j*E0); + temp_x = (E0*A + B + C)/mof_j; + } + x[j] = temp_x; + } + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an apply_u_node. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +//------------------------------------------------------------------------------ + apply_u_node(graph::shared_leaf x, + graph::shared_leaf i, + graph::shared_leaf rand, + graph::shared_leaf mof, + graph::shared_leaf tbnu_e_dt, + graph::shared_leaf A, + graph::shared_leaf B) : + graph::no_derivative> ({x, i, rand, mof, tbnu_e_dt, A, B}, + apply_u_node::to_string({x, i, rand, mof, tbnu_e_dt, A, B})) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of the applying the u operator. +/// +/// result = apply_u(x, i, rand, mof, tbnu_e_dt, A, B) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer x = this->branches[0]->evaluate(); + const backend::buffer i = this->branches[1]->evaluate(); + const backend::buffer rand = this->branches[2]->evaluate(); + const backend::buffer mof = this->branches[3]->evaluate(); + const backend::buffer tbnu_e_dt = this->branches[4]->evaluate(); + const backend::buffer A = this->branches[5]->evaluate(); + const backend::buffer B = this->branches[6]->evaluate(); + + apply_u_node::func(x, i, rand, mof, tbnu_e_dt, A, B); + return x; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the apply_u(x, i, rand, mof, tbnu_e_dt, A, B). +/// +/// @returns Reduced graph from apply_u. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf reduce() { + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + int &avail_const_mem) { + if (visited.find(this) == visited.end()) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); + } + + stream << "void apply_u("; + jit::add_type (stream); + stream << " &x, const uint8_t i, const "; + if constexpr (std::same_as) { + stream << "uint32_t"; + } else { + stream << "uint64_t"; + } + stream << " rand, const "; + jit::add_type (stream); + stream << " mof, const "; + jit::add_type (stream); + stream << " tbnu_e_dt, const "; + jit::add_type (stream); + stream << " A, const "; + jit::add_type (stream); + stream << " B) {" + << " for (uint8_t j = 0; j < i; j++) {" << std::endl + << " const "; + jit::add_type (stream); + stream << "E0 = mof*x;" << std::endl + << " const uint8_t rm = 4*((rand >> j) & 1) - 2;" << std::endl + << " const "; + jit::add_type (stream); + stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl + << " x = (E0*A + B + C)/mof;" << std::endl + << " }" << std::endl + << "}"; + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); + auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); + auto rand = this->branches[2]->compile(stream, registers, thread_mem, usage); + auto mof = this->branches[3]->compile(stream, registers, thread_mem, usage); + auto tbnu_e_dt = this->branches[4]->compile(stream, registers, thread_mem, usage); + auto A = this->branches[5]->compile(stream, registers, thread_mem, usage); + auto B = this->branches[6]->compile(stream, registers, thread_mem, usage); + + registers[this] = registers[x.get()]; + stream << " apply_u(" + << registers[x.get()] << ", " + << registers[i.get()] << ", " + << registers[rand.get()] << ", " + << registers[mof.get()] << ", " + << registers[tbnu_e_dt.get()] << ", " + << registers[A.get()] << ", " + << registers[B.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(graph::shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = apply_u_cast(x); + bool temp = true; + if (x_cast.get()) { + for (size_t i = 0; i < 7; i++) { + temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); + } + } + + return temp; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "\\apply_u{\\left("; + this->branches[0]->to_latex(); + for (uint8_t i = 1; i < 7; i++) { + std::cout << ", " << this->branches[i]->to_latex(); + } + std::cout << "\\right)}"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return apply_u(this->branches[0]->remove_pseudo(), + this->branches[1]->remove_pseudo(), + this->branches[2]->remove_pseudo(), + this->branches[3]->remove_pseudo(), + this->branches[4]->remove_pseudo(), + this->branches[5]->remove_pseudo(), + this->branches[6]->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"apply_u\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + for (auto &b : this->branches) { + auto temp = b->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[temp.get()] << ";" << std::endl; + } + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] mof Mass over 2 e. +/// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. +/// @param[in] A A collision factor. +/// @param[in] B B collision factor. +/// @returns A reduced apply_u node. +//------------------------------------------------------------------------------ + template + graph::shared_leaf apply_u(graph::shared_leaf x, + graph::shared_leaf i, + graph::shared_leaf rand, + graph::shared_leaf mof, + graph::shared_leaf tbnu_e_dt, + graph::shared_leaf A, + graph::shared_leaf B) { + auto temp = std::make_shared> (x, i, rand, mof, + tbnu_e_dt, A, B)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (graph::leaf_node::caches.nodes.find(i) == + graph::leaf_node::caches.nodes.end()) { + graph::leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { + return graph::leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared sqrt nodes. + template + using shared_apply_u = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_apply_u apply_u_cast(graph::shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //------------------------------------------------------------------------------ /// @brief Mesh class. /// diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index 083551b..1adbfa4 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -308,7 +308,7 @@ namespace graph { if constexpr (jit::use_metal ()) { stream << "device "; } - stream <<"mt_state &state) {" << std::endl + stream << "mt_state &state) {" << std::endl << " uint16_t k = state.index;" << std::endl << " uint16_t j = (k + 1) % 624;" << std::endl << " uint32_t x = (state.array[k] & 0x80000000U) |" << std::endl From 37c8c3c438c354db9d950bebfb51d4d98d43fec8 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 31 Aug 2026 16:25:45 -0400 Subject: [PATCH 42/51] The input for x would be computed as const floating_point type so we cannot update it by reference. Instead make the function apply_u return an updated value. --- graph_framework/particle_in_cell.hpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 943ba13..0b077e4 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -436,10 +436,11 @@ namespace pic { textures1d, textures2d, avail_const_mem); } - - stream << "void apply_u("; + + jit::add_type (stream); + stream << " apply_u(const "; jit::add_type (stream); - stream << " &x, const uint8_t i, const "; + stream << " x, const uint8_t i, const "; if constexpr (std::same_as) { stream << "uint32_t"; } else { @@ -454,16 +455,20 @@ namespace pic { stream << " A, const "; jit::add_type (stream); stream << " B) {" + << " "; + jit::add_type (stream); + stream << " temp_x = x;" << std::endl << " for (uint8_t j = 0; j < i; j++) {" << std::endl << " const "; jit::add_type (stream); - stream << "E0 = mof*x;" << std::endl + stream << "E0 = mof*temp_x;" << std::endl << " const uint8_t rm = 4*((rand >> j) & 1) - 2;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl - << " x = (E0*A + B + C)/mof;" << std::endl + << " temp_x = (E0*A + B + C)/mof;" << std::endl << " }" << std::endl + << " return temp_x;" << "}"; visited.insert(this); @@ -497,8 +502,10 @@ namespace pic { auto A = this->branches[5]->compile(stream, registers, thread_mem, usage); auto B = this->branches[6]->compile(stream, registers, thread_mem, usage); - registers[this] = registers[x.get()]; - stream << " apply_u(" + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = apply_u(" << registers[x.get()] << ", " << registers[i.get()] << ", " << registers[rand.get()] << ", " From 2b8abaac46981fc427413c6441d49faed58c3f92 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Mon, 31 Aug 2026 17:43:02 -0400 Subject: [PATCH 43/51] Add apply_xi custom node. --- graph_framework/particle_in_cell.hpp | 446 ++++++++++++++++++++++++--- 1 file changed, 403 insertions(+), 43 deletions(-) diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 0b077e4..4302745 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -343,19 +343,35 @@ namespace pic { const backend::buffer &tbnu_e_dt, const backend::buffer &A, const backend::buffer &B) { - assert(x.size() == i.size() == rand.size() == mof.size() == tbnu_e_dt.size() && + const size_t size = x.size(); + assert(size == i.size() && + size == rand.size() && + size == mof.size() && + size == tbnu_e_dt.size() && + size == A.size() && + size == B.size() && "Expected all arguments to have the same length."); - for (size_t j = 0, ej = x.size(); j < ej; j++) { + for (size_t j = 0; j < size; j++) { const T mof_j = mof[j]; T temp_x = x[j]; - const uint32_t rand_j = reintrepet_cast (rand[j]); const T tbnu_e_dt_j = tbnu_e_dt[j]; - for (uint8_t k = 0, ke = i[j]; k < ke; k++) { - const T E0 = mof_j*temp_x; - const int8_t rm = 4*((rand_j >> k) & 1) - 2; - const T C = rm*std::sqrt(tbnu_e_dt_j*E0); - temp_x = (E0*A + B + C)/mof_j; + if constexpr (std::same_as) { + uint32_t rand_j = reintrepet_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T E0 = mof_j*temp_x; + const int8_t rm = 4*(rand_j & 1) - 2; + const T C = rm*std::sqrt(tbnu_e_dt_j*E0); + temp_x = (E0*A + B + C)/mof_j; + } + } else { + uint64_t rand_j = reintrepet_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T E0 = mof_j*temp_x; + const int8_t rm = 4*(rand_j & 1) - 2; + const T C = rm*std::sqrt(tbnu_e_dt_j*E0); + temp_x = (E0*A + B + C)/mof_j; + } } x[j] = temp_x; } @@ -367,7 +383,7 @@ namespace pic { /// /// @param[in] x Argument to apply collision to. /// @param[in] i Number of collision iterations. -/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] rand A random value of 0s and 1s. /// @param[in] mof Mass over 2 e. /// @param[in] tbnu_e_dt Temperature of species b times normalized collision rate. /// @param[in] A A collision factor. @@ -440,7 +456,7 @@ namespace pic { jit::add_type (stream); stream << " apply_u(const "; jit::add_type (stream); - stream << " x, const uint8_t i, const "; + stream << " x, const uint8_t i, "; if constexpr (std::same_as) { stream << "uint32_t"; } else { @@ -458,11 +474,11 @@ namespace pic { << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl - << " for (uint8_t j = 0; j < i; j++) {" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); stream << "E0 = mof*temp_x;" << std::endl - << " const uint8_t rm = 4*((rand >> j) & 1) - 2;" << std::endl + << " const uint8_t rm = 4*(rand & 1) - 2;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl @@ -507,8 +523,14 @@ namespace pic { jit::add_type (stream); stream << " " << registers[this] << " = apply_u(" << registers[x.get()] << ", " - << registers[i.get()] << ", " - << registers[rand.get()] << ", " + << registers[i.get()] << ", reinterpret_cast<"; + if constexpr (std::same_as) { + stream << "uint32_t"; + } else { + stream << "uint64_t"; + } + stream << "> (" + << registers[rand.get()] << "), " << registers[mof.get()] << ", " << registers[tbnu_e_dt.get()] << ", " << registers[A.get()] << ", " @@ -531,9 +553,10 @@ namespace pic { } auto x_cast = apply_u_cast(x); - bool temp = true; + bool temp; if (x_cast.get()) { - for (size_t i = 0; i < 7; i++) { + temp = this->branches[0]->is_match(x_cast->get_arg(0)); + for (size_t i = 1; i < 7 && temp; i++) { temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); } } @@ -610,37 +633,37 @@ namespace pic { /// @param[in] B B collision factor. /// @returns A reduced apply_u node. //------------------------------------------------------------------------------ - template - graph::shared_leaf apply_u(graph::shared_leaf x, - graph::shared_leaf i, - graph::shared_leaf rand, - graph::shared_leaf mof, - graph::shared_leaf tbnu_e_dt, - graph::shared_leaf A, - graph::shared_leaf B) { - auto temp = std::make_shared> (x, i, rand, mof, - tbnu_e_dt, A, B)->reduce(); + template + graph::shared_leaf apply_u(graph::shared_leaf x, + graph::shared_leaf i, + graph::shared_leaf rand, + graph::shared_leaf mof, + graph::shared_leaf tbnu_e_dt, + graph::shared_leaf A, + graph::shared_leaf B) { + auto temp = std::make_shared> (x, i, rand, mof, + tbnu_e_dt, A, B)->reduce(); // Test for hash collisions. - for (size_t i = temp->get_hash(); - i < std::numeric_limits::max(); i++) { - if (graph::leaf_node::caches.nodes.find(i) == - graph::leaf_node::caches.nodes.end()) { - graph::leaf_node::caches.nodes[i] = temp; - return temp; - } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { - return graph::leaf_node::caches.nodes[i]; - } + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (graph::leaf_node::caches.nodes.find(i) == + graph::leaf_node::caches.nodes.end()) { + graph::leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { + return graph::leaf_node::caches.nodes[i]; } + } #if defined(__clang__) || defined(__GNUC__) - __builtin_unreachable(); + __builtin_unreachable(); #else - assert(false && "Should never reach."); + assert(false && "Should never reach."); #endif - } + } /// Convenience type alias for shared sqrt nodes. - template - using shared_apply_u = std::shared_ptr>; + template + using shared_apply_u = std::shared_ptr>; //------------------------------------------------------------------------------ /// @brief Cast to a apply_u node. @@ -650,11 +673,348 @@ namespace pic { /// @param[in] x Leaf node to attempt cast. /// @returns An attempted dynamic case. //------------------------------------------------------------------------------ - template - shared_apply_u apply_u_cast(graph::shared_leaf x) { - return std::dynamic_pointer_cast> (x); + template + shared_apply_u apply_u_cast(graph::shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//------------------------------------------------------------------------------ +/// @brief Xi Collision node. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class apply_xi_node final : public graph::no_derivative, + 4> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] branches Array of branches. +/// @return A string rep of a the node. +//------------------------------------------------------------------------------ + static std::string to_string(std::array, 4> &branches) { + std::string s = "apply_xi"; + for (auto &b : branches) { + s += jit::format_to_string(reinterpret_cast (b.get())); + } + return s; + } + +//------------------------------------------------------------------------------ +/// @brief Define a CPU evaluator. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +//------------------------------------------------------------------------------ + static void func(backend::buffer &x, + const backend::buffer &i, + const backend::buffer &rand, + const backend::buffer &nu_D_dt) { + const size_t size = x.size(); + assert(size == i.size() && + size == rand.size() && + size == nu_D_dt.size() && + "Expected all arguments to have the same length."); + + for (size_t j = 0; j < size; j++) { + const T nu_D_dt_j = nu_D_dt[j]; + T temp_x = x[j]; + if constexpr (std::same_as) { + uint32_t rand_j = reintrepet_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T A = -temp_x*nu_D_dt_j; + const int8_t rm = 2*(rand_j & 1) - 1; + const T C = rm*std::sqrt((1 - temp_x*temp_x)*nu_D_dt_j); + temp_x += A + C; + } + } else { + uint64_t rand_j = reintrepet_cast (rand[j]); + for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { + const T A = -temp_x*nu_D_dt_j; + const int8_t rm = 2*(rand_j & 1) - 1; + const T C = rm*std::sqrt((1 - temp_x*temp_x)*nu_D_dt_j); + temp_x += A + C; + } + } + x[j] = temp_x; + } + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an apply_xi_node. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +//------------------------------------------------------------------------------ + apply_xi_node(graph::shared_leaf x, + graph::shared_leaf i, + graph::shared_leaf rand, + graph::shared_leaf nu_D_dt) : + graph::no_derivative> ({x, i, rand, nu_D_dt}, + apply_xi_node::to_string({x, i, rand, nu_D_dt})) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of the applying the xi operator. +/// +/// result = apply_xi(x, i, rand, nu_D_dt) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer x = this->branches[0]->evaluate(); + const backend::buffer i = this->branches[1]->evaluate(); + const backend::buffer rand = this->branches[2]->evaluate(); + const backend::buffer nu_D_dt = this->branches[3]->evaluate(); + + apply_xi_node::func(x, i, rand, nu_D_dt); + return x; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the apply_zi(x, i, rand, nu_D_dt). +/// +/// @returns Reduced graph from apply_xi. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf reduce() { + return this->shared_from_this(); } +//------------------------------------------------------------------------------ +/// @brief Compile preamble. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in,out] visited List of visited nodes. +/// @param[in,out] usage List of register usage count. +/// @param[in,out] textures1d List of 1D textures. +/// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] avail_const_mem Available constant memory. +//------------------------------------------------------------------------------ + virtual void compile_preamble(std::ostringstream &stream, + jit::register_map ®isters, + jit::visiter_map &visited, + jit::register_usage &usage, + jit::texture1d_list &textures1d, + jit::texture2d_list &textures2d, + int &avail_const_mem) { + if (visited.find(this) == visited.end()) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + avail_const_mem); + } + + jit::add_type (stream); + stream << " apply_xi(const "; + jit::add_type (stream); + stream << " x, const uint8_t i, "; + if constexpr (std::same_as) { + stream << "uint32_t"; + } else { + stream << "uint64_t"; + } + stream << " rand, const "; + jit::add_type (stream); + stream << " nu_D_dt) {" + << " "; + jit::add_type (stream); + stream << " temp_x = x;" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " const "; + jit::add_type (stream); + stream << "A = -temp_x*nu_D_dt;" << std::endl + << " const uint8_t rm = 2*(rand & 1) - 1;" << std::endl + << " const "; + jit::add_type (stream); + stream << " C = rm*sqrt((1 - temp_x*temp_x)*nu_D_dt);" << std::endl + << " temp_x += A + C;" << std::endl + << " }" << std::endl + << " return temp_x;" + << "}"; + + visited.insert(this); +#ifdef SHOW_USE_COUNT + usage[this] = 1; + } else { + ++usage[this]; +#endif + } + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); + auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); + auto rand = this->branches[2]->compile(stream, registers, thread_mem, usage); + auto nu_D_dt = this->branches[3]->compile(stream, registers, thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = apply_xi(" + << registers[x.get()] << ", " + << registers[i.get()] << ", reinterpret_cast<"; + if constexpr (std::same_as) { + stream << "uint32_t"; + } else { + stream << "uint64_t"; + } + stream << "> (" + << registers[rand.get()] << "), " + << registers[nu_D_dt.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(graph::shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = apply_xi_cast(x); + bool temp; + if (x_cast.get()) { + temp = this->branches[0]->is_match(x_cast->get_arg(0)); + for (size_t i = 1; i < 4 && temp; i++) { + temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); + } + } + + return temp; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "\\apply_xi{\\left("; + this->branches[0]->to_latex(); + for (uint8_t i = 1; i < 4; i++) { + std::cout << ", " << this->branches[i]->to_latex(); + } + std::cout << "\\right)}"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return apply_xi(this->branches[0]->remove_pseudo(), + this->branches[1]->remove_pseudo(), + this->branches[2]->remove_pseudo(), + this->branches[3]->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"apply_xi\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + for (auto &b : this->branches) { + auto temp = b->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[temp.get()] << ";" << std::endl; + } + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build apply_xi node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Argument to apply collision to. +/// @param[in] i Number of collision iterations. +/// @param[in] rand A random value of 32 0s and 1s. +/// @param[in] nu_D_dt Normalized step rate. +/// @returns A reduced apply_xi node. +//------------------------------------------------------------------------------ + template + graph::shared_leaf apply_xi(graph::shared_leaf x, + graph::shared_leaf i, + graph::shared_leaf rand, + graph::shared_leaf nu_D_dt) { + auto temp = std::make_shared> (x, i, rand, + nu_D_dt)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (graph::leaf_node::caches.nodes.find(i) == + graph::leaf_node::caches.nodes.end()) { + graph::leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { + return graph::leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared sqrt nodes. + template + using shared_apply_xi = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a apply_u node. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic case. +//------------------------------------------------------------------------------ + template + shared_apply_xi apply_u_cast(graph::shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //------------------------------------------------------------------------------ /// @brief Mesh class. /// From 2dac57f9b82f25c4b27d50a30a94f826c83929bc Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 3 Sep 2026 17:25:29 -0400 Subject: [PATCH 44/51] Add nodes needed to impliement coordinate conversions and fix reductions for coordinate conversion creates an exact graph. --- graph_framework/arithmetic.hpp | 161 +++++- graph_framework/backend.hpp | 280 ++++++++-- graph_framework/math.hpp | 732 ++++++++++++++++++++++++++- graph_framework/particle_in_cell.hpp | 325 ++++++------ graph_framework/trigonometry.hpp | 2 +- graph_tests/arithmetic_test.cpp | 92 ++++ graph_tests/backend_test.cpp | 88 ++++ graph_tests/jit_test.cpp | 6 + graph_tests/math_test.cpp | 106 ++++ graph_tests/pic_test.cpp | 17 + 10 files changed, 1606 insertions(+), 203 deletions(-) diff --git a/graph_framework/arithmetic.hpp b/graph_framework/arithmetic.hpp index 9be6ad1..efba008 100644 --- a/graph_framework/arithmetic.hpp +++ b/graph_framework/arithmetic.hpp @@ -676,12 +676,10 @@ namespace graph { auto x_cast = add_cast(x); if (x_cast.get()) { // Addition is commutative. - if ((this->left->is_match(x_cast->get_left()) && - this->right->is_match(x_cast->get_right())) || - (this->right->is_match(x_cast->get_left()) && - this->left->is_match(x_cast->get_right()))) { - return true; - } + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); } return false; @@ -1061,6 +1059,24 @@ namespace graph { } } +// (a + b) - a -> b +// (a + b) - b -> a +// a - (a + b) -> -b +// b - (a + b) -> -a + if (la.get()) { + if (la->get_left()->is_match(this->right)) { + return la->get_right(); + } else if (la->get_right()->is_match(this->right)) { + return la->get_left(); + } + } else if (ra.get()) { + if (ra->get_left()->is_match(this->left)) { + return none ()*ra->get_right(); + } else if (ra->get_right()->is_match(this->left)) { + return none ()*ra->get_left(); + } + } + // Assume constants are on the left. // v1 - -c*v2 -> v1 + c*v2 if (rm.get() && @@ -2458,6 +2474,96 @@ namespace graph { } } +// Sqrt(a)*Sqrt(b) -> Sqrt(a*b) + auto lsqr = sqrt_cast(this->left); + auto rsqr = sqrt_cast(this->right); + if (lsqr.get() && rsqr.get()) { + return sqrt(lsqr->get_arg()*rsqr->get_arg()); + } + + if constexpr (std::floating_point) { +// hypot(b,c)*Sqrt(a) -> Sqrt((b^2 + c^2)*a) +// Sqrt(a)*hypot(b,c) -> Sqrt(a*(b^2 + c^2)) + auto lhypot = hypot_cast(this->left); + auto rhypot = hypot_cast(this->right); + if (lhypot.get() && rsqr.get()) { + return sqrt((pow(lhypot->get_left(), static_cast (2)) + + pow(lhypot->get_right(), static_cast (2))) * + rsqr->get_arg()); + } else if (rhypot.get() && lsqr.get()) { + return sqrt((pow(rhypot->get_left(), static_cast (2)) + + pow(rhypot->get_right(), static_cast (2))) * + lsqr->get_arg()); + } + +// Sqrt(x^2)*copysign(1,x) -> x +// copysign(1,x)*Sqrt(x^2) -> x + auto lcsc = copysign_cast(this->left); + auto rcsc = copysign_cast(this->right); + if (rcsc.get() && lsqr.get()) { + auto rcsclc = constant_cast(rcsc->get_left()); + auto lsqrpc = pow_cast(lsqr->get_arg()); + if (lsqrpc.get() && rcsclc.get() && rcsclc->is(1) && + lsqrpc->get_left()->is_match(rcsc->get_right())) { + return rcsc->get_right(); + } + } else if (lcsc.get() && rsqr.get()) { + auto lcsclc = constant_cast(lcsc->get_left()); + auto rsqrpc = pow_cast(rsqr->get_arg()); + if (rsqrpc.get() && lcsclc.get() && lcsclc->is(1) && + rsqrpc->get_left()->is_match(lcsc->get_right())) { + return lcsc->get_right(); + } + } + } + +// (a + b/c)*c -> fma(a,c,b) +// c*(a + b/c) -> fma(a,c,b) +// (b/c + a)*c -> fma(a,c,b) +// c*(b/c + a) -> fma(a,c,b) + auto la = add_cast(this->left); + if (la.get()) { + auto lald = divide_cast(la->get_left()); + auto lard = divide_cast(la->get_right()); + if (lald.get() && lald->get_right()->is_match(this->right)) { + return fma(la->get_right(), this->right, lald->get_left()); + } else if (lard.get() && lard->get_right()->is_match(this->right)) { + return fma(la->get_left(), this->right, lard->get_left()); + } + } else if (ra.get()) { + auto rald = divide_cast(ra->get_left()); + auto rard = divide_cast(ra->get_right()); + if (rald.get() && rald->get_right()->is_match(this->left)) { + return fma(ra->get_right(), this->left, rald->get_left()); + } else if (rard.get() && rard->get_right()->is_match(this->left)) { + return fma(ra->get_left(), this->left, rard->get_left()); + } + } + +// (a - b/c)*c -> a*c - b +// c*(a - b/c) -> a*c - b +// (b/c - a)*c -> b - a*c +// c*(b/c - a) -> b - a*c + auto ls = subtract_cast(this->left); + auto rs = subtract_cast(this->right); + if (ls.get()) { + auto lsld = divide_cast(ls->get_left()); + auto lsrd = divide_cast(ls->get_right()); + if (lsld.get() && lsld->get_right()->is_match(this->right)) { + return lsld->get_left() - ls->get_right()*this->right; + } else if (lsrd.get() && lsrd->get_right()->is_match(this->right)) { + return ls->get_left()*this->right - lsrd->get_left(); + } + } else if (rs.get()) { + auto rsld = divide_cast(rs->get_left()); + auto rsrd = divide_cast(rs->get_right()); + if (rsld.get() && rsld->get_right()->is_match(this->left)) { + return rsld->get_right() - rs->get_right()*this->left; + } else if (rsrd.get() && rsrd->get_right()->is_match(this->left)) { + return rs->get_left()*this->left - rsld->get_left(); + } + } + // Cases like // (c/exp(a))*(exp(b)/d) -> (c/d)*(exp(b)/exp(a)) // (c/exp(a))*(d/exp(b)) -> (c*e)/(exp(b)*exp(a)) @@ -5012,6 +5118,49 @@ namespace graph { } } +// fma(sqrt(a),sqrt(b),c) -> sqrt(a*b) + c + auto lsqr = sqrt_cast(this->left); + auto msqr = sqrt_cast(this->middle); + if (lsqr.get() && msqr.get()) { + return sqrt(lsqr->get_arg()*msqr->get_arg()) + this->right; + } + + if constexpr (std::floating_point) { +// fma(hypot(b,c),Sqrt(a),d) -> Sqrt((b^2 + c^2)*a) + d +// fma(Sqrt(a),hypot(b,c),d) -> Sqrt(a*(b^2 + c^2)) + d + auto lhypot = hypot_cast(this->left); + auto mhypot = hypot_cast(this->middle); + if (lhypot.get() && msqr.get()) { + return sqrt((pow(lhypot->get_left(), static_cast (2)) + + pow(lhypot->get_right(), static_cast (2))) * + msqr->get_arg()) + this->right; + } else if (mhypot.get() && lsqr.get()) { + return sqrt((pow(mhypot->get_left(), static_cast (2)) + + pow(mhypot->get_right(), static_cast (2))) * + lsqr->get_arg()) + this->right; + } + +// fma(Sqrt(x^2),copysign(1,x),d) -> x +// fma(copysign(1,x),Sqrt(x^2),d) -> x + auto lcsc = copysign_cast(this->left); + auto mcsc = copysign_cast(this->middle); + if (mcsc.get() && lsqr.get()) { + auto mcsclc = constant_cast(mcsc->get_left()); + auto lsqrpc = pow_cast(lsqr->get_arg()); + if (lsqrpc.get() && mcsclc.get() && mcsclc->is(1) && + lsqrpc->get_left()->is_match(mcsc->get_right())) { + return mcsc->get_right() + this->right; + } + } else if (lcsc.get() && msqr.get()) { + auto lcsclc = constant_cast(lcsc->get_left()); + auto msqrpc = pow_cast(msqr->get_arg()); + if (msqrpc.get() && lcsclc.get() && lcsclc->is(1) && + msqrpc->get_left()->is_match(lcsc->get_right())) { + return lcsc->get_right() + this->right; + } + } + } + return this->shared_from_this(); } diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index e926874..151dd4e 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -627,6 +627,150 @@ if (size() > x.size()) { \ } } +//------------------------------------------------------------------------------ +/// @brief Hypot row operation. +/// +/// Computes Hypot(m_ij, v_i) or Hypot(v_i, m_ij). This will resize the buffer +/// if it needs to be. +/// +/// @param[in] x The right operand. +//------------------------------------------------------------------------------ + void hypot_row(const buffer &x) requires(std::floating_point) { + if (size() > x.size()) { + assert(size()%x.size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + const size_t num_columns = size()/x.size(); + const size_t num_rows = x.size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + (*this)[i*num_columns + j] = std::hypot((*this)[i*num_columns + j], x[i]); + } + } + } else { + assert(x.size()%size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + std::vector m(x.size()); + const size_t num_columns = x.size()/size(); + const size_t num_rows = size(); + for (size_t i = 0; i < num_columns; i++) { + for (size_t j = 0; j < num_rows; j++) { + m[i*num_columns + j] = std::hypot((*this)[i], x[i*num_columns + j]); + } + } + *this = m; + } + } + +//------------------------------------------------------------------------------ +/// @brief Hypot col operation. +/// +/// Computes Hypot(m_ij, v_j) or Hypot(v_j, m_ij). This will resize the buffer +/// if it needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void hypot_col(const buffer &x) requires(std::floating_point) { + if (size() > x.size()) { + assert(size()%x.size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + const size_t num_columns = size()/x.size(); + const size_t num_rows = x.size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + (*this)[i*num_columns + j] = std::hypot((*this)[i*num_columns + j], x[j]); + } + } + } else { + assert(x.size()%size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + std::vector m(x.size()); + const size_t num_columns = x.size()/size(); + const size_t num_rows = size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + m[i*num_columns + j] = std::hypot((*this)[j], x[i*num_columns + j]); + } + } + *this = m; + } + } + +//------------------------------------------------------------------------------ +/// @brief copysign row operation. +/// +/// Computes copysign(m_ij, v_i) or copysign(v_i, m_ij). This will resize the +/// buffer if it needs to be. +/// +/// @param[in] x The right operand. +//------------------------------------------------------------------------------ + void copysign_row(const buffer &x) requires(std::floating_point) { + if (size() > x.size()) { + assert(size()%x.size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + const size_t num_columns = size()/x.size(); + const size_t num_rows = x.size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + (*this)[i*num_columns + j] = std::copysign((*this)[i*num_columns + j], x[i]); + } + } + } else { + assert(x.size()%size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + std::vector m(x.size()); + const size_t num_columns = x.size()/size(); + const size_t num_rows = size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + m[i*num_columns + j] = std::copysign((*this)[i], x[i*num_columns + j]); + } + } + *this = m; + } + } + +//------------------------------------------------------------------------------ +/// @brief copysign col operation. +/// +/// Computes atan(m_ij, v_j) or atan(v_j, m_ij). This will resize the buffer if +/// it needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void copysign_col(const buffer &x) { + if (size() > x.size()) { + assert(size()%x.size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + const size_t num_columns = size()/x.size(); + const size_t num_rows = x.size(); + for (size_t i = 0; i < num_columns; i++) { + for (size_t j = 0; j < num_rows; j++) { + (*this)[i*num_columns + j] = std::copysign((*this)[i*num_columns + j], x[j]); + } + } + } else { + assert(x.size()%size() == 0 && + "Vector operand size is not a multiple of matrix operand size"); + + std::vector m(x.size()); + const size_t num_columns = x.size()/size(); + const size_t num_rows = size(); + for (size_t i = 0; i < num_rows; i++) { + for (size_t j = 0; j < num_columns; j++) { + m[i*num_columns + j] = std::copysign((*this)[j], x[i*num_columns + j]); + } + } + *this = m; + } + } + //------------------------------------------------------------------------------ /// @brief Not operation. /// @@ -650,8 +794,7 @@ if (size() > x.size()) { \ /// @params[in] t True condition. /// @params[in] f False condition. //------------------------------------------------------------------------------ - buffer if_(const buffer &t, - const buffer &f) { + buffer if_(const buffer &t, const buffer &f) { if (size() == 1) { if constexpr (std::floating_point) { return (*this)[0] ? t : f; @@ -744,8 +887,7 @@ for (T &d : *this) { \ /// @returns a == b. //------------------------------------------------------------------------------ template - inline bool operator==(const buffer &a, - const buffer &b) { + inline bool operator==(const buffer &a, const buffer &b) { if (a.size() != b.size()) { return false; } @@ -798,8 +940,7 @@ return a; /// @returns max(a, b). //------------------------------------------------------------------------------ template - inline buffer max(buffer &a, - buffer &b) { + inline buffer max(buffer &a, buffer &b) { build_assoc_func(std::max); } @@ -813,8 +954,7 @@ return a; /// @returns min(a, b). //------------------------------------------------------------------------------ template - inline buffer min(buffer &a, - buffer &b) { + inline buffer min(buffer &a, buffer &b) { build_assoc_func(std::min); } @@ -855,8 +995,7 @@ return a; /// @returns a + b. //------------------------------------------------------------------------------ template - inline buffer operator+(buffer &a, - buffer &b) { + inline buffer operator+(buffer &a, buffer &b) { build_assoc_op(+=) } @@ -898,8 +1037,7 @@ return a; /// @returns a - b. //------------------------------------------------------------------------------ template - inline buffer operator-(buffer &a, - buffer &b) { + inline buffer operator-(buffer &a, buffer &b) { build_non_assoc_op(-, -=) } @@ -913,8 +1051,7 @@ return a; /// @returns a * b. //------------------------------------------------------------------------------ template - inline buffer operator*(buffer &a, - buffer &b) { + inline buffer operator*(buffer &a, buffer &b) { build_assoc_op(*=) } @@ -928,11 +1065,51 @@ return a; /// @returns a / b. //------------------------------------------------------------------------------ template - inline buffer operator/(buffer &a, - buffer &b) { + inline buffer operator/(buffer &a, buffer &b) { build_non_assoc_op(/, /=) } +//------------------------------------------------------------------------------ +/// @brief Applies an associative function. +/// +/// @param func The function to apply. +//------------------------------------------------------------------------------ +#define build_assoc_func(func) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (T &l : a) { \ + l = func(l, right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (T &r : b) { \ + r = func(r, left); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = func(a[i], b[i]); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief hypot operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Left operand. +/// @param[in] b Right operand. +/// @returns hypot(a,b) +//------------------------------------------------------------------------------ + template + inline buffer hypot(buffer &a, buffer &b) { + build_assoc_func(std::hypot) + } + //------------------------------------------------------------------------------ /// @brief Fused multiply add operation. /// @@ -944,9 +1121,7 @@ return a; /// @returns a*b + c. //------------------------------------------------------------------------------ template - inline buffer fma(buffer &a, - buffer &b, - buffer &c) { + inline buffer fma(buffer &a, buffer &b, buffer &c) { constexpr bool use_fma = !jit::complex_scalar && #ifdef FP_FAST_FMA true; @@ -1051,8 +1226,7 @@ return a; /// @returns a % b. //------------------------------------------------------------------------------ template - inline buffer operator%(buffer &a, - buffer &b) { + inline buffer operator%(buffer &a, buffer &b) { if (b.size() == 1) { const T right = b[0]; for (size_t i = 0, ie = a.size(); i < ie; i++) { @@ -1112,8 +1286,7 @@ return a; /// @returns a == b. //------------------------------------------------------------------------------ template - inline buffer operator==(buffer &a, - buffer &b) { + inline buffer operator==(buffer &a, buffer &b) { logic_op(==) } @@ -1127,8 +1300,7 @@ return a; /// @returns a == b. //------------------------------------------------------------------------------ template - inline buffer operator!=(buffer &a, - buffer &b) { + inline buffer operator!=(buffer &a, buffer &b) { logic_op(!=) } @@ -1142,8 +1314,7 @@ return a; /// @returns a > b. //------------------------------------------------------------------------------ template - inline buffer operator>(buffer &a, - buffer &b) { + inline buffer operator>(buffer &a, buffer &b) { logic_op(>) } @@ -1157,8 +1328,7 @@ return a; /// @returns a < b. //------------------------------------------------------------------------------ template - inline buffer operator<(buffer &a, - buffer &b) { + inline buffer operator<(buffer &a, buffer &b) { logic_op(<) } @@ -1172,8 +1342,7 @@ return a; /// @returns a >= b. //------------------------------------------------------------------------------ template - inline buffer operator>=(buffer &a, - buffer &b) { + inline buffer operator>=(buffer &a, buffer &b) { logic_op(>=) } @@ -1187,8 +1356,7 @@ return a; /// @returns a <= b. //------------------------------------------------------------------------------ template - inline buffer operator<=(buffer &a, - buffer &b) { + inline buffer operator<=(buffer &a, buffer &b) { logic_op(<=) } @@ -1202,8 +1370,7 @@ return a; /// @returns a && b. //------------------------------------------------------------------------------ template - inline buffer operator&&(buffer &a, - buffer &b) { + inline buffer operator&&(buffer &a, buffer &b) { logic_op(&&) } @@ -1217,8 +1384,7 @@ return a; /// @returns a || b. //------------------------------------------------------------------------------ template - inline buffer operator||(buffer &a, - buffer &b) { + inline buffer operator||(buffer &a, buffer &b) { logic_op(||) } @@ -1232,8 +1398,7 @@ return a; /// @returns base^exponent. //------------------------------------------------------------------------------ template - inline buffer pow(buffer &base, - buffer &exponent) { + inline buffer pow(buffer &base, buffer &exponent) { if (exponent.size() == 1) { const T right = exponent[0]; if (std::imag(right) == 0) { @@ -1305,8 +1470,7 @@ return a; /// @returns atan2(y, x) //------------------------------------------------------------------------------ template - inline buffer atan(buffer &x, - buffer &y) { + inline buffer atan(buffer &x, buffer &y) { if (y.size() == 1) { const T right = y[0]; for (size_t i = 0, ie = x.size(); i < ie; i++) { @@ -1340,6 +1504,40 @@ return a; } return x; } + +//------------------------------------------------------------------------------ +/// @brief Copy the sign of x and apply it to y. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] x X argument. +/// @param[in] y Y argument. +/// @returns copysign(x, y) +//------------------------------------------------------------------------------ + template + inline buffer copysign(buffer &x, + buffer &y) { + if (y.size() == 1) { + const T right = y[0]; + for (size_t i = 0, ie = x.size(); i < ie; i++) { + x[i] = std::copysign(x[i], right); + } + return x; + } else if (x.size() == 1) { + const T left = x[0]; + for (size_t i = 0, ie = y.size(); i < ie; i++) { + y[i] = std::copysign(left, y[i]); + } + return y; + } + + assert(x.size() == y.size() && + "Left and right sizes are incompatible."); + for (size_t i = 0, ie = x.size(); i < ie; i++) { + x[i] = std::copysign(x[i], y[i]); + } + return x; + } } #endif /* backend_h */ diff --git a/graph_framework/math.hpp b/graph_framework/math.hpp index abc51f0..f2ab1b7 100644 --- a/graph_framework/math.hpp +++ b/graph_framework/math.hpp @@ -7,6 +7,7 @@ #define math_h #include +#include #include "node.hpp" @@ -87,24 +88,37 @@ namespace graph { } // Handle cases like sqrt(c*x) where c is constant or cases like sqrt((x^a)*y). -// Note that we need to disable this reduction C is a negative real. +// Note that we need to disable this reduction if C is a negative real or +// a == 2. auto am = multiply_cast(this->arg); if (am.get()) { - if (pow_cast(am->get_left()).get() || - am->get_left()->is_constant() || - pow_cast(am->get_right()).get() || - am->get_right()->is_constant()) { + if (am->get_left()->is_constant()) { if constexpr (jit::complex_scalar) { return sqrt(am->get_left()) * sqrt(am->get_right()); } else { - if (am->get_left()->is_constant() && - !am->get_left()->evaluate().is_negative()) { + if (!am->get_left()->evaluate().is_negative()) { return sqrt(am->get_left()) * sqrt(am->get_right()); } } } + + auto amlp = pow_cast(am->get_left()); + auto amrp = pow_cast(am->get_right()); + if (amlp.get()) { + auto amlprc = constant_cast(amlp->get_right()); + if (amlprc.get() && !amlprc->is(2)) { + return sqrt(am->get_left()) * + sqrt(am->get_right()); + } + } else if (amrp.get()) { + auto amrprc = constant_cast(amrp->get_right()); + if (amrprc.get() && !amrprc->is(2)) { + return sqrt(am->get_left()) * + sqrt(am->get_right()); + } + } } auto ad = divide_cast(this->arg); @@ -846,7 +860,7 @@ namespace graph { public: //------------------------------------------------------------------------------ -/// @brief Construct an power node. +/// @brief Construct a power node. /// /// @param[in] l Left branch. /// @param[in] r Right branch. @@ -857,7 +871,7 @@ namespace graph { r.get())) {} //------------------------------------------------------------------------------ -/// @brief Evaluate the results of addition. +/// @brief Evaluate the results of pow. /// /// result = l^r /// @@ -1146,13 +1160,38 @@ namespace graph { return exp(this->right*temp->get_arg()); } + if constexpr (std::floating_point) { +// hypot(a,b)^2 -> a^2 + b^2 + auto lhp = hypot_cast(this->left); + if (lhp.get() && rc.get() && rc->is(2)) { + return pow(lhp->get_left(), this->right) + + pow(lhp->get_right(), this->right); + } + +// (a/hypot(b,c))^2 -> a^2/(b^2 + c^2) +// (hypot(b,c)/a)^2 -> (b^2 + c^2)/a^2 + if (ld.get() && rc.get() && rc->is(2)) { + auto ldlhp = hypot_cast(ld->get_left()); + auto ldrhp = hypot_cast(ld->get_right()); + if (ldlhp.get()) { + return (pow(ldlhp->get_left(), static_cast (2)) + + pow(ldlhp->get_right(), static_cast (2))) / + pow(ld->get_right(), static_cast (2)); + } else if (ldrhp.get()) { + return pow(ld->get_left(), static_cast (2)) / + (pow(ldrhp->get_left(), static_cast (2)) + + pow(ldrhp->get_right(), static_cast (2))); + } + } + } + return this->shared_from_this(); } //------------------------------------------------------------------------------ /// @brief Transform node to derivative. /// -/// d a^b dx = b*a^(b-1)*da/dx + ln(a)a^b*db/dx +/// d a^b/ dx = b*a^(b-1)*da/dx + ln(a)a^b*db/dx /// /// @param[in] x The variable to take the derivative to. /// @returns The derivative of the node. @@ -1643,6 +1682,679 @@ namespace graph { shared_erfi erfi_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// Hypot node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A hypot node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class hypot_node final : public branch_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Argument node pointer. +/// @param[in] r Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "hypot" + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a hypot node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + hypot_node(shared_leaf l, + shared_leaf r) : + branch_node (l, r, hypot_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of hypot. +/// +/// result = hypot(l, r) +/// +/// @returns The value of hypot(l, r) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::hypot(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a hypot node. +/// +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (rc.get() && rc->is(0)) { + return sqrt(pow(this->left, static_cast (2))); + } else if (lc.get() && lc->is(0)) { + return sqrt(pow(this->right, static_cast (2))); + } else if (rc.get() && lc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.hypot_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.hypot_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.hypot_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.hypot_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + +// hypot(sqrt(a), sqrt(b)) -> sqrt(a + b) +// hypot(a, sqrt(b)) -> sqrt(a^2 + b) +// hypot(sqrt(a), b) -> sqrt(a + b^2) + auto sql = sqrt_cast(this->left); + auto sqr = sqrt_cast(this->right); + if (sql.get() && sqr.get()) { + return sqrt(sql->get_arg() + sqr->get_arg()); + } else if (sql.get()) { + return sqrt(sql->get_arg() + pow(this->right, + static_cast (2))); + } else if (sqr.get()) { + return sqrt(sqr->get_arg() + pow(this->left, + static_cast (2))); + } + +// hypoy(a,a) -> sqrt(2)sqrt(a^2) + if (this->left->is_match(this->right)) { + return std::numbers::sqrt2_v*sqrt(pow(this->left, + static_cast (2))); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d hypot(a,b)/ dx = (a*da/dx + b*db/dx)/hypot(a,b) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = (this->left*this->left->df(x) + + this->right*this->right->df(x)) + / this->shared_from_this(); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = "; + if constexpr (jit::use_metal ()) { + stream << "length({"; + } else { + stream << "hypot("; + } + stream << registers[l.get()] << ", " << registers[r.get()]; + if constexpr (jit::use_metal ()) { + stream << "})"; + } else { + stream << ")"; + } + + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = hypot_cast(x); + if (x_cast.get()) { +// Hypot is commutative. + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "hypot\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return hypot(this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"hypot\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(const L l, + shared_leaf r) { + return hypot(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build hypot node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced hypot node. +//------------------------------------------------------------------------------ + template + shared_leaf hypot(shared_leaf l, + const R r) { + return hypot(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared hypot nodes. + template + using shared_hypot = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a hypot node. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_hypot hypot_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + +//****************************************************************************** +// copysign node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A copysign node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class copysign_node final : public no_derivative> { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Argument node pointer. +/// @param[in] r Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "copysign" + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a hypot node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + copysign_node(shared_leaf l, + shared_leaf r) : + no_derivative> (l, r, + copysign_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of copysign. +/// +/// result = copysign(l, r) +/// +/// @returns The value of copysign(l, r) +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::copysign(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a copysign node. +/// +/// @returns A reduced copysign node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (rc.get() && rc->is(0)) { + return sqrt(pow(this->left, static_cast (2))); + } else if (lc.get() && lc->is(0)) { + return sqrt(pow(this->right, static_cast (2))); + } else if (rc.get() && lc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.copysign_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.copysign_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.copysign_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.copysign_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->right->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = copysign(" + << registers[l.get()] << ", " + << registers[r.get()] << ")"; + + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = hypot_cast(x); + if (x_cast.get()) { + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "copysign\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return copysign(this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"copysign\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build copysign node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build copysign node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(const L l, + shared_leaf r) { + return copysign(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build power node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns copysign a reduced node. +//------------------------------------------------------------------------------ + template + shared_leaf copysign(shared_leaf l, + const R r) { + return copysign(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared copysign nodes. + template + using shared_copysign = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a copysign node. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_copysign copysign_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* math_h */ diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 4302745..e86def4 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -156,151 +156,6 @@ namespace pic { dt(dt/norms.t), t_para(t_para), t_perp(t_perp) {} }; -//------------------------------------------------------------------------------ -/// @brief ion class. -/// -/// These values need to be initialized using normalized quantities. -/// -/// @tparam T Base type of the calculation. -//------------------------------------------------------------------------------ - template - class ion { - public: -/// Atomic number. - const T z; -/// Charge - const T charge; -/// Particle mass - const T mass; -/// Normalized Position - graph::shared_leaf x; -/// Normalized Parallel velocity. - graph::shared_leaf v_para; -/// Normalized Perpendicular velocity. - graph::shared_leaf v_perp; -/// Mesh Weights - std::array, 3> weights; -/// Number of real particles - const T num_real; - -//------------------------------------------------------------------------------ -/// @brief Construct an ion object. -/// -/// @param[in] mass Ion mass. -/// @param[in] z Ion Z. -/// @param[in] num_ions Number of ions. -/// @param[in] num_real Number of real particles. -/// @param[in] norms A @ref pic::characteristics object. -//------------------------------------------------------------------------------ - ion(const T mass, - const uint8_t z, - const size_t num_ions, - const T num_real, - const characteristics &norms) : - z(z), charge(z*pic::q/norms.q), - mass(mass), num_real(num_real), - x(graph::variable (num_ions, "x")), - v_para(graph::variable (num_ions, "v_{||}")), - v_perp(graph::variable (num_ions, "v_{\\perp}")), - weights({ - graph::variable (num_ions, "w_{0}"), - graph::variable (num_ions, "w_{1}"), - graph::variable (num_ions, "w_{2}") - }) {} - -//------------------------------------------------------------------------------ -/// @brief Get x case as variable. -/// -/// @return x cast as a variable. -//------------------------------------------------------------------------------ - graph::shared_variable get_x() const { - return graph::variable_cast(x); - } - -//------------------------------------------------------------------------------ -/// @brief Get the number of computational ions. -/// -/// @return The number of particles. -//------------------------------------------------------------------------------ - size_t size() const { - return get_x()->size(); - } - -//------------------------------------------------------------------------------ -/// @brief Get the data for x. -/// -/// @return The number of particles. -//------------------------------------------------------------------------------ - T *x_data() const { - return get_x()->data(); - } - -//------------------------------------------------------------------------------ -/// @brief Get x case as variable. -/// -/// @return x cast as a variable. -//------------------------------------------------------------------------------ - graph::shared_variable get_v_para() const { - return graph::variable_cast(v_para); - } - -//------------------------------------------------------------------------------ -/// @brief Get the data for the parallel velocity. -/// -/// @return The number of particles. -//------------------------------------------------------------------------------ - T *v_para_data() const { - return get_v_para()->data(); - } - -//------------------------------------------------------------------------------ -/// @brief Get x case as variable. -/// -/// @return x cast as a variable. -//------------------------------------------------------------------------------ - graph::shared_variable get_v_perp() const { - return graph::variable_cast(v_perp); - } - -//------------------------------------------------------------------------------ -/// @brief Get the data for the perpendicular velocity. -/// -/// @return The number of particles. -//------------------------------------------------------------------------------ - T *v_perp_data() const { - return graph::variable_cast(v_perp)->data(); - } - -//------------------------------------------------------------------------------ -/// @brief Conversion factor from super particles to real particles. -/// -/// @returns The super to real conversion factor. -//------------------------------------------------------------------------------ - T super_to_real() const { - return num_real/size(); - } - -//------------------------------------------------------------------------------ -/// @brief Define variables. -/// -/// @param[in] file A @ref output::result_file object to define variables. -/// @param[in,out] data A @ref output::data_set object to create variable. -/// @param[in,out] work A @ref workflow::manager object where data was -/// computed. -/// @param[in] tag Unique identity for give the ion species. -//------------------------------------------------------------------------------ - void define_variables(const output::result_file &file, - output::data_set &data, - workflow::manager &work, - const std::string tag) { - data.create_variable(file, "x_" + tag, x, work.get_context()); - data.create_variable(file, "vpara_" + tag, v_para, - work.get_context()); - data.create_variable(file, "vperp_" + tag, v_perp, - work.get_context()); - } - }; - //------------------------------------------------------------------------------ /// @brief U Collision node. /// @@ -1015,6 +870,151 @@ namespace pic { return std::dynamic_pointer_cast> (x); } +//------------------------------------------------------------------------------ +/// @brief ion class. +/// +/// These values need to be initialized using normalized quantities. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class ion { + public: +/// Atomic number. + const T z; +/// Charge + const T charge; +/// Particle mass + const T mass; +/// Normalized Position + graph::shared_leaf x; +/// Normalized Parallel velocity. + graph::shared_leaf v_para; +/// Normalized Perpendicular velocity. + graph::shared_leaf v_perp; +/// Mesh Weights + std::array, 3> weights; +/// Number of real particles + const T num_real; + +//------------------------------------------------------------------------------ +/// @brief Construct an ion object. +/// +/// @param[in] mass Ion mass. +/// @param[in] z Ion Z. +/// @param[in] num_ions Number of ions. +/// @param[in] num_real Number of real particles. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + ion(const T mass, + const uint8_t z, + const size_t num_ions, + const T num_real, + const characteristics &norms) : + z(z), charge(z*pic::q/norms.q), + mass(mass), num_real(num_real), + x(graph::variable (num_ions, "x")), + v_para(graph::variable (num_ions, "v_{||}")), + v_perp(graph::variable (num_ions, "v_{\\perp}")), + weights({ + graph::variable (num_ions, "w_{0}"), + graph::variable (num_ions, "w_{1}"), + graph::variable (num_ions, "w_{2}") + }) {} + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_x() const { + return graph::variable_cast(x); + } + +//------------------------------------------------------------------------------ +/// @brief Get the number of computational ions. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + size_t size() const { + return get_x()->size(); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for x. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *x_data() const { + return get_x()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_para() const { + return graph::variable_cast(v_para); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the parallel velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_para_data() const { + return get_v_para()->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Get x case as variable. +/// +/// @return x cast as a variable. +//------------------------------------------------------------------------------ + graph::shared_variable get_v_perp() const { + return graph::variable_cast(v_perp); + } + +//------------------------------------------------------------------------------ +/// @brief Get the data for the perpendicular velocity. +/// +/// @return The number of particles. +//------------------------------------------------------------------------------ + T *v_perp_data() const { + return graph::variable_cast(v_perp)->data(); + } + +//------------------------------------------------------------------------------ +/// @brief Conversion factor from super particles to real particles. +/// +/// @returns The super to real conversion factor. +//------------------------------------------------------------------------------ + T super_to_real() const { + return num_real/size(); + } + +//------------------------------------------------------------------------------ +/// @brief Define variables. +/// +/// @param[in] file A @ref output::result_file object to define variables. +/// @param[in,out] data A @ref output::data_set object to create variable. +/// @param[in,out] work A @ref workflow::manager object where data was +/// computed. +/// @param[in] tag Unique identity for give the ion species. +//------------------------------------------------------------------------------ + void define_variables(const output::result_file &file, + output::data_set &data, + workflow::manager &work, + const std::string tag) { + data.create_variable(file, "x_" + tag, x, work.get_context()); + data.create_variable(file, "vpara_" + tag, v_para, + work.get_context()); + data.create_variable(file, "vperp_" + tag, v_perp, + work.get_context()); + } + }; + //------------------------------------------------------------------------------ /// @brief Mesh class. /// @@ -1260,6 +1260,41 @@ namespace pic { } }; +//------------------------------------------------------------------------------ +/// @brief Convert from cartesian to sphereical coordinates. +/// +/// @param[in] x +/// @param[in] y +/// @returns The coordinates as sphereical coordinates. +//------------------------------------------------------------------------------ + template + std::array, 3> cartesian_to_sphereical(graph::shared_leaf x, + graph::shared_leaf y) { + auto w = graph::hypot(x, y); + return { + w, x/w, + graph::none ()*graph::copysign(static_cast (1), y) + }; + } + +//------------------------------------------------------------------------------ +/// @brief Convert from sphereical to cartesian coordinates. +/// +/// @param[in] w +/// @param[in] xi +/// @param[in] sinphi +/// @returns The coordinates as cartesian coordinates. +//------------------------------------------------------------------------------ + template + std::array, 2> sphereical_to_cartesian(graph::shared_leaf w, + graph::shared_leaf xi, + graph::shared_leaf sinphi) { + return { + w*xi, + graph::none ()*w*graph::sqrt(static_cast (1) - xi*xi)*sinphi + }; + } + //------------------------------------------------------------------------------ /// @brief Build initialization. /// diff --git a/graph_framework/trigonometry.hpp b/graph_framework/trigonometry.hpp index ead690a..c3ee845 100644 --- a/graph_framework/trigonometry.hpp +++ b/graph_framework/trigonometry.hpp @@ -843,7 +843,7 @@ namespace graph { return atan(l, constant (static_cast (r))); } -/// Convenience type alias for shared add nodes. +/// Convenience type alias for shared atan nodes. template using shared_atan = std::shared_ptr>; diff --git a/graph_tests/arithmetic_test.cpp b/graph_tests/arithmetic_test.cpp index 98e9c6e..bf95f27 100644 --- a/graph_tests/arithmetic_test.cpp +++ b/graph_tests/arithmetic_test.cpp @@ -1026,6 +1026,17 @@ template void test_subtract() { "Expected 3 on the left."); assert(constant_combine6_cast->get_right()->is_match(var_a) && "Expected a on the right."); + +// (a + b) - a -> b + assert(((var_a + var_b) - var_a)->is_match(var_b) && "Expected b."); +// (a + b) - b -> a + assert(((var_a + var_b) - var_b)->is_match(var_a) && "Expected a."); +// a - (a + b) -> -b + assert((var_a - (var_a + var_b))->is_match(graph::none ()*var_b) && + "Expected b."); +// b - (a + b) -> -a + assert((var_b - (var_a + var_b))->is_match(graph::none ()*var_a) && + "Expected a."); } //------------------------------------------------------------------------------ @@ -2036,6 +2047,55 @@ template void test_multiply() { v1, 3.0)*v2) && "Expected fma(fma(fma(50,x,4),x,3),x,3)*y"); + +// Sqrt(a)*Sqrt(b) -> Sqrt(a*b) + auto sqsq = graph::sqrt(v1)*graph::sqrt(v2); + assert(sqsq->is_match(graph::sqrt(v1*v2)) && "Expected Sqrt(a*b)"); + + auto v3 = graph::variable (1, "v3"); + if constexpr (std::floating_point) { +// hypot(b,c)*Sqrt(a) -> Sqrt((b^2 + c^2)*a) + auto hypotsq = graph::hypot(v1, v2)*graph::sqrt(v3); + assert(hypotsq->is_match(graph::sqrt((v1*v1 + v2*v2)*v3)) && + "Expected Sqrt((b^2 + c^2)*a)"); +// Sqrt(a)*hypot(b,c) -> Sqrt(a*(b^2 + c^2)) + auto sqhypot = graph::sqrt(v3)*graph::hypot(v1, v2); + assert(sqhypot->is_match(graph::sqrt((v1*v1 + v2*v2)*v3)) && + "Expected Sqrt((b^2 + c^2)*a)"); + +// Sqrt(x^2)*copysign(1,x) -> x + auto sqcs = graph::sqrt(v1*v1)*graph::copysign(static_cast (1), v1); + assert(sqcs->is_match(v1) && "Expected x"); +// copysign(1,x)*Sqrt(x^2) -> x + auto cssq = graph::copysign(static_cast (1), v1)*graph::sqrt(v1*v1); + assert(cssq->is_match(v1) && "Expected x"); + } + +// (a + b/c)*c -> fma(a,c,b) + auto result = (v1 + v2/v3)*v3; + assert(result->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// c*(a + b/c) -> fma(a,c,b) + auto result2 = v3*(v1 + v2/v3); + assert(result2->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// (b/c + a)*c -> fma(a,c,b) + auto result3 = (v2/v3 + v1)*v3; + assert(result3->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); +// c*(b/c + a) -> fma(a,c,b) + auto result4 = v3*(v2/v3 + v1); + assert(result4->is_match(graph::fma(v1, v3, v2)) && "Expected fma(a,c,b)"); + +// (a - b/c)*c -> a*c - b + auto result5 = (v1 - v2/v3)*v3; + assert(result5->is_match(v1*v3 - v2) && "Expected a*c - b"); +// c*(a - b/c) -> a*c - b + auto result6 = v3*(v1 - v2/v3); + assert(result6->is_match(v1*v3 - v2) && "Expected a*c - b"); +// (b/c - a)*c -> b - a*c + auto result7 = (v2/v3 - v1)*v3; + assert(result7->is_match(v2 - v1*v3) && "Expected b - a*c"); +// c*(b/c - a) -> b - a*c + auto result8 = v3*(v2/v3 - v1); + assert(result8->is_match(v2 - v1*v3) && "Expected b - a*c"); } //------------------------------------------------------------------------------ @@ -3895,6 +3955,38 @@ template void test_fma() { -49.0))) && "Expected fma(fma(fma(fma(2,x,20),x,30),x,50),b,fma(fma(fma(2,x,-19),-29),-49)"); */ + + +// fma(sqrt(a),sqrt(b),c) -> sqrt(a*b) + c + auto sqsq = graph::fma(graph::sqrt(var_a),graph::sqrt(var_b),var_c); + assert(sqsq->is_match(graph::sqrt(var_a*var_b) + var_c) && + "Expected Sqrt(a*b) + c"); + + if constexpr (std::floating_point) { +// fma(hypot(b,c),Sqrt(a),d) -> Sqrt((b^2 + c^2)*a) + d + auto hypotsq = graph::fma(graph::hypot(var_a,var_b),graph::sqrt(var_c),var_d); + assert(hypotsq->is_match(graph::sqrt((var_a*var_a + + var_b*var_b)*var_c) + var_d) && + "Expected Sqrt((b^2 + c^2)*a) + d"); +// fma(Sqrt(a),hypot(b,c),d) -> Sqrt(a*(b^2 + c^2)) + d + auto sqhypot = fma(graph::sqrt(var_c), + graph::hypot(var_a, var_b), + var_d); + assert(sqhypot->is_match(graph::sqrt((var_a*var_a + + var_b*var_b)*var_c) + var_d) && + "Expected Sqrt((b^2 + c^2)*a) + d"); + +// fma(Sqrt(x^2),copysign(1,x),y) -> x + y + auto sqcs = fma(graph::sqrt(var_a*var_a), + graph::copysign(static_cast (1), var_a), + var_b); + assert(sqcs->is_match(var_a + var_b) && "Expected x"); +// fma(copysign(1,x),Sqrt(x^2),y) -> x + y + auto cssq = fma(graph::copysign(static_cast (1), var_a), + graph::sqrt(var_a*var_a), + var_b); + assert(cssq->is_match(var_a + var_b) && "Expected x"); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/backend_test.cpp b/graph_tests/backend_test.cpp index 50bbabd..ea9f983 100644 --- a/graph_tests/backend_test.cpp +++ b/graph_tests/backend_test.cpp @@ -330,6 +330,77 @@ template void test_backend() { static_cast (1.0), static_cast (2.0) })); + avec.sin(); + assert(avec.size() == 2 && "Expected a size of 2"); + assert(avec.at(0) == std::sin(static_cast (1.0)) && + "Expected a value of sin(1)."); + assert(avec.at(1) == std::sin(static_cast (2.0)) && + "Expected a value of sin(2)."); + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + avec.cos(); + assert(avec.size() == 2 && "Expected a size of 2"); + assert(avec.at(0) == std::cos(static_cast (1.0)) && + "Expected a value of cos(1)."); + assert(avec.at(1) == std::cos(static_cast (2.0)) && + "Expected a value of cos(2)."); + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); + const backend::buffer arctanvec = backend::atan(avec, bvec); + assert(arctanvec.size() == 2 && "Expected a size of 2"); + if constexpr (jit::complex_scalar) { + assert(arctanvec.at(0) == std::atan(static_cast (3.0)/ + static_cast (1.0)) && + "Expected a value of atan(3/1)."); + assert(arctanvec.at(1) == std::atan(static_cast (4.0)/ + static_cast (2.0)) && + "Expected a value of atan(4/2)."); + } else { + assert(arctanvec.at(0) == std::atan2(static_cast (3.0), + static_cast (1.0)) && + "Expected a value of atan2(3,1)."); + assert(arctanvec.at(1) == std::atan2(static_cast (4.0), + static_cast (2.0)) && + "Expected a value of atan2(4,2)."); + } + + if constexpr (std::floating_point) { + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); + const backend::buffer hypotvec = backend::hypot(avec, bvec); + assert(hypotvec.size() == 2 && "Expected a size of 2"); + assert(hypotvec.at(0) == std::hypot(static_cast (1.0), + static_cast (3.0)) && + "Expected a value of hypot(1,3)."); + assert(hypotvec.at(1) == std::hypot(static_cast (2.0), + static_cast (4.0)) && + "Expected a value of hypot(2,4)."); + } + + avec.set(std::vector ({ + static_cast (1.0), + static_cast (2.0) + })); + bvec.set(std::vector ({ + static_cast (3.0), + static_cast (4.0) + })); const backend::buffer fma_vec_scale_scale = backend::fma(avec, bscalar, cscalar); assert(fma_vec_scale_scale.size() == 2 && "Expected a size of 2"); assert(fma_vec_scale_scale.at(0) == static_cast (-2.0) && @@ -564,6 +635,23 @@ template void test_backend() { static_cast (NAN) })); assert(!nan_vec.is_normal() && "Expected a NaN."); + + if constexpr (std::floating_point) { + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + bvec.set(std::vector ({ + static_cast (-3.0), + static_cast (0.30) + })); + const backend::buffer copysignvec = backend::copysign(avec, bvec); + assert(copysignvec.size() == 2 && "Expected a size of 2"); + assert(copysignvec.at(0) == static_cast (-4.0) && + "Expected a value of -4."); + assert(copysignvec.at(1) == static_cast (2.0) && + "Expected a value of 2."); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index e40f60a..f64edd2 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -373,6 +373,12 @@ template void run_math_tests() { graph::variable_cast(v1), graph::variable_cast(v2) }, {if_node}, {}, true_v->evaluate().at(0), 0.0); + + auto hypot_node = graph::hypot(v1, v2); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {hypot_node}, {}, hypot_node->evaluate().at(0), 0.0); } } diff --git a/graph_tests/math_test.cpp b/graph_tests/math_test.cpp index 3a0f65a..4af547f 100644 --- a/graph_tests/math_test.cpp +++ b/graph_tests/math_test.cpp @@ -504,6 +504,14 @@ void test_pow() { graph::pow(expr_a, 2.0) * graph::pow(expr_c, 2.0)) && "Expected b*c^2*d^2."); + +// hypot(a,b)^2 -> a^2 + b^2 + if constexpr (std::floating_point) { + assert((graph::pow(graph::hypot(var_a, var_b), + static_cast(2))->is_match(var_a*var_a + + var_b*var_b)) && + "Expected a^2 + b^2"); + } } //------------------------------------------------------------------------------ @@ -556,6 +564,101 @@ void test_erfi() { assert(!erfi->is_power_like() && "Did not expect a power like."); } +//------------------------------------------------------------------------------ +/// @brief Tests for hypot nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_hypot() { + auto a = graph::constant (0.5); + auto b = graph::constant (1.2); + + auto result = graph::hypot (a, b); + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant."); + assert(result_cast->is(std::hypot(static_cast (0.5), + static_cast (1.2))) && + "Expected hypot(0.5, 1.2)"); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + + assert(graph::hypot (v1, v2)->is_match(graph::hypot (v2, v1)) && + "Expected match."); + +// hypot(sqrt(a), sqrt(b)) -> sqrt(a + b) + auto result2 = graph::hypot(sqrt(v1), sqrt(v2)); + auto result2_cast = graph::sqrt_cast(result2); + assert(result2_cast.get() && "Expected a sqrt node."); + assert(result2->is_match(graph::sqrt(v1 + v2)) && + "Expected sqrt(a + b)."); + +// hypot(a, sqrt(b)) -> sqrt(a^2 + b) + auto result3 = graph::hypot(v1, sqrt(v2)); + auto result3_cast = graph::sqrt_cast(result3); + assert(result3_cast.get() && "Expected a sqrt node."); + assert(result3->is_match(graph::sqrt(v1*v1 + v2)) && + "Expected sqrt(a^2 + b)."); + +// hypot(sqrt(a), b) -> sqrt(a + b^2) + auto result4 = graph::hypot(sqrt(v1), v2); + auto result4_cast = graph::sqrt_cast(result4); + assert(result4_cast.get() && "Expected a sqrt node."); + assert(result4->is_match(graph::sqrt(v1 + v2*v2)) && + "Expected sqrt(a + b^2)."); + +// hypoy(a,a) -> sqrt(2)sqrt(a^2) + auto result5 = graph::hypot(v1, v1); + auto result5_cast = graph::multiply_cast(result5); + assert(result5_cast.get() && "Expected a multiply node."); + assert(result5->is_match(std::numbers::sqrt2_v*graph::sqrt(v1*v1)) && + "Expected sqrt(2)sqrt(a^2)."); + +// d hypoy(a,b)/dx -> 0 + auto result6 = graph::hypot(v1, v2)->df(a); + auto result6_cast = graph::constant_cast(result6); + assert(result6_cast.get() && "Expected a constant."); + assert(result6_cast->is(0) && "Expected zero"); + +// d hypoy(a,b)/da -> 0 + auto result7 = graph::hypot(v1, v2)->df(v1); + auto result7_cast = graph::divide_cast(result7); + assert(result7_cast.get() && "Expected a divide node."); + assert(result7_cast->is_match(v1/graph::hypot(v1, v2)) && + "v1/hypot(v1, v2)"); + +// d hypoy(a,b)/db -> 0 + auto result8 = graph::hypot(v1, v2)->df(v2); + auto result8_cast = graph::divide_cast(result8); + assert(result8_cast.get() && "Expected a divide node."); + assert(result8_cast->is_match(v2/graph::hypot(v1, v2)) && + "v2/hypot(v1, v2)"); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for copysign nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_copysign() { + auto a = graph::constant (0.5); + auto b = graph::constant (-1.2); + + auto result = graph::copysign (a, b); + auto result_cast = graph::constant_cast(result); + assert(result_cast.get() && "Expected a constant."); + assert(result_cast->is(-0.5) && "Expected -0.5"); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + + auto result2 = graph::copysign(v1, v2); + auto result2_cast = graph::copysign_cast(result2); + assert(result2_cast.get() && "Expected a copysign node."); +} + //------------------------------------------------------------------------------ /// @brief Tests function for variable like expressions. /// @@ -593,6 +696,9 @@ template void run_tests() { if constexpr (jit::complex_scalar) { test_erfi (); } + if constexpr (std::floating_point) { + test_hypot (); + } } //------------------------------------------------------------------------------ diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 871ac0a..5c6847c 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -299,6 +299,22 @@ template void run_field_solve_test() { } } +//------------------------------------------------------------------------------ +/// @brief Coordinate tests. +//------------------------------------------------------------------------------ +template void run_coord_tests() { + auto x = graph::variable (1, "x"); + auto y = graph::variable (1, "y"); + + std::array, 3> sphere = pic::cartesian_to_sphereical(x, y); + std::array, 2> cart = pic::sphereical_to_cartesian(sphere[0], + sphere[1], + sphere[2]); + + assert(cart[0]->is_match(x) && "x not converted correctly."); + assert(cart[1]->is_match(y) && "y not converted correctly."); +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified precision. /// @@ -307,6 +323,7 @@ template void run_field_solve_test() { template void run_tests() { run_interpolation_test (); run_field_solve_test (); + run_coord_tests (); } //------------------------------------------------------------------------------ From 97d46787e5203a0f2da8f0969b7a8902f946f7a2 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Wed, 16 Sep 2026 16:03:02 -0400 Subject: [PATCH 45/51] Add collision operators to pic code. To accomplish this, needed to add min and erf nodes. Refactor precompile methods to allow a unquie function to bedefined before the kernels. Add debug checks for duplicate nodes in kernel arguments. --- graph_c_binding/graph_c_binding.cpp | 2 +- graph_docs/discription.dox | 6 +- graph_framework.xcodeproj/project.pbxproj | 2 + graph_framework/backend.hpp | 140 +++- graph_framework/cpu_context.hpp | 21 +- graph_framework/cuda_context.hpp | 4 +- graph_framework/jit.hpp | 18 +- graph_framework/logical.hpp | 345 +++++++++- graph_framework/math.hpp | 252 ++++++- graph_framework/metal_context.hpp | 18 +- graph_framework/node.hpp | 76 ++- graph_framework/particle_in_cell.hpp | 778 +++++++++++++++------- graph_framework/piecewise.hpp | 16 +- graph_framework/random.hpp | 24 +- graph_framework/register.hpp | 16 +- graph_framework/workflow.hpp | 3 +- graph_pic/xpic.cpp | 44 ++ graph_tests/backend_test.cpp | 74 +- graph_tests/jit_test.cpp | 6 + graph_tests/logical_test.cpp | 37 + graph_tests/math_test.cpp | 30 +- graph_tests/no_derivative_test.cpp | 26 +- 22 files changed, 1592 insertions(+), 346 deletions(-) diff --git a/graph_c_binding/graph_c_binding.cpp b/graph_c_binding/graph_c_binding.cpp index 826ded2..f1c508d 100644 --- a/graph_c_binding/graph_c_binding.cpp +++ b/graph_c_binding/graph_c_binding.cpp @@ -19,7 +19,7 @@ template struct graph_c_context_type : public graph_c_context { /// Variables nodes. - std::map> nodes; + std::unordered_map> nodes; /// Workflow manager. workflow::manager work; diff --git a/graph_docs/discription.dox b/graph_docs/discription.dox index 1f48961..9ff9378 100644 --- a/graph_docs/discription.dox +++ b/graph_docs/discription.dox @@ -39,9 +39,9 @@ * expression nodes. The factory method checks a node_cache to avoid building * duplicate sub-graphs. Identification of duplicate graphs is performed by * computing a hash of the sub-graph. This hash can be rapidly checked if the - * same hash already exists in a std::map container. If the sub-graph - * already exists, the existing graph is returned otherwise a new sub-graph is - * registered in the node_cache. + * same hash already exists in a std::unordered_map container. If the + * sub-graph already exists, the existing graph is returned otherwise a new + * sub-graph is registered in the node_cache. * * Each time an expression is built, the reduce method is called to simplify the * graph. For instance, a graph consisting of constant added to a constant will diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index cb6fe6a..d3a76b0 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -2324,6 +2324,7 @@ "-lclangParse", "-lclangAPINotes", "-lclangOptions", + "-lclangCodeGenUtils", "-lclangCodeGen", "-rpath", /usr/local/lib, @@ -2501,6 +2502,7 @@ "-lclangParse", "-lclangAPINotes", "-lclangOptions", + "-lclangCodeGenUtils", "-lclangCodeGen", "-rpath", /usr/local/lib, diff --git a/graph_framework/backend.hpp b/graph_framework/backend.hpp index 151dd4e..9246802 100644 --- a/graph_framework/backend.hpp +++ b/graph_framework/backend.hpp @@ -245,6 +245,13 @@ for (T &d : *this) { \ apply_op(std::real) } +//------------------------------------------------------------------------------ +/// @brief Take erf. +//------------------------------------------------------------------------------ + void erf() requires(std::floating_point) { + apply_op(std::erf) + } + //------------------------------------------------------------------------------ /// @brief Take erfi. //------------------------------------------------------------------------------ @@ -306,7 +313,7 @@ for (T &d : *this) { \ } //------------------------------------------------------------------------------ -/// @brief Applies an operatator along a row. +/// @brief Applies an operator along a row. /// /// @param opp The operation to apply. /// @param oppeq The assignment operator to apply. @@ -351,7 +358,7 @@ if (size() > x.size()) { \ } //------------------------------------------------------------------------------ -/// @brief Applies an operatator along a column. +/// @brief Applies an operator along a column. /// /// @param opp The operation to apply. /// @param oppeq The assignment operator to apply. @@ -467,6 +474,94 @@ if (size() > x.size()) { \ col_op(/, /=) } +//------------------------------------------------------------------------------ +/// @brief Applies a function along a row. +/// +/// @param fn The function to apply. +//------------------------------------------------------------------------------ + #define row_fn(fn) \ + if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] = fn((*this)[i*num_columns + j], x[i]); \ + } \ + } \ + } else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = fn((*this)[i], x[i*num_columns + j]); \ + } \ + } \ + *this = m; \ + } + +//------------------------------------------------------------------------------ +/// @brief Min row operation. +/// +/// Takes Min(m_ij, v_i) or Min(v_i, m_ij). This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void min_row(const buffer &x) { + row_fn(std::min) + } + +//------------------------------------------------------------------------------ +/// @brief Applies a function along a column. +/// +/// @param fn The function to apply. +//------------------------------------------------------------------------------ + #define col_fn(fn) \ + if (size() > x.size()) { \ + assert(size()%x.size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + const size_t num_columns = size()/x.size(); \ + const size_t num_rows = x.size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + (*this)[i*num_columns + j] = fn((*this)[i*num_columns + j], x[j]); \ + } \ + } \ + } else { \ + assert(x.size()%size() == 0 && \ + "Vector operand size is not a multiple of matrix operand size"); \ + \ + std::vector m(x.size()); \ + const size_t num_columns = x.size()/size(); \ + const size_t num_rows = size(); \ + for (size_t i = 0; i < num_rows; i++) { \ + for (size_t j = 0; j < num_columns; j++) { \ + m[i*num_columns + j] = fn((*this)[j], x[i*num_columns + j]); \ + } \ + } \ + *this = m; \ + } + +//------------------------------------------------------------------------------ +/// @brief Min col operation. +/// +/// Takes Min(m_ij, v_j) or Min(v_j, m_ij). This will resize the buffer if it +/// needs to be. +/// +/// @param[in] x The other operand. +//------------------------------------------------------------------------------ + void min_col(const buffer &x) { + col_fn(std::min) + } + //------------------------------------------------------------------------------ /// @brief Atan row operation. /// @@ -1388,6 +1483,47 @@ return a; logic_op(||) } +//------------------------------------------------------------------------------ +/// @brief Applies a function with two operands. +/// +/// @param op The operation to apply. +//------------------------------------------------------------------------------ +#define branch_fn(fn) \ +if (b.size() == 1) { \ + const T right = b[0]; \ + for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = fn(a[i], right); \ + } \ + return a; \ +} else if (a.size() == 1) { \ + const T left = a[0]; \ + for (size_t i = 0, ie = b.size(); i < ie; i++) { \ + b[i] = fn(left, b[i]); \ + } \ + return b; \ +} \ + \ +assert(a.size() == b.size() && \ + "Left and right sizes are incompatible."); \ +for (size_t i = 0, ie = a.size(); i < ie; i++) { \ + a[i] = fn(a[i], b[i]); \ +} \ +return a; + +//------------------------------------------------------------------------------ +/// @brief Min operation. +/// +/// @tparam T Base type of the calculation. +/// +/// @param[in] a Numerator. +/// @param[in] b Denominator. +/// @returns min(a, b). +//------------------------------------------------------------------------------ + template + inline buffer min(buffer &a, buffer &b) { + branch_fn(std::min) + } + //------------------------------------------------------------------------------ /// @brief Take the power. /// diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index 94bef50..d776d9c 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -85,11 +85,11 @@ namespace gpu { /// Handle for the dynamic library. std::unique_ptr jit; /// Argument map. - std::map *, std::vector> kernel_arguments; + std::unordered_map *, std::vector> kernel_arguments; /// Host buffer map. - std::map *, std::vector> host_buffers; + std::unordered_map *, std::vector> host_buffers; /// Argument index map. - std::map *, size_t> arg_index; + std::unordered_map *, size_t> arg_index; public: /// Size of random state needed. @@ -242,7 +242,7 @@ namespace gpu { const jit::texture2d_list &tex2d_list) { auto entry = std::move(jit->lookup(kernel_name)).get(); - std::map buffers; + std::unordered_map buffers; for (auto &input : inputs) { if (!kernel_arguments.contains(input.get())) { @@ -271,7 +271,7 @@ namespace gpu { } if (state.get()) { - auto kernel = entry.toPtr &, typename graph::random_state_node::mt_state *)> (); + auto kernel = entry.toPtr &, typename graph::random_state_node::mt_state *)> (); if (!kernel) { std::cerr << "Failed to load function. " << kernel_name @@ -299,7 +299,7 @@ namespace gpu { #endif }; } else { - auto kernel = entry.toPtr &)> (); + auto kernel = entry.toPtr &)> (); if (!kernel) { std::cerr << "Failed to load function. " << kernel_name @@ -523,9 +523,10 @@ namespace gpu { /// @param[in,out] source_buffer Source buffer stream. //------------------------------------------------------------------------------ void create_header(std::ostringstream &source_buffer) { - source_buffer << "#include " << std::endl - << "#include " << std::endl - << "#include " << std::endl; + source_buffer << "#include " << std::endl + << "#include " << std::endl + << "#include " << std::endl + << "#include " << std::endl; if (jit::complex_scalar) { source_buffer << "#include " << std::endl; source_buffer << "#include " << std::endl; @@ -573,7 +574,7 @@ namespace gpu { source_buffer << std::endl; source_buffer << "extern \"C\" void " << name << "(" << std::endl; - source_buffer << " map (source_buffer); source_buffer << " *> &args"; if (state.get()) { diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index cb7b19e..6475b2e 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -80,10 +80,10 @@ namespace gpu { /// The cuda code library. CUmodule module; /// Argument map. - std::map *, CUdeviceptr> kernel_arguments; + std::unordered_map *, CUdeviceptr> kernel_arguments; #ifdef USE_CUDA_TEXTURES /// Textures. - std::map texture_arguments; + std::unordered_map texture_arguments; #endif /// Result buffer. CUdeviceptr result_buffer; diff --git a/graph_framework/jit.hpp b/graph_framework/jit.hpp index cbdde38..1338f25 100644 --- a/graph_framework/jit.hpp +++ b/graph_framework/jit.hpp @@ -52,12 +52,14 @@ namespace jit { std::ostringstream source_buffer; /// Nodes that have been jitted. register_map registers; +/// Prefunctions that have been defined. + preamble_map pre_funcs; /// Kernel names. std::vector kernel_names; /// Kernel textures. - std::map kernel_1dtextures; + std::unordered_map kernel_1dtextures; /// Kernel textures. - std::map kernel_2dtextures; + std::unordered_map kernel_2dtextures; /// Type for the GPU context. using gpu_context_type = typename std::conditional (), @@ -72,8 +74,6 @@ namespace jit { /// GPU Context. gpu_context_type gpu_context; -/// Used random. - bool used_random; public: /// Size of random state needed. @@ -108,7 +108,7 @@ namespace jit { /// /// @param[in] index Concurrent index. Not used. //------------------------------------------------------------------------------ - context(const size_t index) : gpu_context(index), used_random(false) { + context(const size_t index) : gpu_context(index) { source_buffer << std::setprecision(max_digits10 ()); gpu_context.create_header(source_buffer); } @@ -137,12 +137,6 @@ namespace jit { const size_t iterations=1) { kernel_names.push_back(name); - if (state.get() && !used_random) { - used_random = true; - graph::random_state_node::compile_random_state(source_buffer); - graph::random_node::compile_random(source_buffer); - } - std::vector is_constant(inputs.size(), true); visiter_map visited; register_usage usage; @@ -159,6 +153,7 @@ namespace jit { visited, usage, kernel_1dtextures[name], kernel_2dtextures[name], + pre_funcs, gpu_context.remaining_const_memory); } for (auto &out : outputs) { @@ -166,6 +161,7 @@ namespace jit { visited, usage, kernel_1dtextures[name], kernel_2dtextures[name], + pre_funcs, gpu_context.remaining_const_memory); } diff --git a/graph_framework/logical.hpp b/graph_framework/logical.hpp index 90b717c..97cc7af 100644 --- a/graph_framework/logical.hpp +++ b/graph_framework/logical.hpp @@ -2770,10 +2770,10 @@ namespace graph { static std::string to_string(leaf_node *c, leaf_node *t, leaf_node *f) { - return "if(" + - jit::format_to_string(reinterpret_cast (c)) + "," + - jit::format_to_string(reinterpret_cast (t)) + "," + - jit::format_to_string(reinterpret_cast (f)) + ")"; + return "if" + + jit::format_to_string(reinterpret_cast (c)) + + jit::format_to_string(reinterpret_cast (t)) + + jit::format_to_string(reinterpret_cast (f)); } public: @@ -2957,6 +2957,7 @@ namespace graph { /// @param[in] c Condition branch. /// @param[in] t True branch. /// @param[in] f False branch. +/// @returns A reduced if node. //------------------------------------------------------------------------------ template shared_leaf if_(shared_leaf c, @@ -2998,6 +2999,342 @@ namespace graph { shared_if if_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } + +//****************************************************************************** +// Min node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief A Min node. +/// +/// Note use templates here to defer this so it can use the operator functions. +/// +/// @tparam T Base type of the operands. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +//------------------------------------------------------------------------------ + template + class min_node final : public branch_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *l, + leaf_node *r) { + return "min" + + jit::format_to_string(reinterpret_cast (l)) + + jit::format_to_string(reinterpret_cast (r)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct an equal node. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +//------------------------------------------------------------------------------ + min_node(shared_leaf l, + shared_leaf r) : + branch_node (l, r, + min_node::to_string(l.get(), + r.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of if. +/// +/// result = min(l, r) +/// +/// @returns The value of min(l, r). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer l_result = this->left->evaluate(); + backend::buffer r_result = this->right->evaluate(); + return backend::min(l_result, r_result); + } + +//------------------------------------------------------------------------------ +/// @brief Reduce a min node. +/// +/// @returns A reduced equal node. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { +// Constant reductions. + auto lc = constant_cast(this->left); + auto rc = constant_cast(this->right); + + if (lc.get() && rc.get()) { + return constant (this->evaluate()); + } + + auto pl1 = piecewise_1D_cast(this->left); + auto pr1 = piecewise_1D_cast(this->right); + if (pl1.get() && (rc.get() || pl1->is_arg_match(this->right))) { + return piecewise_1D(this->evaluate(), pl1->get_arg()); + } else if (pr1.get() && (lc.get() || pr1->is_arg_match(this->left))) { + return piecewise_1D(this->evaluate(), pr1->get_arg()); + } + + auto pl2 = piecewise_2D_cast(this->left); + auto pr2 = piecewise_2D_cast(this->right); + if (pl2.get() && (rc.get() || pl2->is_arg_match(this->right))) { + return piecewise_2D(this->evaluate(), + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pr2.get() && (lc.get() || pr2->is_arg_match(this->left))) { + return piecewise_2D(this->evaluate(), + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } + +// Combine 2D and 1D piecewise constants if a row or column matches. + if (pr2.get() && pr2->is_row_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.min_row(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pr2.get() && pr2->is_col_match(this->left)) { + backend::buffer result = pl1->evaluate(); + result.min_col(pr2->evaluate()); + return piecewise_2D(result, + pr2->get_num_columns(), + pr2->get_left(), + pr2->get_right()); + } else if (pl2.get() && pl2->is_row_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.min_row(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } else if (pl2.get() && pl2->is_col_match(this->right)) { + backend::buffer result = pl2->evaluate(); + result.min_col(pr1->evaluate()); + return piecewise_2D(result, + pl2->get_num_columns(), + pl2->get_left(), + pl2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d min(l,r)/dx = min(dl/dx,dr/dx) +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf + df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = min (this->left->df(x), + this->right->df(x)); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto l = this->left->compile(stream, registers, + thread_mem, usage); + auto r = this->left->compile(stream, registers, + thread_mem, usage); + registers[this] = jit::to_string('r', this); + + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = min(" + << registers[l.get()] << ", " + << registers[r.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = min_cast(x); + if (x_cast.get()) { +// Min is commutative. + return (this->left->is_match(x_cast->get_left()) && + this->right->is_match(x_cast->get_right())) || + (this->right->is_match(x_cast->get_left()) && + this->left->is_match(x_cast->get_right())); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "min\\left("; + this->left->to_latex(); + std::cout << ","; + this->right->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return min (this->left->remove_pseudo(), + this->right->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"min\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto l = this->left->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[l.get()] << ";" << std::endl; + auto r = this->right->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[r.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Build a max node from a condition and two leaves. +/// +/// Note use templates here to defer this so it can be used in the above +/// classes. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(shared_leaf l, + shared_leaf r) { + auto temp = std::make_shared> (l, r)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +//------------------------------------------------------------------------------ +/// @brief Build min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam L Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(const L l, + shared_leaf r) { + return min(constant (static_cast (l)), r); + } + +//------------------------------------------------------------------------------ +/// @brief Build min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam R Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] l Left branch. +/// @param[in] r Right branch. +/// @returns A reduced min node. +//------------------------------------------------------------------------------ + template + shared_leaf min(shared_leaf l, + const R r) { + return min(l, constant (static_cast (r))); + } + +/// Convenience type alias for shared min nodes. + template + using shared_min = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to an min node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_min min_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } } #endif /* logical_h */ diff --git a/graph_framework/math.hpp b/graph_framework/math.hpp index f2ab1b7..5f9651f 100644 --- a/graph_framework/math.hpp +++ b/graph_framework/math.hpp @@ -1449,6 +1449,241 @@ namespace graph { return std::dynamic_pointer_cast> (x); } +//****************************************************************************** +// Erf node. +//****************************************************************************** +//------------------------------------------------------------------------------ +/// @brief An error function node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// Note use templates here to defer this so it can use the operator functions. +//------------------------------------------------------------------------------ + template + class erf_node final : public straight_node { + private: +//------------------------------------------------------------------------------ +/// @brief Convert node pointer to a string. +/// +/// @param[in] a Argument node pointer. +/// @return A string rep of the node. +//------------------------------------------------------------------------------ + static std::string to_string(leaf_node *a) { + return "erf" + jit::format_to_string(reinterpret_cast (a)); + } + + public: +//------------------------------------------------------------------------------ +/// @brief Construct a erf node. +/// +/// @param[in] x Argument. +//------------------------------------------------------------------------------ + erf_node(shared_leaf x) : + straight_node (x, erf_node::to_string(x.get())) {} + +//------------------------------------------------------------------------------ +/// @brief Evaluate the results of erf. +/// +/// result = erf(x) +/// +/// @returns The value of erf(x). +//------------------------------------------------------------------------------ + virtual backend::buffer evaluate() { + backend::buffer result = this->arg->evaluate(); + result.erf(); + return result; + } + +//------------------------------------------------------------------------------ +/// @brief Reduce the erf(x). +/// +/// @returns Reduced graph from erf. +//------------------------------------------------------------------------------ + virtual shared_leaf reduce() { + if (constant_cast(this->arg).get()) { + return constant (this->evaluate()); + } + + auto ap1 = piecewise_1D_cast(this->arg); + if (ap1.get()) { + return piecewise_1D(this->evaluate(), + ap1->get_arg()); + } + + auto ap2 = piecewise_2D_cast(this->arg); + if (ap2.get()) { + return piecewise_2D(this->evaluate(), + ap2->get_num_columns(), + ap2->get_left(), + ap2->get_right()); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Transform node to derivative. +/// +/// d erf(y)/dx = 2/sqrt(pi)Exp(-y^2)*dy/dx +/// +/// @param[in] x The variable to take the derivative to. +/// @returns The derivative of the node. +//------------------------------------------------------------------------------ + virtual shared_leaf df(shared_leaf x) { + if (this->is_match(x)) { + return one (); + } + + const size_t hash = reinterpret_cast (x.get()); + if (this->df_cache.find(hash) == this->df_cache.end()) { + this->df_cache[hash] = static_cast (2) + * std::numbers::inv_sqrtpi_v + * exp(this->arg*this->arg)*this->arg->df(x); + } + return this->df_cache[hash]; + } + +//------------------------------------------------------------------------------ +/// @brief Compile the node. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @param[in] thread_mem List of defined thread memory registers. +/// @param[in] usage List of register usage count. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf + compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { + if (registers.find(this) == registers.end()) { + auto a = this->arg->compile(stream, registers, + thread_mem, usage); + + registers[this] = jit::to_string('r', this); + stream << " const "; + jit::add_type (stream); + stream << " " << registers[this] << " = erf(" + << registers[a.get()] << ")"; + this->endline(stream, usage); + } + + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Query if the nodes match. +/// +/// @param[in] x Other graph to check if it is a match. +/// @returns True if the nodes are a match. +//------------------------------------------------------------------------------ + virtual bool is_match(shared_leaf x) { + if (this == x.get()) { + return true; + } + + auto x_cast = erf_cast(x); + if (x_cast.get()) { + return this->arg->is_match(x_cast->get_arg()); + } + + return false; + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to latex. +//------------------------------------------------------------------------------ + virtual void to_latex() const { + std::cout << "erf\\left("; + this->arg->to_latex(); + std::cout << "\\right)"; + } + +//------------------------------------------------------------------------------ +/// @brief Remove pseudo variable nodes. +/// +/// @returns A tree without variable nodes. +//------------------------------------------------------------------------------ + virtual shared_leaf remove_pseudo() { + if (this->has_pseudo()) { + return erf(this->arg->remove_pseudo()); + } + return this->shared_from_this(); + } + +//------------------------------------------------------------------------------ +/// @brief Convert the node to vizgraph. +/// +/// @param[in,out] stream String buffer stream. +/// @param[in,out] registers List of defined registers. +/// @returns The current node. +//------------------------------------------------------------------------------ + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { + if (registers.find(this) == registers.end()) { + const std::string name = jit::to_string('r', this); + registers[this] = name; + stream << " " << name + << " [label = \"erf\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + + auto a = this->arg->to_vizgraph(stream, registers); + stream << " " << name << " -- " << registers[a.get()] << ";" << std::endl; + } + + return this->shared_from_this(); + } + }; + +//------------------------------------------------------------------------------ +/// @brief Define erf convenience function. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Argument. +/// @returns A reduced exp node. +//------------------------------------------------------------------------------ + template + shared_leaf erf(shared_leaf x) { + auto temp = std::make_shared> (x)->reduce(); +// Test for hash collisions. + for (size_t i = temp->get_hash(); + i < std::numeric_limits::max(); i++) { + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; + return temp; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; + } + } +#if defined(__clang__) || defined(__GNUC__) + __builtin_unreachable(); +#else + assert(false && "Should never reach."); +#endif + } + +/// Convenience type alias for shared erf nodes. + template + using shared_erf = std::shared_ptr>; + +//------------------------------------------------------------------------------ +/// @brief Cast to a erf node. +/// +/// @tparam T Base type of the calculation. +/// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. +/// +/// @param[in] x Leaf node to attempt cast. +/// @returns An attempted dynamic cast. +//------------------------------------------------------------------------------ + template + shared_erf erf_cast(shared_leaf x) { + return std::dynamic_pointer_cast> (x); + } + //****************************************************************************** // Erfi node. //****************************************************************************** @@ -1475,7 +1710,7 @@ namespace graph { public: //------------------------------------------------------------------------------ -/// @brief Construct a exp node. +/// @brief Construct a erfi node. /// /// @param[in] x Argument. //------------------------------------------------------------------------------ @@ -1498,7 +1733,7 @@ namespace graph { //------------------------------------------------------------------------------ /// @brief Reduce the erfi(x). /// -/// @returns Reduced graph from exp. +/// @returns Reduced graph from erfi. //------------------------------------------------------------------------------ virtual shared_leaf reduce() { if (constant_cast(this->arg).get()) { @@ -1537,7 +1772,14 @@ namespace graph { const size_t hash = reinterpret_cast (x.get()); if (this->df_cache.find(hash) == this->df_cache.end()) { - this->df_cache[hash] = 2.0/std::sqrt(M_PI) + T invsqpi; + if constexpr(std::same_as>) { + invsqpi = std::numbers::inv_sqrtpi_v; + } else { + invsqpi = std::numbers::inv_sqrtpi_v; + } + this->df_cache[hash] = static_cast (2) + * invsqpi * exp(this->arg*this->arg)*this->arg->df(x); } return this->df_cache[hash]; @@ -1665,12 +1907,12 @@ namespace graph { #endif } -/// Convenience type alias for shared exp nodes. +/// Convenience type alias for shared erfi nodes. template using shared_erfi = std::shared_ptr>; //------------------------------------------------------------------------------ -/// @brief Cast to a exp node. +/// @brief Cast to a erfi node. /// /// @tparam T Base type of the calculation. /// @tparam SAFE_MATH Use @ref general_concepts_safe_math operations. diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 2571b02..7956c3c 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -29,15 +29,15 @@ namespace gpu { /// The metal command queue. id queue; /// Argument map. - std::map *, id> kernel_arguments; + std::unordered_map *, id> kernel_arguments; /// Textures. - std::map> texture_arguments; + std::unordered_map> texture_arguments; /// Metal command buffer. id command_buffer; /// Metal library. id library; /// Buffer mutability descriptor. - std::map> bufferMutability; + std::unordered_map> bufferMutability; public: /// Random state size multiplier. @@ -151,7 +151,7 @@ namespace gpu { } std::vector> buffers; - std::set *> needed_buffers; + std::unordered_set *> needed_buffers; const size_t buffer_element_size = sizeof(float); for (graph::shared_variable &input : inputs) { @@ -672,6 +672,9 @@ namespace gpu { used_args.insert(inputs[i].get()); } } + assert(used_args.size() == inputs.size() && + "Kernel inputs contain duplicates."); + for (size_t i = 0, ie = outputs.size(); i < ie; i++) { if (!used_args.contains(outputs[i].get()) && !graph::atomic_accumulate_1D_cast(outputs[i]).get()) { @@ -683,6 +686,9 @@ namespace gpu { used_args.insert(outputs[i].get()); } } + assert(used_args.size() == inputs.size() + outputs.size() && + "Kernel outputs contain duplicates."); + for (size_t i = 0, ie = atomics.size(); i < ie; i++) { if (!used_args.contains(atomics[i].get())) { bufferMutability[name].push_back(MTLMutabilityMutable); @@ -693,6 +699,10 @@ namespace gpu { used_args.insert(atomics[i].get()); } } + assert(used_args.size() == inputs.size() + outputs.size() + + atomics.size() && + "Kernel atomics contain duplicates."); + if (state.get()) { bufferMutability[name].push_back(MTLMutabilityMutable); source_buffer << " device mt_state *" diff --git a/graph_framework/node.hpp b/graph_framework/node.hpp index 63b6d27..d085dad 100644 --- a/graph_framework/node.hpp +++ b/graph_framework/node.hpp @@ -194,20 +194,15 @@ /// jit::register_usage &usage, /// jit::texture1d_list &textures1d, /// jit::texture2d_list &textures2d, +/// jit::preamble_map &pre_funcs, /// int &avail_const_mem) { -/// if (visited.find(this) == visited.end()) { +/// if (!visited.contains(this)) { /// this->arg->compile_preamble(stream, registers, /// visited, usage, /// textures1d, textures2d, +/// pre_funcs, /// avail_const_mem); /// -/// jit::add_type (stream); -/// stream << " foo(const " -/// jit::add_type (stream); -/// stream << "x) {" -/// << " return 2*x;" -/// << "}"; -/// /// visited.insert(this); /// #ifdef SHOW_USE_COUNT /// usage[this] = 1; @@ -215,6 +210,17 @@ /// ++usage[this]; /// #endif /// } +/// +/// if (!pre_funcs.contains("foo")) { +/// visited.insert("foo"); +/// +/// jit::add_type (stream); +/// stream << " foo(const " +/// jit::add_type (stream); +/// stream << "x) {" << std::endl +/// << " return 2*x;" << std::endl +/// << "}" << std::endl; +/// } /// } /// @endcode /// The compile methods generate kernel source code. In this case we created a @@ -369,7 +375,7 @@ namespace graph { /// Graph complexity. const size_t complexity; /// Cache derivative terms. - std::map>> df_cache; + std::unordered_map>> df_cache; /// Node contains pseudo variables. const bool contains_pseudo; @@ -433,6 +439,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -441,6 +448,7 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { #ifdef SHOW_USE_COUNT if (usage.find(this) == usage.end()) { @@ -666,9 +674,9 @@ namespace graph { //------------------------------------------------------------------------------ struct caches_t { /// Cache of node. - std::map>> nodes; + std::unordered_map>> nodes; /// Cache of backend buffers. - std::map> backends; + std::unordered_map> backends; }; /// A per thread instance of the cache structure. @@ -1243,6 +1251,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1251,12 +1260,13 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { arg->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1373,6 +1383,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1381,16 +1392,17 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { left->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); right->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1488,6 +1500,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1496,20 +1509,21 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - this->left->compile_preamble(stream, registers, + if (!visited.contains(this)) { + this->left->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->middle->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->right->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -1611,6 +1625,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1619,13 +1634,14 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { for (auto &b : branches) { b->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); } visited.insert(this); @@ -1661,6 +1677,16 @@ namespace graph { } return true; } + +//------------------------------------------------------------------------------ +/// @brief Get the exponent of a power. +/// +/// @returns Returns a power of one. +//------------------------------------------------------------------------------ + virtual std::shared_ptr> + get_power_exponent() const { + return one (); + } }; //------------------------------------------------------------------------------ @@ -1752,7 +1778,7 @@ namespace graph { /// @param[in] b Array of branches. /// @param[in] s Node string to hash. //------------------------------------------------------------------------------ - no_derivative(std::array, N> &b, + no_derivative(std::array, N> b, const std::string s) requires(std::is_base_of_v, no_derivative +#include #include "piecewise.hpp" #include "workflow.hpp" #include "random.hpp" #include "logical.hpp" -namespace pic { -// FIXME: This should be in a separate file of physics constants. -/// Speed of light m/s. - template - constexpr T c = static_cast (299792458.0); -/// Vacuum permitivity F/m. - template - constexpr T epsilon0 = static_cast (8.8541878188E-12); -/// Fundamental charge coulombs. - template - constexpr T q = static_cast (1.602176634E-19); -/// Hydrogen mass kg. - template - constexpr T m_hydrogen = static_cast (1.67362192595E-27); -/// Atomic mass - template - constexpr T m_atomic = static_cast (1.66053906892E-27); -/// Electron mass kg. - template - constexpr T m_electron = static_cast (9.1093837139E-31); -/// Boltzman constant. - template - constexpr T kb = static_cast (1.380649E-23); - -//------------------------------------------------------------------------------ -/// @brief Characteristic factors. -/// -/// @tparam T Base type of the calculation. -//------------------------------------------------------------------------------ - template - class characteristics { - private: -//------------------------------------------------------------------------------ -/// @brief Compute the characteristic mass. -/// -/// @param[in] ion_masses Ion masses. -/// @returns (∑(m_i) + me)/(n_i + 1); -//------------------------------------------------------------------------------ - T make_m(const std::vector &ion_masses) { - T total_m = static_cast (0); - for (const T &mass : ion_masses) { - total_m += mass; - } - return total_m/ion_masses.size(); - } - -//------------------------------------------------------------------------------ -/// @brief Compute the characteristic mass. -/// -/// @param[in] ion_zs Ion Z. -/// @returns (∑(Z_i)*q + q)/(n_i + 1); -//------------------------------------------------------------------------------ - T make_q(const std::vector &ion_zs) { - T total_q = static_cast (0); - for (const uint8_t &z : ion_zs) { - total_q += z*pic::q; - } - return total_q/ion_zs.size(); - } - - public: -/// Mass - const T m; -/// Charge - const T q; -/// Electron density. - const T ne; -/// Plasma Frequency. - const T wpe; -/// Time. - const T t; -/// Length - const T l; -/// Velocity - const T v; -/// Electron temperature; - const T te; -/// Electric field; - const T efield; -/// Magnetic field; - const T bfield; - -//------------------------------------------------------------------------------ -/// @brief Construct the characteristics. -/// -/// @param[in] ion_masses Ion masses for all species. -/// @param[in] ion_zs Ion Z effective all species. -/// @param[in] ne Characteristic density. -//------------------------------------------------------------------------------ - characteristics(const std::vector &ion_masses, - const std::vector &ion_zs, - const T ne) : - m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), - wpe(std::sqrt(ne*q*q/(m*epsilon0))), - t(1/wpe), l(c/wpe), v(c), te(m*v*v/kb), efield(m*c/(q*t)), - bfield(efield/c) {} - }; - -//------------------------------------------------------------------------------ -/// @brief Parameter Class -//------------------------------------------------------------------------------ - template - class parameters { - public: -/// Initial magnetic field - const T b0; -/// Geometry - const T a0; -/// Filter Iterations. - const size_t filter_iterations; -/// Smoothing parameters. - const T smoothing; -/// Time step. - const T dt; -/// Parallel temperature. - const T t_para; -/// Perpendicular temperature. - const T t_perp; - -//------------------------------------------------------------------------------ -/// @brief Construct a parameters object. -/// -/// @param[in] b0 Initial magnetic field. -/// @param[in] r1 -/// @param[in] r2 -/// @param[in] filter_iterations Number of times to apply smoothing filter. -/// @param[in] smoothing Smoothing parameter. -/// @param[in] dt Time step. -/// @param[in] norms A @ref pic::characteristics object -//------------------------------------------------------------------------------ - parameters(const T b0, const T r1, const T r2, - const size_t filter_iterations, - const T smoothing, const T dt, - const T t_para, const T t_perp, - const characteristics &norms) : - b0(b0/norms.bfield), - a0(std::numbers::pi_v*(r2*r2 - r1*r1)/(norms.l*norms.l)), - filter_iterations(filter_iterations), smoothing(smoothing), - dt(dt/norms.t), t_para(t_para), t_perp(t_perp) {} - }; - +namespace graph { //------------------------------------------------------------------------------ /// @brief U Collision node. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template - class apply_u_node final : public graph::no_derivative, - 7> { + class apply_u_node final : public no_derivative, + 7> { private: //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. @@ -172,7 +33,7 @@ namespace pic { /// @param[in] branches Array of branches. /// @return A string rep of a the node. //------------------------------------------------------------------------------ - static std::string to_string(std::array, 7> &branches) { + static std::string to_string(std::array, 7> branches) { std::string s = "apply_u"; for (auto &b : branches) { s += jit::format_to_string(reinterpret_cast (b.get())); @@ -209,23 +70,25 @@ namespace pic { for (size_t j = 0; j < size; j++) { const T mof_j = mof[j]; + const T a_j = A[j]; + const T b_j = B[j]; T temp_x = x[j]; const T tbnu_e_dt_j = tbnu_e_dt[j]; if constexpr (std::same_as) { - uint32_t rand_j = reintrepet_cast (rand[j]); + uint32_t rand_j = std::bit_cast (rand[j]); for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { const T E0 = mof_j*temp_x; const int8_t rm = 4*(rand_j & 1) - 2; const T C = rm*std::sqrt(tbnu_e_dt_j*E0); - temp_x = (E0*A + B + C)/mof_j; + temp_x = (E0*a_j + b_j + C)/mof_j; } } else { - uint64_t rand_j = reintrepet_cast (rand[j]); + uint64_t rand_j = std::bit_cast (rand[j]); for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { const T E0 = mof_j*temp_x; const int8_t rm = 4*(rand_j & 1) - 2; const T C = rm*std::sqrt(tbnu_e_dt_j*E0); - temp_x = (E0*A + B + C)/mof_j; + temp_x = (E0*a_j + b_j + C)/mof_j; } } x[j] = temp_x; @@ -244,16 +107,16 @@ namespace pic { /// @param[in] A A collision factor. /// @param[in] B B collision factor. //------------------------------------------------------------------------------ - apply_u_node(graph::shared_leaf x, - graph::shared_leaf i, - graph::shared_leaf rand, - graph::shared_leaf mof, - graph::shared_leaf tbnu_e_dt, - graph::shared_leaf A, - graph::shared_leaf B) : - graph::no_derivative> ({x, i, rand, mof, tbnu_e_dt, A, B}, - apply_u_node::to_string({x, i, rand, mof, tbnu_e_dt, A, B})) {} + apply_u_node(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf mof, + shared_leaf tbnu_e_dt, + shared_leaf A, + shared_leaf B) : + no_derivative, 7> ({x, i, rand, mof, tbnu_e_dt, A, B}, + apply_u_node::to_string({x, i, rand, mof, tbnu_e_dt, A, B})) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of the applying the u operator. @@ -278,7 +141,7 @@ namespace pic { /// /// @returns Reduced graph from apply_u. //------------------------------------------------------------------------------ - virtual graph::shared_leaf reduce() { + virtual shared_leaf reduce() { return this->shared_from_this(); } @@ -291,6 +154,7 @@ namespace pic { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -299,14 +163,10 @@ namespace pic { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - for (auto &b : this->branches) { - b->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); - } + if (!pre_funcs.contains("apply_u")) { + pre_funcs.insert("apply_u"); jit::add_type (stream); stream << " apply_u(const "; @@ -325,22 +185,31 @@ namespace pic { jit::add_type (stream); stream << " A, const "; jit::add_type (stream); - stream << " B) {" + stream << " B) {" << std::endl << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); - stream << "E0 = mof*temp_x;" << std::endl + stream << " E0 = mof*temp_x;" << std::endl << " const uint8_t rm = 4*(rand & 1) - 2;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl << " temp_x = (E0*A + B + C)/mof;" << std::endl << " }" << std::endl - << " return temp_x;" - << "}"; + << " return temp_x;" << std::endl + << "}" << std::endl; + } + + if (!visited.contains(this)) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + } visited.insert(this); #ifdef SHOW_USE_COUNT @@ -360,10 +229,10 @@ namespace pic { /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ - virtual graph::shared_leaf compile(std::ostringstream &stream, - jit::register_map ®isters, - const jit::register_map &thread_mem, - const jit::register_usage &usage) { + virtual shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); @@ -378,7 +247,12 @@ namespace pic { jit::add_type (stream); stream << " " << registers[this] << " = apply_u(" << registers[x.get()] << ", " - << registers[i.get()] << ", reinterpret_cast<"; + << registers[i.get()] << ", "; + if (jit::use_metal ()) { + stream << "as_type<"; + } else { + stream << "bit_cast<"; + } if constexpr (std::same_as) { stream << "uint32_t"; } else { @@ -402,7 +276,7 @@ namespace pic { /// @param[in] x Other graph to check if it is a match. /// @returns True if the nodes are a match. //------------------------------------------------------------------------------ - virtual bool is_match(graph::shared_leaf x) { + virtual bool is_match(shared_leaf x) { if (this == x.get()) { return true; } @@ -415,7 +289,7 @@ namespace pic { temp = temp && this->branches[i]->is_match(x_cast->get_arg(i)); } } - + return temp; } @@ -426,7 +300,8 @@ namespace pic { std::cout << "\\apply_u{\\left("; this->branches[0]->to_latex(); for (uint8_t i = 1; i < 7; i++) { - std::cout << ", " << this->branches[i]->to_latex(); + std::cout << ", "; + this->branches[i]->to_latex(); } std::cout << "\\right)}"; } @@ -436,7 +311,7 @@ namespace pic { /// /// @returns A tree without variable nodes. //------------------------------------------------------------------------------ - virtual graph::shared_leaf remove_pseudo() { + virtual shared_leaf remove_pseudo() { if (this->has_pseudo()) { return apply_u(this->branches[0]->remove_pseudo(), this->branches[1]->remove_pseudo(), @@ -456,8 +331,8 @@ namespace pic { /// @param[in,out] registers List of defined registers. /// @returns The current node. //------------------------------------------------------------------------------ - virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, - jit::register_map ®isters) { + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { if (registers.find(this) == registers.end()) { const std::string name = jit::to_string('r', this); registers[this] = name; @@ -488,25 +363,25 @@ namespace pic { /// @param[in] B B collision factor. /// @returns A reduced apply_u node. //------------------------------------------------------------------------------ - template - graph::shared_leaf apply_u(graph::shared_leaf x, - graph::shared_leaf i, - graph::shared_leaf rand, - graph::shared_leaf mof, - graph::shared_leaf tbnu_e_dt, - graph::shared_leaf A, - graph::shared_leaf B) { - auto temp = std::make_shared> (x, i, rand, mof, - tbnu_e_dt, A, B)->reduce(); + template + shared_leaf apply_u(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf mof, + shared_leaf tbnu_e_dt, + shared_leaf A, + shared_leaf B) { + auto temp = std::make_shared> (x, i, rand, mof, + tbnu_e_dt, A, B)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { - if (graph::leaf_node::caches.nodes.find(i) == - graph::leaf_node::caches.nodes.end()) { - graph::leaf_node::caches.nodes[i] = temp; + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; return temp; - } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { - return graph::leaf_node::caches.nodes[i]; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; } } #if defined(__clang__) || defined(__GNUC__) @@ -517,7 +392,7 @@ namespace pic { } /// Convenience type alias for shared sqrt nodes. - template + template using shared_apply_u = std::shared_ptr>; //------------------------------------------------------------------------------ @@ -528,8 +403,8 @@ namespace pic { /// @param[in] x Leaf node to attempt cast. /// @returns An attempted dynamic case. //------------------------------------------------------------------------------ - template - shared_apply_u apply_u_cast(graph::shared_leaf x) { + template + shared_apply_u apply_u_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } @@ -539,9 +414,9 @@ namespace pic { /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ template - class apply_xi_node final : public graph::no_derivative, - 4> { + class apply_xi_node final : public no_derivative, + 4> { private: //------------------------------------------------------------------------------ /// @brief Convert node pointer to a string. @@ -549,7 +424,7 @@ namespace pic { /// @param[in] branches Array of branches. /// @return A string rep of a the node. //------------------------------------------------------------------------------ - static std::string to_string(std::array, 4> &branches) { + static std::string to_string(std::array, 4> branches) { std::string s = "apply_xi"; for (auto &b : branches) { s += jit::format_to_string(reinterpret_cast (b.get())); @@ -579,7 +454,7 @@ namespace pic { const T nu_D_dt_j = nu_D_dt[j]; T temp_x = x[j]; if constexpr (std::same_as) { - uint32_t rand_j = reintrepet_cast (rand[j]); + uint32_t rand_j = std::bit_cast (rand[j]); for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { const T A = -temp_x*nu_D_dt_j; const int8_t rm = 2*(rand_j & 1) - 1; @@ -587,7 +462,7 @@ namespace pic { temp_x += A + C; } } else { - uint64_t rand_j = reintrepet_cast (rand[j]); + uint64_t rand_j = std::bit_cast (rand[j]); for (uint8_t k = 0, ke = i[j]; k < ke; k++, rand_j >>= 1) { const T A = -temp_x*nu_D_dt_j; const int8_t rm = 2*(rand_j & 1) - 1; @@ -608,13 +483,13 @@ namespace pic { /// @param[in] rand A random value of 32 0s and 1s. /// @param[in] nu_D_dt Normalized step rate. //------------------------------------------------------------------------------ - apply_xi_node(graph::shared_leaf x, - graph::shared_leaf i, - graph::shared_leaf rand, - graph::shared_leaf nu_D_dt) : - graph::no_derivative> ({x, i, rand, nu_D_dt}, - apply_xi_node::to_string({x, i, rand, nu_D_dt})) {} + apply_xi_node(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf nu_D_dt) : + no_derivative, 4> ({x, i, rand, nu_D_dt}, + apply_xi_node::to_string({x, i, rand, nu_D_dt})) {} //------------------------------------------------------------------------------ /// @brief Evaluate the results of the applying the xi operator. @@ -636,7 +511,7 @@ namespace pic { /// /// @returns Reduced graph from apply_xi. //------------------------------------------------------------------------------ - virtual graph::shared_leaf reduce() { + virtual shared_leaf reduce() { return this->shared_from_this(); } @@ -649,6 +524,7 @@ namespace pic { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -657,14 +533,10 @@ namespace pic { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { - for (auto &b : this->branches) { - b->compile_preamble(stream, registers, - visited, usage, - textures1d, textures2d, - avail_const_mem); - } + if (!pre_funcs.contains("apply_xi")) { + pre_funcs.insert("apply_xi"); jit::add_type (stream); stream << " apply_xi(const "; @@ -677,22 +549,31 @@ namespace pic { } stream << " rand, const "; jit::add_type (stream); - stream << " nu_D_dt) {" + stream << " nu_D_dt) {" << std::endl << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); - stream << "A = -temp_x*nu_D_dt;" << std::endl + stream << " A = -temp_x*nu_D_dt;" << std::endl << " const uint8_t rm = 2*(rand & 1) - 1;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt((1 - temp_x*temp_x)*nu_D_dt);" << std::endl << " temp_x += A + C;" << std::endl << " }" << std::endl - << " return temp_x;" - << "}"; + << " return temp_x;" << std::endl + << "}" << std::endl; + } + + if (!visited.contains(this)) { + for (auto &b : this->branches) { + b->compile_preamble(stream, registers, + visited, usage, + textures1d, textures2d, + pre_funcs, avail_const_mem); + } visited.insert(this); #ifdef SHOW_USE_COUNT @@ -712,10 +593,10 @@ namespace pic { /// @param[in] usage List of register usage count. /// @returns The current node. //------------------------------------------------------------------------------ - virtual graph::shared_leaf compile(std::ostringstream &stream, - jit::register_map ®isters, - const jit::register_map &thread_mem, - const jit::register_usage &usage) { + virtual shared_leaf compile(std::ostringstream &stream, + jit::register_map ®isters, + const jit::register_map &thread_mem, + const jit::register_usage &usage) { if (registers.find(this) == registers.end()) { auto x = this->branches[0]->compile(stream, registers, thread_mem, usage); auto i = this->branches[1]->compile(stream, registers, thread_mem, usage); @@ -727,7 +608,12 @@ namespace pic { jit::add_type (stream); stream << " " << registers[this] << " = apply_xi(" << registers[x.get()] << ", " - << registers[i.get()] << ", reinterpret_cast<"; + << registers[i.get()] << ", "; + if (jit::use_metal ()) { + stream << "as_type<"; + } else { + stream << "bit_cast<"; + } if constexpr (std::same_as) { stream << "uint32_t"; } else { @@ -748,7 +634,7 @@ namespace pic { /// @param[in] x Other graph to check if it is a match. /// @returns True if the nodes are a match. //------------------------------------------------------------------------------ - virtual bool is_match(graph::shared_leaf x) { + virtual bool is_match(shared_leaf x) { if (this == x.get()) { return true; } @@ -772,7 +658,8 @@ namespace pic { std::cout << "\\apply_xi{\\left("; this->branches[0]->to_latex(); for (uint8_t i = 1; i < 4; i++) { - std::cout << ", " << this->branches[i]->to_latex(); + std::cout << ", "; + this->branches[i]->to_latex(); } std::cout << "\\right)}"; } @@ -782,7 +669,7 @@ namespace pic { /// /// @returns A tree without variable nodes. //------------------------------------------------------------------------------ - virtual graph::shared_leaf remove_pseudo() { + virtual shared_leaf remove_pseudo() { if (this->has_pseudo()) { return apply_xi(this->branches[0]->remove_pseudo(), this->branches[1]->remove_pseudo(), @@ -799,13 +686,13 @@ namespace pic { /// @param[in,out] registers List of defined registers. /// @returns The current node. //------------------------------------------------------------------------------ - virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, - jit::register_map ®isters) { + virtual shared_leaf to_vizgraph(std::stringstream &stream, + jit::register_map ®isters) { if (registers.find(this) == registers.end()) { const std::string name = jit::to_string('r', this); registers[this] = name; stream << " " << name - << " [label = \"apply_xi\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; + << " [label = \"apply_xi\", shape = oval, style = filled, fillcolor = blue, fontcolor = white];" << std::endl; for (auto &b : this->branches) { auto temp = b->to_vizgraph(stream, registers); @@ -828,22 +715,22 @@ namespace pic { /// @param[in] nu_D_dt Normalized step rate. /// @returns A reduced apply_xi node. //------------------------------------------------------------------------------ - template - graph::shared_leaf apply_xi(graph::shared_leaf x, - graph::shared_leaf i, - graph::shared_leaf rand, - graph::shared_leaf nu_D_dt) { - auto temp = std::make_shared> (x, i, rand, - nu_D_dt)->reduce(); + template + shared_leaf apply_xi(shared_leaf x, + shared_leaf i, + shared_leaf rand, + shared_leaf nu_D_dt) { + auto temp = std::make_shared> (x, i, rand, + nu_D_dt)->reduce(); // Test for hash collisions. for (size_t i = temp->get_hash(); i < std::numeric_limits::max(); i++) { - if (graph::leaf_node::caches.nodes.find(i) == - graph::leaf_node::caches.nodes.end()) { - graph::leaf_node::caches.nodes[i] = temp; + if (leaf_node::caches.nodes.find(i) == + leaf_node::caches.nodes.end()) { + leaf_node::caches.nodes[i] = temp; return temp; - } else if (temp->is_match(graph::leaf_node::caches.nodes[i])) { - return graph::leaf_node::caches.nodes[i]; + } else if (temp->is_match(leaf_node::caches.nodes[i])) { + return leaf_node::caches.nodes[i]; } } #if defined(__clang__) || defined(__GNUC__) @@ -854,7 +741,7 @@ namespace pic { } /// Convenience type alias for shared sqrt nodes. - template + template using shared_apply_xi = std::shared_ptr>; //------------------------------------------------------------------------------ @@ -865,10 +752,161 @@ namespace pic { /// @param[in] x Leaf node to attempt cast. /// @returns An attempted dynamic case. //------------------------------------------------------------------------------ - template - shared_apply_xi apply_u_cast(graph::shared_leaf x) { + template + shared_apply_xi apply_xi_cast(shared_leaf x) { return std::dynamic_pointer_cast> (x); } +} + +namespace pic { +// FIXME: This should be in a separate file of physics constants. +/// Speed of light m/s. + template + constexpr T c = static_cast (299792458.0); +/// Vacuum permitivity F/m. + template + constexpr T epsilon0 = static_cast (8.8541878188E-12); +/// Fundamental charge coulombs. + template + constexpr T q = static_cast (1.602176634E-19); +/// Hydrogen mass kg. + template + constexpr T m_hydrogen = static_cast (1.67362192595E-27); +/// Atomic mass + template + constexpr T m_atomic = static_cast (1.66053906892E-27); +/// Electron mass kg. + template + constexpr T m_electron = static_cast (9.1093837139E-31); +/// Boltzman constant. + template + constexpr T kb = static_cast (1.380649E-23); + +//------------------------------------------------------------------------------ +/// @brief Characteristic factors. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ + template + class characteristics { + private: +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_masses Ion masses. +/// @returns (∑(m_i) + me)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_m(const std::vector &ion_masses) { + T total_m = static_cast (0); + for (const T &mass : ion_masses) { + total_m += mass; + } + return total_m/ion_masses.size(); + } + +//------------------------------------------------------------------------------ +/// @brief Compute the characteristic mass. +/// +/// @param[in] ion_zs Ion Z. +/// @returns (∑(Z_i)*q + q)/(n_i + 1); +//------------------------------------------------------------------------------ + T make_q(const std::vector &ion_zs) { + T total_q = static_cast (0); + for (const uint8_t &z : ion_zs) { + total_q += z*pic::q; + } + return total_q/ion_zs.size(); + } + + public: +/// Mass + const T m; +/// Charge + const T q; +/// Electron density. + const T ne; +/// Plasma Frequency. + const T wpe; +/// Time. + const T t; +/// Length + const T l; +/// Velocity + const T v; +/// Electron temperature; + const T te; +/// Electric field; + const T efield; +/// Magnetic field; + const T bfield; + +//------------------------------------------------------------------------------ +/// @brief Construct the characteristics. +/// +/// @param[in] ion_masses Ion masses for all species. +/// @param[in] ion_zs Ion Z effective all species. +/// @param[in] ne Characteristic density. +//------------------------------------------------------------------------------ + characteristics(const std::vector &ion_masses, + const std::vector &ion_zs, + const T ne) : + m(make_m(ion_masses)), q(make_q(ion_zs)), ne(ne), + wpe(std::sqrt(ne*q*q/(m*epsilon0))), + t(1/wpe), l(c/wpe), v(c), te(m*v*v/kb), efield(m*c/(q*t)), + bfield(efield/c) {} + +//------------------------------------------------------------------------------ +/// @brief Get the characteristic volume. +/// +/// @return l^3. +//------------------------------------------------------------------------------ + T get_volume() const { + return l*l*l; + } + }; + +//------------------------------------------------------------------------------ +/// @brief Parameter Class +//------------------------------------------------------------------------------ + template + class parameters { + public: +/// Initial magnetic field + const T b0; +/// Geometry + const T a0; +/// Filter Iterations. + const size_t filter_iterations; +/// Smoothing parameters. + const T smoothing; +/// Time step. + const T dt; +/// Parallel temperature. + const T t_para; +/// Perpendicular temperature. + const T t_perp; + +//------------------------------------------------------------------------------ +/// @brief Construct a parameters object. +/// +/// @param[in] b0 Initial magnetic field. +/// @param[in] r1 +/// @param[in] r2 +/// @param[in] filter_iterations Number of times to apply smoothing filter. +/// @param[in] smoothing Smoothing parameter. +/// @param[in] dt Time step. +/// @param[in] norms A @ref pic::characteristics object. +//------------------------------------------------------------------------------ + parameters(const T b0, const T r1, const T r2, + const size_t filter_iterations, + const T smoothing, const T dt, + const T t_para, const T t_perp, + const characteristics &norms) : + b0(b0/norms.bfield), + a0(std::numbers::pi_v*(r2*r2 - r1*r1)/(norms.l*norms.l)), + filter_iterations(filter_iterations), smoothing(smoothing), + dt(dt/norms.t), t_para(t_para), t_perp(t_perp) {} + }; //------------------------------------------------------------------------------ /// @brief ion class. @@ -881,7 +919,7 @@ namespace pic { class ion { public: /// Atomic number. - const T z; + const uint8_t z; /// Charge const T charge; /// Particle mass @@ -1013,6 +1051,16 @@ namespace pic { data.create_variable(file, "vperp_" + tag, v_perp, work.get_context()); } + +//------------------------------------------------------------------------------ +/// @brief Build a profile. +/// +/// @param[in] func The profile function. +/// @returns The parallel temperature profile. +//------------------------------------------------------------------------------ + graph::shared_leaf build_profile(std::function(graph::shared_leaf)> func) const { + return func(x); + } }; //------------------------------------------------------------------------------ @@ -1038,10 +1086,10 @@ namespace pic { const T scale, const size_t iterations=0) const { auto low = iterations ? build_y_index (x - dx, scale, iterations - 1) : - graph::index_1D(y[I], x, dx, xmin + dx); + graph::index_1D(y[I], x, dx, xmin + dx); auto center = graph::index_1D(y[I], x, dx, xmin); auto high = iterations ? build_y_index (x + dx, scale, iterations - 1) : - graph::index_1D(y[I], x, dx, xmin - dx); + graph::index_1D(y[I], x, dx, xmin - dx); const T center_w = static_cast (0.5); const T side_w = static_cast (0.25); @@ -1239,7 +1287,7 @@ namespace pic { auto w0 = static_cast (0.5)*xnorm1*xnorm1; auto w1 = static_cast (0.75) - xnorm2*xnorm2; auto w2 = static_cast (0.5)*xnorm3*xnorm3; - + return {w0, w1, w2}; } @@ -1263,6 +1311,8 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Convert from cartesian to sphereical coordinates. /// +/// @tparam T Base type of the calculation. +/// /// @param[in] x /// @param[in] y /// @returns The coordinates as sphereical coordinates. @@ -1280,6 +1330,8 @@ namespace pic { //------------------------------------------------------------------------------ /// @brief Convert from sphereical to cartesian coordinates. /// +/// @tparam T Base type of the calculation. +/// /// @param[in] w /// @param[in] xi /// @param[in] sinphi @@ -1295,6 +1347,216 @@ namespace pic { }; } +//------------------------------------------------------------------------------ +/// @brief Model for coulomb scattering operators. +//------------------------------------------------------------------------------ + enum model { + /// From Hinton 1983 EQ 92 and T.S. Chen 1988 EQ 50 + hinton, + /// From T.S. Chen 1988 Report EQ 57 commonly used for NBI + chen + }; + +//------------------------------------------------------------------------------ +/// @brief Calculate ν_{E}. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] xab +/// @param[in] mass_a Mass of particle a. +/// @param[in] mass_b Mass of particle b. +/// @param[in] gb +/// @param[in] nuab0 +/// @param[in] erfp_xab Derivaive of error function. +/// @returns ν_{E}. +//------------------------------------------------------------------------------ + template + graph::shared_leaf build_colision_rate(graph::shared_leaf xab, + const T mass_a, + const T mass_b, + graph::shared_leaf gb, + graph::shared_leaf nuab0, + graph::shared_leaf erfp_xab) { + auto mass_ratio = static_cast (2)*mass_a/mass_b*gb; + auto nu = nuab0/xab; + if constexpr (M == model::hinton) { + return nu*(mass_ratio - erfp_xab/xab); + } else { + return nu*mass_ratio; + } + } + +//------------------------------------------------------------------------------ +/// @brief Build ion ion collision. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] ion_b Ions for species b. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in,out] total_density Accumulated density. +/// @param[in,out] total_flux Accumulated total_flux. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_ion_ion_collision(const ion &ion_a, + const ion &ion_b, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + graph::shared_leaf total_density, + graph::shared_leaf total_flux, + const graph::shared_random_state state) { + const T dt = params.dt*norms.t; + + const T mass_b = ion_b.mass*norms.m; + const uint8_t zb2 = ion_b.z*ion_b.z; + + auto nb = build_density(ion_a.x, ion_b, mesh, norms, params)/norms.get_volume(); + auto tpara = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + }); + auto tperp = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + }); + auto tb = static_cast (0.5)*(tpara + tperp)*norms.te*kb/q; + auto nv = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + })*norms.v/norms.get_volume(); + auto uxb = nv/nb; + + total_density = total_density + nb; + total_flux = total_flux + nv; + + return build_common_collision (ion_a, norms, uxb, tb, nb, mass_b, + zb2, dt, state); + } + +//------------------------------------------------------------------------------ +/// @brief Build ion electron collision. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] mesh A @ref pic::mesh object. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] params A @ref pic::parameters object. +/// @param[in] total_density Accumulated density. +/// @param[in] total_flux Accumulated total_flux. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_ion_electron_collision(const ion &ion_a, + const mesh &mesh, + const characteristics &norms, + const parameters ¶ms, + graph::shared_leaf total_density, + graph::shared_leaf total_flux, + const graph::shared_random_state state) { + const T dt = params.dt*norms.t; + + const T mass_b = m_electron; + const uint8_t zb2 = 1; + + auto nb = total_density; + auto tb = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { + return graph::one (); + })*norms.te; + + auto uxb = total_flux/total_density; + + return build_common_collision (ion_a, norms, uxb, tb, nb, mass_b, + zb2, dt, state); + } + +//------------------------------------------------------------------------------ +/// @brief Build common collisions graphs for all species. +/// +/// @tparam T Base type of the calculation. +/// @tparam M The @ref pic::model of the calculation. +/// +/// @param[in] ion_a Ions for species a. +/// @param[in] norms A @ref pic::characteristics object. +/// @param[in] uxb +/// @param[in] tb Temperature of species b. +/// @param[in] nb Density of species b. +/// @param[in] mass_b Mass of species b. +/// @param[in] zb2 Z effective squared. +/// @param[in] dt Time step. +/// @param[in] state Random state node. +//------------------------------------------------------------------------------ + template + std::array, 2> build_common_collision(const ion &ion_a, + const characteristics &norms, + graph::shared_leaf uxb, + graph::shared_leaf tb, + graph::shared_leaf nb, + const T mass_b, + const uint8_t zb2, + const T dt, + const graph::shared_random_state state) { + const T mass_a = ion_a.mass*norms.m; + const uint8_t za2 = ion_a.z*ion_a.z; + +// Convert to b frame. + auto wxa = ion_a.v_para*norms.v - uxb; + auto wya = ion_a.v_perp*norms.v; + +// Convert to sphereical. + auto sphere = cartesian_to_sphereical(wxa, wya); + +// Apply Monte-Carlo collision operator. + auto wtb = graph::sqrt(static_cast (2)*q*tb/mass_b); + auto xab = sphere[0]/wtb; + + auto erf_xab = graph::erf(xab); + auto erfp_xab = erf_xab->df(xab); + auto erfpp_xab = erfp_xab->df(xab); + + auto gb = (erf_xab - xab*erfp_xab)/(static_cast (2)*xab*xab); + auto logA = static_cast (30) - static_cast (0.5)*graph::log(nb/(tb*graph::sqrt(tb))); + auto nuab0 = nb*q*q*q*q*static_cast (za2*zb2)*logA + / (static_cast (2)*std::numbers::pi_v*mass_a*mass_a*epsilon0*epsilon0*wtb*wtb*wtb); + +// Velocity Scattering operator. + auto nu_e_dt = build_colision_rate (xab, mass_a, mass_b, gb, nuab0, erfp_xab)*dt; + auto steps = graph::min(nu_e_dt*2.5, static_cast (sizeof(T)*8)); + + nu_e_dt = nu_e_dt*steps; + + auto E_nuE_d_nu_E_dE = static_cast (0.5)*((static_cast (3.0)*(xab*erfp_xab - erf_xab) - xab*xab*erfpp_xab)/(erf_xab - xab*erfp_xab)); + + const T mof = mass_a/(static_cast (2)*q); + auto A = static_cast (1) - static_cast (2)*nu_e_dt; + auto tbnu_e_df = tb*nu_e_dt; + auto B = static_cast (2)*tbnu_e_df*(static_cast (1.5) + E_nuE_d_nu_E_dE); + + auto rand1 = graph::random (state); + auto u_op = graph::apply_u(sphere[0]*sphere[0], steps, rand1, + graph::constant(mof), tbnu_e_df, A, B); + sphere[0] = graph::sqrt(u_op); + + auto nu_D_dt = nuab0*(erf_xab - gb)/(xab*xab*xab)*dt; + steps = graph::min(nu_D_dt, static_cast (sizeof(T)*8)); + nu_D_dt = nu_D_dt/steps; + + auto rand2 = graph::random (state); + sphere[1] = graph::apply_xi(sphere[1], steps, rand2, nu_D_dt); + + sphere[1] = graph::if_(sphere[1]*sphere[1] > static_cast (1), + graph::copysign(static_cast (1), sphere[1]) - + sphere[1] % graph::copysign(static_cast (1), sphere[1]), + sphere[1]); + + auto cart = sphereical_to_cartesian(sphere[0], sphere[1], sphere[2]); + return {(cart[0] + uxb)/norms.v, cart[1]/norms.v}; + } + //------------------------------------------------------------------------------ /// @brief Build initialization. /// @@ -1304,7 +1566,7 @@ namespace pic { /// @param[in] mesh A @ref pic::mesh object. /// @param[in] norms A @ref pic::characteristics object. /// @param[in] params A @ref pic::parameters object. -/// @param[in] state Random state node. +/// @param[in] state Random state node. /// @returns Initialized normalized values for x, v||, and v⟂ //------------------------------------------------------------------------------ template @@ -1425,7 +1687,7 @@ namespace pic { // Scale factor. const T sf = ion.super_to_real()/(params.a0*mesh.dx); - return ion.z*y*cf*sf; + return static_cast (ion.z)*y*cf*sf; } //------------------------------------------------------------------------------ @@ -1456,7 +1718,7 @@ namespace pic { // Scale factor. const T sf = ion.super_to_real()/(params.a0*mesh.dx); - return ion.z*y*cf*sf; + return static_cast (ion.z)*y*cf*sf; } //------------------------------------------------------------------------------ diff --git a/graph_framework/piecewise.hpp b/graph_framework/piecewise.hpp index ea99b6b..d13c822 100644 --- a/graph_framework/piecewise.hpp +++ b/graph_framework/piecewise.hpp @@ -506,6 +506,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -514,12 +515,14 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->arg->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); + if (registers.find(leaf_node::caches.backends[data_hash].data()) == registers.end()) { registers[leaf_node::caches.backends[data_hash].data()] = jit::to_string('a', leaf_node::caches.backends[data_hash].data()); @@ -1058,6 +1061,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -1066,16 +1070,18 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->left->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); this->right->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); + if (registers.find(leaf_node::caches.backends[data_hash].data()) == registers.end()) { registers[leaf_node::caches.backends[data_hash].data()] = jit::to_string('a', leaf_node::caches.backends[data_hash].data()); diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index 1adbfa4..7b22df4 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -95,6 +95,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -103,8 +104,15 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!pre_funcs.contains("random_state")) { + pre_funcs.insert("random_state"); + + random_state_node::compile_random_state(stream); + } + + if (!visited.contains(this)) { visited.insert(this); #ifdef SHOW_USE_COUNT usage[this] = 1; @@ -369,6 +377,7 @@ namespace graph { /// @param[in,out] usage List of register usage count. /// @param[in,out] textures1d List of 1D textures. /// @param[in,out] textures2d List of 2D textures. +/// @param[in,out] pre_funcs Set of preamble functions. /// @param[in,out] avail_const_mem Available constant memory. //------------------------------------------------------------------------------ virtual void compile_preamble(std::ostringstream &stream, @@ -377,12 +386,13 @@ namespace graph { jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, + jit::preamble_map &pre_funcs, int &avail_const_mem) { - if (visited.find(this) == visited.end()) { + if (!visited.contains(this)) { this->arg->compile_preamble(stream, registers, visited, usage, textures1d, textures2d, - avail_const_mem); + pre_funcs, avail_const_mem); visited.insert(this); #ifdef SHOW_USE_COUNT @@ -391,6 +401,14 @@ namespace graph { ++usage[this]; #endif } + +// Need to do this after visited was checked so the random_state is created +// first. + if (!pre_funcs.contains("random")) { + pre_funcs.insert("random"); + + random_node::compile_random(stream); + } } //------------------------------------------------------------------------------ diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index 90195aa..457c6c4 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -8,8 +8,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -256,15 +256,17 @@ namespace jit { } /// Type alias for mapping node pointers to register names. - typedef std::map register_map; + typedef std::unordered_map register_map; /// Type alias for counting register usage. - typedef std::map register_usage; + typedef std::unordered_map register_usage; /// Type alias for listing visited nodes. - typedef std::set visiter_map; + typedef std::unordered_set visiter_map; /// Type alias for indexing 1D textures. - typedef std::map texture1d_list; + typedef std::unordered_map texture1d_list; /// Type alias for indexing 2D textures. - typedef std::map> texture2d_list; + typedef std::unordered_map> texture2d_list; +/// Type alias for preamble defined functions. + typedef std::unordered_set preamble_map; /// Type for tacking thread shared memory. typedef std::unordered_set argument_set; diff --git a/graph_framework/workflow.hpp b/graph_framework/workflow.hpp index edf6510..16a7ff3 100644 --- a/graph_framework/workflow.hpp +++ b/graph_framework/workflow.hpp @@ -545,7 +545,8 @@ namespace workflow { const std::string name, const size_t size, const size_t iterations) { if constexpr (O == pre_run_item) { - preitems.push_back(std::make_unique> (in, out, maps, + preitems.push_back(std::make_unique> (in, out, + maps, atomics, state, name, size, diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index ffded51..353b075 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -194,6 +194,50 @@ void run_pic() { }, {}, { graph::variable_cast(mesh.y[0]) }, NULL, "sum_weights_" + ion_tag, num_particles); + + graph::shared_leaf total_density = graph::zero (); + graph::shared_leaf total_flux = graph::zero (); + for (size_t j = 0; j < num_ions; j++) { + const std::string inner_ion_tag = jit::format_to_string(j); + + auto coll = pic::build_ion_ion_collision (ions[i], + ions[j], + mesh, + norms, + params, + total_density, + total_flux, + graph::random_state_cast(state)); + + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp(), + graph::variable_cast(mesh.y[0]) + }, {}, { + {coll[0], ions[i].get_v_para()}, + {coll[1], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "ion_ion_" + ion_tag + "_" + inner_ion_tag, num_particles); + } + + auto coll = pic::build_ion_electron_collision (ions[i], + mesh, + norms, + params, + total_density, + total_flux, + graph::random_state_cast(state)); + + work.add_item({ + ions[i].get_x(), + ions[i].get_v_para(), + ions[i].get_v_perp() + }, {}, { + {coll[0], ions[i].get_v_para()}, + {coll[1], ions[i].get_v_perp()} + }, {}, graph::random_state_cast(state), + "ion_elec_" + ion_tag, num_particles); } init.print(); diff --git a/graph_tests/backend_test.cpp b/graph_tests/backend_test.cpp index ea9f983..98bb593 100644 --- a/graph_tests/backend_test.cpp +++ b/graph_tests/backend_test.cpp @@ -571,7 +571,7 @@ template void test_backend() { })); exp_vec.set(std::vector ({ static_cast (-4.0), - static_cast (0.30) + static_cast (0.3) })); const backend::buffer vec_vec = backend::pow(base_vec, exp_vec); assert(vec_vec.size() == 2 && "Expected a size of 2"); @@ -581,8 +581,8 @@ template void test_backend() { std::abs(static_cast (8.6736173798840355e-19)) && "Expected 4^-4."); assert(vec_vec.at(1) == std::pow(static_cast (2.0), - static_cast (0.30)) && - "Expected 2^0.30."); + static_cast (0.3)) && + "Expected 2^0.3."); base_scalar.set(static_cast (4.0)); base_scalar.log(); @@ -636,6 +636,50 @@ template void test_backend() { })); assert(!nan_vec.is_normal() && "Expected a NaN."); + if constexpr (jit::complex_scalar) { + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + avec.erfi(); + assert(avec.at(0) == static_cast (special::erfi(static_cast (4.0))) && + "Expected a value of Erfi(4 + 0i)."); + assert(avec.at(1) == static_cast (special::erfi(static_cast (-2))) && + "Expected a value of Erfi(-2 + 0i)."); + } + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + bvec.set(std::vector ({ + static_cast (-3.0), + static_cast (0.3) + })); + assert((avec == avec).at(0) == static_cast (1) && "Expected true."); + assert((avec == avec).at(1) == static_cast (1) && "Expected true."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec == bvec).at(0) == static_cast (0) && "Expected false."); + assert((avec == bvec).at(1) == static_cast (0) && "Expected false."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec != avec).at(0) == static_cast (0) && "Expected false."); + assert((avec != avec).at(1) == static_cast (0) && "Expected false."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + assert((avec != bvec).at(0) == static_cast (1) && "Expected true."); + assert((avec != bvec).at(1) == static_cast (1) && "Expected true."); + if constexpr (std::floating_point) { avec.set(std::vector ({ static_cast (4.0), @@ -643,7 +687,7 @@ template void test_backend() { })); bvec.set(std::vector ({ static_cast (-3.0), - static_cast (0.30) + static_cast (0.3) })); const backend::buffer copysignvec = backend::copysign(avec, bvec); assert(copysignvec.size() == 2 && "Expected a size of 2"); @@ -651,6 +695,28 @@ template void test_backend() { "Expected a value of -4."); assert(copysignvec.at(1) == static_cast (2.0) && "Expected a value of 2."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + avec.erf(); + assert(avec.at(0) == static_cast (std::erf(static_cast (4.0))) && + "Expected a value of Erf(4)."); + assert(avec.at(1) == static_cast (std::erf(static_cast (-2.0))) && + "Expected a value of Erf(-2)."); + + avec.set(std::vector ({ + static_cast (4.0), + static_cast (-2.0) + })); + const backend::buffer modvec = avec % bvec; + assert((modvec.at(0) == static_cast (std::fmod(static_cast (4.0), + static_cast (-3.0)))) && + "Expected a value of 4 % -3."); + assert((modvec.at(1) == static_cast (std::fmod(static_cast (-2.0), + static_cast (0.3)))) && + "Expected a value of -2 % 0.3."); } } diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index f64edd2..dc8240e 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -379,6 +379,12 @@ template void run_math_tests() { graph::variable_cast(v1), graph::variable_cast(v2) }, {hypot_node}, {}, hypot_node->evaluate().at(0), 0.0); + + auto min_node = graph::min(v1, v2); + compile ({ + graph::variable_cast(v1), + graph::variable_cast(v2) + }, {min_node}, {}, min_node->evaluate().at(0), 0.0); } } diff --git a/graph_tests/logical_test.cpp b/graph_tests/logical_test.cpp index 977490a..fc21be3 100644 --- a/graph_tests/logical_test.cpp +++ b/graph_tests/logical_test.cpp @@ -302,6 +302,42 @@ template void test_if() { assert(test_not_cast->get_right()->is_match(v1) && "Expected v1"); } +//------------------------------------------------------------------------------ +/// @brief Test for min nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template void test_min() { + auto one = graph::one (); + auto none = graph::none (); + + assert(graph::min(one, none)->is_match(none) && + "Expected -1."); + assert(graph::min(none, one)->is_match(none) && + "Expected -1."); + assert(graph::min(none, none)->is_match(none) && + "Expected -1."); + assert(graph::min(one, one)->is_match(one) && + "Expected 1."); + + auto v1 = graph::variable (1, ""); + auto v2 = graph::variable (1, ""); + auto zero = graph::zero (); + + auto min_result = graph::min(v1, v2); + assert(min_result->df(v1)->is_match(zero) && + "Expected 1."); + assert(min_result->df(v2)->is_match(zero) && + "Expected 1."); + + auto v3 = graph::variable (1, ""); + assert(min_result->df(v3)->is_match(zero) && + "Expected 0."); + + assert(min_result->df(min_result)->is_match(one) && + "Expected 1"); +} + //------------------------------------------------------------------------------ /// @brief Run tests with a specified backend. /// @@ -319,6 +355,7 @@ template void run_tests() { test_and (); test_or (); test_if (); + test_min (); } } diff --git a/graph_tests/math_test.cpp b/graph_tests/math_test.cpp index 4af547f..48428e9 100644 --- a/graph_tests/math_test.cpp +++ b/graph_tests/math_test.cpp @@ -538,7 +538,34 @@ void test_log() { } //------------------------------------------------------------------------------ -/// @brief Tests for log nodes. +/// @brief Tests for erfi nodes. +/// +/// @tparam T Base type of the calculation. +//------------------------------------------------------------------------------ +template +void test_erf() { + auto a = graph::variable (1, ""); + auto erf = graph::erf(a); + + assert(graph::erf_cast(erf) && + "Expected an erf node."); + + auto derfda = erf->df(a); + assert(graph::multiply_cast(derfda) && + "Expected a multiply node."); + + auto erfc = graph::erf(graph::one ()); + assert(graph::constant_cast(erfc) && + "Expected a constant node."); + +// Test node properties. + assert(!erf->is_constant() && "Did not expect a constant."); + assert(erf->is_all_variables() && "Expected a variable."); + assert(!erf->is_power_like() && "Did not expect a power like."); +} + +//------------------------------------------------------------------------------ +/// @brief Tests for erfi nodes. /// /// @tparam T Base type of the calculation. //------------------------------------------------------------------------------ @@ -697,6 +724,7 @@ template void run_tests() { test_erfi (); } if constexpr (std::floating_point) { + test_erf (); test_hypot (); } } diff --git a/graph_tests/no_derivative_test.cpp b/graph_tests/no_derivative_test.cpp index f20362a..91e8479 100644 --- a/graph_tests/no_derivative_test.cpp +++ b/graph_tests/no_derivative_test.cpp @@ -10,12 +10,26 @@ //------------------------------------------------------------------------------ class dummy : public graph::no_derivative> { public: - dummy() : graph::no_derivative> ("") {} +//------------------------------------------------------------------------------ +/// @brief A dummy constructor. +//------------------------------------------------------------------------------ + dummy() : + graph::no_derivative> ("") {} +//------------------------------------------------------------------------------ +/// @brief Dummy evaluate method. +/// +/// @returns An empty buffer. +//------------------------------------------------------------------------------ virtual backend::buffer evaluate() { return backend::buffer (); }; +//------------------------------------------------------------------------------ +/// @brief Dummy reduce method. +/// +/// @returns Returns the dummy node. +//------------------------------------------------------------------------------ virtual graph::shared_leaf compile(std::ostringstream &stream, jit::register_map ®isters, @@ -24,11 +38,21 @@ class dummy : public graph::no_derivativeshared_from_this(); } +//------------------------------------------------------------------------------ +/// @brief Dummy to vizgraph method. +/// +/// @returns A reference to this. +//------------------------------------------------------------------------------ virtual graph::shared_leaf to_vizgraph(std::stringstream &stream, jit::register_map ®isters) { return this->shared_from_this(); } +//------------------------------------------------------------------------------ +/// @brief Dummy get power exponent method. +/// +/// @returns One. +//------------------------------------------------------------------------------ virtual graph::shared_leaf get_power_exponent() const { return graph::one (); } From b6707b475786fa6384f9a9e048450f4f908782dd Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 17 Sep 2026 14:58:19 -0400 Subject: [PATCH 46/51] Refactor random numbers to avoid casting to a floating point type first. This should keep the bits intack and avoid type conversions in the collision operator. Update unit tests as a result. --- graph_framework.xcodeproj/project.pbxproj | 315 +++++++++++++++++++++- graph_framework/metal_context.hpp | 6 +- graph_framework/particle_in_cell.hpp | 46 +--- graph_framework/random.hpp | 47 ++-- graph_pic/xpic.cpp | 148 +++++----- graph_tests/backend_test.cpp | 16 +- graph_tests/c_binding_test.c | 6 +- graph_tests/f_binding_test.f90 | 9 +- 8 files changed, 439 insertions(+), 154 deletions(-) diff --git a/graph_framework.xcodeproj/project.pbxproj b/graph_framework.xcodeproj/project.pbxproj index d3a76b0..c9c7a90 100644 --- a/graph_framework.xcodeproj/project.pbxproj +++ b/graph_framework.xcodeproj/project.pbxproj @@ -55,6 +55,9 @@ C7D12D9A2DBAB31F00925420 /* random.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C7D12D992DBAB31F00925420 /* random.hpp */; }; C7D371132A0595A40074676E /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; C7DC9EEC2E39790100524F6F /* graph_c_binding.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7DC9EE22E39768300524F6F /* graph_c_binding.cpp */; }; + C7DF2CC7305C635500E8491C /* c_binding_test.c in Sources */ = {isa = PBXBuildFile; fileRef = C7DC9EF12E3A688F00524F6F /* c_binding_test.c */; }; + C7DF2CC8305C636000E8491C /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C71342682947F36100672AD4 /* Metal.framework */; }; + C7DF2CCD305C63DE00E8491C /* libgraph_c.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C7DC9EE82E39789900524F6F /* libgraph_c.a */; }; C7E5644528A2A1AA000F31A2 /* backend_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E7328074F540033B488 /* backend_test.cpp */; }; C7E5645128A2A1DD000F31A2 /* dispersion_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E6B28073BCA0033B488 /* dispersion_test.cpp */; }; C7E5645D28A2A21D000F31A2 /* solver_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C7931E6C28073BCA0033B488 /* solver_test.cpp */; }; @@ -166,6 +169,20 @@ remoteGlobalIDString = C79141A522DA9BF200E0BA0D; remoteInfo = graph_framework; }; + C7DF2CC9305C63A900E8491C /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = C791419E22DA9BF200E0BA0D /* Project object */; + proxyType = 1; + remoteGlobalIDString = C7DC9EE72E39789900524F6F; + remoteInfo = graph_c; + }; + C7DF2CCB305C63B100E8491C /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = C791419E22DA9BF200E0BA0D /* Project object */; + proxyType = 1; + remoteGlobalIDString = C79141A522DA9BF200E0BA0D; + remoteInfo = graph_framework; + }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -286,6 +303,15 @@ ); runOnlyForDeploymentPostprocessing = 1; }; + C7DF2CBE305C632700E8491C /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; C7E5643C28A2A16F000F31A2 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -462,6 +488,7 @@ C7DC9EF12E3A688F00524F6F /* c_binding_test.c */ = {isa = PBXFileReference; explicitFileType = sourcecode.c.objc; path = c_binding_test.c; sourceTree = ""; }; C7DD87D32E664B440058BA66 /* code_structure.dox */ = {isa = PBXFileReference; lastKnownFileType = text; path = code_structure.dox; sourceTree = ""; }; C7DD87D42E665E260058BA66 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; + C7DF2CC0305C632700E8491C /* c_binding_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = c_binding_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E134492A3CB3EC0083F6A7 /* output.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = output.hpp; sourceTree = ""; }; C7E5643E28A2A16F000F31A2 /* backend_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = backend_test; sourceTree = BUILT_PRODUCTS_DIR; }; C7E5644A28A2A1C5000F31A2 /* dispersion_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = dispersion_test; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -594,6 +621,15 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C7DF2CBD305C632700E8491C /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C7DF2CCD305C63DE00E8491C /* libgraph_c.a in Frameworks */, + C7DF2CC8305C636000E8491C /* Metal.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7E5643B28A2A16F000F31A2 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -791,6 +827,7 @@ C74F2AE32F6DE8C500B48216 /* workflow_test */, C715C7942FD09E29003EEFF4 /* pic_test */, C76263402FE0882200F283DF /* logical_test */, + C7DF2CC0305C632700E8491C /* c_binding_test */, ); name = Products; sourceTree = ""; @@ -1203,6 +1240,27 @@ productReference = C7DC9EE82E39789900524F6F /* libgraph_c.a */; productType = "com.apple.product-type.library.static"; }; + C7DF2CBF305C632700E8491C /* c_binding_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = C7DF2CC6305C632700E8491C /* Build configuration list for PBXNativeTarget "c_binding_test" */; + buildPhases = ( + C7DF2CBC305C632700E8491C /* Sources */, + C7DF2CBD305C632700E8491C /* Frameworks */, + C7DF2CBE305C632700E8491C /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + C7DF2CCC305C63B100E8491C /* PBXTargetDependency */, + C7DF2CCA305C63A900E8491C /* PBXTargetDependency */, + ); + name = c_binding_test; + packageProductDependencies = ( + ); + productName = c_binding_test; + productReference = C7DF2CC0305C632700E8491C /* c_binding_test */; + productType = "com.apple.product-type.tool"; + }; C7E5643D28A2A16F000F31A2 /* backend_test */ = { isa = PBXNativeTarget; buildConfigurationList = C7E5644228A2A16F000F31A2 /* Build configuration list for PBXNativeTarget "backend_test" */; @@ -1420,6 +1478,9 @@ C7DC9EE72E39789900524F6F = { CreatedOnToolsVersion = 16.4; }; + C7DF2CBF305C632700E8491C = { + CreatedOnToolsVersion = 27.0; + }; C7E5643D28A2A16F000F31A2 = { CreatedOnToolsVersion = 13.4; }; @@ -1486,6 +1547,7 @@ C74F2AE22F6DE8C500B48216 /* workflow_test */, C715C7932FD09E29003EEFF4 /* pic_test */, C762633F2FE0882200F283DF /* logical_test */, + C7DF2CBF305C632700E8491C /* c_binding_test */, ); }; /* End PBXProject section */ @@ -1610,6 +1672,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C7DF2CBC305C632700E8491C /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C7DF2CC7305C635500E8491C /* c_binding_test.c in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; C7E5643A28A2A16F000F31A2 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1755,6 +1825,16 @@ target = C79141A522DA9BF200E0BA0D /* graph_framework */; targetProxy = C7DC9EED2E39791C00524F6F /* PBXContainerItemProxy */; }; + C7DF2CCA305C63A900E8491C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = C7DC9EE72E39789900524F6F /* graph_c */; + targetProxy = C7DF2CC9305C63A900E8491C /* PBXContainerItemProxy */; + }; + C7DF2CCC305C63B100E8491C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = C79141A522DA9BF200E0BA0D /* graph_framework */; + targetProxy = C7DF2CCB305C63B100E8491C /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ @@ -2611,7 +2691,7 @@ "$(inherited)", ); LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.5; + MACOSX_DEPLOYMENT_TARGET = 26.6; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; }; @@ -2626,12 +2706,234 @@ EXECUTABLE_PREFIX = lib; GCC_C_LANGUAGE_STANDARD = gnu23; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.5; + MACOSX_DEPLOYMENT_TARGET = 26.6; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; }; name = Release; }; + C7DF2CC4305C632700E8491C /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.7; + OTHER_LDFLAGS = ( + "-lc++", + "-lnetcdf", + "-ld_classic", + "-L/Users/m4c/Projects/graph_framework/build/_deps/llvm-build/lib", + "-lz", + "-lLLVMCoverage", + "-lLLVMSupport", + "-lLLVMDebugInfoCodeView", + "-lLLVMRemarks", + "-lLLVMJITLink", + "-lLLVMLinker", + "-lLLVMTextAPI", + "-lLLVMRuntimeDyld", + "-lLLVMOrcShared", + "-lLLVMOrcDebugging", + "-lLLVMOrcTargetProcess", + "-lLLVMOrcJIT", + "-lLLVMHipStdPar", + "-lLLVMAggressiveInstCombine", + "-lLLVMVectorize", + "-lLLVMAsmParser", + "-lLLVMOption", + "-lLLVMLTO", + "-lLLVMObject", + "-lLLVMWindowsDriver", + "-lLLVMDemangle", + "-lLLVMIRReader", + "-lLLVMIRPrinter", + "-lLLVMInstCombine", + "-lLLVMBinaryFormat", + "-lLLVMCoroutines", + "-lLLVMBitstreamReader", + "-lLLVMBitReader", + "-lLLVMBitWriter", + "-lLLVMDebugInfoDWARF", + "-lLLVMInstrumentation", + "-lLLVMCFGuard", + "-lLLVMObjCARCOpts", + "-lLLVMipo", + "-lLLVMGlobalISel", + "-lLLVMExecutionEngine", + "-lLLVMFrontendDriver", + "-lLLVMFrontendHLSL", + "-lLLVMFrontendOpenMP", + "-lLLVMFrontendDirective", + "-lLLVMFrontendOffloading", + "-lLLVMSelectionDAG", + "-lLLVMProfileData", + "-lLLVMAnalysis", + "-lLLVMScalarOpts", + "-lLLVMCodeGenTypes", + "-lLLVMCodeGen", + "-lLLVMTargetParser", + "-lLLVMScalarOpts", + "-lLLVMTarget", + "-lLLVMTransformUtils", + "-lLLVMPasses", + "-lLLVMSupport", + "-lLLVMMCParser", + "-lLLVMMC", + "-lLLVMCore", + "-lLLVMAsmPrinter", + "-lLLVMAArch64Utils", + "-lLLVMAArch64Info", + "-lLLVMAArch64Desc", + "-lLLVMAArch64AsmParser", + "-lLLVMDebugInfoDWARFLowLevel", + "-lLLVMAArch64CodeGen", + "-lLLVMCGData", + "-lLLVMSandboxIR", + "-lLLVMObjectYAML", + "-lLLVMPlugins", + "-lLLVMABI", + "-lLLVMFrontendAtomic", + "-lclangFrontend", + "-lclangBasic", + "-lclangEdit", + "-lclangLex", + "-lclangDriver", + "-lclangSerialization", + "-lclangAST", + "-lclangSema", + "-lclangAnalysisLifetimeSafety", + "-lclangAnalysis", + "-lclangASTMatchers", + "-lclangSupport", + "-lclangParse", + "-lclangAPINotes", + "-lclangOptions", + "-lclangCodeGenUtils", + "-lclangCodeGen", + "-rpath", + /usr/local/lib, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + C7DF2CC5305C632700E8491C /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu17; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 26.7; + OTHER_LDFLAGS = ( + "-lc++", + "-lnetcdf", + "-ld_classic", + "-L/Users/m4c/Projects/graph_framework/build/_deps/llvm-build/lib", + "-lz", + "-lLLVMCoverage", + "-lLLVMSupport", + "-lLLVMDebugInfoCodeView", + "-lLLVMRemarks", + "-lLLVMJITLink", + "-lLLVMLinker", + "-lLLVMTextAPI", + "-lLLVMRuntimeDyld", + "-lLLVMOrcShared", + "-lLLVMOrcDebugging", + "-lLLVMOrcTargetProcess", + "-lLLVMOrcJIT", + "-lLLVMHipStdPar", + "-lLLVMAggressiveInstCombine", + "-lLLVMVectorize", + "-lLLVMAsmParser", + "-lLLVMOption", + "-lLLVMLTO", + "-lLLVMObject", + "-lLLVMWindowsDriver", + "-lLLVMDemangle", + "-lLLVMIRReader", + "-lLLVMIRPrinter", + "-lLLVMInstCombine", + "-lLLVMBinaryFormat", + "-lLLVMCoroutines", + "-lLLVMBitstreamReader", + "-lLLVMBitReader", + "-lLLVMBitWriter", + "-lLLVMDebugInfoDWARF", + "-lLLVMInstrumentation", + "-lLLVMCFGuard", + "-lLLVMObjCARCOpts", + "-lLLVMipo", + "-lLLVMGlobalISel", + "-lLLVMExecutionEngine", + "-lLLVMFrontendDriver", + "-lLLVMFrontendHLSL", + "-lLLVMFrontendOpenMP", + "-lLLVMFrontendDirective", + "-lLLVMFrontendOffloading", + "-lLLVMSelectionDAG", + "-lLLVMProfileData", + "-lLLVMAnalysis", + "-lLLVMScalarOpts", + "-lLLVMCodeGenTypes", + "-lLLVMCodeGen", + "-lLLVMTargetParser", + "-lLLVMScalarOpts", + "-lLLVMTarget", + "-lLLVMTransformUtils", + "-lLLVMPasses", + "-lLLVMSupport", + "-lLLVMMCParser", + "-lLLVMMC", + "-lLLVMCore", + "-lLLVMAsmPrinter", + "-lLLVMAArch64Utils", + "-lLLVMAArch64Info", + "-lLLVMAArch64Desc", + "-lLLVMAArch64AsmParser", + "-lLLVMDebugInfoDWARFLowLevel", + "-lLLVMAArch64CodeGen", + "-lLLVMCGData", + "-lLLVMSandboxIR", + "-lLLVMObjectYAML", + "-lLLVMPlugins", + "-lLLVMABI", + "-lLLVMFrontendAtomic", + "-lclangFrontend", + "-lclangBasic", + "-lclangEdit", + "-lclangLex", + "-lclangDriver", + "-lclangSerialization", + "-lclangAST", + "-lclangSema", + "-lclangAnalysisLifetimeSafety", + "-lclangAnalysis", + "-lclangASTMatchers", + "-lclangSupport", + "-lclangParse", + "-lclangAPINotes", + "-lclangOptions", + "-lclangCodeGenUtils", + "-lclangCodeGen", + "-rpath", + /usr/local/lib, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; C7E5644328A2A16F000F31A2 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -3030,6 +3332,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + C7DF2CC6305C632700E8491C /* Build configuration list for PBXNativeTarget "c_binding_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C7DF2CC4305C632700E8491C /* Debug */, + C7DF2CC5305C632700E8491C /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; C7E5644228A2A16F000F31A2 /* Build configuration list for PBXNativeTarget "backend_test" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index 7956c3c..bdbf771 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -932,8 +932,10 @@ namespace gpu { } source_buffer << "index] = "; if constexpr (SAFE_MATH) { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : "; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } } source_buffer << registers[a.get()] << ";" << std::endl; out_registers.insert(out.get()); diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index a109841..11b08ae 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -171,13 +171,7 @@ namespace graph { jit::add_type (stream); stream << " apply_u(const "; jit::add_type (stream); - stream << " x, const uint8_t i, "; - if constexpr (std::same_as) { - stream << "uint32_t"; - } else { - stream << "uint64_t"; - } - stream << " rand, const "; + stream << " x, const uint8_t i, uint32_t rand, const "; jit::add_type (stream); stream << " mof, const "; jit::add_type (stream); @@ -248,18 +242,7 @@ namespace graph { stream << " " << registers[this] << " = apply_u(" << registers[x.get()] << ", " << registers[i.get()] << ", "; - if (jit::use_metal ()) { - stream << "as_type<"; - } else { - stream << "bit_cast<"; - } - if constexpr (std::same_as) { - stream << "uint32_t"; - } else { - stream << "uint64_t"; - } - stream << "> (" - << registers[rand.get()] << "), " + stream << registers[rand.get()] << ", " << registers[mof.get()] << ", " << registers[tbnu_e_dt.get()] << ", " << registers[A.get()] << ", " @@ -541,13 +524,7 @@ namespace graph { jit::add_type (stream); stream << " apply_xi(const "; jit::add_type (stream); - stream << " x, const uint8_t i, "; - if constexpr (std::same_as) { - stream << "uint32_t"; - } else { - stream << "uint64_t"; - } - stream << " rand, const "; + stream << " x, const uint8_t i, uint32_t rand, const "; jit::add_type (stream); stream << " nu_D_dt) {" << std::endl << " "; @@ -609,18 +586,7 @@ namespace graph { stream << " " << registers[this] << " = apply_xi(" << registers[x.get()] << ", " << registers[i.get()] << ", "; - if (jit::use_metal ()) { - stream << "as_type<"; - } else { - stream << "bit_cast<"; - } - if constexpr (std::same_as) { - stream << "uint32_t"; - } else { - stream << "uint64_t"; - } - stream << "> (" - << registers[rand.get()] << "), " + stream << registers[rand.get()] << ", " << registers[nu_D_dt.get()] << ")"; this->endline(stream, usage); } @@ -1525,7 +1491,7 @@ namespace pic { // Velocity Scattering operator. auto nu_e_dt = build_colision_rate (xab, mass_a, mass_b, gb, nuab0, erfp_xab)*dt; - auto steps = graph::min(nu_e_dt*2.5, static_cast (sizeof(T)*8)); + auto steps = graph::min(nu_e_dt*2.5, static_cast (32)); nu_e_dt = nu_e_dt*steps; @@ -1542,7 +1508,7 @@ namespace pic { sphere[0] = graph::sqrt(u_op); auto nu_D_dt = nuab0*(erf_xab - gb)/(xab*xab*xab)*dt; - steps = graph::min(nu_D_dt, static_cast (sizeof(T)*8)); + steps = graph::min(nu_D_dt, static_cast (32)); nu_D_dt = nu_D_dt/steps; auto rand2 = graph::random (state); diff --git a/graph_framework/random.hpp b/graph_framework/random.hpp index 7b22df4..085d4fd 100644 --- a/graph_framework/random.hpp +++ b/graph_framework/random.hpp @@ -311,31 +311,25 @@ namespace graph { /// @param[in,out] stream String buffer stream. //------------------------------------------------------------------------------ static void compile_random(std::ostringstream &stream) { - jit::add_type (stream); - stream << " random("; + stream << "uint32_t random("; if constexpr (jit::use_metal ()) { stream << "device "; } - stream << "mt_state &state) {" << std::endl - << " uint16_t k = state.index;" << std::endl - << " uint16_t j = (k + 1) % 624;" << std::endl - << " uint32_t x = (state.array[k] & 0x80000000U) |" << std::endl - << " (state.array[j] & 0x7fffffffU);" << std::endl - << " uint32_t xA = x >> 1;" << std::endl - << " if (x & 0x00000001U) {" << std::endl - << " xA ^= 0x9908b0dfU;" << std::endl - << " }" << std::endl - << " j = (k + 397) % 624;" << std::endl - << " x = state.array[j]^xA;" << std::endl - << " state.array[k] = x;" << std::endl - << " state.index = (k + 1) % 624;" << std::endl - << " uint32_t y = x^(x >> 11);" << std::endl - << " y = y^((y << 7) & 0x9d2c5680U);" << std::endl - << " y = y^((y << 15) & 0xefc60000U);" << std::endl - << " return static_cast<"; - jit::add_type (stream); - stream << "> (y^(y >> 18));" << std::endl - << "}" << std::endl; + stream << "mt_state &state) {" << std::endl + << " const uint16_t k = state.index;" << std::endl + << " state.index = (k + 1) % 624;" << std::endl + << " uint32_t x = (state.array[k] & 0x80000000U) |" << std::endl + << " (state.array[state.index] & 0x7fffffffU);" << std::endl + << " uint32_t xA = x >> 1;" << std::endl + << " xA = x & 0x1U ? xA^0x9908b0dfU : xA;" << std::endl + << " const uint16_t j = (k + 397) % 624;" << std::endl + << " x = state.array[j]^xA;" << std::endl + << " state.array[k] = x;" << std::endl + << " uint32_t y = x^(x >> 11);" << std::endl + << " y = y^((y << 7) & 0x9d2c5680U);" << std::endl + << " y = y^((y << 15) & 0xefc60000U);" << std::endl + << " return y^(y >> 18);" << std::endl + << "}" << std::endl; } //------------------------------------------------------------------------------ @@ -429,7 +423,14 @@ namespace graph { auto a = this->arg->compile(stream, registers, thread_mem, usage); - registers[this] = "random(" + registers[a.get()] + ")"; + if constexpr (jit::complex_scalar) { + registers[this] = "static_cast<" + + jit::get_type_string () + + "> (random(" + + registers[a.get()] + "))"; + } else { + registers[this] = "random(" + registers[a.get()] + ")"; + } } return this->shared_from_this(); diff --git a/graph_pic/xpic.cpp b/graph_pic/xpic.cpp index 353b075..a6abe04 100644 --- a/graph_pic/xpic.cpp +++ b/graph_pic/xpic.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include "../graph_framework/graph_framework.hpp" @@ -74,22 +74,22 @@ void run_pic() { std::vector> p_datasets(num_ions, output::data_set (p_file)); - std::vector ion_sync(num_ions); - std::mutex mesh_sync; + std::vector ion_sync; + std::thread mesh_sync; + + work.template add_zero_item ({ + graph::variable_cast(mesh.y[0]) + }); + + std::vector, 3>> mesh_solves; for (size_t i = 0; i < num_ions; i++) { const std::string ion_tag = jit::format_to_string(i); - + auto ion_inits = pic::build_initialization (ions[i], mesh, norms, params, graph::random_state_cast(state)); - - if (i == 0) { - work.template add_zero_item ({ - graph::variable_cast(mesh.y[0]) - }); - } - + work.template add_item ({ ions[i].get_x(), ions[i].get_v_para(), ions[i].get_v_perp() }, {}, { @@ -99,49 +99,44 @@ void run_pic() { }, {}, graph::random_state_cast(state), "pre_initization_" + ion_tag, num_particles); - auto mesh_solve = mesh.build_mesh_solve(ions[i]); + work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { + ion_sync.push_back(std::thread([i, &p_file, &p_datasets]() { + p_datasets[i].write(p_file); + })); + }); + + mesh_solves.emplace_back(mesh.build_mesh_solve(ions[i])); work.template add_item ({ ions[i].get_x() }, { - mesh_solve[0], - mesh_solve[1], - mesh_solve[2] + mesh_solves[i][0], + mesh_solves[i][1], + mesh_solves[i][2] }, {}, { graph::variable_cast(mesh.y[0]) }, NULL, "pre_sum_weights_" + ion_tag, num_particles); + } - if (i == ions.size() - 1) { - work.template add_copy_item ({ - {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])}, - {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[2])}, - {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[3])} - }); - } + work.template add_copy_item ({ + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[3])} + }); - work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { - ion_sync[i].lock(); - std::thread async([i, &p_file, &p_datasets, &ion_sync]() { - p_datasets[i].write(p_file); - ion_sync[i].unlock(); - }); - async.detach(); + work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_sync = std::thread([&f_file, &mesh_dataset]() { + mesh_dataset.write(f_file); }); - if (i == 0) { - work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { - mesh_sync.lock(); - std::thread async([&f_file, &mesh_dataset, &mesh_sync]() { - mesh_dataset.write(f_file); - mesh_sync.unlock(); - }); - async.detach(); - }); - } + }); + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); work.add_callback_item([i, &ion_sync]() { - ion_sync[i].lock(); - ion_sync[i].unlock(); + if (ion_sync[i].joinable()) { + ion_sync[i].join(); + } }); - + auto particle_step = pic::build_rk4_step(ions[i], mesh, norms, params); work.add_item({ ions[i].get_x(), @@ -156,7 +151,7 @@ void run_pic() { {particle_step[1], ions[i].get_v_para()}, {particle_step[2], ions[i].get_v_perp()} }, {}, NULL, "particle_push_" + ion_tag, num_particles); - + auto particle_reinject = pic::build_reinjection(ions[i], mesh, norms, params, graph::random_state_cast(state)); work.add_item({ @@ -170,30 +165,55 @@ void run_pic() { }, {}, graph::random_state_cast(state), "particle_reinjection_" + ion_tag, num_particles); - if (i == 0) { - work.add_callback_item([&mesh_sync]() { - mesh_sync.lock(); - mesh_sync.unlock(); - }); - work.add_copy_item({ - {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, - {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, - {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])} - }); - work.add_zero_item({ - graph::variable_cast(mesh.y[0]) + work.template add_callback_item ([i, &p_file, &p_datasets, &ion_sync]() { + ion_sync[i] = std::thread([i, &p_file, &p_datasets]() { + p_datasets[i].write(p_file); }); + }); + } + + work.add_callback_item([&mesh_sync]() { + if (mesh_sync.joinable()) { + mesh_sync.join(); } + }); + work.add_copy_item({ + {graph::variable_cast(mesh.y[2]), graph::variable_cast(mesh.y[3])}, + {graph::variable_cast(mesh.y[1]), graph::variable_cast(mesh.y[2])}, + {graph::variable_cast(mesh.y[0]), graph::variable_cast(mesh.y[1])} + }); + work.add_zero_item({ + graph::variable_cast(mesh.y[0]) + }); + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + work.add_item({ graph::variable_cast(ions[i].x) }, { - mesh_solve[0], - mesh_solve[1], - mesh_solve[2] + mesh_solves[i][0], + mesh_solves[i][1], + mesh_solves[i][2] }, {}, { graph::variable_cast(mesh.y[0]) }, NULL, "sum_weights_" + ion_tag, num_particles); + } + + work.template add_callback_item ([&f_file, &mesh_dataset, &mesh_sync]() { + mesh_sync = std::thread([&f_file, &mesh_dataset]() { + mesh_dataset.write(f_file); + }); + }); + + for (size_t i = 0; i < num_ions; i++) { + const std::string ion_tag = jit::format_to_string(i); + + work.add_callback_item([i, &ion_sync]() { + if (ion_sync[i].joinable()) { + ion_sync[i].join(); + } + }); graph::shared_leaf total_density = graph::zero (); graph::shared_leaf total_flux = graph::zero (); @@ -268,14 +288,10 @@ void run_pic() { #endif const timing::measure_diagnostic run("Run Time"); work.template run (); - work.wait(); - work.template run (); - for (; counter < num_steps; counter++) { for (size_t i = 0; i < num_sub_steps; i++) { work.run(); } - work.wait(); work.template run (); } @@ -284,12 +300,10 @@ void run_pic() { #ifndef PROFILE_KERNELS progress.join(); #endif - for (std::mutex &ion : ion_sync) { - ion.lock(); - ion.unlock(); + for (std::thread &ion : ion_sync) { + ion.join(); } - mesh_sync.lock(); - mesh_sync.unlock(); + mesh_sync.join(); std::cout << "\33[2K\r" << "100% Complete" << std::endl; run.print(); diff --git a/graph_tests/backend_test.cpp b/graph_tests/backend_test.cpp index 98bb593..e9df7a9 100644 --- a/graph_tests/backend_test.cpp +++ b/graph_tests/backend_test.cpp @@ -359,18 +359,18 @@ template void test_backend() { const backend::buffer arctanvec = backend::atan(avec, bvec); assert(arctanvec.size() == 2 && "Expected a size of 2"); if constexpr (jit::complex_scalar) { - assert(arctanvec.at(0) == std::atan(static_cast (3.0)/ - static_cast (1.0)) && + assert(arctanvec.at(0) == static_cast (std::atan(static_cast (3.0)/ + static_cast (1.0))) && "Expected a value of atan(3/1)."); - assert(arctanvec.at(1) == std::atan(static_cast (4.0)/ - static_cast (2.0)) && + assert(arctanvec.at(1) == static_cast (std::atan(static_cast (4.0)/ + static_cast (2.0))) && "Expected a value of atan(4/2)."); } else { - assert(arctanvec.at(0) == std::atan2(static_cast (3.0), - static_cast (1.0)) && + assert(arctanvec.at(0) == static_cast (std::atan2(static_cast (3.0), + static_cast (1.0))) && "Expected a value of atan2(3,1)."); - assert(arctanvec.at(1) == std::atan2(static_cast (4.0), - static_cast (2.0)) && + assert(arctanvec.at(1) == static_cast (std::atan2(static_cast (4.0), + static_cast (2.0))) && "Expected a value of atan2(4,2)."); } diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index 433ae52..e13fbe0 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -272,11 +272,7 @@ void run_tests(const enum graph_type type, assert(value[2] == 2.0f && "Value of dydm does not match."); assert(value[3] == 1.0f && "Value of dydb does not match."); assert(value[4] == 1.0f && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(value[5] == 2546248192.0f && "Value of rand does not match."); - } else { - assert(value[5] == 2357136128.0f && "Value of rand does not match."); - } + assert(value[5] == 2357136128.0f && "Value of rand does not match."); assert(value[6] == 1.0f && "Value of root does not match."); assert(value[7] == 4.0f && "Value of p1 does not match."); assert(value[8] == 8.0f && "Value of p2 does not match."); diff --git a/graph_tests/f_binding_test.f90 b/graph_tests/f_binding_test.f90 index 3ceb8a5..ff4d153 100644 --- a/graph_tests/f_binding_test.f90 +++ b/graph_tests/f_binding_test.f90 @@ -223,13 +223,8 @@ SUBROUTINE run_test_float(use_safe_math) CALL graph%copy_to_host(dydy, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(value(1) .eq. 2546248192.0_C_FLOAT, & - 'Value of rand does not match.') - ELSE - CALL assert(value(1) .eq. 2357136128.0_C_FLOAT, & - 'Value of rand does not match.') - END IF + CALL assert(value(1) .eq. 2357136128.0_C_FLOAT, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of root does not match.') CALL graph%copy_to_host(p1, value) From eafe501bbb9ae116673607aae408ace711c10dc6 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Thu, 17 Sep 2026 16:49:22 -0400 Subject: [PATCH 47/51] Convert device name and architexture to strings and Displaced maximum and currently used memory. --- graph_framework/metal_context.hpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index bdbf771..a5c59a8 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -94,11 +94,13 @@ namespace gpu { } if (jit::verbose) { - std::cout << "Metal GPU info." << std::endl; - std::cout << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl; - std::cout << " Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl; - std::cout << " Device name : " << device.name << std::endl; - std::cout << " Architecture : " << device.architecture << std::endl; + std::cout << "Metal GPU info." << std::endl + << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl + << " Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl + << " Device name : " << [device.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl + << " Architecture : " << [device.architecture.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl + << " Max buffer length : " << device.maxBufferLength << std::endl + << " Max working set : " << device.recommendedMaxWorkingSetSize << std::endl; } } @@ -258,12 +260,13 @@ namespace gpu { NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0); if (jit::verbose) { - std::cout << " Kernel name : " << kernel_name << std::endl; - std::cout << " Thread execution width : " << thread_width << std::endl; - std::cout << " Threads per group : " << threads_per_group << std::endl; - std::cout << " Number of groups : " << thread_groups << std::endl; - std::cout << " Total problem size : " << threads_per_group*thread_groups << std::endl; - std::cout << " Total parallel size : " << total_parallel << std::endl; + std::cout << " Kernel name : " << kernel_name << std::endl + << " Thread execution width : " << thread_width << std::endl + << " Threads per group : " << threads_per_group << std::endl + << " Number of groups : " << thread_groups << std::endl + << " Total problem size : " << threads_per_group*thread_groups << std::endl + << " Total parallel size : " << total_parallel << std::endl + << " Current allocation size : " << device.currentAllocatedSize << std::endl; } if (state.get()) { From 0927a42877004e2ca39f158096069bef6970f5ce Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Fri, 18 Sep 2026 11:52:09 -0400 Subject: [PATCH 48/51] The max working set indicates that we can use more random number states. --- graph_framework/metal_context.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph_framework/metal_context.hpp b/graph_framework/metal_context.hpp index a5c59a8..ff48d3f 100644 --- a/graph_framework/metal_context.hpp +++ b/graph_framework/metal_context.hpp @@ -41,7 +41,7 @@ namespace gpu { public: /// Random state size multiplier. - constexpr static size_t random_state_scale = 1000; + constexpr static size_t random_state_scale = 3000; /// Size of random state needed. constexpr static size_t random_state_size = 1024*random_state_scale; From c14d95750d91b159e9620e0f97482d06481895a2 Mon Sep 17 00:00:00 2001 From: m4c Date: Fri, 18 Sep 2026 15:15:01 -0400 Subject: [PATCH 49/51] Relax test tolarance for complex atan float. --- graph_framework/cuda_context.hpp | 4 +-- graph_framework/particle_in_cell.hpp | 48 ++++++++++++++++++++++++---- graph_tests/backend_test.cpp | 13 ++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 6475b2e..3b2d843 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -109,7 +109,7 @@ namespace gpu { public: /// Random state size multiplyer. - constexpr static size_t random_state_scale = 1000; + constexpr static size_t random_state_scale = 3000; /// Size of random state needed. constexpr static size_t random_state_size = 1024*random_state_scale; @@ -342,7 +342,7 @@ namespace gpu { check_error(cuModuleGetFunction(&function, module, kernel_name.c_str()), "cuModuleGetFunction"); std::vector buffers; - std::set *> needed_buffers; + std::unordered_set *> needed_buffers; const size_t buffer_element_size = sizeof(T); for (auto &input : inputs) { diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 11b08ae..1e5ec52 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -171,7 +171,13 @@ namespace graph { jit::add_type (stream); stream << " apply_u(const "; jit::add_type (stream); - stream << " x, const uint8_t i, uint32_t rand, const "; + stream << " x, const "; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " i, uint32_t rand, const "; jit::add_type (stream); stream << " mof, const "; jit::add_type (stream); @@ -183,11 +189,23 @@ namespace graph { << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl - << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " for ("; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); stream << " E0 = mof*temp_x;" << std::endl - << " const uint8_t rm = 4*(rand & 1) - 2;" << std::endl + << " const "; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " rm = 4*(rand & 1) - 2;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl @@ -524,17 +542,35 @@ namespace graph { jit::add_type (stream); stream << " apply_xi(const "; jit::add_type (stream); - stream << " x, const uint8_t i, uint32_t rand, const "; + stream << " x, const "; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " i, uint32_t rand, const "; jit::add_type (stream); stream << " nu_D_dt) {" << std::endl << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl - << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " for ("; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); stream << " A = -temp_x*nu_D_dt;" << std::endl - << " const uint8_t rm = 2*(rand & 1) - 1;" << std::endl + << " const "; + if constexpr (jit::use_cuda()) { + stream << "unsigned char"; + } else { + stream << "uint8_t"; + } + stream << " rm = 2*(rand & 1) - 1;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt((1 - temp_x*temp_x)*nu_D_dt);" << std::endl diff --git a/graph_tests/backend_test.cpp b/graph_tests/backend_test.cpp index e9df7a9..1ac44ac 100644 --- a/graph_tests/backend_test.cpp +++ b/graph_tests/backend_test.cpp @@ -359,9 +359,18 @@ template void test_backend() { const backend::buffer arctanvec = backend::atan(avec, bvec); assert(arctanvec.size() == 2 && "Expected a size of 2"); if constexpr (jit::complex_scalar) { - assert(arctanvec.at(0) == static_cast (std::atan(static_cast (3.0)/ - static_cast (1.0))) && + const T temp = arctanvec.at(0) + - static_cast (std::atan(static_cast (3.0)/ + static_cast (1.0))); + assert(std::imag(temp) == 0 && "Expected a value of atan(3/1)."); + if constexpr (jit::float_base) { + assert(std::real(temp) < 1.3E-7 && + "Expected a value of atan(3/1)."); + } else { + assert(std::real(temp) == 0 && + "Expected a value of atan(3/1)."); + } assert(arctanvec.at(1) == static_cast (std::atan(static_cast (4.0)/ static_cast (2.0))) && "Expected a value of atan(4/2)."); From 72dcf23cebc35068f33067497057aab0bafc5fac Mon Sep 17 00:00:00 2001 From: m4c Date: Tue, 22 Sep 2026 18:39:29 -0400 Subject: [PATCH 50/51] Fix cuda issues so unit tests pass. --- graph_framework/cuda_context.hpp | 29 +++++++------- graph_framework/particle_in_cell.hpp | 58 ++++++---------------------- graph_framework/register.hpp | 6 +-- graph_tests/jit_test.cpp | 4 +- graph_tests/pic_test.cpp | 2 +- 5 files changed, 30 insertions(+), 69 deletions(-) diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 3b2d843..671c2f0 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -356,8 +356,6 @@ namespace gpu { input->data(), input->size()*sizeof(T)), "cuMemcpyHtoD"); - buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); - needed_buffers.insert(input.get()); } if (!needed_buffers.contains(input.get())) { buffers.push_back(reinterpret_cast (&kernel_arguments[input.get()])); @@ -365,18 +363,18 @@ namespace gpu { } } for (auto &output : outputs) { - if (!kernel_arguments.contains(output.get())) { - kernel_arguments.try_emplace(output.get()); - check_error(cuMemAllocManaged(&kernel_arguments[output.get()], - num_rays*sizeof(T), - CU_MEM_ATTACH_GLOBAL), - "cuMemAllocManaged"); - buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); - needed_buffers.insert(output.get()); - } - if (!needed_buffers.contains(output.get())) { - buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); - needed_buffers.insert(output.get()); + if (!graph::atomic_accumulate_1D_cast(output).get()) { + if (!kernel_arguments.contains(output.get())) { + kernel_arguments.try_emplace(output.get()); + check_error(cuMemAllocManaged(&kernel_arguments[output.get()], + num_rays*sizeof(T), + CU_MEM_ATTACH_GLOBAL), + "cuMemAllocManaged"); + } + if (!needed_buffers.contains(output.get())) { + buffers.push_back(reinterpret_cast (&kernel_arguments[output.get()])); + needed_buffers.insert(output.get()); + } } } for (auto &atomic : atomics) { @@ -390,8 +388,6 @@ namespace gpu { atomic->data(), atomic->size()*sizeof(T)), "cuMemcpyHtoD"); - buffers.push_back(reinterpret_cast (&kernel_arguments[atomic.get()])); - needed_buffers.insert(atomic.get()); } if (!needed_buffers.contains(atomic.get())) { buffers.push_back(reinterpret_cast (&kernel_arguments[atomic.get()])); @@ -883,6 +879,7 @@ namespace gpu { source_buffer << "typedef unsigned int uint32_t;" << std::endl << "typedef unsigned short uint16_t;" << std::endl << "typedef short int16_t;" << std::endl + << "typedef unsigned char uint8_t;" << std::endl << "template" << std::endl << "class array {" << std::endl << "private:" << std::endl diff --git a/graph_framework/particle_in_cell.hpp b/graph_framework/particle_in_cell.hpp index 1e5ec52..1f2f659 100644 --- a/graph_framework/particle_in_cell.hpp +++ b/graph_framework/particle_in_cell.hpp @@ -171,13 +171,7 @@ namespace graph { jit::add_type (stream); stream << " apply_u(const "; jit::add_type (stream); - stream << " x, const "; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " i, uint32_t rand, const "; + stream << " x, const uint8_t i, uint32_t rand, const "; jit::add_type (stream); stream << " mof, const "; jit::add_type (stream); @@ -189,23 +183,11 @@ namespace graph { << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl - << " for ("; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); stream << " E0 = mof*temp_x;" << std::endl - << " const "; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " rm = 4*(rand & 1) - 2;" << std::endl + << " const uint8_t rm = 4*(rand & 1) - 2;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt(tbnu_e_dt*E0);" << std::endl @@ -542,35 +524,17 @@ namespace graph { jit::add_type (stream); stream << " apply_xi(const "; jit::add_type (stream); - stream << " x, const "; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " i, uint32_t rand, const "; + stream << " x, const uint8_t i, uint32_t rand, const "; jit::add_type (stream); stream << " nu_D_dt) {" << std::endl << " "; jit::add_type (stream); stream << " temp_x = x;" << std::endl - << " for ("; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " j = 0; j < i; j++, rand >>= 1) {" << std::endl + << " for (uint8_t j = 0; j < i; j++, rand >>= 1) {" << std::endl << " const "; jit::add_type (stream); stream << " A = -temp_x*nu_D_dt;" << std::endl - << " const "; - if constexpr (jit::use_cuda()) { - stream << "unsigned char"; - } else { - stream << "uint8_t"; - } - stream << " rm = 2*(rand & 1) - 1;" << std::endl + << " const uint8_t rm = 2*(rand & 1) - 1;" << std::endl << " const "; jit::add_type (stream); stream << " C = rm*sqrt((1 - temp_x*temp_x)*nu_D_dt);" << std::endl @@ -1414,10 +1378,10 @@ namespace pic { graph::shared_leaf total_flux, const graph::shared_random_state state) { const T dt = params.dt*norms.t; - + const T mass_b = ion_b.mass*norms.m; const uint8_t zb2 = ion_b.z*ion_b.z; - + auto nb = build_density(ion_a.x, ion_b, mesh, norms, params)/norms.get_volume(); auto tpara = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { return graph::one (); @@ -1430,7 +1394,7 @@ namespace pic { return graph::one (); })*norms.v/norms.get_volume(); auto uxb = nv/nb; - + total_density = total_density + nb; total_flux = total_flux + nv; @@ -1461,10 +1425,10 @@ namespace pic { graph::shared_leaf total_flux, const graph::shared_random_state state) { const T dt = params.dt*norms.t; - + const T mass_b = m_electron; const uint8_t zb2 = 1; - + auto nb = total_density; auto tb = ion_a.build_profile([](graph::shared_leaf x) -> graph::shared_leaf { return graph::one (); diff --git a/graph_framework/register.hpp b/graph_framework/register.hpp index 457c6c4..576d667 100644 --- a/graph_framework/register.hpp +++ b/graph_framework/register.hpp @@ -111,19 +111,19 @@ namespace jit { if constexpr (jit::use_metal ()) { return "ushort"; } else { - return "unsigned char"; + return "uint8_t"; } } else if (max_size <= std::numeric_limits::max()) { if constexpr (jit::use_metal ()) { return "ushort"; } else { - return "unsigned short"; + return "uint16_t"; } } else if (max_size <= std::numeric_limits::max()) { if constexpr (jit::use_metal ()) { return "uint"; } else { - return "unsigned int"; + return "uint32_t"; } } else { if constexpr (jit::use_metal ()) { diff --git a/graph_tests/jit_test.cpp b/graph_tests/jit_test.cpp index dc8240e..114323d 100644 --- a/graph_tests/jit_test.cpp +++ b/graph_tests/jit_test.cpp @@ -367,7 +367,7 @@ template void run_math_tests() { graph::variable_cast(v1), graph::variable_cast(v2) }, {if_node}, {}, false_v->evaluate().at(0), 0.0); - + if_node = graph::if_(v1 < v2, true_v, false_v); compile ({ graph::variable_cast(v1), @@ -378,7 +378,7 @@ template void run_math_tests() { compile ({ graph::variable_cast(v1), graph::variable_cast(v2) - }, {hypot_node}, {}, hypot_node->evaluate().at(0), 0.0); + }, {hypot_node}, {}, hypot_node->evaluate().at(0), 5.0E-16); auto min_node = graph::min(v1, v2); compile ({ diff --git a/graph_tests/pic_test.cpp b/graph_tests/pic_test.cpp index 5c6847c..6ccbf08 100644 --- a/graph_tests/pic_test.cpp +++ b/graph_tests/pic_test.cpp @@ -272,7 +272,7 @@ template void run_field_solve_test() { graph::variable_cast(mesh.y[0]) }); work.add_item({ - graph::variable_cast(ions[0].x) + ions[0].get_x() }, { mesh_solve[0], mesh_solve[1], From 1c2192a320f23d280c03f682d3fd952f1221bc79 Mon Sep 17 00:00:00 2001 From: Cianciosa Date: Tue, 22 Sep 2026 23:05:35 -0400 Subject: [PATCH 51/51] Fix random output during safe_math on the cuda and cpu backends. --- graph_framework/cpu_context.hpp | 58 ++++++++++++++++++++++---------- graph_framework/cuda_context.hpp | 58 +++++++++++++++++++++----------- graph_tests/c_binding_test.c | 20 +++-------- graph_tests/f_binding_test.f90 | 29 ++++------------ 4 files changed, 89 insertions(+), 76 deletions(-) diff --git a/graph_framework/cpu_context.hpp b/graph_framework/cpu_context.hpp index d776d9c..04fc05b 100644 --- a/graph_framework/cpu_context.hpp +++ b/graph_framework/cpu_context.hpp @@ -679,16 +679,27 @@ namespace gpu { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] - << ";" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] + << ";" << std::endl; } } else { source_buffer << registers[a.get()] << ";" << std::endl; @@ -708,16 +719,27 @@ namespace gpu { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] - << ";" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] + << ";" << std::endl; } } else { source_buffer << registers[a.get()] << ";" << std::endl; diff --git a/graph_framework/cuda_context.hpp b/graph_framework/cuda_context.hpp index 671c2f0..c2e36b8 100644 --- a/graph_framework/cuda_context.hpp +++ b/graph_framework/cuda_context.hpp @@ -1279,17 +1279,26 @@ namespace gpu { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" - << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" - << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] << ";" << std::endl; } } else { @@ -1316,17 +1325,26 @@ namespace gpu { if constexpr (jit::complex_scalar) { jit::add_type (source_buffer); source_buffer << " ("; - source_buffer << "isnan(real(" << registers[a.get()] - << ")) ? 0.0 : real(" - << registers[a.get()] - << "), "; - source_buffer << "isnan(imag(" << registers[a.get()] - << ")) ? 0.0 : imag(" - << registers[a.get()] - << "));" << std::endl; + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(real(" + << registers[a.get()] + << ")) ? 0.0 : real(" + << registers[a.get()] + << "), isnan(imag(" + << registers[a.get()] + << ")) ? 0.0 : imag(" + << registers[a.get()] + << ")"; + } else { + source_buffer << registers[a.get()]; + } + source_buffer << ");" << std::endl; } else { - source_buffer << "isnan(" << registers[a.get()] - << ") ? 0.0 : " << registers[a.get()] + if (!graph::random_cast(a).get()) { + source_buffer << "isnan(" << registers[a.get()] + << ") ? 0.0 : "; + } + source_buffer << registers[a.get()] << ";" << std::endl; } } else { diff --git a/graph_tests/c_binding_test.c b/graph_tests/c_binding_test.c index e13fbe0..a3dff5d 100644 --- a/graph_tests/c_binding_test.c +++ b/graph_tests/c_binding_test.c @@ -272,7 +272,7 @@ void run_tests(const enum graph_type type, assert(value[2] == 2.0f && "Value of dydm does not match."); assert(value[3] == 1.0f && "Value of dydb does not match."); assert(value[4] == 1.0f && "Value of dydy does not match."); - assert(value[5] == 2357136128.0f && "Value of rand does not match."); + assert(value[5] == (float)2357136044 && "Value of rand does not match."); assert(value[6] == 1.0f && "Value of root does not match."); assert(value[7] == 4.0f && "Value of p1 does not match."); assert(value[8] == 8.0f && "Value of p2 does not match."); @@ -299,11 +299,7 @@ void run_tests(const enum graph_type type, assert(value[2] == 2.0 && "Value of dydm does not match."); assert(value[3] == 1.0 && "Value of dydb does not match."); assert(value[4] == 1.0 && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(value[5] == 2546248239.0 && "Value of rand does not match."); - } else { - assert(value[5] == 2357136044.0 && "Value of rand does not match."); - } + assert(value[5] == (double)2357136044 && "Value of rand does not match."); assert(value[6] == 1.0 && "Value of root does not match."); assert(value[7] == 4.0 && "Value of p1 does not match."); assert(value[8] == 8.0 && "Value of p2 does not match."); @@ -330,11 +326,7 @@ void run_tests(const enum graph_type type, assert(crealf(value[2]) == 2.0f && "Value of dydm does not match."); assert(crealf(value[3]) == 1.0f && "Value of dydb does not match."); assert(crealf(value[4]) == 1.0f && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(crealf(value[5]) == 2546248192.0f && "Value of rand does not match."); - } else { - assert(crealf(value[5]) == 2357136128.0f && "Value of rand does not match."); - } + assert(crealf(value[5]) == crealf(2357136044) && "Value of rand does not match."); assert(crealf(value[6]) == 1.0f && "Value of root does not match."); assert(crealf(value[7]) == 4.0f && "Value of p1 does not match."); assert(crealf(value[8]) == 8.0f && "Value of p2 does not match."); @@ -361,11 +353,7 @@ void run_tests(const enum graph_type type, assert(creal(value[2]) == 2.0 && "Value of dydm does not match."); assert(creal(value[3]) == 1.0 && "Value of dydb does not match."); assert(creal(value[4]) == 1.0 && "Value of dydy does not match."); - if (c_context->safe_math) { - assert(creal(value[5]) == 2546248239.0 && "Value of rand does not match."); - } else { - assert(creal(value[5]) == 2357136044.0 && "Value of rand does not match."); - } + assert(creal(value[5]) == creal(2357136044) && "Value of rand does not match."); assert(creal(value[6]) == 1.0 && "Value of root does not match."); assert(creal(value[7]) == 4.0 && "Value of p1 does not match."); assert(creal(value[8]) == 8.0 && "Value of p2 does not match."); diff --git a/graph_tests/f_binding_test.f90 b/graph_tests/f_binding_test.f90 index ff4d153..6562ac1 100644 --- a/graph_tests/f_binding_test.f90 +++ b/graph_tests/f_binding_test.f90 @@ -223,7 +223,7 @@ SUBROUTINE run_test_float(use_safe_math) CALL graph%copy_to_host(dydy, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - CALL assert(value(1) .eq. 2357136128.0_C_FLOAT, & + CALL assert(value(1) .eq. 2357136044.0_C_FLOAT, & 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(value(1) .eq. 1.0_C_FLOAT, 'Value of root does not match.') @@ -412,13 +412,8 @@ SUBROUTINE run_test_double(use_safe_math) CALL graph%copy_to_host(dydy, value) CALL assert(value(1) .eq. 1.0_C_DOUBLE, 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(value(1) .eq. 2546248239.0_C_DOUBLE, & - 'Value of rand does not match.') - ELSE - CALL assert(value(1) .eq. 2357136044.0_C_DOUBLE, & - 'Value of rand does not match.') - END IF + CALL assert(value(1) .eq. 2357136044_C_DOUBLE, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(value(1) .eq. 1.0_C_DOUBLE, 'Value of root does not match.') CALL graph%copy_to_host(p1, value) @@ -614,13 +609,8 @@ SUBROUTINE run_test_complex_float(use_safe_math) CALL assert(REAL(value(1)) .eq. 1.0_C_FLOAT, & 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(REAL(value(1)) .eq. 2546248192.0_C_FLOAT, & - 'Value of rand does not match.') - ELSE - CALL assert(REAL(value(1)) .eq. 2357136128.0_C_FLOAT, & - 'Value of rand does not match.') - END IF + CALL assert(REAL(value(1)) .eq. 2357136044.0_C_FLOAT, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(REAL(value(1)) .eq. 1.0_C_FLOAT, & 'Value of root does not match.') @@ -834,13 +824,8 @@ SUBROUTINE run_test_complex_double(use_safe_math) CALL assert(DBLE(value(1)) .eq. 1.0_C_DOUBLE, & 'Value of dydy does not match.') CALL graph%copy_to_host(rand, value) - IF (use_safe_math) THEN - CALL assert(DBLE(value(1)) .eq. 2546248239.0_C_DOUBLE, & - 'Value of rand does not match.') - ELSE - CALL assert(DBLE(value(1)) .eq. 2357136044.0_C_DOUBLE, & - 'Value of rand does not match.') - END IF + CALL assert(DBLE(value(1)) .eq. 2357136044_C_DOUBLE, & + 'Value of rand does not match.') CALL graph%copy_to_host(z, value) CALL assert(DBLE(value(1)) .eq. 1.0_C_DOUBLE, & 'Value of root does not match.')