MadMatrix perf me timing

madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc

@@ -481,7 +481,7 @@ namespace madmatrix
   // helicity loop leaves it. Returns the |M|^2 of the event (before the helicity/color
   // average and the channel weight) and sets selhel; a vanishing |M|^2 returns 0 and
   // leaves selhel alone, as the positive case does.
-  static inline fptype
+  [[maybe_unused]] static inline fptype
   select_helicity_signed( const fptype* ighelSum, // input: running sums over the good helicities
                           const fptype rndhel,    // input: random number for the helicity choice
                           int& selhel )           // output: chosen helicity (Fortran range [1,ncomb])

madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc

@@ -34,7 +34,7 @@ namespace madmatrix
   // is safe to mask on the squared order: sqSoIndex is symmetric (a squared
   // order is the SUM of the two amplitude orders), so a pair and its transpose
   // are always kept or dropped together and the masked total stays real.
-  static void
+  [[maybe_unused]] static void
   color_sum_cpu_splitorders( fptype* allMEs,                  // output: allMEs[nevt], add |M|^2 for one specific helicity
                              const cxtype_amp_sv* allJamp_sv, // input: jamp_sv[njampso] (float/double) or jamp_sv[2*njampso] (mixed) for one specific helicity
                              const int ievt0 )                // input: first event number in current C++ event page
@@ -272,15 +272,15 @@ namespace madmatrix
                  const float* beta, float* c, const int* ldc );
   }
 
-  static inline void
+  [[maybe_unused]] static inline void
   blas_symm( const int m, const int n, const double* a, const double* b, double* c )
   {
     const char side = 'L', uplo = 'U';
     const double alpha = 1, beta = 0;
     dsymm_( &side, &uplo, &m, &n, &alpha, a, &m, b, &m, &beta, c, &m );
   }
 
-  static inline void
+  [[maybe_unused]] static inline void
   blas_symm( const int m, const int n, const float* a, const float* b, float* c )
   {
     const char side = 'L', uplo = 'U';

madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc

@@ -482,7 +482,7 @@ namespace madmatrix
   // helicity loop leaves it. Returns the |M|^2 of the event (before the helicity/color
   // average and the channel weight) and sets selhel; a vanishing |M|^2 returns 0 and
   // leaves selhel alone, as the positive case does.
-  static inline fptype
+  [[maybe_unused]] static inline fptype
   select_helicity_signed( const fptype* ighelSum, // input: running sums over the good helicities
                           const fptype rndhel,    // input: random number for the helicity choice
                           int& selhel )           // output: chosen helicity (Fortran range [1,ncomb])

madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc

@@ -34,7 +34,7 @@ namespace madmatrix
   // is safe to mask on the squared order: sqSoIndex is symmetric (a squared
   // order is the SUM of the two amplitude orders), so a pair and its transpose
   // are always kept or dropped together and the masked total stays real.
-  static void
+  [[maybe_unused]] static void
   color_sum_cpu_splitorders( fptype* allMEs,                  // output: allMEs[nevt], add |M|^2 for one specific helicity
                              const cxtype_amp_sv* allJamp_sv, // input: jamp_sv[njampso] (float/double) or jamp_sv[2*njampso] (mixed) for one specific helicity
                              const int ievt0 )                // input: first event number in current C++ event page
@@ -272,15 +272,15 @@ namespace madmatrix
                  const float* beta, float* c, const int* ldc );
   }
 
-  static inline void
+  [[maybe_unused]] static inline void
   blas_symm( const int m, const int n, const double* a, const double* b, double* c )
   {
     const char side = 'L', uplo = 'U';
     const double alpha = 1, beta = 0;
     dsymm_( &side, &uplo, &m, &n, &alpha, a, &m, b, &m, &beta, c, &m );
   }
 
-  static inline void
+  [[maybe_unused]] static inline void
   blas_symm( const int m, const int n, const float* a, const float* b, float* c )
   {
     const char side = 'L', uplo = 'U';

madgraph/iolibs/template_files/madmatrix/check_sa.cc

@@ -206,6 +206,8 @@ namespace
       << "  --energy <GeV>    (perf only) Ecms for RAMBO (default 1500 GeV).\n"
       << "\n"
       << "Both modes read the parameters, alpha_s included, from ../../Cards/param_card.dat.\n"
+      << "perf times as MatrixElems (3) only the umami_matrix_element call (UMAMI input/output\n"
+      << "handling included); device-to-host copies are listed separately in the timer map.\n"
       << "\n"
       << "perf-mode defaults if positional args are omitted:\n"
       << "  #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n";
@@ -337,7 +339,7 @@ namespace
   )
   {
     constexpr unsigned int UmamiInKeyNum = 3;
-    timermap.start( "3a SigmaKin" );
+    timermap.start( "3a UmamiCall" );
     UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S };
     UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT };
 #ifdef MGONGPUCPP_GPUIMPL
@@ -349,6 +351,10 @@ namespace
 #endif
     UmamiStatus st = umami_matrix_element(
       handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs );
+#ifdef MGONGPUCPP_GPUIMPL
+    // UMAMI only enqueues the kernels; wait for them so the ME time is the computation, not the launch.
+    checkGpu( gpuDeviceSynchronize() );
+#endif
     wavetime += timermap.stop();
     if( st != UMAMI_SUCCESS )
     {
@@ -357,9 +363,10 @@ namespace
     }
 
 #ifdef MGONGPUCPP_GPUIMPL
+    // Device-to-host copies are reported by the timer map, not as ME time.
     timermap.start( "3b CpDTHmes" );
     gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost );
-    wavetime += timermap.stop();
+    timermap.stop();
 #endif
     return true;
   }
@@ -631,10 +638,7 @@ namespace
     constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads;
     const unsigned int nFlavors = CPPProcess::nmaxflavor;
 
-    mgOnGpu::TimerMap timermap;
-
 #ifdef MGONGPUCPP_GPUIMPL
-    timermap.start( "00 GpuInit" );
     GpuRuntime gpuRuntime( false );
 
     PinnedHostBufferRndNumMomenta hstRndmom( nevt );
@@ -751,7 +755,6 @@ namespace
 #ifdef MGONGPUCPP_GPUIMPL
       gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice );
 #endif
-      timermap.start( "3a SigmaKin" );
       UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S };
       UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT };
 #ifdef MGONGPUCPP_GPUIMPL
@@ -763,7 +766,6 @@ namespace
 #endif
       UmamiStatus st = umami_matrix_element(
         umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs );
-      timermap.stop();
       if( st != UMAMI_SUCCESS )
       {
         std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl;
@@ -934,8 +936,8 @@ namespace
     unsigned int nevtZERO = 0;
     double sumME = 0.;
     double sumMEsq = 0.;
-    double minME = std::numeric_limits<double>::infinity();
-    double maxME = -std::numeric_limits<double>::infinity();
+    double minME = std::numeric_limits<double>::max();
+    double maxME = std::numeric_limits<double>::lowest();
     unsigned int nevtALL = 0;
 
     for( unsigned int iiter = 0; iiter < niter; ++iiter )
@@ -956,7 +958,7 @@ namespace
         {
           timermap.start( "1c CpHTDrnd" );
           copyDeviceFromHost( devRndmom, hstRndmom );
-          genrtime += timermap.stop();
+          timermap.stop();
         }
 #endif
 
@@ -965,14 +967,18 @@ namespace
         rambtime += timermap.stop();
         timermap.start( "2b RamboFin" );
         prsk->getMomentaFinal();
+#ifdef MGONGPUCPP_GPUIMPL
+        // Massless RAMBO runs on the device
+        checkGpu( gpuDeviceSynchronize() );
+#endif
         rambtime += timermap.stop();
 #ifdef MGONGPUCPP_GPUIMPL
         // Massive host only (copy)
         if( ramboType != RAMBO_MASSLESS )
         {
           timermap.start( "2c CpHTDmom" );
           copyDeviceFromHost( devMomenta, hstMomenta );
-          rambtime += timermap.stop();
+          timermap.stop();
         }
 #endif
       }
@@ -996,14 +1002,15 @@ namespace
 #ifdef MGONGPUCPP_GPUIMPL
         timermap.start( "2c CpHTDmom" );
         copyDeviceFromHost( devMomenta, hstMomenta );
-        rambtime += timermap.stop();
+        timermap.stop();
 #endif
       }
 
       timermap.start( "2d Aosoa2U " );
 #ifdef MGONGPUCPP_GPUIMPL
       gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt );
       checkGpu( gpuPeekAtLastError() );
+      checkGpu( gpuDeviceSynchronize() );
 #else
       for( std::size_t ievt = 0; ievt < nevt; ++ievt )
         aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt );
@@ -1028,7 +1035,7 @@ namespace
       {
         timermap.start( "3c CpDTHmom" );
         copyHostFromDevice( hstMomenta, devMomenta );
-        wavetime += timermap.stop();
+        timermap.stop();
       }
       const double* mes = hstUmamiMEs.data();
 #else
@@ -1182,9 +1189,8 @@ int main( int argc, char** argv )
       mode = MODE_PERF; // legacy alias
     else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) )
       flavorID = strtoul( argv[++argn], nullptr, 0 );
-    else if( arg == "--rambo-massless"  )
+    else if( arg == "--rambo-massless" )
     {
-      std::string r = argv[++argn];
       ramboType = RAMBO_MASSLESS;
       ramboTypeSet = true;
     }

madgraph/iolibs/template_files/madmatrix/src/rambo/rambo.h

@@ -83,12 +83,11 @@ namespace rambo
     int iranmr, jranmr;
   };
 
-  inline double rn( int idummy )
+  inline double rn( [[maybe_unused]] int idummy )
   {
     static Random rand;
     double ran;
     static int init = 1;
-    if( false ) idummy = idummy; // prevent unused variable warning
     if( init == 1 )
     {
       init = 0;

madgraph/iolibs/template_files/madmatrix/timermap.h

@@ -125,26 +125,23 @@ namespace mgOnGpu
       }
       else
       {
-        // NB: 'setw' affects only the next field (of any type)
         ostr << std::setprecision( 6 ); // set precision (default=6): affects all floats
         ostr << std::fixed;             // fixed format: affects all floats
+        // aligned keys whatever key length.
+        const auto line = [&]( const std::string& key, float seconds )
+        {
+          ostr << std::left << std::setw( maxsize ) << key << std::right << " : "
+               << std::setw( 12 ) << seconds << " sec" << std::endl;
+        };
         for( auto ip: m_partitionTimers )
-          ostr << std::setw( maxsize ) << ip.first << " : "
-               << std::setw( 12 ) << ip.second << " sec" << std::endl;
-        ostr << std::setw( maxsize ) << totalKey << " : "
-             << std::setw( 12 ) << total << " sec" << std::endl
-             << std::setw( maxsize ) << total123Key << " : "
-             << std::setw( 12 ) << total123 << " sec" << std::endl
-             << std::setw( maxsize ) << total23Key << " : "
-             << std::setw( 12 ) << total23 << " sec" << std::endl
-             << std::setw( maxsize ) << total1Key << " : "
-             << std::setw( 12 ) << total1 << " sec" << std::endl
-             << std::setw( maxsize ) << total2Key << " : "
-             << std::setw( 12 ) << total2 << " sec" << std::endl
-             << std::setw( maxsize ) << total3Key << " : "
-             << std::setw( 12 ) << total3 << " sec" << std::endl
-             << std::setw( maxsize ) << total3aKey << " : "
-             << std::setw( 12 ) << total3a << " sec" << std::endl;
+          line( ip.first, ip.second );
+        line( totalKey, total );
+        line( total123Key, total123 );
+        line( total23Key, total23 );
+        line( total1Key, total1 );
+        line( total2Key, total2 );
+        line( total3Key, total3 );
+        line( total3aKey, total3a );
         ostr << std::defaultfloat; // default format: affects all floats
       }
     }