From c1594a0b4ac029129d85b9952d3befa832a38380 Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Fri, 28 Aug 2026 20:25:28 +0200
Subject: [PATCH 1/6] Fix CORTEXA72 Level-3 correctness (SYMM/TRMM/SYRK)

The A72 6x8 DGEMM path left SYMM packer remainders as (n-n/6), wired
DTRMM to a 2x2 ukernel against 6x8 packs, and left GEMM_UNROLL_MN at
MAX(6,8)=8. Correct the packers, add a contiguous 6x8 TRMM ukernel,
and set UNROLL_MN=6. Fixes #5997.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 kernel/arm64/KERNEL.CORTEXA72   |   5 +-
 kernel/generic/symm_lcopy_6.c   |   8 +-
 kernel/generic/symm_ucopy_6.c   |   8 +-
 kernel/generic/trmmkernel_6x8.c | 248 ++++++++++++++++++++++++++++++++
 param.h                         |   3 +
 5 files changed, 264 insertions(+), 8 deletions(-)
 create mode 100644 kernel/generic/trmmkernel_6x8.c

diff --git a/kernel/arm64/KERNEL.CORTEXA72 b/kernel/arm64/KERNEL.CORTEXA72
index f7d247e7ec..7fa9236eea 100644
--- a/kernel/arm64/KERNEL.CORTEXA72
+++ b/kernel/arm64/KERNEL.CORTEXA72
@@ -6,11 +6,12 @@ include $(KERNELDIR)/KERNEL.CORTEXA57
 #
 # Stock generic/gemm_{t,n}copy_6.c packs as 4+2, not contiguous MR=6 —
 # use the panel packers. NR=8 reuses the asm dgemm_{n,t}copy_8.S.
-# No dtrmm_kernel_6x8.S yet; fall back to generic 2x2 for DTRMM.
+# No tuned dtrmm_kernel_6x8.S yet; use a correctness-first generic 6x8
+# TRMM ukernel matching contiguous MR=6 / NR=8 packs (see #5997).
 # Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M; use the
 # UNROLLM6-aware kernels (same idea as loongarch64 UNROLLN6).
 DGEMMKERNEL    =  dgemm_kernel_6x8_cortexa72.S
-DTRMMKERNEL    =  ../generic/trmmkernel_2x2.c
+DTRMMKERNEL    =  ../generic/trmmkernel_6x8.c
 
 DTRSMKERNEL_LN =  trsm_kernel_LN_UNROLLM6.c
 DTRSMKERNEL_LT =  trsm_kernel_LT_UNROLLM6.c
diff --git a/kernel/generic/symm_lcopy_6.c b/kernel/generic/symm_lcopy_6.c
index 3a3e2d5b2f..0f5373820e 100644
--- a/kernel/generic/symm_lcopy_6.c
+++ b/kernel/generic/symm_lcopy_6.c
@@ -92,7 +92,9 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         js --;
     }
 
-    if ((n - n/6) & 4) {
+    /* Remainder must be n%6, not (n - n/6). The latter is wrong for
+     * non-multiples of 6 (e.g. n=7 -> 6 instead of 1) and corrupts SYMM. */
+    if ((n % 6) & 4) {
         offset = posX - posY;
 
         if (offset >  0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda;
@@ -127,7 +129,7 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         posX += 4;
     }
 
-    if ((n - n/6) & 2) {
+    if ((n % 6) & 2) {
 
         offset = posX - posY;
 
@@ -155,7 +157,7 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         posX += 2;
     }
 
-    if ((n - n/6) & 1) {
+    if ((n % 6) & 1) {
 
         offset = posX - posY;
 
diff --git a/kernel/generic/symm_ucopy_6.c b/kernel/generic/symm_ucopy_6.c
index a83d937d0b..18d40e629e 100644
--- a/kernel/generic/symm_ucopy_6.c
+++ b/kernel/generic/symm_ucopy_6.c
@@ -92,7 +92,9 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         js --;
     }
 
-    if ((n - n/6) & 4) {
+    /* Remainder must be n%6, not (n - n/6). The latter is wrong for
+     * non-multiples of 6 (e.g. n=7 -> 6 instead of 1) and corrupts SYMM. */
+    if ((n % 6) & 4) {
 
         offset = posX - posY;
 
@@ -128,7 +130,7 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         posX += 4;
     }
 
-    if ((n - n/6) & 2) {
+    if ((n % 6) & 2) {
         offset = posX - posY;
 
         if (offset >  0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda;
@@ -155,7 +157,7 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLON
         posX += 2;
     }
 
-    if ((n - n/6) & 1) {
+    if ((n % 6) & 1) {
         offset = posX - posY;
 
         if (offset >  0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda;
diff --git a/kernel/generic/trmmkernel_6x8.c b/kernel/generic/trmmkernel_6x8.c
new file mode 100644
index 0000000000..4de13adad7
--- /dev/null
+++ b/kernel/generic/trmmkernel_6x8.c
@@ -0,0 +1,248 @@
+#include "common.h"
+
+/* Correctness-first TRMM microkernel for UNROLL_M=6, UNROLL_N=8.
+ * Matches contiguous MR=6 / NR=8 Goto packs (TARGET=CORTEXA72, #5997).
+ * Not performance-tuned; replaces the incorrect trmmkernel_2x2 fallback.
+ */
+
+int CNAME(BLASLONG bm, BLASLONG bn, BLASLONG bk, FLOAT alpha, FLOAT *ba, FLOAT *bb, FLOAT *C, BLASLONG ldc
+#ifdef TRMMKERNEL
+	  , BLASLONG offset
+#endif
+	  )
+{
+  BLASLONG i, j, k, ii, jj;
+  FLOAT *Cptr, *ptrba, *ptrbb;
+  FLOAT res[6][8];
+  FLOAT a;
+  BLASLONG off, temp;
+  BLASLONG mr, nr, mrem, nrem;
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+  off = -offset;
+#else
+  off = 0;
+#endif
+
+  for (j = 0; j < bn / 8; j++) {
+    Cptr = C;
+#if defined(TRMMKERNEL) && defined(LEFT)
+    off = offset;
+#endif
+    ptrba = ba;
+
+    for (i = 0; i < bm / 6; i++) {
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+      ptrbb = bb;
+#else
+      ptrba += off * 6;
+      ptrbb = bb + off * 8;
+#endif
+      for (ii = 0; ii < 6; ii++)
+	for (jj = 0; jj < 8; jj++)
+	  res[ii][jj] = ZERO;
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+      temp = bk - off;
+#elif defined(LEFT)
+      temp = off + 6;
+#else
+      temp = off + 8;
+#endif
+      for (k = 0; k < temp; k++) {
+	for (ii = 0; ii < 6; ii++) {
+	  a = ptrba[ii];
+	  for (jj = 0; jj < 8; jj++)
+	    res[ii][jj] += a * ptrbb[jj];
+	}
+	ptrba += 6;
+	ptrbb += 8;
+      }
+      for (ii = 0; ii < 6; ii++)
+	for (jj = 0; jj < 8; jj++)
+	  Cptr[ii + jj * ldc] = res[ii][jj] * alpha;
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+      temp = bk - off;
+#ifdef LEFT
+      temp -= 6;
+#else
+      temp -= 8;
+#endif
+      ptrba += temp * 6;
+      ptrbb += temp * 8;
+#endif
+#ifdef LEFT
+      off += 6;
+#endif
+      Cptr += 6;
+    }
+
+    /* M remainder after m/6 panels is 0..5; cascade 4,2,1. */
+    mrem = bm % 6;
+    for (mr = 4; mr >= 1; mr >>= 1) {
+      if (mrem & mr) {
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	ptrbb = bb;
+#else
+	ptrba += off * mr;
+	ptrbb = bb + off * 8;
+#endif
+	for (ii = 0; ii < mr; ii++)
+	  for (jj = 0; jj < 8; jj++)
+	    res[ii][jj] = ZERO;
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	temp = bk - off;
+#elif defined(LEFT)
+	temp = off + mr;
+#else
+	temp = off + 8;
+#endif
+	for (k = 0; k < temp; k++) {
+	  for (ii = 0; ii < mr; ii++) {
+	    a = ptrba[ii];
+	    for (jj = 0; jj < 8; jj++)
+	      res[ii][jj] += a * ptrbb[jj];
+	  }
+	  ptrba += mr;
+	  ptrbb += 8;
+	}
+	for (ii = 0; ii < mr; ii++)
+	  for (jj = 0; jj < 8; jj++)
+	    Cptr[ii + jj * ldc] = res[ii][jj] * alpha;
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	temp = bk - off;
+#ifdef LEFT
+	temp -= mr;
+#else
+	temp -= 8;
+#endif
+	ptrba += temp * mr;
+	ptrbb += temp * 8;
+#endif
+#ifdef LEFT
+	off += mr;
+#endif
+	Cptr += mr;
+      }
+    }
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+    off += 8;
+#endif
+    bb += bk * 8;
+    C += ldc * 8;
+  }
+
+  /* N remainder after n/8 panels is 0..7; cascade 4,2,1. */
+  nrem = bn % 8;
+  for (nr = 4; nr >= 1; nr >>= 1) {
+    if (nrem & nr) {
+      Cptr = C;
+#if defined(TRMMKERNEL) && defined(LEFT)
+      off = offset;
+#endif
+      ptrba = ba;
+
+      for (i = 0; i < bm / 6; i++) {
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	ptrbb = bb;
+#else
+	ptrba += off * 6;
+	ptrbb = bb + off * nr;
+#endif
+	for (ii = 0; ii < 6; ii++)
+	  for (jj = 0; jj < nr; jj++)
+	    res[ii][jj] = ZERO;
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	temp = bk - off;
+#elif defined(LEFT)
+	temp = off + 6;
+#else
+	temp = off + nr;
+#endif
+	for (k = 0; k < temp; k++) {
+	  for (ii = 0; ii < 6; ii++) {
+	    a = ptrba[ii];
+	    for (jj = 0; jj < nr; jj++)
+	      res[ii][jj] += a * ptrbb[jj];
+	  }
+	  ptrba += 6;
+	  ptrbb += nr;
+	}
+	for (ii = 0; ii < 6; ii++)
+	  for (jj = 0; jj < nr; jj++)
+	    Cptr[ii + jj * ldc] = res[ii][jj] * alpha;
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	temp = bk - off;
+#ifdef LEFT
+	temp -= 6;
+#else
+	temp -= nr;
+#endif
+	ptrba += temp * 6;
+	ptrbb += temp * nr;
+#endif
+#ifdef LEFT
+	off += 6;
+#endif
+	Cptr += 6;
+      }
+
+      mrem = bm % 6;
+      for (mr = 4; mr >= 1; mr >>= 1) {
+	if (mrem & mr) {
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	  ptrbb = bb;
+#else
+	  ptrba += off * mr;
+	  ptrbb = bb + off * nr;
+#endif
+	  for (ii = 0; ii < mr; ii++)
+	    for (jj = 0; jj < nr; jj++)
+	      res[ii][jj] = ZERO;
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	  temp = bk - off;
+#elif defined(LEFT)
+	  temp = off + mr;
+#else
+	  temp = off + nr;
+#endif
+	  for (k = 0; k < temp; k++) {
+	    for (ii = 0; ii < mr; ii++) {
+	      a = ptrba[ii];
+	      for (jj = 0; jj < nr; jj++)
+		res[ii][jj] += a * ptrbb[jj];
+	    }
+	    ptrba += mr;
+	    ptrbb += nr;
+	  }
+	  for (ii = 0; ii < mr; ii++)
+	    for (jj = 0; jj < nr; jj++)
+	      Cptr[ii + jj * ldc] = res[ii][jj] * alpha;
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	  temp = bk - off;
+#ifdef LEFT
+	  temp -= mr;
+#else
+	  temp -= nr;
+#endif
+	  ptrba += temp * mr;
+	  ptrbb += temp * nr;
+#endif
+#ifdef LEFT
+	  off += mr;
+#endif
+	  Cptr += mr;
+	}
+      }
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+      off += nr;
+#endif
+      bb += bk * nr;
+      C += ldc * nr;
+    }
+  }
+  return 0;
+}
diff --git a/param.h b/param.h
index f51fd51edd..c0a6bd0b55 100644
--- a/param.h
+++ b/param.h
@@ -3520,6 +3520,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define DGEMM_DEFAULT_UNROLL_M  6
 #define DGEMM_DEFAULT_UNROLL_N  8
+/* Keep UNROLL_MN at MR=6 so SYRK/SYR2K diagonal stepping matches the
+ * 6-wide packers (default MAX(M,N)=8 would mismatch; see #5997). */
+#define DGEMM_DEFAULT_UNROLL_MN 6
 
 #define CGEMM_DEFAULT_UNROLL_M  8
 #define CGEMM_DEFAULT_UNROLL_N  4

From 528e8737f1468141702a39b467c84685a9236580 Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Fri, 28 Aug 2026 20:29:37 +0200
Subject: [PATCH 2/6] syr2k: use division for UNROLL_MN stepping when MN is
 non-PoT

CORTEXA72 sets DGEMM_UNROLL_MN=6 so SYRK diagonals match MR=6 packs.
The ~(MN-1) form only works for power-of-two MN; use the same division
as syrk_kernel.c.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 driver/level3/syr2k_kernel.c | 3 ++-
 param.h                      | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/driver/level3/syr2k_kernel.c b/driver/level3/syr2k_kernel.c
index f9e4a4cda0..c8be0ec83c 100644
--- a/driver/level3/syr2k_kernel.c
+++ b/driver/level3/syr2k_kernel.c
@@ -144,7 +144,8 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha_r,
 
     int mm, nn;
 
-    mm = (loop & ~(GEMM_UNROLL_MN - 1));
+    /* Division form works for non-PoT UNROLL_MN (e.g. CORTEXA72 MN=6). */
+    mm = (loop / GEMM_UNROLL_MN) * GEMM_UNROLL_MN;
     nn = MIN(GEMM_UNROLL_MN, n - loop);
 
 #ifndef LOWER
diff --git a/param.h b/param.h
index c0a6bd0b55..e4fb540835 100644
--- a/param.h
+++ b/param.h
@@ -3520,8 +3520,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define DGEMM_DEFAULT_UNROLL_M  6
 #define DGEMM_DEFAULT_UNROLL_N  8
-/* Keep UNROLL_MN at MR=6 so SYRK/SYR2K diagonal stepping matches the
- * 6-wide packers (default MAX(M,N)=8 would mismatch; see #5997). */
+/* Keep UNROLL_MN at MR=6 so SYRK/SYR2K diagonal blocks match the 6-wide
+ * packers (default MAX(M,N)=8 forces 6+2 A edges on every diagonal tile).
+ * Thread splits use width%~(MN) via (mask+1) with mask=MN-1, so MN need not
+ * be a power of two; syr2k_kernel must not use ~(MN-1) bit clearing. */
 #define DGEMM_DEFAULT_UNROLL_MN 6
 
 #define CGEMM_DEFAULT_UNROLL_M  8

From 1effe1678da409b4d18cc00de2b78e5a4a2bfa51 Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Fri, 28 Aug 2026 20:59:49 +0200
Subject: [PATCH 3/6] CORTEXA72: set DGEMM_UNROLL_MN=8 for SYRK/SYR2K B packs

SYRK packs B in UNROLL_MN-wide strips then calls GEMM with full panel N.
With MN=6 that layout does not match NR=8, so DSYRK/DSYR2K failed on
real A72. MN must be a multiple of UNROLL_N; MR=6 ICOPY handles width 8
as 6+2. Keeps the syr2k division stepping fix.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 param.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/param.h b/param.h
index e4fb540835..406a4b2a0e 100644
--- a/param.h
+++ b/param.h
@@ -3520,11 +3520,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define DGEMM_DEFAULT_UNROLL_M  6
 #define DGEMM_DEFAULT_UNROLL_N  8
-/* Keep UNROLL_MN at MR=6 so SYRK/SYR2K diagonal blocks match the 6-wide
- * packers (default MAX(M,N)=8 forces 6+2 A edges on every diagonal tile).
- * Thread splits use width%~(MN) via (mask+1) with mask=MN-1, so MN need not
- * be a power of two; syr2k_kernel must not use ~(MN-1) bit clearing. */
-#define DGEMM_DEFAULT_UNROLL_MN 6
+/* SYRK/SYR2K pack B in GEMM_UNROLL_MN-wide strips, then call GEMM with the
+ * full panel N. That B buffer must match NR packing, so MN must be a
+ * multiple of UNROLL_N (here 8). MR=6 ICOPY handles width-8 as 6+2.
+ * (MN=6 looked attractive for MR but breaks NR=8 B layout; see #5997.) */
+#define DGEMM_DEFAULT_UNROLL_MN 8
 
 #define CGEMM_DEFAULT_UNROLL_M  8
 #define CGEMM_DEFAULT_UNROLL_N  4

From 4201c787e8c999c25eb8bd0d0243b0fbdaf4c3b5 Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Fri, 28 Aug 2026 21:11:44 +0200
Subject: [PATCH 4/6] CORTEXA72: set DGEMM_UNROLL_MN=LCM(6,8)=24 for SYRK
 panels

SYRK/SYR2K diagonal kernels advance packed A/B by UNROLL_MN and require
panel-aligned starts for both MR and NR. MN=8 fixed NR packing but broke
MR=6 A panels; MN=24 is the common multiple. Keeps non-PoT division stepping.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 driver/level3/syr2k_kernel.c |  2 +-
 param.h                      | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/driver/level3/syr2k_kernel.c b/driver/level3/syr2k_kernel.c
index c8be0ec83c..d3edb6d325 100644
--- a/driver/level3/syr2k_kernel.c
+++ b/driver/level3/syr2k_kernel.c
@@ -144,7 +144,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha_r,
 
     int mm, nn;
 
-    /* Division form works for non-PoT UNROLL_MN (e.g. CORTEXA72 MN=6). */
+    /* Division form works for non-PoT UNROLL_MN (e.g. CORTEXA72 MN=24). */
     mm = (loop / GEMM_UNROLL_MN) * GEMM_UNROLL_MN;
     nn = MIN(GEMM_UNROLL_MN, n - loop);
 
diff --git a/param.h b/param.h
index 406a4b2a0e..3021e856e7 100644
--- a/param.h
+++ b/param.h
@@ -3520,11 +3520,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define DGEMM_DEFAULT_UNROLL_M  6
 #define DGEMM_DEFAULT_UNROLL_N  8
-/* SYRK/SYR2K pack B in GEMM_UNROLL_MN-wide strips, then call GEMM with the
- * full panel N. That B buffer must match NR packing, so MN must be a
- * multiple of UNROLL_N (here 8). MR=6 ICOPY handles width-8 as 6+2.
- * (MN=6 looked attractive for MR but breaks NR=8 B layout; see #5997.) */
-#define DGEMM_DEFAULT_UNROLL_MN 8
+/* SYRK/SYR2K diagonal kernels step packed A/B by GEMM_UNROLL_MN and assume
+ * that lands on both MR and NR panel boundaries (pointer a+loop*k). So MN
+ * must be a common multiple of UNROLL_M and UNROLL_N: LCM(6,8)=24.
+ * MN=6 matched MR but broke NR=8 B packs; MN=8 matched NR but stepped into
+ * the middle of MR=6 A panels (first 6 rows OK, rest wrong). See #5997. */
+#define DGEMM_DEFAULT_UNROLL_MN 24
 
 #define CGEMM_DEFAULT_UNROLL_M  8
 #define CGEMM_DEFAULT_UNROLL_N  4

From bad3679f58f36f39ee5a7ca3fca7d641f62424d3 Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Sat, 29 Aug 2026 05:25:54 +0200
Subject: [PATCH 5/6] CORTEXA72: add ASM DTRMM 6x8 microkernel

Replace the correctness-first generic C trmmkernel_6x8 with an ASM
ukernel derived from dgemm_kernel_6x8_cortexa72.S (TRMM offset/K
stepping and overwrite SAVE). Restores DTRMM throughput vs stock A57
on multicore Graviton1.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 kernel/arm64/KERNEL.CORTEXA72             |    4 +-
 kernel/arm64/dtrmm_kernel_6x8_cortexa72.S | 1608 +++++++++++++++++++++
 2 files changed, 1609 insertions(+), 3 deletions(-)
 create mode 100644 kernel/arm64/dtrmm_kernel_6x8_cortexa72.S

diff --git a/kernel/arm64/KERNEL.CORTEXA72 b/kernel/arm64/KERNEL.CORTEXA72
index 7fa9236eea..7ad8a27092 100644
--- a/kernel/arm64/KERNEL.CORTEXA72
+++ b/kernel/arm64/KERNEL.CORTEXA72
@@ -6,12 +6,10 @@ include $(KERNELDIR)/KERNEL.CORTEXA57
 #
 # Stock generic/gemm_{t,n}copy_6.c packs as 4+2, not contiguous MR=6 —
 # use the panel packers. NR=8 reuses the asm dgemm_{n,t}copy_8.S.
-# No tuned dtrmm_kernel_6x8.S yet; use a correctness-first generic 6x8
-# TRMM ukernel matching contiguous MR=6 / NR=8 packs (see #5997).
 # Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M; use the
 # UNROLLM6-aware kernels (same idea as loongarch64 UNROLLN6).
 DGEMMKERNEL    =  dgemm_kernel_6x8_cortexa72.S
-DTRMMKERNEL    =  ../generic/trmmkernel_6x8.c
+DTRMMKERNEL    =  dtrmm_kernel_6x8_cortexa72.S
 
 DTRSMKERNEL_LN =  trsm_kernel_LN_UNROLLM6.c
 DTRSMKERNEL_LT =  trsm_kernel_LT_UNROLLM6.c
diff --git a/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S
new file mode 100644
index 0000000000..749e58a809
--- /dev/null
+++ b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S
@@ -0,0 +1,1608 @@
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/*                   X0          X1          X2          s0         X3        x4       x5           x6            x7 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */
+
+#define origM		x0
+#define origN		x1
+#define origK		x2
+#define origPA		x3
+#define origPB		x4
+#define pC		x5
+#define LDC		x6
+#define offset		x7
+#define counterL	x8
+#define counterI	x9
+#define counterJ	x10
+#define pB		x11
+#define pCRow0		x12
+#define pCRow1		x13
+#define pCRow2		x14
+#define pA		x15
+#define temp		x16
+#define tempOffset	x19
+#define tempK		x20
+
+/*
+ * Cortex-A72 DTRMM 6x8 microkernel (mr=6, nr=8).
+ * Derived from dgemm_kernel_6x8_cortexa72.S with TRMM offset/K panel
+ * stepping (same scheme as dtrmm_kernel_8x4.S). SAVE overwrites C.
+ */
+#define A_PRE_SIZE	2560
+#define B_PRE_SIZE	448
+#define C_PRE_SIZE	128
+
+#define alpha0		d7
+#define alphaV0		v7.d[0]
+
+.macro INIT6x8
+	fmov		d8, xzr
+	fmov		d9, xzr
+	fmov		d10, xzr
+	fmov		d11, xzr
+	fmov		d12, xzr
+	fmov		d13, xzr
+	fmov		d14, xzr
+	fmov		d15, xzr
+	fmov		d16, xzr
+	fmov		d17, xzr
+	fmov		d18, xzr
+	fmov		d19, xzr
+	fmov		d20, xzr
+	fmov		d21, xzr
+	fmov		d22, xzr
+	fmov		d23, xzr
+	fmov		d24, xzr
+	fmov		d25, xzr
+	fmov		d26, xzr
+	fmov		d27, xzr
+	fmov		d28, xzr
+	fmov		d29, xzr
+	fmov		d30, xzr
+	fmov		d31, xzr
+.endm
+
+.macro KERNEL6x8_SUB
+	ld1	{v0.2d, v1.2d, v2.2d}, [pA], #48
+	ld1	{v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
+	prfm	PLDL1KEEP, [pB, #B_PRE_SIZE]
+	fmla	v8.2d, v0.2d, v3.d[0]
+	fmla	v9.2d, v1.2d, v3.d[0]
+	fmla	v10.2d, v2.2d, v3.d[0]
+	fmla	v11.2d, v0.2d, v3.d[1]
+	fmla	v12.2d, v1.2d, v3.d[1]
+	fmla	v13.2d, v2.2d, v3.d[1]
+	fmla	v14.2d, v0.2d, v4.d[0]
+	fmla	v15.2d, v1.2d, v4.d[0]
+	fmla	v16.2d, v2.2d, v4.d[0]
+	fmla	v17.2d, v0.2d, v4.d[1]
+	fmla	v18.2d, v1.2d, v4.d[1]
+	fmla	v19.2d, v2.2d, v4.d[1]
+	fmla	v20.2d, v0.2d, v5.d[0]
+	fmla	v21.2d, v1.2d, v5.d[0]
+	fmla	v22.2d, v2.2d, v5.d[0]
+	fmla	v23.2d, v0.2d, v5.d[1]
+	fmla	v24.2d, v1.2d, v5.d[1]
+	fmla	v25.2d, v2.2d, v5.d[1]
+	fmla	v26.2d, v0.2d, v6.d[0]
+	fmla	v27.2d, v1.2d, v6.d[0]
+	fmla	v28.2d, v2.2d, v6.d[0]
+	fmla	v29.2d, v0.2d, v6.d[1]
+	fmla	v30.2d, v1.2d, v6.d[1]
+	fmla	v31.2d, v2.2d, v6.d[1]
+.endm
+
+.macro SAVE6x8
+	mov	pCRow1, pCRow0
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v8.2d, alphaV0
+	fmul	v1.2d, v9.2d, alphaV0
+	fmul	v2.2d, v10.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v11.2d, alphaV0
+	fmul	v1.2d, v12.2d, alphaV0
+	fmul	v2.2d, v13.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v14.2d, alphaV0
+	fmul	v1.2d, v15.2d, alphaV0
+	fmul	v2.2d, v16.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v17.2d, alphaV0
+	fmul	v1.2d, v18.2d, alphaV0
+	fmul	v2.2d, v19.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v20.2d, alphaV0
+	fmul	v1.2d, v21.2d, alphaV0
+	fmul	v2.2d, v22.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v23.2d, alphaV0
+	fmul	v1.2d, v24.2d, alphaV0
+	fmul	v2.2d, v25.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v26.2d, alphaV0
+	fmul	v1.2d, v27.2d, alphaV0
+	fmul	v2.2d, v28.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+	fmul	v0.2d, v29.2d, alphaV0
+	fmul	v1.2d, v30.2d, alphaV0
+	fmul	v2.2d, v31.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	add	pCRow0, pCRow0, #48
+.endm
+
+.macro INIT4x8
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+	fmov	d20, xzr
+	fmov	d21, xzr
+	fmov	d22, xzr
+	fmov	d23, xzr
+	fmov	d24, xzr
+	fmov	d25, xzr
+	fmov	d26, xzr
+	fmov	d27, xzr
+	fmov	d28, xzr
+	fmov	d29, xzr
+	fmov	d30, xzr
+	fmov	d31, xzr
+.endm
+
+.macro KERNEL4x8_SUB
+	ld1	{v0.2d, v1.2d}, [pA], #32
+	ld1	{v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v17.2d, v1.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+	fmla	v19.2d, v1.2d, v3.d[1]
+	fmla	v20.2d, v0.2d, v4.d[0]
+	fmla	v21.2d, v1.2d, v4.d[0]
+	fmla	v22.2d, v0.2d, v4.d[1]
+	fmla	v23.2d, v1.2d, v4.d[1]
+	fmla	v24.2d, v0.2d, v5.d[0]
+	fmla	v25.2d, v1.2d, v5.d[0]
+	fmla	v26.2d, v0.2d, v5.d[1]
+	fmla	v27.2d, v1.2d, v5.d[1]
+	fmla	v28.2d, v0.2d, v6.d[0]
+	fmla	v29.2d, v1.2d, v6.d[0]
+	fmla	v30.2d, v0.2d, v6.d[1]
+	fmla	v31.2d, v1.2d, v6.d[1]
+.endm
+
+.macro SAVE4x8
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	fmul	v1.2d, v17.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	fmul	v1.2d, v19.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v20.2d, alphaV0
+	fmul	v1.2d, v21.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v22.2d, alphaV0
+	fmul	v1.2d, v23.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v24.2d, alphaV0
+	fmul	v1.2d, v25.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v26.2d, alphaV0
+	fmul	v1.2d, v27.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v28.2d, alphaV0
+	fmul	v1.2d, v29.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v30.2d, alphaV0
+	fmul	v1.2d, v31.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	add	pCRow0, pCRow0, #32
+.endm
+
+.macro INIT2x8
+	fmov	d16, xzr
+	fmov	d18, xzr
+	fmov	d20, xzr
+	fmov	d22, xzr
+	fmov	d24, xzr
+	fmov	d26, xzr
+	fmov	d28, xzr
+	fmov	d30, xzr
+.endm
+
+.macro KERNEL2x8_SUB
+	ld1	{v0.2d}, [pA], #16
+	ld1	{v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+	fmla	v20.2d, v0.2d, v4.d[0]
+	fmla	v22.2d, v0.2d, v4.d[1]
+	fmla	v24.2d, v0.2d, v5.d[0]
+	fmla	v26.2d, v0.2d, v5.d[1]
+	fmla	v28.2d, v0.2d, v6.d[0]
+	fmla	v30.2d, v0.2d, v6.d[1]
+.endm
+
+.macro SAVE2x8
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v20.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v22.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v24.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v26.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v28.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v30.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	add	pCRow0, pCRow0, #16
+.endm
+
+.macro INIT1x8
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+	fmov	d20, xzr
+	fmov	d21, xzr
+	fmov	d22, xzr
+	fmov	d23, xzr
+.endm
+
+.macro KERNEL1x8_SUB
+	ldr	d0, [pA], #8
+	ld1	{v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
+	fmadd	d16, d0, d3, d16
+	fmul	d1, d0, v3.d[1]
+	fadd	d17, d17, d1
+	fmadd	d18, d0, d4, d18
+	fmul	d1, d0, v4.d[1]
+	fadd	d19, d19, d1
+	fmadd	d20, d0, d5, d20
+	fmul	d1, d0, v5.d[1]
+	fadd	d21, d21, d1
+	fmadd	d22, d0, d6, d22
+	fmul	d1, d0, v6.d[1]
+	fadd	d23, d23, d1
+.endm
+
+.macro SAVE1x8
+	mov	pCRow1, pCRow0
+	fmul	d0, d16, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d17, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d18, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d19, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d20, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d21, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d22, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d23, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	add	pCRow0, pCRow0, #8
+.endm
+
+.macro INIT6x4
+	fmov	d8, xzr
+	fmov	d9, xzr
+	fmov	d10, xzr
+	fmov	d11, xzr
+	fmov	d12, xzr
+	fmov	d13, xzr
+	fmov	d14, xzr
+	fmov	d15, xzr
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+.endm
+
+.macro KERNEL6x4_SUB
+	ld1	{v0.2d, v1.2d, v2.2d}, [pA], #48
+	ld1	{v3.2d, v4.2d}, [pB], #32
+	fmla	v8.2d, v0.2d, v3.d[0]
+	fmla	v9.2d, v1.2d, v3.d[0]
+	fmla	v10.2d, v2.2d, v3.d[0]
+	fmla	v11.2d, v0.2d, v3.d[1]
+	fmla	v12.2d, v1.2d, v3.d[1]
+	fmla	v13.2d, v2.2d, v3.d[1]
+	fmla	v14.2d, v0.2d, v4.d[0]
+	fmla	v15.2d, v1.2d, v4.d[0]
+	fmla	v16.2d, v2.2d, v4.d[0]
+	fmla	v17.2d, v0.2d, v4.d[1]
+	fmla	v18.2d, v1.2d, v4.d[1]
+	fmla	v19.2d, v2.2d, v4.d[1]
+.endm
+
+.macro SAVE6x4
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v8.2d, alphaV0
+	fmul	v1.2d, v9.2d, alphaV0
+	fmul	v2.2d, v10.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v11.2d, alphaV0
+	fmul	v1.2d, v12.2d, alphaV0
+	fmul	v2.2d, v13.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v14.2d, alphaV0
+	fmul	v1.2d, v15.2d, alphaV0
+	fmul	v2.2d, v16.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v17.2d, alphaV0
+	fmul	v1.2d, v18.2d, alphaV0
+	fmul	v2.2d, v19.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	add	pCRow0, pCRow0, #48
+.endm
+
+.macro INIT6x2
+	fmov	d8, xzr
+	fmov	d9, xzr
+	fmov	d10, xzr
+	fmov	d11, xzr
+	fmov	d12, xzr
+	fmov	d13, xzr
+.endm
+
+.macro KERNEL6x2_SUB
+	ld1	{v0.2d, v1.2d, v2.2d}, [pA], #48
+	ld1	{v3.2d}, [pB], #16
+	fmla	v8.2d, v0.2d, v3.d[0]
+	fmla	v9.2d, v1.2d, v3.d[0]
+	fmla	v10.2d, v2.2d, v3.d[0]
+	fmla	v11.2d, v0.2d, v3.d[1]
+	fmla	v12.2d, v1.2d, v3.d[1]
+	fmla	v13.2d, v2.2d, v3.d[1]
+.endm
+
+.macro SAVE6x2
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v8.2d, alphaV0
+	fmul	v1.2d, v9.2d, alphaV0
+	fmul	v2.2d, v10.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v11.2d, alphaV0
+	fmul	v1.2d, v12.2d, alphaV0
+	fmul	v2.2d, v13.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow1]
+	add	pCRow0, pCRow0, #48
+.endm
+
+.macro INIT6x1
+	fmov	d8, xzr
+	fmov	d9, xzr
+	fmov	d10, xzr
+.endm
+
+.macro KERNEL6x1_SUB
+	ld1	{v0.2d, v1.2d, v2.2d}, [pA], #48
+	ldr	d3, [pB], #8
+	fmla	v8.2d, v0.2d, v3.d[0]
+	fmla	v9.2d, v1.2d, v3.d[0]
+	fmla	v10.2d, v2.2d, v3.d[0]
+.endm
+
+.macro SAVE6x1
+	fmul	v0.2d, v8.2d, alphaV0
+	fmul	v1.2d, v9.2d, alphaV0
+	fmul	v2.2d, v10.2d, alphaV0
+	st1	{v0.2d, v1.2d, v2.2d}, [pCRow0]
+	add	pCRow0, pCRow0, #48
+.endm
+
+
+.macro INIT4x4
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+	fmov	d20, xzr
+	fmov	d21, xzr
+	fmov	d22, xzr
+	fmov	d23, xzr
+.endm
+.macro KERNEL4x4_SUB
+	ld1	{v0.2d, v1.2d}, [pA], #32
+	ld1	{v3.2d, v4.2d}, [pB], #32
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v17.2d, v1.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+	fmla	v19.2d, v1.2d, v3.d[1]
+	fmla	v20.2d, v0.2d, v4.d[0]
+	fmla	v21.2d, v1.2d, v4.d[0]
+	fmla	v22.2d, v0.2d, v4.d[1]
+	fmla	v23.2d, v1.2d, v4.d[1]
+.endm
+.macro SAVE4x4
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	fmul	v1.2d, v17.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	fmul	v1.2d, v19.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v20.2d, alphaV0
+	fmul	v1.2d, v21.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v22.2d, alphaV0
+	fmul	v1.2d, v23.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow0, pCRow0, #32
+.endm
+
+.macro INIT2x4
+	fmov	d16, xzr
+	fmov	d18, xzr
+	fmov	d20, xzr
+	fmov	d22, xzr
+.endm
+.macro KERNEL2x4_SUB
+	ld1	{v0.2d}, [pA], #16
+	ld1	{v3.2d, v4.2d}, [pB], #32
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+	fmla	v20.2d, v0.2d, v4.d[0]
+	fmla	v22.2d, v0.2d, v4.d[1]
+.endm
+.macro SAVE2x4
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v20.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v22.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow0, pCRow0, #16
+.endm
+
+.macro INIT1x4
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+.endm
+.macro KERNEL1x4_SUB
+	ldr	d0, [pA], #8
+	ld1	{v3.2d, v4.2d}, [pB], #32
+	fmadd	d16, d0, d3, d16
+	fmul	d1, d0, v3.d[1]
+	fadd	d17, d17, d1
+	fmadd	d18, d0, d4, d18
+	fmul	d1, d0, v4.d[1]
+	fadd	d19, d19, d1
+.endm
+.macro SAVE1x4
+	mov	pCRow1, pCRow0
+	fmul	d0, d16, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d17, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d18, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d19, alpha0
+	str	d0, [pCRow1]
+	add	pCRow0, pCRow0, #8
+.endm
+
+.macro INIT4x2
+	fmov	d16, xzr
+	fmov	d17, xzr
+	fmov	d18, xzr
+	fmov	d19, xzr
+.endm
+.macro KERNEL4x2_SUB
+	ld1	{v0.2d, v1.2d}, [pA], #32
+	ld1	{v3.2d}, [pB], #16
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v17.2d, v1.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+	fmla	v19.2d, v1.2d, v3.d[1]
+.endm
+.macro SAVE4x2
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	fmul	v1.2d, v17.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	fmul	v1.2d, v19.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow1]
+	add	pCRow0, pCRow0, #32
+.endm
+
+.macro INIT2x2
+	fmov	d16, xzr
+	fmov	d18, xzr
+.endm
+.macro KERNEL2x2_SUB
+	ld1	{v0.2d}, [pA], #16
+	ld1	{v3.2d}, [pB], #16
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v18.2d, v0.2d, v3.d[1]
+.endm
+.macro SAVE2x2
+	mov	pCRow1, pCRow0
+	fmul	v0.2d, v16.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	v0.2d, v18.2d, alphaV0
+	str	q0, [pCRow1]
+	add	pCRow0, pCRow0, #16
+.endm
+
+.macro INIT1x2
+	fmov	d16, xzr
+	fmov	d17, xzr
+.endm
+.macro KERNEL1x2_SUB
+	ldr	d0, [pA], #8
+	ld1	{v3.2d}, [pB], #16
+	fmadd	d16, d0, d3, d16
+	fmul	d1, d0, v3.d[1]
+	fadd	d17, d17, d1
+.endm
+.macro SAVE1x2
+	mov	pCRow1, pCRow0
+	fmul	d0, d16, alpha0
+	str	d0, [pCRow1]
+	add	pCRow1, pCRow1, LDC
+	fmul	d0, d17, alpha0
+	str	d0, [pCRow1]
+	add	pCRow0, pCRow0, #8
+.endm
+
+.macro INIT4x1
+	fmov	d16, xzr
+	fmov	d17, xzr
+.endm
+.macro KERNEL4x1_SUB
+	ld1	{v0.2d, v1.2d}, [pA], #32
+	ldr	d3, [pB], #8
+	fmla	v16.2d, v0.2d, v3.d[0]
+	fmla	v17.2d, v1.2d, v3.d[0]
+.endm
+.macro SAVE4x1
+	fmul	v0.2d, v16.2d, alphaV0
+	fmul	v1.2d, v17.2d, alphaV0
+	st1	{v0.2d, v1.2d}, [pCRow0]
+	add	pCRow0, pCRow0, #32
+.endm
+
+.macro INIT2x1
+	fmov	d16, xzr
+.endm
+.macro KERNEL2x1_SUB
+	ld1	{v0.2d}, [pA], #16
+	ldr	d3, [pB], #8
+	fmla	v16.2d, v0.2d, v3.d[0]
+.endm
+.macro SAVE2x1
+	fmul	v0.2d, v16.2d, alphaV0
+	str	q0, [pCRow0]
+	add	pCRow0, pCRow0, #16
+.endm
+
+.macro INIT1x1
+	fmov	d16, xzr
+.endm
+.macro KERNEL1x1_SUB
+	ldr	d0, [pA], #8
+	ldr	d3, [pB], #8
+	fmadd	d16, d0, d3, d16
+.endm
+.macro SAVE1x1
+	fmul	d0, d16, alpha0
+	str	d0, [pCRow0]
+	add	pCRow0, pCRow0, #8
+.endm
+
+	PROLOGUE
+	.align 5
+	add	sp, sp, #-(11 * 16)
+	stp	d8, d9, [sp, #(0 * 16)]
+	stp	d10, d11, [sp, #(1 * 16)]
+	stp	d12, d13, [sp, #(2 * 16)]
+	stp	d14, d15, [sp, #(3 * 16)]
+	stp	d16, d17, [sp, #(4 * 16)]
+	stp	x18, x19, [sp, #(5 * 16)]
+	stp	x20, x21, [sp, #(6 * 16)]
+	stp	x22, x23, [sp, #(7 * 16)]
+	stp	x24, x25, [sp, #(8 * 16)]
+	stp	x26, x27, [sp, #(9 * 16)]
+	str	x28, [sp, #(10 * 16)]
+
+	fmov	alpha0, d0
+	lsl	LDC, LDC, #3
+
+#if !defined(LEFT)
+	neg	tempOffset, offset
+#endif
+
+	mov	counterJ, origN
+	asr	counterJ, counterJ, #3
+	cbz	counterJ, .Ldtrmm6x8_L4_BEGIN
+
+.Ldtrmm6x8_L8_BEGIN:
+	mov	pCRow0, pC
+	add	pC, pC, LDC, lsl #3
+#if defined(LEFT)
+	mov	tempOffset, offset
+#endif
+	mov	pA, origPA
+
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	cbz	counterI, .Ldtrmm6x8_L8_Mrem
+
+.Ldtrmm6x8_L8_M6:
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #48
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #6
+	#else
+	add	tempK, tempOffset, #8
+	#endif
+
+	INIT6x8
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L8_M6s
+.Ldtrmm6x8_L8_M6l:
+	KERNEL6x8_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L8_M6l
+.Ldtrmm6x8_L8_M6s:
+	SAVE6x8
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #6
+	#else
+	sub	tempK, tempK, #8
+	#endif
+	mov	temp, #48
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #6
+	#endif
+
+	subs	counterI, counterI, #1
+	bgt	.Ldtrmm6x8_L8_M6
+
+.Ldtrmm6x8_L8_Mrem:
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	msub	counterI, counterI, temp, origM
+	cmp	counterI, #4
+	blt	.Ldtrmm6x8_L8_M2c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #4
+	#else
+	add	tempK, tempOffset, #8
+	#endif
+
+	INIT4x8
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L8_M4s
+.Ldtrmm6x8_L8_M4l:
+	KERNEL4x8_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L8_M4l
+.Ldtrmm6x8_L8_M4s:
+	SAVE4x8
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #4
+	#else
+	sub	tempK, tempK, #8
+	#endif
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #4
+	#endif
+
+	sub	counterI, counterI, #4
+.Ldtrmm6x8_L8_M2c:
+	cmp	counterI, #2
+	blt	.Ldtrmm6x8_L8_M1c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #2
+	#else
+	add	tempK, tempOffset, #8
+	#endif
+
+	INIT2x8
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L8_M2s
+.Ldtrmm6x8_L8_M2l:
+	KERNEL2x8_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L8_M2l
+.Ldtrmm6x8_L8_M2s:
+	SAVE2x8
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #2
+	#else
+	sub	tempK, tempK, #8
+	#endif
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #2
+	#endif
+
+	sub	counterI, counterI, #2
+.Ldtrmm6x8_L8_M1c:
+	cbz	counterI, .Ldtrmm6x8_L8_END
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #1
+	#else
+	add	tempK, tempOffset, #8
+	#endif
+
+	INIT1x8
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L8_M1s
+.Ldtrmm6x8_L8_M1l:
+	KERNEL1x8_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L8_M1l
+.Ldtrmm6x8_L8_M1s:
+	SAVE1x8
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #1
+	#else
+	sub	tempK, tempK, #8
+	#endif
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #64
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #1
+	#endif
+
+.Ldtrmm6x8_L8_END:
+	lsl	temp, origK, #6
+	add	origPB, origPB, temp
+#if !defined(LEFT)
+	add	tempOffset, tempOffset, #8
+#endif
+	subs	counterJ, counterJ, #1
+	bgt	.Ldtrmm6x8_L8_BEGIN
+
+.Ldtrmm6x8_L4_BEGIN:
+	tst	origN, #7
+	beq	.Ldtrmm6x8_L999
+	tst	origN, #4
+	beq	.Ldtrmm6x8_L2_BEGIN
+
+	mov	pCRow0, pC
+	add	pC, pC, LDC, lsl #2
+#if defined(LEFT)
+	mov	tempOffset, offset
+#endif
+	mov	pA, origPA
+
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	cbz	counterI, .Ldtrmm6x8_L4_Mrem
+.Ldtrmm6x8_L4_M6:
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #48
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #6
+	#else
+	add	tempK, tempOffset, #4
+	#endif
+
+	INIT6x4
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L4_M6s
+.Ldtrmm6x8_L4_M6l:
+	KERNEL6x4_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L4_M6l
+.Ldtrmm6x8_L4_M6s:
+	SAVE6x4
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #6
+	#else
+	sub	tempK, tempK, #4
+	#endif
+	mov	temp, #48
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #6
+	#endif
+
+	subs	counterI, counterI, #1
+	bgt	.Ldtrmm6x8_L4_M6
+.Ldtrmm6x8_L4_Mrem:
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	msub	counterI, counterI, temp, origM
+	cmp	counterI, #4
+	blt	.Ldtrmm6x8_L4_M2c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #4
+	#else
+	add	tempK, tempOffset, #4
+	#endif
+
+	INIT4x4
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L4_M4s
+.Ldtrmm6x8_L4_M4l:
+	KERNEL4x4_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L4_M4l
+.Ldtrmm6x8_L4_M4s:
+	SAVE4x4
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #4
+	#else
+	sub	tempK, tempK, #4
+	#endif
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #4
+	#endif
+
+	sub	counterI, counterI, #4
+.Ldtrmm6x8_L4_M2c:
+	cmp	counterI, #2
+	blt	.Ldtrmm6x8_L4_M1c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #2
+	#else
+	add	tempK, tempOffset, #4
+	#endif
+
+	INIT2x4
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L4_M2s
+.Ldtrmm6x8_L4_M2l:
+	KERNEL2x4_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L4_M2l
+.Ldtrmm6x8_L4_M2s:
+	SAVE2x4
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #2
+	#else
+	sub	tempK, tempK, #4
+	#endif
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #2
+	#endif
+
+	sub	counterI, counterI, #2
+.Ldtrmm6x8_L4_M1c:
+	cbz	counterI, .Ldtrmm6x8_L4_END
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #1
+	#else
+	add	tempK, tempOffset, #4
+	#endif
+
+	INIT1x4
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L4_M1s
+.Ldtrmm6x8_L4_M1l:
+	KERNEL1x4_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L4_M1l
+.Ldtrmm6x8_L4_M1s:
+	SAVE1x4
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #1
+	#else
+	sub	tempK, tempK, #4
+	#endif
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #1
+	#endif
+
+.Ldtrmm6x8_L4_END:
+	add	origPB, origPB, origK, lsl #5
+#if !defined(LEFT)
+	add	tempOffset, tempOffset, #4
+#endif
+
+.Ldtrmm6x8_L2_BEGIN:
+	tst	origN, #2
+	beq	.Ldtrmm6x8_L1_BEGIN
+	mov	pCRow0, pC
+	add	pC, pC, LDC, lsl #1
+#if defined(LEFT)
+	mov	tempOffset, offset
+#endif
+	mov	pA, origPA
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	cbz	counterI, .Ldtrmm6x8_L2_Mrem
+.Ldtrmm6x8_L2_M6:
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #48
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #6
+	#else
+	add	tempK, tempOffset, #2
+	#endif
+
+	INIT6x2
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L2_M6s
+.Ldtrmm6x8_L2_M6l:
+	KERNEL6x2_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L2_M6l
+.Ldtrmm6x8_L2_M6s:
+	SAVE6x2
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #6
+	#else
+	sub	tempK, tempK, #2
+	#endif
+	mov	temp, #48
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #6
+	#endif
+
+	subs	counterI, counterI, #1
+	bgt	.Ldtrmm6x8_L2_M6
+.Ldtrmm6x8_L2_Mrem:
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	msub	counterI, counterI, temp, origM
+	cmp	counterI, #4
+	blt	.Ldtrmm6x8_L2_M2c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #4
+	#else
+	add	tempK, tempOffset, #2
+	#endif
+
+	INIT4x2
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L2_M4s
+.Ldtrmm6x8_L2_M4l:
+	KERNEL4x2_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L2_M4l
+.Ldtrmm6x8_L2_M4s:
+	SAVE4x2
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #4
+	#else
+	sub	tempK, tempK, #2
+	#endif
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #4
+	#endif
+
+	sub	counterI, counterI, #4
+.Ldtrmm6x8_L2_M2c:
+	cmp	counterI, #2
+	blt	.Ldtrmm6x8_L2_M1c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #2
+	#else
+	add	tempK, tempOffset, #2
+	#endif
+
+	INIT2x2
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L2_M2s
+.Ldtrmm6x8_L2_M2l:
+	KERNEL2x2_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L2_M2l
+.Ldtrmm6x8_L2_M2s:
+	SAVE2x2
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #2
+	#else
+	sub	tempK, tempK, #2
+	#endif
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #2
+	#endif
+
+	sub	counterI, counterI, #2
+.Ldtrmm6x8_L2_M1c:
+	cbz	counterI, .Ldtrmm6x8_L2_END
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #1
+	#else
+	add	tempK, tempOffset, #2
+	#endif
+
+	INIT1x2
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L2_M1s
+.Ldtrmm6x8_L2_M1l:
+	KERNEL1x2_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L2_M1l
+.Ldtrmm6x8_L2_M1s:
+	SAVE1x2
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #1
+	#else
+	sub	tempK, tempK, #2
+	#endif
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #1
+	#endif
+
+.Ldtrmm6x8_L2_END:
+	add	origPB, origPB, origK, lsl #4
+#if !defined(LEFT)
+	add	tempOffset, tempOffset, #2
+#endif
+
+.Ldtrmm6x8_L1_BEGIN:
+	tst	origN, #1
+	beq	.Ldtrmm6x8_L999
+	mov	pCRow0, pC
+#if defined(LEFT)
+	mov	tempOffset, offset
+#endif
+	mov	pA, origPA
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	cbz	counterI, .Ldtrmm6x8_L1_Mrem
+.Ldtrmm6x8_L1_M6:
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #48
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #6
+	#else
+	add	tempK, tempOffset, #1
+	#endif
+
+	INIT6x1
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L1_M6s
+.Ldtrmm6x8_L1_M6l:
+	KERNEL6x1_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L1_M6l
+.Ldtrmm6x8_L1_M6s:
+	SAVE6x1
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #6
+	#else
+	sub	tempK, tempK, #1
+	#endif
+	mov	temp, #48
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #6
+	#endif
+
+	subs	counterI, counterI, #1
+	bgt	.Ldtrmm6x8_L1_M6
+.Ldtrmm6x8_L1_Mrem:
+	mov	temp, #6
+	udiv	counterI, origM, temp
+	msub	counterI, counterI, temp, origM
+	cmp	counterI, #4
+	blt	.Ldtrmm6x8_L1_M2c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #32
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #4
+	#else
+	add	tempK, tempOffset, #1
+	#endif
+
+	INIT4x1
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L1_M4s
+.Ldtrmm6x8_L1_M4l:
+	KERNEL4x1_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L1_M4l
+.Ldtrmm6x8_L1_M4s:
+	SAVE4x1
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #4
+	#else
+	sub	tempK, tempK, #1
+	#endif
+	mov	temp, #32
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #4
+	#endif
+
+	sub	counterI, counterI, #4
+.Ldtrmm6x8_L1_M2c:
+	cmp	counterI, #2
+	blt	.Ldtrmm6x8_L1_M1c
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #16
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #2
+	#else
+	add	tempK, tempOffset, #1
+	#endif
+
+	INIT2x1
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L1_M2s
+.Ldtrmm6x8_L1_M2l:
+	KERNEL2x1_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L1_M2l
+.Ldtrmm6x8_L1_M2s:
+	SAVE2x1
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #2
+	#else
+	sub	tempK, tempK, #1
+	#endif
+	mov	temp, #16
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #2
+	#endif
+
+	sub	counterI, counterI, #2
+.Ldtrmm6x8_L1_M1c:
+	cbz	counterI, .Ldtrmm6x8_L999
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	mov	pB, origPB
+	#else
+	mov	pB, origPB
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempOffset, temp
+	add	pB, pB, temp
+	#endif
+	#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#elif defined(LEFT)
+	add	tempK, tempOffset, #1
+	#else
+	add	tempK, tempOffset, #1
+	#endif
+
+	INIT1x1
+	mov	counterL, tempK
+	cbz	counterL, .Ldtrmm6x8_L1_M1s
+.Ldtrmm6x8_L1_M1l:
+	KERNEL1x1_SUB
+	subs	counterL, counterL, #1
+	bgt	.Ldtrmm6x8_L1_M1l
+.Ldtrmm6x8_L1_M1s:
+	SAVE1x1
+	#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+	sub	tempK, origK, tempOffset
+	#if defined(LEFT)
+	sub	tempK, tempK, #1
+	#else
+	sub	tempK, tempK, #1
+	#endif
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pA, pA, temp
+	mov	temp, #8
+	mul	temp, tempK, temp
+	add	pB, pB, temp
+	#endif
+	#if defined(LEFT)
+	add	tempOffset, tempOffset, #1
+	#endif
+
+.Ldtrmm6x8_L999:
+	mov	x0, #0
+	ldp	d8, d9, [sp, #(0 * 16)]
+	ldp	d10, d11, [sp, #(1 * 16)]
+	ldp	d12, d13, [sp, #(2 * 16)]
+	ldp	d14, d15, [sp, #(3 * 16)]
+	ldp	d16, d17, [sp, #(4 * 16)]
+	ldp	x18, x19, [sp, #(5 * 16)]
+	ldp	x20, x21, [sp, #(6 * 16)]
+	ldp	x22, x23, [sp, #(7 * 16)]
+	ldp	x24, x25, [sp, #(8 * 16)]
+	ldp	x26, x27, [sp, #(9 * 16)]
+	ldr	x28, [sp, #(10 * 16)]
+	add	sp, sp, #(11 * 16)
+	ret
+	EPILOGUE

From c8a6e793a35355bff6b9942feeabcf86768b8b1d Mon Sep 17 00:00:00 2001
From: Hugo Meiland <hugo@meiland.nl>
Date: Sat, 29 Aug 2026 05:38:50 +0200
Subject: [PATCH 6/6] CORTEXA72: fix DTRSM LN rem indexing for non-PoT UNROLL_M

Rem panels of size 4/5 used (m & ~(i-1))-i, which assumes power-of-two
M alignment. With MR=6 that misplaces the rem-4 block (cblas N=5/35).
Walk the rem start position from the end instead.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 kernel/arm64/trsm_kernel_LN_UNROLLM6.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/kernel/arm64/trsm_kernel_LN_UNROLLM6.c b/kernel/arm64/trsm_kernel_LN_UNROLLM6.c
index 44fd8e8619..2a22507821 100644
--- a/kernel/arm64/trsm_kernel_LN_UNROLLM6.c
+++ b/kernel/arm64/trsm_kernel_LN_UNROLLM6.c
@@ -212,10 +212,14 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k,  FLOAT dummy1,
     kk = m + offset;
 
     if (mmodM) {
+      /* Walk rem panels from the end. PoT bitmasks (m & ~(i-1))-i are wrong
+       * when UNROLL_M is not a power of two (e.g. rem=4/5 with M=6). */
+      BLASLONG pos = m;
       for (i = 1; i < GEMM_UNROLL_M; i *= 2){
 	if (mmodM & i) {
-	  aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;
-	  cc = c + ((m & ~(i - 1)) - i)     * COMPSIZE;
+	  pos -= i;
+	  aa = a + pos * k * COMPSIZE;
+	  cc = c + pos     * COMPSIZE;
 
 	  if (k - kk > 0) {
 	    GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,
@@ -281,10 +285,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k,  FLOAT dummy1,
 	kk = m + offset;
 
 	if (mmodM) {
+	  BLASLONG pos = m;
 	  for (i = 1; i < GEMM_UNROLL_M; i *= 2){
 	    if (mmodM & i) {
-	      aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;
-	      cc = c + ((m & ~(i - 1)) - i)     * COMPSIZE;
+	      pos -= i;
+	      aa = a + pos * k * COMPSIZE;
+	      cc = c + pos     * COMPSIZE;
 
 	      if (k - kk > 0) {
 		GEMM_KERNEL(i, j, k - kk, dm1,