From dece35c11d75d60ae8887d4b57a9abff43237a50 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 1 Jul 2026 15:09:57 +0800 Subject: [PATCH 1/8] LATX: inline glibc pcmpistri string modes Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-simd.c | 177 +++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 2 deletions(-) diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 9e97cead69..0f8550f77b 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3646,6 +3646,169 @@ bool translate_pcmpestrm(IR1_INST *pir1) return true; } +static void tr_gen_inline_pcmpistri_prefix_mask(IR2_OPND prefix, + IR2_OPND vec, + IR2_OPND vtmp, + IR2_OPND tmp) +{ + la_vor_v(vtmp, vec, vec); + la_vmsknz_b(vtmp, vtmp); + la_vpickve2gr_wu(prefix, vtmp, 0); + la_bstrpick_d(prefix, prefix, 15, 0); + + /* + * Prefix mask before the first NUL: + * 0b...10111 -> ((mask ^ (mask + 1)) >> 1) == 0b00111 + * 0xffff -> 0xffff + */ + la_addi_d(tmp, prefix, 1); + la_xor(prefix, prefix, tmp); + la_srli_d(prefix, prefix, 1); + la_bstrpick_d(prefix, prefix, 15, 0); +} + +static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND flags = ra_alloc_itemp(); + + /* + * CF = res != 0, OF = res[0], ZF/SF indicate whether src/dst contain + * a NUL in the first 16 bytes. AF/PF are cleared. + */ + la_mov64(flags, zero_ir2_opnd); + la_sltu(tmp, zero_ir2_opnd, res); + la_or(flags, flags, tmp); + + la_andi(tmp, res, 1); + la_bstrins_w(flags, tmp, OF_BIT_INDEX, OF_BIT_INDEX); + + li_d(tmp, 0xffff); + la_xor(tmp, prefix_s, tmp); + la_sltu(tmp, zero_ir2_opnd, tmp); + la_bstrins_w(flags, tmp, ZF_BIT_INDEX, ZF_BIT_INDEX); + + li_d(tmp, 0xffff); + la_xor(tmp, prefix_d, tmp); + la_sltu(tmp, zero_ir2_opnd, tmp); + la_bstrins_w(flags, tmp, SF_BIT_INDEX, SF_BIT_INDEX); + + la_x86mtflag(flags, CF_USEDEF_BIT | PF_USEDEF_BIT | AF_USEDEF_BIT | + ZF_USEDEF_BIT | SF_USEDEF_BIT | OF_USEDEF_BIT); + + /* ECX = res ? ctz(res) : 16 */ + la_ctz_w(tmp, res); + la_addi_d(flags, zero_ir2_opnd, 16); + la_maskeqz(tmp, tmp, res); + la_masknez(flags, flags, res); + la_or(tmp, tmp, flags); + la_mov64(ra_alloc_gpr(ecx_index), tmp); + + ra_free_temp(flags); + ra_free_temp(tmp); +} + +static void tr_gen_inline_pcmpistri_equal_each_negative(IR2_OPND vd, + IR2_OPND vs) +{ + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND vcmp = ra_alloc_ftemp(); + + IR2_OPND eq_mask = ra_alloc_itemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + la_vseq_b(vcmp, vd, vs); + la_vmsknz_b(vcmp, vcmp); + la_vpickve2gr_wu(eq_mask, vcmp, 0); + la_bstrpick_d(eq_mask, eq_mask, 15, 0); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + + ra_free_temp(vcmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + ra_free_temp(tmp); + + la_or(res, prefix_d, prefix_s); + la_nor(eq_mask, eq_mask, zero_ir2_opnd); + la_bstrpick_d(eq_mask, eq_mask, 15, 0); + la_and(res, res, eq_mask); + + tr_gen_inline_pcmpistri_finish(res, prefix_d, prefix_s); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(eq_mask); +} + +static void tr_gen_inline_pcmpistri_self_nul_probe(IR2_OPND vd) +{ + IR2_OPND vtmp = ra_alloc_ftemp(); + IR2_OPND prefix = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix, vd, vtmp, tmp); + ra_free_temp(vtmp); + ra_free_temp(tmp); + + la_nor(res, prefix, zero_ir2_opnd); + la_bstrpick_d(res, res, 15, 0); + + tr_gen_inline_pcmpistri_finish(res, prefix, prefix); + + ra_free_temp(res); + ra_free_temp(prefix); +} + +/* + * glibc string fast paths use: + * pcmpistri xmm, xmm/m128, 0x18/0x1a + * Equal Each, negative polarity, least-significant index. + * 0x18 vs 0x1a differs only in unsigned/signed byte elements, which does + * not change byte equality. + * pcmpistri xmm, xmm, 0x3a + * Equal Each, masked negative polarity, least-significant index. glibc + * uses only the self-compare form as a NUL-position probe. + * + * Other modes have different aggregation/polarity/index semantics and stay on + * the helper path. + */ +static bool tr_gen_inline_pcmpistri(IR1_INST *pir1, IR2_OPND vd, IR2_OPND vs, + int ctrl) +{ + if (ctrl == 0x18 || ctrl == 0x1a) { + tr_gen_inline_pcmpistri_equal_each_negative(vd, vs); + return true; + } + + if (ctrl == 0x3a) { + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + + if (ir1_opnd_is_xmm(opnd1) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + tr_gen_inline_pcmpistri_self_nul_probe(vd); + return true; + } + } + + return false; +} + +static bool tr_pcmpistri_mem_can_inline(int ctrl) +{ + return ctrl == 0x18 || ctrl == 0x1a; +} + #ifndef CONFIG_LATX_AVX_OPT bool translate_pcmpistri(IR1_INST *pir1) #else @@ -3659,8 +3822,18 @@ bool translate_vpcmpistri(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); if (ir1_opnd_is_xmm(opnd1)) { int s = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistri_xmm, d, s, imm, - LOAD_HELPER_PCMPISTRI_XMM); + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_xmm(s); + if (!tr_gen_inline_pcmpistri(pir1, vd, vs, imm)) { + tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistri_xmm, d, s, imm, + LOAD_HELPER_PCMPISTRI_XMM); + } + } else if (tr_pcmpistri_mem_can_inline(imm)) { + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_ftemp(); + load_freg128_from_ir1_mem(vs, opnd1); + tr_gen_inline_pcmpistri(pir1, vd, vs, imm); + ra_free_temp(vs); } else { IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src = ra_alloc_xmm((d + 1) % 8); From 4e198a6b85a37c1f8b6f82ce739ff28663d84f09 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 09:49:31 +0800 Subject: [PATCH 2/8] LATX: add fast REP MOVSB/STOSB lowering Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-mov.c | 160 +++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index 52e0164560..dc9f4e8fd3 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -645,6 +645,158 @@ static void load_step_to_reg_ir1(IR2_OPND *p_step_opnd, IR1_OPND *opnd_ir1) ra_free_temp(tmp_step); } +#ifdef TARGET_X86_64 +static bool tr_can_fast_rep_string_b(IR1_INST *pir1, IR1_OPCODE insn) +{ + return CODEIS64 && + ir1_prefix(pir1) == dt_X86_PREFIX_REP && + pir1->info->id == insn && + pir1->info->x86.prefix[3] != 0x67; +} + +static void tr_emit_rep_string_df_guard(IR2_OPND label_slow) +{ + IR2_OPND df_opnd = ra_alloc_itemp(); + IR2_OPND eflags_opnd = ra_alloc_eflags(); + + la_andi(df_opnd, eflags_opnd, 1 << DF_BIT_INDEX); + la_bne(df_opnd, zero_ir2_opnd, label_slow); + + ra_free_temp(df_opnd); +} + +static void tr_emit_finish_fast_rep_string(IR2_OPND ecx_opnd, + IR2_OPND label_exit) +{ + la_or(ecx_opnd, zero_ir2_opnd, zero_ir2_opnd); + store_ireg_to_ir1(ecx_opnd, &rcx_ir1_opnd, false); + la_b(label_exit); +} + +static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, + IR2_OPND rdi_opnd, IR2_OPND rcx_opnd, + IR2_OPND label_exit) +{ + if (!tr_can_fast_rep_string_b(pir1, dt_X86_INS_MOVSB)) { + return false; + } + + IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_no_overlap = ra_alloc_label(); + IR2_OPND label_qword_done = ra_alloc_label(); + IR2_OPND label_qword_loop = ra_alloc_label(); + IR2_OPND label_tail_done = ra_alloc_label(); + IR2_OPND label_tail_loop = ra_alloc_label(); + IR2_OPND qwords = ra_alloc_itemp(); + IR2_OPND tail = ra_alloc_itemp(); + IR2_OPND value = ra_alloc_itemp(); + IR2_OPND overlap = ra_alloc_itemp(); + + tr_emit_rep_string_df_guard(label_slow); + + la_bgeu(rsi_opnd, rdi_opnd, label_no_overlap); + la_sub_d(overlap, rdi_opnd, rsi_opnd); + la_bltu(overlap, rcx_opnd, label_slow); + + la_label(label_no_overlap); + la_srli_d(qwords, rcx_opnd, 3); + la_andi(tail, rcx_opnd, 7); + + la_beq(qwords, zero_ir2_opnd, label_qword_done); + la_label(label_qword_loop); + la_ld_d(value, rsi_opnd, 0); + la_st_d(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, 8); + la_addi_d(rdi_opnd, rdi_opnd, 8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_qword_loop); + + la_label(label_qword_done); + la_beq(tail, zero_ir2_opnd, label_tail_done); + la_label(label_tail_loop); + la_ld_bu(value, rsi_opnd, 0); + la_st_b(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, 1); + la_addi_d(rdi_opnd, rdi_opnd, 1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_tail_loop); + + la_label(label_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + + la_label(label_slow); + + ra_free_temp(qwords); + ra_free_temp(tail); + ra_free_temp(value); + ra_free_temp(overlap); + return true; +} + +static void tr_replicate_byte_to_dword(IR2_OPND dst, IR2_OPND src, + IR2_OPND tmp) +{ + la_andi(dst, src, 0xff); + la_slli_d(tmp, dst, 8); + la_or(dst, dst, tmp); + la_slli_d(tmp, dst, 16); + la_or(dst, dst, tmp); + la_slli_d(tmp, dst, 32); + la_or(dst, dst, tmp); +} + +static bool tr_emit_fast_rep_stosb(IR1_INST *pir1, IR2_OPND rdi_opnd, + IR2_OPND rcx_opnd, IR2_OPND value_opnd, + IR2_OPND label_exit) +{ + if (!tr_can_fast_rep_string_b(pir1, dt_X86_INS_STOSB)) { + return false; + } + + IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_qword_done = ra_alloc_label(); + IR2_OPND label_qword_loop = ra_alloc_label(); + IR2_OPND label_tail_done = ra_alloc_label(); + IR2_OPND label_tail_loop = ra_alloc_label(); + IR2_OPND qwords = ra_alloc_itemp(); + IR2_OPND tail = ra_alloc_itemp(); + IR2_OPND fill = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_emit_rep_string_df_guard(label_slow); + + tr_replicate_byte_to_dword(fill, value_opnd, tmp); + la_srli_d(qwords, rcx_opnd, 3); + la_andi(tail, rcx_opnd, 7); + + la_beq(qwords, zero_ir2_opnd, label_qword_done); + la_label(label_qword_loop); + la_st_d(fill, rdi_opnd, 0); + la_addi_d(rdi_opnd, rdi_opnd, 8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_qword_loop); + + la_label(label_qword_done); + la_beq(tail, zero_ir2_opnd, label_tail_done); + la_label(label_tail_loop); + la_st_b(fill, rdi_opnd, 0); + la_addi_d(rdi_opnd, rdi_opnd, 1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_tail_loop); + + la_label(label_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + + la_label(label_slow); + + ra_free_temp(qwords); + ra_free_temp(tail); + ra_free_temp(fill); + ra_free_temp(tmp); + return true; +} +#endif + bool translate_movs(IR1_INST *pir1) { IR2_OPND esi_opnd = ra_alloc_gpr(esi_index); @@ -769,6 +921,10 @@ bool translate_movs(IR1_INST *pir1) lsassert(0); } +#ifdef TARGET_X86_64 + tr_emit_fast_rep_movsb(pir1, esi_opnd, edi_opnd, ecx_opnd, label_exit); +#endif + /* 2. preparations outside the loop */ IR2_OPND step_opnd = ra_alloc_itemp(); load_step_to_reg_ir1(&step_opnd, mem_di); @@ -970,6 +1126,10 @@ bool translate_stos(IR1_INST *pir1) } IR2_OPND eax_value_opnd = load_ireg_from_ir1(opnd_eax, SIGN_EXTENSION, false); +#ifdef TARGET_X86_64 + tr_emit_fast_rep_stosb(pir1, edi_opnd, ecx_opnd, eax_value_opnd, + label_exit); +#endif IR2_OPND step_opnd = ra_alloc_itemp(); load_step_to_reg_ir1(&step_opnd, opnd_di); From e73acbb9b39fed8b8e948e791e3910ce6b3e1d58 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 10:23:40 +0800 Subject: [PATCH 3/8] LATX: use LSX loops for REP MOVSB/STOSB Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-mov.c | 60 ++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index dc9f4e8fd3..647930b883 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -683,6 +683,8 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, IR2_OPND label_slow = ra_alloc_label(); IR2_OPND label_no_overlap = ra_alloc_label(); + IR2_OPND label_block_done = ra_alloc_label(); + IR2_OPND label_block_loop = ra_alloc_label(); IR2_OPND label_qword_done = ra_alloc_label(); IR2_OPND label_qword_loop = ra_alloc_label(); IR2_OPND label_tail_done = ra_alloc_label(); @@ -691,6 +693,10 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, IR2_OPND tail = ra_alloc_itemp(); IR2_OPND value = ra_alloc_itemp(); IR2_OPND overlap = ra_alloc_itemp(); + IR2_OPND vec0 = ra_alloc_ftemp(); + IR2_OPND vec1 = ra_alloc_ftemp(); + IR2_OPND vec2 = ra_alloc_ftemp(); + IR2_OPND vec3 = ra_alloc_ftemp(); tr_emit_rep_string_df_guard(label_slow); @@ -699,8 +705,27 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, la_bltu(overlap, rcx_opnd, label_slow); la_label(label_no_overlap); - la_srli_d(qwords, rcx_opnd, 3); - la_andi(tail, rcx_opnd, 7); + la_srli_d(qwords, rcx_opnd, 6); + la_andi(tail, rcx_opnd, 63); + + la_beq(qwords, zero_ir2_opnd, label_block_done); + la_label(label_block_loop); + la_vld(vec0, rsi_opnd, 0); + la_vld(vec1, rsi_opnd, 16); + la_vld(vec2, rsi_opnd, 32); + la_vld(vec3, rsi_opnd, 48); + la_vst(vec0, rdi_opnd, 0); + la_vst(vec1, rdi_opnd, 16); + la_vst(vec2, rdi_opnd, 32); + la_vst(vec3, rdi_opnd, 48); + la_addi_d(rsi_opnd, rsi_opnd, 64); + la_addi_d(rdi_opnd, rdi_opnd, 64); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_block_loop); + + la_label(label_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); la_beq(qwords, zero_ir2_opnd, label_qword_done); la_label(label_qword_loop); @@ -730,6 +755,10 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, ra_free_temp(tail); ra_free_temp(value); ra_free_temp(overlap); + ra_free_temp(vec0); + ra_free_temp(vec1); + ra_free_temp(vec2); + ra_free_temp(vec3); return true; } @@ -754,6 +783,8 @@ static bool tr_emit_fast_rep_stosb(IR1_INST *pir1, IR2_OPND rdi_opnd, } IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_block_done = ra_alloc_label(); + IR2_OPND label_block_loop = ra_alloc_label(); IR2_OPND label_qword_done = ra_alloc_label(); IR2_OPND label_qword_loop = ra_alloc_label(); IR2_OPND label_tail_done = ra_alloc_label(); @@ -762,12 +793,32 @@ static bool tr_emit_fast_rep_stosb(IR1_INST *pir1, IR2_OPND rdi_opnd, IR2_OPND tail = ra_alloc_itemp(); IR2_OPND fill = ra_alloc_itemp(); IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND vfill = ra_alloc_ftemp(); tr_emit_rep_string_df_guard(label_slow); tr_replicate_byte_to_dword(fill, value_opnd, tmp); - la_srli_d(qwords, rcx_opnd, 3); - la_andi(tail, rcx_opnd, 7); + la_vreplgr2vr_b(vfill, value_opnd); + la_srli_d(qwords, rcx_opnd, 7); + la_andi(tail, rcx_opnd, 127); + + la_beq(qwords, zero_ir2_opnd, label_block_done); + la_label(label_block_loop); + la_vst(vfill, rdi_opnd, 0); + la_vst(vfill, rdi_opnd, 16); + la_vst(vfill, rdi_opnd, 32); + la_vst(vfill, rdi_opnd, 48); + la_vst(vfill, rdi_opnd, 64); + la_vst(vfill, rdi_opnd, 80); + la_vst(vfill, rdi_opnd, 96); + la_vst(vfill, rdi_opnd, 112); + la_addi_d(rdi_opnd, rdi_opnd, 128); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_block_loop); + + la_label(label_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); la_beq(qwords, zero_ir2_opnd, label_qword_done); la_label(label_qword_loop); @@ -793,6 +844,7 @@ static bool tr_emit_fast_rep_stosb(IR1_INST *pir1, IR2_OPND rdi_opnd, ra_free_temp(tail); ra_free_temp(fill); ra_free_temp(tmp); + ra_free_temp(vfill); return true; } #endif From c75e16d51bf1991420d6ee01cac9caa4c2a7a812 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 11:56:42 +0800 Subject: [PATCH 4/8] LATX: add backward REP MOVSB fast path Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-mov.c | 43 +++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index 647930b883..88ab1252e6 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -682,6 +682,12 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, } IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_backward = ra_alloc_label(); + IR2_OPND label_backward_no_overlap = ra_alloc_label(); + IR2_OPND label_backward_qword_done = ra_alloc_label(); + IR2_OPND label_backward_qword_loop = ra_alloc_label(); + IR2_OPND label_backward_tail_done = ra_alloc_label(); + IR2_OPND label_backward_tail_loop = ra_alloc_label(); IR2_OPND label_no_overlap = ra_alloc_label(); IR2_OPND label_block_done = ra_alloc_label(); IR2_OPND label_block_loop = ra_alloc_label(); @@ -698,7 +704,11 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, IR2_OPND vec2 = ra_alloc_ftemp(); IR2_OPND vec3 = ra_alloc_ftemp(); - tr_emit_rep_string_df_guard(label_slow); + IR2_OPND df_opnd = ra_alloc_itemp(); + IR2_OPND eflags_opnd = ra_alloc_eflags(); + la_andi(df_opnd, eflags_opnd, 1 << DF_BIT_INDEX); + la_bne(df_opnd, zero_ir2_opnd, label_backward); + ra_free_temp(df_opnd); la_bgeu(rsi_opnd, rdi_opnd, label_no_overlap); la_sub_d(overlap, rdi_opnd, rsi_opnd); @@ -749,6 +759,37 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, la_label(label_tail_done); tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + la_label(label_backward); + la_bgeu(rdi_opnd, rsi_opnd, label_backward_no_overlap); + la_sub_d(overlap, rsi_opnd, rdi_opnd); + la_bltu(overlap, rcx_opnd, label_slow); + + la_label(label_backward_no_overlap); + la_srli_d(qwords, rcx_opnd, 3); + la_andi(tail, rcx_opnd, 7); + + la_beq(qwords, zero_ir2_opnd, label_backward_qword_done); + la_label(label_backward_qword_loop); + la_ld_d(value, rsi_opnd, -7); + la_st_d(value, rdi_opnd, -7); + la_addi_d(rsi_opnd, rsi_opnd, -8); + la_addi_d(rdi_opnd, rdi_opnd, -8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_backward_qword_loop); + + la_label(label_backward_qword_done); + la_beq(tail, zero_ir2_opnd, label_backward_tail_done); + la_label(label_backward_tail_loop); + la_ld_bu(value, rsi_opnd, 0); + la_st_b(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, -1); + la_addi_d(rdi_opnd, rdi_opnd, -1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_backward_tail_loop); + + la_label(label_backward_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + la_label(label_slow); ra_free_temp(qwords); From a5fadd25d8a1219b55483fb982c001af55ed7357 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 12:12:04 +0800 Subject: [PATCH 5/8] LATX: vectorize backward REP MOVSB Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-mov.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index 88ab1252e6..1d97c075db 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -684,6 +684,8 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, IR2_OPND label_slow = ra_alloc_label(); IR2_OPND label_backward = ra_alloc_label(); IR2_OPND label_backward_no_overlap = ra_alloc_label(); + IR2_OPND label_backward_block_done = ra_alloc_label(); + IR2_OPND label_backward_block_loop = ra_alloc_label(); IR2_OPND label_backward_qword_done = ra_alloc_label(); IR2_OPND label_backward_qword_loop = ra_alloc_label(); IR2_OPND label_backward_tail_done = ra_alloc_label(); @@ -765,8 +767,27 @@ static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, la_bltu(overlap, rcx_opnd, label_slow); la_label(label_backward_no_overlap); - la_srli_d(qwords, rcx_opnd, 3); - la_andi(tail, rcx_opnd, 7); + la_srli_d(qwords, rcx_opnd, 6); + la_andi(tail, rcx_opnd, 63); + + la_beq(qwords, zero_ir2_opnd, label_backward_block_done); + la_label(label_backward_block_loop); + la_vld(vec0, rsi_opnd, -63); + la_vld(vec1, rsi_opnd, -47); + la_vld(vec2, rsi_opnd, -31); + la_vld(vec3, rsi_opnd, -15); + la_vst(vec0, rdi_opnd, -63); + la_vst(vec1, rdi_opnd, -47); + la_vst(vec2, rdi_opnd, -31); + la_vst(vec3, rdi_opnd, -15); + la_addi_d(rsi_opnd, rsi_opnd, -64); + la_addi_d(rdi_opnd, rdi_opnd, -64); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_backward_block_loop); + + la_label(label_backward_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); la_beq(qwords, zero_ir2_opnd, label_backward_qword_done); la_label(label_backward_qword_loop); From aafa6b5e2b7551ee4ac8f0a57f1b12a7332d3314 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 14:35:29 +0800 Subject: [PATCH 6/8] LATX: inline PCMPxSTR equal-any string modes Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-simd.c | 151 +++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 6 deletions(-) diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 0f8550f77b..c5a2e6d799 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3667,9 +3667,9 @@ static void tr_gen_inline_pcmpistri_prefix_mask(IR2_OPND prefix, la_bstrpick_d(prefix, prefix, 15, 0); } -static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, - IR2_OPND prefix_d, - IR2_OPND prefix_s) +static void tr_gen_inline_pcmpistr_set_flags(IR2_OPND res, + IR2_OPND prefix_d, + IR2_OPND prefix_s) { IR2_OPND tmp = ra_alloc_itemp(); IR2_OPND flags = ra_alloc_itemp(); @@ -3698,6 +3698,19 @@ static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, la_x86mtflag(flags, CF_USEDEF_BIT | PF_USEDEF_BIT | AF_USEDEF_BIT | ZF_USEDEF_BIT | SF_USEDEF_BIT | OF_USEDEF_BIT); + ra_free_temp(flags); + ra_free_temp(tmp); +} + +static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + tr_gen_inline_pcmpistr_set_flags(res, prefix_d, prefix_s); + + IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND flags = ra_alloc_itemp(); + /* ECX = res ? ctz(res) : 16 */ la_ctz_w(tmp, res); la_addi_d(flags, zero_ir2_opnd, 16); @@ -3710,6 +3723,46 @@ static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, ra_free_temp(tmp); } +static void tr_gen_inline_pcmpistr_equal_any_mask(IR2_OPND res, + IR2_OPND vd, + IR2_OPND vs, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + IR2_OPND vmatch = ra_alloc_ftemp(); + IR2_OPND vcmp = ra_alloc_ftemp(); + IR2_OPND vbyte = ra_alloc_ftemp(); + IR2_OPND vvalid = ra_alloc_ftemp(); + IR2_OPND byte = ra_alloc_itemp(); + IR2_OPND valid = ra_alloc_itemp(); + + la_vxor_v(vmatch, vmatch, vmatch); + for (int i = 0; i < 16; i++) { + la_vpickve2gr_bu(byte, vd, i); + la_vreplgr2vr_b(vbyte, byte); + la_vseq_b(vcmp, vs, vbyte); + + la_srli_d(valid, prefix_d, i); + la_andi(valid, valid, 1); + la_sub_d(valid, zero_ir2_opnd, valid); + la_vreplgr2vr_b(vvalid, valid); + la_vand_v(vcmp, vcmp, vvalid); + la_vor_v(vmatch, vmatch, vcmp); + } + + la_vmsknz_b(vmatch, vmatch); + la_vpickve2gr_wu(res, vmatch, 0); + la_bstrpick_d(res, res, 15, 0); + la_and(res, res, prefix_s); + + ra_free_temp(valid); + ra_free_temp(byte); + ra_free_temp(vvalid); + ra_free_temp(vbyte); + ra_free_temp(vcmp); + ra_free_temp(vmatch); +} + static void tr_gen_inline_pcmpistri_equal_each_negative(IR2_OPND vd, IR2_OPND vs) { @@ -3749,6 +3802,36 @@ static void tr_gen_inline_pcmpistri_equal_each_negative(IR2_OPND vd, ra_free_temp(eq_mask); } +static void tr_gen_inline_pcmpistri_equal_any(IR2_OPND vd, IR2_OPND vs, + int ctrl) +{ + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + + tr_gen_inline_pcmpistr_equal_any_mask(res, vd, vs, prefix_d, prefix_s); + if (ctrl == 0x12) { + li_d(tmp, 0xffff); + la_xor(res, res, tmp); + la_bstrpick_d(res, res, 15, 0); + } + + tr_gen_inline_pcmpistri_finish(res, prefix_d, prefix_s); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(tmp); +} + static void tr_gen_inline_pcmpistri_self_nul_probe(IR2_OPND vd) { IR2_OPND vtmp = ra_alloc_ftemp(); @@ -3778,6 +3861,9 @@ static void tr_gen_inline_pcmpistri_self_nul_probe(IR2_OPND vd) * pcmpistri xmm, xmm, 0x3a * Equal Each, masked negative polarity, least-significant index. glibc * uses only the self-compare form as a NUL-position probe. + * pcmpistri xmm, xmm/m128, 0x02/0x12 + * Equal Any, positive/negative polarity, least-significant index. glibc + * uses these for strcspn/strpbrk/strspn delimiter-set scans. * * Other modes have different aggregation/polarity/index semantics and stay on * the helper path. @@ -3790,6 +3876,11 @@ static bool tr_gen_inline_pcmpistri(IR1_INST *pir1, IR2_OPND vd, IR2_OPND vs, return true; } + if (ctrl == 0x02 || ctrl == 0x12) { + tr_gen_inline_pcmpistri_equal_any(vd, vs, ctrl); + return true; + } + if (ctrl == 0x3a) { IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); @@ -3806,7 +3897,7 @@ static bool tr_gen_inline_pcmpistri(IR1_INST *pir1, IR2_OPND vd, IR2_OPND vs, static bool tr_pcmpistri_mem_can_inline(int ctrl) { - return ctrl == 0x18 || ctrl == 0x1a; + return ctrl == 0x02 || ctrl == 0x12 || ctrl == 0x18 || ctrl == 0x1a; } #ifndef CONFIG_LATX_AVX_OPT @@ -3855,6 +3946,44 @@ bool translate_vpcmpistri(IR1_INST *pir1) return true; } +static bool tr_gen_inline_pcmpistrm(IR2_OPND vd, IR2_OPND vs, int ctrl) +{ + if (ctrl != 0x02 && ctrl != 0x12) { + return false; + } + + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + + tr_gen_inline_pcmpistr_equal_any_mask(res, vd, vs, prefix_d, prefix_s); + if (ctrl == 0x12) { + li_d(tmp, 0xffff); + la_xor(res, res, tmp); + la_bstrpick_d(res, res, 15, 0); + } + + tr_gen_inline_pcmpistr_set_flags(res, prefix_d, prefix_s); + + IR2_OPND xmm0 = ra_alloc_xmm(0); + la_vxor_v(xmm0, xmm0, xmm0); + la_vinsgr2vr_d(xmm0, res, 0); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(tmp); + return true; +} + bool translate_pcmpistrm(IR1_INST *pir1) { IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); @@ -3864,8 +3993,18 @@ bool translate_pcmpistrm(IR1_INST *pir1) int imm = ir1_opnd_uimm(opnd2); if (ir1_opnd_is_xmm(opnd1)) { int s = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistrm_xmm, d, s, imm, - LOAD_HELPER_PCMPISTRM_XMM); + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_xmm(s); + if (!tr_gen_inline_pcmpistrm(vd, vs, imm)) { + tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistrm_xmm, d, s, imm, + LOAD_HELPER_PCMPISTRM_XMM); + } + } else if (imm == 0x02 || imm == 0x12) { + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_ftemp(); + load_freg128_from_ir1_mem(vs, opnd1); + tr_gen_inline_pcmpistrm(vd, vs, imm); + ra_free_temp(vs); } else { IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src = ra_alloc_xmm((d + 1) % 7 + 1); From f13247bc30ef8731a672e169301a6b5bdc44166f Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 15:28:03 +0800 Subject: [PATCH 7/8] LATX: shorten PCMPxSTR equal-any lowering Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-simd.c | 50 ++++++++++++++++++++------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index c5a2e6d799..6427125bce 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3723,6 +3723,29 @@ static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, ra_free_temp(tmp); } +static void tr_gen_inline_pcmpistr_equal_any_rounds(IR2_OPND vmatch, + IR2_OPND vd, + IR2_OPND vs, + IR2_OPND prefix_d, + IR2_OPND vcmp, + IR2_OPND vbyte, + IR2_OPND vvalid, + IR2_OPND valid, + int rounds) +{ + for (int i = 0; i < rounds; i++) { + la_vreplvei_b(vbyte, vd, i); + la_vseq_b(vcmp, vs, vbyte); + + la_srli_d(valid, prefix_d, i); + la_andi(valid, valid, 1); + la_sub_d(valid, zero_ir2_opnd, valid); + la_vreplgr2vr_b(vvalid, valid); + la_vand_v(vcmp, vcmp, vvalid); + la_vor_v(vmatch, vmatch, vcmp); + } +} + static void tr_gen_inline_pcmpistr_equal_any_mask(IR2_OPND res, IR2_OPND vd, IR2_OPND vs, @@ -3733,30 +3756,31 @@ static void tr_gen_inline_pcmpistr_equal_any_mask(IR2_OPND res, IR2_OPND vcmp = ra_alloc_ftemp(); IR2_OPND vbyte = ra_alloc_ftemp(); IR2_OPND vvalid = ra_alloc_ftemp(); - IR2_OPND byte = ra_alloc_itemp(); IR2_OPND valid = ra_alloc_itemp(); + IR2_OPND limit = ra_alloc_itemp(); + IR2_OPND label_long = ra_alloc_label(); + IR2_OPND label_done = ra_alloc_label(); la_vxor_v(vmatch, vmatch, vmatch); - for (int i = 0; i < 16; i++) { - la_vpickve2gr_bu(byte, vd, i); - la_vreplgr2vr_b(vbyte, byte); - la_vseq_b(vcmp, vs, vbyte); + li_d(limit, 0x100); + la_bgeu(prefix_d, limit, label_long); - la_srli_d(valid, prefix_d, i); - la_andi(valid, valid, 1); - la_sub_d(valid, zero_ir2_opnd, valid); - la_vreplgr2vr_b(vvalid, valid); - la_vand_v(vcmp, vcmp, vvalid); - la_vor_v(vmatch, vmatch, vcmp); - } + tr_gen_inline_pcmpistr_equal_any_rounds(vmatch, vd, vs, prefix_d, vcmp, + vbyte, vvalid, valid, 8); + la_b(label_done); + + la_label(label_long); + tr_gen_inline_pcmpistr_equal_any_rounds(vmatch, vd, vs, prefix_d, vcmp, + vbyte, vvalid, valid, 16); + la_label(label_done); la_vmsknz_b(vmatch, vmatch); la_vpickve2gr_wu(res, vmatch, 0); la_bstrpick_d(res, res, 15, 0); la_and(res, res, prefix_s); + ra_free_temp(limit); ra_free_temp(valid); - ra_free_temp(byte); ra_free_temp(vvalid); ra_free_temp(vbyte); ra_free_temp(vcmp); From a89dca58822943f50275f8dea99e47a847fe24a6 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 2 Jul 2026 17:42:10 +0800 Subject: [PATCH 8/8] LATX: tune glibc string ifunc defaults Signed-off-by: Lu Zeng --- linux-user/main.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/linux-user/main.c b/linux-user/main.c index e397b76516..f61a6cc6b8 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -151,6 +151,44 @@ static bool enable_strace_error; */ static int last_log_mask; +#ifdef CONFIG_LATX +#define GLIBC_TUNABLES_ENV "GLIBC_TUNABLES" +#define GLIBC_TUNABLES_ENV_PREFIX GLIBC_TUNABLES_ENV "=" +#define LATX_GLIBC_TUNABLES_DEFAULT \ + "glibc.cpu.hwcaps=Fast_Unaligned_Load,Fast_Unaligned_Copy," \ + "Slow_SSE4_2,-SSE4_2" + +static bool latx_glibc_tunables_is_empty(envlist_t *envlist) +{ + g_auto(GStrv) env = envlist_to_environ(envlist, NULL); + size_t prefix_len = strlen(GLIBC_TUNABLES_ENV_PREFIX); + + for (char **entry = env; *entry != NULL; entry++) { + if (strncmp(*entry, GLIBC_TUNABLES_ENV_PREFIX, prefix_len) == 0) { + return (*entry)[prefix_len] == '\0'; + } + } + + return true; +} + +static void latx_apply_glibc_tunables(envlist_t *envlist) +{ + g_autofree char *glibc_tuneables = NULL; + + if (!latx_glibc_tunables_is_empty(envlist)) { + return; + } + + glibc_tuneables = g_strdup_printf("%s%s", GLIBC_TUNABLES_ENV_PREFIX, + LATX_GLIBC_TUNABLES_DEFAULT); + if (envlist_setenv(envlist, glibc_tuneables) != 0) { + fprintf(stderr, "Unable to set GLIBC_TUNABLES for guest\n"); + exit(EXIT_FAILURE); + } +} +#endif + /* * When running 32-on-64 we should make sure we can fit all of the possible * guest address space into a contiguous chunk of virtual host memory. @@ -1374,6 +1412,10 @@ int main(int argc, char **argv, char **envp) } } +#ifdef CONFIG_LATX + latx_apply_glibc_tunables(envlist); +#endif + target_environ = envlist_to_environ(envlist, NULL); envlist_free(envlist);