diff --git a/linux-user/main.c b/linux-user/main.c index e397b76516..f61a6cc6b8 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -151,6 +151,44 @@ static bool enable_strace_error; */ static int last_log_mask; +#ifdef CONFIG_LATX +#define GLIBC_TUNABLES_ENV "GLIBC_TUNABLES" +#define GLIBC_TUNABLES_ENV_PREFIX GLIBC_TUNABLES_ENV "=" +#define LATX_GLIBC_TUNABLES_DEFAULT \ + "glibc.cpu.hwcaps=Fast_Unaligned_Load,Fast_Unaligned_Copy," \ + "Slow_SSE4_2,-SSE4_2" + +static bool latx_glibc_tunables_is_empty(envlist_t *envlist) +{ + g_auto(GStrv) env = envlist_to_environ(envlist, NULL); + size_t prefix_len = strlen(GLIBC_TUNABLES_ENV_PREFIX); + + for (char **entry = env; *entry != NULL; entry++) { + if (strncmp(*entry, GLIBC_TUNABLES_ENV_PREFIX, prefix_len) == 0) { + return (*entry)[prefix_len] == '\0'; + } + } + + return true; +} + +static void latx_apply_glibc_tunables(envlist_t *envlist) +{ + g_autofree char *glibc_tuneables = NULL; + + if (!latx_glibc_tunables_is_empty(envlist)) { + return; + } + + glibc_tuneables = g_strdup_printf("%s%s", GLIBC_TUNABLES_ENV_PREFIX, + LATX_GLIBC_TUNABLES_DEFAULT); + if (envlist_setenv(envlist, glibc_tuneables) != 0) { + fprintf(stderr, "Unable to set GLIBC_TUNABLES for guest\n"); + exit(EXIT_FAILURE); + } +} +#endif + /* * When running 32-on-64 we should make sure we can fit all of the possible * guest address space into a contiguous chunk of virtual host memory. @@ -1374,6 +1412,10 @@ int main(int argc, char **argv, char **envp) } } +#ifdef CONFIG_LATX + latx_apply_glibc_tunables(envlist); +#endif + target_environ = envlist_to_environ(envlist, NULL); envlist_free(envlist); diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index 52e0164560..1d97c075db 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -645,6 +645,272 @@ static void load_step_to_reg_ir1(IR2_OPND *p_step_opnd, IR1_OPND *opnd_ir1) ra_free_temp(tmp_step); } +#ifdef TARGET_X86_64 +static bool tr_can_fast_rep_string_b(IR1_INST *pir1, IR1_OPCODE insn) +{ + return CODEIS64 && + ir1_prefix(pir1) == dt_X86_PREFIX_REP && + pir1->info->id == insn && + pir1->info->x86.prefix[3] != 0x67; +} + +static void tr_emit_rep_string_df_guard(IR2_OPND label_slow) +{ + IR2_OPND df_opnd = ra_alloc_itemp(); + IR2_OPND eflags_opnd = ra_alloc_eflags(); + + la_andi(df_opnd, eflags_opnd, 1 << DF_BIT_INDEX); + la_bne(df_opnd, zero_ir2_opnd, label_slow); + + ra_free_temp(df_opnd); +} + +static void tr_emit_finish_fast_rep_string(IR2_OPND ecx_opnd, + IR2_OPND label_exit) +{ + la_or(ecx_opnd, zero_ir2_opnd, zero_ir2_opnd); + store_ireg_to_ir1(ecx_opnd, &rcx_ir1_opnd, false); + la_b(label_exit); +} + +static bool tr_emit_fast_rep_movsb(IR1_INST *pir1, IR2_OPND rsi_opnd, + IR2_OPND rdi_opnd, IR2_OPND rcx_opnd, + IR2_OPND label_exit) +{ + if (!tr_can_fast_rep_string_b(pir1, dt_X86_INS_MOVSB)) { + return false; + } + + IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_backward = ra_alloc_label(); + IR2_OPND label_backward_no_overlap = ra_alloc_label(); + IR2_OPND label_backward_block_done = ra_alloc_label(); + IR2_OPND label_backward_block_loop = ra_alloc_label(); + IR2_OPND label_backward_qword_done = ra_alloc_label(); + IR2_OPND label_backward_qword_loop = ra_alloc_label(); + IR2_OPND label_backward_tail_done = ra_alloc_label(); + IR2_OPND label_backward_tail_loop = ra_alloc_label(); + IR2_OPND label_no_overlap = ra_alloc_label(); + IR2_OPND label_block_done = ra_alloc_label(); + IR2_OPND label_block_loop = ra_alloc_label(); + IR2_OPND label_qword_done = ra_alloc_label(); + IR2_OPND label_qword_loop = ra_alloc_label(); + IR2_OPND label_tail_done = ra_alloc_label(); + IR2_OPND label_tail_loop = ra_alloc_label(); + IR2_OPND qwords = ra_alloc_itemp(); + IR2_OPND tail = ra_alloc_itemp(); + IR2_OPND value = ra_alloc_itemp(); + IR2_OPND overlap = ra_alloc_itemp(); + IR2_OPND vec0 = ra_alloc_ftemp(); + IR2_OPND vec1 = ra_alloc_ftemp(); + IR2_OPND vec2 = ra_alloc_ftemp(); + IR2_OPND vec3 = ra_alloc_ftemp(); + + IR2_OPND df_opnd = ra_alloc_itemp(); + IR2_OPND eflags_opnd = ra_alloc_eflags(); + la_andi(df_opnd, eflags_opnd, 1 << DF_BIT_INDEX); + la_bne(df_opnd, zero_ir2_opnd, label_backward); + ra_free_temp(df_opnd); + + la_bgeu(rsi_opnd, rdi_opnd, label_no_overlap); + la_sub_d(overlap, rdi_opnd, rsi_opnd); + la_bltu(overlap, rcx_opnd, label_slow); + + la_label(label_no_overlap); + la_srli_d(qwords, rcx_opnd, 6); + la_andi(tail, rcx_opnd, 63); + + la_beq(qwords, zero_ir2_opnd, label_block_done); + la_label(label_block_loop); + la_vld(vec0, rsi_opnd, 0); + la_vld(vec1, rsi_opnd, 16); + la_vld(vec2, rsi_opnd, 32); + la_vld(vec3, rsi_opnd, 48); + la_vst(vec0, rdi_opnd, 0); + la_vst(vec1, rdi_opnd, 16); + la_vst(vec2, rdi_opnd, 32); + la_vst(vec3, rdi_opnd, 48); + la_addi_d(rsi_opnd, rsi_opnd, 64); + la_addi_d(rdi_opnd, rdi_opnd, 64); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_block_loop); + + la_label(label_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); + + la_beq(qwords, zero_ir2_opnd, label_qword_done); + la_label(label_qword_loop); + la_ld_d(value, rsi_opnd, 0); + la_st_d(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, 8); + la_addi_d(rdi_opnd, rdi_opnd, 8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_qword_loop); + + la_label(label_qword_done); + la_beq(tail, zero_ir2_opnd, label_tail_done); + la_label(label_tail_loop); + la_ld_bu(value, rsi_opnd, 0); + la_st_b(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, 1); + la_addi_d(rdi_opnd, rdi_opnd, 1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_tail_loop); + + la_label(label_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + + la_label(label_backward); + la_bgeu(rdi_opnd, rsi_opnd, label_backward_no_overlap); + la_sub_d(overlap, rsi_opnd, rdi_opnd); + la_bltu(overlap, rcx_opnd, label_slow); + + la_label(label_backward_no_overlap); + la_srli_d(qwords, rcx_opnd, 6); + la_andi(tail, rcx_opnd, 63); + + la_beq(qwords, zero_ir2_opnd, label_backward_block_done); + la_label(label_backward_block_loop); + la_vld(vec0, rsi_opnd, -63); + la_vld(vec1, rsi_opnd, -47); + la_vld(vec2, rsi_opnd, -31); + la_vld(vec3, rsi_opnd, -15); + la_vst(vec0, rdi_opnd, -63); + la_vst(vec1, rdi_opnd, -47); + la_vst(vec2, rdi_opnd, -31); + la_vst(vec3, rdi_opnd, -15); + la_addi_d(rsi_opnd, rsi_opnd, -64); + la_addi_d(rdi_opnd, rdi_opnd, -64); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_backward_block_loop); + + la_label(label_backward_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); + + la_beq(qwords, zero_ir2_opnd, label_backward_qword_done); + la_label(label_backward_qword_loop); + la_ld_d(value, rsi_opnd, -7); + la_st_d(value, rdi_opnd, -7); + la_addi_d(rsi_opnd, rsi_opnd, -8); + la_addi_d(rdi_opnd, rdi_opnd, -8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_backward_qword_loop); + + la_label(label_backward_qword_done); + la_beq(tail, zero_ir2_opnd, label_backward_tail_done); + la_label(label_backward_tail_loop); + la_ld_bu(value, rsi_opnd, 0); + la_st_b(value, rdi_opnd, 0); + la_addi_d(rsi_opnd, rsi_opnd, -1); + la_addi_d(rdi_opnd, rdi_opnd, -1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_backward_tail_loop); + + la_label(label_backward_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + + la_label(label_slow); + + ra_free_temp(qwords); + ra_free_temp(tail); + ra_free_temp(value); + ra_free_temp(overlap); + ra_free_temp(vec0); + ra_free_temp(vec1); + ra_free_temp(vec2); + ra_free_temp(vec3); + return true; +} + +static void tr_replicate_byte_to_dword(IR2_OPND dst, IR2_OPND src, + IR2_OPND tmp) +{ + la_andi(dst, src, 0xff); + la_slli_d(tmp, dst, 8); + la_or(dst, dst, tmp); + la_slli_d(tmp, dst, 16); + la_or(dst, dst, tmp); + la_slli_d(tmp, dst, 32); + la_or(dst, dst, tmp); +} + +static bool tr_emit_fast_rep_stosb(IR1_INST *pir1, IR2_OPND rdi_opnd, + IR2_OPND rcx_opnd, IR2_OPND value_opnd, + IR2_OPND label_exit) +{ + if (!tr_can_fast_rep_string_b(pir1, dt_X86_INS_STOSB)) { + return false; + } + + IR2_OPND label_slow = ra_alloc_label(); + IR2_OPND label_block_done = ra_alloc_label(); + IR2_OPND label_block_loop = ra_alloc_label(); + IR2_OPND label_qword_done = ra_alloc_label(); + IR2_OPND label_qword_loop = ra_alloc_label(); + IR2_OPND label_tail_done = ra_alloc_label(); + IR2_OPND label_tail_loop = ra_alloc_label(); + IR2_OPND qwords = ra_alloc_itemp(); + IR2_OPND tail = ra_alloc_itemp(); + IR2_OPND fill = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND vfill = ra_alloc_ftemp(); + + tr_emit_rep_string_df_guard(label_slow); + + tr_replicate_byte_to_dword(fill, value_opnd, tmp); + la_vreplgr2vr_b(vfill, value_opnd); + la_srli_d(qwords, rcx_opnd, 7); + la_andi(tail, rcx_opnd, 127); + + la_beq(qwords, zero_ir2_opnd, label_block_done); + la_label(label_block_loop); + la_vst(vfill, rdi_opnd, 0); + la_vst(vfill, rdi_opnd, 16); + la_vst(vfill, rdi_opnd, 32); + la_vst(vfill, rdi_opnd, 48); + la_vst(vfill, rdi_opnd, 64); + la_vst(vfill, rdi_opnd, 80); + la_vst(vfill, rdi_opnd, 96); + la_vst(vfill, rdi_opnd, 112); + la_addi_d(rdi_opnd, rdi_opnd, 128); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_block_loop); + + la_label(label_block_done); + la_srli_d(qwords, tail, 3); + la_andi(tail, tail, 7); + + la_beq(qwords, zero_ir2_opnd, label_qword_done); + la_label(label_qword_loop); + la_st_d(fill, rdi_opnd, 0); + la_addi_d(rdi_opnd, rdi_opnd, 8); + la_addi_d(qwords, qwords, -1); + la_bne(qwords, zero_ir2_opnd, label_qword_loop); + + la_label(label_qword_done); + la_beq(tail, zero_ir2_opnd, label_tail_done); + la_label(label_tail_loop); + la_st_b(fill, rdi_opnd, 0); + la_addi_d(rdi_opnd, rdi_opnd, 1); + la_addi_d(tail, tail, -1); + la_bne(tail, zero_ir2_opnd, label_tail_loop); + + la_label(label_tail_done); + tr_emit_finish_fast_rep_string(rcx_opnd, label_exit); + + la_label(label_slow); + + ra_free_temp(qwords); + ra_free_temp(tail); + ra_free_temp(fill); + ra_free_temp(tmp); + ra_free_temp(vfill); + return true; +} +#endif + bool translate_movs(IR1_INST *pir1) { IR2_OPND esi_opnd = ra_alloc_gpr(esi_index); @@ -769,6 +1035,10 @@ bool translate_movs(IR1_INST *pir1) lsassert(0); } +#ifdef TARGET_X86_64 + tr_emit_fast_rep_movsb(pir1, esi_opnd, edi_opnd, ecx_opnd, label_exit); +#endif + /* 2. preparations outside the loop */ IR2_OPND step_opnd = ra_alloc_itemp(); load_step_to_reg_ir1(&step_opnd, mem_di); @@ -970,6 +1240,10 @@ bool translate_stos(IR1_INST *pir1) } IR2_OPND eax_value_opnd = load_ireg_from_ir1(opnd_eax, SIGN_EXTENSION, false); +#ifdef TARGET_X86_64 + tr_emit_fast_rep_stosb(pir1, edi_opnd, ecx_opnd, eax_value_opnd, + label_exit); +#endif IR2_OPND step_opnd = ra_alloc_itemp(); load_step_to_reg_ir1(&step_opnd, opnd_di); diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 9e97cead69..6427125bce 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3646,6 +3646,284 @@ bool translate_pcmpestrm(IR1_INST *pir1) return true; } +static void tr_gen_inline_pcmpistri_prefix_mask(IR2_OPND prefix, + IR2_OPND vec, + IR2_OPND vtmp, + IR2_OPND tmp) +{ + la_vor_v(vtmp, vec, vec); + la_vmsknz_b(vtmp, vtmp); + la_vpickve2gr_wu(prefix, vtmp, 0); + la_bstrpick_d(prefix, prefix, 15, 0); + + /* + * Prefix mask before the first NUL: + * 0b...10111 -> ((mask ^ (mask + 1)) >> 1) == 0b00111 + * 0xffff -> 0xffff + */ + la_addi_d(tmp, prefix, 1); + la_xor(prefix, prefix, tmp); + la_srli_d(prefix, prefix, 1); + la_bstrpick_d(prefix, prefix, 15, 0); +} + +static void tr_gen_inline_pcmpistr_set_flags(IR2_OPND res, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND flags = ra_alloc_itemp(); + + /* + * CF = res != 0, OF = res[0], ZF/SF indicate whether src/dst contain + * a NUL in the first 16 bytes. AF/PF are cleared. + */ + la_mov64(flags, zero_ir2_opnd); + la_sltu(tmp, zero_ir2_opnd, res); + la_or(flags, flags, tmp); + + la_andi(tmp, res, 1); + la_bstrins_w(flags, tmp, OF_BIT_INDEX, OF_BIT_INDEX); + + li_d(tmp, 0xffff); + la_xor(tmp, prefix_s, tmp); + la_sltu(tmp, zero_ir2_opnd, tmp); + la_bstrins_w(flags, tmp, ZF_BIT_INDEX, ZF_BIT_INDEX); + + li_d(tmp, 0xffff); + la_xor(tmp, prefix_d, tmp); + la_sltu(tmp, zero_ir2_opnd, tmp); + la_bstrins_w(flags, tmp, SF_BIT_INDEX, SF_BIT_INDEX); + + la_x86mtflag(flags, CF_USEDEF_BIT | PF_USEDEF_BIT | AF_USEDEF_BIT | + ZF_USEDEF_BIT | SF_USEDEF_BIT | OF_USEDEF_BIT); + + ra_free_temp(flags); + ra_free_temp(tmp); +} + +static void tr_gen_inline_pcmpistri_finish(IR2_OPND res, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + tr_gen_inline_pcmpistr_set_flags(res, prefix_d, prefix_s); + + IR2_OPND tmp = ra_alloc_itemp(); + IR2_OPND flags = ra_alloc_itemp(); + + /* ECX = res ? ctz(res) : 16 */ + la_ctz_w(tmp, res); + la_addi_d(flags, zero_ir2_opnd, 16); + la_maskeqz(tmp, tmp, res); + la_masknez(flags, flags, res); + la_or(tmp, tmp, flags); + la_mov64(ra_alloc_gpr(ecx_index), tmp); + + ra_free_temp(flags); + ra_free_temp(tmp); +} + +static void tr_gen_inline_pcmpistr_equal_any_rounds(IR2_OPND vmatch, + IR2_OPND vd, + IR2_OPND vs, + IR2_OPND prefix_d, + IR2_OPND vcmp, + IR2_OPND vbyte, + IR2_OPND vvalid, + IR2_OPND valid, + int rounds) +{ + for (int i = 0; i < rounds; i++) { + la_vreplvei_b(vbyte, vd, i); + la_vseq_b(vcmp, vs, vbyte); + + la_srli_d(valid, prefix_d, i); + la_andi(valid, valid, 1); + la_sub_d(valid, zero_ir2_opnd, valid); + la_vreplgr2vr_b(vvalid, valid); + la_vand_v(vcmp, vcmp, vvalid); + la_vor_v(vmatch, vmatch, vcmp); + } +} + +static void tr_gen_inline_pcmpistr_equal_any_mask(IR2_OPND res, + IR2_OPND vd, + IR2_OPND vs, + IR2_OPND prefix_d, + IR2_OPND prefix_s) +{ + IR2_OPND vmatch = ra_alloc_ftemp(); + IR2_OPND vcmp = ra_alloc_ftemp(); + IR2_OPND vbyte = ra_alloc_ftemp(); + IR2_OPND vvalid = ra_alloc_ftemp(); + IR2_OPND valid = ra_alloc_itemp(); + IR2_OPND limit = ra_alloc_itemp(); + IR2_OPND label_long = ra_alloc_label(); + IR2_OPND label_done = ra_alloc_label(); + + la_vxor_v(vmatch, vmatch, vmatch); + li_d(limit, 0x100); + la_bgeu(prefix_d, limit, label_long); + + tr_gen_inline_pcmpistr_equal_any_rounds(vmatch, vd, vs, prefix_d, vcmp, + vbyte, vvalid, valid, 8); + la_b(label_done); + + la_label(label_long); + tr_gen_inline_pcmpistr_equal_any_rounds(vmatch, vd, vs, prefix_d, vcmp, + vbyte, vvalid, valid, 16); + + la_label(label_done); + la_vmsknz_b(vmatch, vmatch); + la_vpickve2gr_wu(res, vmatch, 0); + la_bstrpick_d(res, res, 15, 0); + la_and(res, res, prefix_s); + + ra_free_temp(limit); + ra_free_temp(valid); + ra_free_temp(vvalid); + ra_free_temp(vbyte); + ra_free_temp(vcmp); + ra_free_temp(vmatch); +} + +static void tr_gen_inline_pcmpistri_equal_each_negative(IR2_OPND vd, + IR2_OPND vs) +{ + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND vcmp = ra_alloc_ftemp(); + + IR2_OPND eq_mask = ra_alloc_itemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + la_vseq_b(vcmp, vd, vs); + la_vmsknz_b(vcmp, vcmp); + la_vpickve2gr_wu(eq_mask, vcmp, 0); + la_bstrpick_d(eq_mask, eq_mask, 15, 0); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + + ra_free_temp(vcmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + ra_free_temp(tmp); + + la_or(res, prefix_d, prefix_s); + la_nor(eq_mask, eq_mask, zero_ir2_opnd); + la_bstrpick_d(eq_mask, eq_mask, 15, 0); + la_and(res, res, eq_mask); + + tr_gen_inline_pcmpistri_finish(res, prefix_d, prefix_s); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(eq_mask); +} + +static void tr_gen_inline_pcmpistri_equal_any(IR2_OPND vd, IR2_OPND vs, + int ctrl) +{ + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + + tr_gen_inline_pcmpistr_equal_any_mask(res, vd, vs, prefix_d, prefix_s); + if (ctrl == 0x12) { + li_d(tmp, 0xffff); + la_xor(res, res, tmp); + la_bstrpick_d(res, res, 15, 0); + } + + tr_gen_inline_pcmpistri_finish(res, prefix_d, prefix_s); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(tmp); +} + +static void tr_gen_inline_pcmpistri_self_nul_probe(IR2_OPND vd) +{ + IR2_OPND vtmp = ra_alloc_ftemp(); + IR2_OPND prefix = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix, vd, vtmp, tmp); + ra_free_temp(vtmp); + ra_free_temp(tmp); + + la_nor(res, prefix, zero_ir2_opnd); + la_bstrpick_d(res, res, 15, 0); + + tr_gen_inline_pcmpistri_finish(res, prefix, prefix); + + ra_free_temp(res); + ra_free_temp(prefix); +} + +/* + * glibc string fast paths use: + * pcmpistri xmm, xmm/m128, 0x18/0x1a + * Equal Each, negative polarity, least-significant index. + * 0x18 vs 0x1a differs only in unsigned/signed byte elements, which does + * not change byte equality. + * pcmpistri xmm, xmm, 0x3a + * Equal Each, masked negative polarity, least-significant index. glibc + * uses only the self-compare form as a NUL-position probe. + * pcmpistri xmm, xmm/m128, 0x02/0x12 + * Equal Any, positive/negative polarity, least-significant index. glibc + * uses these for strcspn/strpbrk/strspn delimiter-set scans. + * + * Other modes have different aggregation/polarity/index semantics and stay on + * the helper path. + */ +static bool tr_gen_inline_pcmpistri(IR1_INST *pir1, IR2_OPND vd, IR2_OPND vs, + int ctrl) +{ + if (ctrl == 0x18 || ctrl == 0x1a) { + tr_gen_inline_pcmpistri_equal_each_negative(vd, vs); + return true; + } + + if (ctrl == 0x02 || ctrl == 0x12) { + tr_gen_inline_pcmpistri_equal_any(vd, vs, ctrl); + return true; + } + + if (ctrl == 0x3a) { + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + + if (ir1_opnd_is_xmm(opnd1) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + tr_gen_inline_pcmpistri_self_nul_probe(vd); + return true; + } + } + + return false; +} + +static bool tr_pcmpistri_mem_can_inline(int ctrl) +{ + return ctrl == 0x02 || ctrl == 0x12 || ctrl == 0x18 || ctrl == 0x1a; +} + #ifndef CONFIG_LATX_AVX_OPT bool translate_pcmpistri(IR1_INST *pir1) #else @@ -3659,8 +3937,18 @@ bool translate_vpcmpistri(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); if (ir1_opnd_is_xmm(opnd1)) { int s = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistri_xmm, d, s, imm, - LOAD_HELPER_PCMPISTRI_XMM); + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_xmm(s); + if (!tr_gen_inline_pcmpistri(pir1, vd, vs, imm)) { + tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistri_xmm, d, s, imm, + LOAD_HELPER_PCMPISTRI_XMM); + } + } else if (tr_pcmpistri_mem_can_inline(imm)) { + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_ftemp(); + load_freg128_from_ir1_mem(vs, opnd1); + tr_gen_inline_pcmpistri(pir1, vd, vs, imm); + ra_free_temp(vs); } else { IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src = ra_alloc_xmm((d + 1) % 8); @@ -3682,6 +3970,44 @@ bool translate_vpcmpistri(IR1_INST *pir1) return true; } +static bool tr_gen_inline_pcmpistrm(IR2_OPND vd, IR2_OPND vs, int ctrl) +{ + if (ctrl != 0x02 && ctrl != 0x12) { + return false; + } + + IR2_OPND vd_nz = ra_alloc_ftemp(); + IR2_OPND vs_nz = ra_alloc_ftemp(); + IR2_OPND prefix_d = ra_alloc_itemp(); + IR2_OPND prefix_s = ra_alloc_itemp(); + IR2_OPND res = ra_alloc_itemp(); + IR2_OPND tmp = ra_alloc_itemp(); + + tr_gen_inline_pcmpistri_prefix_mask(prefix_d, vd, vd_nz, tmp); + tr_gen_inline_pcmpistri_prefix_mask(prefix_s, vs, vs_nz, tmp); + ra_free_temp(vs_nz); + ra_free_temp(vd_nz); + + tr_gen_inline_pcmpistr_equal_any_mask(res, vd, vs, prefix_d, prefix_s); + if (ctrl == 0x12) { + li_d(tmp, 0xffff); + la_xor(res, res, tmp); + la_bstrpick_d(res, res, 15, 0); + } + + tr_gen_inline_pcmpistr_set_flags(res, prefix_d, prefix_s); + + IR2_OPND xmm0 = ra_alloc_xmm(0); + la_vxor_v(xmm0, xmm0, xmm0); + la_vinsgr2vr_d(xmm0, res, 0); + + ra_free_temp(res); + ra_free_temp(prefix_s); + ra_free_temp(prefix_d); + ra_free_temp(tmp); + return true; +} + bool translate_pcmpistrm(IR1_INST *pir1) { IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); @@ -3691,8 +4017,18 @@ bool translate_pcmpistrm(IR1_INST *pir1) int imm = ir1_opnd_uimm(opnd2); if (ir1_opnd_is_xmm(opnd1)) { int s = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistrm_xmm, d, s, imm, - LOAD_HELPER_PCMPISTRM_XMM); + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_xmm(s); + if (!tr_gen_inline_pcmpistrm(vd, vs, imm)) { + tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_pcmpistrm_xmm, d, s, imm, + LOAD_HELPER_PCMPISTRM_XMM); + } + } else if (imm == 0x02 || imm == 0x12) { + IR2_OPND vd = ra_alloc_xmm(d); + IR2_OPND vs = ra_alloc_ftemp(); + load_freg128_from_ir1_mem(vs, opnd1); + tr_gen_inline_pcmpistrm(vd, vs, imm); + ra_free_temp(vs); } else { IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src = ra_alloc_xmm((d + 1) % 7 + 1);