diff --git a/linux-user/main.c b/linux-user/main.c index f8168cfdec..0d99775340 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -659,6 +659,19 @@ static void handle_arg_latx_prlimit(const char *arg) option_prlimit = strtol(arg, NULL, 0); } +static void handle_arg_latx_eflags_cross(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + fprintf(stderr, + "LATX_EFLAGS_CROSS must be exactly 0 or 1 (got '%s')\n", + arg); + exit(EXIT_FAILURE); + } + option_eflags_cross = value; +} + #ifdef CONFIG_LATX_AVX_OPT static void handle_arg_latx_avx_cpuid(const char *arg) { @@ -920,6 +933,9 @@ static const struct qemu_argument arg_table[] = { "", "enable rounding opt"}, {"latx-cvt-opt", "LATX_CVT_OPT", true, handle_arg_latx_cvt_opt, "", "enable cvt opt"}, + {"latx-eflags-cross", "LATX_EFLAGS_CROSS", true, + handle_arg_latx_eflags_cross, "0|1", + "enable TU EFLAGS analysis across direct CALL and RET"}, #if defined(CONFIG_LATX_AVX_OPT) {"latx-avx-cpuid", "LATX_AVX_CPUID", true, handle_arg_latx_avx_cpuid, "", "enable avx cpuid"}, diff --git a/target/i386/latx/include/flag-reduction.h b/target/i386/latx/include/flag-reduction.h index 23cd63e4c6..c4fea6036e 100644 --- a/target/i386/latx/include/flag-reduction.h +++ b/target/i386/latx/include/flag-reduction.h @@ -40,11 +40,14 @@ typedef struct { #define __ALL_EFLAGS (__OF | __SF | __ZF | __AF | __PF | __CF) void flag_gen(IR1_INST *pir1); +uint8 flag_reduction_get_arch_use(IR1_INST *pir1); #ifdef CONFIG_LATX_FLAG_REDUCTION uint8 pending_use_of_succ(void *tb, int indirect_depth, int max_depth); void flag_reduction(IR1_INST *pir1, uint8 *pending_use); uint8 flag_reduction_check(TranslationBlock *tb); +void flag_reduction_get_tb_summary(TranslationBlock *tb, + uint8 *use_before_def, uint8 *must_def); #define DEF_FLAG_RDTN(_prex) \ uint8 _prex##_pending_use = __ALL_EFLAGS; diff --git a/target/i386/latx/include/insts-pattern.h b/target/i386/latx/include/insts-pattern.h index 781a6677ca..6cacb4afff 100644 --- a/target/i386/latx/include/insts-pattern.h +++ b/target/i386/latx/include/insts-pattern.h @@ -163,4 +163,13 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *ir1, int index, bool try_translate_instptn(IR1_INST *pir1); +static inline uint8_t instptn_replaced_eflag_use(IR1_INST *ir1) +{ +#ifdef CONFIG_LATX_INSTS_PATTERN + return ir1->instptn.replaced_eflag_use; +#else + return 0; +#endif +} + #endif diff --git a/target/i386/latx/include/ir1.h b/target/i386/latx/include/ir1.h index 60524add16..82d869901f 100644 --- a/target/i386/latx/include/ir1.h +++ b/target/i386/latx/include/ir1.h @@ -143,6 +143,8 @@ typedef struct IR1_INST { #ifdef CONFIG_LATX_INSTS_PATTERN struct { int opc; + /* EFLAGS reads replaced by the native pattern translation. */ + uint8_t replaced_eflag_use; struct IR1_INST * next; /* index of IR1 list */ } instptn; #endif diff --git a/target/i386/latx/include/latx-config.h b/target/i386/latx/include/latx-config.h index 2badbe3350..315168452d 100644 --- a/target/i386/latx/include/latx-config.h +++ b/target/i386/latx/include/latx-config.h @@ -44,6 +44,7 @@ void latx_handle_args(char *filename); #ifdef CONFIG_LATX_TU void target_disasm(struct TranslationBlock *tb, int max_insns); +extern __thread bool suppress_disasm_side_effects; #endif #ifdef CONFIG_LATX_LARGE_CC diff --git a/target/i386/latx/include/latx-options.h b/target/i386/latx/include/latx-options.h index e4d86cf46a..6d9b4d102a 100644 --- a/target/i386/latx/include/latx-options.h +++ b/target/i386/latx/include/latx-options.h @@ -27,6 +27,9 @@ extern int option_instptn; #ifdef CONFIG_LATX_FLAG_REDUCTION extern int option_flag_reduction; #endif + +/* Proof-driven TU EFLAGS analysis across direct CALL/RET. */ +extern int option_eflags_cross; #ifdef CONFIG_LATX_TU extern int option_tu_link; #endif @@ -132,6 +135,7 @@ extern unsigned long long counter_mips_tr; ENVFUN(LATX_PRLIMIT, handle_arg_latx_prlimit) \ ENVFUN(LATX_ROUNDING_OPT, handle_arg_latx_rounding) \ ENVFUN(LATX_CVT_OPT, handle_arg_latx_cvt_opt) \ + ENVFUN(LATX_EFLAGS_CROSS, handle_arg_latx_eflags_cross) \ ENVFUN(LATX_FPUTAG, handle_arg_latx_fputag) \ ENVFUN(SAVE_XMM, handle_arg_save_xmm) \ ENVFUN(LATX_JRRA, handle_arg_latx_jrra) \ diff --git a/target/i386/latx/ir1/ir1.c b/target/i386/latx/ir1/ir1.c index 41ba0ee23b..2b465816f6 100644 --- a/target/i386/latx/ir1/ir1.c +++ b/target/i386/latx/ir1/ir1.c @@ -390,6 +390,7 @@ ADDRX ir1_disasm(IR1_INST *ir1, uint8_t *addr, ADDRX t_pc, int ir1_num, void *pi } #ifdef CONFIG_LATX_INSTS_PATTERN ir1->instptn.opc = INSTPTN_OPC_NONE; + ir1->instptn.replaced_eflag_use = 0; ir1->instptn.next = NULL; #endif return (ADDRX)(ir1->info->address + ir1->info->size); diff --git a/target/i386/latx/latx-config.c b/target/i386/latx/latx-config.c index 3d067e169b..92cb01990d 100644 --- a/target/i386/latx/latx-config.c +++ b/target/i386/latx/latx-config.c @@ -27,15 +27,16 @@ void target_disasm(struct TranslationBlock *tb, int max_insns) { /* max_insns = 100; */ - counter_tb_tr += 1; - - trace_xtm_tr_tb((void *)tb, (void *)tb->tc.ptr, - (void *)(unsigned long long)tb->pc); - - if (option_dump) { - qemu_log("=====================================\n"); - qemu_log("|| TB translation : %14p ||\n", tb); - qemu_log("=====================================\n"); + if (!suppress_disasm_side_effects) { + counter_tb_tr += 1; + trace_xtm_tr_tb((void *)tb, (void *)tb->tc.ptr, + (void *)(unsigned long long)tb->pc); + + if (option_dump) { + qemu_log("=====================================\n"); + qemu_log("|| TB translation : %14p ||\n", tb); + qemu_log("=====================================\n"); + } } /* target => IR1 @@ -62,14 +63,18 @@ void target_disasm(struct TranslationBlock *tb, int max_insns) #endif #ifdef CONFIG_LATX_DEBUG - counter_ir1_tr += tb->icount; + if (!suppress_disasm_side_effects) { + counter_ir1_tr += tb->icount; + } #endif /* tr_disasm(tb, max_insns); */ #ifdef CONFIG_LATX_PROFILER - qatomic_set(&prof->tr_disasm_time, - prof->tr_disasm_time + profile_getclock() - ti); + if (!suppress_disasm_side_effects) { + qatomic_set(&prof->tr_disasm_time, + prof->tr_disasm_time + profile_getclock() - ti); + } #endif /* ir1_optimization(tb); */ diff --git a/target/i386/latx/latx-options.c b/target/i386/latx/latx-options.c index 43671bbb55..9555a76320 100644 --- a/target/i386/latx/latx-options.c +++ b/target/i386/latx/latx-options.c @@ -31,6 +31,7 @@ int option_avx_cpuid = 0; #ifdef CONFIG_LATX_FLAG_REDUCTION int option_flag_reduction = 1; #endif +int option_eflags_cross; int option_lative = 0; #if defined(CONFIG_LATX_JRRA_STACK) && defined(CONFIG_LATX_LSFPU) @@ -236,6 +237,7 @@ void options_init(void) option_latx_disassemble_trace_cmp = 0; option_enable_lasx = 1; option_vpaes = 0; + option_eflags_cross = 0; counter_tb_exec = 0; counter_tb_tr = 0; diff --git a/target/i386/latx/optimization/flag-reduction.c b/target/i386/latx/optimization/flag-reduction.c index c695f1be65..b6db0c740d 100644 --- a/target/i386/latx/optimization/flag-reduction.c +++ b/target/i386/latx/optimization/flag-reduction.c @@ -8,6 +8,7 @@ #include "ir1.h" #include "translate.h" #include "flag-reduction.h" +#include "insts-pattern.h" /** * @brief ir1 opcode (x86) per instruction eflags using table @@ -293,6 +294,19 @@ static const IR1_EFLAG_USEDEF *ir1_opcode_to_eflag_usedef(IR1_INST *ir1) return ir1_opcode_eflag_usedef + (ir1_opcode(ir1) - dt_X86_INS_INVALID); } +uint8 flag_reduction_get_arch_use(IR1_INST *ir1) +{ + return ir1_opcode_to_eflag_usedef(ir1)->use & __ALL_EFLAGS; +} + +static inline IR1_EFLAG_USEDEF ir1_effective_eflag_usedef(IR1_INST *ir1) +{ + IR1_EFLAG_USEDEF usedef = *ir1_opcode_to_eflag_usedef(ir1); + + usedef.use &= ~instptn_replaced_eflag_use(ir1); + return usedef; +} + #ifdef CONFIG_LATX_FLAG_REDUCTION static inline uint32_t rotate_shift_get_masked_imm(IR1_OPND *d, IR1_OPND *s) @@ -362,6 +376,87 @@ static inline bool cmp_scas_need_zf(IR1_INST *pir1) ir1_opcode(pir1) == dt_X86_INS_SCASQ); } +/* + * Some instructions only update EFLAGS when their runtime count is non-zero. + * Such an update is not a must-def and therefore cannot stop EFLAGS liveness + * propagated from a successor TB. + */ +static bool eflags_def_may_be_skipped(IR1_INST *pir1) +{ + IR1_OPCODE op = ir1_opcode(pir1); + + if ((op == dt_X86_INS_CMPSB || op == dt_X86_INS_CMPSW || + op == dt_X86_INS_CMPSD || op == dt_X86_INS_CMPSQ || + op == dt_X86_INS_SCASB || op == dt_X86_INS_SCASW || + op == dt_X86_INS_SCASD || op == dt_X86_INS_SCASQ) && + (ir1_prefix(pir1) == dt_X86_PREFIX_REPE || + ir1_prefix(pir1) == dt_X86_PREFIX_REPNE)) { + /* RCX may be zero, in which case no comparison takes place. */ + return true; + } + + if (op == dt_X86_INS_RCL || op == dt_X86_INS_RCR) { + /* The effective count also depends on the operand width. */ + return true; + } + + if (op == dt_X86_INS_ROL || op == dt_X86_INS_ROR || + op == dt_X86_INS_SAL || op == dt_X86_INS_SAR || + op == dt_X86_INS_SHL || op == dt_X86_INS_SHR || + op == dt_X86_INS_SHLD || op == dt_X86_INS_SHRD) { + int opnd_num = ir1_get_opnd_num(pir1); + + /* A one-operand form has an implicit count of one. */ + if (opnd_num == 1) { + return false; + } + + IR1_OPND *count = ir1_get_opnd(pir1, opnd_num - 1); + if (!ir1_opnd_is_imm(count)) { + return true; + } + + IR1_OPND *dest = ir1_get_opnd(pir1, 0); + return rotate_shift_get_masked_imm(dest, count) == 0; + } + + return false; +} + +/* + * Summarize a straight-line TB using the same use/def table as the existing + * flag-reduction pass. + * + * use_before_def: flags read before this TB certainly overwrites them. + * must_def: flags certainly overwritten on every normal execution. + */ +void flag_reduction_get_tb_summary(TranslationBlock *tb, + uint8 *use_before_def, uint8 *must_def) +{ + uint8 preserve = __ALL_EFLAGS; + uint8 use = __NONE; + + for (int i = 0; i < tb_ir1_num(tb); i++) { + IR1_INST *pir1 = tb_ir1_inst(tb, i); + IR1_EFLAG_USEDEF usedef = ir1_effective_eflag_usedef(pir1); + uint8 curr_use = usedef.use & __ALL_EFLAGS; + uint8 curr_def = (usedef.def | usedef.undef) & __ALL_EFLAGS; + + if (cmp_scas_need_zf(pir1)) { + curr_use |= __ZF; + } + + use |= curr_use & preserve; + if (eflags_def_may_be_skipped(pir1)) { + curr_def = __NONE; + } + preserve &= ~curr_def; + } + + *use_before_def = use; + *must_def = __ALL_EFLAGS & ~preserve; +} + /** * @brief Find if we have enough information by scan TB * @@ -383,16 +478,16 @@ static bool flag_reduction_pass1(void *tb) /* scanning if this insts will def ALL_EFLAGS */ for (int i = tb_ir1_num(ptb) - 1; i >= 0; --i) { pir1 = tb_ir1_inst(ptb, i); - const IR1_EFLAG_USEDEF *usedef = ir1_opcode_to_eflag_usedef(pir1); + IR1_EFLAG_USEDEF usedef = ir1_effective_eflag_usedef(pir1); /* * NOTE: if you find some insts will use ALL_EFLAGS * you can add this case: - * if (usedef->use == __ALL_EFLAGS) return false; + * if (usedef.use == __ALL_EFLAGS) return false; */ - if (usedef->use != __NONE) { + if (usedef.use != __NONE) { goto _false_path; - } else if (usedef->def == __NONE) { + } else if (usedef.def == __NONE) { /* curr_inst not def any flags */ continue; } else { @@ -442,7 +537,7 @@ void flag_reduction(IR1_INST *pir1, uint8 *pending_use) * - flag use: current inst will use flags * - flag undef: current inst mark undef flags */ - IR1_EFLAG_USEDEF curr_usedef = *ir1_opcode_to_eflag_usedef(pir1); + IR1_EFLAG_USEDEF curr_usedef = ir1_effective_eflag_usedef(pir1); #ifndef CONFIG_LATX_RADICAL_EFLAGS current_def = curr_usedef.def; diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 4bac7d65c2..7837617656 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -14,6 +14,7 @@ #include "reg-alloc.h" #include "translate.h" #include "insts-pattern.h" +#include "flag-reduction.h" #ifdef CONFIG_LATX_INSTS_PATTERN @@ -184,6 +185,8 @@ static int inst_pattern(TranslationBlock *tb, pir1->instptn.opc = INSTPTN_OPC_CMP_SBB; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; + ir1->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1); // ir1->instptn.next = NULL; return 1; } @@ -212,6 +215,8 @@ static int inst_pattern(TranslationBlock *tb, pir1->instptn.opc = INSTPTN_OPC_CMP_XXCC; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; + ir1->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1); // ir1->instptn.next = NULL; return 1; default: @@ -256,6 +261,8 @@ static int inst_pattern(TranslationBlock *tb, pir1->instptn.opc = INSTPTN_OPC_TEST_XXCC; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; + ir1->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1); // ir1->instptn.next = NULL; return 1; default: @@ -342,6 +349,8 @@ static int inst_pattern(TranslationBlock *tb, pir1->instptn.opc = INSTPTN_OPC_UCOMISD_SETA; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; + ir1->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1); // ir1->instptn.next = NULL; return 1; default: @@ -384,6 +393,11 @@ static int inst_pattern(TranslationBlock *tb, pir1->instptn.opc = INSTPTN_OPC_NEG_CMOVCC; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; + if (!ir1_is_prefix_lock(pir1) || + !ir1_opnd_is_mem(ir1_get_opnd(pir1, 0))) { + ir1->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1); + } return 1; default: return 0; @@ -462,6 +476,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_CMP_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_cmp_xx_jcc_0(); @@ -469,6 +485,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_CMP_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -502,6 +520,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_TEST_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_test_xx_jcc_0(); @@ -509,6 +529,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_TEST_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -529,6 +551,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_BT_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_bt_xx_jcc_0(); @@ -536,6 +560,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_BT_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -561,6 +587,11 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_SUB_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + if (!ir1_is_prefix_lock(pir1) || + !ir1_opnd_is_mem(ir1_get_opnd(pir1, 0))) { + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); + } // ir1_jcc->instptn.next = NULL; } return false; @@ -580,6 +611,13 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_SHR_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + uint32_t count_mask = + ir1_opnd_size(ir1_get_opnd(pir1, 0)) == 64 ? + 0x3f : 0x1f; + if ((ir1_opnd_uimm(opnd1) & count_mask) != 0) { + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); + } // ir1_jcc->instptn.next = NULL; } return false; @@ -596,6 +634,11 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_AND_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + if (!ir1_is_prefix_lock(pir1) || + !ir1_opnd_is_mem(ir1_get_opnd(pir1, 0))) { + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); + } // ir1_jcc->instptn.next = NULL; } return false; @@ -622,6 +665,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_COMISD_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_comisd_xx_jcc_0(); @@ -629,6 +674,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_COMISD_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -654,6 +701,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_COMISS_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_comiss_xx_jcc_0(); @@ -661,6 +710,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_COMISS_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -686,6 +737,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_UCOMISD_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); // ir1_jcc->instptn.next = NULL; } else { instptn_check_ucomisd_xx_jcc_0(); @@ -693,6 +746,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_UCOMISD_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; @@ -718,6 +773,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.opc = INSTPTN_OPC_UCOMISS_JCC; pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); ir1_jcc->instptn.next = NULL; } else { instptn_check_ucomiss_xx_jcc_0(); @@ -725,6 +782,8 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_UCOMISS_XX_JCC; ir1_jcc->instptn.next = tb_ir1_inst(tb, pir1_index); + ir1_jcc->instptn.replaced_eflag_use = + flag_reduction_get_arch_use(ir1_jcc); tb->has_jcc_end_ptn = true; } return false; diff --git a/target/i386/latx/optimization/ir1-optimization.c b/target/i386/latx/optimization/ir1-optimization.c index 99b32fcc13..c6b900d22f 100644 --- a/target/i386/latx/optimization/ir1-optimization.c +++ b/target/i386/latx/optimization/ir1-optimization.c @@ -28,29 +28,47 @@ * **Also, this function only analysis the information and set IR1_INST!** */ +static void ir1_pattern_discovery(TranslationBlock *tb) +{ + if (!tb->icount) { + return; + } + + DEF_INSTS_PTN(ptn); + for (int i = tb_ir1_num(tb) - 1; i >= 0; --i) { + IR1_INST *ir1 = tb_ir1_inst(tb, i); + OPT_INSTS_PTN(tb, ir1, i, ptn); + } +} + #ifdef CONFIG_LATX_TU /* static int opt, noopt; */ -static void ir1_optimization_over_tb(TranslationBlock *tb) +static void ir1_optimization_over_tb(TranslationBlock *tb, + bool use_calculated_live_out) { + (void)use_calculated_live_out; if (!tb->icount) { return; } + ir1_pattern_discovery(tb); + IR1_INST *ir1 = NULL; /* cross scanning var defination */ DEF_FLAG_RDTN(rdtn); - DEF_INSTS_PTN(ptn); - /* check if need cross tb analyze */ - CHK_FLAG_RDTN(rdtn, tb); #ifdef CONFIG_LATX_FLAG_REDUCTION - rdtn_pending_use &= tb->s_data->eflag_out; + if (use_calculated_live_out) { + rdtn_pending_use = tb->s_data->eflag_out; + } else { + /* check if need cross tb analyze */ + CHK_FLAG_RDTN(rdtn, tb); + rdtn_pending_use &= tb->s_data->eflag_out; + } #endif /* scanning instructions in reverse order */ for (int i = tb_ir1_num(tb) - 1; i >= 0; --i) { ir1 = tb_ir1_inst(tb, i); /* do core optimize */ OPT_FLAG_RDTN(rdtn, ir1); - /* TODO: TU */ - OPT_INSTS_PTN(tb, ir1, i, ptn); } SAVE_FLAG_TO_TB(rdtn, tb); } @@ -71,6 +89,14 @@ static void get_eflag_out(TranslationBlock *tb) } break; case IR1_TYPE_JUMP: + if (tb->s_data->next_tb[TU_TB_INDEX_TARGET]) { + TranslationBlock *tmp_tb = + (TranslationBlock *)tb->s_data->next_tb[TU_TB_INDEX_TARGET]; + tb->s_data->eflag_out |= tmp_tb->eflag_use; + } else { + tb->s_data->eflag_out |= __ALL_EFLAGS; + } + break; case IR1_TYPE_CALL: if (tb->s_data->next_tb[TU_TB_INDEX_TARGET]) { TranslationBlock *tmp_tb = @@ -101,6 +127,256 @@ static void get_eflag_out(TranslationBlock *tb) } } +#ifdef CONFIG_LATX_FLAG_REDUCTION +static int tb_index(TranslationBlock **tb_list, int tb_num, + TranslationBlock *target) +{ + if (!target) { + return -1; + } + + for (int i = 0; i < tb_num; i++) { + if (tb_list[i] == target) { + return i; + } + } + return -1; +} + +static int tb_pc_index(TranslationBlock **tb_list, int tb_num, + TranslationBlock *from, target_ulong pc) +{ + for (int i = 0; i < tb_num; i++) { + TranslationBlock *tb = tb_list[i]; + + if (tb->pc == pc && + (tb->cflags & CF_PARALLEL) == (from->cflags & CF_PARALLEL) && + (tb->bool_flags & IS_CODE64) == (from->bool_flags & IS_CODE64)) { + return i; + } + } + return -1; +} + +static bool tb_is_near_call(TranslationBlock *tb) +{ + return ir1_opcode(tb_ir1_inst_last(tb)) == dt_X86_INS_CALL; +} + +static bool tb_is_near_ret(TranslationBlock *tb) +{ + return ir1_opcode(tb_ir1_inst_last(tb)) == dt_X86_INS_RET; +} + +static void unknown_eflags_summary(uint8 *use, uint8 *preserve) +{ + *use = __ALL_EFLAGS; + *preserve = __ALL_EFLAGS; +} + +/* + * Get the EFLAGS summary after a TB. RET is a symbolic function exit here: + * its continuation will be supplied at each direct CALL site later. This + * opt-in analysis assumes the normal ABI rule that a near RET uses the return + * slot created by its matching direct CALL. + */ +static void get_summary_after_tb(TranslationBlock **tb_list, int tb_num, + int tb_id, uint8 *summary_use, uint8 *summary_preserve, + uint8 *use, uint8 *preserve) +{ + TranslationBlock *tb = tb_list[tb_id]; + int next_id = tb_index(tb_list, tb_num, + tb->s_data->next_tb[TU_TB_INDEX_NEXT]); + int target_id = tb_index(tb_list, tb_num, + tb->s_data->next_tb[TU_TB_INDEX_TARGET]); + + switch (tb->s_data->last_ir1_type) { + case IR1_TYPE_BRANCH: + if (next_id < 0 || target_id < 0) { + unknown_eflags_summary(use, preserve); + } else { + *use = summary_use[next_id] | summary_use[target_id]; + *preserve = summary_preserve[next_id] | + summary_preserve[target_id]; + } + break; + case IR1_TYPE_JUMP: + if (target_id < 0) { + unknown_eflags_summary(use, preserve); + } else { + *use = summary_use[target_id]; + *preserve = summary_preserve[target_id]; + } + break; + case IR1_TYPE_CALL: { + if (!tb_is_near_call(tb)) { + unknown_eflags_summary(use, preserve); + break; + } + + int call_target_id = tb_pc_index(tb_list, tb_num, tb, + tb->s_data->target_pc); + if (call_target_id < 0 || next_id < 0) { + unknown_eflags_summary(use, preserve); + } else { + *use = summary_use[call_target_id] | + (summary_preserve[call_target_id] & summary_use[next_id]); + *preserve = summary_preserve[call_target_id] & + summary_preserve[next_id]; + } + break; + } + case IR1_TYPE_NORMAL: + if (next_id < 0) { + unknown_eflags_summary(use, preserve); + } else { + *use = summary_use[next_id]; + *preserve = summary_preserve[next_id]; + } + break; + case IR1_TYPE_RET: + if (tb_is_near_ret(tb)) { + *use = __NONE; + *preserve = __ALL_EFLAGS; + } else { + unknown_eflags_summary(use, preserve); + } + break; + case IR1_TYPE_CALLIN: + case IR1_TYPE_JUMPIN: + case IR1_TYPE_SYSCALL: + unknown_eflags_summary(use, preserve); + break; + default: + lsassert(0); + } +} + +/* + * Calculate which incoming flags a direct callee may read or preserve until + * RET. This is a second fixed point over the existing TU graph and does not + * change execution linking. + */ +static void get_tu_eflags_summary(TranslationBlock **tb_list, int tb_num, + uint8 *block_use, uint8 *block_must_def, + uint8 *summary_use, uint8 *summary_preserve) +{ + bool unfinished = true; + + memset(summary_use, 0, tb_num * sizeof(*summary_use)); + memset(summary_preserve, 0, tb_num * sizeof(*summary_preserve)); + + while (unfinished) { + unfinished = false; + for (int i = tb_num - 1; i >= 0; i--) { + uint8 succ_use = __NONE; + uint8 succ_preserve = __NONE; + uint8 block_preserve = __ALL_EFLAGS & ~block_must_def[i]; + + get_summary_after_tb(tb_list, tb_num, i, + summary_use, summary_preserve, + &succ_use, &succ_preserve); + + uint8 new_use = block_use[i] | (block_preserve & succ_use); + uint8 new_preserve = block_preserve & succ_preserve; + if ((new_use & ~summary_use[i]) || + (new_preserve & ~summary_preserve[i])) { + summary_use[i] |= new_use; + summary_preserve[i] |= new_preserve; + unfinished = true; + } + } + } +} + +static uint8 get_tu_live_out(TranslationBlock **tb_list, int tb_num, + int tb_id, uint8 *summary_use, uint8 *summary_preserve, + uint8 *live_in) +{ + TranslationBlock *tb = tb_list[tb_id]; + int next_id = tb_index(tb_list, tb_num, + tb->s_data->next_tb[TU_TB_INDEX_NEXT]); + int target_id = tb_index(tb_list, tb_num, + tb->s_data->next_tb[TU_TB_INDEX_TARGET]); + + switch (tb->s_data->last_ir1_type) { + case IR1_TYPE_BRANCH: + if (next_id < 0 || target_id < 0) { + return __ALL_EFLAGS; + } + return live_in[next_id] | live_in[target_id]; + case IR1_TYPE_JUMP: + return target_id < 0 ? __ALL_EFLAGS : live_in[target_id]; + case IR1_TYPE_CALL: { + if (!tb_is_near_call(tb)) { + return __ALL_EFLAGS; + } + int call_target_id = tb_pc_index(tb_list, tb_num, tb, + tb->s_data->target_pc); + if (call_target_id < 0 || next_id < 0) { + return __ALL_EFLAGS; + } + return summary_use[call_target_id] | + (summary_preserve[call_target_id] & live_in[next_id]); + } + case IR1_TYPE_NORMAL: + return next_id < 0 ? __ALL_EFLAGS : live_in[next_id]; + case IR1_TYPE_SYSCALL: + return __NONE; + case IR1_TYPE_CALLIN: + case IR1_TYPE_JUMPIN: + case IR1_TYPE_RET: + return __ALL_EFLAGS; + default: + lsassert(0); + return __ALL_EFLAGS; + } +} + +static void optimize_tu_calls(TranslationBlock **tb_list, int tb_num) +{ + uint8 block_use[tb_num]; + uint8 block_must_def[tb_num]; + uint8 summary_use[tb_num]; + uint8 summary_preserve[tb_num]; + uint8 live_in[tb_num]; + uint8 live_out[tb_num]; + + for (int i = 0; i < tb_num; i++) { + flag_reduction_get_tb_summary(tb_list[i], + &block_use[i], &block_must_def[i]); + } + + get_tu_eflags_summary(tb_list, tb_num, + block_use, block_must_def, summary_use, summary_preserve); + + memset(live_in, 0, sizeof(live_in)); + memset(live_out, 0, sizeof(live_out)); + + bool unfinished = true; + while (unfinished) { + unfinished = false; + for (int i = tb_num - 1; i >= 0; i--) { + uint8 new_out = get_tu_live_out(tb_list, tb_num, i, + summary_use, summary_preserve, live_in); + uint8 new_in = block_use[i] | + (new_out & ~block_must_def[i]); + + if ((new_out & ~live_out[i]) || (new_in & ~live_in[i])) { + live_out[i] |= new_out; + live_in[i] |= new_in; + unfinished = true; + } + } + } + + for (int i = 0; i < tb_num; i++) { + tb_list[i]->s_data->eflag_out = live_out[i]; + ir1_optimization_over_tb(tb_list[i], true); + } +} +#endif + void over_tb_rfd(TranslationBlock **tb_list, int tb_num) { TranslationBlock *tb; @@ -142,8 +418,14 @@ void over_tb_rfd(TranslationBlock **tb_list, int tb_num) for (int i = 0; i < tb_num; i++) { tb = tb_list[i]; /* fprintf(stderr, "pc %lx %x\n", tb->pc, (tb->s_data->eflag_out)); */ - ir1_optimization_over_tb(tb); + ir1_optimization_over_tb(tb, false); } + +#ifdef CONFIG_LATX_FLAG_REDUCTION + if (option_eflags_cross) { + optimize_tu_calls(tb_list, tb_num); + } +#endif } #endif @@ -155,7 +437,9 @@ void ir1_optimization(TranslationBlock *tb) IR1_INST *ir1 = NULL; /* cross scanning var defination */ DEF_FLAG_RDTN(rdtn); - DEF_INSTS_PTN(ptn); + + ir1_pattern_discovery(tb); + /* check if need cross tb analyze */ CHK_FLAG_RDTN(rdtn, tb); /* scanning instructions in reverse order */ @@ -163,8 +447,6 @@ void ir1_optimization(TranslationBlock *tb) ir1 = tb_ir1_inst(tb, i); /* do core optimize */ OPT_FLAG_RDTN(rdtn, ir1); - /* TODO: TU */ - OPT_INSTS_PTN(tb, ir1, i, ptn); } SAVE_FLAG_TO_TB(rdtn, tb); } diff --git a/target/i386/latx/optimization/ts.c b/target/i386/latx/optimization/ts.c index 08f9a1f589..0a1bf92744 100644 --- a/target/i386/latx/optimization/ts.c +++ b/target/i386/latx/optimization/ts.c @@ -645,6 +645,152 @@ static inline void get_ts_queue(CPUState *cpu, target_ulong cs_base, lsassert(*tb_num_in_tu <= MAX_TB_IN_TS); } +#ifdef CONFIG_LATX_FLAG_REDUCTION +#define MAX_EFLAGS_ANALYSIS_TBS 32 + +static bool eflags_analysis_allocator_has_space(void) +{ + uintptr_t align = qemu_icache_linesize; + uintptr_t tb_addr; + uintptr_t s_data_addr; + uintptr_t next; + + if (option_split_tb) { + tb_addr = ROUND_UP((uintptr_t)tcg_ctx->tb_gen_ptr, align); + s_data_addr = ROUND_UP(tb_addr + sizeof(TranslationBlock), align); + next = ROUND_UP(s_data_addr + sizeof(struct separated_data), align); + if (next > (uintptr_t)tcg_ctx->tb_gen_highwater) { + return false; + } + return ROUND_UP((uintptr_t)tcg_ctx->code_gen_ptr, align) <= + (uintptr_t)tcg_ctx->code_gen_highwater; + } + + tb_addr = ROUND_UP((uintptr_t)tcg_ctx->code_gen_ptr, align); + s_data_addr = ROUND_UP(tb_addr + sizeof(TranslationBlock), align); + next = ROUND_UP(s_data_addr + sizeof(struct separated_data), align); + return next <= (uintptr_t)tcg_ctx->code_gen_highwater; +} + +static bool eflags_analysis_has_space(uint32_t tb_limit, int max_insns) +{ + return tu_data->tb_num < tb_limit && + tu_data->ir1_num_in_tu + max_insns < MAX_IR1_IN_TU && + eflags_analysis_allocator_has_space(); +} + +/* + * Extend the final TU with same-page direct callees after HBR has finished. + * CALL itself keeps only its normal execution edge; the EFLAGS pass finds the + * callee in tb_list by target_pc. + */ +static void get_eflags_call_queue(CPUState *cpu, target_ulong cs_base, + uint32_t flags, int cflags, int max_insns) +{ + TranslationBlock **tb_list = tu_data->tb_list; + uint32_t *tb_num_in_tu = &tu_data->tb_num; + uint32_t original_tb_num = *tb_num_in_tu; + uint32_t tb_limit = original_tb_num + MAX_EFLAGS_ANALYSIS_TBS; + + if (tb_limit > MAX_TB_IN_CACHE) { + tb_limit = MAX_TB_IN_CACHE; + } + + for (uint32_t i = 0; i < original_tb_num; i++) { + TranslationBlock *tb = tb_list[i]; + if (!eflags_analysis_has_space(tb_limit, max_insns)) { + break; + } + if (tb->s_data->last_ir1_type != IR1_TYPE_CALL || + ir1_opcode(tb_ir1_inst_last(tb)) != dt_X86_INS_CALL || + get_tb_id(tb->s_data->target_pc, cflags) < 0) { + continue; + } + + get_target_tb(tb, cpu, cs_base, flags, cflags, max_insns); + tb->s_data->next_tb[TU_TB_INDEX_TARGET] = NULL; + tb->tu_jmp[TU_TB_INDEX_TARGET] = TB_JMP_RESET_OFFSET_INVALID; + } + + for (uint32_t i = original_tb_num; + i < *tb_num_in_tu && i < tb_limit; i++) { + TranslationBlock *tb = tb_list[i]; + + if (!eflags_analysis_has_space(tb_limit, max_insns)) { + return; + } + if (is_bad_tb(tb)) { + continue; + } + + switch (tb->s_data->last_ir1_type) { + case IR1_TYPE_BRANCH: + if (get_tb_id(tb->s_data->next_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_next_tb(tb, cpu, cs_base, flags, cflags, max_insns); + } + if (get_tb_id(tb->s_data->target_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_target_tb(tb, cpu, cs_base, flags, cflags, max_insns); + } + break; + case IR1_TYPE_CALL: + if (get_tb_id(tb->s_data->next_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_next_tb(tb, cpu, cs_base, flags, cflags, max_insns); + } + if (ir1_opcode(tb_ir1_inst_last(tb)) == dt_X86_INS_CALL && + get_tb_id(tb->s_data->target_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_target_tb(tb, cpu, cs_base, flags, cflags, max_insns); + tb->s_data->next_tb[TU_TB_INDEX_TARGET] = NULL; + tb->tu_jmp[TU_TB_INDEX_TARGET] = + TB_JMP_RESET_OFFSET_INVALID; + } + break; + case IR1_TYPE_CALLIN: + case IR1_TYPE_NORMAL: + case IR1_TYPE_SYSCALL: + if (get_tb_id(tb->s_data->next_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_next_tb(tb, cpu, cs_base, flags, cflags, max_insns); + } + break; + case IR1_TYPE_JUMP: + if (get_tb_id(tb->s_data->target_pc, cflags) >= 0 && + eflags_analysis_has_space(tb_limit, max_insns)) { + get_target_tb(tb, cpu, cs_base, flags, cflags, max_insns); + } + break; + case IR1_TYPE_JUMPIN: + case IR1_TYPE_RET: + break; + default: + lsassert(0); + } + } +} + +static void discard_eflags_call_queue(uint32_t original_tb_num, + uint32_t original_ir1_num) +{ + TranslationBlock **tb_list = tu_data->tb_list; + uint32_t *tb_num_in_tu = &tu_data->tb_num; + + for (uint32_t i = original_tb_num; i < *tb_num_in_tu; i++) { + TranslationBlock *tb = tb_list[i]; + int tb_id = get_tb_id(tb->pc, tb->cflags); + + if (tb_id >= 0 && curr_tb_message_vector[tb_id].tb == tb) { + curr_tb_message_vector[tb_id].tb = NULL; + } + tb_list[i] = NULL; + } + *tb_num_in_tu = original_tb_num; + tu_data->ir1_num_in_tu = original_ir1_num; +} +#endif + /* static int max_tb_num; */ static void delet_static_tb(TranslationBlock **tb_list, uint32_t *tb_num_in_tu) @@ -721,7 +867,29 @@ static void ts_tb_explore(CPUState *cpu, target_ulong cs_base, qsort(tb_list, *tb_num_in_tu, sizeof(TranslationBlock *), tb_sort_cmp); solve_tb_overlap(*tb_num_in_tu, tb_list, max_insns); - tu_ir1_optimization(tb_list, *tb_num_in_tu); + +#ifdef CONFIG_LATX_FLAG_REDUCTION + if (option_eflags_cross) { + uint32_t original_tb_num = *tb_num_in_tu; + uint32_t original_ir1_num = *ir1_num_in_tu; + void *original_code_gen_ptr = tcg_ctx->code_gen_ptr; + void *original_tb_gen_ptr = tcg_ctx->tb_gen_ptr; + void *original_data_gen_ptr = tcg_ctx->data_gen_ptr; + bool original_suppress_disasm = suppress_disasm_side_effects; + + suppress_disasm_side_effects = true; + get_eflags_call_queue(cpu, cs_base, flags, cflags, max_insns); + suppress_disasm_side_effects = original_suppress_disasm; + tu_ir1_optimization(tb_list, *tb_num_in_tu); + discard_eflags_call_queue(original_tb_num, original_ir1_num); + qatomic_set(&tcg_ctx->code_gen_ptr, original_code_gen_ptr); + qatomic_set(&tcg_ctx->tb_gen_ptr, original_tb_gen_ptr); + tcg_ctx->data_gen_ptr = original_data_gen_ptr; + } else +#endif + { + tu_ir1_optimization(tb_list, *tb_num_in_tu); + } delet_static_tb(tb_list, tb_num_in_tu); for (int i = 0; i < *tb_num_in_tu; i++) { @@ -842,4 +1010,3 @@ uint64 translate_lib(seg_info **seg_info_vector, int begin_id, in_pre_translate = 0; return tb_num_in_ts; } - diff --git a/target/i386/latx/optimization/tu.c b/target/i386/latx/optimization/tu.c index 35c749c9cc..7b8643905e 100644 --- a/target/i386/latx/optimization/tu.c +++ b/target/i386/latx/optimization/tu.c @@ -143,6 +143,7 @@ void tu_reset_tb(TranslationBlock *tb) tb->jmp_list_next[1] = (uintptr_t)NULL; /* init top in and top out */ + tb->s_data->eflag_out = 0; tb->s_data->_top_out = -1; tb->s_data->_top_in = -1; tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index 2bddc00471..d11ffaee93 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -1411,30 +1411,33 @@ static bool translate_ucomisd_seta(IR1_INST *pir1) IR1_OPND *opnd1 = ir1_get_opnd(curr, 1); IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src = load_freg128_from_ir1(opnd1); - /* 0. set flag = 0 */ - IR2_OPND flag_zf = ra_alloc_itemp(); - IR2_OPND flag_pf = ra_alloc_itemp(); IR2_OPND flag = ra_alloc_itemp(); - la_mov64(flag, zero_ir2_opnd); - /* 1. check ZF, are they equal & unordered? */ - la_fcmp_cond_d(fcc0_ir2_opnd, dest, src, FCMP_COND_CUEQ); - la_movcf2gr(flag_zf, fcc0_ir2_opnd); + if (ir1_need_calculate_any_flag(curr)) { + /* 0. set flag = 0 */ + IR2_OPND flag_zf = ra_alloc_itemp(); + IR2_OPND flag_pf = ra_alloc_itemp(); + la_mov64(flag, zero_ir2_opnd); - /* 2. check CF, are they less & unordered? */ - la_fcmp_cond_d(fcc2_ir2_opnd, dest, src, FCMP_COND_CULT); - la_movcf2gr(flag, fcc2_ir2_opnd); + /* 1. check ZF, are they equal & unordered? */ + la_fcmp_cond_d(fcc0_ir2_opnd, dest, src, FCMP_COND_CUEQ); + la_movcf2gr(flag_zf, fcc0_ir2_opnd); - /* 3. check PF, are they unordered? (= ZF & CF) */ - la_and(flag_pf, flag, flag_zf); + /* 2. check CF, are they less & unordered? */ + la_fcmp_cond_d(fcc2_ir2_opnd, dest, src, FCMP_COND_CULT); + la_movcf2gr(flag, fcc2_ir2_opnd); - la_bstrins_w(flag, flag_zf, ZF_BIT_INDEX, ZF_BIT_INDEX); - la_bstrins_w(flag, flag_pf, PF_BIT_INDEX, PF_BIT_INDEX); + /* 3. check PF, are they unordered? (= ZF & CF) */ + la_and(flag_pf, flag, flag_zf); - ra_free_temp(flag_pf); - ra_free_temp(flag_zf); - /* 4. mov flag to EFLAGS */ - la_x86mtflag(flag, 0x3f); + la_bstrins_w(flag, flag_zf, ZF_BIT_INDEX, ZF_BIT_INDEX); + la_bstrins_w(flag, flag_pf, PF_BIT_INDEX, PF_BIT_INDEX); + + ra_free_temp(flag_pf); + ra_free_temp(flag_zf); + /* 4. mov flag to EFLAGS */ + la_x86mtflag(flag, 0x3f); + } lsenv->tr_data->curr_ir1_inst = next; lsenv->tr_data->curr_ir1_count++; diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 0125e1c6ab..404a6db4ad 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -184,6 +184,8 @@ static char insn_info[MAX_IR1_NUM_PER_TB * IR1_INST_SIZE] = {0}; static IR1_INST ir1_list[MAX_IR1_NUM_PER_TB]; #endif +__thread bool suppress_disasm_side_effects; + IR1_INST *get_ir1_list(struct TranslationBlock *tb, ADDRX pc, int max_insns) { static uint8_t inst_cache[TCG_MAX_INSNS]; @@ -289,7 +291,8 @@ IR1_INST *get_ir1_list(struct TranslationBlock *tb, ADDRX pc, int max_insns) next_pir1->info = NULL; } #endif - if (pir1->info != NULL && ir1_num == 2 && ir1_is_return(pir1) && + if (!suppress_disasm_side_effects && pir1->info != NULL && + ir1_num == 2 && ir1_is_return(pir1) && ir1_opcode(&ir1_list[0]) == dt_X86_INS_MOV) { IR1_INST *insert_ir1 = &ir1_list[0]; IR1_OPND *opnd1 = ir1_get_opnd(insert_ir1, 1); diff --git a/tests/integration/eflags-cross-boundary.S b/tests/integration/eflags-cross-boundary.S new file mode 100644 index 0000000000..e962831dd4 --- /dev/null +++ b/tests/integration/eflags-cross-boundary.S @@ -0,0 +1,126 @@ +.equ __NR_exit, 60 + +.section .text +.global _start +.type _start, @function +_start: + xor %r12d, %r12d + mov $10000, %r13d +.Ltest_loop: + + /* CALL must preserve an input flag when a transparent callee returns. */ + clc + mov $-1, %eax + add $1, %eax + call direct_transparent + jc .Ldirect_transparent_ok + or $1, %r12d +.Ldirect_transparent_ok: + + /* The caller observes carry generated by the callee before RET. */ + clc + call direct_sets_cf + jc .Ldirect_ret_ok + or $2, %r12d +.Ldirect_ret_ok: + + /* A callee definition kills the incoming carry and supplies a new one. */ + stc + mov $-1, %eax + add $1, %eax + call direct_clears_cf + jnc .Ldirect_overwrite_ok + or $4, %r12d +.Ldirect_overwrite_ok: + + /* A callee may consume an incoming flag before defining any flags. */ + clc + mov $-1, %eax + add $1, %eax + call direct_reads_cf + + /* One transparent callee can return to continuations needing two flags. */ + clc + mov $-1, %eax + add $1, %eax + call shared_transparent + jc .Lshared_cf_ok + or $16, %r12d +.Lshared_cf_ok: + mov $1, %eax + sub $1, %eax + call shared_transparent + jz .Lshared_zf_ok + or $16, %r12d +.Lshared_zf_ok: + + /* A dead producer in a predecessor TB can be reduced through CALL/RET. */ + mov $1, %eax + cmp $0, %eax + jmp .Lpredecessor_dead_call +.Lpredecessor_dead_call: + call direct_transparent + xor %eax, %eax + + /* A live predecessor flag must still cross the transparent call. */ + clc + mov $-1, %eax + add $1, %eax + jmp .Lpredecessor_live_call +.Lpredecessor_live_call: + call direct_transparent + jc .Lpredecessor_live_ok + or $128, %r12d +.Lpredecessor_live_ok: + + /* An indirect JMP is not necessarily an ABI function boundary. */ + clc + mov $-1, %eax + add $1, %eax + lea .Lindirect_jmp_target(%rip), %rax + jmp *%rax +.Lindirect_jmp_target: + jc .Lindirect_jmp_ok + or $64, %r12d +.Lindirect_jmp_ok: + + dec %r13d + jne .Ltest_loop + + mov %r12d, %edi + mov $__NR_exit, %eax + syscall +.size _start, .-_start + +.type direct_transparent, @function +direct_transparent: + mov %rcx, %rcx + ret +.size direct_transparent, .-direct_transparent + +.type direct_sets_cf, @function +direct_sets_cf: + mov $-1, %eax + add $1, %eax + ret +.size direct_sets_cf, .-direct_sets_cf + +.type direct_clears_cf, @function +direct_clears_cf: + xor %eax, %eax + ret +.size direct_clears_cf, .-direct_clears_cf + +.type direct_reads_cf, @function +direct_reads_cf: + jc .Ldirect_read_ok + or $8, %r12d +.Ldirect_read_ok: + ret +.size direct_reads_cf, .-direct_reads_cf + +.type shared_transparent, @function +shared_transparent: + mov %rdx, %rdx + ret +.size shared_transparent, .-shared_transparent diff --git a/tests/integration/eflags-pattern-consumer.S b/tests/integration/eflags-pattern-consumer.S new file mode 100644 index 0000000000..ebd30db943 --- /dev/null +++ b/tests/integration/eflags-pattern-consumer.S @@ -0,0 +1,897 @@ +.equ __NR_exit, 60 + +.section .text + +/* CMP+SETcc is the only consumer, so CMP flags can be eliminated. */ +.global cmp_setcc_elide +.type cmp_setcc_elide, @function +cmp_setcc_elide: + mov $1, %eax + cmp $1, %eax + sete %dil + xor $1, %dil + mov $__NR_exit, %eax + syscall +.size cmp_setcc_elide, .-cmp_setcc_elide + +/* TEST+CMOVcc is the only consumer, so TEST flags can be eliminated. */ +.global test_cmov_elide +.type test_cmov_elide, @function +test_cmov_elide: + mov $0, %rax + mov $1, %rdi + mov $0, %rsi + test %rax, %rax + cmove %rdi, %rsi + lea -1(%rsi), %rdi + mov $__NR_exit, %eax + syscall +.size test_cmov_elide, .-test_cmov_elide + +/* ADC still consumes CF after the native SETcc condition. */ +.global cmp_setcc_keep_cf +.type cmp_setcc_keep_cf, @function +cmp_setcc_keep_cf: + clc + mov $0, %eax + cmp $1, %eax + setb %cl + mov $0, %edx + adc $0, %edx + movzx %cl, %edi + xor %edx, %edi + mov $__NR_exit, %eax + syscall +.size cmp_setcc_keep_cf, .-cmp_setcc_keep_cf + +/* LAHF still consumes TEST flags after the native CMOVcc condition. */ +.global test_cmov_keep_flags +.type test_cmov_keep_flags, @function +test_cmov_keep_flags: + mov $1, %eax + cmp $0, %eax + mov $0, %rbx + mov $1, %rdi + mov $0, %rsi + test %rbx, %rbx + cmove %rdi, %rsi + lahf + movzx %ah, %edi + and $0x41, %edi + xor $0x40, %edi + lea -1(%rsi), %edx + or %edx, %edi + mov $__NR_exit, %eax + syscall +.size test_cmov_keep_flags, .-test_cmov_keep_flags + +/* CMP+SBB consumes native carry; neither CMP nor SBB flags escape. */ +.global cmp_sbb_elide +.type cmp_sbb_elide, @function +cmp_sbb_elide: + mov $0, %edi + mov $0, %eax + cmp $1, %eax + sbb %edi, %edi + lea 1(%rdi), %rdi + mov $__NR_exit, %eax + syscall +.size cmp_sbb_elide, .-cmp_sbb_elide + +/* SETE still consumes ZF produced by the transformed SBB. */ +.global cmp_sbb_keep_output_zf +.type cmp_sbb_keep_output_zf, @function +cmp_sbb_keep_output_zf: + mov $0, %edi + mov $2, %eax + cmp $1, %eax + sbb %edi, %edi + sete %cl + movzx %cl, %edi + xor $1, %edi + mov $__NR_exit, %eax + syscall +.size cmp_sbb_keep_output_zf, .-cmp_sbb_keep_output_zf + +/* CMP+JE is native and neither successor consumes its flags. */ +.global cmp_je_elide +.type cmp_je_elide, @function +cmp_je_elide: + mov $1, %eax + cmp $1, %eax + je .Lcmp_je_taken + xor %edi, %edi + jmp .Lcmp_je_exit +.Lcmp_je_taken: + xor %edi, %edi +.Lcmp_je_exit: + mov $__NR_exit, %eax + syscall +.size cmp_je_elide, .-cmp_je_elide + +/* CMP+JNE is native and neither successor consumes its flags. */ +.global cmp_jne_elide +.type cmp_jne_elide, @function +cmp_jne_elide: + mov $1, %eax + cmp $2, %eax + jne .Lcmp_jne_taken + xor %edi, %edi + jmp .Lcmp_jne_exit +.Lcmp_jne_taken: + xor %edi, %edi +.Lcmp_jne_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jne_elide, .-cmp_jne_elide + +/* CMP+JB is native and neither successor consumes its flags. */ +.global cmp_jb_elide +.type cmp_jb_elide, @function +cmp_jb_elide: + mov $0, %eax + cmp $1, %eax + jb .Lcmp_jb_taken + xor %edi, %edi + jmp .Lcmp_jb_exit +.Lcmp_jb_taken: + xor %edi, %edi +.Lcmp_jb_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jb_elide, .-cmp_jb_elide + +/* CMP+JAE is native and neither successor consumes its flags. */ +.global cmp_jae_elide +.type cmp_jae_elide, @function +cmp_jae_elide: + mov $1, %eax + cmp $1, %eax + jae .Lcmp_jae_taken + xor %edi, %edi + jmp .Lcmp_jae_exit +.Lcmp_jae_taken: + xor %edi, %edi +.Lcmp_jae_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jae_elide, .-cmp_jae_elide + +/* TEST+JE is native and neither successor consumes its flags. */ +.global test_je_elide +.type test_je_elide, @function +test_je_elide: + xor %eax, %eax + test %eax, %eax + je .Ltest_je_taken + xor %edi, %edi + jmp .Ltest_je_exit +.Ltest_je_taken: + xor %edi, %edi +.Ltest_je_exit: + mov $__NR_exit, %eax + syscall +.size test_je_elide, .-test_je_elide + +/* TEST+JNE is native and neither successor consumes its flags. */ +.global test_jne_elide +.type test_jne_elide, @function +test_jne_elide: + mov $1, %eax + test %eax, %eax + jne .Ltest_jne_taken + xor %edi, %edi + jmp .Ltest_jne_exit +.Ltest_jne_taken: + xor %edi, %edi +.Ltest_jne_exit: + mov $__NR_exit, %eax + syscall +.size test_jne_elide, .-test_jne_elide + +/* The taken successor still consumes ZF produced by CMP. */ +.global cmp_je_keep_zf +.type cmp_je_keep_zf, @function +cmp_je_keep_zf: + mov $1, %eax + cmp $1, %eax + je .Lcmp_je_keep_taken + mov $1, %edi + jmp .Lcmp_je_keep_exit +.Lcmp_je_keep_taken: + lahf + movzx %ah, %edi + and $0x40, %edi + xor $0x40, %edi +.Lcmp_je_keep_exit: + mov $__NR_exit, %eax + syscall +.size cmp_je_keep_zf, .-cmp_je_keep_zf + +/* The taken successor still consumes ZF produced by TEST. */ +.global test_jne_keep_zf +.type test_jne_keep_zf, @function +test_jne_keep_zf: + mov $1, %eax + test %eax, %eax + jne .Ltest_jne_keep_taken + mov $1, %edi + jmp .Ltest_jne_keep_exit +.Ltest_jne_keep_taken: + lahf + movzx %ah, %edi + and $0x40, %edi +.Ltest_jne_keep_exit: + mov $__NR_exit, %eax + syscall +.size test_jne_keep_zf, .-test_jne_keep_zf + +/* CMP+JBE is native and neither successor consumes its flags. */ +.global cmp_jbe_elide +.type cmp_jbe_elide, @function +cmp_jbe_elide: + mov $1, %eax + cmp $1, %eax + jbe .Lcmp_jbe_taken + xor %edi, %edi + jmp .Lcmp_jbe_exit +.Lcmp_jbe_taken: + xor %edi, %edi +.Lcmp_jbe_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jbe_elide, .-cmp_jbe_elide + +/* CMP+JA is native and neither successor consumes its flags. */ +.global cmp_ja_elide +.type cmp_ja_elide, @function +cmp_ja_elide: + mov $2, %eax + cmp $1, %eax + ja .Lcmp_ja_taken + xor %edi, %edi + jmp .Lcmp_ja_exit +.Lcmp_ja_taken: + xor %edi, %edi +.Lcmp_ja_exit: + mov $__NR_exit, %eax + syscall +.size cmp_ja_elide, .-cmp_ja_elide + +/* CMP+JL is native and neither successor consumes its flags. */ +.global cmp_jl_elide +.type cmp_jl_elide, @function +cmp_jl_elide: + mov $-1, %eax + cmp $0, %eax + jl .Lcmp_jl_taken + xor %edi, %edi + jmp .Lcmp_jl_exit +.Lcmp_jl_taken: + xor %edi, %edi +.Lcmp_jl_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jl_elide, .-cmp_jl_elide + +/* CMP+JGE is native and neither successor consumes its flags. */ +.global cmp_jge_elide +.type cmp_jge_elide, @function +cmp_jge_elide: + xor %eax, %eax + cmp $0, %eax + jge .Lcmp_jge_taken + xor %edi, %edi + jmp .Lcmp_jge_exit +.Lcmp_jge_taken: + xor %edi, %edi +.Lcmp_jge_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jge_elide, .-cmp_jge_elide + +/* CMP+JLE is native and neither successor consumes its flags. */ +.global cmp_jle_elide +.type cmp_jle_elide, @function +cmp_jle_elide: + xor %eax, %eax + cmp $0, %eax + jle .Lcmp_jle_taken + xor %edi, %edi + jmp .Lcmp_jle_exit +.Lcmp_jle_taken: + xor %edi, %edi +.Lcmp_jle_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jle_elide, .-cmp_jle_elide + +/* CMP+JG is native and neither successor consumes its flags. */ +.global cmp_jg_elide +.type cmp_jg_elide, @function +cmp_jg_elide: + mov $1, %eax + cmp $0, %eax + jg .Lcmp_jg_taken + xor %edi, %edi + jmp .Lcmp_jg_exit +.Lcmp_jg_taken: + xor %edi, %edi +.Lcmp_jg_exit: + mov $__NR_exit, %eax + syscall +.size cmp_jg_elide, .-cmp_jg_elide + +/* TEST+JS is native and neither successor consumes its flags. */ +.global test_js_elide +.type test_js_elide, @function +test_js_elide: + mov $-1, %eax + test %eax, %eax + js .Ltest_js_taken + xor %edi, %edi + jmp .Ltest_js_exit +.Ltest_js_taken: + xor %edi, %edi +.Ltest_js_exit: + mov $__NR_exit, %eax + syscall +.size test_js_elide, .-test_js_elide + +/* TEST+JNS is native and neither successor consumes its flags. */ +.global test_jns_elide +.type test_jns_elide, @function +test_jns_elide: + mov $1, %eax + test %eax, %eax + jns .Ltest_jns_taken + xor %edi, %edi + jmp .Ltest_jns_exit +.Ltest_jns_taken: + xor %edi, %edi +.Ltest_jns_exit: + mov $__NR_exit, %eax + syscall +.size test_jns_elide, .-test_jns_elide + +/* TEST+JLE is native and neither successor consumes its flags. */ +.global test_jle_elide +.type test_jle_elide, @function +test_jle_elide: + xor %eax, %eax + test %eax, %eax + jle .Ltest_jle_taken + xor %edi, %edi + jmp .Ltest_jle_exit +.Ltest_jle_taken: + xor %edi, %edi +.Ltest_jle_exit: + mov $__NR_exit, %eax + syscall +.size test_jle_elide, .-test_jle_elide + +/* TEST+JG is native and neither successor consumes its flags. */ +.global test_jg_elide +.type test_jg_elide, @function +test_jg_elide: + mov $1, %eax + test %eax, %eax + jg .Ltest_jg_taken + xor %edi, %edi + jmp .Ltest_jg_exit +.Ltest_jg_taken: + xor %edi, %edi +.Ltest_jg_exit: + mov $__NR_exit, %eax + syscall +.size test_jg_elide, .-test_jg_elide + +/* TEST clears OF, so JNO is always taken without reading EFLAGS. */ +.global test_jno_elide +.type test_jno_elide, @function +test_jno_elide: + mov $1, %eax + test %eax, %eax + jno .Ltest_jno_taken + mov $1, %edi + jmp .Ltest_jno_exit +.Ltest_jno_taken: + xor %edi, %edi +.Ltest_jno_exit: + mov $__NR_exit, %eax + syscall +.size test_jno_elide, .-test_jno_elide + +/* TEST clears OF, so JO is never taken without reading EFLAGS. */ +.global test_jo_elide +.type test_jo_elide, @function +test_jo_elide: + mov $1, %eax + test %eax, %eax + jo .Ltest_jo_bad + xor %edi, %edi + jmp .Ltest_jo_exit +.Ltest_jo_bad: + mov $1, %edi +.Ltest_jo_exit: + mov $__NR_exit, %eax + syscall +.size test_jo_elide, .-test_jo_elide + +/* TEST clears CF, so JB is never taken without reading EFLAGS. */ +.global test_jb_elide +.type test_jb_elide, @function +test_jb_elide: + mov $1, %eax + test %eax, %eax + jb .Ltest_jb_bad + xor %edi, %edi + jmp .Ltest_jb_exit +.Ltest_jb_bad: + mov $1, %edi +.Ltest_jb_exit: + mov $__NR_exit, %eax + syscall +.size test_jb_elide, .-test_jb_elide + +/* TEST clears CF, so JBE is equivalent to JE. */ +.global test_jbe_elide +.type test_jbe_elide, @function +test_jbe_elide: + xor %eax, %eax + test %eax, %eax + jbe .Ltest_jbe_taken + mov $1, %edi + jmp .Ltest_jbe_exit +.Ltest_jbe_taken: + xor %edi, %edi +.Ltest_jbe_exit: + mov $__NR_exit, %eax + syscall +.size test_jbe_elide, .-test_jbe_elide + +/* TEST clears CF, so JA is equivalent to JNE. */ +.global test_ja_elide +.type test_ja_elide, @function +test_ja_elide: + mov $1, %eax + test %eax, %eax + ja .Ltest_ja_taken + mov $1, %edi + jmp .Ltest_ja_exit +.Ltest_ja_taken: + xor %edi, %edi +.Ltest_ja_exit: + mov $__NR_exit, %eax + syscall +.size test_ja_elide, .-test_ja_elide + +/* TEST clears CF, so JAE is always taken without reading EFLAGS. */ +.global test_jae_elide +.type test_jae_elide, @function +test_jae_elide: + mov $1, %eax + test %eax, %eax + jae .Ltest_jae_taken + mov $1, %edi + jmp .Ltest_jae_exit +.Ltest_jae_taken: + xor %edi, %edi +.Ltest_jae_exit: + mov $__NR_exit, %eax + syscall +.size test_jae_elide, .-test_jae_elide + +/* BT+JB reads the tested bit directly; its CF does not escape. */ +.global bt_jb_elide +.type bt_jb_elide, @function +bt_jb_elide: + mov $1, %eax + bt $0, %eax + jb .Lbt_jb_taken + mov $1, %edi + jmp .Lbt_jb_exit +.Lbt_jb_taken: + xor %edi, %edi +.Lbt_jb_exit: + mov $__NR_exit, %eax + syscall +.size bt_jb_elide, .-bt_jb_elide + +/* BT+JAE reads the tested bit directly; its CF does not escape. */ +.global bt_jae_elide +.type bt_jae_elide, @function +bt_jae_elide: + xor %eax, %eax + bt $0, %eax + jae .Lbt_jae_taken + mov $1, %edi + jmp .Lbt_jae_exit +.Lbt_jae_taken: + xor %edi, %edi +.Lbt_jae_exit: + mov $__NR_exit, %eax + syscall +.size bt_jae_elide, .-bt_jae_elide + +/* SUB+JNE compares the original operands directly; its flags do not escape. */ +.global sub_jne_elide +.type sub_jne_elide, @function +sub_jne_elide: + mov $2, %eax + sub $1, %eax + jne .Lsub_jne_taken + mov $1, %edi + jmp .Lsub_jne_exit +.Lsub_jne_taken: + xor %edi, %edi +.Lsub_jne_exit: + mov $__NR_exit, %eax + syscall +.size sub_jne_elide, .-sub_jne_elide + +/* SUB+JL compares signed operands directly; its flags do not escape. */ +.global sub_jl_elide +.type sub_jl_elide, @function +sub_jl_elide: + xor %eax, %eax + sub $1, %eax + jl .Lsub_jl_taken + mov $1, %edi + jmp .Lsub_jl_exit +.Lsub_jl_taken: + xor %edi, %edi +.Lsub_jl_exit: + mov $__NR_exit, %eax + syscall +.size sub_jl_elide, .-sub_jl_elide + +/* A locked SUB falls back to the ordinary EFLAGS path. */ +.global lock_sub_jne_keep_flags +.type lock_sub_jne_keep_flags, @function +lock_sub_jne_keep_flags: + lock subl $1, lock_sub_value(%rip) + jne .Llock_sub_taken + mov $1, %edi + jmp .Llock_sub_exit +.Llock_sub_taken: + xor %edi, %edi +.Llock_sub_exit: + mov $__NR_exit, %eax + syscall +.size lock_sub_jne_keep_flags, .-lock_sub_jne_keep_flags + +/* A nonzero SHR count supplies JNE natively; its flags do not escape. */ +.global shr_jne_elide +.type shr_jne_elide, @function +shr_jne_elide: + mov $2, %eax + shr $1, %eax + jne .Lshr_jne_taken + mov $1, %edi + jmp .Lshr_jne_exit +.Lshr_jne_taken: + xor %edi, %edi +.Lshr_jne_exit: + mov $__NR_exit, %eax + syscall +.size shr_jne_elide, .-shr_jne_elide + +/* A masked-zero SHR count preserves the flags produced by CMP. */ +.global shr_zero_jne_keep_flags +.type shr_zero_jne_keep_flags, @function +shr_zero_jne_keep_flags: + mov $1, %eax + cmp $0, %eax + shr $32, %eax + jne .Lshr_zero_taken + mov $1, %edi + jmp .Lshr_zero_exit +.Lshr_zero_taken: + xor %edi, %edi +.Lshr_zero_exit: + mov $__NR_exit, %eax + syscall +.size shr_zero_jne_keep_flags, .-shr_zero_jne_keep_flags + +/* AND+JNE tests the result directly; its flags do not escape. */ +.global and_jne_elide +.type and_jne_elide, @function +and_jne_elide: + mov $3, %eax + and $1, %eax + jne .Land_jne_taken + mov $1, %edi + jmp .Land_jne_exit +.Land_jne_taken: + xor %edi, %edi +.Land_jne_exit: + mov $__NR_exit, %eax + syscall +.size and_jne_elide, .-and_jne_elide + +/* A locked AND falls back to the ordinary EFLAGS path. */ +.global lock_and_jne_keep_flags +.type lock_and_jne_keep_flags, @function +lock_and_jne_keep_flags: + lock andl $1, lock_and_value(%rip) + jne .Llock_and_taken + mov $1, %edi + jmp .Llock_and_exit +.Llock_and_taken: + xor %edi, %edi +.Llock_and_exit: + mov $__NR_exit, %eax + syscall +.size lock_and_jne_keep_flags, .-lock_and_jne_keep_flags + +/* COMISD+JE uses the native floating-point condition. */ +.global comisd_je_elide +.type comisd_je_elide, @function +comisd_je_elide: + movq fp_one_double(%rip), %xmm0 + movapd %xmm0, %xmm1 + comisd %xmm1, %xmm0 + je .Lcomisd_je_taken + mov $1, %edi + jmp .Lcomisd_je_exit +.Lcomisd_je_taken: + xor %edi, %edi +.Lcomisd_je_exit: + mov $__NR_exit, %eax + syscall +.size comisd_je_elide, .-comisd_je_elide + +/* COMISS+JNE uses the native floating-point condition. */ +.global comiss_jne_elide +.type comiss_jne_elide, @function +comiss_jne_elide: + movd fp_one_single(%rip), %xmm0 + movd fp_two_single(%rip), %xmm1 + comiss %xmm1, %xmm0 + jne .Lcomiss_jne_taken + mov $1, %edi + jmp .Lcomiss_jne_exit +.Lcomiss_jne_taken: + xor %edi, %edi +.Lcomiss_jne_exit: + mov $__NR_exit, %eax + syscall +.size comiss_jne_elide, .-comiss_jne_elide + +/* UCOMISD+JB handles unordered input through the native condition. */ +.global ucomisd_jb_nan_elide +.type ucomisd_jb_nan_elide, @function +ucomisd_jb_nan_elide: + movq fp_qnan_double(%rip), %xmm0 + movq fp_one_double(%rip), %xmm1 + ucomisd %xmm1, %xmm0 + jb .Lucomisd_jb_taken + mov $1, %edi + jmp .Lucomisd_jb_exit +.Lucomisd_jb_taken: + xor %edi, %edi +.Lucomisd_jb_exit: + mov $__NR_exit, %eax + syscall +.size ucomisd_jb_nan_elide, .-ucomisd_jb_nan_elide + +/* UCOMISS+JNE uses the native floating-point condition. */ +.global ucomiss_jne_elide +.type ucomiss_jne_elide, @function +ucomiss_jne_elide: + movd fp_one_single(%rip), %xmm0 + movd fp_two_single(%rip), %xmm1 + ucomiss %xmm1, %xmm0 + jne .Lucomiss_jne_taken + mov $1, %edi + jmp .Lucomiss_jne_exit +.Lucomiss_jne_taken: + xor %edi, %edi +.Lucomiss_jne_exit: + mov $__NR_exit, %eax + syscall +.size ucomiss_jne_elide, .-ucomiss_jne_elide + +/* A successor LAHF still requires the flags produced by UCOMISD. */ +.global ucomisd_je_keep_zf +.type ucomisd_je_keep_zf, @function +ucomisd_je_keep_zf: + movq fp_one_double(%rip), %xmm0 + movapd %xmm0, %xmm1 + ucomisd %xmm1, %xmm0 + je .Lucomisd_keep_taken + mov $1, %edi + jmp .Lucomisd_keep_exit +.Lucomisd_keep_taken: + lahf + movzx %ah, %edi + and $0x40, %edi + xor $0x40, %edi +.Lucomisd_keep_exit: + mov $__NR_exit, %eax + syscall +.size ucomisd_je_keep_zf, .-ucomisd_je_keep_zf + +/* A MOV may separate CMP from its native JNE consumer. */ +.global cmp_xx_jne_elide +.type cmp_xx_jne_elide, @function +cmp_xx_jne_elide: + mov $2, %eax + cmp $1, %eax + mov $1, %eax + jne .Lcmp_xx_jne_taken + mov $1, %edi + jmp .Lcmp_xx_jne_exit +.Lcmp_xx_jne_taken: + xor %edi, %edi +.Lcmp_xx_jne_exit: + mov $__NR_exit, %eax + syscall +.size cmp_xx_jne_elide, .-cmp_xx_jne_elide + +/* A MOV may separate TEST from its native JE consumer. */ +.global test_xx_je_elide +.type test_xx_je_elide, @function +test_xx_je_elide: + xor %eax, %eax + test %eax, %eax + mov $1, %eax + je .Ltest_xx_je_taken + mov $1, %edi + jmp .Ltest_xx_je_exit +.Ltest_xx_je_taken: + xor %edi, %edi +.Ltest_xx_je_exit: + mov $__NR_exit, %eax + syscall +.size test_xx_je_elide, .-test_xx_je_elide + +/* A MOV may separate BT from its native JB consumer. */ +.global bt_xx_jb_elide +.type bt_xx_jb_elide, @function +bt_xx_jb_elide: + mov $1, %eax + bt $0, %eax + mov $0, %eax + jb .Lbt_xx_jb_taken + mov $1, %edi + jmp .Lbt_xx_jb_exit +.Lbt_xx_jb_taken: + xor %edi, %edi +.Lbt_xx_jb_exit: + mov $__NR_exit, %eax + syscall +.size bt_xx_jb_elide, .-bt_xx_jb_elide + +/* MOVAPD may separate UCOMISD from its native JE consumer. */ +.global ucomisd_xx_je_elide +.type ucomisd_xx_je_elide, @function +ucomisd_xx_je_elide: + movq fp_one_double(%rip), %xmm0 + movapd %xmm0, %xmm1 + ucomisd %xmm1, %xmm0 + movq fp_qnan_double(%rip), %xmm0 + je .Lucomisd_xx_je_taken + mov $1, %edi + jmp .Lucomisd_xx_je_exit +.Lucomisd_xx_je_taken: + xor %edi, %edi +.Lucomisd_xx_je_exit: + mov $__NR_exit, %eax + syscall +.size ucomisd_xx_je_elide, .-ucomisd_xx_je_elide + +/* A non-adjacent successor LAHF still requires CMP flags. */ +.global cmp_xx_je_keep_zf +.type cmp_xx_je_keep_zf, @function +cmp_xx_je_keep_zf: + mov $1, %eax + cmp $1, %eax + mov $2, %eax + je .Lcmp_xx_keep_taken + mov $1, %edi + jmp .Lcmp_xx_keep_exit +.Lcmp_xx_keep_taken: + lahf + movzx %ah, %edi + and $0x40, %edi + xor $0x40, %edi +.Lcmp_xx_keep_exit: + mov $__NR_exit, %eax + syscall +.size cmp_xx_je_keep_zf, .-cmp_xx_je_keep_zf + +/* NEG+CMOVS uses the result sign directly; NEG flags do not escape. */ +.global neg_cmovs_elide +.type neg_cmovs_elide, @function +neg_cmovs_elide: + mov $1, %eax + mov $7, %ebx + xor %ecx, %ecx + neg %eax + cmovs %ebx, %ecx + sub $7, %ecx + mov %ecx, %edi + mov $__NR_exit, %eax + syscall +.size neg_cmovs_elide, .-neg_cmovs_elide + +/* A locked NEG falls back to the ordinary EFLAGS CMOV path. */ +.global lock_neg_cmovs_keep_flags +.type lock_neg_cmovs_keep_flags, @function +lock_neg_cmovs_keep_flags: + mov $7, %ebx + xor %ecx, %ecx + lock negl lock_neg_value(%rip) + cmovs %ebx, %ecx + sub $7, %ecx + mov %ecx, %edi + mov $__NR_exit, %eax + syscall +.size lock_neg_cmovs_keep_flags, .-lock_neg_cmovs_keep_flags + +/* A later LAHF still requires the flags produced by NEG. */ +.global neg_cmovs_keep_sf +.type neg_cmovs_keep_sf, @function +neg_cmovs_keep_sf: + mov $1, %eax + mov $7, %ebx + xor %ecx, %ecx + neg %eax + cmovs %ebx, %ecx + lahf + movzx %ah, %edi + and $0x80, %edi + xor $0x80, %edi + mov $__NR_exit, %eax + syscall +.size neg_cmovs_keep_sf, .-neg_cmovs_keep_sf + +/* UCOMISD+SETA uses a native floating-point comparison. */ +.global ucomisd_seta_elide +.type ucomisd_seta_elide, @function +ucomisd_seta_elide: + movq fp_one_double(%rip), %xmm0 + movq fp_zero_double(%rip), %xmm1 + ucomisd %xmm1, %xmm0 + seta %dil + xor $1, %dil + mov $__NR_exit, %eax + syscall +.size ucomisd_seta_elide, .-ucomisd_seta_elide + +/* A later LAHF still requires the flags produced by UCOMISD+SETA. */ +.global ucomisd_seta_keep_zf +.type ucomisd_seta_keep_zf, @function +ucomisd_seta_keep_zf: + movq fp_one_double(%rip), %xmm0 + movapd %xmm0, %xmm1 + ucomisd %xmm1, %xmm0 + seta %dil + lahf + movzx %ah, %edi + and $0x40, %edi + xor $0x40, %edi + mov $__NR_exit, %eax + syscall +.size ucomisd_seta_keep_zf, .-ucomisd_seta_keep_zf + +.section .data +.align 8 +lock_sub_value: + .long 2 +lock_and_value: + .long 3 +lock_neg_value: + .long 1 +fp_one_double: + .quad 0x3ff0000000000000 +fp_zero_double: + .quad 0x0000000000000000 +fp_qnan_double: + .quad 0x7ff8000000000000 +fp_one_single: + .long 0x3f800000 +fp_two_single: + .long 0x40000000 diff --git a/tests/integration/meson.build b/tests/integration/meson.build index be47597268..4ade9ec93a 100644 --- a/tests/integration/meson.build +++ b/tests/integration/meson.build @@ -1,6 +1,7 @@ latx_integration_tests = [] if host_machine.cpu_family() == 'loongarch64' + subdir('registrations/eflags') subdir('registrations/x11-kzt') subdir('registrations/sandbox') subdir('registrations/process') diff --git a/tests/integration/registrations/eflags/meson.build b/tests/integration/registrations/eflags/meson.build new file mode 100644 index 0000000000..b22ceaec45 --- /dev/null +++ b/tests/integration/registrations/eflags/meson.build @@ -0,0 +1,30 @@ +if 'x86_64-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-eflags-pattern-consumer', + 'runner': find_program('../../test-eflags-pattern-consumer.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../eflags-pattern-consumer.S'), + ], + }, { + 'name': 'test-eflags-cross-boundary-off', + 'runner': find_program('../../test-eflags-cross-boundary.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../eflags-cross-boundary.S'), + '0', + '0', + '1', + ], + }, { + 'name': 'test-eflags-cross-boundary-on', + 'runner': find_program('../../test-eflags-cross-boundary.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../eflags-cross-boundary.S'), + '1', + '0', + '1', + ], + }] +endif diff --git a/tests/integration/test-eflags-cross-boundary.sh b/tests/integration/test-eflags-cross-boundary.sh new file mode 100755 index 0000000000..d3ecd51557 --- /dev/null +++ b/tests/integration/test-eflags-cross-boundary.sh @@ -0,0 +1,67 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +mask=${3:-0} +expected=${4:-0} +aot_mode=${5:-0} +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM +test_home="$workdir/home" +mkdir -p "$test_home" + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + "$source_file" -o "$workdir/eflags-cross-boundary" + +run_guest() +{ + set +e + ( + cd "$workdir" + HOME="$test_home" LATX_AOT="$aot_mode" LATX_KZT=0 \ + LATX_EFLAGS_CROSS="$mask" \ + "$emulator" ./eflags-cross-boundary + ) + ret=$? + set -e +} + +run_guest + +if [ "$aot_mode" -ne 0 ] && [ "$ret" -eq "$expected" ]; then + cache_file="" + count=0 + while [ "$count" -lt 100 ]; do + cache_file=$(find "$test_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + if [ -n "$cache_file" ]; then + break + fi + count=$((count + 1)) + sleep 0.1 + done + if [ -z "$cache_file" ]; then + echo "FAIL: AOT cache was not generated" >&2 + exit 1 + fi + + # The first run profiles and generates the TU; the second executes it. + run_guest +fi + +if [ "$ret" -ne "$expected" ]; then + echo "FAIL: mask=$mask aot=$aot_mode expected=$expected actual=$ret" >&2 + exit 1 +fi + +echo "PASS: mask=$mask aot=$aot_mode result=$ret" diff --git a/tests/integration/test-eflags-pattern-consumer.sh b/tests/integration/test-eflags-pattern-consumer.sh new file mode 100755 index 0000000000..b4666993cf --- /dev/null +++ b/tests/integration/test-eflags-pattern-consumer.sh @@ -0,0 +1,92 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +run_case() +{ + entry=$1 + guest="$workdir/$entry" + + "$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none -Wl,-e,"$entry" "$source_file" -o "$guest" + + set +e + HOME="$workdir/home-$entry" LATX_AOT=0 LATX_KZT=0 \ + "$emulator" "$guest" + ret=$? + set -e + if [ "$ret" -ne 0 ]; then + echo "FAIL: $entry expected=0 actual=$ret" >&2 + exit 1 + fi +} + +run_case cmp_setcc_elide +run_case test_cmov_elide +run_case cmp_setcc_keep_cf +run_case test_cmov_keep_flags +run_case cmp_sbb_elide +run_case cmp_sbb_keep_output_zf +run_case cmp_je_elide +run_case cmp_jne_elide +run_case cmp_jb_elide +run_case cmp_jae_elide +run_case test_je_elide +run_case test_jne_elide +run_case cmp_je_keep_zf +run_case test_jne_keep_zf +run_case cmp_jbe_elide +run_case cmp_ja_elide +run_case cmp_jl_elide +run_case cmp_jge_elide +run_case cmp_jle_elide +run_case cmp_jg_elide +run_case test_js_elide +run_case test_jns_elide +run_case test_jle_elide +run_case test_jg_elide +run_case test_jno_elide +run_case test_jo_elide +run_case test_jb_elide +run_case test_jbe_elide +run_case test_ja_elide +run_case test_jae_elide +run_case bt_jb_elide +run_case bt_jae_elide +run_case sub_jne_elide +run_case sub_jl_elide +run_case lock_sub_jne_keep_flags +run_case shr_jne_elide +run_case shr_zero_jne_keep_flags +run_case and_jne_elide +run_case lock_and_jne_keep_flags +run_case comisd_je_elide +run_case comiss_jne_elide +run_case ucomisd_jb_nan_elide +run_case ucomiss_jne_elide +run_case ucomisd_je_keep_zf +run_case cmp_xx_jne_elide +run_case test_xx_je_elide +run_case bt_xx_jb_elide +run_case ucomisd_xx_je_elide +run_case cmp_xx_je_keep_zf +run_case neg_cmovs_elide +run_case lock_neg_cmovs_keep_flags +run_case neg_cmovs_keep_sf +run_case ucomisd_seta_elide +run_case ucomisd_seta_keep_zf + +echo "PASS: native pattern consumers preserve required EFLAGS semantics"