Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/download-bundled/jit-ir.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.."
tmp_dir=/tmp/php-src-download-bundled/jit-ir
rm -rf "$tmp_dir"

revision=00bec1ca490b8bc51f636a42040c5e9f64c1a91b
revision=93f92f5b5a49a02295685e340ebb0b83562cbe79

git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir"

Expand Down
4 changes: 3 additions & 1 deletion ext/opcache/jit/ir/ir_emit.c
Original file line number Diff line number Diff line change
Expand Up @@ -1528,7 +1528,9 @@ static ir_reg _get_free_reg2(ir_ctx *ctx, ir_type type, ir_reg_alloc_simple_data

reg = IR_REGSET_FIRST(available);
if (IR_REGSET_IN(x->preserved_regs, reg)) {
IR_REGSET_INCL(ctx->used_preserved_regs, reg);
ir_regset tmp = ctx->used_preserved_regs;
IR_REGSET_INCL(tmp, reg);
ctx->used_preserved_regs = tmp;
}
return reg;
}
Expand Down
10 changes: 4 additions & 6 deletions ext/opcache/jit/ir/ir_fold.h
Original file line number Diff line number Diff line change
Expand Up @@ -1840,8 +1840,7 @@ IR_FOLD(REPLACE(LONG_CONST, _))
if (IR_IS_CONST_REF(op3)
&& IR_IS_CONST_REF(op2)
&& IR_IS_TYPE_INT(op2_insn->type)
&& op2_insn->val.i64 >= 0
&& op2_insn->val.i64 < IR_VECTOR_LENGTH(op1_insn->type)) {
&& op2_insn->val.u64 < (uint64_t)IR_VECTOR_LENGTH(op1_insn->type)) {
IR_ASSERT(IR_IS_TYPE_VECTOR(op1_insn->type)
&& (IR_VECTOR_BASE_TYPE(op1_insn->type) == op3_insn->type
|| (IR_IS_TYPE_INT(IR_VECTOR_BASE_TYPE(op1_insn->type))
Expand Down Expand Up @@ -1942,8 +1941,7 @@ IR_FOLD(EXTRACT(LONG_CONST, _))
{
if (IR_IS_CONST_REF(op2)
&& IR_IS_TYPE_INT(op2_insn->type)
&& op2_insn->val.i64 >= 0
&& op2_insn->val.i64 < IR_VECTOR_LENGTH(op1_insn->type)) {
&& op2_insn->val.u64 < (uint64_t)IR_VECTOR_LENGTH(op1_insn->type)) {
uint32_t idx = op2_insn->val.u32;
void *ptr;

Expand Down Expand Up @@ -3212,7 +3210,7 @@ IR_FOLD(LT(ABS, C_FLOAT))
IR_FOLD(LT(ABS, C_DOUBLE))
{
if (op2_insn->val.u64 == 0) {
/* abs() < 0 => false */
/* abs() < 0 => false (abs(INT_MIN) is UB, so this optimization may change the result) */
IR_FOLD_COPY(IR_FALSE);
}
IR_FOLD_NEXT;
Expand All @@ -3226,7 +3224,7 @@ IR_FOLD(GE(ABS, C_FLOAT))
IR_FOLD(GE(ABS, C_DOUBLE))
{
if (op2_insn->val.u64 == 0) {
/* abs() >= 0 => true */
/* abs() >= 0 => true (abs(INT_MIN) is UB, so this optimization may change the result) */
IR_FOLD_COPY(IR_TRUE);
}
IR_FOLD_NEXT;
Expand Down
27 changes: 16 additions & 11 deletions ext/opcache/jit/ir/ir_private.h
Original file line number Diff line number Diff line change
Expand Up @@ -1273,24 +1273,29 @@ typedef struct _ir_use_pos ir_use_pos;
#define IR_USE_MUST_BE_IN_REG (1<<0)
#define IR_USE_SHOULD_BE_IN_REG (1<<1)
#define IR_HINT_TWO_REGS (1<<2)
#define IR_HINT_NEEDS_HOLE (1<<3) /* create a fake hole in live ranges to avoid conflict with fxed reg */

#define IR_DEF_REUSES_OP1_REG (1<<3)
#define IR_DEF_CONFLICTS_WITH_INPUT_REGS (1<<4)
#define IR_EXTEND_INPUTS_TO_NEXT (1<<5) /* used for SNAPSHOT followed by GUARD */

#define IR_FUSED_USE (1<<6)
#define IR_PHI_USE (1<<7)

#define IR_OP1_MUST_BE_IN_REG (1<<8)
#define IR_OP1_SHOULD_BE_IN_REG (1<<9)
#define IR_OP1_HINT_TWO_REGS (1<<10)
#define IR_OP2_MUST_BE_IN_REG (1<<11)
#define IR_OP2_SHOULD_BE_IN_REG (1<<12)
#define IR_OP2_HINT_TWO_REGS (1<<13)
#define IR_OP3_MUST_BE_IN_REG (1<<14)
#define IR_OP3_SHOULD_BE_IN_REG (1<<15)
#define IR_OP3_HINT_TWO_REGS (1<<16)

#define IR_USE_FLAGS(def_flags, op_num) (((def_flags) >> (5 + (IR_MIN((op_num), 3) * 3))) & 7)
#define IR_OP1_MUST_BE_IN_REG (1<<16)
#define IR_OP1_SHOULD_BE_IN_REG (1<<17)
#define IR_OP1_HINT_TWO_REGS (1<<18)
#define IR_OP1_HINT_NEEDS_HOLE (1<<19)
#define IR_OP2_MUST_BE_IN_REG (1<<20)
#define IR_OP2_SHOULD_BE_IN_REG (1<<21)
#define IR_OP2_HINT_TWO_REGS (1<<22)
#define IR_OP2_HINT_NEEDS_HOLE (1<<23)
#define IR_OP3_MUST_BE_IN_REG (1<<24)
#define IR_OP3_SHOULD_BE_IN_REG (1<<25)
#define IR_OP3_HINT_TWO_REGS (1<<26)
#define IR_OP3_HINT_NEEDS_HOLE (1<<27)

#define IR_USE_FLAGS(def_flags, op_num) (((def_flags) >> (12 + (IR_MIN((op_num), 3) * 4))) & 15)

struct _ir_use_pos {
uint16_t op_num; /* 0 - means result */
Expand Down
49 changes: 39 additions & 10 deletions ext/opcache/jit/ir/ir_ra.c
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,10 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block
/* intervals[opd].addRange(b.from, op.id) */
ival = ir_add_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
} else if (reg != IR_REG_NONE && (IR_USE_FLAGS(def_flags, j) & IR_HINT_NEEDS_HOLE)) {
ir_fix_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos + IR_DEF_SUB_REF);
ival = ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
} else {
ival = ctx->live_intervals[v];
}
Expand Down Expand Up @@ -692,14 +696,15 @@ int ir_compute_live_ranges(ir_ctx *ctx)
if (input > 0) {
uint32_t v = ctx->vregs[input];

/* live.add(phi.inputOf(b)) */
IR_ASSERT(v);
ir_bitset_incl(live, v);
/* intervals[phi.inputOf(b)].addRange(b.from, b.to) */
ival = ir_add_prev_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start),
IR_END_LIVE_POS_FROM_REF(bb->end));
ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use);
if (v) {
/* live.add(phi.inputOf(b)) */
ir_bitset_incl(live, v);
/* intervals[phi.inputOf(b)].addRange(b.from, b.to) */
ival = ir_add_prev_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start),
IR_END_LIVE_POS_FROM_REF(bb->end));
ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use);
}
}
}
}
Expand Down Expand Up @@ -766,8 +771,6 @@ int ir_compute_live_ranges(ir_ctx *ctx)
insn = &ctx->ir_base[ref];
v = ctx->vregs[ref];
if (v) {
IR_ASSERT(ir_bitset_in(live, v));

if (insn->op != IR_PHI) {
ir_live_pos def_pos;
ir_ref hint_ref = 0;
Expand Down Expand Up @@ -878,6 +881,17 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else
#endif
ir_add_fixed_live_range(ctx, reg, use_pos, use_pos + IR_USE_SUB_REF);
if ((use_flags & IR_HINT_NEEDS_HOLE) && ir_bitset_in(live, v)) {
/* This is a special case for x86 SHIFT instructions that uses %rcx for op2.
* SHIFT adds short fixed live range for %rcx and this makes a conflict with
* operand live interval, if the operand is still alive after the SHIFT.
* To avoid the conflict we create a "fake" hole in the operand live interval.
* See: ./tests/debug/ra_004.irt
*/
ir_fix_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos + IR_DEF_SUB_REF);
ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
}
} else if (def_flags & IR_DEF_REUSES_OP1_REG) {
if (j == 1) {
use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref);
Expand Down Expand Up @@ -1286,6 +1300,10 @@ static void ir_add_fusion_ranges(ir_ctx *ctx, ir_ref ref, ir_ref input, ir_block
/* intervals[opd].addRange(b.from, op.id) */
ival = ir_add_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
} else if (reg != IR_REG_NONE && (IR_USE_FLAGS(def_flags, j) & IR_HINT_NEEDS_HOLE)) {
ir_fix_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos + IR_DEF_SUB_REF);
ival = ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
} else {
ival = ctx->live_intervals[v];
}
Expand Down Expand Up @@ -1559,6 +1577,17 @@ int ir_compute_live_ranges(ir_ctx *ctx)
} else
#endif
ir_add_fixed_live_range(ctx, reg, use_pos, use_pos + IR_USE_SUB_REF);
if ((use_flags & IR_HINT_NEEDS_HOLE) && IS_LIVE_IN_BLOCK(v, b)) {
/* This is a special case for x86 SHIFT instructions that uses %rcx for op2.
* SHIFT adds short fixed live range for %rcx and this makes a conflict with
* operand live interval, if the operand is still alive after the SHIFT.
* To avoid the conflict we create a "fake" hole in the operand live interval.
* See: ./tests/debug/ra_004.irt
*/
ir_fix_live_range(ctx, v,
IR_START_LIVE_POS_FROM_REF(bb->start), use_pos + IR_DEF_SUB_REF);
ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos);
}
} else if (def_flags & IR_DEF_REUSES_OP1_REG) {
if (j == 1) {
if (def_flags & IR_DEF_CONFLICTS_WITH_INPUT_REGS) {
Expand Down
19 changes: 19 additions & 0 deletions ext/opcache/jit/ir/ir_sccp.c
Original file line number Diff line number Diff line change
Expand Up @@ -2451,6 +2451,20 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn)
&& ir_is_loop_invariant(ctx, insn->op1, ctx->ir_base[insn->op2].op1)) {
return ir_try_promote_induction_var_ext(ctx, ext_ref, insn->op2, ref);
}
} else if (insn->op == IR_COND
&& IR_IS_CONST_REF(insn->op2)
&& IR_IS_CONST_REF(insn->op3)
&& ctx->use_lists[ref].count == 1) {
IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op));
IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op3].op));

ir_op op = ctx->ir_base[ext_ref].op;
ir_type type = ctx->ir_base[ext_ref].type;

insn->type = type;
insn->op2 = ir_ext_const(ctx, &ctx->ir_base[insn->op2], op, type);;
insn->op3 = ir_ext_const(ctx, &ctx->ir_base[insn->op3], op, type);;
ir_iter_replace_insn(ctx, ext_ref, ref);
}

return 0;
Expand Down Expand Up @@ -2837,6 +2851,8 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
MAKE_NOP(end2); CLEAR_USES(end2_ref);
MAKE_NOP(merge); CLEAR_USES(merge_ref);

ir_bitqueue_add(ctx->iter_worklist, ref);
ir_iter_add_uses(ctx, ref, ctx->iter_worklist);
if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) {
ir_bitqueue_add(ctx->iter_worklist, next->op1);
}
Expand Down Expand Up @@ -2928,6 +2944,8 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
MAKE_NOP(merge); CLEAR_USES(merge_ref);
MAKE_NOP(&ctx->ir_base[neg_ref]); CLEAR_USES(neg_ref);

ir_bitqueue_add(ctx->iter_worklist, ref);
ir_iter_add_uses(ctx, ref, ctx->iter_worklist);
if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) {
ir_bitqueue_add(ctx->iter_worklist, next->op1);
}
Expand Down Expand Up @@ -2997,6 +3015,7 @@ static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_re
MAKE_NOP(merge); CLEAR_USES(merge_ref);

ir_bitqueue_add(ctx->iter_worklist, ref);
ir_iter_add_uses(ctx, ref, ctx->iter_worklist);
if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) {
ir_bitqueue_add(ctx->iter_worklist, next->op1);
}
Expand Down
55 changes: 36 additions & 19 deletions ext/opcache/jit/ir/ir_x86.dasc
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,11 @@ IR_ALWAYS_INLINE ir_mem IR_MEM(ir_reg base, int32_t offset, ir_reg index, int32_
|| }
|.endmacro

/* Zero a register. A 32-bit xor has a shorter encoding and also clears the upper 32 bits. */
|.macro ASM_REG_ZERO, type, reg
| xor Rd(reg), Rd(reg)
|.endmacro

|.macro ASM_REG_REG_OP2, op, type, op1, op2
|| switch (ir_type_size[type]) {
|| default:
Expand Down Expand Up @@ -1851,9 +1856,10 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
break;
case IR_SHIFT:
if (rule & IR_FUSED) {
flags = IR_OP2_MUST_BE_IN_REG;
flags = IR_OP2_MUST_BE_IN_REG | IR_OP2_HINT_NEEDS_HOLE;
} else {
flags = IR_DEF_REUSES_OP1_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS | IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG;
flags = IR_DEF_REUSES_OP1_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS | IR_USE_MUST_BE_IN_REG |
IR_OP1_SHOULD_BE_IN_REG | IR_OP2_SHOULD_BE_IN_REG | IR_OP2_HINT_NEEDS_HOLE;
}
constraints->hints[1] = IR_REG_NONE;
constraints->hints[2] = IR_REG_RCX;
Expand Down Expand Up @@ -3268,6 +3274,15 @@ static bool ir_match_has_flags_deps(ir_ctx *ctx, ir_ref ref, ir_ref root)
return pos != ref;
}

static uint32_t ir_match_fuse_load_const_addr(ir_ctx *ctx, ir_ref ref, const ir_insn *addr_insn)
{
if (ir_may_fuse_addr(ctx, addr_insn)) {
return IR_FUSED | IR_SIMPLE | IR_LOAD;
}
/* The address doesn't fit into the displacement. Load it into a temporary register. */
return IR_FUSED | (IR_IS_TYPE_INT(ctx->ir_base[ref].type) ? IR_LOAD_INT : IR_LOAD_FP);
}

static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root)
{
if (ir_in_same_block(ctx, ref) &&
Expand All @@ -3279,10 +3294,8 @@ static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root)
ir_insn *addr_insn = &ctx->ir_base[addr_ref];

if (IR_IS_CONST_REF(addr_ref)) {
if (ir_may_fuse_addr(ctx, addr_insn)) {
ctx->rules[ref] = IR_FUSED | IR_SIMPLE | IR_LOAD;
return;
}
ctx->rules[ref] = ir_match_fuse_load_const_addr(ctx, ref, addr_insn);
return;
} else if (addr_insn->op == IR_TLS_ADDR) {
// TODO: try to fuse static TLS addr ???
return;
Expand All @@ -3307,10 +3320,8 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root)
ir_insn *addr_insn = &ctx->ir_base[addr_ref];

if (IR_IS_CONST_REF(addr_ref)) {
if (ir_may_fuse_addr(ctx, addr_insn)) {
ctx->rules[ref] = IR_FUSED | IR_SIMPLE | IR_LOAD;
return 1;
}
ctx->rules[ref] = ir_match_fuse_load_const_addr(ctx, ref, addr_insn);
return 1;
} else if (addr_insn->op == IR_TLS_ADDR) {
// TODO: try to fuse static TLS addr ???
return 0;
Expand Down Expand Up @@ -5348,7 +5359,11 @@ static void ir_match_insn2(ir_ctx *ctx, ir_ref ref, uint32_t rule)
if (rule == IR_LEA_IB) {
if (!ir_match_try_revert_lea_to_add(ctx, ref) && !(ctx->flags & IR_OPT_CODEGEN)) {
/* revert to ADD to avoid extra register spill load with -O0 */
ctx->rules[ref] = IR_BINOP_INT | IR_MAY_SWAP;
ir_insn *insn = &ctx->ir_base[ref];

if (ctx->ir_base[insn->op1].op != IR_ALLOCA && ctx->ir_base[insn->op2].op != IR_ALLOCA) {
ctx->rules[ref] = IR_BINOP_INT | IR_MAY_SWAP;
}
}
}
}
Expand Down Expand Up @@ -5458,7 +5473,7 @@ static void ir_emit_load_imm_int(ir_ctx *ctx, ir_type type, ir_reg reg, int64_t

IR_ASSERT(IR_IS_TYPE_INT(type));
if (val == 0) {
| ASM_REG_REG_OP xor, type, reg, reg
| ASM_REG_ZERO type, reg
} else {
ir_emit_mov_imm_int(ctx, type, reg, val);
}
Expand Down Expand Up @@ -8234,7 +8249,7 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (ir_type_size[type] == 1) {
| movzx ax, al
} else {
| ASM_REG_REG_OP xor, type, IR_REG_RDX, IR_REG_RDX
| ASM_REG_ZERO type, IR_REG_RDX
}
if (op2_reg != IR_REG_NONE) {
| ASM_REG_OP div, type, op2_reg
Expand Down Expand Up @@ -8801,7 +8816,7 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (IR_IS_CONST_REF(op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2].op) && ctx->ir_base[op2].val.u64 == 0) {
if (op == IR_ULT) {
/* always false */
| xor Ra(def_reg), Ra(def_reg)
| xor Rd(def_reg), Rd(def_reg)
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
Expand Down Expand Up @@ -14572,7 +14587,7 @@ static void ir_emit_cmp_i64(ir_ctx *ctx, ir_ref def, ir_insn *insn)
if (IR_IS_CONST_REF(op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2].op) && ctx->ir_base[op2].val.u64 == 0) {
if (op == IR_ULT) {
/* always false */
| xor Ra(def_reg), Ra(def_reg)
| xor Rd(def_reg), Rd(def_reg)
if (IR_REG_SPILLED(ctx->regs[def][0])) {
ir_emit_store(ctx, insn->type, def, def_reg);
}
Expand Down Expand Up @@ -22205,7 +22220,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else if (element_size == 1) {
| movzx ax, al
} else {
| ASM_REG_REG_OP xor, element_type, IR_REG_RDX, IR_REG_RDX
| ASM_REG_ZERO element_type, IR_REG_RDX
}
if (IR_IS_TYPE_SIGNED(element_type)) {
| ASM_REG_OP idiv, element_type, tmp2_reg
Expand Down Expand Up @@ -22652,7 +22667,7 @@ static void ir_emit_vector_binop_expand(ir_ctx *ctx, ir_ref def, ir_insn *insn)
} else if (element_size == 1) {
| movzx ax, al
} else {
| ASM_REG_REG_OP xor, element_type, IR_REG_RDX, IR_REG_RDX
| ASM_REG_ZERO element_type, IR_REG_RDX
}
if (IR_IS_TYPE_SIGNED(element_type)) {
| ASM_MEM_OP idiv, element_type, mem
Expand Down Expand Up @@ -25988,8 +26003,10 @@ void ir_fix_stack_frame(ir_ctx *ctx)

#ifdef IR_TARGET_X86
if (ctx->flags2 & IR_HAS_MEMCPY) {
IR_REGSET_INCL(ctx->used_preserved_regs, IR_REG_RSI);
IR_REGSET_INCL(ctx->used_preserved_regs, IR_REG_RDI);
ir_regset tmp = ctx->used_preserved_regs;
IR_REGSET_INCL(tmp, IR_REG_RSI);
IR_REGSET_INCL(tmp, IR_REG_RDI);
ctx->used_preserved_regs = tmp;
}
#endif

Expand Down
Loading