Skip to content

Commit

Permalink
Optimize tag tests in is_eq_exact and is_ne_exact
Browse files Browse the repository at this point in the history
  • Loading branch information
@bjorng
bjorng committed Nov 25, 2023
1 parent 9b0455b commit add6abb
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 70 deletions.
43 changes: 43 additions & 0 deletions erts/emulator/beam/jit/arm/beam_asm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1774,6 +1774,49 @@ class BeamModuleAssembler : public BeamAssembler,
a.ldp(gp1, gp2, arm::Mem(SUPER_TMP));
}
}

/* Set the Z flag if Reg1 and Reg2 are definitely not equal based
* on their tags alone. (They may still be equal if both are
* immediates and all other bits are equal too.) */
void emit_is_unequal_based_on_tags(Label Unequal,
const ArgVal &Src1,
arm::Gp Reg1,
const ArgVal &Src2,
arm::Gp Reg2) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) ==
TAG_PRIMARY_IMMED1);

if (always_one_of<BeamTypeId::AlwaysBoxed>(Src1)) {
emit_is_boxed(Unequal, Reg2);
} else if (always_one_of<BeamTypeId::AlwaysBoxed>(Src2)) {
emit_is_boxed(Unequal, Reg1);
} else if (exact_type<BeamTypeId::Cons>(Src1)) {
emit_is_cons(Unequal, Reg2);
} else if (exact_type<BeamTypeId::Cons>(Src2)) {
emit_is_cons(Unequal, Reg1);
} else {
a.orr(SUPER_TMP, Reg1, Reg2);

if (never_one_of<BeamTypeId::Cons>(Src1) ||
never_one_of<BeamTypeId::Cons>(Src2)) {
emit_is_boxed(Unequal, SUPER_TMP);
} else if (never_one_of<BeamTypeId::AlwaysBoxed>(Src1) ||
never_one_of<BeamTypeId::AlwaysBoxed>(Src2)) {
emit_is_cons(Unequal, SUPER_TMP);
} else {
a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));

/*
* SUPER_TMP will be now be TAG_PRIMARY_IMMED1 if
* either one or both registers are immediates, or if
* one register is a list and the other a boxed.
*/
a.b_eq(Unequal);
}
}
}
};

void beamasm_metadata_update(std::string module_name,
Expand Down
26 changes: 6 additions & 20 deletions erts/emulator/beam/jit/arm/instr_common.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1525,18 +1525,13 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail,
emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP1);
} else if (always_same_types(X, Y)) {
comment("skipped tag test since they are always equal");
} else if (Y.isLiteral()) {
/* Fail immediately unless X is the same type of pointer as
* the literal Y.
*/
Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get());
Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK);
int bitNumber = Support::ctz<Eterm>(tag_test);
a.tbnz(x.reg, imm(bitNumber), resolve_beam_label(Fail, disp32K));
} else {
/* Fail immediately if the pointer tags are not equal. */
emit_is_unequal_based_on_tags(x.reg, y.reg);
a.b_eq(resolve_beam_label(Fail, disp1MB));
emit_is_unequal_based_on_tags(resolve_beam_label(Fail, dispUnknown),
X,
x.reg,
Y,
y.reg);
}

/* Both operands are pointers having the same tag. Must do a
Expand Down Expand Up @@ -1599,19 +1594,10 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail,
emit_is_boxed(next, TMP1);
} else if (always_same_types(X, Y)) {
comment("skipped tag test since they are always equal");
} else if (Y.isLiteral()) {
/* Succeed immediately if X is not the same type of pointer as
* the literal Y.
*/
Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get());
Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK);
int bitNumber = Support::ctz<Eterm>(tag_test);
a.tbnz(x.reg, imm(bitNumber), next);
} else {
/* Test whether the terms are definitely unequal based on the tags
* alone. */
emit_is_unequal_based_on_tags(x.reg, y.reg);
a.b_eq(next);
emit_is_unequal_based_on_tags(next, X, x.reg, Y, y.reg);
}

/* Both operands are pointers having the same tag. Must do a
Expand Down
65 changes: 41 additions & 24 deletions erts/emulator/beam/jit/x86/beam_asm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -855,30 +855,6 @@ struct BeamAssembler : public BeamAssemblerCommon {
}
}

/* Set the Z flag if Reg1 and Reg2 are definitely not equal based on their
* tags alone. (They may still be equal if both are immediates and all other
* bits are equal too.) */
void emit_is_unequal_based_on_tags(x86::Gp Reg1, x86::Gp Reg2) {
ASSERT(Reg1 != RET && Reg2 != RET);
emit_is_unequal_based_on_tags(Reg1, Reg2, RET);
}

void emit_is_unequal_based_on_tags(x86::Gp Reg1,
x86::Gp Reg2,
const x86::Gp &spill) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) ==
TAG_PRIMARY_IMMED1);
a.mov(RETd, Reg1.r32());
a.or_(RETd, Reg2.r32());
a.and_(RETb, imm(_TAG_PRIMARY_MASK));

/* RET will be now be TAG_PRIMARY_IMMED1 if either one or both
* registers are immediates, or if one register is a list and the other
* a boxed. */
a.cmp(RETb, imm(TAG_PRIMARY_IMMED1));
}

/*
* Generate the shortest instruction for setting a register to an immediate
* value. May clear flags.
Expand Down Expand Up @@ -1617,6 +1593,47 @@ class BeamModuleAssembler : public BeamAssembler,
mov_arg(getArgRef(to), from);
}
}

/* Set the Z flag if Reg1 and Reg2 are definitely not equal based
* on their tags alone. (They may still be equal if both are
* immediates and all other bits are equal too.)
*
* Clobbers RET.
*/
void emit_is_unequal_based_on_tags(Label Unequal,
const ArgVal &Src1,
x86::Gp Reg1,
const ArgVal &Src2,
x86::Gp Reg2,
Distance dist = dLong) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) ==
TAG_PRIMARY_IMMED1);

if (always_one_of<BeamTypeId::AlwaysBoxed>(Src1)) {
emit_is_boxed(Unequal, Reg2, dist);
} else if (always_one_of<BeamTypeId::AlwaysBoxed>(Src2)) {
emit_is_boxed(Unequal, Reg1, dist);
} else if (exact_type<BeamTypeId::Cons>(Src1)) {
emit_is_cons(Unequal, Reg2, dist);
} else if (exact_type<BeamTypeId::Cons>(Src2)) {
emit_is_cons(Unequal, Reg1, dist);
} else {
a.mov(RETd, Reg1.r32());
a.or_(RETd, Reg2.r32());
a.and_(RETb, imm(_TAG_PRIMARY_MASK));

/* RET will be now be TAG_PRIMARY_IMMED1 if either one or
* both registers are immediates, or if one register is a
* list and the other a boxed. */
a.cmp(RETb, imm(TAG_PRIMARY_IMMED1));
if (dist == dShort) {
a.short_().je(Unequal);
} else {
a.je(Unequal);
}
}
}
};

void beamasm_metadata_update(std::string module_name,
Expand Down
33 changes: 7 additions & 26 deletions erts/emulator/beam/jit/x86/instr_common.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1439,18 +1439,13 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgLabel &Fail,
a.jne(resolve_beam_label(Fail));
} else if (always_same_types(X, Y)) {
comment("skipped tag test since they are always equal");
} else if (Y.isLiteral()) {
/* Fail immediately unless X is the same type of pointer as
* the literal Y.
*/
Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get());
Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK);
a.test(ARG1.r8(), imm(tag_test));
a.jne(resolve_beam_label(Fail));
} else {
/* Fail immediately if the pointer tags are not equal. */
emit_is_unequal_based_on_tags(ARG1, ARG2);
a.je(resolve_beam_label(Fail));
emit_is_unequal_based_on_tags(resolve_beam_label(Fail),
X,
ARG1,
Y,
ARG2);
}

/* Both operands are pointers having the same tag. Must do a
Expand Down Expand Up @@ -1515,27 +1510,13 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgLabel &Fail,
a.short_().jne(next);
} else if (always_same_types(X, Y)) {
comment("skipped tag test since they are always equal");
} else if (Y.isLiteral()) {
/* Succeed immediately if X is not the same type of pointer as
* the literal Y.
*/
Eterm literal = beamfile_get_literal(beam, Y.as<ArgLiteral>().get());
Uint tag_test = _TAG_PRIMARY_MASK - (literal & _TAG_PRIMARY_MASK);
a.test(ARG1.r8(), imm(tag_test));
#ifdef JIT_HARD_DEBUG
a.jne(next);
#else
a.short_().jne(next);
#endif
} else {
/* Test whether the terms are definitely unequal based on the tags
* alone. */
emit_is_unequal_based_on_tags(ARG1, ARG2);

#ifdef JIT_HARD_DEBUG
a.je(next);
emit_is_unequal_based_on_tags(next, X, ARG1, Y, ARG2);
#else
a.short_().je(next);
emit_is_unequal_based_on_tags(next, X, ARG1, Y, ARG2, dShort);
#endif
}

Expand Down

0 comments on commit add6abb

Please sign in to comment.