diff --git a/erts/emulator/beam/jit/arm/instr_common.cpp b/erts/emulator/beam/jit/arm/instr_common.cpp index 1ad65f19d7fa..57e0b3ad8af4 100644 --- a/erts/emulator/beam/jit/arm/instr_common.cpp +++ b/erts/emulator/beam/jit/arm/instr_common.cpp @@ -603,25 +603,31 @@ void BeamModuleAssembler::emit_move_trim(const ArgSource &Src, } void BeamModuleAssembler::emit_store_two_values(const ArgSource &Src1, - const ArgYRegister &Dst1, + const ArgRegister &Dst1, const ArgSource &Src2, - const ArgYRegister &Dst2) { + const ArgRegister &Dst2) { auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2); auto dst1 = init_destination(Dst1, src1.reg); auto dst2 = init_destination(Dst2, src2.reg); + ASSERT(!isRegisterBacked(Dst1)); + ASSERT(!isRegisterBacked(Dst2)); + flush_vars(dst1, dst2); } -void BeamModuleAssembler::emit_load_two_xregs(const ArgYRegister &Src1, +void BeamModuleAssembler::emit_load_two_xregs(const ArgRegister &Src1, const ArgXRegister &Dst1, - const ArgYRegister &Src2, + const ArgRegister &Src2, const ArgXRegister &Dst2) { ASSERT(ArgVal::memory_relation(Src1, Src2) == ArgVal::Relation::consecutive); auto dst1 = init_destination(Dst1, TMP1); auto dst2 = init_destination(Dst2, TMP2); + ASSERT(!isRegisterBacked(Src1)); + ASSERT(!isRegisterBacked(Src2)); + safe_ldp(dst1.reg, dst2.reg, Src1, Src2); flush_vars(dst1, dst2); } diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab index 8411bc665e51..210833dcdb7d 100644 --- a/erts/emulator/beam/jit/arm/ops.tab +++ b/erts/emulator/beam/jit/arm/ops.tab @@ -340,17 +340,29 @@ move S1=y D1=x | move S2=y D2=x | distinct(D1, D2) => load_two_xregs S1 D1 S2 D2 +move S1=x D1=x | move S2=x D2=x | + consecutive_xregs(S1, S2) | + distinct(D1, D2) => + load_two_xregs S1 D1 S2 D2 + move S1=y D1=x | move S2=y D2=x | consecutive_words(S2, S1) | + distinct(D1, S2) | + distinct(D1, D2) => + load_two_xregs S2 D2 S1 D1 + +move S1=x D1=x | move S2=x D2=x | + consecutive_xregs(S2, S1) | + distinct(D1, S2) | distinct(D1, D2) => load_two_xregs S2 D2 S1 D1 # Optimize storing two values in Y registers when destination Y # registers are consecutive. -move S1 D1=y | move S2 D2=y | consecutive_words(D1, D2) => +move S1 D1=y | move S2 D2=y | consecutive_words(D1, D2) | distinct(D1, S2) => store_two_values S1 D1 S2 D2 -move S1 D1=y | move S2 D2=y | consecutive_words(D2, D1) => +move S1 D1=y | move S2 D2=y | consecutive_words(D2, D1) | distinct(D1, S2) => store_two_values S2 D2 S1 D1 move S1=y D1 | move S2=y D2 | consecutive_words(S1, S2) | trim N u => @@ -359,6 +371,12 @@ move S1=y D1 | move S2=y D2 | consecutive_words(S1, S2) | trim N u => move S2=y D2 | move S1=y D1 | consecutive_words(S1, S2) | trim N u => move_two_trim S1 D1 S2 D2 N +move S1 D1=x | move S2 D2=x | consecutive_xregs(D1, D2) | distinct(D1, S2) => + store_two_values S1 D1 S2 D2 + +move S1 D1=x | move S2 D2=x | consecutive_xregs(D2, D1) | distinct(D1, S2) => + store_two_values S2 D2 S1 D1 + move Src Dst | trim N u => move_trim Src Dst N move_two_trim y d y d t @@ -368,8 +386,8 @@ move_trim s d t move Src Dst => i_move Src Dst i_move s d -store_two_values s y s y -load_two_xregs y x y x +store_two_values s d s d +load_two_xregs d x d x # # Swap instructions. diff --git a/erts/emulator/beam/jit/arm/predicates.tab b/erts/emulator/beam/jit/arm/predicates.tab index 8a3ae485ece8..d08684a99255 100644 --- a/erts/emulator/beam/jit/arm/predicates.tab +++ b/erts/emulator/beam/jit/arm/predicates.tab @@ -81,6 +81,10 @@ pred.never_fails(Bif) { return 0; } +pred.consecutive_xregs(A1, A2) { + return A1.type == TAG_x && A2.type == TAG_x && A1.val + 1 == A2.val && A1.val >= 6; +} + pred.consecutive_words(A1, A2) { return A1.type == A2.type && A1.val + 1 == A2.val; }