From 03c61a43739effe7a8d364bffcf81b7d2ca171db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 8 May 2024 12:13:49 +0200 Subject: [PATCH] Get rid of macros for bit syntax construction The first implementation of the bit syntax was done in Erlang/OTP R7B before the SMP emulator (the multi-threaded runtime system). Therefore, global variables were used to keep track of the binary being constructed. When the SMP emulator was introduced, the state of the binary being constructed needed to be local for each scheduler thread. To reduce the need for extensive rewrites, macros were used to automagically pass references to the binary construction state to the helper functions in erl_bits.c. Now bite the bullet and remove these macros. Also take the opportunity to remove the "new" part from function names. They are not exactly new. --- erts/emulator/beam/emu/bs_instrs.tab | 35 +++-- erts/emulator/beam/erl_bits.c | 149 +++++++++++---------- erts/emulator/beam/erl_bits.h | 60 ++++----- erts/emulator/beam/jit/arm/instr_bs.cpp | 84 ++++++------ erts/emulator/beam/jit/beam_jit_common.cpp | 10 +- erts/emulator/beam/jit/beam_jit_common.hpp | 2 +- erts/emulator/beam/jit/x86/instr_bs.cpp | 76 ++++++----- 7 files changed, 209 insertions(+), 207 deletions(-) diff --git a/erts/emulator/beam/emu/bs_instrs.tab b/erts/emulator/beam/emu/bs_instrs.tab index 1150b1172781..0024bae6e34f 100644 --- a/erts/emulator/beam/emu/bs_instrs.tab +++ b/erts/emulator/beam/emu/bs_instrs.tab @@ -373,7 +373,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { const BeamInstr* p; Uint alloc = $Alloc; Eterm new_binary; - ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ + ErlBitsState* EBS = ERL_BITS_EBS_FROM_REG(reg); /* We count the total number of bits in an unsigned integer. To avoid * having to check for overflow when adding to `num_bits`, we ensure that @@ -546,7 +546,6 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { } /* Allocate binary. */ - ERL_BITS_RELOAD_STATEP(c_p); p = p_start; if (p[0] == BSC_APPEND) { Uint live = $Live; @@ -565,15 +564,13 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { } p_start += BSC_NUM_ARGS; } else if (p[0] == BSC_PRIVATE_APPEND) { - Uint unit; Eterm Src; $test_heap(alloc, $Live); - $BS_LOAD_UNIT(p, unit); $BS_LOAD_SRC(p, Src); - new_binary = erts_bs_private_append_checked(c_p, Src, num_bits, unit); + new_binary = erts_bs_private_append_checked(EBS, c_p, Src, num_bits); if (is_non_value(new_binary)) { $BS_FAIL_INFO($Fail, c_p->freason, c_p->fvalue, Src); @@ -589,7 +586,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { /* num_bits = Number of bits to build * alloc = Total number of words to allocate on heap */ - erts_bin_offset = 0; + EBS->erts_bin_offset = 0; if (num_bits <= ERL_ONHEAP_BITS_LIMIT) { ErlHeapBits *hb; @@ -598,7 +595,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { HTOP += heap_bits_size(num_bits); hb->thing_word = header_heap_bits(num_bits); hb->size = num_bits; - erts_current_bin = (byte *) hb->data; + EBS->erts_current_bin = (byte *) hb->data; new_binary = make_bitstring(hb); } else { Binary* bptr; @@ -608,7 +605,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { $Live); bptr = erts_bin_nrml_alloc(NBYTES(num_bits)); - erts_current_bin = (byte *)bptr->orig_bytes; + EBS->erts_current_bin = (byte *)bptr->orig_bytes; LIGHT_SWAPOUT; @@ -616,7 +613,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { &MSO(c_p).overhead, &HEAP_TOP(c_p), bptr, - erts_current_bin, + EBS->erts_current_bin, 0, num_bits); @@ -640,7 +637,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { byte* string; $BS_LOAD_STRING_SRC(p, string); $BS_LOAD_FIXED_SIZE(p, Size); - erts_new_bs_put_string(ERL_BITS_ARGS_2(string, Size)); + erts_bs_put_string(EBS, string, Size); continue; } @@ -649,7 +646,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { switch (p[0]) { case BSC_BINARY_ALL: $BS_LOAD_UNIT(p, unit); - if (!erts_new_bs_put_binary_all(c_p, Src, unit)) { + if (!erts_bs_put_binary_all(EBS, c_p, Src, unit)) { $BS_FAIL_INFO($Fail, BADARG, am_unit, Src); } break; @@ -658,14 +655,14 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { $BS_LOAD_FLAGS(p, flags); $BS_LOAD_SIZE(p, Size); $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size); - if (!erts_new_bs_put_binary(c_p, Src, _size)) { + if (!erts_bs_put_binary(EBS, c_p, Src, _size)) { Eterm reason = is_bitstring(Src) ? am_short : am_type; $BS_FAIL_INFO($Fail, BADARG, reason, Src); } break; case BSC_BINARY_FIXED_SIZE: $BS_LOAD_FIXED_SIZE(p, Size); - if (!erts_new_bs_put_binary(c_p, Src, Size)) { + if (!erts_bs_put_binary(EBS, c_p, Src, Size)) { Eterm reason = is_bitstring(Src) ? am_short : am_type; $BS_FAIL_INFO($Fail, BADARG, reason, Src); } @@ -675,7 +672,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { $BS_LOAD_FLAGS(p, flags); $BS_LOAD_SIZE(p, Size); $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size); - Src = erts_new_bs_put_float(c_p, Src, _size, flags); + Src = erts_bs_put_float(EBS, c_p, Src, _size, flags); if (is_value(Src)) { $BS_FAIL_INFO($Fail, BADARG, c_p->fvalue, Src); } @@ -683,7 +680,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { case BSC_FLOAT_FIXED_SIZE: $BS_LOAD_FLAGS(p, flags); $BS_LOAD_FIXED_SIZE(p, Size); - Src = erts_new_bs_put_float(c_p, Src, Size, flags); + Src = erts_bs_put_float(EBS, c_p, Src, Size, flags); if (is_value(Src)) { $BS_FAIL_INFO($Fail, BADARG, c_p->fvalue, Src); } @@ -696,7 +693,7 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { $BS_LOAD_FLAGS(p, flags); $BS_LOAD_SIZE(p, Size); $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size); - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(Src, _size, flags))) { + if (!erts_bs_put_integer(EBS, Src, _size, flags)) { $BS_FAIL_INFO($Fail, BADARG, am_type, Src); } } @@ -705,19 +702,19 @@ i_bs_create_bin(Fail, Alloc, Live, Dst, N) { case BSC_UTF32: $BS_LOAD_FLAGS(p, flags); $BS_LOAD_FIXED_SIZE(p, Size); - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(Src, Size, flags))) { + if (!erts_bs_put_integer(EBS, Src, Size, flags)) { $BS_FAIL_INFO($Fail, BADARG, am_type, Src); } break; case BSC_UTF8: - if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(Src))) { + if (!erts_bs_put_utf8(EBS, Src)) { $BS_FAIL_INFO($Fail, BADARG, am_type, Src); } break; case BSC_UTF16: $BS_LOAD_FLAGS(p, flags); $BS_LOAD_SRC(p, Src); - if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(Src, flags))) { + if (!erts_bs_put_utf16(EBS, Src, flags)) { $BS_FAIL_INFO($Fail, BADARG, am_type, Src); } break; diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 75413225b9cf..0d7e6b1f5cf3 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -732,9 +732,10 @@ fmt_big(byte *buf, Uint num_bytes, Eterm val, Uint num_bits, Uint flags) } int -erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flags)) +erts_bs_put_integer(ErlBitsState *EBS, Eterm arg, Uint num_bits, unsigned flags) { - Uint bin_offset = erts_bin_offset; + byte* dst_bin = EBS->erts_current_bin; + Uint bin_offset = EBS->erts_bin_offset; Uint bit_offset; Uint b; byte *iptr; @@ -749,7 +750,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag /* * All bits are in the same byte. */ - iptr = erts_current_bin+BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); b = *iptr & (0xff << rbits); b |= (signed_val(arg) & ((1 << num_bits)-1)) << (rbits-num_bits); *iptr = b; @@ -757,7 +758,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag /* * More than one bit, starting at a byte boundary. */ - iptr = erts_current_bin + BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); fmt_small(iptr, NBYTES(num_bits), arg, num_bits, flags); } else if (flags & BSF_LITTLE) { /* @@ -771,7 +772,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag Uint count = (num_bits - rbits) / 8; Uint bits, bits1; - iptr = erts_current_bin+BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); if (BIT_OFFSET(num_bits) == 0) { bits = val; @@ -830,7 +831,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag */ Uint shift_count = num_bits - rbits; Sint val = signed_val(arg); - iptr = erts_current_bin+BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); b = *iptr & (0xff << rbits); /* @@ -855,7 +856,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag * Big number, aligned on a byte boundary. We can format the * integer directly into the binary. */ - fmt_big(erts_current_bin+BYTE_OFFSET(bin_offset), + fmt_big(dst_bin + BYTE_OFFSET(bin_offset), NBYTES(num_bits), arg, num_bits, flags); } else if (is_big(arg) && bit_offset + num_bits <= 8) { /* @@ -866,7 +867,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag ErtsDigit* dp = big_v(arg); Uint val = sign ? -*dp : *dp; - iptr = erts_current_bin+BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); b = *iptr & (0xff << rbits); b |= (val & ((1 << num_bits)-1)) << (rbits-num_bits); *iptr = b; @@ -888,7 +889,7 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag * Format the integer byte-aligned using the binary itself as * a temporary buffer. */ - iptr = erts_current_bin + BYTE_OFFSET(bin_offset); + iptr = dst_bin + BYTE_OFFSET(bin_offset); b = *iptr; fmt_big(iptr, NBYTES(num_bits), arg, num_bits, flags); @@ -919,15 +920,15 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag /* Not an integer. */ return 0; } - erts_bin_offset = bin_offset + num_bits; + EBS->erts_bin_offset = bin_offset + num_bits; return 1; } #if !defined(BEAMASM) int -erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg)) +erts_bs_put_utf8(ErlBitsState *EBS, Eterm arg) { - Uint bin_offset = erts_bin_offset; + Uint bin_offset = EBS->erts_bin_offset; Uint bit_offset; Uint num_bits; byte tmp_buf[4]; @@ -944,7 +945,7 @@ erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg)) if ((bit_offset = BIT_OFFSET(bin_offset)) == 0) { /* We can write directly into the destination binary. */ - dst = erts_current_bin+BYTE_OFFSET(bin_offset); + dst = EBS->erts_current_bin + BYTE_OFFSET(bin_offset); } else { /* Unaligned destination binary. Must use a temporary buffer. */ dst = tmp_buf; @@ -975,19 +976,19 @@ erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg)) } if (bin_offset != 0) { - erts_copy_bits(dst, 0, 1, erts_current_bin, bin_offset, 1, num_bits); + erts_copy_bits(dst, 0, 1, EBS->erts_current_bin, bin_offset, 1, num_bits); } - erts_bin_offset += num_bits; + EBS->erts_bin_offset += num_bits; return 1; } #endif int -erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags)) +erts_bs_put_utf16(ErlBitsState *EBS, Eterm arg, Uint flags) { - Uint bin_offset = erts_bin_offset; + Uint bin_offset = EBS->erts_bin_offset; Uint bit_offset; Uint num_bits; byte tmp_buf[4]; @@ -1004,7 +1005,7 @@ erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags)) if ((bit_offset = BIT_OFFSET(bin_offset)) == 0) { /* We can write directly into the destination binary. */ - dst = erts_current_bin+BYTE_OFFSET(bin_offset); + dst = EBS->erts_current_bin + BYTE_OFFSET(bin_offset); } else { /* Unaligned destination binary. Must use a temporary buffer. */ dst = tmp_buf; @@ -1040,17 +1041,16 @@ erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags)) } if (bin_offset != 0) { - erts_copy_bits(dst, 0, 1, erts_current_bin, bin_offset, 1, num_bits); + erts_copy_bits(dst, 0, 1, EBS->erts_current_bin, bin_offset, 1, num_bits); } - erts_bin_offset += num_bits; + EBS->erts_bin_offset += num_bits; return 1; } int -erts_new_bs_put_binary(Process *c_p, Eterm arg, Uint num_bits) +erts_bs_put_binary(ErlBitsState *EBS, Process *c_p, Eterm arg, Uint num_bits) { - ERL_BITS_DEFINE_STATEP(c_p); Uint offset, size; byte *base; @@ -1066,18 +1066,17 @@ erts_new_bs_put_binary(Process *c_p, Eterm arg, Uint num_bits) return 0; } - copy_binary_to_buffer(erts_current_bin, erts_bin_offset, + copy_binary_to_buffer(EBS->erts_current_bin, EBS->erts_bin_offset, base, offset, num_bits); - erts_bin_offset += num_bits; + EBS->erts_bin_offset += num_bits; BUMP_REDS(c_p, num_bits / BITS_PER_REDUCTION); return 1; } int -erts_new_bs_put_binary_all(Process *c_p, Eterm arg, Uint unit) +erts_bs_put_binary_all(ErlBitsState *EBS, Process *c_p, Eterm arg, Uint unit) { - ERL_BITS_DEFINE_STATEP(c_p); Uint offset, size; byte *base; @@ -1091,9 +1090,9 @@ erts_new_bs_put_binary_all(Process *c_p, Eterm arg, Uint unit) return 0; } - copy_binary_to_buffer(erts_current_bin, erts_bin_offset, + copy_binary_to_buffer(EBS->erts_current_bin, EBS->erts_bin_offset, base, offset, size); - erts_bin_offset += size; + EBS->erts_bin_offset += size; BUMP_REDS(c_p, size / BITS_PER_REDUCTION); return 1; @@ -1106,11 +1105,9 @@ erts_new_bs_put_binary_all(Process *c_p, Eterm arg, Uint unit) * and sets c_p-fvalue to 'type', 'no_float', or 'invalid'. */ Eterm -erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) +erts_bs_put_float(ErlBitsState *EBS, Process *c_p, Eterm arg, Uint num_bits, int flags) { - ERL_BITS_DEFINE_STATEP(c_p); - - if (BIT_OFFSET(erts_bin_offset) == 0) { + if (BIT_OFFSET(EBS->erts_bin_offset) == 0) { Uint32 a; Uint32 b; @@ -1222,7 +1219,7 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) } if (BIT_IS_MACHINE_ENDIAN(flags)) { - byte* t = erts_current_bin+BYTE_OFFSET(erts_bin_offset); + byte* t = EBS->erts_current_bin + BYTE_OFFSET(EBS->erts_bin_offset); #ifdef WORDS_BIGENDIAN if (num_bits == 16) { t[0] = a >> 8; @@ -1255,7 +1252,9 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) } #endif } else { - byte* t = erts_current_bin+BYTE_OFFSET(erts_bin_offset) + NBYTES(num_bits); + byte* t = EBS->erts_current_bin + + BYTE_OFFSET(EBS->erts_bin_offset) + + NBYTES(num_bits); #ifdef WORDS_BIGENDIAN if (num_bits == 16) { t[-1] = a >> 8; @@ -1385,27 +1384,32 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) } if (BIT_IS_MACHINE_ENDIAN(flags)) { erts_copy_bits(bptr, 0, 1, - erts_current_bin, - erts_bin_offset, 1, num_bits); + EBS->erts_current_bin, + EBS->erts_bin_offset, 1, num_bits); } else { erts_copy_bits(bptr+NBYTES(num_bits)-1, 0, -1, - erts_current_bin, erts_bin_offset, 1, + EBS->erts_current_bin, EBS->erts_bin_offset, 1, num_bits); } } - erts_bin_offset += num_bits; + EBS->erts_bin_offset += num_bits; return THE_NON_VALUE; } -void -erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes)) +void +erts_bs_put_string(ErlBitsState* EBS, byte* iptr, Uint num_bytes) { - if (BIT_OFFSET(erts_bin_offset) != 0) { - erts_copy_bits(iptr, 0, 1, erts_current_bin, erts_bin_offset, 1, num_bytes*8); + byte* dst_bin = EBS->erts_current_bin; + Uint dst_offset = EBS->erts_bin_offset; + + EBS->erts_bin_offset = dst_offset + num_bytes * 8; + if (BIT_OFFSET(dst_offset) != 0) { + erts_copy_bits(iptr, 0, 1, + dst_bin, dst_offset, 1, + num_bytes*8); } else { - sys_memcpy(erts_current_bin+BYTE_OFFSET(erts_bin_offset), iptr, num_bytes); + sys_memcpy(dst_bin + BYTE_OFFSET(dst_offset), iptr, num_bytes); } - erts_bin_offset += num_bytes*8; } static ERTS_INLINE @@ -1488,8 +1492,9 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, BinRef* br; Binary* binp; Uint heap_need; + Uint position; Uint used_size_in_bits; - ERL_BITS_DEFINE_STATEP(c_p); + ErlBitsState* EBS = ERL_BITS_EBS_FROM_REG(reg); /* * Check the binary argument. @@ -1528,10 +1533,10 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, * OK, the binary is writable. */ ASSERT(sb->start == 0); - erts_bin_offset = sb->end; + EBS->erts_bin_offset = position = sb->end; if (unit > 1) { - if ((unit == 8 && (erts_bin_offset & 7) != 0) || - (unit != 8 && (erts_bin_offset % unit) != 0)) { + if ((unit == 8 && (position & 7) != 0) || + (unit != 8 && (position % unit) != 0)) { c_p->fvalue = am_unit; goto badarg; } @@ -1548,13 +1553,13 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, return bin; } - if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + if ((ERTS_UINT_MAX - build_size_in_bits) < position) { c_p->fvalue = am_size; c_p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } - used_size_in_bits = erts_bin_offset + build_size_in_bits; + used_size_in_bits = position + build_size_in_bits; /* Make sure that no one else can append to the incoming bitstring. */ erl_sub_bits_clear_writable(sb); @@ -1569,11 +1574,11 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, binp = erts_bin_realloc(binp, new_size); br->val = binp; - BUMP_REDS(c_p, erts_bin_offset / BITS_PER_REDUCTION); + BUMP_REDS(c_p, position / BITS_PER_REDUCTION); } binp->intern.apparent_size = NBYTES(used_size_in_bits); - erts_current_bin = (byte*)binp->orig_bytes; + EBS->erts_current_bin = (byte*)binp->orig_bytes; /* Allocate heap space and build a new sub binary. */ reg[live] = sb->orig; @@ -1592,7 +1597,7 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, erl_sub_bits_init(sb, ERL_SUB_BITS_FLAGS_WRITABLE, reg[live], - erts_current_bin, + EBS->erts_current_bin, 0, used_size_in_bits); @@ -1656,10 +1661,10 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, &br, &sb); - erts_current_bin = (byte*)(br->val)->orig_bytes; - erts_bin_offset = src_size; + EBS->erts_current_bin = (byte*)(br->val)->orig_bytes; + EBS->erts_bin_offset = src_size; - copy_binary_to_buffer(erts_current_bin, + copy_binary_to_buffer(EBS->erts_current_bin, 0, src_bytes, src_offset, @@ -1671,15 +1676,14 @@ erts_bs_append_checked(Process* c_p, Eterm* reg, Uint live, } Eterm -erts_bs_private_append_checked(Process* p, Eterm bin, Uint build_size_in_bits, Uint unit) +erts_bs_private_append_checked(ErlBitsState* EBS, Process* p, + Eterm bin, Uint build_size_in_bits) { - Uint new_position, new_size, used_size; + Uint old_position, new_position, used_size; Binary *refc_binary; ErlSubBits *sb; BinRef *br; - ERL_BITS_DEFINE_STATEP(p); - sb = (ErlSubBits*)bitstring_val(bin); ASSERT(sb->thing_word == HEADER_SUB_BITS); @@ -1688,21 +1692,24 @@ erts_bs_private_append_checked(Process* p, Eterm bin, Uint build_size_in_bits, U /* Calculate new size in bits. */ ASSERT(sb->start == 0); - erts_bin_offset = sb->end; + EBS->erts_bin_offset = old_position = sb->end; - if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { +#ifdef BEAMASM + ASSERT(ERTS_UINT_MAX - build_size_in_bits >= old_position); +#else + if (ERTS_UINT_MAX - build_size_in_bits < old_position) { p->fvalue = am_size; p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } +#endif refc_binary = br->val; - new_position = erts_bin_offset + build_size_in_bits; - update_wb_overhead(p, br, sb->end, new_position); - + new_position = old_position + build_size_in_bits; used_size = NBYTES(new_position); - new_size = GROW_PROC_BIN_SIZE(used_size); + + update_wb_overhead(p, br, old_position, new_position); if (refc_binary->intern.flags & BIN_FLAG_WRITABLE) { /* This is the normal case - the binary is writable. There are no other @@ -1711,10 +1718,11 @@ erts_bs_private_append_checked(Process* p, Eterm bin, Uint build_size_in_bits, U ASSERT(erl_sub_bits_is_writable(sb)); ASSERT(erts_refc_read(&refc_binary->intern.refc, 1) == 1); if (refc_binary->orig_size < used_size) { + Uint new_size = GROW_PROC_BIN_SIZE(used_size); refc_binary = erts_bin_realloc(refc_binary, new_size); br->val = refc_binary; - BUMP_REDS(p, erts_bin_offset / BITS_PER_REDUCTION); + BUMP_REDS(p, EBS->erts_bin_offset / BITS_PER_REDUCTION); } ASSERT(sb->start == 0); @@ -1730,10 +1738,11 @@ erts_bs_private_append_checked(Process* p, Eterm bin, Uint build_size_in_bits, U * binary and make a copy of the data. * * We'll also make a new BinRef as the old one may have been moved from - * the `wrt_bins` list to the regular `off_heap` list by the GC. To + * the `wrt_bins` list to the regular `off_heap` list by the GC. * To move it back would mean traversing the `off_heap` list from the * start, so we'll create a new BinRef instead for this (hopefully) * rare case. */ + Uint new_size = GROW_PROC_BIN_SIZE(used_size); Binary *new_binary = erts_bin_nrml_alloc(new_size); Eterm *hp = HeapFragOnlyAlloc(p, ERL_REFC_BITS_SIZE); @@ -1749,12 +1758,12 @@ erts_bs_private_append_checked(Process* p, Eterm bin, Uint build_size_in_bits, U refc_binary->orig_bytes, MIN(refc_binary->orig_size, new_size)); - BUMP_REDS(p, erts_bin_offset / BITS_PER_REDUCTION); + BUMP_REDS(p, EBS->erts_bin_offset / BITS_PER_REDUCTION); refc_binary = new_binary; } ASSERT(refc_binary->intern.flags & BIN_FLAG_WRITABLE); - erts_current_bin = (byte*)&refc_binary->orig_bytes[0]; + EBS->erts_current_bin = (byte*)&refc_binary->orig_bytes[0]; return make_bitstring(sb); } diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 619f6157e4ec..0700fb01f053 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -168,43 +168,28 @@ struct erl_bits_state { /* * Pointer to the beginning of the current binary. */ - byte* erts_current_bin_; + byte* erts_current_bin; /* * Offset in bits into the current binary. */ - Uint erts_bin_offset_; + Uint erts_bin_offset; }; +typedef struct erl_bits_state ErlBitsState; + /* - * Reentrant API with the state passed as a parameter. - * (Except when the current Process* already is a parameter.) + * The bit syntax construction state resides in the current process's + * schduler data. The following macro retrieves the pointer to that + * state given a pointer to the X register array. */ -/* the state resides in the current process' scheduler data */ -#define ERL_BITS_DECLARE_STATEP struct erl_bits_state *EBS - -#define ERL_BITS_RELOAD_STATEP(P) \ - do { \ - EBS = &erts_proc_sched_data((P))->registers->aux_regs.d.erl_bits_state; \ - } while(0) - -#define ERL_BITS_DEFINE_STATEP(P) \ - struct erl_bits_state *EBS = \ - &erts_proc_sched_data((P))->registers->aux_regs.d.erl_bits_state - -#define ErlBitsState (*EBS) - -#define ERL_BITS_PROTO_0 struct erl_bits_state *EBS -#define ERL_BITS_PROTO_1(PARM1) struct erl_bits_state *EBS, PARM1 -#define ERL_BITS_PROTO_2(PARM1,PARM2) struct erl_bits_state *EBS, PARM1, PARM2 -#define ERL_BITS_PROTO_3(PARM1,PARM2,PARM3) struct erl_bits_state *EBS, PARM1, PARM2, PARM3 -#define ERL_BITS_ARGS_0 EBS -#define ERL_BITS_ARGS_1(ARG1) EBS, ARG1 -#define ERL_BITS_ARGS_2(ARG1,ARG2) EBS, ARG1, ARG2 -#define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3) EBS, ARG1, ARG2, ARG3 -#define erts_bin_offset (ErlBitsState.erts_bin_offset_) -#define erts_current_bin (ErlBitsState.erts_current_bin_) +#define ERL_BITS_EBS_FROM_REG(Reg) \ + ((ErlBitsState *) ((char *)(Reg) + \ + (offsetof(ErtsSchedulerRegisters, \ + aux_regs.d.erl_bits_state) - \ + offsetof(ErtsSchedulerRegisters, \ + x_reg_array.d)))) /* * Return number of Eterm words needed for allocation with HAlloc(), @@ -231,22 +216,25 @@ Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlSubBits Eterm erts_bs_get_binary_all_2(Process *p, ErlSubBits* sb); /* Binary construction, new instruction set. */ -int erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm Integer, Uint num_bits, unsigned flags)); +int erts_bs_put_integer(ErlBitsState *EBS, Eterm Integer, Uint num_bits, + unsigned flags); #if !defined(BEAMASM) -int erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm Integer)); +int erts_bs_put_utf8(ErlBitsState *EBS, Eterm Integer); #endif -int erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm Integer, Uint flags)); -int erts_new_bs_put_binary(Process *c_p, Eterm Bin, Uint num_bits); -int erts_new_bs_put_binary_all(Process *c_p, Eterm Bin, Uint unit); -Eterm erts_new_bs_put_float(Process *c_p, Eterm Float, Uint num_bits, int flags); -void erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes)); +int erts_bs_put_utf16(ErlBitsState *EBS, Eterm Integer, Uint flags); +int erts_bs_put_binary(ErlBitsState *EBS, Process *c_p, Eterm Bin, Uint num_bits); +int erts_bs_put_binary_all(ErlBitsState* EBS, Process *c_p, Eterm Bin, Uint unit); +Eterm erts_bs_put_float(ErlBitsState *EBS, Process *c_p, Eterm Float, + Uint num_bits, int flags); +void erts_bs_put_string(ErlBitsState *EBS, byte* iptr, Uint num_bytes); Uint32 erts_bs_get_unaligned_uint32(ErlSubBits* sb); Eterm erts_bs_get_utf8(ErlSubBits* sb); Eterm erts_bs_get_utf16(ErlSubBits* sb, Uint flags); Eterm erts_bs_append_checked(Process* p, Eterm* reg, Uint live, Uint size, Uint extra_words, Uint unit); -Eterm erts_bs_private_append_checked(Process* p, Eterm bin, Uint size, Uint unit); +Eterm erts_bs_private_append_checked(ErlBitsState* EBS, Process* p, + Eterm bin, Uint size); Eterm erts_bs_init_writable(Process* p, Eterm sz); /* ************************************************************************* */ diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp index f6a84591c998..e0080466c47f 100644 --- a/erts/emulator/beam/jit/arm/instr_bs.cpp +++ b/erts/emulator/beam/jit/arm/instr_bs.cpp @@ -1199,7 +1199,7 @@ void BeamModuleAssembler::update_bin_state(a64::Gp bin_offset, Sint size, a64::Gp size_reg) { int cur_bin_offset = offsetof(ErtsSchedulerRegisters, - aux_regs.d.erl_bits_state.erts_current_bin_); + aux_regs.d.erl_bits_state.erts_current_bin); arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset); arm::Mem mem_bin_offset = arm::Mem(scheduler_registers, cur_bin_offset + sizeof(Eterm)); @@ -1207,8 +1207,8 @@ void BeamModuleAssembler::update_bin_state(a64::Gp bin_offset, if (bit_offset % 8 != 0) { /* The bit offset is unknown or not byte-aligned. */ ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state, - erts_current_bin_, - erts_bin_offset_); + erts_current_bin, + erts_bin_offset); a.ldp(TMP2, bin_offset, mem_bin_base); if (size_reg.isValid()) { @@ -2021,14 +2021,14 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, BscSegment seg = segments[0]; comment("private append to binary"); ASSERT(Alloc.get() == 0); - mov_arg(ARG2, seg.src); + load_erl_bits_state(ARG1); + a.mov(ARG2, c_p); + mov_arg(ARG3, seg.src); if (sizeReg.isValid()) { - a.mov(ARG3, sizeReg); + a.mov(ARG4, sizeReg); } else { - mov_imm(ARG3, num_bits); + mov_imm(ARG4, num_bits); } - a.mov(ARG4, seg.unit); - a.mov(ARG1, c_p); emit_enter_runtime(Live.get()); runtime_call<4>(erts_bs_private_append_checked); emit_leave_runtime(Live.get()); @@ -2036,7 +2036,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } else if (estimated_num_bits <= ERL_ONHEAP_BITS_LIMIT) { static constexpr auto cur_bin_offset = offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + - offsetof(struct erl_bits_state, erts_current_bin_); + offsetof(struct erl_bits_state, erts_current_bin); Uint need; arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset); @@ -2104,8 +2104,8 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, /* Initialize the erl_bin_state struct. */ ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state, - erts_current_bin_, - erts_bin_offset_); + erts_current_bin, + erts_bin_offset); a.stp(HTOP, ZERO, mem_bin_base); /* Update HTOP. */ @@ -2158,11 +2158,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, comment("construct a binary segment"); if (seg.effectiveSize >= 0) { /* The segment has a literal size. */ - mov_imm(ARG3, seg.effectiveSize); - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); + load_erl_bits_state(ARG1); + a.mov(ARG2, c_p); + mov_arg(ARG3, seg.src); + mov_imm(ARG4, seg.effectiveSize); emit_enter_runtime(Live.get()); - runtime_call<3>(erts_new_bs_put_binary); + runtime_call<4>(erts_bs_put_binary); emit_leave_runtime(Live.get()); error_info = beam_jit_update_bsc_reason_info(seg.error_info, BSC_REASON_BADARG, @@ -2172,12 +2173,13 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, seg.size.as().get() == am_all) { /* Include the entire binary/bitstring in the * resulting binary. */ - a.mov(ARG3, seg.unit); - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); + load_erl_bits_state(ARG1); + a.mov(ARG2, c_p); + mov_arg(ARG3, seg.src); + mov_imm(ARG4, seg.unit); emit_enter_runtime(Live.get()); - runtime_call<3>(erts_new_bs_put_binary_all); + runtime_call<4>(erts_bs_put_binary_all); emit_leave_runtime(Live.get()); error_info = beam_jit_update_bsc_reason_info(seg.error_info, @@ -2195,17 +2197,18 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, * the value is a non-negative small in the * appropriate range. Multiply the size with the * unit. */ - auto r = load_source(seg.size, ARG3); - a.asr(ARG3, r.reg, imm(_TAG_IMMED1_SIZE)); + auto r = load_source(seg.size, ARG4); + a.asr(ARG4, r.reg, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(TMP1, seg.unit); - a.mul(ARG3, ARG3, TMP1); + a.mul(ARG4, ARG4, TMP1); } - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); + load_erl_bits_state(ARG1); + a.mov(ARG2, c_p); + mov_arg(ARG3, seg.src); emit_enter_runtime(Live.get()); - runtime_call<3>(erts_new_bs_put_binary); + runtime_call<4>(erts_bs_put_binary); emit_leave_runtime(Live.get()); error_info = beam_jit_update_bsc_reason_info(seg.error_info, @@ -2225,21 +2228,22 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, case am_float: comment("construct float segment"); if (seg.effectiveSize >= 0) { - mov_imm(ARG3, seg.effectiveSize); + mov_imm(ARG4, seg.effectiveSize); } else { - auto r = load_source(seg.size, ARG3); - a.asr(ARG3, r.reg, imm(_TAG_IMMED1_SIZE)); + auto r = load_source(seg.size, ARG4); + a.asr(ARG4, r.reg, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(TMP1, seg.unit); - a.mul(ARG3, ARG3, TMP1); + a.mul(ARG4, ARG4, TMP1); } } - mov_arg(ARG2, seg.src); - mov_imm(ARG4, seg.flags); - a.mov(ARG1, c_p); + load_erl_bits_state(ARG1); + a.mov(ARG2, c_p); + mov_arg(ARG3, seg.src); + mov_imm(ARG5, seg.flags); emit_enter_runtime(Live.get()); - runtime_call<4>(erts_new_bs_put_float); + runtime_call<5>(erts_bs_put_float); emit_leave_runtime(Live.get()); if (Fail.get() == 0) { @@ -2490,12 +2494,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, } else { /* Call the helper function to fetch and store the * integer into the binary. */ + load_erl_bits_state(ARG1); mov_arg(ARG2, seg.src); mov_imm(ARG4, seg.flags); - load_erl_bits_state(ARG1); emit_enter_runtime(Live.get()); - runtime_call<4>(erts_new_bs_put_integer); + runtime_call<4>(erts_bs_put_integer); emit_leave_runtime(Live.get()); if (exact_type(seg.src)) { @@ -2522,12 +2526,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, comment("insert string"); ASSERT(seg.effectiveSize >= 0); - mov_imm(ARG3, seg.effectiveSize / 8); - mov_arg(ARG2, string_ptr); load_erl_bits_state(ARG1); + mov_arg(ARG2, string_ptr); + mov_imm(ARG3, seg.effectiveSize / 8); emit_enter_runtime(Live.get()); - runtime_call<3>(erts_new_bs_put_string); + runtime_call<3>(erts_bs_put_string); emit_leave_runtime(Live.get()); break; } @@ -2556,13 +2560,13 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, a.cbz(ARG1, resolve_label(error, disp1MB)); break; case am_utf32: + load_erl_bits_state(ARG1); mov_arg(ARG2, seg.src); mov_imm(ARG3, 4 * 8); a.mov(ARG4, seg.flags); - load_erl_bits_state(ARG1); emit_enter_runtime(Live.get()); - runtime_call<4>(erts_new_bs_put_integer); + runtime_call<4>(erts_bs_put_integer); emit_leave_runtime(Live.get()); if (Fail.get() == 0) { diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp index fde678606ffa..3b348db4b9ad 100644 --- a/erts/emulator/beam/jit/beam_jit_common.cpp +++ b/erts/emulator/beam/jit/beam_jit_common.cpp @@ -808,7 +808,7 @@ void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B) { Eterm beam_jit_bs_init_bits(Process *c_p, Eterm *reg, - ERL_BITS_DECLARE_STATEP, + ErlBitsState *EBS, Uint num_bits, Uint alloc, unsigned Live) { @@ -818,7 +818,7 @@ Eterm beam_jit_bs_init_bits(Process *c_p, alloc += ERL_REFC_BITS_SIZE; } - erts_bin_offset = 0; + EBS->erts_bin_offset = 0; if (num_bits <= ERL_ONHEAP_BITS_LIMIT) { ErlHeapBits *hb; @@ -830,7 +830,7 @@ Eterm beam_jit_bs_init_bits(Process *c_p, hb->thing_word = header_heap_bits(num_bits); hb->size = num_bits; - erts_current_bin = (byte *)hb->data; + EBS->erts_current_bin = (byte *)hb->data; return make_bitstring(hb); } else { const Uint num_bytes = NBYTES(num_bits); @@ -839,13 +839,13 @@ Eterm beam_jit_bs_init_bits(Process *c_p, test_bin_vheap(c_p, reg, num_bytes / sizeof(Eterm), alloc, Live); new_binary = erts_bin_nrml_alloc(num_bytes); - erts_current_bin = (byte *)new_binary->orig_bytes; + EBS->erts_current_bin = (byte *)new_binary->orig_bytes; return erts_wrap_refc_bitstring(&MSO(c_p).first, &MSO(c_p).overhead, &HEAP_TOP(c_p), new_binary, - erts_current_bin, + EBS->erts_current_bin, 0, num_bits); } diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp index 934821d8f532..3d34674153e0 100644 --- a/erts/emulator/beam/jit/beam_jit_common.hpp +++ b/erts/emulator/beam/jit/beam_jit_common.hpp @@ -600,7 +600,7 @@ void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size); void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B); Eterm beam_jit_bs_init_bits(Process *c_p, Eterm *reg, - ERL_BITS_DECLARE_STATEP, + ErlBitsState *EBS, Uint num_bits, Uint alloc, unsigned Live); diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index d3c9e1c050d6..b43351b871ba 100644 --- a/erts/emulator/beam/jit/x86/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -1262,10 +1262,10 @@ void BeamModuleAssembler::update_bin_state(x86::Gp bin_offset, const int x_reg_offset = offsetof(ErtsSchedulerRegisters, x_reg_array.d); const int cur_bin_base = offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + - offsetof(struct erl_bits_state, erts_current_bin_); + offsetof(struct erl_bits_state, erts_current_bin); const int cur_bin_offset = offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + - offsetof(struct erl_bits_state, erts_bin_offset_); + offsetof(struct erl_bits_state, erts_bin_offset); x86::Mem mem_bin_base = x86::Mem(registers, cur_bin_base - x_reg_offset, sizeof(UWord)); @@ -1822,10 +1822,10 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, offsetof(ErtsSchedulerRegisters, x_reg_array.d); const int cur_bin_base = offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + - offsetof(struct erl_bits_state, erts_current_bin_); + offsetof(struct erl_bits_state, erts_current_bin); const int cur_bin_offset = offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) + - offsetof(struct erl_bits_state, erts_bin_offset_); + offsetof(struct erl_bits_state, erts_bin_offset); x86::Mem mem_bin_base = x86::qword_ptr(registers, cur_bin_base - x_reg_offset); x86::Mem mem_bin_offset = @@ -2195,14 +2195,14 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("private append to binary"); ASSERT(Alloc.get() == 0); - mov_arg(ARG2, seg.src); + mov_arg(ARG3, seg.src); if (sizeReg.isValid()) { - a.mov(ARG3, sizeReg); + a.mov(ARG4, sizeReg); } else { - mov_imm(ARG3, num_bits); + mov_imm(ARG4, num_bits); } - a.mov(ARG4, seg.unit); - a.mov(ARG1, c_p); + a.mov(ARG2, c_p); + load_erl_bits_state(ARG1); runtime_call<4>(erts_bs_private_append_checked); /* There is no way the call can fail on a 64-bit architecture. */ a.mov(TMP_MEM1q, RET); @@ -2251,10 +2251,11 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, comment("construct a binary segment"); if (seg.effectiveSize >= 0) { /* The segment has a literal size. */ - mov_imm(ARG3, seg.effectiveSize); - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); - runtime_call<3>(erts_new_bs_put_binary); + mov_imm(ARG4, seg.effectiveSize); + mov_arg(ARG3, seg.src); + a.mov(ARG2, c_p); + load_erl_bits_state(ARG1); + runtime_call<4>(erts_bs_put_binary); error_info = beam_jit_update_bsc_reason_info(seg.error_info, BSC_REASON_BADARG, BSC_INFO_DEPENDS, @@ -2263,10 +2264,11 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, seg.size.as().get() == am_all) { /* Include the entire binary/bitstring in the * resulting binary. */ - a.mov(ARG3, seg.unit); - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); - runtime_call<3>(erts_new_bs_put_binary_all); + mov_imm(ARG4, seg.unit); + mov_arg(ARG3, seg.src); + a.mov(ARG2, c_p); + load_erl_bits_state(ARG1); + runtime_call<4>(erts_bs_put_binary_all); error_info = beam_jit_update_bsc_reason_info(seg.error_info, BSC_REASON_BADARG, BSC_INFO_UNIT, @@ -2282,16 +2284,17 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, * the value is a non-negative small in the * appropriate range. Multiply the size with the * unit. */ - mov_arg(ARG3, seg.size); - a.sar(ARG3, imm(_TAG_IMMED1_SIZE)); + mov_arg(ARG4, seg.size); + a.sar(ARG4, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(RET, seg.unit); - a.mul(ARG3); /* CLOBBERS RDX = ARG3! */ - a.mov(ARG3, RET); + a.mul(ARG4); /* CLOBBERS RDX = ARG3! */ + a.mov(ARG4, RET); } - mov_arg(ARG2, seg.src); - a.mov(ARG1, c_p); - runtime_call<3>(erts_new_bs_put_binary); + mov_arg(ARG3, seg.src); + a.mov(ARG2, c_p); + load_erl_bits_state(ARG1); + runtime_call<4>(erts_bs_put_binary); error_info = beam_jit_update_bsc_reason_info(seg.error_info, BSC_REASON_BADARG, BSC_INFO_DEPENDS, @@ -2311,20 +2314,21 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, runtime_entered = bs_maybe_enter_runtime(runtime_entered); comment("construct float segment"); if (seg.effectiveSize >= 0) { - mov_imm(ARG3, seg.effectiveSize); + mov_imm(ARG4, seg.effectiveSize); } else { - mov_arg(ARG3, seg.size); - a.sar(ARG3, imm(_TAG_IMMED1_SIZE)); + mov_arg(ARG4, seg.size); + a.sar(ARG4, imm(_TAG_IMMED1_SIZE)); if (seg.unit != 1) { mov_imm(RET, seg.unit); - a.mul(ARG3); /* CLOBBERS RDX = ARG3! */ - a.mov(ARG3, RET); + a.mul(ARG4); /* CLOBBERS RDX = ARG3! */ + a.mov(ARG4, RET); } } - mov_arg(ARG2, seg.src); - mov_imm(ARG4, seg.flags); - a.mov(ARG1, c_p); - runtime_call<4>(erts_new_bs_put_float); + mov_arg(ARG3, seg.src); + mov_imm(ARG5, seg.flags); + a.mov(ARG2, c_p); + load_erl_bits_state(ARG1); + runtime_call<5>(erts_bs_put_float); if (Fail.get() == 0) { mov_imm(ARG4, beam_jit_update_bsc_reason_info(seg.error_info, @@ -2597,7 +2601,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, mov_arg(ARG2, seg.src); mov_imm(ARG4, seg.flags); load_erl_bits_state(ARG1); - runtime_call<4>(erts_new_bs_put_integer); + runtime_call<4>(erts_bs_put_integer); if (exact_type(seg.src)) { comment("skipped test for success because construction " "can't fail"); @@ -2628,7 +2632,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, mov_imm(ARG3, seg.effectiveSize / 8); mov_arg(ARG2, string_ptr); load_erl_bits_state(ARG1); - runtime_call<3>(erts_new_bs_put_string); + runtime_call<3>(erts_bs_put_string); } break; case am_utf8: { runtime_entered = bs_maybe_enter_runtime(runtime_entered); @@ -2658,7 +2662,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail, mov_imm(ARG3, 4 * 8); a.mov(ARG4, seg.flags); load_erl_bits_state(ARG1); - runtime_call<4>(erts_new_bs_put_integer); + runtime_call<4>(erts_bs_put_integer); if (Fail.get() == 0) { mov_arg(ARG1, seg.src); mov_imm(ARG4,