compiler: Add support for debug information in BEAM files

The new `beam_debug_info` compiler option will insert `debug_line`
instructions in roughly the same places that `line_coverage` would
insert `executable_line` instructions, and it will maintain
information about which variables the BEAM registers contain at each
`debug_line` instruction. This information will be inserted into a
"DbgB" chunk in the BEAM file.

When a `debug_line` is executed, the current stack frame (if any) is
guaranteed to be fully initialized. The number of live X registers is
given as the second operand for the `debug_line` instruction (it is
guaranteed that there are no "holes").

Here is an example where the debug information translated to text has
been inserted as comments before the lines they apply to:

    sum(A, B, _Ignored) ->
	%% no stack frame; A in x0, B in x1, _Ignored in x2
	C = A + B,

	%% no stack frame; B in x1, C in x0
	io:format("~p\n", [C]),

	%% stack frame size is 1; C in y0
	D = 10 * C,

	%% stack frame size is 1; C in y0, D in x0
	{ok,D}.

Note that not all variables are available in the debug information.

For example, before the call to `io:format/2`, the sum of A and B have
overwritten the register that used to hold the value of A, and the value
for _Ignore was wiped out by the `+` operation.

The size of the current stack frame is also given at each `debug_line`
to be able to easly find the beginning of the previous stack frame.

erts/emulator/beam/emu/ops.tab

@@ -97,6 +97,8 @@ line I
 
 executable_line _Id _Line => _
 
+debug_line u u u => _
+
 # For the JIT, the init_yregs/1 instruction allows generation of better code.
 # For the BEAM interpreter, though, it will probably be more efficient to
 # translate all uses of init_yregs/1 back to the instructions that the compiler

erts/emulator/beam/jit/arm/ops.tab

@@ -88,6 +88,8 @@ line I
 
 executable_line I I
 
+debug_line u u u => _
+
 allocate t t
 allocate_heap t I t
 

erts/emulator/beam/jit/x86/ops.tab

@@ -88,6 +88,8 @@ line I
 
 executable_line I I
 
+debug_line u u u => _
+
 allocate t t
 allocate_heap t I t
 

lib/compiler/src/beam_asm.erl

@@ -27,11 +27,14 @@
 
 -export_type([fail/0,label/0,src/0,module_code/0,function_name/0]).
 
--import(lists, [append/1,duplicate/2,map/2,member/2,keymember/3,splitwith/2]).
+-import(lists, [append/1,duplicate/2,keymember/3,last/1,map/2,
+                member/2,splitwith/2]).
 
 -include("beam_opcodes.hrl").
 -include("beam_asm.hrl").
 
+-define(BEAM_DEBUG_INFO_VERSION, 0).
+
 %% Common types for describing operands for BEAM instructions.
 -type src() :: beam_reg() |
 	       {'literal',term()} |
@@ -60,23 +63,24 @@
 -define(BEAMFILE_EXECUTABLE_LINE, 1).
 -define(BEAMFILE_FORCE_LINE_COUNTERS, 2).
 
--spec module(module_code(), [{binary(), binary()}], [{atom(),term()}], [compile:option()]) ->
-                    {'ok',binary()}.
-
-module(Code, ExtraChunks, CompileInfo, CompilerOpts) ->
-    {ok,assemble(Code, ExtraChunks, CompileInfo, CompilerOpts)}.
+-spec module(module_code(), [{binary(), binary()}],
+             [{atom(),term()}], [compile:option()]) ->
+          {'ok',binary()}.
 
-assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, CompileInfo, CompilerOpts) ->
+module(Code0, ExtraChunks, CompileInfo, CompilerOpts) ->
+    {Mod,Exp0,Attr0,Asm0,NumLabels} = Code0,
     {1,Dict0} = beam_dict:atom(Mod, beam_dict:new()),
     {0,Dict1} = beam_dict:fname(atom_to_list(Mod) ++ ".erl", Dict0),
     {0,Dict2} = beam_dict:type(any, Dict1),
     Dict3 = reject_unsupported_versions(Dict2),
+
     NumFuncs = length(Asm0),
     {Asm,Attr} = on_load(Asm0, Attr0),
     Exp = sets:from_list(Exp0),
-    {Code,Dict} = assemble_1(Asm, Exp, Dict3, []),
-    build_file(Code, Attr, Dict, NumLabels, NumFuncs,
-               ExtraChunks, CompileInfo, CompilerOpts).
+    {Code,Dict} = assemble(Asm, Exp, Dict3, []),
+    Beam = build_file(Code, Attr, Dict, NumLabels, NumFuncs,
+                      ExtraChunks, CompileInfo, CompilerOpts),
+    {ok,Beam}.
 
 reject_unsupported_versions(Dict) ->
     %% Emit an instruction that was added in our lowest supported
@@ -106,16 +110,16 @@ insert_on_load_instruction(Is0, Entry) ->
 		  end, Is0),
     Bef ++ [El,on_load|Is].
 
-assemble_1([{function,Name,Arity,Entry,Asm}|T], Exp, Dict0, Acc) ->
+assemble([{function,Name,Arity,Entry,Asm}|T], Exp, Dict0, Acc) ->
     Dict1 = case sets:is_element({Name,Arity}, Exp) of
 		true ->
 		    beam_dict:export(Name, Arity, Entry, Dict0);
 		false ->
 		    beam_dict:local(Name, Arity, Entry, Dict0)
 	    end,
     {Code, Dict2} = assemble_function(Asm, Acc, Dict1),
-    assemble_1(T, Exp, Dict2, Code);
-assemble_1([], _Exp, Dict0, Acc) ->
+    assemble(T, Exp, Dict2, Code);
+assemble([], _Exp, Dict0, Acc) ->
     {IntCodeEnd,Dict1} = make_op(int_code_end, Dict0),
     {list_to_binary(lists:reverse(Acc, [IntCodeEnd])),Dict1}.
 
@@ -125,17 +129,23 @@ assemble_function([H|T], Acc, Dict0) ->
 assemble_function([], Code, Dict) ->
     {Code, Dict}.
 
-build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks0, CompileInfo, CompilerOpts) ->
+build_file(Code, Attr, Dict0, NumLabels, NumFuncs, ExtraChunks0,
+           CompileInfo, CompilerOpts) ->
     %% Create the code chunk.
 
     CodeChunk = chunk(<<"Code">>,
 		      <<16:32,
 		       (beam_opcodes:format_number()):32,
-		       (beam_dict:highest_opcode(Dict)):32,
+		       (beam_dict:highest_opcode(Dict0)):32,
 		       NumLabels:32,
 		       NumFuncs:32>>,
 		      Code),
 
+    %% Build the BEAM debug information chunk. It is important
+    %% to build it early, because it will add entries to the
+    %% atom and literal tables.
+    {ExtraChunks1,Dict} = build_beam_debug_info(ExtraChunks0, CompilerOpts, Dict0),
+
     %% Create the atom table chunk.
     AtomChunk = build_atom_table(CompilerOpts, Dict),
 
@@ -186,13 +196,14 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks0, CompileInfo, Com
                       TypeTab),
 
     %% Create the meta chunk
-    Meta = proplists:get_value(<<"Meta">>, ExtraChunks0, empty),
+    Meta = proplists:get_value(<<"Meta">>, ExtraChunks1, empty),
     MetaChunk = case Meta of
                     empty -> [];
                     Meta -> chunk(<<"Meta">>, Meta)
                 end,
+
     %% Remove Meta chunk from ExtraChunks since it is essential
-    ExtraChunks = ExtraChunks0 -- [{<<"Meta">>, Meta}],
+    ExtraChunks = ExtraChunks1 -- [{<<"Meta">>, Meta}],
 
     %% Create the attributes and compile info chunks.
 
@@ -381,6 +392,103 @@ filter_essentials([<<>>|T]) ->
     filter_essentials(T);
 filter_essentials([]) -> [].
 
+%%%
+%%% Build the BEAM debug information chunk.
+%%%
+
+build_beam_debug_info(ExtraChunks, CompilerOpts, Dict) ->
+    case member(beam_debug_info, CompilerOpts) of
+        true ->
+            build_beam_debug_info_1(ExtraChunks, Dict);
+        false ->
+            {ExtraChunks,Dict}
+    end.
+
+build_beam_debug_info_1(ExtraChunks0, Dict0) ->
+    DebugTab0 = beam_dict:debug_table(Dict0),
+    DebugTab1 = [{Index,Info} ||
+                    Index := Info <- maps:iterator(DebugTab0, ordered)],
+    DebugTab = build_bdi_fill_holes(DebugTab1),
+    NumVars = lists:sum([length(Vs) || {_,Vs} <- DebugTab]),
+    {Contents0,Dict} = build_bdi(DebugTab, Dict0),
+    NumItems = length(Contents0),
+    Contents1 = iolist_to_binary(Contents0),
+
+    0 = NumItems bsr 31,                        %Assertion.
+    0 = NumVars bsr 31,                         %Assertion.
+
+    Contents = <<?BEAM_DEBUG_INFO_VERSION:32,
+                 NumItems:32,
+                 NumVars:32,
+                 Contents1/binary>>,
+    ExtraChunks = [{~"DbgB",Contents}|ExtraChunks0],
+    {ExtraChunks,Dict}.
+
+build_bdi_fill_holes([]) ->
+    [];
+build_bdi_fill_holes([{_,Item}]) ->
+    [Item];
+build_bdi_fill_holes([{I0,Item}|[{I1,_}|_]=T]) ->
+    case I0 + 1 of
+        I1 ->
+            [Item|build_bdi_fill_holes(T)];
+        Next ->
+            NewPair = {Next,{none,[]}},
+            [Item|build_bdi_fill_holes([NewPair|T])]
+    end.
+
+build_bdi([{FrameSize0,Vars0}|Items], Dict0) ->
+    %% The debug information utilizes the encoding machinery for BEAM
+    %% instructions. The debug information for `debug_line`
+    %% instructions is translated to:
+    %%
+    %%    {call,FrameSize,{list,[VariableName,Where,...]}}
+    %%
+    %% Where:
+    %%
+    %%    FrameSize := 'none' | 0..1023
+    %%    VariableName := binary()
+    %%    Where := {x,0..1023} | {y,0..1023} | {literal,_} |
+    %%             {integer,_} | {atom,_} | {float,_} | nil
+    %%
+    %% The only reason the `call` instruction is used is because it
+    %% has two operands.
+    %%
+    %% The debug information in the following example:
+    %%
+    %%    {debug_line,[...],1,
+    %%       {4, [{'Args',[{y,3}]},
+    %%            {'Line',[{y,2}]},
+    %%            {'Live',[{x,0},{y,1}]}]},
+    %%       1}
+    %%
+    %% will be translated to the following instruction:
+    %%
+    %%     {call,4,{list,[{literal,<<"Args">>},{y,3},
+    %%                    {literal,<<"Line">>},{y,2},
+    %%                    {literal,<<"Live">>},{y,1}]}}
+    %%
+    %% Note that only one register is given for each variable. It
+    %% is always the last register listed.
+
+    FrameSize = case FrameSize0 of
+                    none -> nil;
+                    _ -> FrameSize0
+                end,
+    Vars1 = [[{literal,atom_to_binary(Name)},last(Regs)] ||
+                {Name,[_|_]=Regs} <:- Vars0],
+    Vars = append(Vars1),
+    Instr0 = {call,FrameSize,{list,Vars}},
+    {Instr,Dict1} = make_op(Instr0, Dict0),
+    {Tail,Dict2} = build_bdi(Items, Dict1),
+    {[Instr|Tail],Dict2};
+build_bdi([], Dict) ->
+    {[],Dict}.
+
+%%%
+%%% Functions for assembling BEAM instruction.
+%%%
+
 bif_type(fnegate, 1) -> {op,fnegate};
 bif_type(fadd, 2)   -> {op,fadd};
 bif_type(fsub, 2)   -> {op,fsub};
@@ -397,6 +505,10 @@ make_op({line=Op,Location}, Dict0) ->
 make_op({executable_line=Op,Location,Index}, Dict0) ->
     {LocationIndex,Dict} = beam_dict:line(Location, Dict0, Op),
     encode_op(executable_line, [LocationIndex,Index], Dict);
+make_op({debug_line=Op,Location,Index,Live,DebugInfo}, Dict0) ->
+    {LocationIndex,Dict1} = beam_dict:line(Location, Dict0, Op),
+    Dict = beam_dict:debug_info(Index, DebugInfo, Dict1),
+    encode_op(debug_line, [LocationIndex,Index,Live], Dict);
 make_op({bif, Bif, {f,_}, [], Dest}, Dict) ->
     %% BIFs without arguments cannot fail.
     encode_op(bif0, [{extfunc, erlang, Bif, 0}, Dest], Dict);

lib/compiler/src/beam_block.erl

@@ -25,7 +25,7 @@
 -include("beam_asm.hrl").
 
 -export([module/2]).
--import(lists, [keysort/2,member/2,reverse/1,reverse/2,
+-import(lists, [flatmap/2,keysort/2,member/2,reverse/1,reverse/2,
                 splitwith/2,usort/1]).
 
 -spec module(beam_utils:module_code(), [compile:option()]) ->
@@ -172,12 +172,17 @@ collect({put_map,{f,0},Op,S,D,R,{list,Puts}}) ->
 collect({fmove,S,D})         -> {set,[D],[S],fmove};
 collect({fconv,S,D})         -> {set,[D],[S],fconv};
 collect({executable_line,_,_}=Line) -> {set,[],[],Line};
+collect({debug_line,_,_,_,_}=Line) -> collect_debug_line(Line);
 collect({swap,D1,D2})        ->
     Regs = [D1,D2],
     {set,Regs,Regs,swap};
 collect({make_fun3,F,I,U,D,{list,Ss}}) -> {set,[D],Ss,{make_fun3,F,I,U}};
 collect(_)                   -> error.
 
+collect_debug_line({debug_line,_Loc,_Index,_Live,{_,Vars}}=I) ->
+    Ss = flatmap(fun({_Name,Regs}) -> Regs end, Vars),
+    {set,[],Ss,I}.
+
 %% embed_lines([Instruction]) -> [Instruction]
 %%  Combine blocks that would be split by line/1 instructions.
 %%  Also move a line instruction before a block into the block,