diff --git a/Directory.Build.props b/Directory.Build.props index f826a619d..e635050c9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -61,7 +61,7 @@ $(MSBuildProjectName.Contains('Test')) - + diff --git a/Samples/Kaleidoscope/Chapter2/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter2/CodeGenerator.cs index 75b4decce..aaa33257d 100644 --- a/Samples/Kaleidoscope/Chapter2/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter2/CodeGenerator.cs @@ -4,29 +4,24 @@ using System; using Antlr4.Runtime; -using Antlr4.Runtime.Misc; using Antlr4.Runtime.Tree; using Kaleidoscope.Grammar; using Kaleidoscope.Runtime; -using static Kaleidoscope.Grammar.KaleidoscopeParser; - namespace Kaleidoscope { /// Static extension methods to perform LLVM IR Code generation from the Kaleidoscope AST /// - /// This doesn't actually generate any code. The only thing it does is to record any user defined operators - /// in the so that subsequent parsing takes the operator precedence into - /// account. (If the language didn't support user defined precedence this would not be needed at all) + /// This doesn't actually generate any code. It simply satisfies the generator contract with an + /// effective NOP /// internal sealed class CodeGenerator : KaleidoscopeBaseVisitor , IDisposable , IKaleidoscopeCodeGenerator { - public CodeGenerator( DynamicRuntimeState globalState ) + public CodeGenerator( ) { - RuntimeState = globalState; } public void Dispose( ) @@ -37,34 +32,12 @@ public int Generate( Parser parser, IParseTree tree, DiagnosticRepresentations a { if( parser.NumberOfSyntaxErrors > 0 ) { - return 0; - } - - return Visit( tree ); - } - - public override int VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context ) - { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - - return 0; - } - - public override int VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context ) - { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); + return 1; } return 0; } protected override int DefaultResult => 0; - - private readonly DynamicRuntimeState RuntimeState; } } diff --git a/Samples/Kaleidoscope/Chapter2/Program.cs b/Samples/Kaleidoscope/Chapter2/Program.cs index 5152e398d..d99990fca 100644 --- a/Samples/Kaleidoscope/Chapter2/Program.cs +++ b/Samples/Kaleidoscope/Chapter2/Program.cs @@ -23,8 +23,8 @@ public static void Main( string[ ] args ) // Using language level that includes the complete set // of language features to allow exploring and verifying // - var parser = new ReplParserStack( LanguageLevel.MutableVariables ); - using( var generator = new CodeGenerator( parser.GlobalState ) ) + var parser = new ParserStack( LanguageLevel.MutableVariables ); + using( var generator = new CodeGenerator( ) ) { Console.WriteLine( "LLVM Kaleidoscope Syntax Viewer - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter3/Program.cs b/Samples/Kaleidoscope/Chapter3/Program.cs index 7022e3ff7..fc2a0cde1 100644 --- a/Samples/Kaleidoscope/Chapter3/Program.cs +++ b/Samples/Kaleidoscope/Chapter3/Program.cs @@ -39,7 +39,7 @@ public static void Main( string[ ] args ) RegisterNative( ); // - var parser = new ReplParserStack( LanguageLevel.SimpleExpressions ); + var parser = new ParserStack( LanguageLevel.SimpleExpressions ); using( var generator = new CodeGenerator( parser.GlobalState ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Interpreter - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter4/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter4/CodeGenerator.cs index f6938838d..85421fc58 100644 --- a/Samples/Kaleidoscope/Chapter4/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter4/CodeGenerator.cs @@ -51,6 +51,7 @@ public void Dispose( ) Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -60,17 +61,21 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return Visit( tree ); } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { return Context.CreateConstant( context.Value ); } + // + // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) { string varName = context.Name; @@ -81,6 +86,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte return value; } + // public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context ) { @@ -94,6 +100,7 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { return context.Signature.Accept( this ); @@ -103,6 +110,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -129,6 +137,7 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte } // + // public override Value VisitExpression( [NotNull] ExpressionContext context ) { // Expression: PrimaryExpression (op expression)* @@ -141,9 +150,11 @@ public override Value VisitExpression( [NotNull] ExpressionContext context ) return lhs; } + // protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { var rhs = rightTree.Accept( this ); @@ -185,6 +196,7 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ throw new CodeGeneratorException( $"Invalid binary operator {op.Token.Text}" ); } } + // // private void InitializeModuleAndPassManager( ) diff --git a/Samples/Kaleidoscope/Chapter4/Program.cs b/Samples/Kaleidoscope/Chapter4/Program.cs index 0c87b1cbb..2226d10c1 100644 --- a/Samples/Kaleidoscope/Chapter4/Program.cs +++ b/Samples/Kaleidoscope/Chapter4/Program.cs @@ -39,7 +39,7 @@ public static void Main( string[ ] args ) RegisterNative( ); // - var parser = new ReplParserStack( LanguageLevel.SimpleExpressions ); + var parser = new ParserStack( LanguageLevel.SimpleExpressions ); using( var generator = new CodeGenerator( parser.GlobalState ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Interpreter - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter5/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter5/CodeGenerator.cs index 54c3f1c33..3c5e99fda 100644 --- a/Samples/Kaleidoscope/Chapter5/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter5/CodeGenerator.cs @@ -51,6 +51,7 @@ public void Dispose( ) Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -60,17 +61,21 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return Visit( tree ); } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { return Context.CreateConstant( context.Value ); } + // + // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) { string varName = context.Name; @@ -81,6 +86,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte return value; } + // public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context ) { @@ -94,6 +100,7 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { return context.Signature.Accept( this ); @@ -103,6 +110,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -129,6 +137,7 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte } // + // public override Value VisitExpression( [NotNull] ExpressionContext context ) { // Expression: PrimaryExpression (op expression)* @@ -141,6 +150,7 @@ public override Value VisitExpression( [NotNull] ExpressionContext context ) return lhs; } + // // public override Value VisitConditionalExpression( [NotNull] ConditionalExpressionContext context ) @@ -158,7 +168,7 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio var thenBlock = Context.CreateBasicBlock( "then", function ); var elseBlock = Context.CreateBasicBlock( "else" ); - var phiMergeBlock = Context.CreateBasicBlock( "ifcont" ); + var continueBlock = Context.CreateBasicBlock( "ifcont" ); InstructionBuilder.Branch( condBool, thenBlock, elseBlock ); // generate then block @@ -169,7 +179,7 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio return null; } - InstructionBuilder.Branch( phiMergeBlock ); + InstructionBuilder.Branch( continueBlock ); // capture the insert in case generating else adds new blocks thenBlock = InstructionBuilder.InsertBlock; @@ -183,12 +193,12 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio return null; } - InstructionBuilder.Branch( phiMergeBlock ); + InstructionBuilder.Branch( continueBlock ); elseBlock = InstructionBuilder.InsertBlock; - // generate PHI merge block - function.BasicBlocks.Add( phiMergeBlock ); - InstructionBuilder.PositionAtEnd( phiMergeBlock ); + // generate continue block + function.BasicBlocks.Add( continueBlock ); + InstructionBuilder.PositionAtEnd( continueBlock ); var phiNode = InstructionBuilder.PhiNode( function.Context.DoubleType ) .RegisterName( "iftmp" ); @@ -294,6 +304,7 @@ public override Value VisitForExpression( [NotNull] ForExpressionContext context protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { var rhs = rightTree.Accept( this ); @@ -335,6 +346,7 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ throw new CodeGeneratorException( $"Invalid binary operator {op.Token.Text}" ); } } + // // private void InitializeModuleAndPassManager( ) diff --git a/Samples/Kaleidoscope/Chapter5/Program.cs b/Samples/Kaleidoscope/Chapter5/Program.cs index bac66d022..881067abf 100644 --- a/Samples/Kaleidoscope/Chapter5/Program.cs +++ b/Samples/Kaleidoscope/Chapter5/Program.cs @@ -39,7 +39,7 @@ public static void Main( string[ ] args ) RegisterNative( ); // - var parser = new ReplParserStack( LanguageLevel.ControlFlow ); + var parser = new ParserStack( LanguageLevel.ControlFlow ); using( var generator = new CodeGenerator( parser.GlobalState ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Interpreter - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter6/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter6/CodeGenerator.cs index df32216ad..ed45c5ebd 100644 --- a/Samples/Kaleidoscope/Chapter6/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter6/CodeGenerator.cs @@ -51,6 +51,7 @@ public void Dispose( ) Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -60,17 +61,21 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return Visit( tree ); } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { return Context.CreateConstant( context.Value ); } + // + // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) { string varName = context.Name; @@ -81,6 +86,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte return value; } + // public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context ) { @@ -94,6 +100,7 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { return context.Signature.Accept( this ); @@ -103,6 +110,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -129,6 +137,7 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte } // + // public override Value VisitExpression( [NotNull] ExpressionContext context ) { // Expression: PrimaryExpression (op expression)* @@ -141,6 +150,7 @@ public override Value VisitExpression( [NotNull] ExpressionContext context ) return lhs; } + // // public override Value VisitConditionalExpression( [NotNull] ConditionalExpressionContext context ) @@ -315,28 +325,18 @@ public override Value VisitUnaryOpExpression( [NotNull] UnaryOpExpressionContext public override Value VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } public override Value VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) ) - { - // should never get here now that grammar distinguishes built-in operators - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } // protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { var rhs = rightTree.Accept( this ); @@ -396,6 +396,7 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ // } } + // // private void InitializeModuleAndPassManager( ) diff --git a/Samples/Kaleidoscope/Chapter6/Program.cs b/Samples/Kaleidoscope/Chapter6/Program.cs index b68719741..9d627acf8 100644 --- a/Samples/Kaleidoscope/Chapter6/Program.cs +++ b/Samples/Kaleidoscope/Chapter6/Program.cs @@ -39,7 +39,7 @@ public static void Main( string[ ] args ) RegisterNative( ); // - var parser = new ReplParserStack( LanguageLevel.UserDefinedOperators ); + var parser = new ParserStack( LanguageLevel.UserDefinedOperators ); using( var generator = new CodeGenerator( parser.GlobalState ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Interpreter - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs index 89e9b2ad8..5a2dfc749 100644 --- a/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs @@ -30,11 +30,12 @@ internal sealed class CodeGenerator , IKaleidoscopeCodeGenerator { // - public CodeGenerator( DynamicRuntimeState globalState ) + public CodeGenerator( DynamicRuntimeState globalState, bool disableOptimization = false ) { RuntimeState = globalState; Context = new Context( ); JIT = new KaleidoscopeJIT( ); + DisableOptimizations = disableOptimization; InitializeModuleAndPassManager( ); InstructionBuilder = new InstructionBuilder( Context ); FunctionPrototypes = new PrototypeCollection( ); @@ -43,14 +44,13 @@ public CodeGenerator( DynamicRuntimeState globalState ) } // - public bool DisableOptimizations { get; set; } - public void Dispose( ) { JIT.Dispose( ); Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -60,16 +60,19 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return Visit( tree ); } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { return Context.CreateConstant( context.Value ); } + // // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) @@ -97,6 +100,7 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { return context.Signature.Accept( this ); @@ -106,6 +110,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -219,7 +224,7 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio function.BasicBlocks.Add( continueBlock ); InstructionBuilder.PositionAtEnd( continueBlock ); return InstructionBuilder.Load( result ) - .RegisterName("ifresult"); + .RegisterName( "ifresult" ); } // @@ -309,7 +314,7 @@ public override Value VisitForExpression( [NotNull] ForExpressionContext context InstructionBuilder.Branch( endCondition, loopBlock, afterBlock ); InstructionBuilder.PositionAtEnd( afterBlock ); - // for expr always returns 0.0 for consistency, there is no 'void' + // for expression always returns 0.0 for consistency, there is no 'void' return Context.DoubleType.GetNullValue( ); } } @@ -338,22 +343,11 @@ public override Value VisitUnaryOpExpression( [NotNull] UnaryOpExpressionContext public override Value VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } public override Value VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) ) - { - // should never get here now that grammar distinguishes built-in operators - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } // @@ -384,6 +378,7 @@ public override Value VisitVarInExpression( [NotNull] VarInExpressionContext con protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { var rhs = rightTree.Accept( this ); @@ -447,19 +442,25 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ // } } + // // private void InitializeModuleAndPassManager( ) { Module = Context.CreateBitcodeModule( ); Module.Layout = JIT.TargetMachine.TargetData; - FunctionPassManager = new FunctionPassManager( Module ); - FunctionPassManager.AddPromoteMemoryToRegisterPass( ) - .AddInstructionCombiningPass( ) - .AddReassociatePass( ) - .AddGVNPass( ) - .AddCFGSimplificationPass( ) - .Initialize( ); + FunctionPassManager = new FunctionPassManager( Module ) + .AddPromoteMemoryToRegisterPass( ); + + if( !DisableOptimizations ) + { + FunctionPassManager.AddInstructionCombiningPass( ) + .AddReassociatePass( ) + .AddGVNPass( ) + .AddCFGSimplificationPass( ); + } + + FunctionPassManager.Initialize( ); } // @@ -582,6 +583,7 @@ private static Alloca CreateEntryBlockAlloca( Function theFunction, string varNa // // + private readonly bool DisableOptimizations; private readonly DynamicRuntimeState RuntimeState; private static int AnonNameIndex; private readonly Context Context; diff --git a/Samples/Kaleidoscope/Chapter7/Program.cs b/Samples/Kaleidoscope/Chapter7/Program.cs index a256d4411..abd98bc87 100644 --- a/Samples/Kaleidoscope/Chapter7/Program.cs +++ b/Samples/Kaleidoscope/Chapter7/Program.cs @@ -39,7 +39,7 @@ public static void Main( string[ ] args ) RegisterNative( ); // - var parser = new ReplParserStack( LanguageLevel.MutableVariables ); + var parser = new ParserStack( LanguageLevel.MutableVariables ); using( var generator = new CodeGenerator( parser.GlobalState ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Interpreter - {0}", parser.LanguageLevel ); diff --git a/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs index 1e7597c78..6ed685e76 100644 --- a/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs @@ -3,6 +3,7 @@ // using System; +using System.Collections.Generic; using System.Linq; using Antlr4.Runtime; using Antlr4.Runtime.Misc; @@ -27,20 +28,20 @@ internal sealed class CodeGenerator , IKaleidoscopeCodeGenerator { // - public CodeGenerator( DynamicRuntimeState globalState, TargetMachine machine ) + public CodeGenerator( DynamicRuntimeState globalState, TargetMachine machine, bool disableOptimization = false ) { RuntimeState = globalState; Context = new Context( ); TargetMachine = machine; + DisableOptimizations = disableOptimization; InitializeModuleAndPassManager( ); InstructionBuilder = new InstructionBuilder( Context ); FunctionPrototypes = new PrototypeCollection( ); NamedValues = new ScopeStack( ); + AnonymousFunctions = new List( ); } // - public bool DisableOptimizations { get; set; } - public BitcodeModule Module { get; private set; } public void Dispose( ) @@ -48,6 +49,7 @@ public void Dispose( ) Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -55,18 +57,47 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return null; } - return Visit( tree ); + Visit( tree ); + if( AnonymousFunctions.Count > 0 ) + { + var mainFunction = Module.AddFunction( "main", Context.GetFunctionType( Context.VoidType ) ); + var block = mainFunction.AppendBasicBlock( "entry" ); + var irBuilder = new InstructionBuilder( block ); + var printdFunc = Module.AddFunction( "printd", Context.GetFunctionType( Context.DoubleType, Context.DoubleType ) ); + foreach( var anonFunc in AnonymousFunctions ) + { + var value = irBuilder.Call( anonFunc ); + irBuilder.Call( printdFunc, value ); + } + + irBuilder.Return( ); + + // Use always inline and Dead Code Elimination module passes to inline all of the + // anonymous functions. This effectively strips all the calls just generated for main() + // and inlines each of the anonymous functions directly into main, dropping the now + // unused original anonymous functions all while retaining all of the original source + // debug information locations. + var mpm = new ModulePassManager( ) + .AddAlwaysInlinerPass( ) + .AddGlobalDCEPass( ); + mpm.Run( Module ); + } + + return null; } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { return Context.CreateConstant( context.Value ); } + // // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) @@ -94,6 +125,7 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { return context.Signature.Accept( this ); @@ -103,6 +135,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -120,6 +153,7 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte , isAnonymous: true ); + AnonymousFunctions.Add( function ); return DefineFunction( function, context.expression( ) ); } // @@ -211,7 +245,7 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio function.BasicBlocks.Add( continueBlock ); InstructionBuilder.PositionAtEnd( continueBlock ); return InstructionBuilder.Load( result ) - .RegisterName("ifresult"); + .RegisterName( "ifresult" ); } // @@ -301,7 +335,7 @@ public override Value VisitForExpression( [NotNull] ForExpressionContext context InstructionBuilder.Branch( endCondition, loopBlock, afterBlock ); InstructionBuilder.PositionAtEnd( afterBlock ); - // for expr always returns 0.0 for consistency, there is no 'void' + // for expression always returns 0.0 for consistency, there is no 'void' return Context.DoubleType.GetNullValue( ); } } @@ -330,22 +364,11 @@ public override Value VisitUnaryOpExpression( [NotNull] UnaryOpExpressionContext public override Value VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } public override Value VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) ) - { - // should never get here now that grammar distinguishes built-in operators - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } // @@ -376,6 +399,7 @@ public override Value VisitVarInExpression( [NotNull] VarInExpressionContext con protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { var rhs = rightTree.Accept( this ); @@ -439,6 +463,7 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ // } } + // // private void InitializeModuleAndPassManager( ) @@ -446,13 +471,18 @@ private void InitializeModuleAndPassManager( ) Module = Context.CreateBitcodeModule( ); Module.TargetTriple = TargetMachine.Triple; Module.Layout = TargetMachine.TargetData; - FunctionPassManager = new FunctionPassManager( Module ); - FunctionPassManager.AddPromoteMemoryToRegisterPass( ) - .AddInstructionCombiningPass( ) - .AddReassociatePass( ) - .AddGVNPass( ) - .AddCFGSimplificationPass( ) - .Initialize( ); + FunctionPassManager = new FunctionPassManager( Module ) + .AddPromoteMemoryToRegisterPass( ); + + if( !DisableOptimizations ) + { + FunctionPassManager.AddInstructionCombiningPass( ) + .AddReassociatePass( ) + .AddGVNPass( ) + .AddCFGSimplificationPass( ); + } + + FunctionPassManager.Initialize( ); } // @@ -488,7 +518,16 @@ private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = f var llvmSignature = Context.GetFunctionType( Context.DoubleType, prototype.Parameters.Select( _ => Context.DoubleType ) ); var retVal = Module.AddFunction( prototype.Identifier.Name, llvmSignature ); - retVal.Linkage( Linkage.External ); + // mark anonymous functions as always-inline and private so they can be inlined and then removed + if( isAnonymous ) + { + retVal.AddAttribute( FunctionAttributeIndex.Function, AttributeKind.AlwaysInline ) + .Linkage( Linkage.Private ); + } + else + { + retVal.Linkage( Linkage.External ); + } int index = 0; foreach( var argId in prototype.Parameters ) @@ -536,10 +575,7 @@ private Function DefineFunction( Function function, ExpressionContext body ) function.Verify( ); } - if( !DisableOptimizations ) - { - FunctionPassManager.Run( function ); - } + FunctionPassManager.Run( function ); return function; } @@ -560,6 +596,7 @@ private static Alloca CreateEntryBlockAlloca( Function theFunction, string varNa // // + private readonly bool DisableOptimizations; private readonly DynamicRuntimeState RuntimeState; private static int AnonNameIndex; private readonly Context Context; @@ -568,6 +605,7 @@ private static Alloca CreateEntryBlockAlloca( Function theFunction, string varNa private FunctionPassManager FunctionPassManager; private TargetMachine TargetMachine; private readonly PrototypeCollection FunctionPrototypes; + private readonly List AnonymousFunctions; // } } diff --git a/Samples/Kaleidoscope/Chapter8/Program.cs b/Samples/Kaleidoscope/Chapter8/Program.cs index a8bced440..33f645f1d 100644 --- a/Samples/Kaleidoscope/Chapter8/Program.cs +++ b/Samples/Kaleidoscope/Chapter8/Program.cs @@ -8,7 +8,6 @@ using Kaleidoscope.Grammar; using Kaleidoscope.Runtime; using Llvm.NET; -using Llvm.NET.Values; using static Kaleidoscope.Runtime.Utilities; using static Llvm.NET.StaticState; @@ -36,7 +35,7 @@ public static class Program /// public static int Main( string[ ] args ) { - (string sourceFilePath, bool waitForDebugger, int exitCode) = ProcessArgs( args ); + (string sourceFilePath, int exitCode) = ProcessArgs( args ); if( exitCode != 0 ) { return exitCode; @@ -44,6 +43,7 @@ public static int Main( string[ ] args ) string objFilePath = Path.ChangeExtension( sourceFilePath, ".o" ); string irFilePath = Path.ChangeExtension( sourceFilePath, ".ll" ); + string asmPath = Path.ChangeExtension( sourceFilePath, ".s" ); using( TextReader rdr = File.OpenText( sourceFilePath ) ) using( InitializeLLVM( ) ) @@ -51,18 +51,16 @@ public static int Main( string[ ] args ) RegisterNative( ); var machine = new TargetMachine( Triple.HostTriple ); - var parser = new ReplParserStack( LanguageLevel.MutableVariables ); + var parser = new ParserStack( LanguageLevel.MutableVariables ); using( var generator = new CodeGenerator( parser.GlobalState, machine ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Compiler - {0}", parser.LanguageLevel ); Console.WriteLine( "Compiling {0}", sourceFilePath ); - var replLoop = new ReplLoop( generator, parser, DiagnosticRepresentations.None, rdr ); - replLoop.CodeGenerationError += OnGeneratorError; - // time the parse and code generation var timer = System.Diagnostics.Stopwatch.StartNew( ); - replLoop.Run( ); + var (parseTree, recognizer) = parser.Parse( rdr, DiagnosticRepresentations.DebugTraceParser ); + generator.Generate( recognizer, parseTree, DiagnosticRepresentations.None ); if( !generator.Module.Verify( out string errMsg ) ) { Console.Error.WriteLine( errMsg ); @@ -78,6 +76,8 @@ public static int Main( string[ ] args ) Console.Error.WriteLine( msg ); return -1; } + + machine.EmitToFile( generator.Module, asmPath, CodeGenFileType.AssemblySource ); Console.WriteLine( "CopmilationTiorTime: {0}", timer.Elapsed ); } } @@ -105,7 +105,7 @@ private static void OnGeneratorError( object sender, CodeGenerationExceptionArgs // // really simple command line handling, just loops through the args - private static (string SourceFilePath, bool WaitForDebugger, int ExitCode) ProcessArgs( string[ ] args ) + private static (string SourceFilePath, int ExitCode) ProcessArgs( string[ ] args ) { bool waitforDebugger = false; string sourceFilePath = string.Empty; @@ -130,16 +130,16 @@ private static (string SourceFilePath, bool WaitForDebugger, int ExitCode) Proce if( string.IsNullOrWhiteSpace( sourceFilePath ) ) { Console.Error.WriteLine( "Missing source file name!" ); - return (null, false, -1); + return (null, -1); } if( !File.Exists( sourceFilePath ) ) { Console.Error.WriteLine( "Source file '{0}' - not found!", sourceFilePath ); - return (null, false, -2); + return (null, -2); } - return (sourceFilePath, waitforDebugger, 0); + return (sourceFilePath, 0); } // } diff --git a/Samples/Kaleidoscope/Chapter9/Chapter9.csproj b/Samples/Kaleidoscope/Chapter9/Chapter9.csproj index a6fa23c5d..3e53b4ec9 100644 --- a/Samples/Kaleidoscope/Chapter9/Chapter9.csproj +++ b/Samples/Kaleidoscope/Chapter9/Chapter9.csproj @@ -6,7 +6,11 @@ Kaleidoscope latest - + + + PreserveNewest + + Llvm.NET diff --git a/Samples/Kaleidoscope/Chapter9/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter9/CodeGenerator.cs index 4d76d1d47..a6818c279 100644 --- a/Samples/Kaleidoscope/Chapter9/CodeGenerator.cs +++ b/Samples/Kaleidoscope/Chapter9/CodeGenerator.cs @@ -4,8 +4,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; -using System.Runtime.InteropServices; using Antlr4.Runtime; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Tree; @@ -30,21 +31,21 @@ internal sealed class CodeGenerator , IKaleidoscopeCodeGenerator { // - public CodeGenerator( DynamicRuntimeState globalState, TargetMachine machine ) + public CodeGenerator( DynamicRuntimeState globalState, TargetMachine machine, string sourcePath, bool disableOptimization = false ) { RuntimeState = globalState; - LexicalBlocks = new Stack( ); // TODO: combine this with ScopeStack as they work in tandem + LexicalBlocks = new Stack( ); Context = new Context( ); TargetMachine = machine; - InitializeModuleAndPassManager( ); + DisableOptimizations = disableOptimization; + InitializeModuleAndPassManager( sourcePath ); InstructionBuilder = new InstructionBuilder( Context ); FunctionPrototypes = new PrototypeCollection( ); NamedValues = new ScopeStack( ); + AnonymousFunctions = new List( ); } // - public bool DisableOptimizations { get; set; } - public BitcodeModule Module { get; private set; } public void Dispose( ) @@ -52,6 +53,7 @@ public void Dispose( ) Context.Dispose( ); } + // public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations additionalDiagnostics ) { if( parser.NumberOfSyntaxErrors > 0 ) @@ -59,8 +61,36 @@ public Value Generate( Parser parser, IParseTree tree, DiagnosticRepresentations return null; } - return Visit( tree ); + Visit( tree ); + if( AnonymousFunctions.Count > 0 ) + { + var mainFunction = Module.AddFunction( "main", Context.GetFunctionType( Context.VoidType ) ); + var block = mainFunction.AppendBasicBlock( "entry" ); + var irBuilder = new InstructionBuilder( block ); + var printdFunc = Module.AddFunction( "printd", Context.GetFunctionType( Context.DoubleType, Context.DoubleType ) ); + foreach( var anonFunc in AnonymousFunctions ) + { + var value = irBuilder.Call( anonFunc ); + irBuilder.Call( printdFunc, value ); + } + + irBuilder.Return( ); + + // Use always inline and Dead Code Elimination module passes to inline all of the + // anonymous functions. This effectively strips all the calls just generated for main() + // and inlines each of the anonymous functions directly into main, dropping the now + // unused original anonymous functions all while retaining all of the original source + // debug information locations. + var mpm = new ModulePassManager( ) + .AddAlwaysInlinerPass( ) + .AddGlobalDCEPass( ); + mpm.Run( Module ); + } + + Module.DIBuilder.Finish( ); + return null; } + // public override Value VisitParenExpression( [NotNull] ParenExpressionContext context ) { @@ -68,11 +98,13 @@ public override Value VisitParenExpression( [NotNull] ParenExpressionContext con return context.Expression.Accept( this ); } + // public override Value VisitConstExpression( [NotNull] ConstExpressionContext context ) { EmitLocation( context ); return Context.CreateConstant( context.Value ); } + // // public override Value VisitVariableExpression( [NotNull] VariableExpressionContext context ) @@ -92,6 +124,7 @@ public override Value VisitVariableExpression( [NotNull] VariableExpressionConte public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpressionContext context ) { EmitLocation( context ); + var function = FindCallTarget( context.CaleeName ); if( function == null ) { @@ -99,9 +132,12 @@ public override Value VisitFunctionCallExpression( [NotNull] FunctionCallExpress } var args = context.Args.Select( ctx => ctx.Accept( this ) ).ToArray( ); + + EmitLocation( context ); return InstructionBuilder.Call( function, args ).RegisterName( "calltmp" ); } + // public override Value VisitExternalDeclaration( [NotNull] ExternalDeclarationContext context ) { EmitLocation( context ); @@ -112,6 +148,7 @@ public override Value VisitFunctionPrototype( [NotNull] FunctionPrototypeContext { return GetOrDeclareFunction( new Prototype( context ) ); } + // // public override Value VisitFunctionDefinition( [NotNull] FunctionDefinitionContext context ) @@ -129,6 +166,7 @@ public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionConte , isAnonymous: true ); + AnonymousFunctions.Add( function ); return DefineFunction( function, context.expression( ) ); } // @@ -180,6 +218,8 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio return null; } + EmitLocation( context ); + var condBool = InstructionBuilder.Compare( RealPredicate.OrderedAndNotEqual, condition, Context.CreateConstant( 0.0 ) ) .RegisterName( "ifcond" ); @@ -221,7 +261,7 @@ public override Value VisitConditionalExpression( [NotNull] ConditionalExpressio function.BasicBlocks.Add( continueBlock ); InstructionBuilder.PositionAtEnd( continueBlock ); return InstructionBuilder.Load( result ) - .RegisterName("ifresult"); + .RegisterName( "ifresult" ); } // @@ -231,7 +271,7 @@ public override Value VisitForExpression( [NotNull] ForExpressionContext context EmitLocation( context ); var function = InstructionBuilder.InsertBlock.ContainingFunction; string varName = context.Initializer.Name; - var allocaVar = CreateEntryBlockAlloca( function, varName ); + var allocaVar = CreateEntryBlockAlloca( function, context.Initializer ); // Emit the start code first, without 'variable' in scope. Value startVal = null; @@ -312,7 +352,7 @@ public override Value VisitForExpression( [NotNull] ForExpressionContext context InstructionBuilder.Branch( endCondition, loopBlock, afterBlock ); InstructionBuilder.PositionAtEnd( afterBlock ); - // for expr always returns 0.0 for consistency, there is no 'void' + // for expression always returns 0.0 for consistency, there is no 'void' return Context.DoubleType.GetNullValue( ); } } @@ -343,22 +383,11 @@ public override Value VisitUnaryOpExpression( [NotNull] UnaryOpExpressionContext public override Value VisitBinaryPrototype( [NotNull] BinaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.InfixLeftAssociative, context.Precedence ) ) - { - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } public override Value VisitUnaryPrototype( [NotNull] UnaryPrototypeContext context ) { - if( !RuntimeState.TryAddOperator( context.OpToken, OperatorKind.PreFix, 0 ) ) - { - // should never get here now that grammar distinguishes built-in operators - throw new CodeGeneratorException( "Cannot replace built-in operators" ); - } - return GetOrDeclareFunction( new Prototype( context, context.Name ) ); } // @@ -377,7 +406,7 @@ public override Value VisitVarInExpression( [NotNull] VarInExpressionContext con initValue = initializer.Value.Accept( this ); } - var alloca = CreateEntryBlockAlloca( function, initializer.Name ); + var alloca = CreateEntryBlockAlloca( function, initializer ); InstructionBuilder.Store( initValue, alloca ); NamedValues[ initializer.Name ] = alloca; } @@ -389,6 +418,7 @@ public override Value VisitVarInExpression( [NotNull] VarInExpressionContext con protected override Value DefaultResult => null; + // private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree rightTree ) { EmitLocation( op ); @@ -453,28 +483,34 @@ private Value EmitBinaryOperator( Value lhs, BinaryopContext op, IParseTree righ // } } + // // - private void InitializeModuleAndPassManager( ) + private void InitializeModuleAndPassManager( string sourcePath ) { - Module = Context.CreateBitcodeModule( ); + Module = Context.CreateBitcodeModule( Path.GetFileName( sourcePath ), SourceLanguage.C, sourcePath, "Kaleidoscope Compiler" ); Module.TargetTriple = TargetMachine.Triple; Module.Layout = TargetMachine.TargetData; DoubleType = new DebugBasicType( Context.DoubleType, Module, "double", DiTypeKind.Float ); - FunctionPassManager = new FunctionPassManager( Module ); - FunctionPassManager.AddPromoteMemoryToRegisterPass( ) - .AddInstructionCombiningPass( ) - .AddReassociatePass( ) - .AddGVNPass( ) - .AddCFGSimplificationPass( ) - .Initialize( ); + FunctionPassManager = new FunctionPassManager( Module ) + .AddPromoteMemoryToRegisterPass( ); + + if( !DisableOptimizations ) + { + FunctionPassManager.AddInstructionCombiningPass( ) + .AddReassociatePass( ) + .AddGVNPass( ) + .AddCFGSimplificationPass( ); + } + FunctionPassManager.Initialize( ); } // + // private void EmitLocation( IParseTree context ) { - if( !(context is ParserRuleContext ruleContext ) ) + if( !( context is ParserRuleContext ruleContext ) ) { InstructionBuilder.SetDebugLocation( 0, 0 ); } @@ -489,6 +525,7 @@ private void EmitLocation( IParseTree context ) InstructionBuilder.SetDebugLocation( ( uint )ruleContext.Start.Line, ( uint )ruleContext.Start.Column, scope ); } } + // // private Function FindCallTarget( string name ) @@ -519,15 +556,24 @@ private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = f return function; } - var debugFile = Module.DIBuilder.CreateFile( Module.DICompileUnit.File.FileName, Module.DICompileUnit.File.Directory ); - var signature = Context.CreateFunctionType( Module.DIBuilder, DoubleType, prototype.Parameters.Select( _ => DoubleType ) ); - var lastParam = prototype.Parameters.LastOrDefault( ); - if( lastParam == default ) + // extern declarations don't get debug information + Function retVal; + if( prototype.IsExtern ) { - lastParam = prototype.Identifier; + var llvmSignature = Context.GetFunctionType( Context.DoubleType, prototype.Parameters.Select( _ => Context.DoubleType ) ); + retVal = Module.AddFunction( prototype.Identifier.Name, llvmSignature ); } + else + { + var debugFile = Module.DIBuilder.CreateFile( Module.DICompileUnit.File.FileName, Module.DICompileUnit.File.Directory ); + var signature = Context.CreateFunctionType( Module.DIBuilder, DoubleType, prototype.Parameters.Select( _ => DoubleType ) ); + var lastParam = prototype.Parameters.LastOrDefault( ); + if( lastParam == default ) + { + lastParam = prototype.Identifier; + } - var retVal = Module.CreateFunction( Module.DICompileUnit + retVal = Module.CreateFunction( Module.DICompileUnit , prototype.Identifier.Name , null , debugFile @@ -536,11 +582,21 @@ private Function GetOrDeclareFunction( Prototype prototype, bool isAnonymous = f , false , true , ( uint )lastParam.Span.EndLine - , DebugInfoFlags.Prototyped + , isAnonymous ? DebugInfoFlags.Artificial : DebugInfoFlags.Prototyped , false ); + } - retVal.Linkage( Linkage.External ); + // mark anonymous functions as always-inline and private so they can be inlined and then removed + if( isAnonymous ) + { + retVal.AddAttribute( FunctionAttributeIndex.Function, AttributeKind.AlwaysInline ) + .Linkage( Linkage.Private ); + } + else + { + retVal.Linkage( Linkage.External ); + } int index = 0; foreach( var argId in prototype.Parameters ) @@ -566,12 +622,9 @@ private Function DefineFunction( Function function, ExpressionContext body ) throw new CodeGeneratorException( $"Function {function.Name} cannot be redefined in the same module" ); } - var proto = FunctionPrototypes[ function.Name ]; var basicBlock = function.AppendBasicBlock( "entry" ); InstructionBuilder.PositionAtEnd( basicBlock ); - var diFile = Module.DICompileUnit.File; - var scope = Module.DICompileUnit; LexicalBlocks.Push( function.DISubProgram ); // Unset the location for the prologue emission (leading instructions with no @@ -579,28 +632,14 @@ private Function DefineFunction( Function function, ExpressionContext body ) // will run past them when breaking on a function) EmitLocation( null ); + FunctionPrototypes.TryGetValue( function.Name, out Prototype proto ); + using( NamedValues.EnterScope( ) ) { foreach( var arg in function.Parameters ) { - uint line = ( uint )proto.Parameters[ ( int )( arg.Index ) ].Span.StartLine; - uint col = ( uint )proto.Parameters[ ( int )( arg.Index ) ].Span.StartColumn; - - var argSlot = CreateEntryBlockAlloca( function, arg.Name ); - DILocalVariable debugVar = Module.DIBuilder.CreateArgument( function.DISubProgram - , arg.Name - , diFile - , line - , DoubleType - , true - , DebugInfoFlags.None - , checked(( ushort )(arg.Index + 1)) // Debug index starts at 1! - ); - Module.DIBuilder.InsertDeclare( argSlot - , debugVar - , new DILocation( Context, line, col, function.DISubProgram ) - , InstructionBuilder.InsertBlock - ); + var paramNode = proto.Parameters[ ( int )( arg.Index ) ]; + var argSlot = CreateEntryBlockAlloca( function, paramNode, arg.Index ); InstructionBuilder.Store( arg, argSlot ); NamedValues[ arg.Name ] = argSlot; @@ -620,16 +659,58 @@ private Function DefineFunction( Function function, ExpressionContext body ) function.Verify( ); } - if( !DisableOptimizations ) - { - FunctionPassManager.Run( function ); - } + FunctionPassManager.Run( function ); return function; } // // + private Alloca CreateEntryBlockAlloca( Function function, Identifier paramIdentifier, uint argIndex ) + { + var argSlot = CreateEntryBlockAlloca( function, paramIdentifier.Name ); + uint line = ( uint )paramIdentifier.Span.StartLine; + uint col = ( uint )paramIdentifier.Span.StartColumn; + + DILocalVariable debugVar = Module.DIBuilder.CreateArgument( function.DISubProgram + , paramIdentifier.Name + , function.DISubProgram.File + , line + , DoubleType + , true + , DebugInfoFlags.None + , checked(( ushort )( argIndex + 1 )) // Debug index starts at 1! + ); + Module.DIBuilder.InsertDeclare( argSlot + , debugVar + , new DILocation( Context, line, col, function.DISubProgram ) + , InstructionBuilder.InsertBlock + ); + return argSlot; + } + + private Alloca CreateEntryBlockAlloca( Function function, InitializerContext initializer ) + { + var argSlot = CreateEntryBlockAlloca( function, initializer.Name ); + uint line = ( uint )initializer.Start.Line; + uint col = ( uint )initializer.Start.Column; + + DILocalVariable debugVar = Module.DIBuilder.CreateLocalVariable( function.DISubProgram + , initializer.Name + , function.DISubProgram.File + , line + , DoubleType + , false + , DebugInfoFlags.None + ); + Module.DIBuilder.InsertDeclare( argSlot + , debugVar + , new DILocation( Context, line, col, function.DISubProgram ) + , InstructionBuilder.InsertBlock + ); + return argSlot; + } + private static Alloca CreateEntryBlockAlloca( Function theFunction, string varName ) { var tmpBldr = new InstructionBuilder( theFunction.EntryBlock ); @@ -644,6 +725,7 @@ private static Alloca CreateEntryBlockAlloca( Function theFunction, string varNa // // + private readonly bool DisableOptimizations; private readonly DynamicRuntimeState RuntimeState; private static int AnonNameIndex; private readonly Context Context; @@ -652,7 +734,7 @@ private static Alloca CreateEntryBlockAlloca( Function theFunction, string varNa private FunctionPassManager FunctionPassManager; private TargetMachine TargetMachine; private readonly PrototypeCollection FunctionPrototypes; - + private readonly List AnonymousFunctions; private DebugBasicType DoubleType; private Stack LexicalBlocks; // diff --git a/Samples/Kaleidoscope/Chapter9/Program.cs b/Samples/Kaleidoscope/Chapter9/Program.cs index e395b9813..a63ce7c21 100644 --- a/Samples/Kaleidoscope/Chapter9/Program.cs +++ b/Samples/Kaleidoscope/Chapter9/Program.cs @@ -8,7 +8,6 @@ using Kaleidoscope.Grammar; using Kaleidoscope.Runtime; using Llvm.NET; -using Llvm.NET.Values; using static Kaleidoscope.Runtime.Utilities; using static Llvm.NET.StaticState; @@ -36,7 +35,7 @@ public static class Program /// public static int Main( string[ ] args ) { - (string sourceFilePath, bool waitForDebugger, int exitCode) = ProcessArgs( args ); + (string sourceFilePath, int exitCode) = ProcessArgs( args ); if( exitCode != 0 ) { return exitCode; @@ -44,6 +43,7 @@ public static int Main( string[ ] args ) string objFilePath = Path.ChangeExtension( sourceFilePath, ".o" ); string irFilePath = Path.ChangeExtension( sourceFilePath, ".ll" ); + string asmPath = Path.ChangeExtension( sourceFilePath, ".s" ); using( TextReader rdr = File.OpenText( sourceFilePath ) ) using( InitializeLLVM( ) ) @@ -51,20 +51,17 @@ public static int Main( string[ ] args ) RegisterNative( ); var machine = new TargetMachine( Triple.HostTriple ); - var parser = new ReplParserStack( LanguageLevel.MutableVariables ); - using( var generator = new CodeGenerator( parser.GlobalState, machine ) ) + var parser = new ParserStack( LanguageLevel.MutableVariables ); + using( var generator = new CodeGenerator( parser.GlobalState, machine, sourceFilePath, true ) ) { Console.WriteLine( "Llvm.NET Kaleidoscope Compiler - {0}", parser.LanguageLevel ); Console.WriteLine( "Compiling {0}", sourceFilePath ); - var replLoop = new ReplLoop( generator, parser, DiagnosticRepresentations.None, rdr ); - replLoop.CodeGenerationError += OnGeneratorError; - // time the parse and code generation var timer = System.Diagnostics.Stopwatch.StartNew( ); - replLoop.Run( ); - generator.Module.DIBuilder.Finish( ); - if( !generator.Module.Verify(out string errMsg ) ) + var (parseTree, recognizer) = parser.Parse( rdr, DiagnosticRepresentations.DebugTraceParser ); + generator.Generate( recognizer, parseTree, DiagnosticRepresentations.None ); + if( !generator.Module.Verify( out string errMsg ) ) { Console.Error.WriteLine( errMsg ); } @@ -79,6 +76,8 @@ public static int Main( string[ ] args ) Console.Error.WriteLine( msg ); return -1; } + + machine.EmitToFile( generator.Module, asmPath, CodeGenFileType.AssemblySource ); Console.WriteLine( "CopmilationTiorTime: {0}", timer.Elapsed ); } } @@ -106,7 +105,7 @@ private static void OnGeneratorError( object sender, CodeGenerationExceptionArgs // // really simple command line handling, just loops through the args - private static (string SourceFilePath, bool WaitForDebugger, int ExitCode) ProcessArgs( string[ ] args ) + private static (string SourceFilePath, int ExitCode) ProcessArgs( string[ ] args ) { bool waitforDebugger = false; string sourceFilePath = string.Empty; @@ -131,16 +130,16 @@ private static (string SourceFilePath, bool WaitForDebugger, int ExitCode) Proce if( string.IsNullOrWhiteSpace( sourceFilePath ) ) { Console.Error.WriteLine( "Missing source file name!" ); - return (null, false, -1); + return (null, -1); } if( !File.Exists( sourceFilePath ) ) { Console.Error.WriteLine( "Source file '{0}' - not found!", sourceFilePath ); - return (null, false, -2); + return (null, -2); } - return (sourceFilePath, waitforDebugger, 0); + return (sourceFilePath, 0); } // } diff --git a/Samples/Kaleidoscope/Kaleidoscope.Parser/DebugTraceListener.cs b/Samples/Kaleidoscope/Kaleidoscope.Parser/DebugTraceListener.cs index 0e27b1416..2eb750d17 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Parser/DebugTraceListener.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Parser/DebugTraceListener.cs @@ -22,13 +22,13 @@ public DebugTraceListener( Parser parser ) /// public virtual void EnterEveryRule( ParserRuleContext ctx ) { - Trace.TraceInformation( $"enter[{ctx.SourceInterval}] {Parser.RuleNames[ ctx.RuleIndex ]} Lt(1)='{( ( ITokenStream )Parser.InputStream ).Lt( 1 ).Text}'" ); + Trace.TraceInformation( $"enter[{ctx.SourceInterval}] {Parser.RuleNames[ ctx.RuleIndex ]} [{ctx.GetType( ).Name}] Lt(1)='{( ( ITokenStream )Parser.InputStream ).Lt( 1 ).Text}'" ); } /// public virtual void ExitEveryRule( ParserRuleContext ctx ) { - Trace.TraceInformation( $"exit[{ctx.SourceInterval}] {Parser.RuleNames[ ctx.RuleIndex ]} Lt(1)='{( ( ITokenStream )Parser.InputStream ).Lt( 1 ).Text}'"); + Trace.TraceInformation( $"exit[{ctx.SourceInterval}] {Parser.RuleNames[ ctx.RuleIndex ]} [{ctx.GetType( ).Name}] Lt(1)='{( ( ITokenStream )Parser.InputStream ).Lt( 1 ).Text}'"); } /// diff --git a/Samples/Kaleidoscope/Kaleidoscope.Parser/Kaleidoscope.g4 b/Samples/Kaleidoscope/Kaleidoscope.Parser/Kaleidoscope.g4 index 716744ebc..71cd0bf27 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Parser/Kaleidoscope.g4 +++ b/Samples/Kaleidoscope/Kaleidoscope.Parser/Kaleidoscope.g4 @@ -180,7 +180,12 @@ prototype repl : DEF prototype expression[0] # FunctionDefinition - | EXTERN prototype # ExternalDeclaration + | EXTERN prototype # ExternalDeclaration | expression[0] # TopLevelExpression - | SEMICOLON # TopLevelSemicolon + | SEMICOLON # TopLevelSemicolon ; + +// Full source parse accepts a series of definitions or prototypes, all top level expressions +// are generated into a single function called Main() +fullsrc + : repl*; diff --git a/Samples/Kaleidoscope/Kaleidoscope.Parser/KaleidoscopeParser.cs b/Samples/Kaleidoscope/Kaleidoscope.Parser/KaleidoscopeParser.cs index 6626f65fa..a4eb74814 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Parser/KaleidoscopeParser.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Parser/KaleidoscopeParser.cs @@ -38,11 +38,11 @@ public partial class KaleidoscopeParser /// public DynamicRuntimeState GlobalState { get; set; } - private bool FeatureControlFlow => IsFeatureEnabled( LanguageLevel.ControlFlow ); + public bool FeatureControlFlow => IsFeatureEnabled( LanguageLevel.ControlFlow ); - private bool FeatureMutableVars => IsFeatureEnabled( LanguageLevel.MutableVariables ); + public bool FeatureMutableVars => IsFeatureEnabled( LanguageLevel.MutableVariables ); - private bool FeatureUserOperators => IsFeatureEnabled( LanguageLevel.UserDefinedOperators ); + public bool FeatureUserOperators => IsFeatureEnabled( LanguageLevel.UserDefinedOperators ); private bool IsFeatureEnabled( LanguageLevel feature ) => LanguageLevel >= feature; diff --git a/Samples/Kaleidoscope/Kaleidoscope.Parser/ProtoType.cs b/Samples/Kaleidoscope/Kaleidoscope.Parser/ProtoType.cs index 813845a08..d2e4aae4d 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Parser/ProtoType.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Parser/ProtoType.cs @@ -47,6 +47,7 @@ public Prototype( Identifier name, params Identifier[ ] parameters ) public Prototype( FunctionPrototypeContext ctx ) : this( new Identifier( ctx.Name, ctx.GetSourceSpan( ) ), ctx.Parameters.Select( i => new Identifier( i.Name, i.Span ) ) ) { + IsExtern = ctx.parent is ExternalDeclarationContext; } /// Initializes a new instance of the class. @@ -65,6 +66,8 @@ public Prototype( UnaryPrototypeContext ctx, string linkageName ) { } + public bool IsExtern { get; } + public Identifier Identifier { get; } public IReadOnlyList Parameters { get; } diff --git a/Samples/Kaleidoscope/Kaleidoscope.Runtime/IKaleidoscopeParser.cs b/Samples/Kaleidoscope/Kaleidoscope.Runtime/IKaleidoscopeParser.cs index fc0b11818..3763ce9b7 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Runtime/IKaleidoscopeParser.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Runtime/IKaleidoscopeParser.cs @@ -36,7 +36,7 @@ public interface IKaleidoscopeParser /// (IParseTree parseTree, Parser recognizer) Parse( string txt, DiagnosticRepresentations additionalDiagnostics ); - /// Try parsing the given input text + /// Try parsing the given input text as full source, potentially containing multiple definitions /// TextReader to parse /// Additional diagnostics to generate /// Parse tree and the parser that was used to generate it as a diff --git a/Samples/Kaleidoscope/Kaleidoscope.Runtime/KaleidoscopeUserOperatorListener.cs b/Samples/Kaleidoscope/Kaleidoscope.Runtime/KaleidoscopeUserOperatorListener.cs new file mode 100644 index 000000000..dfccc00d1 --- /dev/null +++ b/Samples/Kaleidoscope/Kaleidoscope.Runtime/KaleidoscopeUserOperatorListener.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) .NET Foundation. All rights reserved. +// + +using Antlr4.Runtime.Misc; +using Kaleidoscope.Grammar; + +namespace Kaleidoscope.Runtime +{ + // Parse listener to handle updating runtime state on successfully parsing a user defined + // operator definition. + internal class KaleidoscopeUserOperatorListener + : KaleidoscopeBaseListener + { + public KaleidoscopeUserOperatorListener( DynamicRuntimeState state ) + { + RuntimeState = state; + } + + // upon successful parse of a function definition check if it is a user defined operator + // and update the RuntimeState accordingly, if it is. + public override void ExitFunctionDefinition( [NotNull] KaleidoscopeParser.FunctionDefinitionContext context ) + { + switch( context.Signature ) + { + case KaleidoscopeParser.UnaryPrototypeContext unaryProto: + RuntimeState.TryAddOperator( unaryProto.OpToken, OperatorKind.PreFix, 0 ); + break; + + case KaleidoscopeParser.BinaryPrototypeContext binaryProto: + RuntimeState.TryAddOperator( binaryProto.OpToken, OperatorKind.InfixLeftAssociative, binaryProto.Precedence ); + break; + + default: + base.ExitFunctionDefinition( context ); + break; + } + } + + private readonly DynamicRuntimeState RuntimeState; + } +} diff --git a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplParserStack.cs b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ParserStack.cs similarity index 86% rename from Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplParserStack.cs rename to Samples/Kaleidoscope/Kaleidoscope.Runtime/ParserStack.cs index 92bbf0fee..66a4f5b71 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplParserStack.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ParserStack.cs @@ -10,30 +10,36 @@ namespace Kaleidoscope.Runtime { + public enum ParseMode + { + ReplLoop, + FullSource, + } + /// Combined Lexer and Parser that can support REPL usage - public class ReplParserStack + public class ParserStack : IKaleidoscopeParser { - /// Initializes a new instance of the class configured for the specified language level + /// Initializes a new instance of the class configured for the specified language level /// for the parser - public ReplParserStack( LanguageLevel level ) + public ParserStack( LanguageLevel level ) : this( level, new FormattedConsoleErrorListener( ) ) { } - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// for the parser /// Combined error listener for lexer and parser errors - public ReplParserStack( LanguageLevel level, IUnifiedErrorListener listener ) + public ParserStack( LanguageLevel level, IUnifiedErrorListener listener ) : this( level, listener, listener) { } - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// for the parser /// Error listener for Lexer errors /// Error listener for parer errors - public ReplParserStack( LanguageLevel level + public ParserStack( LanguageLevel level , IAntlrErrorListener lexErrorListener , IAntlrErrorListener parseErrorListener ) @@ -57,16 +63,16 @@ public LanguageLevel LanguageLevel /// public (IParseTree parseTree, Parser recognizer) Parse( string txt, DiagnosticRepresentations aditionalDiagnostics ) { - return Parse( new AntlrInputStream( txt ), aditionalDiagnostics ); + return Parse( new AntlrInputStream( txt ), aditionalDiagnostics, ParseMode.ReplLoop ); } /// public (IParseTree parseTree, Parser recognizer) Parse( TextReader reader, DiagnosticRepresentations aditionalDiagnostics ) { - return Parse( new AntlrInputStream( reader ), aditionalDiagnostics ); + return Parse( new AntlrInputStream( reader ), aditionalDiagnostics, ParseMode.FullSource ); } - private (IParseTree parseTree, Parser recognizer) Parse( AntlrInputStream inputStream, DiagnosticRepresentations aditionalDiagnostics ) + private (IParseTree parseTree, Parser recognizer) Parse( AntlrInputStream inputStream, DiagnosticRepresentations aditionalDiagnostics, ParseMode mode ) { try { @@ -94,13 +100,18 @@ public LanguageLevel LanguageLevel Parser.AddParseListener( new DebugTraceListener( Parser ) ); } + if( Parser.FeatureUserOperators ) + { + Parser.AddParseListener( new KaleidoscopeUserOperatorListener( GlobalState ) ); + } + if( ParseErrorListener != null ) { Parser.RemoveErrorListeners( ); Parser.AddErrorListener( ParseErrorListener ); } - var parseTree = Parser.repl( ); + var parseTree = mode == ParseMode.ReplLoop ? (IParseTree)Parser.repl( ) : Parser.fullsrc(); if( aditionalDiagnostics.HasFlag( DiagnosticRepresentations.Xml ) ) { diff --git a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplErrorStrategy.cs b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplErrorStrategy.cs index 6ffc9938b..03937b6f3 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplErrorStrategy.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplErrorStrategy.cs @@ -9,11 +9,6 @@ namespace Kaleidoscope.Runtime /// Error handling strategy for REPL scenarios /// /// This strategy handles creating more meaningful messages on feature predicate failures. - /// Additionally, error recovery that results from reaching EOF on an otherwise successful parse - /// is translated into a to prevent any further attempts - /// at recovery. This allows for detection of the incomplete statements when used in an interactive - /// REPL application. In such a case the incomplete code could be combined with new text from the - /// user until it either succeeds or fails due to a syntax error. /// public class ReplErrorStrategy : DefaultErrorStrategy diff --git a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplLoop.cs b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplLoop.cs index 1f87f1beb..102bc6894 100644 --- a/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplLoop.cs +++ b/Samples/Kaleidoscope/Kaleidoscope.Runtime/ReplLoop.cs @@ -12,7 +12,7 @@ namespace Kaleidoscope.Runtime public class ReplLoop { public ReplLoop( IKaleidoscopeCodeGenerator generator, LanguageLevel languageLevel ) - : this( generator, new ReplParserStack( languageLevel ), DiagnosticRepresentations.None, Console.In ) + : this( generator, new ParserStack( languageLevel ), DiagnosticRepresentations.None, Console.In ) { } diff --git a/docfx/Llvm.Net.Docfx.csproj b/docfx/Llvm.Net.Docfx.csproj index 7605a7ac5..2d39cafbd 100644 --- a/docfx/Llvm.Net.Docfx.csproj +++ b/docfx/Llvm.Net.Docfx.csproj @@ -9,8 +9,8 @@ - - + + @@ -54,7 +54,7 @@ - + diff --git a/docfx/articles/Samples/Kaleidoscope-ch2.md b/docfx/articles/Samples/Kaleidoscope-ch2.md index 6822fbe09..3e2bf9800 100644 --- a/docfx/articles/Samples/Kaleidoscope-ch2.md +++ b/docfx/articles/Samples/Kaleidoscope-ch2.md @@ -367,7 +367,7 @@ In particular Chapters 3-7 only really differ in the language level support. [!code-csharp[Program.cs](../../../Samples/Kaleidoscope/Chapter2/Program.cs#generatorloop)] -The ReplParserStack contains the support for parsing the Kaleidoscope language from the REPL loop interactive input. +The ParserStack contains the support for parsing the Kaleidoscope language from the REPL loop interactive input. The stack also maintains the global state of the runtime, which controls the language features enabled, and if user defined operators are enabled, contains the operators defined along with their precedence. diff --git a/docfx/articles/Samples/Kaleidoscope-ch6.md b/docfx/articles/Samples/Kaleidoscope-ch6.md index 38722eadf..9b976d057 100644 --- a/docfx/articles/Samples/Kaleidoscope-ch6.md +++ b/docfx/articles/Samples/Kaleidoscope-ch6.md @@ -126,18 +126,28 @@ internal int GetNextPrecedence( int tokenType ) } ``` -This provides the core ability for looking up and handling precedence. Though as shown so far it is a rather convoluted form of what ANTLR4 gives -us for free. The real point of this runtime state is the ability of the application to add or remove user operators. (The built-in operators are -not removable) By adding operators to the runtime state the lookup process will include them during parsing. Thus, whenever visiting an operator -definition it is generated as a normal function with a specialized name and the operator and precedence are added to the runtime state. Any future -use of the operator in an expression will detect the correct precedence and the code generation treats it as a function call with the appropriate +This provides the core ability for looking up and handling precedence. Though, as shown so far, it is just a rather convoluted form of what +ANTLR4 gives us for free. The real point of this runtime state is the ability of the language to dynamically add user operators. By adding +operators to the runtime state the lookup process will include them during parsing. Thus, whenever visiting an operator definition it is +generated as a normal function with a specialized name and the operator and precedence are added to the runtime state. Any future use of +the operator in an expression will detect the correct precedence and the code generation treats it as a function call with the appropriate left and right hand argument values. -[!code-csharp[Main](../../../Samples/Kaleidoscope/Chapter6/CodeGenerator.cs#VisitUserOperators)] +Actually adding the operators to the table is handled in the parsing process itself using a feature of the ANTLR generated parser called a +"Parse Listener". A parse listener is attached to the parser and effectively monitors the entire parsing process. For the user operators the +listener will listen for the specific case of a complete function definition that defines a user operator. When it detects such a case it will +update the runtime table accordingly. + +[!code-csharp[UserOperatorListener](../../../Samples/Kaleidoscope/Kaleidoscope.Runtime/KaleidoscopeUserOperatorListener.cs)] + +With the use of the listener the dynamic precedence is contained in the parsing allowing the generator and later stages to remain blissfully ignorant +of the issue of precedence. Operator definitions are treated as function definitions. + +[!code-csharp[VisitUserOperators](../../../Samples/Kaleidoscope/Chapter6/CodeGenerator.cs#VisitUserOperators)] Finally, the default case for EmitBinaryOperator() is updated to actually emit a call to the binary operator function generated. -[!code-csharp[Main](../../../Samples/Kaleidoscope/Chapter6/CodeGenerator.cs#EmitUserOperator)] +[!code-csharp[EmitUserOperator](../../../Samples/Kaleidoscope/Chapter6/CodeGenerator.cs#EmitUserOperator)] That completes the support for user defined operators. diff --git a/docfx/articles/Samples/Kaleidoscope-ch8.md b/docfx/articles/Samples/Kaleidoscope-ch8.md index 43f003334..525300be4 100644 --- a/docfx/articles/Samples/Kaleidoscope-ch8.md +++ b/docfx/articles/Samples/Kaleidoscope-ch8.md @@ -12,178 +12,78 @@ LLVM uses a "Triple" string to describe the target used for code generation. Thi Fortunately, it is normally not required to build such strings directly. +## Grammar +In the preceding chapters the Kaleidoscope implementation provided an interactive JIT based on the classic +Read Evaluate Print Loop (REPL). So the grammar focused on a top level rule "repl" that processes individual +expressions one at a time. For native compilation this complicates the process of parsing and processing a +complete file. To handle these two distinct scenarios the grammar has different rules. For the interactive +scenario the previously mentioned "repl" rule is used. When parsing a full source file the "fullsrc" rule +is used. + +```antlr +// Full source parse accepts a series of definitions or prototypes, all top level expressions +// are generated into a single function called Main() +fullsrc + : repl*; +``` + +This rule simply accepts any number of expressions so that a single source file is parsed to a single +complete parse tree. (This particular point will become even more valuable when generating debug information +in [Chapter 9](Kaleidoscope-ch9.md) as the parse tree nodes contain the source location information based on the input stream). ## Code Generation Changes The changes in code generation are fairly straight forward and consist of the following basic steps. 1. Remove JIT engine support 2. Expose the bit code module generated, so it is available to the "driver". 3. Saving the target machine (since it doesn't come from the JIT anymore) - -The changes are shown below in classic 'diff' form since each change doesn't really need a detailed explanation. -(Nothing even remotely close to "Rocket Science" here. 8^) ) -```diff -diff --git a/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs b/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs -index 89e9b2ad8..1e7597c78 100644 ---- a/Samples/Kaleidoscope/Chapter7/CodeGenerator.cs -+++ b/Samples/Kaleidoscope/Chapter8/CodeGenerator.cs -@@ -3,9 +3,7 @@ - // - - using System; --using System.Collections.Generic; - using System.Linq; --using System.Runtime.InteropServices; - using Antlr4.Runtime; - using Antlr4.Runtime.Misc; - using Antlr4.Runtime.Tree; -@@ -13,7 +11,6 @@ using Kaleidoscope.Grammar; - using Kaleidoscope.Runtime; - using Llvm.NET; - using Llvm.NET.Instructions; --using Llvm.NET.JIT; - using Llvm.NET.Transforms; - using Llvm.NET.Values; - -@@ -30,24 +27,24 @@ namespace Kaleidoscope - , IKaleidoscopeCodeGenerator - { - // -- public CodeGenerator( DynamicRuntimeState globalState ) -+ public CodeGenerator( DynamicRuntimeState globalState, TargetMachine machine ) - { - RuntimeState = globalState; - Context = new Context( ); -- JIT = new KaleidoscopeJIT( ); -+ TargetMachine = machine; - InitializeModuleAndPassManager( ); - InstructionBuilder = new InstructionBuilder( Context ); - FunctionPrototypes = new PrototypeCollection( ); -- FunctionModuleMap = new Dictionary( ); - NamedValues = new ScopeStack( ); - } - // - - public bool DisableOptimizations { get; set; } - -+ public BitcodeModule Module { get; private set; } -+ - public void Dispose( ) - { -- JIT.Dispose( ); - Context.Dispose( ); - } - -@@ -112,23 +109,18 @@ namespace Kaleidoscope - { - return DefineFunction( ( Function )context.Signature.Accept( this ) - , context.BodyExpression -- ).Function; -+ ); - } - // - - // - public override Value VisitTopLevelExpression( [NotNull] TopLevelExpressionContext context ) - { -- var proto = new Prototype( $"anon_expr_{AnonNameIndex++}" ); -- var function = GetOrDeclareFunction( proto, isAnonymous: true ); -- -- var (_, jitHandle) = DefineFunction( function, context.expression( ) ); -+ var function = GetOrDeclareFunction( new Prototype( $"anon_expr_{AnonNameIndex++}" ) -+ , isAnonymous: true -+ ); - -- var nativeFunc = JIT.GetDelegateForFunction( proto.Identifier.Name ); -- var retVal = Context.CreateConstant( nativeFunc( ) ); -- FunctionModuleMap.Remove( function.Name ); -- JIT.RemoveModule( jitHandle ); -- return retVal; -+ return DefineFunction( function, context.expression( ) ); - } - // - -@@ -452,7 +444,8 @@ namespace Kaleidoscope - private void InitializeModuleAndPassManager( ) - { - Module = Context.CreateBitcodeModule( ); -- Module.Layout = JIT.TargetMachine.TargetData; -+ Module.TargetTriple = TargetMachine.Triple; -+ Module.Layout = TargetMachine.TargetData; - FunctionPassManager = new FunctionPassManager( Module ); - FunctionPassManager.AddPromoteMemoryToRegisterPass( ) - .AddInstructionCombiningPass( ) -@@ -514,25 +507,13 @@ namespace Kaleidoscope - // - - // -- private (Function Function, IJitModuleHandle JitHandle) DefineFunction( Function function, ExpressionContext body ) -+ private Function DefineFunction( Function function, ExpressionContext body ) - { - if( !function.IsDeclaration ) - { - throw new CodeGeneratorException( $"Function {function.Name} cannot be redefined in the same module" ); - } - -- // Destroy any previously generated module for this function. -- // This allows re-definition as the new module will provide the -- // implementation. This is needed, otherwise both the MCJIT -- // and OrcJit engines will resolve to the original module, despite -- // claims to the contrary in the official tutorial text. (Though, -- // to be fair it may have been true in the original JIT and might -- // still be true for the interpreter) -- if( FunctionModuleMap.Remove( function.Name, out IJitModuleHandle handle ) ) -- { -- JIT.RemoveModule( handle ); -- } -- - var basicBlock = function.AppendBasicBlock( "entry" ); - InstructionBuilder.PositionAtEnd( basicBlock ); - using( NamedValues.EnterScope( ) ) -@@ -548,7 +529,7 @@ namespace Kaleidoscope - if( funcReturn == null ) - { - function.EraseFromParent( ); -- return (null, default); -+ return null; - } - - InstructionBuilder.Return( funcReturn ); -@@ -560,10 +541,7 @@ namespace Kaleidoscope - FunctionPassManager.Run( function ); - } - -- var jitHandle = JIT.AddModule( Module ); -- FunctionModuleMap.Add( function.Name, jitHandle ); -- InitializeModuleAndPassManager( ); -- return (function, jitHandle); -+ return function; - } - // - -@@ -585,18 +563,11 @@ namespace Kaleidoscope - private readonly DynamicRuntimeState RuntimeState; - private static int AnonNameIndex; - private readonly Context Context; -- private BitcodeModule Module; - private readonly InstructionBuilder InstructionBuilder; - private readonly ScopeStack NamedValues; -- private readonly KaleidoscopeJIT JIT; -- private readonly Dictionary FunctionModuleMap; - private FunctionPassManager FunctionPassManager; -+ private TargetMachine TargetMachine; - private readonly PrototypeCollection FunctionPrototypes; -- -- /// Delegate type to allow execution of a JIT'd TopLevelExpression -- /// Result of evaluating the expression -- [UnmanagedFunctionPointer( System.Runtime.InteropServices.CallingConvention.Cdecl )] -- private delegate double AnonExpressionFunc( ); - // - } - } +4. Keep track of all generated top level anonymous expressions +5. Once generating from the parse tree is complete generate a main() that includes calls to all the +previously generated anonymous expressions. + +Most of these steps are pretty straight forward. The anonymous function handling is a bit distinct. +Since the language syntax allows anonymous expressions throughout the source file, and they don't +actually execute during generation they need to be organized into an executable form. Thus, a new +list of the generated functions is maintained and after the tree is generated a new main() function +is created and a call to each anonymous expression is made with a second call to printd() to show +the results - just like they would appear if typed in an interactive console. A trick used in the +code generation is to mark each of the anonymous functions as private and always inline. + +```C# +// mark anonymous functions as always-inline and private so they can be inlined and then removed +if( isAnonymous ) +{ + retVal.AddAttribute( FunctionAttributeIndex.Function, AttributeKind.AlwaysInline ) + .Linkage( Linkage.Private ); +} +else +{ + retVal.Linkage( Linkage.External ); +} ``` +These settings are leveraged after generating from the tree to create the main function. A simple +loop generate a call to each expression and then generates the call to print the results. Once, that +is completed a [ModulePassManager](xref:Llvm.NET.Transforms.ModulePassManager) is created to run +the Always inliner and a global dead code elimination pass. The always inliner will inline the functions +marked as inline and the dead code elimination pass will eliminate unused internal/private global symbols. +This has the effect of generating the main function with all top level expressions inlined and the originally +generated anonymous functions removed. + +[!code-csharp[Generate](../../../Samples/Kaleidoscope/Chapter8/CodeGenerator.cs#Generate)] + +Most of the rest of the changes are pretty straightforward following the steps listed previously. The larger +changes are shown here. You can run your favorite diff tool between the two sample folders to see all the little +changes. + +### VisitTopLevelExpression +As previously mentioned when generating the top level expression the resulting function is added to the +list of anonymous functions to generate a call to it from main(). + +[!code-csharp[VisitTopLevelExpression](../../../Samples/Kaleidoscope/Chapter8/CodeGenerator.cs#VisitTopLevelExpression)] + + ## Driver changes -The bulk of the work needed to generate object files is in the "driver" application code. The changes +To support generating object files the "driver" application code needs some alterations. The changes fall into two general categories: 1. Command line argument handling @@ -191,7 +91,7 @@ fall into two general categories: ### Adding Command Line handling To allow providing a file like a traditional compiler the driver app needs to have some basic -command line argument handling. ("Basic" in this case means truly rudimentary ::Grin:: ) +command line argument handling. ("Basic" in this case means truly rudimentary :grin: ) Generally this just gets a viable file path to use for the source code. [!code-csharp[ProcessArgs](../../../Samples/Kaleidoscope/Chapter8/Program.cs#ProcessArgs)] @@ -202,17 +102,19 @@ here either. The general plan is: 1. Process the arguments to get the path to compile 2. Open the file for reading 3. Create a new target machine from the default triple of the host -4. Build the parser loop - specifying the file as the input source (instead of the default console) -5. Remove the REPL loop ready state console output handling as compilation isn't interactive -6. Once the parsing has completed, verify the module and emit the object file -7. For diagnostics use, also emit the LLVM IR textual form +4. Create the parser stack +5. Parse the input file +6. Generate the IR code from the parse tree +7. Once the parsing has completed, verify the module and emit the object file +8. For diagnostics use, also emit the LLVM IR textual form and assembly files [!code-csharp[Main](../../../Samples/Kaleidoscope/Chapter8/Program.cs#Main)] ## Conclusion -That's it - seriously! Very little change was needed, mostly deleting code. Looking at the changes -it should be clear that it is possible to support runtime choice between JIT and full native compilation -instead of deleting the JIT code. (Implementing this feature is "left as an exercise for the reader" ::Grin::) +That's it - seriously! Very little change was needed, mostly deleting code and adding the special handling of the +anonymous expressions. Looking at the changes it should be clear that it is possible to support runtime choice +between JIT and full native compilation instead of deleting the JIT code. (Implementing this feature is +"left as an exercise for the reader" :wink:) diff --git a/docfx/articles/Samples/Kaleidoscope-ch9.md b/docfx/articles/Samples/Kaleidoscope-ch9.md index fc3f6d8b4..46092a5d1 100644 --- a/docfx/articles/Samples/Kaleidoscope-ch9.md +++ b/docfx/articles/Samples/Kaleidoscope-ch9.md @@ -1,5 +1,111 @@ -# Topic Header -Summary +# 9. Kaleidoscop: Adding Debug Information +So far in the progress of the Kaleidoscope tutorials we've covered the basics of the language as a JIT engine +and even added ahead of time compilation into the mix so it is a full static compiled language. But what happens +if something goes wrong in one of the programs written in Kaleidoscope? How can a developer debug applications +written in this wonderful new language? Up until now, the answer is, you can't. This chapter will add debugging +information to the generated object file so that it is available for debuggers. -## Sub Section -... \ No newline at end of file +Source level debugging uses formatted data bound into the output binaries that helps the debugger map the state +of the application to the original source code that created it. The exact format of the data depends on the target +platform but the general idea holds for all of them. In order to abstract the actual format from LLVM front-end +developers LLVM uses an abstract form of debug data that is based on the common DWARF debugging format. Internally, +the LLVM target will transform the abstract representation into the actual target binary form. + +>[!NOTE] +>Debugging JIT code is rather complex as it requires awareness of the runtime within the debugger to +>control the execution state. Therefore, the focus in this chapter is on the static compilation scenarion (building +>on the suport created in [Chapter 8](Kaleidoscope-ch8.md)) + +## Why is it a hard problem? +Debugging is a tough problem for a number of reasons, mostly revolving around optimized code. Optimizations make +keeping source level information more difficult. In LLVM the original source location information is attached to +each LLVM IR instruction. Optimization passes should keep the source location for any new instructions created, +but merged instructions only get to keep a single source location. This is generally the cause of the observed +"jumping around" when debugging optimized code. Additionally, optimizations can move variables in ways that are +either optimized out, shared in memory, in registers or otherwise difficult to track. Thus, for the purposes of +this tutorial we'll disable optimizations. (The DisableOptimizations property of the CodeGenerator was added previously +to aid in observing the effects of optimizations and will serve to disable the optimizations for debugging in this +chapter.) + +## Setup for emitting debug information +Debug information in Llvm.NET is created with the [DebugInfoBuilder](xref:Llvm.NET.DebugInfo.DebugInfoBuilder). +This is similar to the [InstructionBuilder](xref:Llvm.NET.Instructions.InstructionBuilder). Using the DebugInfoBuilder +requires a bit more knowledge on the general concepts of the DWARF debugging format, and in particular the [Debugging +Metadata](xref:llvm_sourcelevel_debugging) in LLVM. In Llvm.NET you don't need to, and in fact can't, create an instance +of the DebugInfoBuilder class. Instead it is lazy constructed internally to a [BitcodeModule](xref:Llvm.NET.BitcodeModule) +and accessible through the [DIBuilder](xref:Llvm.NET.BitcodeModule.DIBuilder) property. This simplifies creating the +builder since it is bound to the module. + +Another important item for debug information is called the Compilation Unit. In Llvm.NET that is the [DICompileUnit](xref:Llvm.NET.DebugInfo.DICompileUnit). +The compile unit is the top level scope for storing debug information, there is only ever one per module and generally +it represents the full source file that was used to create the module. Since the compile unit, like the builder is +really tied to the module it is exposed as the [DIBuilder](xref:Llvm.NET.BitcodeModule.DIBuilder) property. However, +unlike a builder it isn't something that a module can automatically construct without more information. Therefore, +Llvm.NET provides overloads for the creation of a module that includes the additional data needed to create the DICompileUnit +for you. + +The updated InitializeModuleAndPassManager() function looks like this: + +[!code-csharp[InitializeModuleAndPassManager](../../../Samples/Kaleidoscope/Chapter9/CodeGenerator.cs#InitializeModuleAndPassManager)] + +There are a few points of interest here. First the compile unit is created for the Kaleidoscope language, however it is using +the [SourceLanguage.C](xref:Llvm.NET.DebugInfo.SourceLanguage.C) value. This is because a debugger won't likely understand the +Kaleidoscope language, runtime, or calling conventions. (We just invented it and only now setting up debugger support after all!) +The good news is that the language follows the C language ABI in the code generation (generally a good idea unless you ave a really +good reason not to). Therefore, the C language is fairly accurate. This allows calling functions from the debugger and it will execute +them. + +Another point to note is that the module ID is derived from the source file path and the source file path is provided so that it becomes +the root compile unit. + +>[!IMPORTANT] +>It is important to note that when using the DIBuilder it must be "finalized" inorder to resolve internal forward references in the +>debug metadata. The exact details of this aren't generally relevant, just remember that somewhere after generating all code and debug +>information to call the [Finish](xref:Llvm.NET.DebugInfo.DebugInfoBuilder.Finish(Llvm.NET.DebugInfo.DISubProgram)) method. (In Llvm.NET +> this method is called Finish() to avoid conflicts with the .NET runtime defined Finalize() and to avoid confusion on the term as the +> idea of "finalization" has a very different meaning in .NET then what happens here). + +The tutorial takes care of finishing the debug information in Main after the running the parsing loop to process all the input text for +the source file. + +```C# +replLoop.Run( ); +generator.Module.DIBuilder.Finish( ); +``` +## Functions +With the basics of the DIBuilder and DICompile unit setup for the module it is time to focus on providing debug information for functions. +This requires adding a few extra calls to build the context of the debug information for the function. The DWARF debug format that LLVM's +debug metadata is based on calls these "SubPrograms". The new code builds a representation of the file the code is contained in as a new +[DIFile](xref:Llvm.NET.DebugInfo.DIFile). In this case that is a bit redundant as all the code comes from a single file and the compile +unit already has the file info. However, that's not always true for all languages (i.e. some sort of Include mechanism) so the file is +created. It's not a problem as LLVM will intern the file definition so that it won't actually end up with duplicates. + + +[!code-csharp[GetIrDeclareFunction](../../../Samples/Kaleidoscope/Chapter9/CodeGenerator.cs#GetOrDeclareFunction)] + +## Debug Locations +The parse tree contains full location information for each parsed node in the tree. This allows building debug location information for +each node fairly easily. The general idea is to set the location in the InstructionBuilder so that it is applied to all instructions +emitted until it is changed. This saves on manually adding the location on every instruction. + +[!code-csharp[EmitLocation](../../../Samples/Kaleidoscope/Chapter9/CodeGenerator.cs#EmitLocation)] + +## DefineFunction +The next step is to update the function definition with attached debug information. The definition starts by pushing a new lexical scope +that is the functions declaration. This serves as the parent scope for all the debug information generated for the function's implementation. +The debug location info is cleared from the builder to set up all the parameter variables with alloca, as before. Additionally, the debug +information for each parameter is constructed. After the function is fully generated the debug information for the function is finalized, this +is needed to allow for any optimizations to occur at the function level. + +[!code-csharp[DefineFunction](../../../Samples/Kaleidoscope/Chapter9/CodeGenerator.cs#DefineFunction)] + +## Debug info for Parameters and Local Variables +Debug information for parameters and local variables is similar but not quite identical. Thus two new overloads of the CreateEntryBlockAlloca() +handle attaching the correct debug information for parameters and local variables. + +[!code-csharp[CreateEntryBlockAlloca](../../../Samples/Kaleidoscope/Chapter9/CodeGenerator.cs#CreateEntryBlockAlloca)] + +## Conclusion +Adding debugging information in LLVM IR is rather straight forward. The bulk of the problem is in tracking the source location information in the +parser. Fortunately for us, ANTLR4 generated parsers do this for us already, so combining the parser with Llvm.NET makes building a full compiler +for custom languages, including debug support. diff --git a/src/Llvm.NET/BitcodeModule.cs b/src/Llvm.NET/BitcodeModule.cs index 6a1848f72..a89d3882a 100644 --- a/src/Llvm.NET/BitcodeModule.cs +++ b/src/Llvm.NET/BitcodeModule.cs @@ -134,6 +134,7 @@ public IReadOnlyDictionary ModuleFlags } /// Gets the used to create debug information for this module + /// The builder returned from this property is lazy constructed on first access so doesn't consume resources unless used. public DebugInfoBuilder DIBuilder { get