Merge pull request #18 from bash/newlines-as-trivia

Broccolini.Test/ParserTest.cs

@@ -28,9 +28,7 @@ public Property ParsesArbitraryComment(string commentValue)
     {
         var input = $"; {commentValue}";
         var document = Parse(input);
-        return (document.NodesOutsideSection.Count == 1
-                && document.NodesOutsideSection.First() is CommentIniNode triviaNode
-                && triviaNode.ToString() == input)
+        return (document.NodesOutsideSection is [CommentIniNode commentNode] && commentNode.ToString() == input)
             .ToProperty()
             .When(!input.Contains('\r') && !input.Contains('\n'));
     }
@@ -67,16 +65,14 @@ public void ParsesSectionNames(string name, string input)
     public bool ParsesArbitrarySectionName(SectionName name, Whitespace ws1, Whitespace ws2, Whitespace ws3, InlineText trailing)
     {
         var document = Parse($"{ws1.Value}[{ws2.Value}{name.Value}{ws3.Value}]{trailing.Value}");
-        return (document.Sections.Count == 1
-            && document.Sections[0].Name == name.Value);
+        return document.Sections is [{ Name: var actualName }] && actualName == name.Value;
     }
 
     [Property]
     public bool ParsesArbitrarySectionNameWithoutClosingBracket(SectionName name, Whitespace ws1, Whitespace ws2, Whitespace ws3)
     {
         var document = Parse($"{ws1.Value}[{ws2.Value}{name.Value}{ws3.Value}");
-        return (document.Sections.Count == 1
-            && document.Sections[0].Name == name.Value);
+        return document.Sections is [{ Name: var actualName }] && actualName == name.Value;
     }
 
     public static TheoryData<string, string> GetSectionNameData()

Broccolini.Test/RoundtripTest.cs

@@ -28,7 +28,7 @@ public static TheoryData<string> PreservesFormattingData()
         => Sequence.Concat(
             CommentNodes,
             GarbageNodes,
-            LeadingNodes,
+            LeadingNodesOrTrivia,
             NewLines,
             SectionsWithNames.Select(s => s.Input),
             KeyValuePairsWithKeyAndValue.Select(s => s.Input)).ToTheoryData();

Broccolini.Test/TestData.cs

@@ -1,3 +1,6 @@
+using Broccolini.Syntax;
+using Broccolini.Tokenization;
+
 namespace Broccolini.Test;
 
 internal static class TestData
@@ -26,9 +29,19 @@ public static IEnumerable<string> LeadingNodes
             "; comment\r\n" +
             "garbage\r\n",
             "[section]\r\n",
-            "\r\n",
             "key = value\r\n");
 
+    public static IEnumerable<string> InlineTrivia
+        => ["\t", "    ", ""];
+
+    public static IEnumerable<string> LineBreakingTrivia
+        => ContextFreeNewLines
+            .Concat(ContextFreeNewLines.SelectMany(_ => InlineTrivia, (nl, inline) => $"{nl}{inline}{nl}"))
+            .Append("");
+
+    public static IEnumerable<string> LeadingNodesOrTrivia
+        => LeadingNodes.Concat(InlineTrivia.SelectMany(_ => LineBreakingTrivia, (inline, breaking) => inline + breaking));
+
     public static IEnumerable<SectionWithName> SectionsWithNames
         => Sequence.Return(
             new SectionWithName("[", string.Empty),
@@ -70,7 +83,9 @@ public static IEnumerable<KeyValuePairWithKeyAndValue> KeyValuePairsWithKeyAndVa
             .SelectMany(VaryLeadingNewLines);
             // TODO: vary leading and trailing whitespace and line break
 
-    public static IEnumerable<string> NewLines => Sequence.Return("\r\n", "\r", "\n");
+    public static IEnumerable<string> NewLines => ["\r\n", "\r", "\n"];
+
+    public static IEnumerable<string> ContextFreeNewLines => ["\r\n", "\n"];
 
     public static IEnumerable<CaseSensitivityInput> CaseSensitivityInputs
         => Sequence.Return(
@@ -89,6 +104,15 @@ public static IEnumerable<char> WhiteSpace
             .Select(n => (char)n)
             .Except(Sequence.Return('\r', '\n'));
 
+    public static IEnumerable<ExampleNode> ExampleNodes
+        => [
+            new UnrecognizedIniNode(Tokenizer.Tokenize("garbage")) { NewLine = new IniToken.NewLine("\n") },
+            new CommentIniNode("comment") { NewLine = new IniToken.NewLine("\n") },
+            new KeyValueIniNode("key", "value") { NewLine = new IniToken.NewLine("\n") },
+            new SectionIniNode(new SectionHeaderIniNode("section") { NewLine = new IniToken.NewLine("\n") }, []),
+            new SectionIniNode(new SectionHeaderIniNode("section") { NewLine = new IniToken.NewLine("\n") }, [new KeyValueIniNode("child-key", "value") { NewLine = new IniToken.NewLine("\n") }]),
+           ];
+
     private static IEnumerable<KeyValuePairWithKeyAndValue> KeyValuePairsWithQuotes
         => Sequence.Return(
             new KeyValuePairWithKeyAndValue("\"quoted key\" = \"quoted value\"", "\"quoted key\"", "quoted value"),
@@ -151,3 +175,8 @@ public sealed record SectionWithName(string Input, string Name);
 public sealed record KeyValuePairWithKeyAndValue(string Input, string Key, string Value);
 
 public sealed record CaseSensitivityInput(string Variant1, string Variant2, bool ShouldBeEqual);
+
+public sealed record ExampleNode(IniNode Value)
+{
+    public static implicit operator ExampleNode(IniNode node) => new(node);
+}

Broccolini.Test/TriviaTest.cs

@@ -0,0 +1,146 @@
+using System.Diagnostics;
+using Broccolini.Editing;
+using Broccolini.Syntax;
+using FsCheck;
+using Xunit;
+using static Broccolini.IniParser;
+using static Broccolini.Test.TestData;
+using static Broccolini.Tokenization.Tokenizer;
+
+namespace Broccolini.Test;
+
+public sealed class TriviaTest
+{
+    [Fact]
+    public void Example()
+    {
+        var input =
+            """
+            \t
+            \t[section]\t
+            \t
+            \tkey=value\t
+            \t\n
+            """
+            .Replace(@"\t", "\t")
+            .Replace(@"\n", "\n")
+            .ReplaceLineEndings("\n");
+
+        var sectionHeader = new SectionHeaderIniNode("section")
+        {
+            LeadingTrivia = Tokenize("\t"),
+            TrailingTrivia = Tokenize("\t\n\t"),
+            NewLine = new IniToken.NewLine("\n"),
+        };
+
+        var keyValue = new KeyValueIniNode("key", "value")
+        {
+            LeadingTrivia = Tokenize("\t"),
+            TrailingTrivia = Tokenize("\t"),
+            NewLine = new IniToken.NewLine("\n"),
+        };
+
+        var section = new SectionIniNode(sectionHeader, [keyValue])
+        {
+            LeadingTrivia = Tokenize("\t\n"),
+            TrailingTrivia = Tokenize("\t\n"),
+        };
+
+        var expectedDocument = IniDocument.Empty with
+        {
+            Sections = [section],
+        };
+        Assert.Equal(expectedDocument.ToString(), input); // Sanity check
+        var parsed = Parse(input);
+        Assert.Equal(expectedDocument, parsed);
+    }
+
+    [Theory]
+    [MemberData(nameof(LeadingTriviaData))]
+    public void RecognizedLeadingWhitespaceAndNewLinesAsTrivia(IniNode node)
+    {
+        var expectedDocument = ToIniDocument(node);
+        var parsedDocument = Parse(node.ToString());
+        Assert.Equal(expectedDocument, parsedDocument);
+    }
+
+    public static TheoryData<IniNode> LeadingTriviaData()
+       => (from node in ExampleNodes
+           from breaking in LineBreakingTrivia
+           from inline in InlineTrivia
+           from inlineBeforeBreaking in InlineTrivia
+           where (inlineBeforeBreaking.Length == 0) == (breaking.Length == 0)
+           select ApplyLeadingTrivia(node.Value, inlineBeforeBreaking + breaking, inline)).ToTheoryData();
+
+    private static IniNode ApplyLeadingTrivia(IniNode node, string trivia, string inlineTrivia)
+        => node switch
+        {
+            SectionIniNode section => section with { LeadingTrivia = Tokenize(trivia), Header = section.Header with { LeadingTrivia = Tokenize(inlineTrivia) } },
+            _ => node with { LeadingTrivia = Tokenize(trivia + inlineTrivia) },
+        };
+
+    [Theory]
+    [MemberData(nameof(TrailingTriviaData))]
+    public void RecognizedTrailingWhitespaceAndNewLinesAsTrivia(IniNode node)
+    {
+        var expectedDocument = ToIniDocument(node);
+        var parsedDocument = Parse(expectedDocument.ToString());
+        Assert.Equal(expectedDocument.ToString(), parsedDocument.ToString()); // Sanity check
+        Assert.Equal(expectedDocument, parsedDocument);
+    }
+
+    private static TheoryData<IniNode> TrailingTriviaData()
+       => (from node in ExampleNodes
+           from inline in InlineTrivia
+           from breaking in LineBreakingTrivia
+           from inlineAfterBreaking in InlineTrivia
+           where (inlineAfterBreaking.Length == 0) == (breaking.Length == 0)
+           select ApplyTrailingTrivia(node.Value, inline, breaking + inlineAfterBreaking)).ToTheoryData();
+
+    [Theory]
+    [MemberData(nameof(TriviaForConsecutiveNodes))]
+    public void RecognizedWhitespaceAndNewLinesAsTriviaForConsecutiveNodes(IniNode a, IniNode b)
+    {
+        var expectedDocument = Append(ToIniDocument(a), b);
+        var parsedDocument = Parse(expectedDocument.ToString());
+        Assert.Equal(expectedDocument.ToString(), parsedDocument.ToString()); // Sanity check
+        Assert.Equal(expectedDocument, parsedDocument);
+    }
+
+    private static TheoryData<IniNode, IniNode> TriviaForConsecutiveNodes()
+       => (from node1 in ExampleNodes
+           from node2 in ExampleNodes
+           from inline in InlineTrivia
+           from breaking in LineBreakingTrivia
+           from inlineLeading in InlineTrivia
+           select (ApplyTrailingTrivia(node1.Value, inline, breaking), ApplyLeadingTrivia(node2.Value, "", inlineLeading))).ToTheoryData();
+
+    private static IniNode ApplyTrailingTrivia(IniNode node, string inlineTrivia, string trivia)
+        => node switch
+        {
+            SectionIniNode section => section with { TrailingTrivia = Tokenize(trivia), Header = section.Header with { TrailingTrivia = Tokenize(inlineTrivia) } },
+            _ => node with { TrailingTrivia = Tokenize(inlineTrivia + trivia) },
+        };
+
+    private static IniDocument ToIniDocument(IniNode node)
+        => Append(IniDocument.Empty, node);
+
+    private static IniDocument Append(IniDocument document, IniNode node)
+    {
+        document = document.EnsureTrailingNewLine(new IniToken.NewLine("\n"));
+        return node switch
+        {
+            SectionIniNode section => document with { Sections = document.Sections.Add(section) },
+            SectionChildIniNode child when document.Sections.Any() => AppendToLastSection(document, child),
+            SectionChildIniNode child => document with { NodesOutsideSection = document.NodesOutsideSection.Add(child) },
+            _ => throw new UnreachableException(),
+        };
+    }
+
+    private static IniDocument AppendToLastSection(IniDocument document, SectionChildIniNode node)
+    {
+        var lastSection = document.Sections.Last();
+        var updatedSection = lastSection with { Children = lastSection.Children.Add(node) };
+        return document with { Sections = document.Sections.SetItem(document.Sections.Count - 1, updatedSection) };
+    }
+}

Broccolini/Broccolini.csproj

@@ -6,7 +6,7 @@
         <IsTrimmable>true</IsTrimmable>
     </PropertyGroup>
     <PropertyGroup Label="NuGet Packing">
-        <Version>1.0.1</Version>
+        <Version>2.0.0-rc.1</Version>
         <Description>Broccolini is a non-destructive parser for INI files compatible with GetPrivateProfileString.</Description>
         <Authors>Tau Gärtli</Authors>
         <PackageLicenseExpression>MIT</PackageLicenseExpression>

Broccolini/Compatibility/DistinctBy.cs

@@ -6,7 +6,7 @@
 
 namespace System.Linq;
 
-internal static class EnumerableCompatibility
+internal static partial class EnumerableCompatibility
 {
     public static IEnumerable<TSource> DistinctBy<TSource, TKey>(this IEnumerable<TSource> source, Func<TSource, TKey> keySelector, IEqualityComparer<TKey>? comparer = null)
     {

Broccolini/Compatibility/SkipLast.cs

@@ -0,0 +1,35 @@
+#if NETSTANDARD2_0
+// Source: https://github.com/dotnet/runtime/blob/v5.0.18/src/libraries/System.Linq/src/System/Linq/Skip.cs
+
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Linq;
+
+internal static partial class EnumerableCompatibility
+{
+    public static IEnumerable<TSource> SkipLast<TSource>(this IEnumerable<TSource> source, int count)
+    {
+        var queue = new Queue<TSource>();
+        using IEnumerator<TSource> e = source.GetEnumerator();
+        while (e.MoveNext())
+        {
+            if (queue.Count == count)
+            {
+                do
+                {
+                    yield return queue.Dequeue();
+                    queue.Enqueue(e.Current);
+                }
+                while (e.MoveNext());
+                break;
+            }
+            else
+            {
+                queue.Enqueue(e.Current);
+            }
+        }
+    }
+}
+
+#endif

Broccolini/Editing/EditingExtensions.Section.cs

@@ -36,20 +36,7 @@ public static SectionIniNode RemoveKeyValue(this SectionIniNode sectionNode, str
 
     private static SectionIniNode AppendChild(this SectionIniNode sectionNode, SectionChildIniNode node)
     {
-        return sectionNode.Children.TryFindIndex(IsBlank, out var index)
-            ? InsertAtIndex(index)
-            : AppendTrailing();
-
-        SectionIniNode InsertAtIndex(int childIndex)
-            => sectionNode with { Children = sectionNode.Children.Insert(childIndex, node.EnsureTrailingNewLine(sectionNode.DetectNewLine())) };
-
-        SectionIniNode AppendTrailing()
-        {
-            var sectionWithNewLine = sectionNode.EnsureTrailingNewLine(sectionNode.DetectNewLine());
-            return sectionWithNewLine with { Children = sectionWithNewLine.Children.Add(node) };
-        }
-
-        static bool IsBlank(SectionChildIniNode node)
-            => node is UnrecognizedIniNode unrecognizedNode && unrecognizedNode.IsBlank();
+        var sectionWithNewLine = sectionNode.EnsureTrailingNewLine(sectionNode.DetectNewLine());
+        return sectionWithNewLine with { Children = sectionWithNewLine.Children.Add(node) };
     }
 }

Broccolini/Editing/NewlineDetectionExtensions.cs

@@ -8,10 +8,11 @@ internal static class NewlineDetectionExtensions
 
     public static IniToken.NewLine DetectNewLine(this IniDocument document)
         => document.GetNodes()
+            .OfType<IIniNodeWithNewLine>()
             .Select(n => n.NewLine)
             .FirstOrDefault()
                 ?? NativeNewLine;
 
     public static IniToken.NewLine DetectNewLine(this SectionIniNode node)
-        => node.NewLine ?? node.NewLineHint ?? NativeNewLine;
+        => node.Header.NewLine ?? node.NewLineHint ?? NativeNewLine;
 }

Broccolini/Editing/NewlineExtensions.cs

@@ -16,7 +16,7 @@ public static SectionIniNode EnsureTrailingNewLine(this SectionIniNode node, Ini
         => node switch
         {
             { Children: { Count: >=1 } children } => node with { Children = children.ReplaceLast(n => EnsureTrailingNewLine(n, newLine)) },
-            { Children.Count: 0, NewLine: null } => node with { NewLine = newLine },
+            { Children.Count: 0, Header: { NewLine: null } header } => node with { Header = header with { NewLine = newLine } },
             _ => node,
         };
 

Broccolini/Parsing/EnumerableExtensions.cs

@@ -0,0 +1,34 @@
+using Broccolini.Syntax;
+
+namespace Broccolini.Parsing;
+
+internal static class EnumerableExtensions
+{
+    public static IniToken FirstOrEpsilon(this IEnumerable<IniToken> tokens)
+        => tokens.FirstOrDefault() ?? IniToken.Epsilon.Instance;
+
+    public static TSource FirstOrDefault<TSource>(this IEnumerable<TSource> source, TSource @default)
+        => source.Append(@default).First();
+
+    public static IEnumerable<TSource> DropLast<TSource>(this IEnumerable<TSource> source, Func<TSource, bool> predicate)
+        => source.WithLast().Where(x => !x.IsLast || !predicate(x.Item)).Select(x => x.Item);
+
+    private static IEnumerable<(TSource Item, bool IsLast)> WithLast<TSource>(this IEnumerable<TSource> source)
+    {
+        using var enumerator = source.GetEnumerator();
+
+        if (!enumerator.MoveNext())
+        {
+            yield break;
+        }
+
+        var current = enumerator.Current;
+        while (enumerator.MoveNext())
+        {
+            yield return (current, false);
+            current = enumerator.Current;
+        }
+
+        yield return (current, true);
+    }
+}

Broccolini/Parsing/IParserInput.cs

@@ -4,7 +4,13 @@ namespace Broccolini.Parsing;
 
 internal interface IParserInput
 {
+    int AvailableLength { get; }
+
     IniToken Peek(int lookAhead = 0);
 
+    IEnumerable<IniToken> PeekRange();
+
     IniToken Read();
+
+    ImmutableArray<IniToken> Read(IEnumerable<IniToken> peeked);
 }