Merge pull request #120 from PositiveTechnologies/dev

TextSpan for reduced nodes and C# conversion fixes.
PositiveTechnologies · Jan 21, 2018 · a605082 · a605082
2 parents b184c3d + 01fa4f5
commit a605082
Show file tree

Hide file tree

Showing 24 changed files with 287 additions and 65 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,8 +1,6 @@
 [submodule "Sources/AspxParser"]
 	path = Sources/AspxParser
 	url = https://github.com/PositiveTechnologies/AspxParser.git
-	branch = master
 [submodule "Sources/antlr-grammars-v4"]
 	path = Sources/antlr-grammars-v4
 	url = https://github.com/PositiveTechnologies/grammars-v4.git
-	branch = master
diff --git a/Sources/PT.PM.CSharpParseTreeUst/Converter/ExpressionVisitor.cs b/Sources/PT.PM.CSharpParseTreeUst/Converter/ExpressionVisitor.cs
@@ -12,7 +12,6 @@
 using PT.PM.Common.Exceptions;
 using System.Collections.Generic;
 using PT.PM.Common.Nodes.Tokens.Literals;
-//using Microsoft.CodeAnalysis.FindSymbols;
 
 namespace PT.PM.CSharpParseTreeUst.RoslynUstVisitor
 {

diff --git a/Sources/PT.PM.CSharpParseTreeUst/Converter/StatementVisitor.cs b/Sources/PT.PM.CSharpParseTreeUst/Converter/StatementVisitor.cs
@@ -28,8 +28,8 @@ public override Ust VisitLocalDeclarationStatement(LocalDeclarationStatementSynt
                 node.Declaration.Variables.Select(v => (AssignmentExpression)VisitAndReturnNullIfError(v))
                 .ToArray();
 
-            var resultExpression = new VariableDeclarationExpression(type, variables, node.GetTextSpan());
-            var result = new ExpressionStatement(resultExpression);
+            var resultExpression = new VariableDeclarationExpression(type, variables, node.Declaration.GetTextSpan());
+            var result = new ExpressionStatement(resultExpression, node.GetTextSpan());
             return result;
         }
 

diff --git a/Sources/PT.PM.Cli/CliParameters.cs b/Sources/PT.PM.Cli/CliParameters.cs
@@ -9,10 +9,10 @@ public class CliParameters
         [Option('f', "files", HelpText = "Input file or directory to be processed")]
         public string InputFileNameOrDirectory { get; set; } = "";
 
-        [Option('l', "languages", HelpText = "Languages to be processed")]
+        [Option('l', "languages", HelpText = "Languages to be processed (at least -f or -p parameter required)")]
         public string Languages { get; set; } = "";
 
-        [Option('p', "patterns", HelpText = "Patterns to be processed (json or base64 encoded)")]
+        [Option('p', "patterns", HelpText = "Patterns to be processed (at least -f or -p parameter required)")]
         public string Patterns { get; set; } = "";
 
         [Option('t', "threads", HelpText = "Number of processing threads")]
@@ -57,9 +57,6 @@ public class CliParameters
         [Option('d', "dump", HelpText = "Stages to be dumped (ParseTree, Ust)")]
         public string DumpStages { get; set; } = "";
 
-        [Option('v', "version", HelpText = "Show version or not")]
-        public bool ShowVersion { get; set; } = true;
-
         [Option('r', "render", HelpText = "Stages to be rendered")]
         public string RenderStages { get; set; } = "";
 

diff --git a/Sources/PT.PM.Cli/CliProcessorBase.cs b/Sources/PT.PM.Cli/CliProcessorBase.cs
@@ -56,12 +56,9 @@ public int Convert(string[] args, CliParameters parameters)
 
             try
             {
-                if (parameters.ShowVersion)
-                {
-                    AssemblyName assemblyName = Assembly.GetEntryAssembly().GetName();
-                    string name = assemblyName.Name.Replace(".Cli", "");
-                    logger.LogInfo($"{name} version: {assemblyName.Version}");
-                }
+                AssemblyName assemblyName = Assembly.GetEntryAssembly().GetName();
+                string name = assemblyName.Name.Replace(".Cli", "");
+                logger.LogInfo($"{name} version: {assemblyName.Version}");
 
                 if (logger is FileLogger abstractLogger)
                 {
@@ -76,7 +73,7 @@ public int Convert(string[] args, CliParameters parameters)
 
                 if (string.IsNullOrEmpty(parameters.InputFileNameOrDirectory) && string.IsNullOrEmpty(parameters.Patterns))
                 {
-                    throw new ArgumentException("at least --files or --patterns parameter required");
+                    throw new ArgumentException("at least -f or -p parameter required");
                 }
 
                 if (!Enum.TryParse(parameters.Stage, true, out Stage pmStage))

diff --git a/Sources/PT.PM.Common/Json/JsonBaseSerializer.cs b/Sources/PT.PM.Common/Json/JsonBaseSerializer.cs
@@ -1,8 +1,6 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
-using PT.PM.Common.Nodes;
 using System.Collections.Generic;
-using System.Linq;
 
 namespace PT.PM.Common.Json
 {

diff --git a/Sources/PT.PM.Common/Json/JsonConverterBase.cs b/Sources/PT.PM.Common/Json/JsonConverterBase.cs
@@ -3,15 +3,20 @@
 using PT.PM.Common.Nodes;
 using PT.PM.Common.Reflection;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
 
 namespace PT.PM.Common.Json
 {
     public abstract class JsonConverterBase : JsonConverter, ILoggable
     {
+        private MultiMap<TextSpan, Ust> existingUsts = new MultiMap<TextSpan, Ust>();
+
         protected const string KindName = "Kind";
 
+        protected JsonSerializer jsonSerializer;
+
         public ILogger Logger { get; set; } = DummyLogger.Instance;
 
         public CodeFile JsonFile { get; } = CodeFile.Empty;
@@ -100,7 +105,7 @@ protected JObject GetJObject(object value, JsonSerializer serializer)
             return jObject;
         }
 
-        protected Ust CreateUst(object jObjectOrToken)
+        protected Ust CreateOrGetUst(object jObjectOrToken)
         {
             JObject jObject = jObjectOrToken as JObject;
             JToken jToken = jObject == null ? jObjectOrToken as JToken : null;
@@ -121,6 +126,26 @@ protected Ust CreateUst(object jObjectOrToken)
                 return null;
             }
 
+            JToken textSpanToken = jObject != null
+                ? jObject.GetValueIgnoreCase(nameof(Ust.TextSpan))
+                : jToken != null
+                ? jToken.GetValueIgnoreCase(nameof(Ust.TextSpan))
+                : null;
+
+            TextSpan textSpan = TextSpan.Empty;
+            if (textSpanToken != null)
+            {
+                textSpan = textSpanToken.ToObject<TextSpan>(jsonSerializer);
+                if (!textSpan.IsEmpty && existingUsts.TryGetValue(textSpan, out List<Ust> usts))
+                {
+                    var sameTypeUst = usts.FirstOrDefault(u => u.GetType() == type);
+                    if (sameTypeUst != null)
+                    {
+                        return sameTypeUst;
+                    }
+                }
+            }
+
             Ust ust;
             if (type == typeof(RootUst))
             {
@@ -141,6 +166,12 @@ protected Ust CreateUst(object jObjectOrToken)
                 ust = (Ust)Activator.CreateInstance(type);
             }
 
+            if (!textSpan.IsEmpty)
+            {
+                ust.TextSpan = textSpan;
+                existingUsts.Add(textSpan, ust);
+            }
+
             return ust;
         }
 

diff --git a/Sources/PT.PM.Common/Json/UstJsonConverter.cs b/Sources/PT.PM.Common/Json/UstJsonConverter.cs
@@ -21,6 +21,8 @@ public override bool CanConvert(Type objectType)
         public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
             JsonSerializer serializer)
         {
+            jsonSerializer = serializer;
+
             if (reader.TokenType == JsonToken.Null)
             {
                 return null;
@@ -35,7 +37,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             {
                 // Try load from Ust subfield.
                 JToken jToken = jObject.GetValueIgnoreCase(nameof(Ust));
-                target = CreateUst(jToken);
+                target = CreateOrGetUst(jToken);
                 if (target == null)
                 {
                     return null;
@@ -44,7 +46,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             }
             else
             {
-                target = CreateUst(jObject);
+                target = CreateOrGetUst(jObject);
                 if (target == null)
                 {
                     return null;

diff --git a/Sources/PT.PM.Common/MultiMap.cs b/Sources/PT.PM.Common/MultiMap.cs
@@ -0,0 +1,59 @@
+using System.Collections.Generic;
+
+namespace PT.PM.Common
+{
+    /// <summary>
+    /// An alternative to .NET Dictionary class that allows duplicate keys, 
+    /// and is still mutable (unlike the Lookup class).
+    /// </summary>
+    /// <remarks>
+    /// Inspired by http://stackoverflow.com/questions/146204/duplicate-keys-in-net-dictionaries
+    /// </remarks>
+    public class MultiMap<TKey, TValue>
+    {
+        private readonly Dictionary<TKey, List<TValue>> storage = new Dictionary<TKey, List<TValue>>();
+
+        public IEnumerable<TKey> Keys => storage.Keys;
+
+        public int Count => storage.Count;
+
+        public MultiMap()
+        {
+        }
+
+        public MultiMap(IEnumerable<KeyValuePair<TKey, TValue>> data)
+        {
+            foreach (KeyValuePair<TKey, TValue> pair in data)
+            {
+                Add(pair.Key, pair.Value);
+            }
+        }
+
+        public void Add(TKey key, TValue value)
+        {
+            if (!storage.TryGetValue(key, out List<TValue> storageValue))
+            {
+                storageValue = new List<TValue>();
+                storage[key] = storageValue;
+            }
+            storageValue.Add(value);
+        }
+
+        public void Clear()
+        {
+            storage.Clear();
+        }
+
+        public bool ContainsKey(TKey key)
+        {
+            return storage.ContainsKey(key);
+        }
+
+        public bool TryGetValue(TKey key, out List<TValue> value) => storage.TryGetValue(key, out value);
+
+        public List<TValue> this[TKey key] =>
+            storage.TryGetValue(key, out List<TValue> storageValue)
+            ? storageValue
+            : new List<TValue>();
+    }
+}
diff --git a/Sources/PT.PM.Common/Nodes/Expressions/AssignmentExpression.cs b/Sources/PT.PM.Common/Nodes/Expressions/AssignmentExpression.cs
@@ -23,7 +23,11 @@ public AssignmentExpression()
 
         public override string ToString()
         {
-            return Right == null ? Left.ToString() : $"{Left} = {Right}";
+            return Right == null
+                ? Left == null
+                ? " = "
+                : Left.ToString()
+                : $"{Left} = {Right}";
         }
     }
 }
diff --git a/Sources/PT.PM.Common/Nodes/Expressions/VariableDeclarationExpression.cs b/Sources/PT.PM.Common/Nodes/Expressions/VariableDeclarationExpression.cs
@@ -11,7 +11,7 @@ public class VariableDeclarationExpression : Expression
         public List<AssignmentExpression> Variables { get; set; }
 
         public VariableDeclarationExpression(TypeToken type, IEnumerable<AssignmentExpression> variables,
-            TextSpan textSpan)
+            TextSpan textSpan = default(TextSpan))
             : base(textSpan)
         {
             Type = type;

diff --git a/Sources/PT.PM.Common/Nodes/GeneralScope/TypeDeclaration.cs b/Sources/PT.PM.Common/Nodes/GeneralScope/TypeDeclaration.cs
@@ -34,6 +34,8 @@ public TypeDeclaration(TypeTypeLiteral type, IdToken name, List<TypeToken> baseT
 
         public TypeDeclaration()
         {
+            BaseTypes = new List<TypeToken>();
+            TypeMembers = new List<EntityDeclaration>();
         }
 
         public override Ust[] GetChildren()

diff --git a/Sources/PT.PM.Common/Nodes/Tokens/Literals/StringLiteral.cs b/Sources/PT.PM.Common/Nodes/Tokens/Literals/StringLiteral.cs
@@ -1,4 +1,7 @@
-using System;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace PT.PM.Common.Nodes.Tokens.Literals
 {
@@ -38,4 +41,19 @@ public override int CompareTo(Ust other)
 
         public override string ToString() => $"\"{Text}\"";
     }
+
+    public static class InitialTextSpanPopulate
+    {
+        public static void Populate(this List<TextSpan> textSpans, StringLiteral stringLiteral)
+        {
+            if(stringLiteral.InitialTextSpans.Any())
+            {
+                textSpans.AddRange(stringLiteral.InitialTextSpans);
+            }
+            else
+            {
+                textSpans.Add(stringLiteral.TextSpan);
+            }
+        }
+    }
 }
diff --git a/Sources/PT.PM.Common/Nodes/Ust.cs b/Sources/PT.PM.Common/Nodes/Ust.cs
@@ -1,5 +1,7 @@
-using System;
+using Newtonsoft.Json;
+using System;
 using System.Diagnostics;
+using System.Collections.Generic;
 using System.Text;
 
 namespace PT.PM.Common.Nodes
@@ -29,6 +31,14 @@ public abstract class Ust : IComparable<Ust>, IEquatable<Ust>, IUst<Ust, RootUst
 
         public string ToStringWithoutLineBreaks() => debuggerPrinter.Print(ToString());
 
+        /// <summary>
+        /// The list of text spans before any UST transformation or reduction.
+        /// For example it won't be empty for concatenation of several strings,
+        /// i.e. "1" + "2" + "3" -> "123".
+        /// These spans map on an original source code.
+        /// </summary>
+        public List<TextSpan> InitialTextSpans { get; set; } = new List<TextSpan>();
+
         protected Ust()
         {
         }