From 04ce02c20abbd493a6de6ab54e8ac07bfc7bc960 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 7 Oct 2024 16:26:27 -0400 Subject: [PATCH] Resolved issue with parallel delete. --- .../Interactions/Management/IndexManager.cs | 4 +- .../StaticAnalysis/StaticAnalyzer.cs | 14 +- TestApplications/Benchmark/PayloadModel.cs | 26 + .../Benchmark/Scripts/0000.0 - Regression.kbs | 472 ++++++++++++++++++ 4 files changed, 510 insertions(+), 6 deletions(-) create mode 100644 TestApplications/Benchmark/PayloadModel.cs create mode 100644 TestApplications/Benchmark/Scripts/0000.0 - Regression.kbs diff --git a/NTDLS.Katzebase.Engine/Interactions/Management/IndexManager.cs b/NTDLS.Katzebase.Engine/Interactions/Management/IndexManager.cs index 7d860fa5..ff9952d1 100644 --- a/NTDLS.Katzebase.Engine/Interactions/Management/IndexManager.cs +++ b/NTDLS.Katzebase.Engine/Interactions/Management/IndexManager.cs @@ -999,7 +999,7 @@ private void RemoveDocumentsFromIndex(Transaction transaction, PhysicalSchema ph { try { - var childPool = _core.ThreadPool.Indexing.CreateChildPool(_core.Settings.IndexingChildThreadPoolQueueDepth); + var childPool = _core.ThreadPool.Indexing.CreateChildPool(_core.Settings.IndexingChildThreadPoolQueueDepth); for (uint indexPartition = 0; indexPartition < physicalIndex.Partitions; indexPartition++) { @@ -1009,7 +1009,7 @@ private void RemoveDocumentsFromIndex(Transaction transaction, PhysicalSchema ph } var ptThreadQueue = transaction.Instrumentation.CreateToken(PerformanceCounter.ThreadQueue); - childPool.Enqueue(() => + childPool.Enqueue(indexPartition, (uint indexPartition) => { #region Thread. diff --git a/NTDLS.Katzebase.Management/StaticAnalysis/StaticAnalyzer.cs b/NTDLS.Katzebase.Management/StaticAnalysis/StaticAnalyzer.cs index 2e50d599..b365aa3d 100644 --- a/NTDLS.Katzebase.Management/StaticAnalysis/StaticAnalyzer.cs +++ b/NTDLS.Katzebase.Management/StaticAnalysis/StaticAnalyzer.cs @@ -111,10 +111,16 @@ static Action AddSyntaxError(TextDocument textDocument, TextMarkerService textMa return; } - var line = textDocument.GetLineByNumber(lineNumber.EnsureNotNull()); - int startOffset = line.Offset; - int length = line.Length; - textMarkerService.Create(startOffset, length, message, Colors.Red); + try + { + var line = textDocument.GetLineByNumber(lineNumber.EnsureNotNull()); + int startOffset = line.Offset; + int length = line.Length; + textMarkerService.Create(startOffset, length, message, Colors.Red); + } + catch (Exception ex) + { + } }; } } diff --git a/TestApplications/Benchmark/PayloadModel.cs b/TestApplications/Benchmark/PayloadModel.cs new file mode 100644 index 00000000..8d58c82d --- /dev/null +++ b/TestApplications/Benchmark/PayloadModel.cs @@ -0,0 +1,26 @@ +namespace Benchmark +{ + internal class PayloadModel + { + public int Id { get; set; } + public int AcceptedAnswerId { get; set; } + public int AnswerCount { get; set; } + public string? Body { get; set; } + public DateTime? ClosedDate { get; set; } + public int CommentCount { get; set; } + public DateTime? CommunityOwnedDate { get; set; } + public DateTime? CreationDate { get; set; } + public int FavoriteCount { get; set; } + public DateTime? LastActivityDate { get; set; } + public DateTime? LastEditDate { get; set; } + public string? LastEditorDisplayName { get; set; } + public int LastEditorUserId { get; set; } + public int OwnerUserId { get; set; } + public int ParentId { get; set; } + public int PostTypeId { get; set; } + public int Score { get; set; } + public string? Tags { get; set; } + public string? Title { get; set; } + public int ViewCount { get; set; } + } +} diff --git a/TestApplications/Benchmark/Scripts/0000.0 - Regression.kbs b/TestApplications/Benchmark/Scripts/0000.0 - Regression.kbs new file mode 100644 index 00000000..09c11de2 --- /dev/null +++ b/TestApplications/Benchmark/Scripts/0000.0 - Regression.kbs @@ -0,0 +1,472 @@ +--Test 'SELECT *' +SELECT + * +FROM + WordList:Language + +--Test WHERE equals. +SELECT + * +FROM + WordList:Language +WHERE + Name = 'French' + +--Test WHERE with equals. +SELECT + * +FROM + WordList:Language as L +WHERE + L.Name = 'French' + +--Test SELECT * with alias. +SELECT + L.* +FROM + WordList:Language as L +WHERE + L.Name = 'French' + +--Test WHERE LIKE '%...'. +SELECT + * +FROM + WordList:Language +WHERE + Name LIKE 'F%' + +--Test WHERE LIKE '...%'. +SELECT + * +FROM + WordList:Language +WHERE + Name LIKE '%h' + +--Test WHERE NOT LIKE '%...'. +SELECT + * +FROM + WordList:Language +WHERE + Name NOT LIKE 'F%' + +--Test WHERE NOT LIKE '...%'. +SELECT + Name +FROM + WordList:Language +WHERE + Name = 'French' + +--Test SELECT column which does not exist. +SELECT + Id, + ColumnWhichDoesNotExist, + Name +FROM + WordList:Language +WHERE + Name = 'French' + +--Test WHERE column which does not exist. +SELECT + Name +FROM + WordList:Language +WHERE + ColumnWhichDoesNotExist = 'French' + +--Test WHERE column which does not exist OR valid condition. +SELECT + Name +FROM + WordList:Language +WHERE + Name = 'French' + OR ColumnWhichDoesNotExist = 'French' + +--Test WHERE column which does not exist AND valid condition +SELECT + Name +FROM + WordList:Language +WHERE + Name = 'French' + OR ColumnWhichDoesNotExist = 'French' + +--Test GROUP BY with ORDER BY using aggregate ORDER BY. +SELECT + sw.Text as SourceWord, + tw.Text as TargetWord, + Avg(sw.Id) as Average, + Count(0) as CountOf +FROM + WordList:Word as sw +INNER JOIN WordList:Language as sl + ON sl.Id = sw.LanguageId +INNER JOIN WordList:Synonym as s + ON s.SourceWordId = sw.Id +INNER JOIN WordList:Word as tw + ON tw.Id = s.TargetWordId +INNER JOIN WordList:Language as tl + ON tl.Id = tw.LanguageId +WHERE + sw.Text = 'Car' + OR sw.Text = 'Gym' +GROUP BY + sw.Text, + tw.Text +ORDER BY + Avg(sw.Id / 60) asc, + sw.Text desc + +--Test aggregate without group by. +SELECT + Avg(sw.Id) as Average, + Count(0) as CountOf +FROM + WordList:Word as sw +INNER JOIN WordList:Language as sl + ON sl.Id = sw.LanguageId +INNER JOIN WordList:Synonym as s + ON s.SourceWordId = sw.Id +INNER JOIN WordList:Word as tw + ON tw.Id = s.TargetWordId +INNER JOIN WordList:Language as tl + ON tl.Id = tw.LanguageId +WHERE + sw.Text = 'Car' + OR sw.Text = 'Gym' + +--Test TOP. +SELECT TOP 10 + * +FROM + WordList:Word as sw +INNER JOIN WordList:Language as sl + ON sl.Id = sw.LanguageId +INNER JOIN WordList:Synonym as s + ON s.SourceWordId = sw.Id +INNER JOIN WordList:Word as tw + ON tw.Id = s.TargetWordId +INNER JOIN WordList:Language as tl + ON tl.Id = tw.LanguageId +WHERE + sw.Text = 'Car' + OR sw.Text = 'Gym' + +--Test TOP with OFFSET. +SELECT TOP 10 + * +FROM + WordList:Word as sw +INNER JOIN WordList:Language as sl + ON sl.Id = sw.LanguageId +INNER JOIN WordList:Synonym as s + ON s.SourceWordId = sw.Id +INNER JOIN WordList:Word as tw + ON tw.Id = s.TargetWordId +INNER JOIN WordList:Language as tl + ON tl.Id = tw.LanguageId +WHERE + sw.Text = 'Car' + OR sw.Text = 'Gym' +OFFSET 100 + +DROP SCHEMA Temporary:Payload1 +DROP SCHEMA Temporary:Payload2 + +--Test SELECT INTO. +SELECT + sw.Id as SourceWordId, + sw.Text as SourceWord, + sl.Name as SourceLanguage, + tw.Id as TargetWordId, + tw.Text as TargetWord, + tl.Name as TargetLanguage +INTO + Temporary:Payload1 +FROM + WordList:Word as sw +INNER JOIN WordList:Language as sl + ON sl.Id = sw.LanguageId +INNER JOIN WordList:Synonym as s + ON s.SourceWordId = sw.Id +INNER JOIN WordList:Word as tw + ON tw.Id = s.TargetWordId +INNER JOIN WordList:Language as tl + ON tl.Id = tw.LanguageId +WHERE + sw.Text = 'Car' + OR sw.Text = 'Gym' + OR sw.Text = 'Moon' + +SELECT * INTO Temporary:Payload2 FROM Temporary:Payload1 + +--Test DELETE from schema. +DELETE FROM Temporary:Payload1 WHERE TargetLanguage = 'French' + +--Test DELETE from alias. +DELETE + p1 +FROM + Temporary:Payload1 as p1 +WHERE + p1.TargetLanguage = 'German' + +--Test DELETE from alias with JOIN. +DELETE + p1 +FROM + Temporary:Payload1 as p1 +INNER JOIN WordList:Language as L + ON L.Name = P1.TargetLanguage +WHERE + L.Id = 1 --English + +--Test JOIN. +SELECT + T1.* +FROM + Temporary:Payload1 as T1 +INNER JOIN WordList:Word as T2 + ON T2.Id = T1.SourceWordId + +--Test UPDATE. +UPDATE + Temporary:Payload1 +SET + TargetLanguage = 'UPDATED ' + TargetLanguage +WHERE + TargetLanguage = 'Finnish' + +--Test UPDATE with join. +UPDATE + p1 +SET + TargetLanguage = l.Name +FROM + Temporary:Payload1 as p1 +INNER JOIN WordList:Word as w + ON w.Id = p1.TargetWordId +INNER JOIN WordList:Language as l + ON l.Id = W.LanguageId +WHERE + p1.TargetLanguage LIKE 'Updated %' + +DROP SCHEMA Temporary:InsertTest1 +CREATE SCHEMA Temporary:InsertTest1 + +--Test VALUES INSERT. +INSERT INTO Temporary:InsertTest1(FirstName, LastName) +VALUES('Jane', 'Doe'),('John', 'Doe') + +--Test VALUES INSERT with expression. +INSERT INTO Temporary:InsertTest1(Id, FirstName, LastName) +VALUES(1, 'Jane', 'Doe'),(2, 'John', 'Doe'),(3, 'Test', Guid() + ' (some expression)') + +DROP SCHEMA Temporary:InsertTest2 +CREATE SCHEMA Temporary:InsertTest2 + +--Test RAGGED INSERT. +INSERT INTO Temporary:InsertTest2 +(FirstName = 'John', LastName = 'Doe'), +(FirstName = 'Fred', MiddleName = 'Joe', LastName = 'Doe'), +(FirstName = 'Jane', LastName = 'Doe') + + +--Test RAGGED INSERT with expression +INSERT INTO Temporary:InsertTest2 +(FirstName = 'John', LastName = 'Doe'), +(FirstName = 'Fred', MiddleName = Guid(), LastName = 'Doe'), +(FirstName = 'Jane', LastName = 'Doe') + +---Test complex SELECT and complex WHERE. +SELECT + Sha1('t'), + 11 ^ (2 + 1) + 'ten' + (Length('A10CharStr') * 10 + 2), + 6 * -1 as Negative, + 10 + 10 + (11 ^ 3) + 10 + '->' + Guid(), + 10 + 10 + 'ten' + 10 * 10, + 'ten (' + 10 * 10 + ') : ' + DateTimeUTC('yyyy/MM/dd hh:mm:ss tt') +FROM + Single +WHERE + 1 = 1 + and 1 != 3 + and ( + 10 = 10 + and 5 = 5 + and ( + 10 = 10 + and length(guid()) = 36 + and 5 = 5 + ) + ) +ORDER BY + 10 + +--Test index creation. +CREATE INDEX ix_Temporary_SourceWord_TargetWord +( + SourceWord, + TargetWord +) ON Temporary:Payload1 + +--Test index rebuild. +REBUILD INDEX ix_Temporary_SourceWord_TargetWord ON Temporary:Payload1 +REBUILD INDEX ix_Temporary_SourceWord_TargetWord ON Temporary:Payload1 WITH (Partitions = 1) +REBUILD INDEX ix_Temporary_SourceWord_TargetWord ON Temporary:Payload1 WITH (Partitions = 10) +REBUILD INDEX ix_Temporary_SourceWord_TargetWord ON Temporary:Payload1 WITH (Partitions = 100) + +--Test index analyze. +ANALYZE INDEX ix_Temporary_SourceWord_TargetWord ON Temporary:Payload1 + +--Test unique key creation. +CREATE UNIQUEKEY uk_Temporary_SourceWordId_TargetWordId +( + SourceWordId, + TargetWordId +) ON Temporary:Payload1 + +--Test unique key rebuild. +REBUILD INDEX uk_Temporary_SourceWordId_TargetWordId ON Temporary:Payload1 +REBUILD INDEX uk_Temporary_SourceWordId_TargetWordId ON Temporary:Payload1 WITH (Partitions = 1) +REBUILD INDEX uk_Temporary_SourceWordId_TargetWordId ON Temporary:Payload1 WITH (Partitions = 10) +REBUILD INDEX uk_Temporary_SourceWordId_TargetWordId ON Temporary:Payload1 WITH (Partitions = 100) + +--Test unique key analyze. +ANALYZE INDEX uk_Temporary_SourceWordId_TargetWordId ON Temporary:Payload1 + +--Test analysis. +ANALYZE SCHEMA Temporary:Payload1 +ANALYZE SCHEMA Temporary:Payload1 WITH (IncludePhysicalPages = false) +ANALYZE SCHEMA Temporary:Payload1 WITH (IncludePhysicalPages = true) + +--Test ALTER schema. +ALTER schema Temporary:Payload1 WITH (PageSize = 100) + +--Test functions: + +exec CheckpointHealthCounters() +exec ClearCacheAllocations() +exec ClearHealthCounters() +exec ReleaseCacheAllocations() +exec ShowAggregateFunctions() +exec ShowBlocks() +exec ShowBlocks(1) +exec ShowBlocks(null) +exec ShowBlockTree() +exec ShowCacheAllocations() +exec ShowCachePages() +exec ShowCachePartitions() +exec ShowHealthCounters() +exec ShowLocks() +exec ShowLocks(1) +exec ShowLocks(null) +exec ShowMemoryUtilization() +exec ShowProcesses() +exec ShowProcesses(1) +exec ShowProcesses(null) +exec ShowScalarFunctions() +exec ShowSystemFunctions() +exec ShowTransactions() +exec ShowTransactions(1) +exec ShowTransactions(null) +exec ShowVersion() +exec ShowVersion(true) +exec ShowVersion(false) +exec ShowWaitingLocks() +exec ShowWaitingLocks(1) +exec ShowWaitingLocks(null) +exec Cancel(0) --pid:0 never exists, so this is not a great test. +exec Terminate(0) --pid:0 never exists, so this is not a great test. +exec ShowThreadPools() + +--Test scalar functions: +SELECT IsBetween(50, 10, 100) FROM Single +SELECT IsBetween(200, 10, 100) FROM Single +SELECT IsEqual('string1', 'string2') FROM Single +SELECT IsGreater(10, 5) FROM Single +SELECT IsGreaterOrEqual(10, 5) FROM Single +SELECT IsLess(10, 5) FROM Single +SELECT IsLessOrEqual(10, 5) FROM Single +SELECT IsLike('Hello World', '%o w%') FROM Single +SELECT IsNotBetween(10, 5, 100) FROM Single +SELECT IsNotEqual('string1', 'string2') FROM Single +SELECT IsNotLike('Hello World', '%o w%') FROM Single +SELECT IsInteger('123456') FROM Single +SELECT IsString('123456') FROM Single +SELECT IsDouble('123456') FROM Single +SELECT Checksum('hello world') FROM Single +SELECT LastIndexOf('text', 'This is some text and this is more text.') FROM Single +SELECT Length('Hello World') FROM Single +SELECT DateTime('yyyy-MM-dd HH:mm:ss.ms') FROM Single +SELECT DateTimeUTC('yyyy-MM-dd HH:mm:ss.ms') FROM Single +SELECT DocumentID('h') FROM Single +SELECT DocumentPage('') FROM Single +SELECT DocumentUID('') FROM Single +SELECT Guid() FROM Single +SELECT IndexOf('text', 'This is some text and this is more text.') FROM Single +SELECT Left('Hello world', 5) FROM Single +SELECT Right('Hello world', 5) FROM Single +SELECT Sha1('Hello world') FROM Single +SELECT Sha256('Hello world') FROM Single +SELECT Sha512('Hello world') FROM Single +SELECT SubString('Hello cruel world', 6, 5) FROM Single +SELECT ToLower('Hello cruel world') FROM Single +SELECT ToProper('Hello cruel world') FROM Single +SELECT ToUpper('Hello cruel world') FROM Single +SELECT Concat('hello', ' ', 'world') FROM Single +SELECT Coalesce(null, null, 'hello', 'world') FROM Single +SELECT Trim('hello world', 'dhle') FROM Single +SELECT Trim(' hello world ') FROM Single +SELECT IIF(IsGreater(Length ('Hello World'), 10), 'Is True', 'Is False') FROM Single + +--Test aggregate functions +SELECT + Avg(L.Id) as Avg, + Count(L.Id) as Count, + CountDistinct(L.Name, true) as CaseSensitiveDistinct, + CountDistinct(L.Name, false) as CaseInsensitiveDistinct, + GeometricMean(L.Id) as GeometricMean, + Min(L.Id) as Min, + Max(L.Id) as Max, + Mean(L.Id) as Mean, + Median(L.Id) as Median, + Mode(L.Id) as Mode, + Sum(L.Id) as Sum, + Variance(L.Id) as Variance, + MinString(L.Name) as MinString, + MaxString(L.Name) as MaxString, + Sha1Agg(L.Name) as Sha1Agg, + Sha256Agg(L.Name) as Sha256Agg, + Sha512Agg(L.Name) as Sha512Agg +FROM + WordList:Language as L + +--Test aggregate functions +SELECT + Avg(L.Id) as Avg, + Count(L.Id) as Count, + CountDistinct(L.Text, true) as CaseSensitiveDistinct, + CountDistinct(L.Text, false) as CaseInsensitiveDistinct, + GeometricMean(L.Id) as GeometricMean, + Min(L.Id) as Min, + Max(L.Id) as Max, + Mean(L.Id) as Mean, + Median(L.Id) as Median, + Mode(L.Id) as Mode, + Sum(L.Id) as Sum, + Variance(L.Id) as Variance, + MinString(L.Text) as MinString, + MaxString(L.Text) as MaxString, + Sha1Agg(L.Text) as Sha1Agg, + Sha256Agg(L.Text) as Sha256Agg, + Sha512Agg(L.Text) as Sha512Agg +FROM + WordList:Word as L +WHERE + L.Text LIKE '%oad%'