Skip to content

Commit

Permalink
With helper paralle foreach
Browse files Browse the repository at this point in the history
  • Loading branch information
sumanthkb committed Jan 10, 2025
1 parent 9560268 commit 1adcbf5
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 86 deletions.
1 change: 1 addition & 0 deletions src/LCT.Common/Constants/Dataconstant.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public static class Dataconstant
public const string Cdx_JfrogRepoPath = "internal:siemens:clearing:jfrog-repo-path";
public const string Cdx_Siemensfilename = "internal:siemens:clearing:siemens:filename";
public const string Cdx_SiemensDirect = "internal:siemens:clearing:siemens:direct";
public const int MaxDegreeOfParallelism = 8;

public static Dictionary<string, string> PurlCheck()
{
Expand Down
15 changes: 15 additions & 0 deletions src/LCT.Common/ProcessAsyncHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Diagnostics;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace LCT.Common
Expand Down Expand Up @@ -153,6 +154,20 @@ public static StringBuilder STDErrorHandler(Process process, List<Task> processT

}


public static async Task ProcessItemsAsync<T>(IEnumerable<T> items, Func<T, CancellationToken, ValueTask> action, ParallelOptions parallelOptions = null, CancellationToken cancellationToken = default)
{
if (parallelOptions == null)
{
parallelOptions = new ParallelOptions();
}

await Parallel.ForEachAsync(items, parallelOptions, async (item, ct) =>
{
await action(item, ct);
});
}

}

}
18 changes: 9 additions & 9 deletions src/LCT.SW360PackageCreator/ComponentCreator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;


Expand Down Expand Up @@ -58,14 +59,7 @@ public async Task<List<ComparisonBomData>> CycloneDxBomParser(CommonAppSettings
private async Task<List<Components>> GetListOfBomData(List<Component> components, CommonAppSettings appSettings)
{
List<Components> lstOfBomDataToBeCompared = new List<Components>();

//Degree for parallel processing for faster execution is set to 4.
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = 4
};

await Parallel.ForEachAsync(components, parallelOptions, async (item, cancellationToken) =>
Func<Component, CancellationToken, ValueTask> action = async (item, ct) =>
{
try
{
Expand Down Expand Up @@ -115,8 +109,14 @@ await Parallel.ForEachAsync(components, parallelOptions, async (item, cancellati
{
Logger.Error($"Error processing item {item.Name}: {ex.Message}", ex);
}
});
};


var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Dataconstant.MaxDegreeOfParallelism
};
await ProcessAsyncHelper.ProcessItemsAsync(components, action, parallelOptions);
return lstOfBomDataToBeCompared;
}

Expand Down
134 changes: 75 additions & 59 deletions src/LCT.SW360PackageCreator/CreatorHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
using System.Net;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using static System.Collections.Specialized.BitVector32;


namespace LCT.SW360PackageCreator
Expand Down Expand Up @@ -218,79 +220,93 @@ public async Task<List<ComparisonBomData>> SetContentsForComparisonBOM(List<Comp
private async Task<List<ComparisonBomData>> GetComparisionBomItems(List<Components> lstComponentForBOM, ISW360Service sw360Service)
{
List<ComparisonBomData> comparisonBomData = new();
ComparisonBomData mapper;
foreach (Components item in lstComponentForBOM)
{
mapper = new ComparisonBomData();
ReleasesInfo releasesInfo = await GetReleaseInfoFromSw360(item, componentsAvailableInSw360, sw360Service);
IRepository repo = new Repository();

mapper.Name = item.Name;
mapper.Group = item.Group;
mapper.Version = item.Version;
mapper.ComponentExternalId = item.ComponentExternalId;
mapper.ReleaseExternalId = item.ReleaseExternalId;
mapper.SourceUrl = item.SourceUrl;
mapper.DownloadUrl = item.DownloadUrl;
mapper.ComponentStatus = GetComponentAvailabilityStatus(componentsAvailableInSw360, item);
mapper.ReleaseStatus = IsReleaseAvailable(item.Name, item.Version, item.ReleaseExternalId);

if(!string.IsNullOrEmpty(mapper.ReleaseExternalId) && mapper.ReleaseStatus == Dataconstant.NotAvailable && string.IsNullOrEmpty(mapper.SourceUrl))
//ComparisonBomData mapper;

Func<Components, CancellationToken, ValueTask> action = async (item, ct) =>
{
try
{
if(item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["NPM"]))
ComparisonBomData mapper = new ComparisonBomData();
ReleasesInfo releasesInfo = await GetReleaseInfoFromSw360(item, componentsAvailableInSw360, sw360Service);
IRepository repo = new Repository();

mapper.Name = item.Name;
mapper.Group = item.Group;
mapper.Version = item.Version;
mapper.ComponentExternalId = item.ComponentExternalId;
mapper.ReleaseExternalId = item.ReleaseExternalId;
mapper.SourceUrl = item.SourceUrl;
mapper.DownloadUrl = item.DownloadUrl;
mapper.ComponentStatus = GetComponentAvailabilityStatus(componentsAvailableInSw360, item);
mapper.ReleaseStatus = IsReleaseAvailable(item.Name, item.Version, item.ReleaseExternalId);

if (!string.IsNullOrEmpty(mapper.ReleaseExternalId) && mapper.ReleaseStatus == Dataconstant.NotAvailable && string.IsNullOrEmpty(mapper.SourceUrl))
{
if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["NPM"]))
{
mapper.SourceUrl = UrlHelper.Instance.GetSourceUrlForNpmPackage(mapper.Name, mapper.Version);
}
else if(item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["NUGET"]))
mapper.SourceUrl = UrlHelper.Instance.GetSourceUrlForNpmPackage(mapper.Name, mapper.Version);
}
else if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["NUGET"]))
{
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForNugetPackage(mapper.Name, mapper.Version);
}
else if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"]))
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForNugetPackage(mapper.Name, mapper.Version);
}
else if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"]))
{
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForPythonPackage(mapper.Name, mapper.Version);
}
else if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["CONAN"]))
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForPythonPackage(mapper.Name, mapper.Version);
}
else if (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["CONAN"]))
{
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForConanPackage(mapper.Name, mapper.Version);
mapper.SourceUrl = await UrlHelper.Instance.GetSourceUrlForConanPackage(mapper.Name, mapper.Version);
}
}
}


mapper.AlpineSource = item.AlpineSourceData;
if (!string.IsNullOrEmpty(item.ReleaseExternalId) && item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["DEBIAN"]))
{
if ((string.IsNullOrEmpty(item.SourceUrl) || item.SourceUrl == Dataconstant.SourceUrlNotFound) && !string.IsNullOrEmpty(releasesInfo.SourceCodeDownloadUrl))
mapper.AlpineSource = item.AlpineSourceData;
if (!string.IsNullOrEmpty(item.ReleaseExternalId) && item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["DEBIAN"]))
{
// If not able to get source details from snapshot.org, try getting source URL from SW360
mapper.SourceUrl = releasesInfo.SourceCodeDownloadUrl;
mapper.DownloadUrl = releasesInfo.SourceCodeDownloadUrl;
if ((string.IsNullOrEmpty(item.SourceUrl) || item.SourceUrl == Dataconstant.SourceUrlNotFound) && !string.IsNullOrEmpty(releasesInfo.SourceCodeDownloadUrl))
{
// If not able to get source details from snapshot.org, try getting source URL from SW360
mapper.SourceUrl = releasesInfo.SourceCodeDownloadUrl;
mapper.DownloadUrl = releasesInfo.SourceCodeDownloadUrl;
}
mapper.PatchURls = item.PatchURLs;
}
mapper.PatchURls = item.PatchURLs;
}
else if (!string.IsNullOrEmpty(item.ReleaseExternalId) && item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["MAVEN"]))
{
mapper.DownloadUrl = GetMavenDownloadUrl(mapper, item, releasesInfo);
}
else if (!string.IsNullOrEmpty(item.ReleaseExternalId) &&
(item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["CONAN"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["ALPINE"])))
{
mapper.DownloadUrl = mapper.SourceUrl;
else if (!string.IsNullOrEmpty(item.ReleaseExternalId) && item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["MAVEN"]))
{
mapper.DownloadUrl = GetMavenDownloadUrl(mapper, item, releasesInfo);
}
else if (!string.IsNullOrEmpty(item.ReleaseExternalId) &&
(item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["CONAN"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["ALPINE"])))
{
mapper.DownloadUrl = mapper.SourceUrl;
}
else
{
mapper.DownloadUrl = GetComponentDownloadUrl(mapper, item, repo, releasesInfo);
}
mapper.ApprovedStatus = GetApprovedStatus(mapper.ComponentStatus, mapper.ReleaseStatus, releasesInfo);
mapper.IsComponentCreated = GetCreatedStatus(mapper.ComponentStatus);
mapper.IsReleaseCreated = GetCreatedStatus(mapper.ReleaseStatus);
mapper.FossologyUploadStatus = GetFossologyUploadStatus(mapper.ApprovedStatus);
mapper.ReleaseAttachmentLink = string.Empty;
mapper.ReleaseLink = GetReleaseLink(componentsAvailableInSw360, item.Name, item.Version);

Logger.Debug($"Sw360 avilability status for Name " + mapper.Name + ":" + mapper.ComponentExternalId + "=" + mapper.ComponentStatus +
"-Version " + mapper.Version + ":" + mapper.ReleaseExternalId + "=" + mapper.ReleaseStatus);
comparisonBomData.Add(mapper);
}
else
catch (Exception ex)
{
mapper.DownloadUrl = GetComponentDownloadUrl(mapper, item, repo, releasesInfo);
Logger.Error($"GetComparisionBomItems() : Error processing item {item.Name}: {ex.Message}", ex);
}
mapper.ApprovedStatus = GetApprovedStatus(mapper.ComponentStatus, mapper.ReleaseStatus, releasesInfo);
mapper.IsComponentCreated = GetCreatedStatus(mapper.ComponentStatus);
mapper.IsReleaseCreated = GetCreatedStatus(mapper.ReleaseStatus);
mapper.FossologyUploadStatus = GetFossologyUploadStatus(mapper.ApprovedStatus);
mapper.ReleaseAttachmentLink = string.Empty;
mapper.ReleaseLink = GetReleaseLink(componentsAvailableInSw360, item.Name, item.Version);
};

Logger.Debug($"Sw360 avilability status for Name " + mapper.Name + ":" + mapper.ComponentExternalId + "=" + mapper.ComponentStatus +
"-Version " + mapper.Version + ":" + mapper.ReleaseExternalId + "=" + mapper.ReleaseStatus);
comparisonBomData.Add(mapper);
}
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Dataconstant.MaxDegreeOfParallelism
};
await ProcessAsyncHelper.ProcessItemsAsync(lstComponentForBOM, action, parallelOptions);
return comparisonBomData;
}

Expand Down
12 changes: 6 additions & 6 deletions src/LCT.Services/Sw360CreatorService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public async Task<ComponentCreateStatus> CreateComponentBasesOFswComaprisonBOM(
else
{
componentCreateStatus.IsCreated = false;
componentCreateStatus.ReleaseStatus.IsCreated = false;
componentCreateStatus.ReleaseStatus.IsCreated = false;
Environment.ExitCode = -1;
Logger.Debug($"CreateComponent():Component Name -{componentInfo.Name}- " +
$"response status code-{response.StatusCode} and reason pharase-{response.ReasonPhrase}");
Expand All @@ -95,7 +95,7 @@ public async Task<ComponentCreateStatus> CreateComponentBasesOFswComaprisonBOM(
}
catch (HttpRequestException e)
{
Logger.Error($"CreateComponent():", e);
Logger.Error($"CreateComponent():", e);
Environment.ExitCode = -1;
componentCreateStatus.IsCreated = false;
componentCreateStatus.ReleaseStatus.IsCreated = false;
Expand Down Expand Up @@ -179,7 +179,7 @@ public async Task<ReleaseCreateStatus> CreateReleaseForComponent(ComparisonBomDa
else
{
createStatus.IsCreated = false;

Environment.ExitCode = -1;
Logger.Debug($"CreateReleaseForComponent():Component Name -{componentInfo.Name}{componentInfo.Version}- " +
$"response status code-{response.StatusCode} and reason pharase-{response.ReasonPhrase}");
Expand Down Expand Up @@ -302,7 +302,7 @@ public async Task<bool> LinkReleasesToProject(List<ReleaseLinked> releasesTobeLi
var response = await m_SW360ApiCommunicationFacade.LinkReleasesToProject(content, sw360ProjectId);
if (!response.IsSuccessStatusCode)
{

Environment.ExitCode = -1;
Logger.Error($"LinkReleasesToProject() : Linking releases to project Id {sw360ProjectId} is failed.");
return false;
Expand Down Expand Up @@ -464,13 +464,13 @@ public async Task<bool> UpdatePurlIdForExistingComponent(ComparisonBomData cbomD
}
catch (HttpRequestException ex)
{
Logger.Error($"UpdateExternalIdForRelease(): {ex}");
Logger.Debug($"UpdateExternalIdForRelease(): {ex}" + " For Componenet :" + cbomData.Name);
Environment.ExitCode = -1;
return false;
}
catch (AggregateException ex)
{
Logger.Error($"UpdateExternalIdForRelease(): {ex}");
Logger.Debug($"UpdateExternalIdForRelease(): {ex}" + " For Componenet :" + cbomData.Name);
Environment.ExitCode = -1;
return false;
}
Expand Down
38 changes: 26 additions & 12 deletions src/LCT.Services/Sw360Service.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace LCT.Services
Expand Down Expand Up @@ -302,23 +303,36 @@ private async Task<List<Components>> GetAvailableComponenentsList(IList<Sw360Rel
return availableComponentList;
}

foreach (Components component in listOfComponentsToBom)
Func<Components, CancellationToken, ValueTask> action = async (component, ct) =>
{
if (await CheckReleaseExistenceByExternalId(component) ||
CheckAvailabilityByNameAndVersion(sw360Releases, component))
{
Logger.Debug($"GetAvailableComponenentsList(): Release Exist : Release name - {component.Name}, version - {component.Version}");
}
else if (await CheckComponentExistenceByExternalId(component) ||
CheckAvailabilityByName(sw360ComponentList, component))
try
{
Logger.Debug($"GetAvailableComponenentsList(): Compoennt Exist : Release name - {component.Name}, version - {component.Version}");
if (await CheckReleaseExistenceByExternalId(component) ||
CheckAvailabilityByNameAndVersion(sw360Releases, component))
{
Logger.Debug($"GetAvailableComponenentsList(): Release Exist : Release name - {component.Name}, version - {component.Version}");
}
else if (await CheckComponentExistenceByExternalId(component) ||
CheckAvailabilityByName(sw360ComponentList, component))
{
Logger.Debug($"GetAvailableComponenentsList(): Compoennt Exist : Release name - {component.Name}, version - {component.Version}");
}
else
{
// Do Nothing or to be implemented
}
}
else
catch (Exception ex)
{
// Do Nothing or to be implemented
Logger.Error($"GetAvailableComponenentsList() : Error processing item {component.Name}: {ex.Message}", ex);
}
}
};

var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Dataconstant.MaxDegreeOfParallelism
};
await ProcessAsyncHelper.ProcessItemsAsync(listOfComponentsToBom, action, parallelOptions);

return availableComponentList;
}
Expand Down

0 comments on commit 1adcbf5

Please sign in to comment.